Update vLLM-fork version to v0.7.2+Gaudi-1.21.0

Signed-off-by: CICD-at-OPEA <CICD@opea.dev>

.github/CODEOWNERS

@@ -1,16 +1,18 @@
-* liang1.lv@intel.com feng.tian@intel.com suyue.chen@intel.com
+# Code owners will review PRs within their respective folders.
+
+* liang1.lv@intel.com feng.tian@intel.com suyue.chen@intel.com kaokao.lv@intel.com minmin.hou@intel.com rita.brugarolas.brufau@intel.com
 /.github/ suyue.chen@intel.com ze.pan@intel.com
-/AgentQnA/ kaokao.lv@intel.com minmin.hou@intel.com
+/AgentQnA/ abolfazl.shahbazi@intel.com kaokao.lv@intel.com minmin.hou@intel.com
 /AudioQnA/ sihan.chen@intel.com wenjiao.yue@intel.com
 /AvatarChatbot/ chun.tao@intel.com kaokao.lv@intel.com
 /ChatQnA/ liang1.lv@intel.com letong.han@intel.com
-/CodeGen/ liang1.lv@intel.com xinyao.wang@intel.com
-/CodeTrans/ sihan.chen@intel.com xinyao.wang@intel.com
+/CodeGen/ liang1.lv@intel.com
+/CodeTrans/ sihan.chen@intel.com
 /DBQnA/ supriya.krishnamurthi@intel.com liang1.lv@intel.com
-/DocIndexRetriever/ kaokao.lv@intel.com chendi.xue@intel.com
-/DocSum/ letong.han@intel.com xinyao.wang@intel.com
+/DocIndexRetriever/ abolfazl.shahbazi@intel.com kaokao.lv@intel.com chendi.xue@intel.com
+/DocSum/ letong.han@intel.com
 /EdgeCraftRAG/ yongbo.zhu@intel.com mingyuan.qi@intel.com
-/FaqGen/ yogesh.pandey@intel.com xinyao.wang@intel.com
+/FinanceAgent/ abolfazl.shahbazi@intel.com kaokao.lv@intel.com minmin.hou@intel.com rita.brugarolas.brufau@intel.com
 /GraphRAG/ rita.brugarolas.brufau@intel.com abolfazl.shahbazi@intel.com
 /InstructionTuning/ xinyu.ye@intel.com kaokao.lv@intel.com
 /MultimodalQnA/ melanie.h.buehler@intel.com tiep.le@intel.com
@@ -19,5 +21,6 @@
 /SearchQnA/ sihan.chen@intel.com letong.han@intel.com
 /Text2Image/ wenjiao.yue@intel.com xinyu.ye@intel.com
 /Translation/ liang1.lv@intel.com sihan.chen@intel.com
-/VideoQnA/ huiling.bao@intel.com xinyao.wang@intel.com
-/VisualQnA/ liang1.lv@intel.com sihan.chen@intel.com
+/VideoQnA/ huiling.bao@intel.com
+/VisualQnA/ liang1.lv@intel.com sihan.chen@intel.com
+/WorkflowExecAgent/ joshua.jian.ern.liew@intel.com kaokao.lv@intel.com

.github/env/_build_image.sh

@@ -0,0 +1,5 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+export VLLM_VER=v0.8.3
+export VLLM_FORK_VER=v0.7.2+Gaudi-1.21.0

.github/workflows/_build_image.yml

@@ -75,17 +75,12 @@ jobs:
         run: |
           cd ${{ github.workspace }}/${{ inputs.example }}/docker_image_build
           docker_compose_path=${{ github.workspace }}/${{ inputs.example }}/docker_image_build/build.yaml
+          source ${{ github.workspace }}/.github/env/_build_image.sh
           if [[ $(grep -c "vllm:" ${docker_compose_path}) != 0 ]]; then
-              git clone https://github.com/vllm-project/vllm.git && cd vllm
-              VLLM_VER=v0.8.3
-              echo "Check out vLLM tag ${VLLM_VER}"
-              git checkout ${VLLM_VER} &> /dev/null && cd ../
+              git clone -b ${VLLM_VER} --single-branch https://github.com/vllm-project/vllm.git
           fi
           if [[ $(grep -c "vllm-gaudi:" ${docker_compose_path}) != 0 ]]; then
-              git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-              VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-              echo "Check out vLLM tag ${VLLM_VER}"
-              git checkout ${VLLM_VER} &> /dev/null && cd ../
+              git clone -b ${VLLM_FORK_VER} --single-branch https://github.com/HabanaAI/vllm-fork.git
           fi
           git clone --depth 1 --branch ${{ inputs.opea_branch }} https://github.com/opea-project/GenAIComps.git
           cd GenAIComps && git rev-parse HEAD && cd ../

.github/workflows/_example-workflow.yml

@@ -76,6 +76,7 @@ jobs:
       example: ${{ inputs.example }}
       hardware: ${{ inputs.node }}
       use_model_cache: ${{ inputs.use_model_cache }}
+      opea_branch: ${{ inputs.opea_branch }}
     secrets: inherit
 
 

.github/workflows/_helm-e2e.yml

@@ -2,7 +2,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 name: Helm Chart E2e Test For Call
-permissions: read-all
+permissions:
+  contents: read
+
 on:
   workflow_call:
     inputs:
@@ -135,16 +137,28 @@ jobs:
         env:
           example: ${{ inputs.example }}
         run: |
-          CHART_NAME="${example,,}"  # CodeGen
-          echo "CHART_NAME=$CHART_NAME" >> $GITHUB_ENV
-          echo "RELEASE_NAME=${CHART_NAME}$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
-          echo "NAMESPACE=${CHART_NAME}-$(head -c 4 /dev/urandom | xxd -p)" >> $GITHUB_ENV
-          echo "ROLLOUT_TIMEOUT_SECONDS=600s" >> $GITHUB_ENV
-          echo "TEST_TIMEOUT_SECONDS=600s" >> $GITHUB_ENV
-          echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV
-          echo "should_cleanup=false" >> $GITHUB_ENV
-          echo "skip_validate=false" >> $GITHUB_ENV
-          echo "CHART_FOLDER=${example}/kubernetes/helm" >> $GITHUB_ENV
+          if [[ ! "$example" =~ ^[a-zA-Z0-9]{1,20}$ ]] || [[ "$example" =~ \.\. ]] || [[ "$example" == -* || "$example" == *- ]]; then
+            echo "Error: Invalid input - only lowercase alphanumeric and internal hyphens allowed"
+            exit 1
+          fi
+          # SAFE_PREFIX="kb-"
+          CHART_NAME="${SAFE_PREFIX}$(echo "$example" | tr '[:upper:]' '[:lower:]')"
+          RAND_SUFFIX=$(openssl rand -hex 2 | tr -dc 'a-f0-9')
+
+          cat <<EOF >> $GITHUB_ENV
+          CHART_NAME=${CHART_NAME}
+          RELEASE_NAME=${CHART_NAME}-$(date +%s)
+          NAMESPACE=ns-${CHART_NAME}-${RAND_SUFFIX}
+          ROLLOUT_TIMEOUT_SECONDS=600s
+          TEST_TIMEOUT_SECONDS=600s
+          KUBECTL_TIMEOUT_SECONDS=60s
+          should_cleanup=false
+          skip_validate=false
+          CHART_FOLDER=${example}/kubernetes/helm
+          EOF
+
+          echo "Generated safe variables:" >> $GITHUB_STEP_SUMMARY
+          echo "- CHART_NAME: ${CHART_NAME}" >> $GITHUB_STEP_SUMMARY
 
       - name: Helm install
         id: install

.github/workflows/_run-docker-compose.yml

@@ -32,6 +32,10 @@ on:
         required: false
         type: boolean
         default: false
+      opea_branch:
+        default: "main"
+        required: false
+        type: string
 jobs:
   get-test-case:
     runs-on: ubuntu-latest
@@ -169,6 +173,7 @@ jobs:
           FINANCIAL_DATASETS_API_KEY: ${{ secrets.FINANCIAL_DATASETS_API_KEY }}
           IMAGE_REPO: ${{ inputs.registry }}
           IMAGE_TAG: ${{ inputs.tag }}
+          opea_branch: ${{ inputs.opea_branch }}
           example: ${{ inputs.example }}
           hardware: ${{ inputs.hardware }}
           test_case: ${{ matrix.test_case }}
@@ -199,6 +204,10 @@ jobs:
           if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
 
           echo "Cleaning up images ..."
+          df -h
+          sleep 1
+          docker system df
+          sleep 1
           if [[ "${{ inputs.hardware }}" == "xeon"* ]]; then
               docker system prune -a -f
           else
@@ -208,7 +217,13 @@ jobs:
               docker images --filter reference="opea/comps-base" -q | xargs -r docker rmi && sleep 1s
               docker system prune -f
           fi
+          sleep 5
           docker images
+          sleep 1
+          df -h
+          sleep 1
+          docker system df
+          sleep 1
 
       - name: Publish pipeline artifact
         if: ${{ !cancelled() }}

.github/workflows/daily-update-vllm-version.yml

@@ -0,0 +1,94 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Daily update vLLM & vLLM-fork version
+
+on:
+  schedule:
+    - cron: "30 22 * * *"
+  workflow_dispatch:
+
+env:
+  BRANCH_NAME: "update"
+  USER_NAME: "CICD-at-OPEA"
+  USER_EMAIL: "CICD@opea.dev"
+
+jobs:
+  freeze-tag:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - repo: vLLM
+            repo_name: vllm-project/vllm
+            ver_name: VLLM_VER
+          - repo: vLLM-fork
+            repo_name: HabanaAI/vllm-fork
+            ver_name: VLLM_FORK_VER
+      fail-fast: false
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.ref }}
+
+      - name: Set up Git
+        run: |
+          git config --global user.name ${{ env.USER_NAME }}
+          git config --global user.email ${{ env.USER_EMAIL }}
+          git remote set-url origin https://${{ env.USER_NAME }}:"${{ secrets.ACTION_TOKEN }}"@github.com/${{ github.repository }}.git
+          git fetch
+
+          if git ls-remote https://github.com/${{ github.repository }}.git "refs/heads/${{ env.BRANCH_NAME }}_${{ matrix.repo }}" | grep -q "refs/heads/${{ env.BRANCH_NAME }}_${{ matrix.repo }}"; then
+            echo "branch ${{ env.BRANCH_NAME }}_${{ matrix.repo }} exists"
+            git checkout ${{ env.BRANCH_NAME }}_${{ matrix.repo }}
+          else
+            echo "branch ${{ env.BRANCH_NAME }}_${{ matrix.repo }} not exists"
+            git checkout -b ${{ env.BRANCH_NAME }}_${{ matrix.repo }}
+            git push origin ${{ env.BRANCH_NAME }}_${{ matrix.repo }}
+            echo "branch ${{ env.BRANCH_NAME }}_${{ matrix.repo }} created successfully"
+          fi
+
+      - name: Run script
+        run: |
+          latest_vllm_ver=$(curl -s "https://api.github.com/repos/${{ matrix.repo_name }}/tags" | jq '.[0].name' -)
+          latest_vllm_ver=$(echo "$latest_vllm_ver" | sed 's/"//g')
+          echo "latest_vllm_ver=${latest_vllm_ver}" >> "$GITHUB_ENV"
+          find . -type f -name "*.sh" -exec sed -i "s/${{ matrix.ver_name }}=.*/${{ matrix.ver_name }}=${latest_vllm_ver}/" {} \;
+
+      - name: Commit changes
+        run: |
+          git add .
+          if git diff-index --quiet HEAD --; then
+            echo "No changes detected, skipping commit."
+            exit 1
+          else
+            git commit -s -m "Update ${{ matrix.repo }} version to ${latest_vllm_ver}"
+            git push --set-upstream origin ${{ env.BRANCH_NAME }}_${{ matrix.repo }}
+          fi
+
+      - name: Create Pull Request
+        env:
+          GH_TOKEN: ${{ secrets.ACTION_TOKEN }}
+        run: |
+          pr_count=$(curl -H "Authorization: token ${{ secrets.ACTION_TOKEN }}" -s "https://api.github.com/repos/${{ github.repository }}/pulls?state=all&head=${{ env.USER_NAME }}:${{ env.BRANCH_NAME }}_${{ matrix.repo }}" | jq '. | length')
+          if [ $pr_count -gt 0 ]; then
+            echo "Pull Request exists"
+            pr_number=$(curl -H "Authorization: token ${{ secrets.ACTION_TOKEN }}" -s "https://api.github.com/repos/${{ github.repository }}/pulls?state=all&head=${{ env.USER_NAME }}:${{ env.BRANCH_NAME }}_${{ matrix.repo }}" | jq '.[0].number')
+            gh pr edit ${pr_number} \
+              --title "Update ${{ matrix.repo }} version to ${latest_vllm_ver}" \
+              --body "Update ${{ matrix.repo }} version to ${latest_vllm_ver}"
+            echo "Pull Request updated successfully"
+          else
+            echo "Pull Request does not exists..."
+            gh pr create \
+              -B main \
+              -H ${{ env.BRANCH_NAME }}_${{ matrix.repo }} \
+              --title "Update ${{ matrix.repo }} version to ${latest_vllm_ver}" \
+              --body "Update ${{ matrix.repo }} version to ${latest_vllm_ver}"
+            echo "Pull Request created successfully"
+          fi

.github/workflows/daily_check_issue_and_pr.yml

@@ -26,3 +26,4 @@ jobs:
           close-pr-message: "This PR was closed because it has been stalled for 7 days with no activity."
           repo-token: ${{ secrets.ACTION_TOKEN }}
           start-date: "2025-03-01T00:00:00Z"
+          exempt-issue-labels: "Backlog"

.github/workflows/pr-chart-e2e.yml

@@ -19,6 +19,9 @@ concurrency:
 jobs:
   job1:
     name: Get-Test-Matrix
+    permissions:
+      contents: read
+      pull-requests: read
     runs-on: ubuntu-latest
     outputs:
       run_matrix: ${{ steps.get-test-matrix.outputs.run_matrix }}

.github/workflows/pr-link-path-scan.yml

@@ -23,6 +23,7 @@ jobs:
       - name: Check the Validity of Hyperlinks
         run: |
           cd ${{github.workspace}}
+          delay=15
           fail="FALSE"
           merged_commit=$(git log -1 --format='%H')
           changed_files="$(git diff --name-status --diff-filter=ARM ${{ github.event.pull_request.base.sha }} ${merged_commit} | awk '/\.md$/ {print $NF}')"
@@ -35,15 +36,20 @@ jobs:
                   # echo $url_line
                   url=$(echo "$url_line"|cut -d '(' -f2 | cut -d ')' -f1|sed 's/\.git$//')
                   path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-)
-                  response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url")|| true
-                  if [ "$response" -ne 200 ]; then
-                    echo "**********Validation failed, try again**********"
-                    response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url")
-                    if [ "$response_retry" -eq 200 ]; then
-                      echo "*****Retry successfully*****"
-                    else
-                      echo "Invalid link from ${{github.workspace}}/$path: $url"
-                      fail="TRUE"
+                  if [[ "$url" == "https://platform.openai.com/api-keys"* ]]; then
+                    echo "Link "$url" from ${{github.workspace}}/$path needs to be verified by a real person."
+                  else
+                    sleep $delay
+                    response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url")|| true
+                    if [ "$response" -ne 200 ]; then
+                      echo "**********Validation failed ($response), try again**********"
+                      response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url")
+                      if [ "$response_retry" -eq 200 ]; then
+                        echo "*****Retry successfully*****"
+                      else
+                        echo "Invalid link ($response_retry) from ${{github.workspace}}/$path: $url"
+                        fail="TRUE"
+                      fi
                     fi
                   fi
                 done
@@ -74,6 +80,7 @@ jobs:
       - name: Checking Relative Path Validity
         run: |
           cd ${{github.workspace}}
+          delay=15
           fail="FALSE"
           repo_name=${{ github.event.pull_request.head.repo.full_name }}
           branch="https://github.com/$repo_name/blob/${{ github.event.pull_request.head.ref }}"
@@ -105,14 +112,15 @@ jobs:
                 if [[ "$png_line" == *#* ]]; then
                   if [ -n "changed_files" ] && echo "$changed_files" | grep -q "^${refer_path}$"; then
                     url_dev=$branch$(echo "$real_path" | sed 's|.*/GenAIExamples||')$png_path
+                    sleep $delay
                     response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url_dev")
                     if [ "$response" -ne 200 ]; then
-                      echo "**********Validation failed, try again**********"
+                      echo "**********Validation failed ($response), try again**********"
                       response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url_dev")
                       if [ "$response_retry" -eq 200 ]; then
                         echo "*****Retry successfully*****"
                       else
-                        echo "Invalid path from ${{github.workspace}}/$refer_path: $png_path"
+                        echo "Invalid path ($response_retry) from ${{github.workspace}}/$refer_path: $png_path"
                         fail="TRUE"
                       fi
                     else

.github/workflows/scripts/codeScan/hadolint.sh

@@ -7,7 +7,7 @@ source /GenAIExamples/.github/workflows/scripts/change_color
 log_dir=/GenAIExamples/.github/workflows/scripts/codeScan
 ERROR_WARN=false
 
-find . -type f \( -name "Dockerfile*" \) -print -exec hadolint --ignore DL3006 --ignore DL3007 --ignore DL3008 --ignore DL3013 {} \; > ${log_dir}/hadolint.log
+find . -type f \( -name "Dockerfile*" \) -print -exec hadolint --ignore DL3006 --ignore DL3007 --ignore DL3008 --ignore DL3013 --ignore DL3018 --ignore DL3016 {} \; > ${log_dir}/hadolint.log
 
 if [[ $(grep -c "error" ${log_dir}/hadolint.log) != 0 ]]; then
     $BOLD_RED && echo "Error!! Please Click on the artifact button to download and check error details." && $RESET

AgentQnA/README.md

@@ -99,7 +99,7 @@ flowchart LR
 
 #### First, clone the `GenAIExamples` repo.
 
-```
+```bash
 export WORKDIR=<your-work-directory>
 cd $WORKDIR
 git clone https://github.com/opea-project/GenAIExamples.git
@@ -109,7 +109,7 @@ git clone https://github.com/opea-project/GenAIExamples.git
 
 ##### For proxy environments only
 
-```
+```bash
 export http_proxy="Your_HTTP_Proxy"
 export https_proxy="Your_HTTPs_Proxy"
 # Example: no_proxy="localhost, 127.0.0.1, 192.168.1.1"
@@ -118,14 +118,24 @@ export no_proxy="Your_No_Proxy"
 
 ##### For using open-source llms
 
-```
+Set up a [HuggingFace](https://huggingface.co/) account and generate a [user access token](https://huggingface.co/docs/transformers.js/en/guides/private#step-1-generating-a-user-access-token).
+
+Then set an environment variable with the token and another for a directory to download the models:
+
+```bash
 export HUGGINGFACEHUB_API_TOKEN=<your-HF-token>
-export HF_CACHE_DIR=<directory-where-llms-are-downloaded> #so that no need to redownload every time
+export HF_CACHE_DIR=<directory-where-llms-are-downloaded> #  to avoid redownloading models
 ```
 
-##### [Optional] OPANAI_API_KEY to use OpenAI models
+##### [Optional] OPENAI_API_KEY to use OpenAI models or Intel® AI for Enterprise Inference
 
-```
+To use OpenAI models, generate a key following these [instructions](https://platform.openai.com/api-keys).
+
+To use a remote server running Intel® AI for Enterprise Inference, contact the cloud service provider or owner of the on-prem machine for a key to access the desired model on the server.
+
+Then set the environment variable `OPENAI_API_KEY` with the key contents:
+
+```bash
 export OPENAI_API_KEY=<your-openai-key>
 ```
 
@@ -133,16 +143,18 @@ export OPENAI_API_KEY=<your-openai-key>
 
 ##### Gaudi
 
-```
+```bash
 source $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/hpu/gaudi/set_env.sh
 ```
 
 ##### Xeon
 
-```
+```bash
 source $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon/set_env.sh
 ```
 
+For running
+
 ### 2. Launch the multi-agent system. </br>
 
 We make it convenient to launch the whole system with docker compose, which includes microservices for LLM, agents, UI, retrieval tool, vector database, dataprep, and telemetry. There are 3 docker compose files, which make it easy for users to pick and choose. Users can choose a different retrieval tool other than the `DocIndexRetriever` example provided in our GenAIExamples repo. Users can choose not to launch the telemetry containers.
@@ -184,14 +196,37 @@ docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/
 
 #### Launch on Xeon
 
-On Xeon, only OpenAI models are supported. The command below will launch the multi-agent system with the `DocIndexRetriever` as the retrieval tool for the Worker RAG agent.
+On Xeon, OpenAI models and models deployed on a remote server are supported. Both methods require an API key.
 
 ```bash
 export OPENAI_API_KEY=<your-openai-key>
 cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon
+```
+
+##### OpenAI Models
+
+The command below will launch the multi-agent system with the `DocIndexRetriever` as the retrieval tool for the Worker RAG agent.
+
+```bash
 docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml -f compose_openai.yaml up -d
 ```
 
+##### Models on Remote Server
+
+When models are deployed on a remote server with Intel® AI for Enterprise Inference, a base URL and an API key are required to access them. To run the Agent microservice on Xeon while using models deployed on a remote server, add `compose_remote.yaml` to the `docker compose` command and set additional environment variables.
+
+###### Notes
+
+- `OPENAI_API_KEY` is already set in a previous step.
+- `model` is used to overwrite the value set for this environment variable in `set_env.sh`.
+- `LLM_ENDPOINT_URL` is the base URL given from the owner of the on-prem machine or cloud service provider. It will follow this format: "https://<DNS>". Here is an example: "https://api.inference.example.com".
+
+```bash
+export model=<name-of-model-card>
+export LLM_ENDPOINT_URL=<http-endpoint-of-remote-server>
+docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml -f compose_openai.yaml -f compose_remote.yaml up -d
+```
+
 ### 3. Ingest Data into the vector database
 
 The `run_ingest_data.sh` script will use an example jsonl file to ingest example documents into a vector database. Other ways to ingest data and other types of documents supported can be found in the OPEA dataprep microservice located in the opea-project/GenAIComps repo.
@@ -208,12 +243,18 @@ bash run_ingest_data.sh
 The UI microservice is launched in the previous step with the other microservices.
 To see the UI, open a web browser to `http://${ip_address}:5173` to access the UI. Note the `ip_address` here is the host IP of the UI microservice.
 
-1. `create Admin Account` with a random value
-2. add opea agent endpoint `http://$ip_address:9090/v1` which is a openai compatible api
+1. Click on the arrow above `Get started`. Create an admin account with a name, email, and password.
+2. Add an OpenAI-compatible API endpoint. In the upper right, click on the circle button with the user's initial, go to `Admin Settings`->`Connections`. Under `Manage OpenAI API Connections`, click on the `+` to add a connection. Fill in these fields:
+
+- **URL**: `http://${ip_address}:9090/v1`, do not forget the `v1`
+- **Key**: any value
+- **Model IDs**: any name i.e. `opea-agent`, then press `+` to add it
+
+Click "Save".
 
 ![opea-agent-setting](assets/img/opea-agent-setting.png)
 
-3. test opea agent with ui
+3. Test OPEA agent with UI. Return to `New Chat` and ensure the model (i.e. `opea-agent`) is selected near the upper left. Enter in any prompt to interact with the agent.
 
 ![opea-agent-test](assets/img/opea-agent-test.png)
 

AgentQnA/docker_compose/intel/cpu/xeon/compose_remote.yaml

@@ -0,0 +1,18 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  worker-rag-agent:
+    environment:
+      llm_endpoint_url: ${LLM_ENDPOINT_URL}
+      api_key: ${OPENAI_API_KEY}
+
+  worker-sql-agent:
+    environment:
+      llm_endpoint_url: ${LLM_ENDPOINT_URL}
+      api_key: ${OPENAI_API_KEY}
+
+  supervisor-react-agent:
+    environment:
+      llm_endpoint_url: ${LLM_ENDPOINT_URL}
+      api_key: ${OPENAI_API_KEY}

AgentQnA/tests/step1_build_images.sh

@@ -37,8 +37,8 @@ function build_agent_docker_image_gaudi_vllm() {
     get_genai_comps
 
     git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.7.2+Gaudi-1.21.0
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build agent image with --no-cache..."
     service_list="agent agent-ui vllm-gaudi"

AgentQnA/ui/docker/Dockerfile

@@ -1,49 +1,203 @@
-# Copyright (C) 2024 Intel Corporation
+# Copyright (C) 2025 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-#FROM python:3.11-slim
-FROM node:22.9.0
+# syntax=docker/dockerfile:1
+# Initialize device type args
+# use build args in the docker build command with --build-arg="BUILDARG=true"
+ARG USE_CUDA=false
+ARG USE_OLLAMA=false
+# Tested with cu117 for CUDA 11 and cu121 for CUDA 12 (default)
+ARG USE_CUDA_VER=cu121
+# any sentence transformer model; models to use can be found at https://huggingface.co/models?library=sentence-transformers
+# Leaderboard: https://huggingface.co/spaces/mteb/leaderboard
+# for better performance and multilangauge support use "intfloat/multilingual-e5-large" (~2.5GB) or "intfloat/multilingual-e5-base" (~1.5GB)
+# IMPORTANT: If you change the embedding model (sentence-transformers/all-MiniLM-L6-v2) and vice versa, you aren't able to use RAG Chat with your previous documents loaded in the WebUI! You need to re-embed them.
+ARG USE_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+ARG USE_RERANKING_MODEL=""
 
-ENV LANG=C.UTF-8
-ARG ARCH=cpu
+# Tiktoken encoding name; models to use can be found at https://huggingface.co/models?library=tiktoken
+ARG USE_TIKTOKEN_ENCODING_NAME="cl100k_base"
 
-RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
-    build-essential \
-    libgl1-mesa-glx \
-    libjemalloc-dev \
-    git \
-    python3-venv
+ARG BUILD_HASH=dev-build
+# Override at your own risk - non-root configurations are untested
+ARG UID=0
+ARG GID=0
 
+######## WebUI frontend ########
+FROM --platform=$BUILDPLATFORM node:22-alpine3.20 AS build
+ARG BUILD_HASH
+
+WORKDIR /app
+
+COPY open_webui_patches /app/patches
+ARG WEBUI_VERSION=v0.5.20
+RUN apk add --no-cache git
+
+# Clone code and use patch
+RUN git config --global user.name "opea" && \
+    git config --global user.email "" && \
+    git clone https://github.com/open-webui/open-webui.git
+
+WORKDIR /app/open-webui
+
+RUN git checkout ${WEBUI_VERSION} && git am /app/patches/*.patch
+
+WORKDIR /app
+
+RUN mv open-webui/* . && rm -fr open-webui && ls -lrth /app/backend/
+
+RUN npm install onnxruntime-node --onnxruntime-node-install-cuda=skip
+RUN apk update && \
+    apk add --no-cache wget && \
+    wget https://github.com/microsoft/onnxruntime/releases/download/v1.20.1/onnxruntime-linux-x64-gpu-1.20.1.tgz
+
+ENV APP_BUILD_HASH=${BUILD_HASH}
+RUN npm run build
+
+######## WebUI backend ########
+FROM python:3.11-slim-bookworm AS base
+
+# Use args
+ARG USE_CUDA
+ARG USE_OLLAMA
+ARG USE_CUDA_VER
+ARG USE_EMBEDDING_MODEL
+ARG USE_RERANKING_MODEL
+ARG UID
+ARG GID
+
+## Basis ##
+ENV ENV=prod \
+    PORT=8080 \
+    # pass build args to the build
+    USE_OLLAMA_DOCKER=${USE_OLLAMA} \
+    USE_CUDA_DOCKER=${USE_CUDA} \
+    USE_CUDA_DOCKER_VER=${USE_CUDA_VER} \
+    USE_EMBEDDING_MODEL_DOCKER=${USE_EMBEDDING_MODEL} \
+    USE_RERANKING_MODEL_DOCKER=${USE_RERANKING_MODEL}
+
+## Basis URL Config ##
+ENV OLLAMA_BASE_URL="/ollama" \
+    OPENAI_API_BASE_URL=""
+
+## API Key and Security Config ##
+ENV OPENAI_API_KEY="" \
+    WEBUI_SECRET_KEY="" \
+    SCARF_NO_ANALYTICS=true \
+    DO_NOT_TRACK=true \
+    ANONYMIZED_TELEMETRY=false
+
+#### Other models #########################################################
+## whisper TTS model settings ##
+ENV WHISPER_MODEL="base" \
+    WHISPER_MODEL_DIR="/app/backend/data/cache/whisper/models"
+
+## RAG Embedding model settings ##
+ENV RAG_EMBEDDING_MODEL="$USE_EMBEDDING_MODEL_DOCKER" \
+    RAG_RERANKING_MODEL="$USE_RERANKING_MODEL_DOCKER" \
+    SENTENCE_TRANSFORMERS_HOME="/app/backend/data/cache/embedding/models"
+
+## Tiktoken model settings ##
+ENV TIKTOKEN_ENCODING_NAME="cl100k_base" \
+    TIKTOKEN_CACHE_DIR="/app/backend/data/cache/tiktoken"
+
+## Hugging Face download cache ##
+ENV HF_HOME="/app/backend/data/cache/embedding/models"
+
+## Torch Extensions ##
+# ENV TORCH_EXTENSIONS_DIR="/.cache/torch_extensions"
+
+#### Other models ##########################################################
+
+COPY --from=build /app/backend /app/backend
+
+WORKDIR /app/backend
 
-WORKDIR /root/
 
 ENV HOME=/root
-ENV VIRTUAL_ENV=$HOME/.env/open-webui
+# Create user and group if not root
+RUN if [ $UID -ne 0 ]; then \
+    if [ $GID -ne 0 ]; then \
+    addgroup --gid $GID app; \
+    fi; \
+    adduser --uid $UID --gid $GID --home $HOME --disabled-password --no-create-home app; \
+    fi
 
-COPY open_webui_patches /root/patches
+RUN mkdir -p $HOME/.cache/chroma
+RUN printf 00000000-0000-0000-0000-000000000000 > $HOME/.cache/chroma/telemetry_user_id
 
-RUN git clone https://github.com/open-webui/open-webui.git && \
-    git config --global user.name "opea" && git config --global user.email "" && \
-    mkdir -p $HOME/.env && python3 -m venv $VIRTUAL_ENV && \
-    $VIRTUAL_ENV/bin/python -m pip install --no-cache-dir --upgrade pip && \
-    $VIRTUAL_ENV/bin/python -m pip install --no-cache-dir build
+# Make sure the user has access to the app and root directory
+RUN chown -R $UID:$GID /app $HOME
 
-WORKDIR /root/open-webui
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN if [ "$USE_OLLAMA" = "true" ]; then \
+    apt-get update && \
+    # Install pandoc and netcat
+    apt-get install -y --no-install-recommends git build-essential pandoc netcat-openbsd curl && \
+    apt-get install -y --no-install-recommends gcc python3-dev && \
+    # for RAG OCR
+    apt-get install -y --no-install-recommends ffmpeg libsm6 libxext6 && \
+    # install helper tools
+    apt-get install -y --no-install-recommends curl jq && \
+    # install ollama
+    curl -fsSL https://ollama.com/install.sh | sh && \
+    # cleanup
+    rm -rf /var/lib/apt/lists/*; \
+    else \
+    apt-get update && \
+    # Install pandoc, netcat and gcc
+    apt-get install -y --no-install-recommends git build-essential pandoc gcc netcat-openbsd curl jq && \
+    apt-get install -y --no-install-recommends gcc python3-dev && \
+    # for RAG OCR
+    apt-get install -y --no-install-recommends ffmpeg libsm6 libxext6 && \
+    # cleanup
+    rm -rf /var/lib/apt/lists/*; \
+    fi
 
-RUN git checkout v0.5.20 && \
-    git am ../patches/*.patch && \
-    python -m build && \
-    pip install --no-cache-dir dist/open_webui-0.5.20-py3-none-any.whl    
+# install python dependencies
+# COPY --chown=$UID:$GID ./backend/requirements.txt ./requirements.txt
+# RUN cp /app/backend/requirements.txt ./requirements.txt
 
-ENV LANG=en_US.UTF-8
-
-WORKDIR /root/
-
-RUN rm -fr /root/open-webui && rm -fr /root/patches
-
-# CMD ["/bin/bash"]
-ENTRYPOINT ["open-webui", "serve"]
+RUN pip3 install --no-cache-dir uv && \
+    if [ "$USE_CUDA" = "true" ]; then \
+    # If you use CUDA the whisper and embedding model will be downloaded on first use
+    pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/$USE_CUDA_DOCKER_VER --no-cache-dir && \
+    uv pip install --system -r requirements.txt --no-cache-dir && \
+    python -c "import os; from sentence_transformers import SentenceTransformer; SentenceTransformer(os.environ['RAG_EMBEDDING_MODEL'], device='cpu')" && \
+    python -c "import os; from faster_whisper import WhisperModel; WhisperModel(os.environ['WHISPER_MODEL'], device='cpu', compute_type='int8', download_root=os.environ['WHISPER_MODEL_DIR'])"; \
+    python -c "import os; import tiktoken; tiktoken.get_encoding(os.environ['TIKTOKEN_ENCODING_NAME'])"; \
+    else \
+    pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu --no-cache-dir && \
+    uv pip install --system -r requirements.txt --no-cache-dir && \
+    python -c "import os; from sentence_transformers import SentenceTransformer; SentenceTransformer(os.environ['RAG_EMBEDDING_MODEL'], device='cpu')" && \
+    python -c "import os; from faster_whisper import WhisperModel; WhisperModel(os.environ['WHISPER_MODEL'], device='cpu', compute_type='int8', download_root=os.environ['WHISPER_MODEL_DIR'])"; \
+    python -c "import os; import tiktoken; tiktoken.get_encoding(os.environ['TIKTOKEN_ENCODING_NAME'])"; \
+    fi; \
+    chown -R $UID:$GID /app/backend/data/
 
 
+
+# copy embedding weight from build
+# RUN mkdir -p /root/.cache/chroma/onnx_models/all-MiniLM-L6-v2
+# COPY --from=build /app/onnx /root/.cache/chroma/onnx_models/all-MiniLM-L6-v2/onnx
+
+# copy built frontend files
+COPY --chown=$UID:$GID --from=build /app/build /app/build
+COPY --chown=$UID:$GID --from=build /app/CHANGELOG.md /app/CHANGELOG.md
+COPY --chown=$UID:$GID --from=build /app/package.json /app/package.json
+
+# copy backend files
+# COPY --chown=$UID:$GID ./backend .
+
+EXPOSE 8080
+
+HEALTHCHECK CMD curl --silent --fail http://localhost:${PORT:-8080}/health | jq -ne 'input.status == true' || exit 1
+
+USER $UID:$GID
+
+ARG BUILD_HASH
+ENV WEBUI_BUILD_VERSION=${BUILD_HASH}
+ENV DOCKER=true
+
+CMD [ "bash", "start.sh"]

AudioQnA/Dockerfile

@@ -3,7 +3,6 @@
 
 ARG IMAGE_REPO=opea
 ARG BASE_TAG=latest
-FROM opea/comps-base:$BASE_TAG
 FROM $IMAGE_REPO/comps-base:$BASE_TAG
 
 COPY ./audioqna.py $HOME/audioqna.py

AudioQnA/Dockerfile.multilang

@@ -3,7 +3,6 @@
 
 ARG IMAGE_REPO=opea
 ARG BASE_TAG=latest
-FROM opea/comps-base:$BASE_TAG
 FROM $IMAGE_REPO/comps-base:$BASE_TAG
 
 COPY ./audioqna_multilang.py $HOME/audioqna_multilang.py

AudioQnA/docker_compose/amd/gpu/rocm/README.md

@@ -1,120 +1,59 @@
-# Build Mega Service of AudioQnA on AMD ROCm GPU
+# Deploying AudioQnA on AMD ROCm GPU
 
-This document outlines the deployment process for a AudioQnA application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice
-pipeline on server on AMD ROCm GPU platform.
+This document outlines the single node deployment process for a AudioQnA application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservices on server with AMD ROCm processing accelerators. The steps include pulling Docker images, container deployment via Docker Compose, and service execution using microservices `llm`.
 
-## Build Docker Images
+Note: The default LLM is `Intel/neural-chat-7b-v3-3`. Before deploying the application, please make sure either you've requested and been granted the access to it on [Huggingface](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) or you've downloaded the model locally from [ModelScope](https://www.modelscope.cn/models).
 
-### 1. Build Docker Image
+## Table of Contents
 
-- #### Create application install directory and go to it:
+1. [AudioQnA Quick Start Deployment](#audioqna-quick-start-deployment)
+2. [AudioQnA Docker Compose Files](#audioqna-docker-compose-files)
+3. [Validate Microservices](#validate-microservices)
+4. [Conclusion](#conclusion)
 
-  ```bash
-  mkdir ~/audioqna-install && cd audioqna-install
-  ```
+## AudioQnA Quick Start Deployment
 
-- #### Clone the repository GenAIExamples (the default repository branch "main" is used here):
+This section describes how to quickly deploy and test the AudioQnA service manually on an AMD ROCm platform. The basic steps are:
 
-  ```bash
-  git clone https://github.com/opea-project/GenAIExamples.git
-  ```
+1. [Access the Code](#access-the-code)
+2. [Configure the Deployment Environment](#configure-the-deployment-environment)
+3. [Deploy the Services Using Docker Compose](#deploy-the-services-using-docker-compose)
+4. [Check the Deployment Status](#check-the-deployment-status)
+5. [Validate the Pipeline](#validate-the-pipeline)
+6. [Cleanup the Deployment](#cleanup-the-deployment)
 
-  If you need to use a specific branch/tag of the GenAIExamples repository, then (v1.3 replace with its own value):
+### Access the Code
 
-  ```bash
-  git clone https://github.com/opea-project/GenAIExamples.git && cd GenAIExamples && git checkout v1.3
-  ```
-
-  We remind you that when using a specific version of the code, you need to use the README from this version:
-
-- #### Go to build directory:
-
-  ```bash
-  cd ~/audioqna-install/GenAIExamples/AudioQnA/docker_image_build
-  ```
-
-- Cleaning up the GenAIComps repository if it was previously cloned in this directory.
-  This is necessary if the build was performed earlier and the GenAIComps folder exists and is not empty:
-
-  ```bash
-  echo Y | rm -R GenAIComps
-  ```
-
-- #### Clone the repository GenAIComps (the default repository branch "main" is used here):
+Clone the GenAIExample repository and access the AudioQnA AMD ROCm platform Docker Compose files and supporting scripts:
 
 ```bash
-git clone https://github.com/opea-project/GenAIComps.git
-cd GenAIComps
+git clone https://github.com/opea-project/GenAIExamples.git
+cd GenAIExamples/AudioQnA
 ```
 
-We remind you that when using a specific version of the code, you need to use the README from this version.
+Then checkout a released version, such as v1.3:
 
-- #### Setting the list of images for the build (from the build file.yaml)
+```bash
+git checkout v1.3
+```
 
-  If you want to deploy a vLLM-based or TGI-based application, then the set of services is installed as follows:
+### Configure the Deployment Environment
 
-  #### vLLM-based application
+#### Docker Compose GPU Configuration
 
-  ```bash
-  service_list="vllm-rocm whisper speecht5 audioqna audioqna-ui"
-  ```
+Consult the section on [AudioQnA Service configuration](#audioqna-configuration) for information on how service specific configuration parameters affect deployments.
 
-  #### TGI-based application
-
-  ```bash
-  service_list="whisper speecht5 audioqna audioqna-ui"
-  ```
-
-- #### Optional. Pull TGI Docker Image (Do this if you want to use TGI)
-
-  ```bash
-  docker pull ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-  ```
-
-- #### Build Docker Images
-
-  ```bash
-  docker compose -f build.yaml build ${service_list} --no-cache
-  ```
-
-  After the build, we check the list of images with the command:
-
-  ```bash
-  docker image ls
-  ```
-
-  The list of images should include:
-
-  ##### vLLM-based application:
-
-  - opea/vllm-rocm:latest
-    - opea/whisper:latest
-    - opea/speecht5:latest
-    - opea/audioqna:latest
-
-  ##### TGI-based application:
-
-  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-    - opea/whisper:latest
-    - opea/speecht5:latest
-    - opea/audioqna:latest
-
----
-
-## Deploy the AudioQnA Application
-
-### Docker Compose Configuration for AMD GPUs
-
-To enable GPU support for AMD GPUs, the following configuration is added to the Docker Compose file:
-
-- compose_vllm.yaml - for vLLM-based application
-- compose.yaml - for TGI-based
+To enable GPU support for AMD GPUs, the following configuration is added to the Docker Compose files (`compose.yaml`, `compose_vllm.yaml`) for the LLM serving container:
 
 ```yaml
+# Example for vLLM service in compose_vllm.yaml
+# Note: Modern docker compose might use deploy.resources syntax instead.
+# Check your docker version and compose file.
 shm_size: 1g
 devices:
   - /dev/kfd:/dev/kfd
   - /dev/dri/:/dev/dri/
+# - /dev/dri/render128:/dev/dri/render128
 cap_add:
   - SYS_PTRACE
 group_add:
@@ -123,131 +62,161 @@ security_opt:
   - seccomp:unconfined
 ```
 
-This configuration forwards all available GPUs to the container. To use a specific GPU, specify its `cardN` and `renderN` device IDs. For example:
+#### Environment Variables (`set_env*.sh`)
 
-```yaml
-shm_size: 1g
-devices:
-  - /dev/kfd:/dev/kfd
-  - /dev/dri/card0:/dev/dri/card0
-  - /dev/dri/render128:/dev/dri/render128
-cap_add:
-  - SYS_PTRACE
-group_add:
-  - video
-security_opt:
-  - seccomp:unconfined
-```
+These scripts (`set_env_vllm.sh` for vLLM, `set_env.sh` for TGI) configure crucial parameters passed to the containers.
 
-**How to Identify GPU Device IDs:**
-Use AMD GPU driver utilities to determine the correct `cardN` and `renderN` IDs for your GPU.
+To set up environment variables for deploying AudioQnA services, set up some parameters specific to the deployment environment and source the `set_env.sh` script in this directory:
 
-### Set deploy environment variables
-
-#### Setting variables in the operating system environment:
-
-##### Set variable HUGGINGFACEHUB_API_TOKEN:
+For TGI inference usage:
 
 ```bash
-### Replace the string 'your_huggingfacehub_token' with your HuggingFacehub repository access token.
-export HUGGINGFACEHUB_API_TOKEN='your_huggingfacehub_token'
+export host_ip="External_Public_IP"           # ip address of the node
+export HUGGINGFACEHUB_API_TOKEN="Your_HuggingFace_API_Token"
+export http_proxy="Your_HTTP_Proxy"           # http proxy if any
+export https_proxy="Your_HTTPs_Proxy"         # https proxy if any
+export no_proxy=localhost,127.0.0.1,$host_ip,whisper-service,speecht5-service,vllm-service,tgi-service,audioqna-xeon-backend-server,audioqna-xeon-ui-server  # additional no proxies if needed
+export NGINX_PORT=${your_nginx_port}          # your usable port for nginx, 80 for example
+source ./set_env.sh
 ```
 
-#### Set variables value in set_env\*\*\*\*.sh file:
-
-Go to Docker Compose directory:
+For vLLM inference usage
 
 ```bash
-cd ~/audioqna-install/GenAIExamples/AudioQnA/docker_compose/amd/gpu/rocm
+export host_ip="External_Public_IP"           # ip address of the node
+export HUGGINGFACEHUB_API_TOKEN="Your_HuggingFace_API_Token"
+export http_proxy="Your_HTTP_Proxy"           # http proxy if any
+export https_proxy="Your_HTTPs_Proxy"         # https proxy if any
+export no_proxy=localhost,127.0.0.1,$host_ip,whisper-service,speecht5-service,vllm-service,tgi-service,audioqna-xeon-backend-server,audioqna-xeon-ui-server  # additional no proxies if needed
+export NGINX_PORT=${your_nginx_port}          # your usable port for nginx, 80 for example
+source ./set_env_vllm.sh
 ```
 
-The example uses the Nano text editor. You can use any convenient text editor:
+### Deploy the Services Using Docker Compose
 
-#### If you use vLLM
-
-```bash
-nano set_env_vllm.sh
-```
-
-#### If you use TGI
-
-```bash
-nano set_env.sh
-```
-
-If you are in a proxy environment, also set the proxy-related environment variables:
-
-```bash
-export http_proxy="Your_HTTP_Proxy"
-export https_proxy="Your_HTTPs_Proxy"
-```
-
-Set the values of the variables:
-
-- **HOST_IP, HOST_IP_EXTERNAL** - These variables are used to configure the name/address of the service in the operating system environment for the application services to interact with each other and with the outside world.
-
-  If your server uses only an internal address and is not accessible from the Internet, then the values for these two variables will be the same and the value will be equal to the server's internal name/address.
-
-  If your server uses only an external, Internet-accessible address, then the values for these two variables will be the same and the value will be equal to the server's external name/address.
-
-  If your server is located on an internal network, has an internal address, but is accessible from the Internet via a proxy/firewall/load balancer, then the HOST_IP variable will have a value equal to the internal name/address of the server, and the EXTERNAL_HOST_IP variable will have a value equal to the external name/address of the proxy/firewall/load balancer behind which the server is located.
-
-  We set these values in the file set_env\*\*\*\*.sh
-
-- **Variables with names like "**\*\*\*\*\*\*\_PORT"\*\* - These variables set the IP port numbers for establishing network connections to the application services.
-  The values shown in the file set_env.sh or set_env_vllm they are the values used for the development and testing of the application, as well as configured for the environment in which the development is performed. These values must be configured in accordance with the rules of network access to your environment's server, and must not overlap with the IP ports of other applications that are already in use.
-
-#### Set variables with script set_env\*\*\*\*.sh
-
-#### If you use vLLM
-
-```bash
-. set_env_vllm.sh
-```
-
-#### If you use TGI
-
-```bash
-. set_env.sh
-```
-
-### Start the services:
-
-#### If you use vLLM
-
-```bash
-docker compose -f compose_vllm.yaml up -d
-```
-
-#### If you use TGI
+To deploy the AudioQnA services, execute the `docker compose up` command with the appropriate arguments. For a default deployment, execute the command below. It uses the 'compose.yaml' file.
+
+for TGI inference deployment
 
 ```bash
+cd docker_compose/amd/gpu/rocm
 docker compose -f compose.yaml up -d
 ```
 
-All containers should be running and should not restart:
+for vLLM inference deployment
 
-##### If you use vLLM:
+```bash
+cd docker_compose/amd/gpu/rocm
+docker compose -f compose_vllm.yaml up -d
+```
 
-- audioqna-vllm-service
-- whisper-service
-- speecht5-service
-- audioqna-backend-server
-- audioqna-ui-server
+> **Note**: developers should build docker image from source when:
+>
+> - Developing off the git main branch (as the container's ports in the repo may be different > from the published docker image).
+> - Unable to download the docker image.
+> - Use a specific version of Docker image.
 
-##### If you use TGI:
+Please refer to the table below to build different microservices from source:
 
-- audioqna-tgi-service
-- whisper-service
-- speecht5-service
-- audioqna-backend-server
-- audioqna-ui-server
+| Microservice | Deployment Guide                                                                                                                  |
+| ------------ | --------------------------------------------------------------------------------------------------------------------------------- |
+| vLLM         | [vLLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/vllm#build-docker)                    |
+| LLM          | [LLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/llms)                                                |
+| WHISPER      | [Whisper build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/asr/src#211-whisper-server-image)                |
+| SPEECHT5     | [SpeechT5 build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/tts/src#211-speecht5-server-image)              |
+| GPT-SOVITS   | [GPT-SOVITS build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/gpt-sovits/src#build-the-image) |
+| MegaService  | [MegaService build guide](../../../../README_miscellaneous.md#build-megaservice-docker-image)                                     |
+| UI           | [Basic UI build guide](../../../../README_miscellaneous.md#build-ui-docker-image)                                                 |
 
----
+### Check the Deployment Status
 
-## Validate the Services
+After running docker compose, check if all the containers launched via docker compose have started:
 
-### 1. Validate the vLLM/TGI Service
+#### For TGI inference deployment
+
+```bash
+docker ps -a
+```
+
+For the default deployment, the following 5 containers should have started:
+
+```
+CONTAINER ID   IMAGE                                                      COMMAND                  CREATED          STATUS          PORTS                                         NAMES
+d8007690868d   opea/audioqna:latest                                       "python audioqna.py"     21 seconds ago   Up 19 seconds   0.0.0.0:3008->8888/tcp, [::]:3008->8888/tcp   audioqna-rocm-backend-server
+87ba9a1d56ae   ghcr.io/huggingface/text-generation-inference:2.4.1-rocm   "/tgi-entrypoint.sh …"   21 seconds ago   Up 20 seconds   0.0.0.0:3006->80/tcp, [::]:3006->80/tcp       tgi-service
+59e869acd742   opea/speecht5:latest                                       "python speecht5_ser…"   21 seconds ago   Up 20 seconds   0.0.0.0:7055->7055/tcp, :::7055->7055/tcp     speecht5-service
+0143267a4327   opea/whisper:latest                                        "python whisper_serv…"   21 seconds ago   Up 20 seconds   0.0.0.0:7066->7066/tcp, :::7066->7066/tcp     whisper-service
+```
+
+### For vLLM inference deployment
+
+```bash
+docker ps -a
+```
+
+For the default deployment, the following 5 containers should have started:
+
+```
+CONTAINER ID   IMAGE                     COMMAND                  CREATED          STATUS          PORTS                                           NAMES
+f3e6893a69fa   opea/audioqna-ui:latest   "docker-entrypoint.s…"   37 seconds ago   Up 35 seconds   0.0.0.0:18039->5173/tcp, [::]:18039->5173/tcp   audioqna-ui-server
+f943e5cd21e9   opea/audioqna:latest      "python audioqna.py"     37 seconds ago   Up 35 seconds   0.0.0.0:18038->8888/tcp, [::]:18038->8888/tcp   audioqna-backend-server
+074e8c418f52   opea/speecht5:latest      "python speecht5_ser…"   37 seconds ago   Up 36 seconds   0.0.0.0:7055->7055/tcp, :::7055->7055/tcp       speecht5-service
+77abe498e427   opea/vllm-rocm:latest     "python3 /workspace/…"   37 seconds ago   Up 36 seconds   0.0.0.0:8081->8011/tcp, [::]:8081->8011/tcp     audioqna-vllm-service
+9074a95bb7a6   opea/whisper:latest       "python whisper_serv…"   37 seconds ago   Up 36 seconds   0.0.0.0:7066->7066/tcp, :::7066->7066/tcp       whisper-service
+```
+
+If any issues are encountered during deployment, refer to the [Troubleshooting](../../../../README_miscellaneous.md#troubleshooting) section.
+
+### Validate the Pipeline
+
+Once the AudioQnA services are running, test the pipeline using the following command:
+
+```bash
+# Test the AudioQnA megaservice by recording a .wav file, encoding the file into the base64 format, and then sending the base64 string to the megaservice endpoint.
+# The megaservice will return a spoken response as a base64 string. To listen to the response, decode the base64 string and save it as a .wav file.
+wget https://github.com/intel/intel-extension-for-transformers/raw/refs/heads/main/intel_extension_for_transformers/neural_chat/assets/audio/sample_2.wav
+base64_audio=$(base64 -w 0 sample_2.wav)
+
+# if you are using speecht5 as the tts service, voice can be "default" or "male"
+# if you are using gpt-sovits for the tts service, you can set the reference audio following https://github.com/opea-project/GenAIComps/blob/main/comps/third_parties/gpt-sovits/src/README.md
+
+curl http://${host_ip}:3008/v1/audioqna \
+  -X POST \
+  -H "Content-Type: application/json" \
+  -d "{\"audio\": \"${base64_audio}\", \"max_tokens\": 64, \"voice\": \"default\"}" \
+  | sed 's/^"//;s/"$//' | base64 -d > output.wav
+```
+
+**Note** : Access the AudioQnA UI by web browser through this URL: `http://${host_ip}:5173`. Please confirm the `5173` port is opened in the firewall. To validate each microservice used in the pipeline refer to the [Validate Microservices](#validate-microservices) section.
+
+### Cleanup the Deployment
+
+To stop the containers associated with the deployment, execute the following command:
+
+#### If you use vLLM
+
+```bash
+cd ~/audioqna-install/GenAIExamples/AudioQnA/docker_compose/amd/gpu/rocm
+docker compose -f compose_vllm.yaml down
+```
+
+#### If you use TGI
+
+```bash
+cd ~/audioqna-install/GenAIExamples/AudioQnA/docker_compose/amd/gpu/rocm
+docker compose -f compose.yaml down
+```
+
+## AudioQnA Docker Compose Files
+
+In the context of deploying an AudioQnA pipeline on an Intel® Xeon® platform, we can pick and choose different large language model serving frameworks, or single English TTS/multi-language TTS component. The table below outlines the various configurations that are available as part of the application. These configurations can be used as templates and can be extended to different components available in [GenAIComps](https://github.com/opea-project/GenAIComps.git).
+
+| File                                     | Description                                                                               |
+| ---------------------------------------- | ----------------------------------------------------------------------------------------- |
+| [compose_vllm.yaml](./compose_vllm.yaml) | Default compose file using vllm as serving framework and redis as vector database         |
+| [compose.yaml](./compose.yaml)           | The LLM serving framework is TGI. All other configurations remain the same as the default |
+
+### Validate the vLLM/TGI Service
 
 #### If you use vLLM:
 
@@ -313,7 +282,7 @@ Checking the response from the service. The response should be similar to JSON:
 If the service response has a meaningful response in the value of the "generated_text" key,
 then we consider the TGI service to be successfully launched
 
-### 2. Validate MegaServices
+### Validate MegaServices
 
 Test the AudioQnA megaservice by recording a .wav file, encoding the file into the base64 format, and then sending the
 base64 string to the megaservice endpoint. The megaservice will return a spoken response as a base64 string. To listen
@@ -327,7 +296,7 @@ curl http://${host_ip}:3008/v1/audioqna \
   -H 'Content-Type: application/json' | sed 's/^"//;s/"$//' | base64 -d > output.wav
 ```
 
-### 3. Validate MicroServices
+### Validate MicroServices
 
 ```bash
 # whisper service
@@ -343,18 +312,6 @@ curl http://${host_ip}:7055/v1/tts \
   -H 'Content-Type: application/json'
 ```
 
-### 4. Stop application
+## Conclusion
 
-#### If you use vLLM
-
-```bash
-cd ~/audioqna-install/GenAIExamples/AudioQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose_vllm.yaml down
-```
-
-#### If you use TGI
-
-```bash
-cd ~/audioqna-install/GenAIExamples/AudioQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose.yaml down
-```
+This guide should enable developers to deploy the default configuration or any of the other compose yaml files for different configurations. It also highlights the configurable parameters that can be set before deployment.

AudioQnA/docker_compose/amd/gpu/rocm/set_env.sh

@@ -6,8 +6,8 @@
 
 # export host_ip=<your External Public IP>    # export host_ip=$(hostname -I | awk '{print $1}')
 
-export host_ip="192.165.1.21"
-export HUGGINGFACEHUB_API_TOKEN=${YOUR_HUGGINGFACEHUB_API_TOKEN}
+export host_ip=${ip_address}
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 # <token>
 
 export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3

AudioQnA/docker_compose/amd/gpu/rocm/set_env_vllm.sh

@@ -6,8 +6,8 @@
 
 # export host_ip=<your External Public IP>    # export host_ip=$(hostname -I | awk '{print $1}')
 
-export host_ip=""
-export external_host_ip=""
+export host_ip=${ip_address}
+export external_host_ip=${ip_address}
 export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 export HF_CACHE_DIR="./data"
 export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"

AudioQnA/docker_compose/intel/cpu/xeon/set_env.sh

@@ -14,7 +14,8 @@ export MEGA_SERVICE_HOST_IP=${host_ip}
 export WHISPER_SERVER_HOST_IP=${host_ip}
 export SPEECHT5_SERVER_HOST_IP=${host_ip}
 export LLM_SERVER_HOST_IP=${host_ip}
-
+export GPT_SOVITS_SERVER_HOST_IP=${host_ip}
+export GPT_SOVITS_SERVER_PORT=9880
 export WHISPER_SERVER_PORT=7066
 export SPEECHT5_SERVER_PORT=7055
 export LLM_SERVER_PORT=3006

AudioQnA/tests/README.md

@@ -0,0 +1,45 @@
+# AudioQnA E2E test scripts
+
+## Set the required environment variable
+
+```bash
+export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token"
+```
+
+## Run test
+
+On Intel Xeon with TGI:
+
+```bash
+bash test_compose_tgi_on_xeon.sh
+```
+
+On Intel Xeon with vLLM:
+
+```bash
+bash test_compose_on_xeon.sh
+```
+
+On Intel Gaudi with TGI:
+
+```bash
+bash test_compose_tgi_on_gaudi.sh
+```
+
+On Intel Gaudi with vLLM:
+
+```bash
+bash test_compose_on_gaudi.sh
+```
+
+On AMD ROCm with TGI:
+
+```bash
+bash test_compose_on_rocm.sh
+```
+
+On AMD ROCm with vLLM:
+
+```bash
+bash test_compose_vllm_on_rocm.sh
+```

AudioQnA/tests/test_compose_multilang_on_xeon.sh

@@ -40,21 +40,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon/
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct
-
-    export MEGA_SERVICE_HOST_IP=${ip_address}
-    export WHISPER_SERVER_HOST_IP=${ip_address}
-    export GPT_SOVITS_SERVER_HOST_IP=${ip_address}
-    export LLM_SERVER_HOST_IP=${ip_address}
-
-    export WHISPER_SERVER_PORT=7066
-    export GPT_SOVITS_SERVER_PORT=9880
-    export LLM_SERVER_PORT=3006
-
-    export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna
     export host_ip=${ip_address}
-
+    source set_env.sh
     # sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
     # Start Docker Containers

AudioQnA/tests/test_compose_on_gaudi.sh

@@ -27,9 +27,9 @@ function build_docker_images() {
 
     git clone https://github.com/HabanaAI/vllm-fork.git
     cd vllm-fork/
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    echo "Check out vLLM tag ${VLLM_VER}"
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.7.2+Gaudi-1.21.0
+    echo "Check out vLLM tag ${VLLM_FORK_VER}"
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="audioqna audioqna-ui whisper-gaudi speecht5-gaudi vllm-gaudi"
@@ -40,24 +40,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct
-    export NUM_CARDS=1
-    export BLOCK_SIZE=128
-    export MAX_NUM_SEQS=256
-    export MAX_SEQ_LEN_TO_CAPTURE=2048
-
-    export MEGA_SERVICE_HOST_IP=${ip_address}
-    export WHISPER_SERVER_HOST_IP=${ip_address}
-    export SPEECHT5_SERVER_HOST_IP=${ip_address}
-    export LLM_SERVER_HOST_IP=${ip_address}
-
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_PORT=3006
-
-    export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna
     export host_ip=${ip_address}
+    source set_env.sh
     # sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
     # Start Docker Containers

AudioQnA/tests/test_compose_on_rocm.sh

@@ -35,20 +35,7 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/amd/gpu/rocm/
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3
-
-    export MEGA_SERVICE_HOST_IP=${ip_address}
-    export WHISPER_SERVER_HOST_IP=${ip_address}
-    export SPEECHT5_SERVER_HOST_IP=${ip_address}
-    export LLM_SERVER_HOST_IP=${ip_address}
-
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_PORT=3006
-
-    export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna
-
+    source set_env.sh
     # Start Docker Containers
     docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
     n=0

AudioQnA/tests/test_compose_on_xeon.sh

@@ -40,21 +40,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon/
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct
-
-    export MEGA_SERVICE_HOST_IP=${ip_address}
-    export WHISPER_SERVER_HOST_IP=${ip_address}
-    export SPEECHT5_SERVER_HOST_IP=${ip_address}
-    export LLM_SERVER_HOST_IP=${ip_address}
-
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_PORT=3006
-
-    export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna
     export host_ip=${ip_address}
-
+    source set_env.sh
     # sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
     # Start Docker Containers

AudioQnA/tests/test_compose_tgi_on_gaudi.sh

@@ -34,21 +34,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct
-
-    export MEGA_SERVICE_HOST_IP=${ip_address}
-    export WHISPER_SERVER_HOST_IP=${ip_address}
-    export SPEECHT5_SERVER_HOST_IP=${ip_address}
-    export LLM_SERVER_HOST_IP=${ip_address}
-
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_PORT=3006
-
-    export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna
     export host_ip=${ip_address}
-
+    source set_env.sh
     # Start Docker Containers
     docker compose -f compose_tgi.yaml up -d > ${LOG_PATH}/start_services_with_compose.log
     n=0

AudioQnA/tests/test_compose_tgi_on_xeon.sh

@@ -34,21 +34,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon/
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct
-
-    export MEGA_SERVICE_HOST_IP=${ip_address}
-    export WHISPER_SERVER_HOST_IP=${ip_address}
-    export SPEECHT5_SERVER_HOST_IP=${ip_address}
-    export LLM_SERVER_HOST_IP=${ip_address}
-
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_PORT=3006
-
-    export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna
     export host_ip=${ip_address}
-
+    source set_env.sh
     # Start Docker Containers
     docker compose -f compose_tgi.yaml up -d > ${LOG_PATH}/start_services_with_compose.log
     n=0

AudioQnA/tests/test_compose_vllm_on_rocm.sh

@@ -33,27 +33,7 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/amd/gpu/rocm/
-
-    export host_ip=${ip_address}
-    export external_host_ip=${ip_address}
-    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-    export HF_CACHE_DIR="./data"
-    export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-    export VLLM_SERVICE_PORT="8081"
-
-    export MEGA_SERVICE_HOST_IP=${host_ip}
-    export WHISPER_SERVER_HOST_IP=${host_ip}
-    export SPEECHT5_SERVER_HOST_IP=${host_ip}
-    export LLM_SERVER_HOST_IP=${host_ip}
-
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_PORT=${VLLM_SERVICE_PORT}
-    export BACKEND_SERVICE_PORT=3008
-    export FRONTEND_SERVICE_PORT=5173
-
-    export BACKEND_SERVICE_ENDPOINT=http://${external_host_ip}:${BACKEND_SERVICE_PORT}/v1/audioqna
-
+    source set_env_vllm.sh
     sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
 
     # Start Docker Containers

AvatarChatbot/Dockerfile

@@ -1,8 +1,9 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+ARG IMAGE_REPO=opea
 ARG BASE_TAG=latest
-FROM opea/comps-base:$BASE_TAG
+FROM $IMAGE_REPO/comps-base:$BASE_TAG
 
 COPY ./avatarchatbot.py $HOME/avatarchatbot.py
 

AvatarChatbot/docker_compose/amd/gpu/rocm/compose.yaml

@@ -42,7 +42,7 @@ services:
     environment:
       TTS_ENDPOINT: ${TTS_ENDPOINT}
   tgi-service:
-    image: ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
+    image: ghcr.io/huggingface/text-generation-inference:2.4.1-rocm
     container_name: tgi-service
     ports:
       - "${TGI_SERVICE_PORT:-3006}:80"
@@ -66,24 +66,6 @@ services:
       - seccomp:unconfined
     ipc: host
     command: --model-id ${LLM_MODEL_ID} --max-input-length 4096 --max-total-tokens 8192
-  llm:
-    image: ${REGISTRY:-opea}/llm-textgen:${TAG:-latest}
-    container_name: llm-tgi-server
-    depends_on:
-      - tgi-service
-    ports:
-      - "3007:9000"
-    ipc: host
-    environment:
-      no_proxy: ${no_proxy}
-      http_proxy: ${http_proxy}
-      https_proxy: ${https_proxy}
-      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
-      LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
-      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
-      HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
-      OPENAI_API_KEY: ${OPENAI_API_KEY}
-    restart: unless-stopped
   wav2lip-service:
     image: ${REGISTRY:-opea}/wav2lip:${TAG:-latest}
     container_name: wav2lip-service
@@ -125,7 +107,7 @@ services:
     container_name: avatarchatbot-backend-server
     depends_on:
       - asr
-      - llm
+      - tgi-service
       - tts
       - animation
     ports:

AvatarChatbot/docker_compose/amd/gpu/rocm/set_env.sh

@@ -30,7 +30,7 @@ export ANIMATION_SERVICE_HOST_IP=${host_ip}
 export MEGA_SERVICE_PORT=8888
 export ASR_SERVICE_PORT=3001
 export TTS_SERVICE_PORT=3002
-export LLM_SERVICE_PORT=3007
+export LLM_SERVICE_PORT=3006
 export ANIMATION_SERVICE_PORT=3008
 
 export DEVICE="cpu"
@@ -41,7 +41,7 @@ export FACE="/home/user/comps/animation/src/assets/img/avatar5.png"
 # export AUDIO='assets/audio/eg3_ref.wav' # audio file path is optional, will use base64str in the post request as input if is 'None'
 export AUDIO='None'
 export FACESIZE=96
-export OUTFILE="/outputs/result.mp4"
+export OUTFILE="./outputs/result.mp4"
 export GFPGAN_MODEL_VERSION=1.4 # latest version, can roll back to v1.3 if needed
 export UPSCALE_FACTOR=1
-export FPS=10
+export FPS=5

AvatarChatbot/docker_compose/intel/cpu/xeon/set_env.sh

@@ -5,3 +5,32 @@
 pushd "../../../../../" > /dev/null
 source .set_env.sh
 popd > /dev/null
+
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export host_ip=$(hostname -I | awk '{print $1}')
+export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3
+export WAV2LIP_ENDPOINT=http://$host_ip:7860
+export MEGA_SERVICE_HOST_IP=${host_ip}
+export WHISPER_SERVER_HOST_IP=${host_ip}
+export WHISPER_SERVER_PORT=7066
+export SPEECHT5_SERVER_HOST_IP=${host_ip}
+export SPEECHT5_SERVER_PORT=7055
+export LLM_SERVER_HOST_IP=${host_ip}
+export LLM_SERVER_PORT=3006
+export ANIMATION_SERVICE_HOST_IP=${host_ip}
+export ANIMATION_SERVICE_PORT=3008
+
+export MEGA_SERVICE_PORT=8888
+
+export DEVICE="cpu"
+export WAV2LIP_PORT=7860
+export INFERENCE_MODE='wav2lip+gfpgan'
+export CHECKPOINT_PATH='/usr/local/lib/python3.11/site-packages/Wav2Lip/checkpoints/wav2lip_gan.pth'
+export FACE="/home/user/comps/animation/src/assets/img/avatar5.png"
+# export AUDIO='assets/audio/eg3_ref.wav' # audio file path is optional, will use base64str in the post request as input if is 'None'
+export AUDIO='None'
+export FACESIZE=96
+export OUTFILE="/outputs/result.mp4"
+export GFPGAN_MODEL_VERSION=1.4 # latest version, can roll back to v1.3 if needed
+export UPSCALE_FACTOR=1
+export FPS=10

AvatarChatbot/docker_compose/intel/hpu/gaudi/set_env.sh

@@ -5,3 +5,35 @@
 pushd "../../../../../" > /dev/null
 source .set_env.sh
 popd > /dev/null
+
+export HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN
+export host_ip=$(hostname -I | awk '{print $1}')
+
+export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3
+
+export WAV2LIP_ENDPOINT=http://$host_ip:7860
+
+export MEGA_SERVICE_HOST_IP=${host_ip}
+export WHISPER_SERVER_HOST_IP=${host_ip}
+export WHISPER_SERVER_PORT=7066
+export SPEECHT5_SERVER_HOST_IP=${host_ip}
+export SPEECHT5_SERVER_PORT=7055
+export LLM_SERVER_HOST_IP=${host_ip}
+export LLM_SERVER_PORT=3006
+export ANIMATION_SERVICE_HOST_IP=${host_ip}
+export ANIMATION_SERVICE_PORT=3008
+
+export MEGA_SERVICE_PORT=8888
+
+export DEVICE="hpu"
+export WAV2LIP_PORT=7860
+export INFERENCE_MODE='wav2lip+gfpgan'
+export CHECKPOINT_PATH='/usr/local/lib/python3.10/dist-packages/Wav2Lip/checkpoints/wav2lip_gan.pth'
+export FACE="/home/user/comps/animation/src/assets/img/avatar1.jpg"
+# export AUDIO='assets/audio/eg3_ref.wav' # audio file path is optional, will use base64str in the post request as input if is 'None'
+export AUDIO='None'
+export FACESIZE=96
+export OUTFILE="/outputs/result.mp4"
+export GFPGAN_MODEL_VERSION=1.4 # latest version, can roll back to v1.3 if needed
+export UPSCALE_FACTOR=1
+export FPS=10

AvatarChatbot/docker_image_build/build.yaml

@@ -5,6 +5,8 @@ services:
   avatarchatbot:
     build:
       args:
+        IMAGE_REPO: ${REGISTRY:-opea}
+        BASE_TAG: ${TAG:-latest}
         http_proxy: ${http_proxy}
         https_proxy: ${https_proxy}
         no_proxy: ${no_proxy}

AvatarChatbot/tests/README.md

@@ -0,0 +1,27 @@
+# AvatarChatbot E2E test scripts
+
+## Set the required environment variable
+
+```bash
+export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token"
+```
+
+## Run test
+
+On Intel Xeon with TGI:
+
+```bash
+bash test_compose_on_xeon.sh
+```
+
+On Intel Gaudi with TGI:
+
+```bash
+bash test_compose_on_gaudi.sh
+```
+
+On AMD ROCm with TGI:
+
+```bash
+bash test_compose_on_rocm.sh
+```

AvatarChatbot/tests/test_compose_on_gaudi.sh

@@ -24,19 +24,13 @@ ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="avatarchatbot whisper-gaudi speecht5-gaudi wav2lip-gaudi animation"
@@ -51,37 +45,7 @@ function build_docker_images() {
 function start_services() {
     cd $WORKPATH/docker_compose/intel/hpu/gaudi
 
-    export HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN
-    export host_ip=$(hostname -I | awk '{print $1}')
-
-    export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3
-
-    export WAV2LIP_ENDPOINT=http://$host_ip:7860
-
-    export MEGA_SERVICE_HOST_IP=${host_ip}
-    export WHISPER_SERVER_HOST_IP=${host_ip}
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_HOST_IP=${host_ip}
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_HOST_IP=${host_ip}
-    export LLM_SERVER_PORT=3006
-    export ANIMATION_SERVICE_HOST_IP=${host_ip}
-    export ANIMATION_SERVICE_PORT=3008
-
-    export MEGA_SERVICE_PORT=8888
-
-    export DEVICE="hpu"
-    export WAV2LIP_PORT=7860
-    export INFERENCE_MODE='wav2lip+gfpgan'
-    export CHECKPOINT_PATH='/usr/local/lib/python3.10/dist-packages/Wav2Lip/checkpoints/wav2lip_gan.pth'
-    export FACE="/home/user/comps/animation/src/assets/img/avatar1.jpg"
-    # export AUDIO='assets/audio/eg3_ref.wav' # audio file path is optional, will use base64str in the post request as input if is 'None'
-    export AUDIO='None'
-    export FACESIZE=96
-    export OUTFILE="/outputs/result.mp4"
-    export GFPGAN_MODEL_VERSION=1.4 # latest version, can roll back to v1.3 if needed
-    export UPSCALE_FACTOR=1
-    export FPS=10
+    source set_env.sh
 
     # Start Docker Containers
     docker compose up -d > ${LOG_PATH}/start_services_with_compose.log
@@ -128,19 +92,29 @@ function stop_docker() {
 
 
 function main() {
+    echo "::group::stop_docker"
     stop_docker
-    echo y | docker builder prune --all
-    echo y | docker image prune
+    echo "::endgroup::"
+    docker builder prune --all -f
+    docker image prune -f
 
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
-    start_services
-    # validate_microservices
-    validate_megaservice
-    # validate_frontend
+    echo "::endgroup::"
 
+    echo "::group::start_services"
+    start_services
+    echo "::endgroup::"
+
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
     stop_docker
-    echo y | docker builder prune --all
-    echo y | docker image prune
+    echo "::endgroup::"
+    docker builder prune --all -f
+    docker image prune -f
 
 }
 

AvatarChatbot/tests/test_compose_on_rocm.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 IMAGE_REPO=${IMAGE_REPO:-"opea"}
 IMAGE_TAG=${IMAGE_TAG:-"latest"}
 echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
@@ -25,9 +25,13 @@ ip_address=$(hostname -I | awk '{print $1}')
 function build_docker_images() {
     cd $WORKPATH/docker_image_build
     git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
-    service_list="avatarchatbot whisper asr llm-textgen speecht5 tts wav2lip animation"
+    service_list="avatarchatbot whisper asr speecht5 tts wav2lip animation"
     docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
     docker pull ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
@@ -38,48 +42,8 @@ function build_docker_images() {
 
 function start_services() {
     cd $WORKPATH/docker_compose/amd/gpu/rocm
-
-    export HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN
     export OPENAI_API_KEY=$OPENAI_API_KEY
-    export host_ip=${ip_address}
-
-    export TGI_SERVICE_PORT=3006
-    export TGI_LLM_ENDPOINT=http://${host_ip}:${TGI_SERVICE_PORT}
-    export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
-
-    export ASR_ENDPOINT=http://${host_ip}:7066
-    export TTS_ENDPOINT=http://${host_ip}:7055
-    export WAV2LIP_ENDPOINT=http://${host_ip}:7860
-
-    export MEGA_SERVICE_HOST_IP=${host_ip}
-    export ASR_SERVICE_HOST_IP=${host_ip}
-    export TTS_SERVICE_HOST_IP=${host_ip}
-    export LLM_SERVICE_HOST_IP=${host_ip}
-    export ANIMATION_SERVICE_HOST_IP=${host_ip}
-    export WHISPER_SERVER_HOST_IP=${host_ip}
-    export WHISPER_SERVER_PORT=7066
-
-    export SPEECHT5_SERVER_HOST_IP=${host_ip}
-    export SPEECHT5_SERVER_PORT=7055
-
-    export MEGA_SERVICE_PORT=8888
-    export ASR_SERVICE_PORT=3001
-    export TTS_SERVICE_PORT=3002
-    export LLM_SERVICE_PORT=3007
-    export ANIMATION_SERVICE_PORT=3008
-
-    export DEVICE="cpu"
-    export WAV2LIP_PORT=7860
-    export INFERENCE_MODE='wav2lip+gfpgan'
-    export CHECKPOINT_PATH='/usr/local/lib/python3.11/site-packages/Wav2Lip/checkpoints/wav2lip_gan.pth'
-    export FACE="/home/user/comps/animation/src/assets/img/avatar5.png"
-    # export AUDIO='assets/audio/eg3_ref.wav' # audio file path is optional, will use base64str in the post request as input if is 'None'
-    export AUDIO='None'
-    export FACESIZE=96
-    export OUTFILE="./outputs/result.mp4"
-    export GFPGAN_MODEL_VERSION=1.4 # latest version, can roll back to v1.3 if needed
-    export UPSCALE_FACTOR=1
-    export FPS=5
+    source set_env.sh
 
     # Start Docker Containers
     docker compose up -d --force-recreate
@@ -138,11 +102,6 @@ function validate_megaservice() {
 }
 
 
-#function validate_frontend() {
-
-#}
-
-
 function stop_docker() {
     cd $WORKPATH/docker_compose/amd/gpu/rocm
     docker compose down && docker compose rm -f
@@ -151,19 +110,27 @@ function stop_docker() {
 
 function main() {
 
-    echo $OPENAI_API_KEY
-    echo $OPENAI_KEY
-
+    echo "::group::stop_docker"
     stop_docker
+    echo "::endgroup::"
+
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
-    start_services
-    # validate_microservices
-    sleep 30
-    validate_megaservice
-    # validate_frontend
-    stop_docker
+    echo "::endgroup::"
 
-    echo y | docker system prune
+    echo "::group::start_services"
+    start_services
+    echo "::endgroup::"
+
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
+    stop_docker
+    echo "::endgroup::"
+
+    docker system prune -f
 
 }
 

AvatarChatbot/tests/test_compose_on_xeon.sh

@@ -24,19 +24,13 @@ ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="avatarchatbot whisper speecht5 wav2lip animation"
@@ -51,37 +45,7 @@ function build_docker_images() {
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon
 
-    export HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN
-    export host_ip=$(hostname -I | awk '{print $1}')
-
-    export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3
-
-    export WAV2LIP_ENDPOINT=http://$host_ip:7860
-
-    export MEGA_SERVICE_HOST_IP=${host_ip}
-    export WHISPER_SERVER_HOST_IP=${host_ip}
-    export WHISPER_SERVER_PORT=7066
-    export SPEECHT5_SERVER_HOST_IP=${host_ip}
-    export SPEECHT5_SERVER_PORT=7055
-    export LLM_SERVER_HOST_IP=${host_ip}
-    export LLM_SERVER_PORT=3006
-    export ANIMATION_SERVICE_HOST_IP=${host_ip}
-    export ANIMATION_SERVICE_PORT=3008
-
-    export MEGA_SERVICE_PORT=8888
-
-    export DEVICE="cpu"
-    export WAV2LIP_PORT=7860
-    export INFERENCE_MODE='wav2lip+gfpgan'
-    export CHECKPOINT_PATH='/usr/local/lib/python3.11/site-packages/Wav2Lip/checkpoints/wav2lip_gan.pth'
-    export FACE="/home/user/comps/animation/src/assets/img/avatar5.png"
-    # export AUDIO='assets/audio/eg3_ref.wav' # audio file path is optional, will use base64str in the post request as input if is 'None'
-    export AUDIO='None'
-    export FACESIZE=96
-    export OUTFILE="/outputs/result.mp4"
-    export GFPGAN_MODEL_VERSION=1.4 # latest version, can roll back to v1.3 if needed
-    export UPSCALE_FACTOR=1
-    export FPS=10
+    source set_env.sh
 
     # Start Docker Containers
     docker compose up -d
@@ -127,16 +91,28 @@ function stop_docker() {
 
 
 function main() {
-    stop_docker
-    if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
-    start_services
-    # validate_microservices
-    validate_megaservice
-    # validate_frontend
-    stop_docker
 
-    echo y | docker builder prune --all
-    echo y | docker image prune
+    echo "::group::stop_docker"
+    stop_docker
+    echo "::endgroup::"
+
+    echo "::group::build_docker_images"
+    if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
+
+    echo "::group::start_services"
+    start_services
+    echo "::endgroup::"
+
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
+    stop_docker
+    echo "::endgroup::"
+
+    docker system prune -f
 
 }
 

ChatQnA/README.md

@@ -96,20 +96,21 @@ flowchart LR
 
 The table below lists currently available deployment options. They outline in detail the implementation of this example on selected hardware.
 
-| Category                | Deployment Option            | Description                                                                                                                                                                                                                                                                          |
-| ----------------------- | ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| On-premise Deployments  | Docker compose               | [ChatQnA deployment on Xeon](./docker_compose/intel/cpu/xeon)                                                                                                                                                                                                                        |
-|                         |                              | [ChatQnA deployment on AI PC](./docker_compose/intel/cpu/aipc)                                                                                                                                                                                                                       |
-|                         |                              | [ChatQnA deployment on Gaudi](./docker_compose/intel/hpu/gaudi)                                                                                                                                                                                                                      |
-|                         |                              | [ChatQnA deployment on Nvidia GPU](./docker_compose/nvidia/gpu)                                                                                                                                                                                                                      |
-|                         |                              | [ChatQnA deployment on AMD ROCm](./docker_compose/amd/gpu/rocm)                                                                                                                                                                                                                      |
-|                         | Kubernetes                   | [Helm Charts](./kubernetes/helm)                                                                                                                                                                                                                                                     |
-| Cloud Service Providers | AWS                          | [Terraform deployment on 4th Gen Intel Xeon with Intel AMX using meta-llama/Meta-Llama-3-8B-Instruct ](https://github.com/intel/terraform-intel-aws-vm/tree/main/examples/gen-ai-xeon-opea-chatqna)                                                                                  |
-|                         |                              | [Terraform deployment on 4th Gen Intel Xeon with Intel AMX using TII Falcon2-11B](https://github.com/intel/terraform-intel-aws-vm/tree/main/examples/gen-ai-xeon-opea-chatqna-falcon11B)                                                                                             |
-|                         | GCP                          | [Terraform deployment on 5th Gen Intel Xeon with Intel AMX(support Confidential AI by using Intel® TDX](https://github.com/intel/terraform-intel-gcp-vm/tree/main/examples/gen-ai-xeon-opea-chatqna)                                                                                |
-|                         | Azure                        | [Terraform deployment on 4th/5th Gen Intel Xeon with Intel AMX & Intel TDX](https://github.com/intel/terraform-intel-azure-linux-vm/tree/main/examples/azure-gen-ai-xeon-opea-chatqna-tdx)                                                                                           |
-|                         | Intel Tiber AI Cloud         | Coming Soon                                                                                                                                                                                                                                                                          |
-|                         | Any Xeon based Ubuntu system | [ChatQnA Ansible Module for Ubuntu 20.04](https://github.com/intel/optimized-cloud-recipes/tree/main/recipes/ai-opea-chatqna-xeon) .Use this if you are not using Terraform and have provisioned your system either manually or with another tool, including directly on bare metal. |
+| Category                                                                                                                       | Deployment Option            | Description                                                                                                                                                                                                                                                                          |
+| ------------------------------------------------------------------------------------------------------------------------------ | ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| On-premise Deployments                                                                                                         | Docker compose               | [ChatQnA deployment on Xeon](./docker_compose/intel/cpu/xeon/README.md)                                                                                                                                                                                                              |
+|                                                                                                                                |                              | [ChatQnA deployment on AI PC](./docker_compose/intel/cpu/aipc/README.md)                                                                                                                                                                                                             |
+|                                                                                                                                |                              | [ChatQnA deployment on Gaudi](./docker_compose/intel/hpu/gaudi/README.md)                                                                                                                                                                                                            |
+|                                                                                                                                |                              | [ChatQnA deployment on Nvidia GPU](./docker_compose/nvidia/gpu/README.md)                                                                                                                                                                                                            |
+|                                                                                                                                |                              | [ChatQnA deployment on AMD ROCm](./docker_compose/amd/gpu/rocm/README.md)                                                                                                                                                                                                            |
+| Cloud Platforms Deployment on AWS, GCP, Azure, IBM Cloud,Oracle Cloud, [Intel® Tiber™ AI Cloud](https://ai.cloud.intel.com/) | Docker Compose               | [Getting Started Guide: Deploy the ChatQnA application across multiple cloud platforms](https://github.com/opea-project/docs/tree/main/getting-started/README.md)                                                                                                                    |
+|                                                                                                                                | Kubernetes                   | [Helm Charts](./kubernetes/helm/README.md)                                                                                                                                                                                                                                           |
+| Automated Terraform Deployment on Cloud Service Providers                                                                      | AWS                          | [Terraform deployment on 4th Gen Intel Xeon with Intel AMX using meta-llama/Meta-Llama-3-8B-Instruct ](https://github.com/intel/terraform-intel-aws-vm/tree/main/examples/gen-ai-xeon-opea-chatqna)                                                                                  |
+|                                                                                                                                |                              | [Terraform deployment on 4th Gen Intel Xeon with Intel AMX using TII Falcon2-11B](https://github.com/intel/terraform-intel-aws-vm/tree/main/examples/gen-ai-xeon-opea-chatqna-falcon11B)                                                                                             |
+|                                                                                                                                | GCP                          | [Terraform deployment on 5th Gen Intel Xeon with Intel AMX(support Confidential AI by using Intel® TDX](https://github.com/intel/terraform-intel-gcp-vm/tree/main/examples/gen-ai-xeon-opea-chatqna)                                                                                |
+|                                                                                                                                | Azure                        | [Terraform deployment on 4th/5th Gen Intel Xeon with Intel AMX & Intel TDX](https://github.com/intel/terraform-intel-azure-linux-vm/tree/main/examples/azure-gen-ai-xeon-opea-chatqna-tdx)                                                                                           |
+|                                                                                                                                | Intel Tiber AI Cloud         | Coming Soon                                                                                                                                                                                                                                                                          |
+|                                                                                                                                | Any Xeon based Ubuntu system | [ChatQnA Ansible Module for Ubuntu 20.04](https://github.com/intel/optimized-cloud-recipes/tree/main/recipes/ai-opea-chatqna-xeon). Use this if you are not using Terraform and have provisioned your system either manually or with another tool, including directly on bare metal. |
 
 ## Monitor and Tracing
 

ChatQnA/benchmark/performance/kubernetes/intel/gaudi/README.md

@@ -1,192 +0,0 @@
-# ChatQnA Benchmarking
-
-This folder contains a collection of Kubernetes manifest files for deploying the ChatQnA service across scalable nodes. It includes a comprehensive [benchmarking tool](https://github.com/opea-project/GenAIEval/blob/main/evals/benchmark/README.md) that enables throughput analysis to assess inference performance.
-
-By following this guide, you can run benchmarks on your deployment and share the results with the OPEA community.
-
-## Purpose
-
-We aim to run these benchmarks and share them with the OPEA community for three primary reasons:
-
-- To offer insights on inference throughput in real-world scenarios, helping you choose the best service or deployment for your needs.
-- To establish a baseline for validating optimization solutions across different implementations, providing clear guidance on which methods are most effective for your use case.
-- To inspire the community to build upon our benchmarks, allowing us to better quantify new solutions in conjunction with current leading llms, serving frameworks etc.
-
-## Metrics
-
-The benchmark will report the below metrics, including:
-
-- Number of Concurrent Requests
-- End-to-End Latency: P50, P90, P99 (in milliseconds)
-- End-to-End First Token Latency: P50, P90, P99 (in milliseconds)
-- Average Next Token Latency (in milliseconds)
-- Average Token Latency (in milliseconds)
-- Requests Per Second (RPS)
-- Output Tokens Per Second
-- Input Tokens Per Second
-
-Results will be displayed in the terminal and saved as CSV file named `1_stats.csv` for easy export to spreadsheets.
-
-## Table of Contents
-
-- [Deployment](#deployment)
-  - [Prerequisites](#prerequisites)
-  - [Deployment Scenarios](#deployment-scenarios)
-    - [Case 1: Baseline Deployment with Rerank](#case-1-baseline-deployment-with-rerank)
-    - [Case 2: Baseline Deployment without Rerank](#case-2-baseline-deployment-without-rerank)
-    - [Case 3: Tuned Deployment with Rerank](#case-3-tuned-deployment-with-rerank)
-- [Benchmark](#benchmark)
-  - [Test Configurations](#test-configurations)
-  - [Test Steps](#test-steps)
-    - [Upload Retrieval File](#upload-retrieval-file)
-    - [Run Benchmark Test](#run-benchmark-test)
-    - [Data collection](#data-collection)
-- [Teardown](#teardown)
-
-## Deployment
-
-### Prerequisites
-
-- Kubernetes installation: Use [kubespray](https://github.com/opea-project/docs/blob/main/guide/installation/k8s_install/k8s_install_kubespray.md) or other official Kubernetes installation guides: 
-- (Optional) [Kubernetes set up guide on Intel Gaudi product](https://github.com/opea-project/GenAIInfra/blob/main/README.md#setup-kubernetes-cluster) 
-- Helm installation: Follow the [Helm documentation](https://helm.sh/docs/intro/install/#helm) to install Helm.
-- Setup Hugging Face Token
-
-  To access models and APIs from Hugging Face, set your token as environment variable.
-  ```bash
-  export HF_TOKEN="insert-your-huggingface-token-here"
-  ```
-- Prepare Shared Models (Optional but Strongly Recommended)
-
-  Downloading models simultaneously to multiple nodes in your cluster can overload resources such as network bandwidth, memory and storage. To prevent resource exhaustion, it's recommended to preload the models in advance.
-  ```bash
-  pip install -U "huggingface_hub[cli]"
-  sudo mkdir -p /mnt/models
-  sudo chmod 777 /mnt/models
-  huggingface-cli download --cache-dir /mnt/models Intel/neural-chat-7b-v3-3
-  export MODEL_DIR=/mnt/models
-  ```
-  Once the models are downloaded, you can consider the following methods for sharing them across nodes:
-  - Persistent Volume Claim (PVC): This is the recommended approach for production setups. For more details on using PVC, refer to [PVC](https://github.com/opea-project/GenAIInfra/blob/main/helm-charts/README.md#using-persistent-volume).
-  - Local Host Path: For simpler testing, ensure that each node involved in the deployment follows the steps above to locally prepare the models. After preparing the models, use `--set global.modelUseHostPath=${MODELDIR}` in the deployment command.
-
-- Label Nodes
-  ```base
-  python deploy.py --add-label --num-nodes 2
-  ```
-
-### Deployment Scenarios
-
-The example below are based on a two-node setup. You can adjust the number of nodes by using the `--num-nodes` option.
-
-By default, these commands use the `default` namespace. To specify a different namespace, use the `--namespace` flag with deploy, uninstall, and kubernetes command. Additionally, update the `namespace` field in `benchmark.yaml` before running the benchmark test.
-
-For additional configuration options, run `python deploy.py --help`
-
-#### Case 1: Baseline Deployment with Rerank
-
-Deploy Command (with node number, Hugging Face token, model directory specified):
-```bash
-python deploy.py --hf-token $HF_TOKEN --model-dir $MODEL_DIR --num-nodes 2 --with-rerank
-```
-Uninstall Command:
-```bash
-python deploy.py --uninstall
-```
-
-#### Case 2: Baseline Deployment without Rerank
-
-```bash
-python deploy.py --hf-token $HFTOKEN --model-dir $MODELDIR --num-nodes 2
-```
-#### Case 3: Tuned Deployment with Rerank
-
-```bash
-python deploy.py --hf-token $HFTOKEN --model-dir $MODELDIR --num-nodes 2 --with-rerank --tuned
-```
-
-## Benchmark
-
-### Test Configurations
-
-| Key      | Value   |
-| -------- | ------- |
-| Workload | ChatQnA |
-| Tag      | V1.1    |
-
-Models configuration
-| Key | Value |
-| ---------- | ------------------ |
-| Embedding | BAAI/bge-base-en-v1.5 |
-| Reranking | BAAI/bge-reranker-base |
-| Inference | Intel/neural-chat-7b-v3-3 |
-
-Benchmark parameters
-| Key | Value |
-| ---------- | ------------------ |
-| LLM input tokens | 1024 |
-| LLM output tokens | 128 |
-
-Number of test requests for different scheduled node number:
-| Node count | Concurrency | Query number |
-| ----- | -------- | -------- |
-| 1 | 128 | 640 |
-| 2 | 256 | 1280 |
-| 4 | 512 | 2560 |
-
-More detailed configuration can be found in configuration file [benchmark.yaml](./benchmark.yaml).
-
-### Test Steps
-
-Use `kubectl get pods` to confirm that all pods are `READY` before starting the test.
-
-#### Upload Retrieval File
-
-Before testing, upload a specified file to make sure the llm input have the token length of 1k.
-
-Get files:
-
-```bash
-wget https://github.com/opea-project/GenAIEval/tree/main/evals/benchmark/data/upload_file.txt
-```
-
-Retrieve the `ClusterIP` of the `chatqna-data-prep` service.
-
-```bash
-kubectl get svc
-```
-Expected output:
-```log
-chatqna-data-prep         ClusterIP   xx.xx.xx.xx    <none>        6007/TCP            51m
-```
-
-Use the following `cURL` command to upload file:
-
-```bash
-cd GenAIEval/evals/benchmark/data
-curl -X POST "http://${cluster_ip}:6007/v1/dataprep/ingest" \
-     -H "Content-Type: multipart/form-data" \
-     -F "chunk_size=3800" \
-     -F "files=@./upload_file.txt"
-```
-
-#### Run Benchmark Test
-
-Run the benchmark test using:
-```bash
-bash benchmark.sh -n 2
-```
-The `-n` argument specifies the number of test nodes. Required dependencies will be automatically installed when running the benchmark for the first time.
-
-#### Data collection
-
-All the test results will come to the folder `GenAIEval/evals/benchmark/benchmark_output`.
-
-## Teardown
-
-After completing the benchmark, use the following command to clean up the environment:
-
-Remove Node Labels:
-```bash
-python deploy.py --delete-label
-```

ChatQnA/benchmark/performance/kubernetes/intel/gaudi/benchmark.sh

@@ -1,102 +0,0 @@
-#!/bin/bash
-
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-
-deployment_type="k8s"
-node_number=1
-service_port=8888
-query_per_node=640
-
-benchmark_tool_path="$(pwd)/GenAIEval"
-
-usage() {
-    echo "Usage: $0 [-d deployment_type] [-n node_number] [-i service_ip] [-p service_port]"
-    echo "  -d deployment_type    ChatQnA deployment type, select between k8s and docker (default: k8s)"
-    echo "  -n node_number        Test node number, required only for k8s deployment_type, (default: 1)"
-    echo "  -i service_ip         chatqna service ip, required only for docker deployment_type"
-    echo "  -p service_port       chatqna service port, required only for docker deployment_type, (default: 8888)"
-    exit 1
-}
-
-while getopts ":d:n:i:p:" opt; do
-    case ${opt} in
-        d )
-            deployment_type=$OPTARG
-            ;;
-        n )
-            node_number=$OPTARG
-            ;;
-        i )
-            service_ip=$OPTARG
-            ;;
-        p )
-            service_port=$OPTARG
-            ;;
-        \? )
-            echo "Invalid option: -$OPTARG" 1>&2
-            usage
-            ;;
-        : )
-            echo "Invalid option: -$OPTARG requires an argument" 1>&2
-            usage
-            ;;
-    esac
-done
-
-if [[ "$deployment_type" == "docker" && -z "$service_ip" ]]; then
-    echo "Error: service_ip is required for docker deployment_type" 1>&2
-    usage
-fi
-
-if [[ "$deployment_type" == "k8s" && ( -n "$service_ip" || -n "$service_port" ) ]]; then
-    echo "Warning: service_ip and service_port are ignored for k8s deployment_type" 1>&2
-fi
-
-function main() {
-    if [[ ! -d ${benchmark_tool_path} ]]; then
-        echo "Benchmark tool not found, setting up..."
-        setup_env
-    fi
-    run_benchmark
-}
-
-function setup_env() {
-    git clone https://github.com/opea-project/GenAIEval.git
-    pushd ${benchmark_tool_path}
-    python3 -m venv stress_venv
-    source stress_venv/bin/activate
-    pip install -r requirements.txt
-    popd
-}
-
-function run_benchmark() {
-    source ${benchmark_tool_path}/stress_venv/bin/activate
-    export DEPLOYMENT_TYPE=${deployment_type}
-    export SERVICE_IP=${service_ip:-"None"}
-    export SERVICE_PORT=${service_port:-"None"}
-    export LOAD_SHAPE=${load_shape:-"constant"}
-    export CONCURRENT_LEVEL=${concurrent_level:-5}
-    export ARRIVAL_RATE=${arrival_rate:-1.0}
-    if [[ -z $USER_QUERIES ]]; then
-        user_query=$((query_per_node*node_number))
-        export USER_QUERIES="[${user_query}, ${user_query}, ${user_query}, ${user_query}]"
-        echo "USER_QUERIES not configured, setting to: ${USER_QUERIES}."
-    fi
-    export WARMUP=$(echo $USER_QUERIES | sed -e 's/[][]//g' -e 's/,.*//')
-    if [[ -z $WARMUP ]]; then export WARMUP=0; fi
-    if [[ -z $TEST_OUTPUT_DIR ]]; then
-        if [[ $DEPLOYMENT_TYPE == "k8s" ]]; then
-            export TEST_OUTPUT_DIR="${benchmark_tool_path}/evals/benchmark/benchmark_output/node_${node_number}"
-        else
-            export TEST_OUTPUT_DIR="${benchmark_tool_path}/evals/benchmark/benchmark_output/docker"
-        fi
-        echo "TEST_OUTPUT_DIR not configured, setting to: ${TEST_OUTPUT_DIR}."
-    fi
-
-    envsubst < ./benchmark.yaml > ${benchmark_tool_path}/evals/benchmark/benchmark.yaml
-    cd ${benchmark_tool_path}/evals/benchmark
-    python benchmark.py
-}
-
-main

ChatQnA/benchmark/performance/kubernetes/intel/gaudi/benchmark.yaml

@@ -1,68 +0,0 @@
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-
-test_suite_config: # Overall configuration settings for the test suite
-  examples: ["chatqna"]  # The specific test cases being tested, e.g., chatqna, codegen, codetrans, faqgen, audioqna, visualqna
-  deployment_type: ${DEPLOYMENT_TYPE}  # Default is "k8s", can also be "docker"
-  service_ip: ${SERVICE_IP}  # Leave as None for k8s, specify for Docker
-  service_port: ${SERVICE_PORT}  # Leave as None for k8s, specify for Docker
-  warm_ups: ${WARMUP}  # Number of test requests for warm-up
-  run_time: 60m  # The max total run time for the test suite
-  seed:  # The seed for all RNGs
-  user_queries: ${USER_QUERIES}  # Number of test requests at each concurrency level
-  query_timeout: 120  # Number of seconds to wait for a simulated user to complete any executing task before exiting. 120 sec by defeult.
-  random_prompt: false  # Use random prompts if true, fixed prompts if false
-  collect_service_metric: false  # Collect service metrics if true, do not collect service metrics if false
-  data_visualization: false # Generate data visualization if true, do not generate data visualization if false
-  llm_model: "Intel/neural-chat-7b-v3-3"  # The LLM model used for the test
-  test_output_dir: "${TEST_OUTPUT_DIR}"  # The directory to store the test output
-  load_shape:              # Tenant concurrency pattern
-    name: ${LOAD_SHAPE}      # poisson or constant(locust default load shape)
-    params:                  # Loadshape-specific parameters
-      constant:                # Constant load shape specific parameters, activate only if load_shape.name is constant
-        concurrent_level: ${CONCURRENT_LEVEL}      # If user_queries is specified, concurrent_level is target number of requests per user. If not, it is the number of simulated users
-      poisson:                 # Poisson load shape specific parameters, activate only if load_shape.name is poisson
-        arrival_rate: ${ARRIVAL_RATE}        # Request arrival rate
-
-test_cases:
-  chatqna:
-    embedding:
-      run_test: false
-      service_name: "chatqna-embedding-usvc"  # Replace with your service name
-    embedserve:
-      run_test: false
-      service_name: "chatqna-tei"  # Replace with your service name
-    retriever:
-      run_test: false
-      service_name: "chatqna-retriever-usvc"  # Replace with your service name
-      parameters:
-        search_type: "similarity"
-        k: 1
-        fetch_k: 20
-        lambda_mult: 0.5
-        score_threshold: 0.2
-    reranking:
-      run_test: false
-      service_name: "chatqna-reranking-usvc"  # Replace with your service name
-      parameters:
-        top_n: 1
-    rerankserve:
-      run_test: false
-      service_name: "chatqna-teirerank"  # Replace with your service name
-    llm:
-      run_test: false
-      service_name: "chatqna-llm-uservice"  # Replace with your service name
-      parameters:
-        max_tokens: 128
-        temperature: 0.01
-        top_k: 10
-        top_p: 0.95
-        repetition_penalty: 1.03
-        stream: true
-    llmserve:
-      run_test: false
-      service_name: "chatqna-tgi"  # Replace with your service name
-    e2e:
-      run_test: true
-      service_name: "chatqna"  # Replace with your service name
-      k: 1

ChatQnA/benchmark/performance/kubernetes/intel/gaudi/deploy.py

@@ -1,278 +0,0 @@
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-
-import argparse
-import glob
-import json
-import os
-import shutil
-import subprocess
-import sys
-
-from generate_helm_values import generate_helm_values
-
-
-def run_kubectl_command(command):
-    """Run a kubectl command and return the output."""
-    try:
-        result = subprocess.run(command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-        return result.stdout
-    except subprocess.CalledProcessError as e:
-        print(f"Error running command: {command}\n{e.stderr}")
-        exit(1)
-
-
-def get_all_nodes():
-    """Get the list of all nodes in the Kubernetes cluster."""
-    command = ["kubectl", "get", "nodes", "-o", "json"]
-    output = run_kubectl_command(command)
-    nodes = json.loads(output)
-    return [node["metadata"]["name"] for node in nodes["items"]]
-
-
-def add_label_to_node(node_name, label):
-    """Add a label to the specified node."""
-    command = ["kubectl", "label", "node", node_name, label, "--overwrite"]
-    print(f"Labeling node {node_name} with {label}...")
-    run_kubectl_command(command)
-    print(f"Label {label} added to node {node_name} successfully.")
-
-
-def add_labels_to_nodes(node_count=None, label=None, node_names=None):
-    """Add a label to the specified number of nodes or to specified nodes."""
-
-    if node_names:
-        # Add label to the specified nodes
-        for node_name in node_names:
-            add_label_to_node(node_name, label)
-    else:
-        # Fetch the node list and label the specified number of nodes
-        all_nodes = get_all_nodes()
-        if node_count is None or node_count > len(all_nodes):
-            print(f"Error: Node count exceeds the number of available nodes ({len(all_nodes)} available).")
-            sys.exit(1)
-
-        selected_nodes = all_nodes[:node_count]
-        for node_name in selected_nodes:
-            add_label_to_node(node_name, label)
-
-
-def clear_labels_from_nodes(label, node_names=None):
-    """Clear the specified label from specific nodes if provided, otherwise from all nodes."""
-    label_key = label.split("=")[0]  # Extract key from 'key=value' format
-
-    # If specific nodes are provided, use them; otherwise, get all nodes
-    nodes_to_clear = node_names if node_names else get_all_nodes()
-
-    for node_name in nodes_to_clear:
-        # Check if the node has the label by inspecting its metadata
-        command = ["kubectl", "get", "node", node_name, "-o", "json"]
-        node_info = run_kubectl_command(command)
-        node_metadata = json.loads(node_info)
-
-        # Check if the label exists on this node
-        labels = node_metadata["metadata"].get("labels", {})
-        if label_key in labels:
-            # Remove the label from the node
-            command = ["kubectl", "label", "node", node_name, f"{label_key}-"]
-            print(f"Removing label {label_key} from node {node_name}...")
-            run_kubectl_command(command)
-            print(f"Label {label_key} removed from node {node_name} successfully.")
-        else:
-            print(f"Label {label_key} not found on node {node_name}, skipping.")
-
-
-def install_helm_release(release_name, chart_name, namespace, values_file, device_type):
-    """Deploy a Helm release with a specified name and chart.
-
-    Parameters:
-    - release_name: The name of the Helm release.
-    - chart_name: The Helm chart name or path, e.g., "opea/chatqna".
-    - namespace: The Kubernetes namespace for deployment.
-    - values_file: The user values file for deployment.
-    - device_type: The device type (e.g., "gaudi") for specific configurations (optional).
-    """
-
-    # Check if the namespace exists; if not, create it
-    try:
-        # Check if the namespace exists
-        command = ["kubectl", "get", "namespace", namespace]
-        subprocess.run(command, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    except subprocess.CalledProcessError:
-        # Namespace does not exist, create it
-        print(f"Namespace '{namespace}' does not exist. Creating it...")
-        command = ["kubectl", "create", "namespace", namespace]
-        subprocess.run(command, check=True)
-        print(f"Namespace '{namespace}' created successfully.")
-
-    # Handle gaudi-specific values file if device_type is "gaudi"
-    hw_values_file = None
-    untar_dir = None
-    if device_type == "gaudi":
-        print("Device type is gaudi. Pulling Helm chart to get gaudi-values.yaml...")
-
-        # Combine chart_name with fixed prefix
-        chart_pull_url = f"oci://ghcr.io/opea-project/charts/{chart_name}"
-
-        # Pull and untar the chart
-        subprocess.run(["helm", "pull", chart_pull_url, "--untar"], check=True)
-
-        # Find the untarred directory
-        untar_dirs = glob.glob(f"{chart_name}*")
-        if untar_dirs:
-            untar_dir = untar_dirs[0]
-            hw_values_file = os.path.join(untar_dir, "gaudi-values.yaml")
-            print("gaudi-values.yaml pulled and ready for use.")
-        else:
-            print(f"Error: Could not find untarred directory for {chart_name}")
-            return
-
-    # Prepare the Helm install command
-    command = ["helm", "install", release_name, chart_name, "--namespace", namespace]
-
-    # Append additional values file for gaudi if it exists
-    if hw_values_file:
-        command.extend(["-f", hw_values_file])
-
-    # Append the main values file
-    command.extend(["-f", values_file])
-
-    # Execute the Helm install command
-    try:
-        print(f"Running command: {' '.join(command)}")  # Print full command for debugging
-        subprocess.run(command, check=True)
-        print("Deployment initiated successfully.")
-    except subprocess.CalledProcessError as e:
-        print(f"Error occurred while deploying Helm release: {e}")
-
-    # Cleanup: Remove the untarred directory
-    if untar_dir and os.path.isdir(untar_dir):
-        print(f"Removing temporary directory: {untar_dir}")
-        shutil.rmtree(untar_dir)
-        print("Temporary directory removed successfully.")
-
-
-def uninstall_helm_release(release_name, namespace=None):
-    """Uninstall a Helm release and clean up resources, optionally delete the namespace if not 'default'."""
-    # Default to 'default' namespace if none is specified
-    if not namespace:
-        namespace = "default"
-
-    try:
-        # Uninstall the Helm release
-        command = ["helm", "uninstall", release_name, "--namespace", namespace]
-        print(f"Uninstalling Helm release {release_name} in namespace {namespace}...")
-        run_kubectl_command(command)
-        print(f"Helm release {release_name} uninstalled successfully.")
-
-        # If the namespace is specified and not 'default', delete it
-        if namespace != "default":
-            print(f"Deleting namespace {namespace}...")
-            delete_namespace_command = ["kubectl", "delete", "namespace", namespace]
-            run_kubectl_command(delete_namespace_command)
-            print(f"Namespace {namespace} deleted successfully.")
-        else:
-            print("Namespace is 'default', skipping deletion.")
-
-    except subprocess.CalledProcessError as e:
-        print(f"Error occurred while uninstalling Helm release or deleting namespace: {e}")
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Manage Helm Deployment.")
-    parser.add_argument(
-        "--release-name",
-        type=str,
-        default="chatqna",
-        help="The Helm release name created during deployment (default: chatqna).",
-    )
-    parser.add_argument(
-        "--chart-name",
-        type=str,
-        default="chatqna",
-        help="The chart name to deploy, composed of repo name and chart name (default: chatqna).",
-    )
-    parser.add_argument("--namespace", default="default", help="Kubernetes namespace (default: default).")
-    parser.add_argument("--hf-token", help="Hugging Face API token.")
-    parser.add_argument(
-        "--model-dir", help="Model directory, mounted as volumes for service access to pre-downloaded models"
-    )
-    parser.add_argument("--user-values", help="Path to a user-specified values.yaml file.")
-    parser.add_argument(
-        "--create-values-only", action="store_true", help="Only create the values.yaml file without deploying."
-    )
-    parser.add_argument("--uninstall", action="store_true", help="Uninstall the Helm release.")
-    parser.add_argument("--num-nodes", type=int, default=1, help="Number of nodes to use (default: 1).")
-    parser.add_argument("--node-names", nargs="*", help="Optional specific node names to label.")
-    parser.add_argument("--add-label", action="store_true", help="Add label to specified nodes if this flag is set.")
-    parser.add_argument(
-        "--delete-label", action="store_true", help="Delete label from specified nodes if this flag is set."
-    )
-    parser.add_argument(
-        "--label", default="node-type=opea-benchmark", help="Label to add/delete (default: node-type=opea-benchmark)."
-    )
-    parser.add_argument("--with-rerank", action="store_true", help="Include rerank service in the deployment.")
-    parser.add_argument(
-        "--tuned",
-        action="store_true",
-        help="Modify resources for services and change extraCmdArgs when creating values.yaml.",
-    )
-    parser.add_argument(
-        "--device-type",
-        type=str,
-        choices=["cpu", "gaudi"],
-        default="gaudi",
-        help="Specify the device type for deployment (choices: 'cpu', 'gaudi'; default: gaudi).",
-    )
-
-    args = parser.parse_args()
-
-    # Adjust num-nodes based on node-names if specified
-    if args.node_names:
-        num_node_names = len(args.node_names)
-        if args.num_nodes != 1 and args.num_nodes != num_node_names:
-            parser.error("--num-nodes must match the number of --node-names if both are specified.")
-        else:
-            args.num_nodes = num_node_names
-
-    # Node labeling management
-    if args.add_label:
-        add_labels_to_nodes(args.num_nodes, args.label, args.node_names)
-        return
-    elif args.delete_label:
-        clear_labels_from_nodes(args.label, args.node_names)
-        return
-
-    # Uninstall Helm release if specified
-    if args.uninstall:
-        uninstall_helm_release(args.release_name, args.namespace)
-        return
-
-    # Prepare values.yaml if not uninstalling
-    if args.user_values:
-        values_file_path = args.user_values
-    else:
-        if not args.hf_token:
-            parser.error("--hf-token are required")
-        node_selector = {args.label.split("=")[0]: args.label.split("=")[1]}
-        values_file_path = generate_helm_values(
-            with_rerank=args.with_rerank,
-            num_nodes=args.num_nodes,
-            hf_token=args.hf_token,
-            model_dir=args.model_dir,
-            node_selector=node_selector,
-            tune=args.tuned,
-        )
-
-    # Read back the generated YAML file for verification
-    with open(values_file_path, "r") as file:
-        print("Generated YAML contents:")
-        print(file.read())
-
-    # Deploy unless --create-values-only is specified
-    if not args.create_values_only:
-        install_helm_release(args.release_name, args.chart_name, args.namespace, values_file_path, args.device_type)
-
-
-if __name__ == "__main__":
-    main()

ChatQnA/benchmark/performance/kubernetes/intel/gaudi/generate_helm_values.py

@@ -1,164 +0,0 @@
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-
-import os
-
-import yaml
-
-
-def generate_helm_values(with_rerank, num_nodes, hf_token, model_dir, node_selector=None, tune=False):
-    """Create a values.yaml file based on the provided configuration."""
-
-    # Log the received parameters
-    print("Received parameters:")
-    print(f"with_rerank: {with_rerank}")
-    print(f"num_nodes: {num_nodes}")
-    print(f"node_selector: {node_selector}")  # Log the node_selector
-    print(f"tune: {tune}")
-
-    if node_selector is None:
-        node_selector = {}
-
-    # Construct the base values dictionary
-    values = {
-        "tei": {"nodeSelector": {key: value for key, value in node_selector.items()}},
-        "tgi": {"nodeSelector": {key: value for key, value in node_selector.items()}},
-        "data-prep": {"nodeSelector": {key: value for key, value in node_selector.items()}},
-        "redis-vector-db": {"nodeSelector": {key: value for key, value in node_selector.items()}},
-        "retriever-usvc": {"nodeSelector": {key: value for key, value in node_selector.items()}},
-        "chatqna-ui": {"nodeSelector": {key: value for key, value in node_selector.items()}},
-        "global": {
-            "HUGGINGFACEHUB_API_TOKEN": hf_token,  # Use passed token
-            "modelUseHostPath": model_dir,  # Use passed model directory
-        },
-        "nodeSelector": {key: value for key, value in node_selector.items()},
-    }
-
-    if with_rerank:
-        values["teirerank"] = {"nodeSelector": {key: value for key, value in node_selector.items()}}
-    else:
-        values["image"] = {"repository": "opea/chatqna-without-rerank"}
-        values["teirerank"] = {"enabled": False}
-
-    default_replicas = [
-        {"name": "chatqna", "replicaCount": 2},
-        {"name": "tei", "replicaCount": 1},
-        {"name": "teirerank", "replicaCount": 1} if with_rerank else None,
-        {"name": "tgi", "replicaCount": 7 if with_rerank else 8},
-        {"name": "data-prep", "replicaCount": 1},
-        {"name": "redis-vector-db", "replicaCount": 1},
-        {"name": "retriever-usvc", "replicaCount": 2},
-    ]
-
-    if num_nodes > 1:
-        # Scale replicas based on number of nodes
-        replicas = [
-            {"name": "chatqna", "replicaCount": 1 * num_nodes},
-            {"name": "tei", "replicaCount": 1 * num_nodes},
-            {"name": "teirerank", "replicaCount": 1} if with_rerank else None,
-            {"name": "tgi", "replicaCount": (8 * num_nodes - 1) if with_rerank else 8 * num_nodes},
-            {"name": "data-prep", "replicaCount": 1},
-            {"name": "redis-vector-db", "replicaCount": 1},
-            {"name": "retriever-usvc", "replicaCount": 1 * num_nodes},
-        ]
-    else:
-        replicas = default_replicas
-
-    # Remove None values for rerank disabled
-    replicas = [r for r in replicas if r]
-
-    # Update values.yaml with replicas
-    for replica in replicas:
-        service_name = replica["name"]
-        if service_name == "chatqna":
-            values["replicaCount"] = replica["replicaCount"]
-            print(replica["replicaCount"])
-        elif service_name in values:
-            values[service_name]["replicaCount"] = replica["replicaCount"]
-
-    # Prepare resource configurations based on tuning
-    resources = []
-    if tune:
-        resources = [
-            {
-                "name": "chatqna",
-                "resources": {
-                    "limits": {"cpu": "16", "memory": "8000Mi"},
-                    "requests": {"cpu": "16", "memory": "8000Mi"},
-                },
-            },
-            {
-                "name": "tei",
-                "resources": {
-                    "limits": {"cpu": "80", "memory": "20000Mi"},
-                    "requests": {"cpu": "80", "memory": "20000Mi"},
-                },
-            },
-            {"name": "teirerank", "resources": {"limits": {"habana.ai/gaudi": 1}}} if with_rerank else None,
-            {"name": "tgi", "resources": {"limits": {"habana.ai/gaudi": 1}}},
-            {"name": "retriever-usvc", "resources": {"requests": {"cpu": "8", "memory": "8000Mi"}}},
-        ]
-
-        # Filter out any None values directly as part of initialization
-        resources = [r for r in resources if r is not None]
-
-        # Add resources for each service if tuning
-        for resource in resources:
-            service_name = resource["name"]
-            if service_name == "chatqna":
-                values["resources"] = resource["resources"]
-            elif service_name in values:
-                values[service_name]["resources"] = resource["resources"]
-
-        # Add extraCmdArgs for tgi service with default values
-        if "tgi" in values:
-            values["tgi"]["extraCmdArgs"] = [
-                "--max-input-length",
-                "1280",
-                "--max-total-tokens",
-                "2048",
-                "--max-batch-total-tokens",
-                "65536",
-                "--max-batch-prefill-tokens",
-                "4096",
-            ]
-
-    yaml_string = yaml.dump(values, default_flow_style=False)
-
-    # Determine the mode based on the 'tune' parameter
-    mode = "tuned" if tune else "oob"
-
-    # Determine the filename based on 'with_rerank' and 'num_nodes'
-    if with_rerank:
-        filename = f"{mode}-{num_nodes}-gaudi-with-rerank-values.yaml"
-    else:
-        filename = f"{mode}-{num_nodes}-gaudi-without-rerank-values.yaml"
-
-    # Write the YAML data to the file
-    with open(filename, "w") as file:
-        file.write(yaml_string)
-
-    # Get the current working directory and construct the file path
-    current_dir = os.getcwd()
-    filepath = os.path.join(current_dir, filename)
-
-    print(f"YAML file {filepath} has been generated.")
-    return filepath  # Optionally return the file path
-
-
-# Main execution for standalone use of create_values_yaml
-if __name__ == "__main__":
-    # Example values for standalone execution
-    with_rerank = True
-    num_nodes = 2
-    hftoken = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-    modeldir = "/mnt/model"
-    node_selector = {"node-type": "opea-benchmark"}
-    tune = True
-
-    filename = generate_helm_values(with_rerank, num_nodes, hftoken, modeldir, node_selector, tune)
-
-    # Read back the generated YAML file for verification
-    with open(filename, "r") as file:
-        print("Generated YAML contents:")
-        print(file.read())

ChatQnA/benchmark_chatqna.yaml

@@ -3,7 +3,7 @@
 
 deploy:
   device: gaudi
-  version: 1.2.0
+  version: 1.3.0
   modelUseHostPath: /mnt/models
   HUGGINGFACEHUB_API_TOKEN: "" # mandatory
   node: [1, 2, 4, 8]

ChatQnA/docker_compose/amd/gpu/rocm/README.md

@@ -1,163 +1,90 @@
-# Build and Deploy ChatQnA Application on AMD GPU (ROCm)
+# Deploying ChatQnA on AMD ROCm GPU
 
-## Build Docker Images
+This document outlines the single node deployment process for a ChatQnA application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservices on Intel Xeon server and AMD GPU. The steps include pulling Docker images, container deployment via Docker Compose, and service execution using microservices `llm`.
 
-### 1. Build Docker Image
+Note: The default LLM is `meta-llama/Meta-Llama-3-8B-Instruct`. Before deploying the application, please make sure either you've requested and been granted the access to it on [Huggingface](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) or you've downloaded the model locally from [ModelScope](https://www.modelscope.cn/models).
 
-- #### Create application install directory and go to it:
+## Table of Contents
 
-  ```bash
-  mkdir ~/chatqna-install && cd chatqna-install
-  ```
+1. [ChatQnA Quick Start Deployment](#chatqna-quick-start-deployment)
+2. [ChatQnA Docker Compose Files](#chatqna-docker-compose-files)
+3. [Validate Microservices](#validate-microservices)
+4. [Conclusion](#conclusion)
 
-- #### Clone the repository GenAIExamples (the default repository branch "main" is used here):
+## ChatQnA Quick Start Deployment
 
-  ```bash
-  git clone https://github.com/opea-project/GenAIExamples.git
-  ```
+This section describes how to quickly deploy and test the ChatQnA service manually on an AMD ROCm GPU. The basic steps are:
 
-  If you need to use a specific branch/tag of the GenAIExamples repository, then (v1.3 replace with its own value):
+1. [Access the Code](#access-the-code)
+2. [Configure the Deployment Environment](#configure-the-deployment-environment)
+3. [Deploy the Services Using Docker Compose](#deploy-the-services-using-docker-compose)
+4. [Check the Deployment Status](#check-the-deployment-status)
+5. [Validate the Pipeline](#validate-the-pipeline)
+6. [Cleanup the Deployment](#cleanup-the-deployment)
 
-  ```bash
-  git clone https://github.com/opea-project/GenAIExamples.git && cd GenAIExamples && git checkout v1.3
-  ```
+### Access the Code
 
-  We remind you that when using a specific version of the code, you need to use the README from this version:
+Clone the GenAIExample repository and access the ChatQnA AMD ROCm GPU platform Docker Compose files and supporting scripts:
 
-- #### Go to build directory:
+```bash
+git clone https://github.com/opea-project/GenAIExamples.git
+cd GenAIExamples/ChatQnA
+```
 
-  ```bash
-  cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_image_build
-  ```
+Then checkout a released version, such as v1.3:
 
-- Cleaning up the GenAIComps repository if it was previously cloned in this directory.
-  This is necessary if the build was performed earlier and the GenAIComps folder exists and is not empty:
+```bash
+git checkout v1.3
+```
 
-  ```bash
-  echo Y | rm -R GenAIComps
-  ```
+### Configure the Deployment Environment
 
-- #### Clone the repository GenAIComps (the default repository branch "main" is used here):
+To set up environment variables for deploying ChatQnA services, set up some parameters specific to the deployment environment and source the `set_env_*.sh` script in this directory:
 
-  ```bash
-  git clone https://github.com/opea-project/GenAIComps.git
-  ```
+- if used vLLM - set_env_vllm.sh
+- if used vLLM with FaqGen - set_env_faqgen_vllm.sh
+- if used TGI - set_env.sh
+- if used TGI with FaqGen - set_env_faqgen.sh
 
-  If you use a specific tag of the GenAIExamples repository,
-  then you should also use the corresponding tag for GenAIComps. (v1.3 replace with its own value):
+Set the values of the variables:
 
-  ```bash
-  git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout v1.3
-  ```
+- **HOST_IP, HOST_IP_EXTERNAL** - These variables are used to configure the name/address of the service in the operating system environment for the application services to interact with each other and with the outside world.
 
-  We remind you that when using a specific version of the code, you need to use the README from this version.
+  If your server uses only an internal address and is not accessible from the Internet, then the values for these two variables will be the same and the value will be equal to the server's internal name/address.
 
-- #### Setting the list of images for the build (from the build file.yaml)
+  If your server uses only an external, Internet-accessible address, then the values for these two variables will be the same and the value will be equal to the server's external name/address.
 
-  If you want to deploy a vLLM-based or TGI-based application, then the set of services is installed as follows:
+  If your server is located on an internal network, has an internal address, but is accessible from the Internet via a proxy/firewall/load balancer, then the HOST_IP variable will have a value equal to the internal name/address of the server, and the EXTERNAL_HOST_IP variable will have a value equal to the external name/address of the proxy/firewall/load balancer behind which the server is located.
 
-  #### vLLM-based application
+  We set these values in the file set_env\*\*\*\*.sh
 
-  ```bash
-  service_list="dataprep retriever vllm-rocm chatqna chatqna-ui nginx"
-  ```
+- **Variables with names like "**\*\*\*\*\*\*\_PORT"\*\* - These variables set the IP port numbers for establishing network connections to the application services.
+  The values shown in the file set_env.sh or set_env_vllm.sh they are the values used for the development and testing of the application, as well as configured for the environment in which the development is performed. These values must be configured in accordance with the rules of network access to your environment's server, and must not overlap with the IP ports of other applications that are already in use.
 
-  #### vLLM-based application with FaqGen
+Setting variables in the operating system environment:
 
-  ```bash
-  service_list="dataprep retriever vllm-rocm llm-faqgen chatqna chatqna-ui nginx"
-  ```
+```bash
+export HUGGINGFACEHUB_API_TOKEN="Your_HuggingFace_API_Token"
+source ./set_env_*.sh # replace the script name with the appropriate one
+```
 
-  #### TGI-based application
+Consult the section on [ChatQnA Service configuration](#chatqna-configuration) for information on how service specific configuration parameters affect deployments.
 
-  ```bash
-  service_list="dataprep retriever chatqna chatqna-ui nginx"
-  ```
+### Deploy the Services Using Docker Compose
 
-  #### TGI-based application with FaqGen
+To deploy the ChatQnA services, execute the `docker compose up` command with the appropriate arguments. For a default deployment with TGI, execute the command below. It uses the 'compose.yaml' file.
 
-  ```bash
-  service_list="dataprep retriever llm-faqgen chatqna chatqna-ui nginx"
-  ```
-
-- #### Pull Docker Images
-
-  ```bash
-  docker pull redis/redis-stack:7.2.0-v9
-  docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  ```
-
-- #### Optional. Pull TGI Docker Image (Do this if you want to use TGI)
-
-  ```bash
-  docker pull ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-  ```
-
-- #### Build Docker Images
-
-  ```bash
-  docker compose -f build.yaml build ${service_list} --no-cache
-  ```
-
-  After the build, we check the list of images with the command:
-
-  ```bash
-  docker image ls
-  ```
-
-  The list of images should include:
-
-  ##### vLLM-based application:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - opea/vllm-rocm:latest
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
-  ##### vLLM-based application with FaqGen:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - opea/vllm-rocm:latest
-  - opea/llm-faqgen:latest
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
-  ##### TGI-based application:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
-  ##### TGI-based application with FaqGen:
-
-  - redis/redis-stack:7.2.0-v9
-  - opea/dataprep:latest
-  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-  - opea/retriever:latest
-  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-  - opea/llm-faqgen:latest
-  - opea/chatqna:latest
-  - opea/chatqna-ui:latest
-  - opea/nginx:latest
-
----
-
-## Deploy the ChatQnA Application
-
-### Docker Compose Configuration for AMD GPUs
+```bash
+cd docker_compose/amd/gpu/rocm
+# if used TGI
+docker compose -f compose.yaml up -d
+# if used TGI with FaqGen
+# docker compose -f compose_faqgen.yaml up -d
+# if used vLLM
+# docker compose -f compose_vllm.yaml up -d
+# if used vLLM with FaqGen
+# docker compose -f compose_faqgen_vllm.yaml up -d
+```
 
 To enable GPU support for AMD GPUs, the following configuration is added to the Docker Compose file:
 
@@ -198,332 +125,103 @@ security_opt:
 **How to Identify GPU Device IDs:**
 Use AMD GPU driver utilities to determine the correct `cardN` and `renderN` IDs for your GPU.
 
-### Set deploy environment variables
+> **Note**: developers should build docker image from source when:
+>
+> - Developing off the git main branch (as the container's ports in the repo may be different > from the published docker image).
+> - Unable to download the docker image.
+> - Use a specific version of Docker image.
 
-#### Setting variables in the operating system environment:
+Please refer to the table below to build different microservices from source:
 
-##### Set variable HUGGINGFACEHUB_API_TOKEN:
+| Microservice    | Deployment Guide                                                                                                   |
+| --------------- | ------------------------------------------------------------------------------------------------------------------ |
+| vLLM            | [vLLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/vllm#build-docker)     |
+| TGI             | [TGI project](https://github.com/huggingface/text-generation-inference.git)                                        |
+| LLM             | [LLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/llms)                                 |
+| Redis Vector DB | [Redis](https://github.com/redis/redis.git)                                                                        |
+| Dataprep        | [Dataprep build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/dataprep/src/README_redis.md)    |
+| TEI Embedding   | [TEI guide](https://github.com/huggingface/text-embeddings-inference.git)                                          |
+| Retriever       | [Retriever build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/retrievers/src/README_redis.md) |
+| TEI Reranking   | [TEI guide](https://github.com/huggingface/text-embeddings-inference.git)                                          |
+| MegaService     | [MegaService guide](../../../../README.md)                                                                         |
+| UI              | [UI guide](../../../../ui/react/README.md)                                                                         |
+| Nginx           | [Nginx guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/nginx)                      |
+
+### Check the Deployment Status
+
+After running docker compose, check if all the containers launched via docker compose have started:
 
 ```bash
-### Replace the string 'your_huggingfacehub_token' with your HuggingFacehub repository access token.
-export HUGGINGFACEHUB_API_TOKEN='your_huggingfacehub_token'
+docker ps -a
 ```
 
-#### Set variables value in set_env\*\*\*\*.sh file:
+For the default deployment with TGI, the following 9 containers should have started:
 
-Go to Docker Compose directory:
-
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
+```
+CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
+eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chatqna-nginx-server
+2fce48a4c0f4   opea/chatqna-ui:latest                                  "docker-entrypoint.s…"   37 seconds ago   Up 5 seconds              0.0.0.0:18101->5173/tcp, [::]:18101->5173/tcp                                              chatqna-ui-server
+613c384979f4   opea/chatqna:latest                                     "bash entrypoint.sh"     37 seconds ago   Up 5 seconds              0.0.0.0:18102->8888/tcp, [::]:18102->8888/tcp                                              chatqna-backend-server
+05512bd29fee   opea/dataprep:latest                                    "sh -c 'python $( [ …"   37 seconds ago   Up 36 seconds (healthy)   0.0.0.0:18103->5000/tcp, [::]:18103->5000/tcp                                              chatqna-dataprep-service
+49844d339d1d   opea/retriever:latest                                   "python opea_retriev…"   37 seconds ago   Up 36 seconds             0.0.0.0:7000->7000/tcp, [::]:7000->7000/tcp                                                chatqna-retriever
+75b698fe7de0   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18808->80/tcp, [::]:18808->80/tcp                                                  chatqna-tei-reranking-service
+342f01bfdbb2   ghcr.io/huggingface/text-generation-inference:2.3.1-rocm"python3 /workspace/…"   37 seconds ago   Up 36 seconds             0.0.0.0:18008->8011/tcp, [::]:18008->8011/tcp                                              chatqna-tgi-service
+6081eb1c119d   redis/redis-stack:7.2.0-v9                              "/entrypoint.sh"         37 seconds ago   Up 36 seconds             0.0.0.0:6379->6379/tcp, [::]:6379->6379/tcp, 0.0.0.0:8001->8001/tcp, [::]:8001->8001/tcp   chatqna-redis-vector-db
+eded17420782   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18090->80/tcp, [::]:18090->80/tcp                                                  chatqna-tei-embedding-service
 ```
 
-The example uses the Nano text editor. You can use any convenient text editor:
+if used TGI with FaqGen:
 
-#### If you use vLLM based application
-
-```bash
-nano set_env_vllm.sh
+```
+CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
+eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chatqna-nginx-server
+2fce48a4c0f4   opea/chatqna-ui:latest                                  "docker-entrypoint.s…"   37 seconds ago   Up 5 seconds              0.0.0.0:18101->5173/tcp, [::]:18101->5173/tcp                                              chatqna-ui-server
+613c384979f4   opea/chatqna:latest                                     "bash entrypoint.sh"     37 seconds ago   Up 5 seconds              0.0.0.0:18102->8888/tcp, [::]:18102->8888/tcp                                              chatqna-backend-server
+e0ef1ea67640   opea/llm-faqgen:latest                                  "bash entrypoint.sh"     37 seconds ago   Up 36 seconds             0.0.0.0:18011->9000/tcp, [::]:18011->9000/tcp                                              chatqna-llm-faqgen
+05512bd29fee   opea/dataprep:latest                                    "sh -c 'python $( [ …"   37 seconds ago   Up 36 seconds (healthy)   0.0.0.0:18103->5000/tcp, [::]:18103->5000/tcp                                              chatqna-dataprep-service
+49844d339d1d   opea/retriever:latest                                   "python opea_retriev…"   37 seconds ago   Up 36 seconds             0.0.0.0:7000->7000/tcp, [::]:7000->7000/tcp                                                chatqna-retriever
+75b698fe7de0   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18808->80/tcp, [::]:18808->80/tcp                                                  chatqna-tei-reranking-service
+342f01bfdbb2   ghcr.io/huggingface/text-generation-inference:2.3.1-rocm"python3 /workspace/…"   37 seconds ago   Up 36 seconds             0.0.0.0:18008->8011/tcp, [::]:18008->8011/tcp                                              chatqna-tgi-service
+6081eb1c119d   redis/redis-stack:7.2.0-v9                              "/entrypoint.sh"         37 seconds ago   Up 36 seconds             0.0.0.0:6379->6379/tcp, [::]:6379->6379/tcp, 0.0.0.0:8001->8001/tcp, [::]:8001->8001/tcp   chatqna-redis-vector-db
+eded17420782   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18090->80/tcp, [::]:18090->80/tcp                                                  chatqna-tei-embedding-service
 ```
 
-#### If you use vLLM based application with FaqGen
+if used vLLM:
 
-```bash
-nano set_env_vllm_faqgen.sh
+```
+CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
+eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chatqna-nginx-server
+2fce48a4c0f4   opea/chatqna-ui:latest                                  "docker-entrypoint.s…"   37 seconds ago   Up 5 seconds              0.0.0.0:18101->5173/tcp, [::]:18101->5173/tcp                                              chatqna-ui-server
+613c384979f4   opea/chatqna:latest                                     "bash entrypoint.sh"     37 seconds ago   Up 5 seconds              0.0.0.0:18102->8888/tcp, [::]:18102->8888/tcp                                              chatqna-backend-server
+05512bd29fee   opea/dataprep:latest                                    "sh -c 'python $( [ …"   37 seconds ago   Up 36 seconds (healthy)   0.0.0.0:18103->5000/tcp, [::]:18103->5000/tcp                                              chatqna-dataprep-service
+49844d339d1d   opea/retriever:latest                                   "python opea_retriev…"   37 seconds ago   Up 36 seconds             0.0.0.0:7000->7000/tcp, [::]:7000->7000/tcp                                                chatqna-retriever
+75b698fe7de0   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18808->80/tcp, [::]:18808->80/tcp                                                  chatqna-tei-reranking-service
+342f01bfdbb2   opea/vllm-rocm:latest                                   "python3 /workspace/…"   37 seconds ago   Up 36 seconds             0.0.0.0:18008->8011/tcp, [::]:18008->8011/tcp                                              chatqna-vllm-service
+6081eb1c119d   redis/redis-stack:7.2.0-v9                              "/entrypoint.sh"         37 seconds ago   Up 36 seconds             0.0.0.0:6379->6379/tcp, [::]:6379->6379/tcp, 0.0.0.0:8001->8001/tcp, [::]:8001->8001/tcp   chatqna-redis-vector-db
+eded17420782   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18090->80/tcp, [::]:18090->80/tcp                                                  chatqna-tei-embedding-service
 ```
 
-#### If you use TGI based application
+if used vLLM with FaqGen:
 
-```bash
-nano set_env.sh
+```
+CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
+eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chatqna-nginx-server
+2fce48a4c0f4   opea/chatqna-ui:latest                                  "docker-entrypoint.s…"   37 seconds ago   Up 5 seconds              0.0.0.0:18101->5173/tcp, [::]:18101->5173/tcp                                              chatqna-ui-server
+613c384979f4   opea/chatqna:latest                                     "bash entrypoint.sh"     37 seconds ago   Up 5 seconds              0.0.0.0:18102->8888/tcp, [::]:18102->8888/tcp                                              chatqna-backend-server
+e0ef1ea67640   opea/llm-faqgen:latest                                  "bash entrypoint.sh"     37 seconds ago   Up 36 seconds             0.0.0.0:18011->9000/tcp, [::]:18011->9000/tcp                                              chatqna-llm-faqgen
+05512bd29fee   opea/dataprep:latest                                    "sh -c 'python $( [ …"   37 seconds ago   Up 36 seconds (healthy)   0.0.0.0:18103->5000/tcp, [::]:18103->5000/tcp                                              chatqna-dataprep-service
+49844d339d1d   opea/retriever:latest                                   "python opea_retriev…"   37 seconds ago   Up 36 seconds             0.0.0.0:7000->7000/tcp, [::]:7000->7000/tcp                                                chatqna-retriever
+75b698fe7de0   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18808->80/tcp, [::]:18808->80/tcp                                                  chatqna-tei-reranking-service
+342f01bfdbb2   opea/vllm-rocm:latest                                   "python3 /workspace/…"   37 seconds ago   Up 36 seconds             0.0.0.0:18008->8011/tcp, [::]:18008->8011/tcp                                              chatqna-vllm-service
+6081eb1c119d   redis/redis-stack:7.2.0-v9                              "/entrypoint.sh"         37 seconds ago   Up 36 seconds             0.0.0.0:6379->6379/tcp, [::]:6379->6379/tcp, 0.0.0.0:8001->8001/tcp, [::]:8001->8001/tcp   chatqna-redis-vector-db
+eded17420782   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   37 seconds ago   Up 36 seconds             0.0.0.0:18090->80/tcp, [::]:18090->80/tcp                                                  chatqna-tei-embedding-service
 ```
 
-#### If you use TGI based application with FaqGen
+If any issues are encountered during deployment, refer to the [Troubleshooting](../../../../README_miscellaneous.md#troubleshooting) section.
 
-```bash
-nano set_env_faqgen.sh
-```
+### Validate the Pipeline
 
-If you are in a proxy environment, also set the proxy-related environment variables:
-
-```bash
-export http_proxy="Your_HTTP_Proxy"
-export https_proxy="Your_HTTPs_Proxy"
-```
-
-Set the values of the variables:
-
-- **HOST_IP, HOST_IP_EXTERNAL** - These variables are used to configure the name/address of the service in the operating system environment for the application services to interact with each other and with the outside world.
-
-  If your server uses only an internal address and is not accessible from the Internet, then the values for these two variables will be the same and the value will be equal to the server's internal name/address.
-
-  If your server uses only an external, Internet-accessible address, then the values for these two variables will be the same and the value will be equal to the server's external name/address.
-
-  If your server is located on an internal network, has an internal address, but is accessible from the Internet via a proxy/firewall/load balancer, then the HOST_IP variable will have a value equal to the internal name/address of the server, and the EXTERNAL_HOST_IP variable will have a value equal to the external name/address of the proxy/firewall/load balancer behind which the server is located.
-
-  We set these values in the file set_env\*\*\*\*.sh
-
-- **Variables with names like "**\*\*\*\*\*\*\_PORT"\*\* - These variables set the IP port numbers for establishing network connections to the application services.
-  The values shown in the file set_env.sh or set_env_vllm they are the values used for the development and testing of the application, as well as configured for the environment in which the development is performed. These values must be configured in accordance with the rules of network access to your environment's server, and must not overlap with the IP ports of other applications that are already in use.
-
-#### Set variables with script set_env\*\*\*\*.sh
-
-#### If you use vLLM based application
-
-```bash
-. set_env_vllm.sh
-```
-
-#### If you use vLLM based application with FaqGen
-
-```bash
-. set_env_faqgen_vllm.sh
-```
-
-#### If you use TGI based application
-
-```bash
-. set_env.sh
-```
-
-#### If you use TGI based application with FaqGen
-
-```bash
-. set_env_faqgen.sh
-```
-
-### Start the services:
-
-#### If you use vLLM based application
-
-```bash
-docker compose -f compose_vllm.yaml up -d
-```
-
-#### If you use vLLM based application with FaqGen
-
-```bash
-docker compose -f compose_faqgen_vllm.yaml up -d
-```
-
-#### If you use TGI based application
-
-```bash
-docker compose -f compose.yaml up -d
-```
-
-#### If you use TGI based application with FaqGen
-
-```bash
-docker compose -f compose_faqgen.yaml up -d
-```
-
-All containers should be running and should not restart:
-
-##### If you use vLLM based application:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-vllm-service
-- chatqna-backend-server
-- chatqna-ui-server
-- chatqna-nginx-server
-
-##### If you use vLLM based application with FaqGen:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-vllm-service
-- chatqna-llm-faqgen
-- chatqna-backend-server
-- chatqna-ui-server
-- chatqna-nginx-server
-
-##### If you use TGI based application:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-tgi-service
-- chatqna-backend-server
-- chatqna-ui-server
-- chaqna-nginx-server
-
-##### If you use TGI based application with FaqGen:
-
-- chatqna-redis-vector-db
-- chatqna-dataprep-service
-- chatqna-tei-embedding-service
-- chatqna-retriever
-- chatqna-tei-reranking-service
-- chatqna-tgi-service
-- chatqna-llm-faqgen
-- chatqna-backend-server
-- chatqna-ui-server
-- chaqna-nginx-server
-
----
-
-## Validate the Services
-
-### 1. Validate TEI Embedding Service
-
-```bash
-curl http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}/embed \
-    -X POST \
-    -d '{"inputs":"What is Deep Learning?"}' \
-    -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to text:
-
-```textmate
-[[0.00037115702,-0.06356819,0.0024758505,..................,0.022725677,0.016026087,-0.02125421,-0.02984927,-0.0049473033]]
-```
-
-If the service response has a meaningful response in the value,
-then we consider the TEI Embedding Service to be successfully launched
-
-### 2. Validate Retriever Microservice
-
-```bash
-export your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
-curl http://${HOST_IP}:${CHATQNA_REDIS_RETRIEVER_PORT}/v1/retrieval \
-  -X POST \
-  -d "{\"text\":\"test\",\"embedding\":${your_embedding}}" \
-  -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-{ "id": "e191846168aed1f80b2ea12df80844d2", "retrieved_docs": [], "initial_query": "test", "top_n": 1, "metadata": [] }
-```
-
-If the response corresponds to the form of the provided JSON, then we consider the
-Retriever Microservice verification successful.
-
-### 3. Validate TEI Reranking Service
-
-```bash
-curl http://${HOST_IP}:${CHATQNA_TEI_RERANKING_PORT}/rerank \
-    -X POST \
-    -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \
-    -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-[
-  { "index": 1, "score": 0.94238955 },
-  { "index": 0, "score": 0.120219156 }
-]
-```
-
-If the response corresponds to the form of the provided JSON, then we consider the TEI Reranking Service
-verification successful.
-
-### 4. Validate the vLLM/TGI Service
-
-#### If you use vLLM:
-
-```bash
-DATA='{"model": "meta-llama/Meta-Llama-3-8B-Instruct", '\
-'"messages": [{"role": "user", "content": "What is a Deep Learning?"}], "max_tokens": 64}'
-
-curl http://${HOST_IP}:${CHATQNA_VLLM_SERVICE_PORT}/v1/chat/completions \
-  -X POST \
-  -d "$DATA" \
-  -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-{
-  "id": "chatcmpl-91003647d1c7469a89e399958f390f67",
-  "object": "chat.completion",
-  "created": 1742877228,
-  "model": "meta-llama/Meta-Llama-3-8B-Instruct",
-  "choices": [
-    {
-      "index": 0,
-      "message": {
-        "role": "assistant",
-        "content": "Deep Learning ( DL) is a subfield of Machine Learning (ML) that focuses on the design of algorithms and architectures inspired by the structure and function of the human brain. These algorithms are designed to analyze and interpret data that is presented in the form of patterns or signals, and they often mimic the way the human brain",
-        "tool_calls": []
-      },
-      "logprobs": null,
-      "finish_reason": "length",
-      "stop_reason": null
-    }
-  ],
-  "usage": { "prompt_tokens": 16, "total_tokens": 80, "completion_tokens": 64, "prompt_tokens_details": null },
-  "prompt_logprobs": null
-}
-```
-
-If the service response has a meaningful response in the value of the "choices.message.content" key,
-then we consider the vLLM service to be successfully launched
-
-#### If you use TGI:
-
-```bash
-DATA='{"inputs":"What is a Deep Learning?",'\
-'"parameters":{"max_new_tokens":64,"do_sample": true}}'
-
-curl http://${HOST_IP}:${CHATQNA_TGI_SERVICE_PORT}/generate \
-  -X POST \
-  -d "$DATA" \
-  -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-{
-  "generated_text": " What is its application in Computer Vision?\nWhat is a Deep Learning?\nDeep learning is a subfield of machine learning that involves the use of artificial neural networks to model high-level abstractions in data. It involves the use of deep neural networks, which are composed of multiple layers, to learn complex patterns in data. The"
-}
-```
-
-If the service response has a meaningful response in the value of the "generated_text" key,
-then we consider the TGI service to be successfully launched
-
-### 5. Validate the LLM Service (if your used application with FaqGen)
-
-```bash
-DATA='{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source '\
-'text embeddings and sequence classification models. TEI enables high-performance extraction for the most '\
-'popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens": 128}'
-
-curl http://${HOST_IP}:${CHATQNA_LLM_FAQGEN_PORT}/v1/faqgen \
-  -X POST \
-  -d "$DATA" \
-  -H 'Content-Type: application/json'
-```
-
-Checking the response from the service. The response should be similar to JSON:
-
-```json
-{
-  "id": "58f0632f5f03af31471b895b0d0d397b",
-  "text": " Q: What is Text Embeddings Inference (TEI)?\n         A: TEI is a toolkit for deploying and serving open source text embeddings and sequence classification models.\n\n         Q: What models does TEI support?\n         A: TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.\n\n         Q: What is the purpose of TEI?\n         A: The purpose of TEI is to enable high-performance extraction for text embeddings and sequence classification models.\n\n         Q: What are the benefits of using TEI?\n         A: The benefits of using TEI include high",
-  "prompt": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."
-}
-```
-
-If the service response has a meaningful response in the value of the "text" key,
-then we consider the LLM service to be successfully launched
-
-### 6. Validate the MegaService
+Once the ChatQnA services are running, test the pipeline using the following command:
 
 ```bash
 curl http://${HOST_IP}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna \
@@ -531,91 +229,105 @@ curl http://${HOST_IP}:${CHATQNA_BACKEND_SERVICE_PORT}/v1/chatqna \
   -d '{"messages": "What is the revenue of Nike in 2023?"}'
 ```
 
-Checking the response from the service. The response should be similar to text:
+**Note** : Access the ChatQnA UI by web browser through this URL: `http://${HOST_IP_EXTERNAL}:${CHATQNA_NGINX_PORT}`
 
-```textmate
-data: b' What'
-data: b' is'
-data: b' the'
-data: b' revenue'
-data: b' of'
-data: b' Nike'
-data: b' in'
-data: b' '
-data: b'202'
-data: b'3'
-data: b'?\n'
-data: b'        '
-data: b' Answer'
-data: b':'
-data: b' According'
-data: b' to'
-data: b' the'
-data: b' search'
-data: b' results'
-data: b','
-data: b' the'
-data: b' revenue'
-data: b' of'
-data: b''
+### Cleanup the Deployment
 
-data: [DONE]
-
-```
-
-If the output lines in the "data" keys contain words (tokens) containing meaning, then the service
-is considered launched successfully.
-
-### 7. Validate the Frontend (UI)
-
-To access the UI, use the URL - http://${EXTERNAL_HOST_IP}:${CHATQNA_NGINX_PORT}
-A page should open when you click through to this address:
-
-![UI start page](../../../../assets/img/ui-starting-page.png)
-
-If a page of this type has opened, then we believe that the service is running and responding,
-and we can proceed to functional UI testing.
-
-Let's enter the task for the service in the "Enter prompt here" field.
-For example, "What is a Deep Learning?" and press Enter.
-After that, a page with the result of the task should open:
-
-#### If used application without FaqGen
-
-![UI result page](../../../../assets/img/ui-result-page.png)
-
-#### If used application with FaqGen
-
-![UI result page](../../../../assets/img/ui-result-page-faqgen.png)
-
-If the result shown on the page is correct, then we consider the verification of the UI service to be successful.
-
-### 5. Stop application
-
-#### If you use vLLM
+To stop the containers associated with the deployment, execute the following command:
 
 ```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose_vllm.yaml down
-```
-
-#### If you use vLLM with FaqGen
-
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose_faqgen_vllm.yaml down
-```
-
-#### If you use TGI
-
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
+# if used TGI
 docker compose -f compose.yaml down
+# if used TGI with FaqGen
+# docker compose -f compose_faqgen.yaml down
+# if used vLLM
+# docker compose -f compose_vllm.yaml down
+# if used vLLM with FaqGen
+# docker compose -f compose_faqgen_vllm.yaml down
+
+
 ```
 
-#### If you use TGI with FaqGen
+## ChatQnA Docker Compose Files
 
-```bash
-cd ~/chatqna-install/GenAIExamples/ChatQnA/docker_compose/amd/gpu/rocm
-docker compose -f compose_faqgen.yaml down
-```
+In the context of deploying an ChatQnA pipeline on an Intel® Xeon® platform, we can pick and choose different large language model serving frameworks, or single English TTS/multi-language TTS component. The table below outlines the various configurations that are available as part of the application. These configurations can be used as templates and can be extended to different components available in [GenAIComps](https://github.com/opea-project/GenAIComps.git).
+
+| File                                                   | Description                                                                                                              |
+| ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------ |
+| [compose.yaml](./compose.yaml)                         | The LLM serving framework is TGI. Default compose file using TGI as serving framework and redis as vector database       |
+| [compose_faqgen.yaml](./compose_faqgen.yaml)           | The LLM serving framework is TGI with FaqGen. All other configurations remain the same as the default                    |
+| [compose_vllm.yaml](./compose_vllm.yaml)               | The LLM serving framework is vLLM. Compose file using vllm as serving framework and redis as vector database             |
+| [compose_faqgen_vllm.yaml](./compose_faqgen_vllm.yaml) | The LLM serving framework is vLLM with FaqGen. Compose file using vllm as serving framework and redis as vector database |
+
+## Validate MicroServices
+
+1. TEI Embedding Service
+
+   ```bash
+   curl http://${HOST_IP}:${CHATQNA_TEI_EMBEDDING_PORT}/embed \
+   -X POST \
+   -d '{"inputs":"What is Deep Learning?"}' \
+   -H 'Content-Type: application/json'
+   ```
+
+2. Retriever Microservice
+
+   ```bash
+   export your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
+   curl http://${HOST_IP}:${CHATQNA_REDIS_RETRIEVER_PORT}/v1/retrieval \
+     -X POST \
+     -d "{\"text\":\"test\",\"embedding\":${your_embedding}}" \
+     -H 'Content-Type: application/json'
+   ```
+
+3. TEI Reranking Service
+
+   ```bash
+   curl http://${HOST_IP}:${CHATQNA_TEI_RERANKING_PORT}/rerank \
+   -X POST \
+   -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \
+   -H 'Content-Type: application/json'
+   ```
+
+4. vLLM/TGI Service
+
+   If you use vLLM:
+
+   ```bash
+   DATA='{"model": "meta-llama/Meta-Llama-3-8B-Instruct", '\
+   '"messages": [{"role": "user", "content": "What is a Deep Learning?"}], "max_tokens": 64}'
+
+   curl http://${HOST_IP}:${CHATQNA_VLLM_SERVICE_PORT}/v1/chat/completions \
+     -X POST \
+     -d "$DATA" \
+     -H 'Content-Type: application/json'
+   ```
+
+   If you use TGI:
+
+   ```bash
+   DATA='{"inputs":"What is a Deep Learning?",'\
+   '"parameters":{"max_new_tokens":64,"do_sample": true}}'
+
+   curl http://${HOST_IP}:${CHATQNA_TGI_SERVICE_PORT}/generate \
+   -X POST \
+   -d "$DATA" \
+   -H 'Content-Type: application/json'
+   ```
+
+5. LLM Service (if your used application with FaqGen)
+
+   ```bash
+   DATA='{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source '\
+   'text embeddings and sequence classification models. TEI enables high-performance extraction for the most '\
+   'popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens": 128}'
+
+   curl http://${HOST_IP}:${CHATQNA_LLM_FAQGEN_PORT}/v1/faqgen \
+     -X POST \
+     -d "$DATA" \
+     -H 'Content-Type: application/json'
+   ```
+
+## Conclusion
+
+This guide should enable developers to deploy the default configuration or any of the other compose yaml files for different configurations. It also highlights the configurable parameters that can be set before deployment.

ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml

@@ -165,7 +165,7 @@ services:
 
   chatqna-nginx-server:
     image: ${REGISTRY:-opea}/nginx:${TAG:-latest}
-    container_name: chaqna-nginx-server
+    container_name: chatqna-nginx-server
     depends_on:
       - chatqna-backend-server
       - chatqna-ui-server

ChatQnA/docker_compose/amd/gpu/rocm/compose_faqgen.yaml

@@ -187,7 +187,7 @@ services:
 
   chatqna-nginx-server:
     image: ${REGISTRY:-opea}/nginx:${TAG:-latest}
-    container_name: chaqna-nginx-server
+    container_name: chatqna-nginx-server
     depends_on:
       - chatqna-backend-server
       - chatqna-ui-server

ChatQnA/docker_compose/amd/gpu/rocm/compose_faqgen_vllm.yaml

@@ -192,7 +192,7 @@ services:
 
   chatqna-nginx-server:
     image: ${REGISTRY:-opea}/nginx:${TAG:-latest}
-    container_name: chaqna-nginx-server
+    container_name: chatqna-nginx-server
     depends_on:
       - chatqna-backend-server
       - chatqna-ui-server

ChatQnA/docker_compose/install_docker.sh

@@ -1,35 +0,0 @@
-#!/bin/bash
-
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-
-# Update the package index
-sudo apt-get -y update
-
-# Install prerequisites
-sudo apt-get -y install ca-certificates curl
-
-# Create the directory for the Docker GPG key
-sudo install -m 0755 -d /etc/apt/keyrings
-
-# Add Docker's official GPG key
-sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
-
-# Set permissions for the GPG key
-sudo chmod a+r /etc/apt/keyrings/docker.asc
-
-# Add Docker repository to the sources list
-echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
-  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-
-# Update the package index with Docker packages
-sudo apt-get -y update
-
-# Install Docker packages
-sudo apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
-
-# add existing user
-sudo usermod -aG docker $USER
-
-# Optional: Verify that Docker is installed correctly
-sudo docker --version

ChatQnA/docker_compose/intel/cpu/aipc/compose.yaml

@@ -32,7 +32,7 @@ services:
       retries: 50
     restart: unless-stopped
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-server
     ports:
       - "6006:80"
@@ -65,7 +65,7 @@ services:
       RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_REDIS"
     restart: unless-stopped
   tei-reranking-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-reranking-server
     ports:
       - "8808:80"

ChatQnA/docker_compose/intel/cpu/xeon/README.md

@@ -156,6 +156,7 @@ In the context of deploying a ChatQnA pipeline on an Intel® Xeon® platform, we
 | [compose_faqgen_tgi.yaml](./compose_faqgen_tgi.yaml)         | Enables FAQ generation using TGI as the LLM serving framework. For more details, refer to [README_faqgen.md](./README_faqgen.md).                                     |
 | [compose.telemetry.yaml](./compose.telemetry.yaml)           | Helper file for telemetry features for vllm. Can be used along with any compose files that serves vllm                                                                |
 | [compose_tgi.telemetry.yaml](./compose_tgi.telemetry.yaml)   | Helper file for telemetry features for tgi. Can be used along with any compose files that serves tgi                                                                  |
+| [compose_mariadb.yaml](./compose_mariadb.yaml)               | Uses MariaDB Server as the vector database. All other configurations remain the same as the default                                                                   |
 
 ## ChatQnA with Conversational UI (Optional)
 

ChatQnA/docker_compose/intel/cpu/xeon/README_mariadb.md

@@ -0,0 +1,259 @@
+# Deploying ChatQnA with MariaDB Vector on Intel® Xeon® Processors
+
+This document outlines the deployment process for a ChatQnA application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel® Xeon® servers. The pipeline integrates **MariaDB Vector** as the vector database and includes microservices such as `embedding`, `retriever`, `rerank`, and `llm`.
+
+---
+
+## Table of Contents
+
+1. [Build Docker Images](#build-docker-images)
+2. [Validate Microservices](#validate-microservices)
+3. [Launch the UI](#launch-the-ui)
+4. [Launch the Conversational UI (Optional)](#launch-the-conversational-ui-optional)
+
+---
+
+## Build Docker Images
+
+First of all, you need to build Docker Images locally and install the python package of it.
+
+```bash
+git clone https://github.com/opea-project/GenAIComps.git
+cd GenAIComps
+```
+
+### 1. Build Retriever Image
+
+```bash
+docker build --no-cache -t opea/retriever:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/Dockerfile .
+```
+
+### 2. Build Dataprep Image
+
+```bash
+docker build --no-cache -t opea/dataprep:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
+cd ..
+```
+
+### 3. Build MegaService Docker Image
+
+To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `chatqna.py` Python script. Build MegaService Docker image via below command:
+
+```bash
+git clone https://github.com/opea-project/GenAIExamples.git
+cd GenAIExamples/ChatQnA/
+docker build --no-cache -t opea/chatqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+cd ../..
+```
+
+### 4. Build UI Docker Image
+
+Build frontend Docker image via below command:
+
+```bash
+cd GenAIExamples/ChatQnA/ui
+docker build --no-cache -t opea/chatqna-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile .
+cd ../../..
+```
+
+### 5. Build Conversational React UI Docker Image (Optional)
+
+Build frontend Docker image that enables Conversational experience with ChatQnA megaservice via below command:
+
+**Export the value of the public IP address of your Xeon server to the `host_ip` environment variable**
+
+```bash
+cd GenAIExamples/ChatQnA/ui
+export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8912/v1/chatqna"
+export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6043/v1/dataprep/ingest"
+docker build --no-cache -t opea/chatqna-conversation-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg DATAPREP_SERVICE_ENDPOINT=$DATAPREP_SERVICE_ENDPOINT -f ./docker/Dockerfile.react .
+cd ../../..
+```
+
+### 6. Build Nginx Docker Image
+
+```bash
+cd GenAIComps
+docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/third_parties/nginx/src/Dockerfile .
+```
+
+Then run the command `docker images`, you will have the following 5 Docker Images:
+
+1. `opea/dataprep:latest`
+2. `opea/retriever:latest`
+3. `opea/chatqna:latest`
+4. `opea/chatqna-ui:latest`
+5. `opea/nginx:latest`
+
+## Start Microservices
+
+### Required Models
+
+By default, the embedding, reranking and LLM models are set to a default value as listed below:
+
+| Service   | Model                               |
+| --------- | ----------------------------------- |
+| Embedding | BAAI/bge-base-en-v1.5               |
+| Reranking | BAAI/bge-reranker-base              |
+| LLM       | meta-llama/Meta-Llama-3-8B-Instruct |
+
+Change the `xxx_MODEL_ID` below for your needs.
+
+### Setup Environment Variables
+
+Since the `compose.yaml` will consume some environment variables, you need to set them up in advance as below.
+
+**Export the value of the public IP address of your Xeon server to the `host_ip` environment variable**
+
+> Change the External_Public_IP below with the actual IPV4 value
+
+```bash
+export host_ip="External_Public_IP"
+```
+
+> Change to your actual Huggingface API Token value
+
+```bash
+export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token"
+```
+
+**Append the value of the public IP address to the no_proxy list if you are in a proxy environment**
+
+```bash
+export no_proxy=${your_no_proxy},chatqna-xeon-ui-server,chatqna-xeon-backend-server,dataprep-mariadb-vector,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service
+```
+
+```bash
+export no_proxy=${your_no_proxy}
+export http_proxy=${your_http_proxy}
+export https_proxy=${your_http_proxy}
+export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
+export RERANK_MODEL_ID="BAAI/bge-reranker-base"
+export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
+export MARIADB_DATABASE="vectordb"
+export MARIADB_USER="chatqna"
+export MARIADB_PASSWORD="password"
+```
+
+Note: Please replace with `host_ip` with you external IP address, do not use localhost.
+
+### Start all the services Docker Containers
+
+> Before running the docker compose command, you need to be in the folder that has the docker compose yaml file
+
+```bash
+cd GenAIExamples/ChatQnA/docker_compose/intel/cpu/xeon/
+docker compose -f compose_mariadb.yaml up -d
+```
+
+### Validate Microservices
+
+Follow the instructions to validate MicroServices.
+For details on how to verify the correctness of the response, refer to [how-to-validate_service](../../hpu/gaudi/how_to_validate_service.md).
+
+1. TEI Embedding Service
+
+   ```bash
+   curl ${host_ip}:6040/embed \
+       -X POST \
+       -d '{"inputs":"What is Deep Learning?"}' \
+       -H 'Content-Type: application/json'
+   ```
+
+2. Retriever Microservice
+
+   To consume the retriever microservice, you need to generate a mock embedding vector by Python script. The length of embedding vector
+   is determined by the embedding model.
+   Here we use the model `EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"`, which vector size is 768.
+
+   Check the vector dimension of your embedding model, set `your_embedding` dimension equals to it.
+
+   ```bash
+   export your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
+   curl http://${host_ip}:6045/v1/retrieval \
+     -X POST \
+     -d '{"text":"What is the revenue of Nike in 2023?","embedding":"'"${your_embedding}"'"}' \
+     -H 'Content-Type: application/json'
+   ```
+
+3. TEI Reranking Service
+
+   ```bash
+   curl http://${host_ip}:6041/rerank \
+       -X POST \
+       -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \
+       -H 'Content-Type: application/json'
+   ```
+
+4. LLM Backend Service
+
+   In the first startup, this service will take more time to download, load and warm up the model. After it's finished, the service will be ready.
+
+   Try the command below to check whether the LLM service is ready.
+
+   ```bash
+   docker logs vllm-service 2>&1 | grep complete
+   ```
+
+   If the service is ready, you will get the response like below.
+
+   ```text
+   INFO: Application startup complete.
+   ```
+
+   Then try the `cURL` command below to validate vLLM service.
+
+   ```bash
+   curl http://${host_ip}:6042/v1/chat/completions \
+     -X POST \
+     -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens":17}' \
+     -H 'Content-Type: application/json'
+   ```
+
+5. MegaService
+
+   ```bash
+   curl http://${host_ip}:8912/v1/chatqna -H "Content-Type: application/json" -d '{
+        "messages": "What is the revenue of Nike in 2023?"
+        }'
+   ```
+
+6. Dataprep Microservice（Optional）
+
+   If you want to update the default knowledge base, you can use the following commands:
+
+   Update Knowledge Base via Local File Upload:
+
+   ```bash
+   curl -X POST "http://${host_ip}:6043/v1/dataprep/ingest" \
+        -H "Content-Type: multipart/form-data" \
+        -F "files=@./your_file.pdf"
+   ```
+
+   This command updates a knowledge base by uploading a local file for processing. Update the file path according to your environment.
+
+   Add Knowledge Base via HTTP Links:
+
+   ```bash
+   curl -X POST "http://${host_ip}:6043/v1/dataprep/ingest" \
+        -H "Content-Type: multipart/form-data" \
+        -F 'link_list=["https://opea.dev"]'
+   ```
+
+## Launch the UI
+
+To access the frontend, open the following URL in your browser: http://{host_ip}:5173. By default, the UI runs on port 5173 internally. If you prefer to use a different host port to access the frontend, you can modify the port mapping in the `compose.yaml` file as shown below:
+
+```yaml
+  chatqna-xeon-ui-server:
+    image: opea/chatqna-ui:latest
+    ...
+    ports:
+      - "80:5173"
+```
+
+![project-screenshot](../../../../assets/img/chat_ui_init.png)
+
+Here is an example of running ChatQnA:
+
+![project-screenshot](../../../../assets/img/chat_ui_response.png)

ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml

@@ -39,7 +39,7 @@ services:
       retries: 50
     restart: unless-stopped
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-server
     ports:
       - "6006:80"
@@ -72,7 +72,7 @@ services:
       RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_REDIS"
     restart: unless-stopped
   tei-reranking-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-reranking-server
     ports:
       - "8808:80"

ChatQnA/docker_compose/intel/cpu/xeon/compose_faqgen.yaml

@@ -32,7 +32,7 @@ services:
       retries: 50
     restart: unless-stopped
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-server
     ports:
       - "6006:80"
@@ -65,7 +65,7 @@ services:
       RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_REDIS"
     restart: unless-stopped
   tei-reranking-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-reranking-server
     ports:
       - "8808:80"

ChatQnA/docker_compose/intel/cpu/xeon/compose_faqgen_tgi.yaml

@@ -32,7 +32,7 @@ services:
       retries: 50
     restart: unless-stopped
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-server
     ports:
       - "6006:80"
@@ -65,7 +65,7 @@ services:
       RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_REDIS"
     restart: unless-stopped
   tei-reranking-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-reranking-server
     ports:
       - "8808:80"

ChatQnA/docker_compose/intel/cpu/xeon/compose_mariadb.yaml

@@ -0,0 +1,185 @@
+# Copyright (C) 2025 MariaDB Foundation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  mariadb-server:
+    image: mariadb:latest
+    container_name: mariadb-server
+    ports:
+      - "3306:3306"
+    environment:
+      - MARIADB_DATABASE=${MARIADB_DATABASE}
+      - MARIADB_USER=${MARIADB_USER}
+      - MARIADB_PASSWORD=${MARIADB_PASSWORD}
+      - MARIADB_RANDOM_ROOT_PASSWORD=1
+    healthcheck:
+      test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
+      start_period: 10s
+      interval: 10s
+      timeout: 5s
+      retries: 3
+  dataprep-mariadb-vector:
+    image: ${REGISTRY:-opea}/dataprep:${TAG:-latest}
+    container_name: dataprep-mariadb-vector
+    depends_on:
+      mariadb-server:
+        condition: service_healthy
+      tei-embedding-service:
+        condition: service_started
+    ports:
+      - "6007:5000"
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      DATAPREP_COMPONENT_NAME: "OPEA_DATAPREP_MARIADBVECTOR"
+      MARIADB_CONNECTION_URL: mariadb+mariadbconnector://${MARIADB_USER}:${MARIADB_PASSWORD}@mariadb-server:3306/${MARIADB_DATABASE}
+      TEI_ENDPOINT: http://tei-embedding-service:80
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://localhost:5000/v1/health_check || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 50
+    restart: unless-stopped
+  tei-embedding-service:
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+    container_name: tei-embedding-server
+    ports:
+      - "6006:80"
+    volumes:
+      - "${MODEL_CACHE:-./data}:/data"
+    shm_size: 1g
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+    command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate
+  retriever:
+    image: ${REGISTRY:-opea}/retriever:${TAG:-latest}
+    container_name: retriever-mariadb-vector
+    depends_on:
+      mariadb-server:
+        condition: service_healthy
+    ports:
+      - "7000:7000"
+    ipc: host
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      MARIADB_CONNECTION_URL: mariadb+mariadbconnector://${MARIADB_USER}:${MARIADB_PASSWORD}@mariadb-server:3306/${MARIADB_DATABASE}
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      LOGFLAG: ${LOGFLAG}
+      RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_MARIADBVECTOR"
+    restart: unless-stopped
+  tei-reranking-service:
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+    container_name: tei-reranking-server
+    ports:
+      - "8808:80"
+    volumes:
+      - "${MODEL_CACHE:-./data}:/data"
+    shm_size: 1g
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      HF_HUB_DISABLE_PROGRESS_BARS: 1
+      HF_HUB_ENABLE_HF_TRANSFER: 0
+    command: --model-id ${RERANK_MODEL_ID} --auto-truncate
+  vllm-service:
+    image: ${REGISTRY:-opea}/vllm:${TAG:-latest}
+    container_name: vllm-service
+    ports:
+      - "9009:80"
+    volumes:
+      - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub"
+    shm_size: 128g
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
+      VLLM_TORCH_PROFILER_DIR: "/mnt"
+      VLLM_CPU_KVCACHE_SPACE: 40
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:9009/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 100
+    command: --model $LLM_MODEL_ID --host 0.0.0.0 --port 80
+  chatqna-xeon-backend-server:
+    image: ${REGISTRY:-opea}/chatqna:${TAG:-latest}
+    container_name: chatqna-xeon-backend-server
+    depends_on:
+      mariadb-server:
+        condition: service_healthy
+      dataprep-mariadb-vector:
+        condition: service_healthy
+      tei-embedding-service:
+        condition: service_started
+      retriever:
+        condition: service_started
+      tei-reranking-service:
+        condition: service_started
+      vllm-service:
+        condition: service_healthy
+    ports:
+      - "8888:8888"
+    environment:
+      - no_proxy=${no_proxy}
+      - https_proxy=${https_proxy}
+      - http_proxy=${http_proxy}
+      - MEGA_SERVICE_HOST_IP=chatqna-xeon-backend-server
+      - EMBEDDING_SERVER_HOST_IP=tei-embedding-service
+      - EMBEDDING_SERVER_PORT=${EMBEDDING_SERVER_PORT:-80}
+      - RETRIEVER_SERVICE_HOST_IP=retriever
+      - RERANK_SERVER_HOST_IP=tei-reranking-service
+      - RERANK_SERVER_PORT=${RERANK_SERVER_PORT:-80}
+      - LLM_SERVER_HOST_IP=vllm-service
+      - LLM_SERVER_PORT=80
+      - LLM_MODEL=${LLM_MODEL_ID}
+      - LOGFLAG=${LOGFLAG}
+    ipc: host
+    restart: always
+  chatqna-xeon-ui-server:
+    image: ${REGISTRY:-opea}/chatqna-ui:${TAG:-latest}
+    container_name: chatqna-xeon-ui-server
+    depends_on:
+      - chatqna-xeon-backend-server
+    ports:
+      - "5173:5173"
+    environment:
+      - no_proxy=${no_proxy}
+      - https_proxy=${https_proxy}
+      - http_proxy=${http_proxy}
+    ipc: host
+    restart: always
+  chatqna-xeon-nginx-server:
+    image: ${REGISTRY:-opea}/nginx:${TAG:-latest}
+    container_name: chatqna-xeon-nginx-server
+    depends_on:
+      - chatqna-xeon-backend-server
+      - chatqna-xeon-ui-server
+    ports:
+      - "${NGINX_PORT:-80}:80"
+    environment:
+      - no_proxy=${no_proxy}
+      - https_proxy=${https_proxy}
+      - http_proxy=${http_proxy}
+      - FRONTEND_SERVICE_IP=chatqna-xeon-ui-server
+      - FRONTEND_SERVICE_PORT=5173
+      - BACKEND_SERVICE_NAME=chatqna
+      - BACKEND_SERVICE_IP=chatqna-xeon-backend-server
+      - BACKEND_SERVICE_PORT=8888
+      - DATAPREP_SERVICE_IP=dataprep-mariadb-vector
+      - DATAPREP_SERVICE_PORT=5000
+    ipc: host
+    restart: always
+
+networks:
+  default:
+    driver: bridge

ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml

@@ -113,7 +113,7 @@ services:
     restart: unless-stopped
 
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-server
     ports:
       - "6006:80"
@@ -127,7 +127,7 @@ services:
     command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate
 
   tei-reranking-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-reranking-server
     ports:
       - "8808:80"

ChatQnA/docker_compose/intel/cpu/xeon/compose_pinecone.yaml

@@ -29,7 +29,7 @@ services:
       retries: 50
     restart: unless-stopped
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-server
     ports:
       - "6006:80"
@@ -60,7 +60,7 @@ services:
       RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_PINECONE"
     restart: unless-stopped
   tei-reranking-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-reranking-server
     ports:
       - "8808:80"

ChatQnA/docker_compose/intel/cpu/xeon/compose_remote.yaml

@@ -33,7 +33,7 @@ services:
       TEI_ENDPOINT: http://tei-embedding-service:80
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-server
     ports:
       - "6006:80"
@@ -66,7 +66,7 @@ services:
       RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_REDIS"
     restart: unless-stopped
   tei-reranking-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-reranking-server
     ports:
       - "8808:80"

ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml

@@ -32,7 +32,7 @@ services:
       retries: 50
     restart: unless-stopped
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-server
     ports:
       - "6006:80"
@@ -65,7 +65,7 @@ services:
       RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_REDIS"
     restart: unless-stopped
   tei-reranking-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-reranking-server
     ports:
       - "8808:80"

ChatQnA/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml

@@ -32,7 +32,7 @@ services:
       retries: 50
     restart: unless-stopped
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-server
     ports:
       - "6006:80"

ChatQnA/docker_compose/intel/cpu/xeon/set_env_mariadb.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2025 MariaDB Foundation
+# SPDX-License-Identifier: Apache-2.0
+
+pushd "../../../../../" > /dev/null
+source .set_env.sh
+popd > /dev/null
+
+if [ -z "${HUGGINGFACEHUB_API_TOKEN}" ]; then
+    echo "Error: HUGGINGFACEHUB_API_TOKEN is not set. Please set HUGGINGFACEHUB_API_TOKEN."
+fi
+
+export host_ip=$(hostname -I | awk '{print $1}')
+export MARIADB_DATABASE="vectordb"
+export MARIADB_USER="chatqna"
+export MARIADB_PASSWORD="password"
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
+export RERANK_MODEL_ID="BAAI/bge-reranker-base"
+export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
+export LOGFLAG=""
+export no_proxy="$no_proxy,chatqna-xeon-ui-server,chatqna-xeon-backend-server,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service,jaeger,prometheus,grafana,node-exporter"
+export LLM_SERVER_PORT=9000
+export NGINX_PORT=80

ChatQnA/docker_compose/intel/hpu/gaudi/README.md

@@ -95,7 +95,7 @@ d560c232b120   opea/retriever:latest
 a1d7ca2d3787   ghcr.io/huggingface/tei-gaudi:1.5.0                                                             "text-embeddings-rou…"   2 minutes ago   Up 2 minutes                0.0.0.0:8808->80/tcp, [::]:8808->80/tcp                                                tei-reranking-gaudi-server
 9a9f3fd4fd4c   opea/vllm-gaudi:latest                                                                          "python3 -m vllm.ent…"   2 minutes ago   Exited (1) 2 minutes ago                                                                                           vllm-gaudi-server
 1ab9bbdf5182   redis/redis-stack:7.2.0-v9                                                                      "/entrypoint.sh"         2 minutes ago   Up 2 minutes                0.0.0.0:6379->6379/tcp, :::6379->6379/tcp, 0.0.0.0:8001->8001/tcp, :::8001->8001/tcp   redis-vector-db
-9ee0789d819e   ghcr.io/huggingface/text-embeddings-inference:cpu-1.6                                           "text-embeddings-rou…"   2 minutes ago   Up 2 minutes                0.0.0.0:8090->80/tcp, [::]:8090->80/tcp                                                tei-embedding-gaudi-server
+9ee0789d819e   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5                                           "text-embeddings-rou…"   2 minutes ago   Up 2 minutes                0.0.0.0:8090->80/tcp, [::]:8090->80/tcp                                                tei-embedding-gaudi-server
 ```
 
 ### Test the Pipeline
@@ -148,7 +148,7 @@ The default deployment utilizes Gaudi devices primarily for the `vllm-service`,
 | ---------------------------- | ----------------------------------------------------- | ------------ |
 | redis-vector-db              | redis/redis-stack:7.2.0-v9                            | No           |
 | dataprep-redis-service       | opea/dataprep:latest                                  | No           |
-| tei-embedding-service        | ghcr.io/huggingface/text-embeddings-inference:cpu-1.6 | No           |
+| tei-embedding-service        | ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 | No           |
 | retriever                    | opea/retriever:latest                                 | No           |
 | tei-reranking-service        | ghcr.io/huggingface/tei-gaudi:1.5.0                   | 1 card       |
 | vllm-service                 | opea/vllm-gaudi:latest                                | Configurable |
@@ -164,7 +164,7 @@ The TGI (Text Generation Inference) deployment and the default deployment differ
 | ---------------------------- | ----------------------------------------------------- | -------------- |
 | redis-vector-db              | redis/redis-stack:7.2.0-v9                            | No             |
 | dataprep-redis-service       | opea/dataprep:latest                                  | No             |
-| tei-embedding-service        | ghcr.io/huggingface/text-embeddings-inference:cpu-1.6 | No             |
+| tei-embedding-service        | ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 | No             |
 | retriever                    | opea/retriever:latest                                 | No             |
 | tei-reranking-service        | ghcr.io/huggingface/tei-gaudi:1.5.0                   | 1 card         |
 | **tgi-service**              | ghcr.io/huggingface/tgi-gaudi:2.3.1                   | Configurable   |
@@ -184,7 +184,7 @@ The TGI (Text Generation Inference) deployment and the default deployment differ
 | ---------------------------- | ----------------------------------------------------- | ------------ |
 | redis-vector-db              | redis/redis-stack:7.2.0-v9                            | No           |
 | dataprep-redis-service       | opea/dataprep:latest                                  | No           |
-| tei-embedding-service        | ghcr.io/huggingface/text-embeddings-inference:cpu-1.6 | No           |
+| tei-embedding-service        | ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 | No           |
 | retriever                    | opea/retriever:latest                                 | No           |
 | tei-reranking-service        | ghcr.io/huggingface/tei-gaudi:1.5.0                   | 1 card       |
 | vllm-service                 | opea/vllm-gaudi:latest                                | Configurable |
@@ -203,7 +203,7 @@ The _compose_without_rerank.yaml_ Docker Compose file is distinct from the defau
 | ---------------------------- | ----------------------------------------------------- | -------------- |
 | redis-vector-db              | redis/redis-stack:7.2.0-v9                            | No             |
 | dataprep-redis-service       | opea/dataprep:latest                                  | No             |
-| tei-embedding-service        | ghcr.io/huggingface/text-embeddings-inference:cpu-1.6 | No             |
+| tei-embedding-service        | ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 | No             |
 | retriever                    | opea/retriever:latest                                 | No             |
 | vllm-service                 | opea/vllm-gaudi:latest                                | Configurable   |
 | chatqna-gaudi-backend-server | opea/chatqna:latest                                   | No             |
@@ -222,7 +222,7 @@ The _compose_guardrails.yaml_ Docker Compose file introduces enhancements over t
 | dataprep-redis-service       | opea/dataprep:latest                                  | No             | No       |
 | _vllm-guardrails-service_    | opea/vllm-gaudi:latest                                | 1 card         | Yes      |
 | _guardrails_                 | opea/guardrails:latest                                | No             | No       |
-| tei-embedding-service        | ghcr.io/huggingface/text-embeddings-inference:cpu-1.6 | No             | No       |
+| tei-embedding-service        | ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 | No             | No       |
 | retriever                    | opea/retriever:latest                                 | No             | No       |
 | tei-reranking-service        | ghcr.io/huggingface/tei-gaudi:1.5.0                   | 1 card         | No       |
 | vllm-service                 | opea/vllm-gaudi:latest                                | Configurable   | Yes      |
@@ -258,7 +258,7 @@ The table provides a comprehensive overview of the ChatQnA services utilized acr
 | ---------------------------- | ----------------------------------------------------- | -------- | -------------------------------------------------------------------------------------------------- |
 | redis-vector-db              | redis/redis-stack:7.2.0-v9                            | No       | Acts as a Redis database for storing and managing data.                                            |
 | dataprep-redis-service       | opea/dataprep:latest                                  | No       | Prepares data and interacts with the Redis database.                                               |
-| tei-embedding-service        | ghcr.io/huggingface/text-embeddings-inference:cpu-1.6 | No       | Provides text embedding services, often using Hugging Face models.                                 |
+| tei-embedding-service        | ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 | No       | Provides text embedding services, often using Hugging Face models.                                 |
 | retriever                    | opea/retriever:latest                                 | No       | Retrieves data from the Redis database and interacts with embedding services.                      |
 | tei-reranking-service        | ghcr.io/huggingface/tei-gaudi:1.5.0                   | Yes      | Reranks text embeddings, typically using Gaudi hardware for enhanced performance.                  |
 | vllm-service                 | opea/vllm-gaudi:latest                                | No       | Handles large language model (LLM) tasks, utilizing Gaudi hardware.                                |
@@ -284,7 +284,7 @@ ChatQnA now supports running the latest DeepSeek models, including [deepseek-ai/
 
 ### tei-embedding-service & tei-reranking-service
 
-The `ghcr.io/huggingface/text-embeddings-inference:cpu-1.6` image supporting `tei-embedding-service` and `tei-reranking-service` depends on the `EMBEDDING_MODEL_ID` or `RERANK_MODEL_ID` environment variables respectively to specify the embedding model and reranking model used for converting text into vector representations and rankings. This choice impacts the quality and relevance of the embeddings rerankings for various applications. Unlike the `vllm-service`, the `tei-embedding-service` and `tei-reranking-service` each typically acquires only one Gaudi device and does not use the `NUM_CARDS` parameter; embedding and reranking tasks generally do not require extensive parallel processing and one Gaudi per service is appropriate. The list of [supported embedding and reranking models](https://github.com/huggingface/tei-gaudi?tab=readme-ov-file#supported-models) can be found at the [huggingface/tei-gaudi](https://github.com/huggingface/tei-gaudi?tab=readme-ov-file#supported-models) website.
+The `ghcr.io/huggingface/text-embeddings-inference:cpu-1.5` image supporting `tei-embedding-service` and `tei-reranking-service` depends on the `EMBEDDING_MODEL_ID` or `RERANK_MODEL_ID` environment variables respectively to specify the embedding model and reranking model used for converting text into vector representations and rankings. This choice impacts the quality and relevance of the embeddings rerankings for various applications. Unlike the `vllm-service`, the `tei-embedding-service` and `tei-reranking-service` each typically acquires only one Gaudi device and does not use the `NUM_CARDS` parameter; embedding and reranking tasks generally do not require extensive parallel processing and one Gaudi per service is appropriate. The list of [supported embedding and reranking models](https://github.com/huggingface/tei-gaudi?tab=readme-ov-file#supported-models) can be found at the [huggingface/tei-gaudi](https://github.com/huggingface/tei-gaudi?tab=readme-ov-file#supported-models) website.
 
 ### tgi-guardrails-service
 

ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml

@@ -39,7 +39,7 @@ services:
       retries: 50
     restart: unless-stopped
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-gaudi-server
     ports:
       - "8090:80"

ChatQnA/docker_compose/intel/hpu/gaudi/compose_faqgen.yaml

@@ -33,7 +33,7 @@ services:
       retries: 50
     restart: unless-stopped
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-gaudi-server
     ports:
       - "8090:80"

ChatQnA/docker_compose/intel/hpu/gaudi/compose_faqgen_tgi.yaml

@@ -33,7 +33,7 @@ services:
       retries: 50
     restart: unless-stopped
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-gaudi-server
     ports:
       - "8090:80"

ChatQnA/docker_compose/intel/hpu/gaudi/compose_guardrails.yaml

@@ -76,7 +76,7 @@ services:
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
     restart: unless-stopped
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-gaudi-server
     ports:
       - "8090:80"

ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml

@@ -32,7 +32,7 @@ services:
       retries: 50
     restart: unless-stopped
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-gaudi-server
     ports:
       - "8090:80"

ChatQnA/docker_compose/intel/hpu/gaudi/compose_without_rerank.yaml

@@ -32,7 +32,7 @@ services:
       retries: 50
     restart: unless-stopped
   tei-embedding-service:
-    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
+    image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
     container_name: tei-embedding-gaudi-server
     ports:
       - "8090:80"

ChatQnA/docker_compose/intel/hpu/gaudi/how_to_validate_service.md

@@ -51,7 +51,7 @@ f810f3b4d329   opea/embedding:latest                               "python embed
 174bd43fa6b5   ghcr.io/huggingface/tei-gaudi:1.5.0                    "text-embeddings-rou…"   2 minutes ago   Up 2 minutes                    0.0.0.0:8090->80/tcp, :::8090->80/tcp                                                  tei-embedding-gaudi-server
 05c40b636239   ghcr.io/huggingface/tgi-gaudi:2.3.1                     "text-generation-lau…"   2 minutes ago   Exited (1) About a minute ago                                                                                          tgi-gaudi-server
 74084469aa33   redis/redis-stack:7.2.0-v9                              "/entrypoint.sh"         2 minutes ago   Up 2 minutes                    0.0.0.0:6379->6379/tcp, :::6379->6379/tcp, 0.0.0.0:8001->8001/tcp, :::8001->8001/tcp   redis-vector-db
-88399dbc9e43   ghcr.io/huggingface/text-embeddings-inference:cpu-1.6   "text-embeddings-rou…"   2 minutes ago   Up 2 minutes                    0.0.0.0:8808->80/tcp, :::8808->80/tcp                                                  tei-reranking-gaudi-server
+88399dbc9e43   ghcr.io/huggingface/text-embeddings-inference:cpu-1.5   "text-embeddings-rou…"   2 minutes ago   Up 2 minutes                    0.0.0.0:8808->80/tcp, :::8808->80/tcp                                                  tei-reranking-gaudi-server
 ```
 
 In this case, `ghcr.io/huggingface/tgi-gaudi:2.3.1` Existed.

ChatQnA/tests/test_compose_faqgen_on_gaudi.sh

@@ -24,15 +24,13 @@ function build_docker_images() {
     docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
     popd && sleep 1s
     git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.7.2+Gaudi-1.21.0
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="chatqna chatqna-ui dataprep retriever llm-faqgen vllm-gaudi nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
-    docker pull ghcr.io/huggingface/tei-gaudi:1.5.0
     docker images && sleep 1s
 }
 

ChatQnA/tests/test_compose_faqgen_on_rocm.sh

@@ -69,9 +69,6 @@ function build_docker_images() {
     service_list="chatqna chatqna-ui dataprep retriever llm-faqgen nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > "${LOG_PATH}"/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
-
     docker images && sleep 1s
 }
 

ChatQnA/tests/test_compose_faqgen_on_xeon.sh

@@ -32,7 +32,6 @@ function build_docker_images() {
     service_list="chatqna chatqna-ui dataprep retriever llm-faqgen vllm nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
     docker images && sleep 1s
 }
 

ChatQnA/tests/test_compose_faqgen_tgi_on_gaudi.sh

@@ -28,9 +28,6 @@ function build_docker_images() {
     service_list="chatqna chatqna-ui dataprep retriever llm-faqgen nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
-    docker pull ghcr.io/huggingface/tei-gaudi:1.5.0
     docker images && sleep 1s
 }
 

ChatQnA/tests/test_compose_faqgen_tgi_on_xeon.sh

@@ -32,8 +32,6 @@ function build_docker_images() {
     service_list="chatqna chatqna-ui dataprep retriever llm-faqgen nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
     docker images && sleep 1s
 }
 

ChatQnA/tests/test_compose_guardrails_on_gaudi.sh

@@ -24,16 +24,13 @@ function build_docker_images() {
     docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
     popd && sleep 1s
     git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.7.2+Gaudi-1.21.0
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="chatqna chatqna-ui dataprep retriever vllm-gaudi guardrails nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
-    docker pull ghcr.io/huggingface/tei-gaudi:1.5.0
-
     docker images && sleep 1s
 }
 

ChatQnA/tests/test_compose_mariadb_on_xeon.sh

@@ -0,0 +1,176 @@
+#!/bin/bash
+# Copyright (C) 2025 MariaDB Foundation
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+export REGISTRY=${IMAGE_REPO}
+export TAG=${IMAGE_TAG}
+export MODEL_CACHE=${model_cache:-"./data"}
+
+WORKPATH=$(dirname "$PWD")
+LOG_PATH="$WORKPATH/tests"
+ip_address=$(hostname -I | awk '{print $1}')
+
+function build_docker_images() {
+    opea_branch=${opea_branch:-"main"}
+    cd $WORKPATH/docker_image_build
+    git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
+    git clone https://github.com/vllm-project/vllm.git && cd vllm
+    VLLM_VER="v0.8.3"
+    echo "Check out vLLM tag ${VLLM_VER}"
+    git checkout ${VLLM_VER} &> /dev/null
+    # make sure NOT change the pwd
+    cd ../
+
+    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
+    service_list="chatqna chatqna-ui dataprep retriever vllm nginx"
+    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
+
+    docker images && sleep 1s
+}
+
+function start_services() {
+    cd $WORKPATH/docker_compose/intel/cpu/xeon
+    export MARIADB_DATABASE="vectordb"
+    export MARIADB_USER="chatqna"
+    export MARIADB_PASSWORD="test"
+    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
+    export RERANK_MODEL_ID="BAAI/bge-reranker-base"
+    export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
+    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+    export host_ip=${ip_address}
+
+    # Start Docker Containers
+    docker compose -f compose_mariadb.yaml up -d > ${LOG_PATH}/start_services_with_compose.log
+    n=0
+    until [[ "$n" -ge 100 ]]; do
+        docker logs vllm-service > ${LOG_PATH}/vllm_service_start.log 2>&1
+        if grep -q complete ${LOG_PATH}/vllm_service_start.log; then
+            break
+        fi
+        sleep 5s
+        n=$((n+1))
+    done
+}
+
+function validate_service() {
+    local URL="$1"
+    local EXPECTED_RESULT="$2"
+    local SERVICE_NAME="$3"
+    local DOCKER_NAME="$4"
+    local INPUT_DATA="$5"
+
+    local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
+    if [ "$HTTP_STATUS" -eq 200 ]; then
+        echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
+
+        local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
+
+        if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
+            echo "[ $SERVICE_NAME ] Content is as expected."
+        else
+            echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
+            docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+            exit 1
+        fi
+    else
+        echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
+        exit 1
+    fi
+    sleep 1s
+}
+
+function validate_microservices() {
+    # Check if the microservices are running correctly.
+    sleep 3m
+
+    # tei for embedding service
+    validate_service \
+        "${ip_address}:6006/embed" \
+        "\[\[" \
+        "tei-embedding" \
+        "tei-embedding-server" \
+        '{"inputs":"What is Deep Learning?"}'
+
+    # retrieval microservice
+    test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
+    validate_service \
+        "${ip_address}:7000/v1/retrieval" \
+        " " \
+        "retrieval" \
+        "retriever-mariadb-vector" \
+        "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}"
+
+    # tei for rerank microservice
+    validate_service \
+        "${ip_address}:8808/rerank" \
+        '{"index":1,"score":' \
+        "tei-rerank" \
+        "tei-reranking-server" \
+        '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}'
+
+    # vllm for llm service
+    validate_service \
+        "${ip_address}:9009/v1/chat/completions" \
+        "content" \
+        "vllm-llm" \
+        "vllm-service" \
+        '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}'
+}
+
+function validate_megaservice() {
+    # Curl the Mega Service
+    validate_service \
+        "${ip_address}:8888/v1/chatqna" \
+        "Nike" \
+        "mega-chatqna" \
+        "chatqna-xeon-backend-server" \
+        '{"messages": "What is the revenue of Nike in 2023?"}'
+
+}
+
+function stop_docker() {
+    cd $WORKPATH/docker_compose/intel/cpu/xeon
+    docker compose -f compose_mariadb.yaml down
+}
+
+function main() {
+
+    echo "::group::stop_docker"
+    stop_docker
+    echo "::endgroup::"
+
+    echo "::group::build_docker_images"
+    if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
+
+    echo "::group::start_services"
+    start_services
+    echo "::endgroup::"
+
+    echo "::group::validate_microservices"
+    validate_microservices
+    echo "::endgroup::"
+
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
+    stop_docker
+    echo "::endgroup::"
+
+    docker system prune -f
+
+}
+
+main

ChatQnA/tests/test_compose_milvus_on_xeon.sh

@@ -35,13 +35,10 @@ function build_docker_images() {
     service_list="chatqna chatqna-ui dataprep retriever vllm nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
-
     docker images && sleep 1s
 }
 function start_services() {
     cd $WORKPATH/docker_compose/intel/cpu/xeon/
-    export no_proxy=${no_proxy},${ip_address}
     export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
     export RERANK_MODEL_ID="BAAI/bge-reranker-base"
     export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"

ChatQnA/tests/test_compose_on_gaudi.sh

@@ -24,15 +24,13 @@ function build_docker_images() {
     docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
     popd && sleep 1s
     git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.7.2+Gaudi-1.21.0
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="chatqna chatqna-ui dataprep retriever vllm-gaudi nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
-    docker pull ghcr.io/huggingface/tei-gaudi:1.5.0
     docker images && sleep 1s
 }
 

ChatQnA/tests/test_compose_on_rocm.sh

@@ -67,9 +67,6 @@ function build_docker_images() {
     service_list="chatqna chatqna-ui dataprep retriever nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > "${LOG_PATH}"/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
-
     docker images && sleep 1s
 }
 

ChatQnA/tests/test_compose_on_xeon.sh

@@ -34,8 +34,6 @@ function build_docker_images() {
     service_list="chatqna chatqna-ui dataprep retriever vllm nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
-
     docker images && sleep 1s
 }
 

ChatQnA/tests/test_compose_pinecone_on_xeon.sh

@@ -35,8 +35,6 @@ function build_docker_images() {
     service_list="chatqna chatqna-ui dataprep retriever vllm nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
-
     docker images && sleep 1s
 }
 

ChatQnA/tests/test_compose_tgi_on_gaudi.sh

@@ -27,10 +27,6 @@ function build_docker_images() {
     service_list="chatqna chatqna-ui dataprep retriever nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/tgi-gaudi:2.3.1
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
-    docker pull ghcr.io/huggingface/tei-gaudi:1.5.0
-
     docker images && sleep 1s
 }
 

ChatQnA/tests/test_compose_tgi_on_xeon.sh

@@ -27,9 +27,6 @@ function build_docker_images() {
     service_list="chatqna chatqna-ui dataprep retriever nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
-
     docker images && sleep 1s
 }
 

ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh

@@ -24,16 +24,13 @@ function build_docker_images() {
     docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
     popd && sleep 1s
     git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.7.2+Gaudi-1.21.0
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="chatqna chatqna-ui dataprep retriever vllm-gaudi nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
-    docker pull ghcr.io/huggingface/tei-gaudi:1.5.0
-
     docker images && sleep 1s
 }
 

ChatQnA/tests/test_compose_without_rerank_on_xeon.sh

@@ -35,8 +35,6 @@ function build_docker_images() {
     service_list="chatqna chatqna-ui dataprep retriever vllm nginx"
     docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
 
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
-
     docker images && sleep 1s
 }
 

CodeGen/Dockerfile

@@ -1,8 +1,9 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+ARG IMAGE_REPO=opea
 ARG BASE_TAG=latest
-FROM opea/comps-base:$BASE_TAG
+FROM $IMAGE_REPO/comps-base:$BASE_TAG
 
 COPY ./codegen.py $HOME/codegen.py
 

CodeGen/docker_compose/amd/gpu/rocm/README.md

@@ -2,78 +2,69 @@
 
 This README provides instructions for deploying the CodeGen application using Docker Compose on a system equipped with AMD GPUs supporting ROCm, detailing the steps to configure, run, and validate the services. This guide defaults to using the **vLLM** backend for LLM serving.
 
+If the service response has a meaningful response in the value of the "choices.text" key,
+then we consider the vLLM service to be successfully launched
+
 ## Table of Contents
 
-- [Steps to Run with Docker Compose (Default vLLM)](#steps-to-run-with-docker-compose-default-vllm)
-- [Service Overview](#service-overview)
+- [Overview](#overview)
+- [Prerequisites](#prerequisites)
+- [Quick Start](#quick-start)
 - [Available Deployment Options](#available-deployment-options)
-  - [compose_vllm.yaml (vLLM - Default)](#compose_vllyaml-vllm---default)
-  - [compose.yaml (TGI)](#composeyaml-tgi)
-- [Configuration Parameters and Usage](#configuration-parameters-and-usage)
-  - [Docker Compose GPU Configuration](#docker-compose-gpu-configuration)
-  - [Environment Variables (`set_env*.sh`)](#environment-variables-set_envsh)
-- [Building Docker Images Locally (Optional)](#building-docker-images-locally-optional)
-  - [1. Setup Build Environment](#1-setup-build-environment)
-  - [2. Clone Repositories](#2-clone-repositories)
-  - [3. Select Services and Build](#3-select-services-and-build)
-- [Validate Service Health](#validate-service-health)
-  - [1. Validate the vLLM/TGI Service](#1-validate-the-vllmtgi-service)
-  - [2. Validate the LLM Service](#2-validate-the-llm-service)
-  - [3. Validate the MegaService (Backend)](#3-validate-the-megaservice-backend)
-  - [4. Validate the Frontend (UI)](#4-validate-the-frontend-ui)
-- [How to Open the UI](#how-to-open-the-ui)
+  - [Default: vLLM-based Deployment (`--profile codegen-xeon-vllm`)](#default-vllm-based-deployment---profile-codegen-xeon-vllm)
+  - [TGI-based Deployment (`--profile codegen-xeon-tgi`)](#tgi-based-deployment---profile-codegen-xeon-tgi)
+- [Configuration Parameters](#configuration-parameters)
+  - [Environment Variables](#environment-variables)
+  - [Compose Profiles](#compose-profiles)
+- [Building Custom Images (Optional)](#building-custom-images-optional)
+- [Validate Services](#validate-services)
+  - [Check Container Status](#check-container-status)
+  - [Run Validation Script/Commands](#run-validation-scriptcommands)
+- [Accessing the User Interface (UI)](#accessing-the-user-interface-ui)
+  - [Gradio UI (Default)](#gradio-ui-default)
+  - [Svelte UI (Optional)](#svelte-ui-optional)
+  - [React UI (Optional)](#react-ui-optional)
+  - [VS Code Extension (Optional)](#vs-code-extension-optional)
 - [Troubleshooting](#troubleshooting)
 - [Stopping the Application](#stopping-the-application)
 - [Next Steps](#next-steps)
 
-## Steps to Run with Docker Compose (Default vLLM)
+## Overview
 
-_This section assumes you are using pre-built images and targets the default vLLM deployment._
+This guide focuses on running the pre-configured CodeGen service using Docker Compose on AMD ROCm processing acelarating platform. It leverages containers optimized for Intel architecture for the CodeGen gateway, LLM serving (vLLM or TGI), and UI.
 
-1.  **Set Deploy Environment Variables:**
+## CodeGen Quick Start Deployment
 
-    - Go to the Docker Compose directory:
-      ```bash
-      # Adjust path if your GenAIExamples clone is located elsewhere
-      cd GenAIExamples/CodeGen/docker_compose/amd/gpu/rocm
-      ```
-    - Setting variables in the operating system environment:
-      - Set variable `HUGGINGFACEHUB_API_TOKEN`:
-        ```bash
-        ### Replace the string 'your_huggingfacehub_token' with your HuggingFacehub repository access token.
-        export HUGGINGFACEHUB_API_TOKEN='your_huggingfacehub_token'
-        ```
-    - Edit the environment script for the **vLLM** deployment (`set_env_vllm.sh`):
-      ```bash
-      nano set_env_vllm.sh
-      ```
-      - Configure `HOST_IP`, `EXTERNAL_HOST_IP`, `*_PORT` variables, and proxies (`http_proxy`, `https_proxy`, `no_proxy`) as described in the Configuration section below.
-    - Source the environment variables:
-      ```bash
-      . set_env_vllm.sh
-      ```
+This section describes how to quickly deploy and test the CodeGen service manually on an AMD GPU (ROCm) platform. The basic steps are:
 
-2.  **Start the Services (vLLM):**
+1. [Prerequisites](#prerequisites)
+2. [Generate a HuggingFace Access Token](#generate-a-huggingface-access-token)
+3. [Configure the Deployment Environment](#configure-the-deployment-environment)
+4. [Deploy the Services Using Docker Compose](#deploy-the-services-using-docker-compose)
+5. [Check the Deployment Status](#check-the-deployment-status)
+6. [Test the Pipeline](#test-the-pipeline)
+7. [Cleanup the Deployment](#cleanup-the-deployment)
 
-    ```bash
-    docker compose -f compose_vllm.yaml up -d
-    ```
+## Prerequisites
 
-3.  **Verify:** Proceed to the [Validate Service Health](#validate-service-health) section after allowing time for services to start.
+- Docker and Docker Compose installed.
+- x86 Intel or AMD CPU.
+- 4x AMD Instinct MI300X Accelerators.
+- Git installed (for cloning repository).
+- Hugging Face Hub API Token (for downloading models).
+- Access to the internet (or a private model cache).
+- Clone the `GenAIExamples` repository:
 
-## Service Overview
+```bash
+  git clone https://github.com/opea-project/GenAIExamples.git
+  cd GenAIExamples/CodeGen/docker_compose/amd/gpu/rocm/
+```
 
-When using the default `compose_vllm.yaml` (vLLM-based), the following services are deployed:
+Checkout a released version, such as v1.3:
 
-| Service Name           | Default Port (Host)                            | Internal Port | Purpose                     |
-| :--------------------- | :--------------------------------------------- | :------------ | :-------------------------- |
-| codegen-vllm-service   | `${CODEGEN_VLLM_SERVICE_PORT}` (e.g., 8028)    | 8000          | LLM Serving (vLLM on ROCm)  |
-| codegen-llm-server     | `${CODEGEN_LLM_SERVICE_PORT}` (e.g., 9000)     | 80            | LLM Microservice Wrapper    |
-| codegen-backend-server | `${CODEGEN_BACKEND_SERVICE_PORT}` (e.g., 7778) | 80            | CodeGen MegaService/Gateway |
-| codegen-ui-server      | `${CODEGEN_UI_SERVICE_PORT}` (e.g., 5173)      | 80            | Frontend User Interface     |
-
-_(Note: Ports are configurable via `set_env_vllm.sh`. Check the script for actual defaults used.)_
-_(Note: The TGI deployment (`compose.yaml`) uses `codegen-tgi-service` instead of `codegen-vllm-service`)_
+```bash
+git checkout v1.3
+```
 
 ## Available Deployment Options
 
@@ -91,6 +82,69 @@ This directory provides different Docker Compose files:
 
 ## Configuration Parameters and Usage
 
+### Environment Variables (`set_env*.sh`)
+
+These scripts (`set_env_vllm.sh` for vLLM, `set_env.sh` for TGI) configure crucial parameters passed to the containers.
+
+This example covers the single-node on-premises deployment of the CodeGen example using OPEA components. There are various ways to enable CodeGen, but this example will focus on four options available for deploying the CodeGen pipeline to AMD ROCm AI Accelerators. This example begins with a Quick Start section and then documents how to modify deployments, leverage new models and configure the number of allocated devices.
+
+This example includes the following sections:
+
+- [CodeGen Quick Start Deployment](#CodeGen-quick-start-deployment): Demonstrates how to quickly deploy a CodeGen application/pipeline on AMD GPU (ROCm) platform.
+- [CodeGen Docker Compose Files](#CodeGen-docker-compose-files): Describes some example deployments and their docker compose files.
+- [CodeGen Service Configuration](#CodeGen-service-configuration): Describes the services and possible configuration changes.
+
+**Note** This example requires access to a properly installed AMD ROCm platform with a functional Docker service configured
+
+## Generate a HuggingFace Access Token
+
+Some HuggingFace resources, such as some models, are only accessible if you have an access token. If you do not already have a HuggingFace access token, you can create one by first creating an account by following the steps provided at [HuggingFace](https://huggingface.co/) and then generating a [user access token](https://huggingface.co/docs/transformers.js/en/guides/private#step-1-generating-a-user-access-token).
+
+## Configure the Deployment Environment
+
+### Environment Variables
+
+Key parameters are configured via environment variables set before running `docker compose up`.
+
+| Environment Variable                    | Description                                                                                                         | Default (Set Externally)                                                                         |
+| :-------------------------------------- | :------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------- |
+| `HOST_IP`                               | External IP address of the host machine. **Required.**                                                              | `your_external_ip_address`                                                                       |
+| `HUGGINGFACEHUB_API_TOKEN`              | Your Hugging Face Hub token for model access. **Required.**                                                         | `your_huggingface_token`                                                                         |
+| `LLM_MODEL_ID`                          | Hugging Face model ID for the CodeGen LLM (used by TGI/vLLM service). Configured within `compose.yaml` environment. | `Qwen/Qwen2.5-Coder-7B-Instruct`                                                                 |
+| `EMBEDDING_MODEL_ID`                    | Hugging Face model ID for the embedding model (used by TEI service). Configured within `compose.yaml` environment.  | `BAAI/bge-base-en-v1.5`                                                                          |
+| `LLM_ENDPOINT`                          | Internal URL for the LLM serving endpoint (used by `codegen-llm-server`). Configured in `compose.yaml`.             | `http://codegen-tgi-server:80/generate` or `http://codegen-vllm-server:8000/v1/chat/completions` |
+| `TEI_EMBEDDING_ENDPOINT`                | Internal URL for the Embedding service. Configured in `compose.yaml`.                                               | `http://codegen-tei-embedding-server:80/embed`                                                   |
+| `DATAPREP_ENDPOINT`                     | Internal URL for the Data Preparation service. Configured in `compose.yaml`.                                        | `http://codegen-dataprep-server:80/dataprep`                                                     |
+| `BACKEND_SERVICE_ENDPOINT`              | External URL for the CodeGen Gateway (MegaService). Derived from `HOST_IP` and port `7778`.                         | `http://${HOST_IP}:7778/v1/codegen`                                                              |
+| `*_PORT` (Internal)                     | Internal container ports (e.g., `80`, `6379`). Defined in `compose.yaml`.                                           | N/A                                                                                              |
+| `http_proxy` / `https_proxy`/`no_proxy` | Network proxy settings (if required).                                                                               | `""`                                                                                             |
+
+To set up environment variables for deploying CodeGen services, source the _setup_env.sh_ script in this directory:
+
+For TGI
+
+```bash
+export host_ip="External_Public_IP"           #ip address of the node
+export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token"
+export http_proxy="Your_HTTP_Proxy"           #http proxy if any
+export https_proxy="Your_HTTPs_Proxy"         #https proxy if any
+export no_proxy=localhost,127.0.0.1,$host_ip  #additional no proxies if needed
+export no_proxy=$no_proxy
+source ./set_env.sh
+```
+
+For vLLM
+
+```bash
+export host_ip="External_Public_IP"           #ip address of the node
+export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token"
+export http_proxy="Your_HTTP_Proxy"           #http proxy if any
+export https_proxy="Your_HTTPs_Proxy"         #https proxy if any
+export no_proxy=localhost,127.0.0.1,$host_ip  #additional no proxies if needed
+export no_proxy=$no_proxy
+source ./set_env_vllm.sh
+```
+
 ### Docker Compose GPU Configuration
 
 To enable GPU support for AMD GPUs, the following configuration is added to the Docker Compose files (`compose.yaml`, `compose_vllm.yaml`) for the LLM serving container:
@@ -103,7 +157,6 @@ shm_size: 1g
 devices:
   - /dev/kfd:/dev/kfd
   - /dev/dri/:/dev/dri/
-# - /dev/dri/render128:/dev/dri/render128
 cap_add:
   - SYS_PTRACE
 group_add:
@@ -112,302 +165,329 @@ security_opt:
   - seccomp:unconfined
 ```
 
-This configuration forwards all available GPUs to the container. To use a specific GPU, specify its `cardN` and `renderN` device IDs (e.g., `/dev/dri/card0:/dev/dri/card0`, `/dev/dri/render128:/dev/dri/render128`). Use AMD GPU driver utilities to identify device IDs.
+This configuration forwards all available GPUs to the container. To use a specific GPU, specify its `cardN` and `renderN` device IDs (e.g., `/dev/dri/card0:/dev/dri/card0`, `/dev/dri/render128:/dev/dri/render128`). For example:
 
-### Environment Variables (`set_env*.sh`)
+```yaml
+shm_size: 1g
+devices:
+  - /dev/kfd:/dev/kfd
+  - /dev/dri/card0:/dev/dri/card0
+  - /dev/dri/render128:/dev/dri/render128
+cap_add:
+  - SYS_PTRACE
+group_add:
+  - video
+security_opt:
+  - seccomp:unconfined
+```
 
-These scripts (`set_env_vllm.sh` for vLLM, `set_env.sh` for TGI) configure crucial parameters passed to the containers.
+**How to Identify GPU Device IDs:**
+Use AMD GPU driver utilities to determine the correct `cardN` and `renderN` IDs for your GPU.
 
-| Environment Variable           | Description                                                                                              | Example Value (Edit in Script)   |
-| :----------------------------- | :------------------------------------------------------------------------------------------------------- | :------------------------------- |
-| `HUGGINGFACEHUB_API_TOKEN`     | Your Hugging Face Hub token for model access. **Required.**                                              | `your_huggingfacehub_token`      |
-| `HOST_IP`                      | Internal/Primary IP address of the host machine. Used for inter-service communication. **Required.**     | `192.168.1.100`                  |
-| `EXTERNAL_HOST_IP`             | External IP/hostname used to access the UI from outside. Same as `HOST_IP` if no proxy/LB. **Required.** | `192.168.1.100`                  |
-| `CODEGEN_LLM_MODEL_ID`         | Hugging Face model ID for the CodeGen LLM.                                                               | `Qwen/Qwen2.5-Coder-7B-Instruct` |
-| `CODEGEN_VLLM_SERVICE_PORT`    | Host port mapping for the vLLM serving endpoint (in `set_env_vllm.sh`).                                  | `8028`                           |
-| `CODEGEN_TGI_SERVICE_PORT`     | Host port mapping for the TGI serving endpoint (in `set_env.sh`).                                        | `8028`                           |
-| `CODEGEN_LLM_SERVICE_PORT`     | Host port mapping for the LLM Microservice wrapper.                                                      | `9000`                           |
-| `CODEGEN_BACKEND_SERVICE_PORT` | Host port mapping for the CodeGen MegaService/Gateway.                                                   | `7778`                           |
-| `CODEGEN_UI_SERVICE_PORT`      | Host port mapping for the UI service.                                                                    | `5173`                           |
-| `http_proxy`                   | Network HTTP Proxy URL (if required).                                                                    | `Your_HTTP_Proxy`                |
-| `https_proxy`                  | Network HTTPS Proxy URL (if required).                                                                   | `Your_HTTPs_Proxy`               |
-| `no_proxy`                     | Comma-separated list of hosts to bypass proxy. Should include `localhost,127.0.0.1,$HOST_IP`.            | `localhost,127.0.0.1`            |
+### Deploy the Services Using Docker Compose
 
-**How to Use:** Edit the relevant `set_env*.sh` file (`set_env_vllm.sh` for the default) with your values, then source it (`. ./set_env*.sh`) before running `docker compose`.
+Please refer to the table below to build different microservices from source:
 
-## Building Docker Images Locally (Optional)
+When using the default `compose_vllm.yaml` (vLLM-based), the following services are deployed:
 
-Follow these steps if you need to build the Docker images from source instead of using pre-built ones.
+| Service Name           | Default Port (Host)                            | Internal Port | Purpose                     |
+| :--------------------- | :--------------------------------------------- | :------------ | :-------------------------- |
+| codegen-vllm-service   | `${CODEGEN_VLLM_SERVICE_PORT}` (e.g., 8028)    | 8000          | LLM Serving (vLLM on ROCm)  |
+| codegen-llm-server     | `${CODEGEN_LLM_SERVICE_PORT}` (e.g., 9000)     | 80            | LLM Microservice Wrapper    |
+| codegen-backend-server | `${CODEGEN_BACKEND_SERVICE_PORT}` (e.g., 7778) | 80            | CodeGen MegaService/Gateway |
+| codegen-ui-server      | `${CODEGEN_UI_SERVICE_PORT}` (e.g., 5173)      | 80            | Frontend User Interface     |
 
-### 1. Setup Build Environment
+To deploy the CodeGen services, execute the `docker compose up` command with the appropriate arguments. For a vLLM deployment, execute:
 
-- #### Create application install directory and go to it:
+```bash
+docker compose -f compose_vllm.sh up -d
+```
 
-  ```bash
-  mkdir ~/codegen-install && cd codegen-install
-  ```
+The CodeGen docker images should automatically be downloaded from the `OPEA registry` and deployed on the AMD GPU (ROCM) Platform:
 
-### 2. Clone Repositories
+```bash
+[+] Running 5/5_default              Created                                                              0.3s
+ ✔ Network rocm_default              Created                                                              0.3s
+ ✔ Container codegen-vllm-service    Healthy                                                              100.9s
+ ✔ Container codegen-llm-server      Started                                                              101.2s
+ ✔ Container codegen-backend-server  Started                                                              101.5s
+ ✔ Container codegen-ui-server       Started                                                              101.9s
+```
 
-- #### Clone the repository GenAIExamples (the default repository branch "main" is used here):
+# To deploy the CodeGen services, execute the `docker compose up` command with the appropriate arguments. For a TGI deployment, execute:
 
-  ```bash
-  git clone https://github.com/opea-project/GenAIExamples.git
-  ```
+```
+docker compose  up -d
+```
 
-  If you need to use a specific branch/tag of the GenAIExamples repository, then (v1.3 replace with its own value):
+The CodeGen docker images should automatically be downloaded from the `OPEA registry` and deployed on the AMD GPU (ROCM) Platform:
 
-  ```bash
-  git clone https://github.com/opea-project/GenAIExamples.git && cd GenAIExamples && git checkout v1.3
-  ```
+```bash
+[+] Running 5/5_default              Created                                                                0.4s
+ ✔ Network rocm_default              Created                                                                0.4s
+ ✔ Container codegen-tgi-service     Healthy                                                                102.6s
+ ✔ Container codegen-llm-server      Started                                                                100.2s
+ ✔ Container codegen-backend-server  Started                                                                103.7s
+ ✔ Container codegen-ui-server       Started                                                                102.9s
+```
 
-  We remind you that when using a specific version of the code, you need to use the README from this version.
+## Building Custom Images (Optional)
 
-- #### Go to build directory:
+If you need to modify the microservices:
 
-  ```bash
-  cd ~/codegen-install/GenAIExamples/CodeGen/docker_image_build
-  ```
+1.  Clone the [OPEA GenAIComps](https://github.com/opea-project/GenAIComps) repository.
+2.  Follow build instructions in the respective component directories (e.g., `comps/llms/text-generation`, `comps/codegen`, `comps/ui/gradio`, etc.). Use the provided Dockerfiles (e.g., `CodeGen/Dockerfile`, `CodeGen/ui/docker/Dockerfile.gradio`).
+3.  Tag your custom images appropriately (e.g., `my-custom-codegen:latest`).
+4.  Update the `image:` fields in the `compose.yaml` file to use your custom image tags.
 
-- Cleaning up the GenAIComps repository if it was previously cloned in this directory.
-  This is necessary if the build was performed earlier and the GenAIComps folder exists and is not empty:
+_Refer to the main [CodeGen README](../../../../README.md) for links to relevant GenAIComps components._
 
-  ```bash
-  echo Y | rm -R GenAIComps
-  ```
+## Validate Services
 
-- #### Clone the repository GenAIComps (the default repository branch "main" is used here):
+### Check the Deployment Status for TGI base deployment
 
-  ```bash
-  git clone https://github.com/opea-project/GenAIComps.git
-  ```
+After running docker compose, check if all the containers launched via docker compose have started:
 
-  If you use a specific tag of the GenAIExamples repository,
-  then you should also use the corresponding tag for GenAIComps. (v1.3 replace with its own value):
+```bash
+docker ps -a
+```
 
-  ```bash
-  git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout v1.3
-  ```
+For the default deployment, the following 10 containers should have started:
 
-  We remind you that when using a specific version of the code, you need to use the README from this version.
+```bash
+CONTAINER ID   IMAGE                                                      COMMAND                  CREATED         STATUS                   PORTS                                           NAMES
+1d08caeae2ed   opea/codegen-ui:latest                                     "docker-entrypoint.s…"   2 minutes ago   Up About a minute        0.0.0.0:18151->5173/tcp, [::]:18151->5173/tcp   codegen-ui-server
+f52adc66c116   opea/codegen:latest                                        "python codegen.py"      2 minutes ago   Up About a minute        0.0.0.0:18150->7778/tcp, [::]:18150->7778/tcp   codegen-backend-server
+4b1cb8f5d4ff   opea/llm-textgen:latest                                    "bash entrypoint.sh"     2 minutes ago   Up About a minute        0.0.0.0:9000->9000/tcp, :::9000->9000/tcp       codegen-llm-server
+3bb4ee0abf15   ghcr.io/huggingface/text-generation-inference:2.4.1-rocm   "/tgi-entrypoint.sh …"   2 minutes ago   Up 2 minutes (healthy)   0.0.0.0:8028->80/tcp, [::]:8028->80/tcp         codegen-tgi-service
+```
 
-### 3. Select Services and Build
+### Check the Deployment Status for vLLM base deployment
 
-- #### Setting the list of images for the build (from the build file.yaml)
+After running docker compose, check if all the containers launched via docker compose have started:
 
-  Select the services corresponding to your desired deployment (vLLM is the default):
+```bash
+docker ps -a
+```
 
-  ##### vLLM-based application (Default)
+For the default deployment, the following 10 containers should have started:
 
-  ```bash
-  service_list="vllm-rocm llm-textgen codegen codegen-ui"
-  ```
+```bash
+CONTAINER ID   IMAGE                     COMMAND                  CREATED          STATUS                    PORTS                                           NAMES
+f100cc326343   opea/codegen-ui:latest    "docker-entrypoint.s…"   16 minutes ago   Up 14 minutes             0.0.0.0:18151->5173/tcp, [::]:18151->5173/tcp   codegen-ui-server
+c59de0b2da5b   opea/codegen:latest       "python codegen.py"      16 minutes ago   Up 14 minutes             0.0.0.0:18150->7778/tcp, [::]:18150->7778/tcp   codegen-backend-server
+dcd83e0e4c0f   opea/llm-textgen:latest   "bash entrypoint.sh"     16 minutes ago   Up 14 minutes             0.0.0.0:9000->9000/tcp, :::9000->9000/tcp       codegen-llm-server
+d091d8f2fab6   opea/vllm-rocm:latest     "python3 /workspace/…"   16 minutes ago   Up 16 minutes (healthy)   0.0.0.0:8028->8011/tcp, [::]:8028->8011/tcp     codegen-vllm-service
+```
 
-  ##### TGI-based application
+### Test the Pipeline
 
-  ```bash
-  service_list="llm-textgen codegen codegen-ui"
-  ```
+### If you use vLLM:
 
-- #### Optional. Pull TGI Docker Image (Do this if you plan to build/use the TGI variant)
+```bash
+DATA='{"model": "Qwen/Qwen2.5-Coder-7B-Instruct", '\
+'"messages": [{"role": "user", "content": "Implement a high-level API for a TODO list application. '\
+'The API takes as input an operation request and updates the TODO list in place. '\
+'If the request is invalid, raise an exception."}], "max_tokens": 256}'
 
-  ```bash
-  docker pull ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-  ```
+curl http://${HOST_IP}:${CODEGEN_VLLM_SERVICE_PORT}/v1/chat/completions \
+  -X POST \
+  -d "$DATA" \
+  -H 'Content-Type: application/json'
+```
 
-- #### Build Docker Images
+Checking the response from the service. The response should be similar to JSON:
 
-  _Ensure you are in the `~/codegen-install/GenAIExamples/CodeGen/docker_image_build` directory._
+````json
+{
+  "id": "chatcmpl-142f34ef35b64a8db3deedd170fed951",
+  "object": "chat.completion",
+  "created": 1742270316,
+  "model": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "choices": [
+    {
+      "index": 0,
+      "message": {
+        "role": "assistant",
+        "content": "```python\nfrom typing import Optional, List, Dict, Union\nfrom pydantic import BaseModel, validator\n\nclass OperationRequest(BaseModel):\n    # Assuming OperationRequest is already defined as per the given text\n    pass\n\nclass UpdateOperation(OperationRequest):\n    new_items: List[str]\n\n    def apply_and_maybe_raise(self, updatable_item: \"Updatable todo list\") -> None:\n        # Assuming updatable_item is an instance of Updatable todo list\n        self.validate()\n        updatable_item.add_items(self.new_items)\n\nclass Updatable:\n    # Abstract class for items that can be updated\n    pass\n\nclass TodoList(Updatable):\n    # Class that represents a todo list\n    items: List[str]\n\n    def add_items(self, new_items: List[str]) -> None:\n        self.items.extend(new_items)\n\ndef handle_request(operation_request: OperationRequest) -> None:\n    # Function to handle an operation request\n    if isinstance(operation_request, UpdateOperation):\n        operation_request.apply_and_maybe_raise(get_todo_list_for_update())\n    else:\n        raise ValueError(\"Invalid operation request\")\n\ndef get_todo_list_for_update() -> TodoList:\n    # Function to get the todo list for update\n    # Assuming this function returns the",
+        "tool_calls": []
+      },
+      "logprobs": null,
+      "finish_reason": "length",
+      "stop_reason": null
+    }
+  ],
+  "usage": { "prompt_tokens": 66, "total_tokens": 322, "completion_tokens": 256, "prompt_tokens_details": null },
+  "prompt_logprobs": null
+}
+````
 
-  ```bash
-  docker compose -f build.yaml build ${service_list} --no-cache
-  ```
+If the service response has a meaningful response in the value of the "choices.message.content" key,
+then we consider the vLLM service to be successfully launched
 
-  After the build, check the list of images with the command:
+### If you use TGI:
 
-  ```bash
-  docker image ls
-  ```
+```bash
+DATA='{"inputs":"Implement a high-level API for a TODO list application. '\
+'The API takes as input an operation request and updates the TODO list in place. '\
+'If the request is invalid, raise an exception.",'\
+'"parameters":{"max_new_tokens":256,"do_sample": true}}'
 
-  The list of images should include (depending on `service_list`):
+curl http://${HOST_IP}:${CODEGEN_TGI_SERVICE_PORT}/generate \
+  -X POST \
+  -d "$DATA" \
+  -H 'Content-Type: application/json'
+```
 
-  ###### vLLM-based application:
+Checking the response from the service. The response should be similar to JSON:
 
-  - opea/vllm-rocm:latest
-  - opea/llm-textgen:latest
-  - opea/codegen:latest
-  - opea/codegen-ui:latest
+````json
+{
+  "generated_text": " The supported operations are \"add_task\", \"complete_task\", and \"remove_task\". Each operation can be defined with a corresponding function in the API.\n\nAdd your API in the following format:\n\n```\nTODO App API\n\nsupported operations:\n\noperation name           description\n-----------------------  ------------------------------------------------\n<operation_name>         <operation description>\n```\n\nUse type hints for function parameters and return values. Specify a text description of the API's supported operations.\n\nUse the following code snippet as a starting point for your high-level API function:\n\n```\nclass TodoAPI:\n    def __init__(self, tasks: List[str]):\n        self.tasks = tasks  # List of tasks to manage\n\n    def add_task(self, task: str) -> None:\n        self.tasks.append(task)\n\n    def complete_task(self, task: str) -> None:\n        self.tasks = [t for t in self.tasks if t != task]\n\n    def remove_task(self, task: str) -> None:\n        self.tasks = [t for t in self.tasks if t != task]\n\n    def handle_request(self, request: Dict[str, str]) -> None:\n        operation = request.get('operation')\n        if operation == 'add_task':\n            self.add_task(request.get('task'))\n        elif"
+}
+````
 
-  ###### TGI-based application:
-
-  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm (if pulled)
-  - opea/llm-textgen:latest
-  - opea/codegen:latest
-  - opea/codegen-ui:latest
-
-  _After building, ensure the `image:` tags in the main `compose_vllm.yaml` or `compose.yaml` (in the `amd/gpu/rocm` directory) match these built images (e.g., `opea/vllm-rocm:latest`)._
-
-## Validate Service Health
-
-Run these checks after starting the services to ensure they are operational. Focus on the vLLM checks first as it's the default.
-
-### 1. Validate the vLLM/TGI Service
-
-#### If you use vLLM (Default - using `compose_vllm.yaml` and `set_env_vllm.sh`)
-
-- **How Tested:** Send a POST request with a sample prompt to the vLLM endpoint.
-- **CURL Command:**
-
-  ```bash
-  DATA='{"model": "Qwen/Qwen2.5-Coder-7B-Instruct", '\
-  '"messages": [{"role": "user", "content": "Implement a high-level API for a TODO list application. '\
-  'The API takes as input an operation request and updates the TODO list in place. '\
-  'If the request is invalid, raise an exception."}], "max_tokens": 256}'
-
-  curl http://${HOST_IP}:${CODEGEN_VLLM_SERVICE_PORT}/v1/chat/completions \
-    -X POST \
-    -d "$DATA" \
-    -H 'Content-Type: application/json'
-  ```
-
-- **Sample Output:**
-  ```json
-  {
-    "id": "chatcmpl-142f34ef35b64a8db3deedd170fed951",
-    "object": "chat.completion"
-    // ... (rest of output) ...
-  }
-  ```
-- **Expected Result:** A JSON response with a `choices[0].message.content` field containing meaningful generated code.
-
-#### If you use TGI (using `compose.yaml` and `set_env.sh`)
-
-- **How Tested:** Send a POST request with a sample prompt to the TGI endpoint.
-- **CURL Command:**
-
-  ```bash
-  DATA='{"inputs":"Implement a high-level API for a TODO list application. '\
-  # ... (data payload as before) ...
-  '"parameters":{"max_new_tokens":256,"do_sample": true}}'
-
-  curl http://${HOST_IP}:${CODEGEN_TGI_SERVICE_PORT}/generate \
-    -X POST \
-    -d "$DATA" \
-    -H 'Content-Type: application/json'
-  ```
-
-- **Sample Output:**
-  ```json
-  {
-    "generated_text": " The supported operations are \"add_task\", \"complete_task\", and \"remove_task\". # ... (generated code) ..."
-  }
-  ```
-- **Expected Result:** A JSON response with a `generated_text` field containing meaningful generated code.
+If the service response has a meaningful response in the value of the "generated_text" key,
+then we consider the TGI service to be successfully launched
 
 ### 2. Validate the LLM Service
 
-- **Service Name:** `codegen-llm-server`
-- **How Tested:** Send a POST request to the LLM microservice wrapper endpoint.
-- **CURL Command:**
+```bash
+DATA='{"query":"Implement a high-level API for a TODO list application. '\
+'The API takes as input an operation request and updates the TODO list in place. '\
+'If the request is invalid, raise an exception.",'\
+'"max_tokens":256,"top_k":10,"top_p":0.95,"typical_p":0.95,"temperature":0.01,'\
+'"repetition_penalty":1.03,"stream":false}'
 
-  ```bash
-  DATA='{"query":"Implement a high-level API for a TODO list application. '\
-  # ... (data payload as before) ...
-  '"repetition_penalty":1.03,"stream":false}'
+curl http://${HOST_IP}:${CODEGEN_LLM_SERVICE_PORT}/v1/chat/completions \
+  -X POST \
+  -d "$DATA" \
+  -H 'Content-Type: application/json'
+```
 
-  curl http://${HOST_IP}:${CODEGEN_LLM_SERVICE_PORT}/v1/chat/completions \
-    -X POST \
-    -d "$DATA" \
-    -H 'Content-Type: application/json'
-  ```
+Checking the response from the service. The response should be similar to JSON:
 
-- **Sample Output:** (Structure may vary slightly depending on whether vLLM or TGI is backend)
-  ```json
-  {
-    "id": "cmpl-4e89a590b1af46bfb37ce8f12b2996f8" // Example ID
-    // ... (output structure depends on backend, check original validation) ...
+````json
+{
+  "id": "cmpl-4e89a590b1af46bfb37ce8f12b2996f8",
+  "choices": [
+    {
+      "finish_reason": "length",
+      "index": 0,
+      "logprobs": null,
+      "text": " The API should support the following operations:\n\n1. Add a new task to the TODO list.\n2. Remove a task from the TODO list.\n3. Mark a task as completed.\n4. Retrieve the list of all tasks.\n\nThe API should also support the following features:\n\n1. The ability to filter tasks based on their completion status.\n2. The ability to sort tasks based on their priority.\n3. The ability to search for tasks based on their description.\n\nHere is an example of how the API can be used:\n\n```python\ntodo_list = []\napi = TodoListAPI(todo_list)\n\n# Add tasks\napi.add_task(\"Buy groceries\")\napi.add_task(\"Finish homework\")\n\n# Mark a task as completed\napi.mark_task_completed(\"Buy groceries\")\n\n# Retrieve the list of all tasks\nprint(api.get_all_tasks())\n\n# Filter tasks based on completion status\nprint(api.filter_tasks(completed=True))\n\n# Sort tasks based on priority\napi.sort_tasks(priority=\"high\")\n\n# Search for tasks based on description\nprint(api.search_tasks(description=\"homework\"))\n```\n\nIn this example, the `TodoListAPI` class is used to manage the TODO list. The `add_task` method adds a new task to the list, the `mark_task_completed` method",
+      "stop_reason": null,
+      "prompt_logprobs": null
+    }
+  ],
+  "created": 1742270567,
+  "model": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "object": "text_completion",
+  "system_fingerprint": null,
+  "usage": {
+    "completion_tokens": 256,
+    "prompt_tokens": 37,
+    "total_tokens": 293,
+    "completion_tokens_details": null,
+    "prompt_tokens_details": null
   }
-  ```
-- **Expected Result:** A JSON response containing meaningful generated code within the `choices` array.
+}
+````
 
-### 3. Validate the MegaService (Backend)
+## Accessing the User Interface (UI)
 
-- **Service Name:** `codegen-backend-server`
-- **How Tested:** Send a POST request to the main CodeGen gateway endpoint.
-- **CURL Command:**
+Multiple UI options can be configured via the `compose.yaml`.
 
-  ```bash
-  DATA='{"messages": "Implement a high-level API for a TODO list application. '\
-  # ... (data payload as before) ...
-  'If the request is invalid, raise an exception."}'
+### Svelte UI (Optional)
 
-  curl http://${HOST_IP}:${CODEGEN_BACKEND_SERVICE_PORT}/v1/codegen \
-    -H "Content-Type: application/json" \
-    -d "$DATA"
-  ```
+1.  Modify `compose.yaml`: Comment out the `codegen-gradio-ui-server` service and uncomment/add the `codegen-xeon-ui-server` (Svelte) service definition, ensuring the port mapping is correct (e.g., `"- 5173:5173"`).
+2.  Restart Docker Compose: `docker compose --profile <profile_name> up -d`
+3.  Access: `http://{HOST_IP}:5173` (or the host port you mapped).
 
-- **Sample Output:**
-  ```textmate
-  data: {"id":"cmpl-...", ...}
-  # ... more data chunks ...
-  data: [DONE]
-  ```
-- **Expected Result:** A stream of server-sent events (SSE) containing JSON data with generated code tokens, ending with `data: [DONE]`.
+![Svelte UI Init](../../../../assets/img/codeGen_ui_init.jpg)
 
-### 4. Validate the Frontend (UI)
+### VS Code Extension (Optional)
 
-- **Service Name:** `codegen-ui-server`
-- **How Tested:** Access the UI URL in a web browser and perform a test query.
-- **Steps:** See [How to Open the UI](#how-to-open-the-ui).
-- **Expected Result:** The UI loads correctly, and submitting a prompt results in generated code displayed on the page.
+Users can interact with the backend service using the `Neural Copilot` VS Code extension.
 
-## How to Open the UI
-
-1.  Determine the UI access URL using the `EXTERNAL_HOST_IP` and `CODEGEN_UI_SERVICE_PORT` variables defined in your sourced `set_env*.sh` file (use `set_env_vllm.sh` for the default vLLM deployment). The default URL format is:
-    `http://${EXTERNAL_HOST_IP}:${CODEGEN_UI_SERVICE_PORT}`
-    (e.g., `http://192.168.1.100:5173`)
-
-2.  Open this URL in your web browser.
-
-3.  You should see the CodeGen starting page:
-    ![UI start page](../../../../assets/img/ui-starting-page.png)
-
-4.  Enter a prompt in the input field (e.g., "Write a Python code that returns the current time and date") and press Enter or click the submit button.
-
-5.  Verify that the generated code appears correctly:
-    ![UI result page](../../../../assets/img/ui-result-page.png)
+1.  **Install:** Find and install `Neural Copilot` from the VS Code Marketplace.
+    ![Install Copilot](../../../../assets/img/codegen_copilot.png)
+2.  **Configure:** Set the "Service URL" in the extension settings to your CodeGen backend endpoint: `http://${HOST_IP}:7778/v1/codegen` (use the correct port if changed).
+    ![Configure Endpoint](../../../../assets/img/codegen_endpoint.png)
+3.  **Usage:**
+    - **Inline Suggestion:** Type a comment describing the code you want (e.g., `# Python function to read a file`) and wait for suggestions.
+      ![Code Suggestion](../../../../assets/img/codegen_suggestion.png)
+    - **Chat:** Use the Neural Copilot panel to chat with the AI assistant about code.
+      ![Chat Dialog](../../../../assets/img/codegen_dialog.png)
 
 ## Troubleshooting
 
-_(No specific troubleshooting steps provided in the original content for this file. Add common issues if known.)_
+- **Model Download Issues:** Check `HUGGINGFACEHUB_API_TOKEN`. Ensure internet connectivity or correct proxy settings. Check logs of `tgi-service`/`vllm-service` and `tei-embedding-server`. Gated models need prior Hugging Face access.
+- **Connection Errors:** Verify `HOST_IP` is correct and accessible. Check `docker ps` for port mappings. Ensure `no_proxy` includes `HOST_IP` if using a proxy. Check logs of the service failing to connect (e.g., `codegen-backend-server` logs if it can't reach `codegen-llm-server`).
+- **"Container name is in use"**: Stop existing containers (`docker compose down`) or change `container_name` in `compose.yaml`.
+- **Resource Issues:** CodeGen models can be memory-intensive. Monitor host RAM usage. Increase Docker resources if needed.
 
-- Check container logs (`docker compose -f <file> logs <service_name>`), especially for `codegen-vllm-service` or `codegen-tgi-service`.
-- Ensure `HUGGINGFACEHUB_API_TOKEN` is correct.
-- Verify ROCm drivers and Docker setup for GPU access.
-- Confirm network connectivity and proxy settings.
-- Ensure `HOST_IP` and `EXTERNAL_HOST_IP` are correctly set and accessible.
-- If building locally, ensure build steps completed without error and image tags match compose file.
+### Cleanup the Deployment
 
-## Stopping the Application
-
-### If you use vLLM (Default)
+To stop the containers associated with the deployment, execute the following command:
 
 ```bash
-# Ensure you are in the correct directory
-# cd GenAIExamples/CodeGen/docker_compose/amd/gpu/rocm
-docker compose -f compose_vllm.yaml down
-```
-
-### If you use TGI
-
-```bash
-# Ensure you are in the correct directory
-# cd GenAIExamples/CodeGen/docker_compose/amd/gpu/rocm
 docker compose -f compose.yaml down
 ```
 
+```bash
+[+] Running 0/1
+[+] Running 1/2degen-ui-server  Stopping                                                                                               0.4s
+[+] Running 2/3degen-ui-server       Removed                                                                                          10.5s
+[+] Running 2/3degen-ui-server       Removed                                                                                          10.5s
+[+] Running 3/4degen-ui-server       Removed                                                                                          10.5s
+[+] Running 5/5degen-ui-server       Removed                                                                                          10.5s
+ ✔ Container codegen-ui-server       Removed                                                                                         10.5s
+ ✔ Container codegen-backend-server  Removed                                                                                         10.4s
+ ✔ Container codegen-llm-server      Removed                                                                                         10.4s
+ ✔ Container codegen-tgi-service     Removed                                                                                          8.0s
+ ✔ Network rocm_default              Removed                                                                                          0.6s
+```
+
+### compose.yaml - TGI Deployment
+
+The TGI (Text Generation Inference) deployment and the default deployment differ primarily in their service configurations and specific focus on handling large language models (LLMs). The TGI deployment includes a unique `codegen-tgi-service`, which utilizes the `ghcr.io/huggingface/text-generation-inference:2.4.1-rocm` image and is specifically configured to run on AMD hardware.
+
+| Service Name           | Image Name                                               | AMD Use |
+| ---------------------- | -------------------------------------------------------- | ------- |
+| codegen-backend-server | opea/codegen:latest                                      | no      |
+| codegen-llm-server     | opea/codegen:latest                                      | no      |
+| codegen-tgi-service    | ghcr.io/huggingface/text-generation-inference:2.4.1-rocm | yes     |
+| codegen-ui-server      | opea/codegen-ui:latest                                   | no      |
+
+### compose_vllm.yaml - vLLM Deployment
+
+The vLLM deployment utilizes AMD devices primarily for the `vllm-service`, which handles large language model (LLM) tasks. This service is configured to maximize the use of AMD's capabilities, potentially allocating multiple devices to enhance parallel processing and throughput.
+
+| Service Name           | Image Name             | AMD Use |
+| ---------------------- | ---------------------- | ------- |
+| codegen-backend-server | opea/codegen:latest    | no      |
+| codegen-llm-server     | opea/codegen:latest    | no      |
+| codegen-vllm-service   | opea/vllm-rocm:latest  | yes     |
+| codegen-ui-server      | opea/codegen-ui:latest | no      |
+
+## CodeGen Service Configuration
+
+The table provides a comprehensive overview of the CodeGen services utilized across various deployments as illustrated in the example Docker Compose files. Each row in the table represents a distinct service, detailing its possible images used to enable it and a concise description of its function within the deployment architecture. These services collectively enable functionalities such as data storage and management, text embedding, retrieval, reranking, and large language model processing.
+
+ex.: (From ChatQna)
+| Service Name | Possible Image Names | Optional | Description
+| redis-vector-db | redis/redis-stack:7.2.0-v9 | No | Acts as a Redis database for storing and managing
+
+## Conclusion
+
+In the configuration of the `vllm-service` and the `tgi-service`, two variables play a primary role in determining the service's performance and functionality. The `LLM_MODEL_ID` parameter specifies the particular large language model (LLM) that the service will utilize, effectively determining the capabilities and characteristics of the language processing tasks it can perform. This model identifier ensures that the service is aligned with the specific requirements of the application, whether it involves text generation, comprehension, or other language-related tasks.
+
+However, developers need to be aware of the models that have been tested with the respective service image supporting the `vllm-service` and `tgi-service`. For example, documentation for the OPEA GenAIComps v1.0 release specify the list of [validated LLM models](https://github.com/opea-project/GenAIComps/blob/v1.0/comps/llms/text-generation/README.md#validated-llm-models) for each AMD ROCm enabled service image. Specific models may have stringent requirements on the number of AMD ROCm devices required to support them.
+
+This guide should enable developer to deploy the default configuration or any of the other compose yaml files for different configurations. It also highlights the configurable parameters that can be set before deployment.
+
 ## Next Steps
 
-- Explore the alternative TGI deployment option if needed.
-- Refer to the main [CodeGen README](../../../../README.md) for architecture details and links to other deployment methods (Kubernetes, Xeon).
 - Consult the [OPEA GenAIComps](https://github.com/opea-project/GenAIComps) repository for details on individual microservices.
+- Refer to the main [CodeGen README](../../../../README.md) for links to benchmarking and Kubernetes deployment options.

CodeGen/docker_image_build/build.yaml

@@ -5,6 +5,8 @@ services:
   codegen:
     build:
       args:
+        IMAGE_REPO: ${REGISTRY}
+        BASE_TAG: ${TAG}
         http_proxy: ${http_proxy}
         https_proxy: ${https_proxy}
         no_proxy: ${no_proxy}
@@ -39,6 +41,7 @@ services:
     build:
       context: GenAIComps
       dockerfile: comps/third_parties/vllm/src/Dockerfile.amd_gpu
+    extends: codegen
     image: ${REGISTRY:-opea}/vllm-rocm:${TAG:-latest}
   vllm:
     build:

CodeGen/tests/test_compose_on_gaudi.sh

@@ -27,25 +27,19 @@ export no_proxy=${no_proxy},${ip_address}
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     # Download Gaudi vllm of latest tag
     git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
-    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
-    echo "Check out vLLM tag ${VLLM_VER}"
-    git checkout ${VLLM_VER} &> /dev/null && cd ../
+    VLLM_FORK_VER=v0.7.2+Gaudi-1.21.0
+    echo "Check out vLLM tag ${VLLM_FORK_VER}"
+    git checkout ${VLLM_FORK_VER} &> /dev/null && cd ../
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="codegen codegen-gradio-ui llm-textgen vllm-gaudi dataprep retriever embedding"
@@ -250,24 +244,36 @@ function main() {
         stop_docker "${docker_compose_profiles[${i}]}"
     done
 
-    # build docker images
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
 
     # loop all profiles
     for ((i = 0; i < len_profiles; i++)); do
         echo "Process [${i}]: ${docker_compose_profiles[$i]}, ${docker_llm_container_names[${i}]}"
+
+        echo "::group::start_services"
         start_services "${docker_compose_profiles[${i}]}" "${docker_llm_container_names[${i}]}"
+        echo "::endgroup::"
         docker ps -a
 
+        echo "::group::validate_microservices"
         validate_microservices "${docker_llm_container_names[${i}]}"
+        echo "::endgroup::"
+
+        echo "::group::validate_megaservice"
         validate_megaservice
+        echo "::endgroup::"
+
+        echo "::group::validate_gradio"
         validate_gradio
+        echo "::endgroup::"
 
         stop_docker "${docker_compose_profiles[${i}]}"
         sleep 5s
     done
 
-    echo y | docker system prune
+    docker system prune -f
 }
 
 main

CodeGen/tests/test_compose_on_rocm.sh

@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="codegen codegen-ui llm-textgen"
@@ -164,18 +158,35 @@ function stop_docker() {
 
 function main() {
 
+    echo "::group::stop_docker"
     stop_docker
+    echo "::endgroup::"
 
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
+
+    echo "::group::start_services"
     start_services
+    echo "::endgroup::"
 
+    echo "::group::validate_microservices"
     validate_microservices
-    validate_megaservice
-    validate_frontend
+    echo "::endgroup::"
 
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::validate_frontend"
+    validate_frontend
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
     stop_docker
-    echo y | docker system prune
-    cd $WORKPATH
+    echo "::endgroup::"
+
+    docker system prune -f
 
 }
 

CodeGen/tests/test_compose_on_xeon.sh

@@ -27,19 +27,13 @@ export no_proxy=${no_proxy},${ip_address}
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     git clone https://github.com/vllm-project/vllm.git && cd vllm
     VLLM_VER="v0.8.3"
@@ -256,17 +250,28 @@ function main() {
     for ((i = 0; i < len_profiles; i++)); do
         echo "Process [${i}]: ${docker_compose_profiles[$i]}, ${docker_llm_container_names[${i}]}"
         docker ps -a
-        start_services "${docker_compose_profiles[${i}]}" "${docker_llm_container_names[${i}]}"
 
+        echo "::group::start_services"
+        start_services "${docker_compose_profiles[${i}]}" "${docker_llm_container_names[${i}]}"
+        echo "::endgroup::"
+
+        echo "::group::validate_microservices"
         validate_microservices "${docker_llm_container_names[${i}]}"
+        echo "::endgroup::"
+
+        echo "::group::validate_megaservice"
         validate_megaservice
+        echo "::endgroup::"
+
+        echo "::group::validate_gradio"
         validate_gradio
+        echo "::endgroup::"
 
         stop_docker "${docker_compose_profiles[${i}]}"
         sleep 5s
     done
 
-    echo y | docker system prune
+    docker system prune -f
 }
 
 main

CodeGen/tests/test_compose_vllm_on_rocm.sh

@@ -17,19 +17,13 @@ ip_address=$(hostname -I | awk '{print $1}')
 
 function build_docker_images() {
     opea_branch=${opea_branch:-"main"}
-    # If the opea_branch isn't main, replace the git clone branch in Dockerfile.
-    if [[ "${opea_branch}" != "main" ]]; then
-        cd $WORKPATH
-        OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
-        NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
-        find . -type f -name "Dockerfile*" | while read -r file; do
-            echo "Processing file: $file"
-            sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
-        done
-    fi
 
     cd $WORKPATH/docker_image_build
     git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
+    pushd GenAIComps
+    echo "GenAIComps test commit is $(git rev-parse HEAD)"
+    docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    popd && sleep 1s
 
     echo "Build all the images with --no-cache, check docker_image_build.log for details..."
     service_list="vllm-rocm llm-textgen codegen codegen-ui"
@@ -164,17 +158,35 @@ function stop_docker() {
 
 function main() {
 
+    echo "::group::stop_docker"
     stop_docker
+    echo "::endgroup::"
+
+    echo "::group::build_docker_images"
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
+    echo "::endgroup::"
+
+    echo "::group::start_services"
     start_services
+    echo "::endgroup::"
 
+    echo "::group::validate_microservices"
     validate_microservices
-    validate_megaservice
-    validate_frontend
+    echo "::endgroup::"
 
+    echo "::group::validate_megaservice"
+    validate_megaservice
+    echo "::endgroup::"
+
+    echo "::group::validate_frontend"
+    validate_frontend
+    echo "::endgroup::"
+
+    echo "::group::stop_docker"
     stop_docker
-    echo y | docker system prune
-    cd $WORKPATH
+    echo "::endgroup::"
+
+    docker system prune -f
 
 }
 

CodeGen/ui/gradio/codegen_ui_gradio.py

@@ -46,13 +46,24 @@ dataprep_get_indices_endpoint = f"{DATAPREP_ENDPOINT}/indices"
 
 
 # Define the functions that will be used in the app
+
+
+def add_to_history(prompt, history):
+    history.append([prompt["text"], ""])
+    return history, ""
+
+
 def conversation_history(prompt, index, use_agent, history):
     print(f"Generating code for prompt: {prompt} using index: {index} and use_agent is {use_agent}")
-    history.append([prompt, ""])
-    response_generator = generate_code(prompt, index, use_agent)
+    history = add_to_history(prompt, history)[0]
+    response_generator = generate_code(prompt["text"], index, use_agent)
     for token in response_generator:
         history[-1][-1] += token
-        yield history
+        yield history, ""
+
+
+def clear_history():
+    return ""
 
 
 def upload_media(media, index=None, chunk_size=1500, chunk_overlap=100):
@@ -287,19 +298,32 @@ def get_file_names(files):
 # Define UI components
 with gr.Blocks() as ui:
     with gr.Tab("Code Generation"):
-        gr.Markdown("### Generate Code from Natural Language")
-        chatbot = gr.Chatbot(label="Chat History")
-        prompt_input = gr.Textbox(label="Enter your query")
-        with gr.Column():
-            with gr.Row(equal_height=True):
+        with gr.Row():
+            with gr.Column(scale=2):
                 database_dropdown = gr.Dropdown(choices=get_indices(), label="Select Index", value="None", scale=10)
                 db_refresh_button = gr.Button("Refresh Dropdown", scale=0.1)
                 db_refresh_button.click(update_indices_dropdown, outputs=database_dropdown)
                 use_agent = gr.Checkbox(label="Use Agent", container=False)
 
-        generate_button = gr.Button("Generate Code")
-        generate_button.click(
-            conversation_history, inputs=[prompt_input, database_dropdown, use_agent, chatbot], outputs=chatbot
+            with gr.Column(scale=9):
+                gr.Markdown("### Generate Code from Natural Language")
+                chatbot = gr.Chatbot(label="Chat History")
+                with gr.Row(equal_height=True):
+                    with gr.Column(scale=8):
+                        prompt_input = gr.MultimodalTextbox(
+                            show_label=False, interactive=True, placeholder="Enter your query", sources=[]
+                        )
+                    with gr.Column(scale=1, min_width=150):
+                        with gr.Row(elem_id="buttons") as button_row:
+                            clear_btn = gr.Button(value="🗑️  Clear", interactive=True)
+                            clear_btn.click(clear_history, None, chatbot)
+
+        prompt_input.submit(add_to_history, inputs=[prompt_input, chatbot], outputs=[chatbot, prompt_input])
+
+        prompt_input.submit(
+            conversation_history,
+            inputs=[prompt_input, database_dropdown, use_agent, chatbot],
+            outputs=[chatbot, prompt_input],
         )
 
     with gr.Tab("Resource Management"):
@@ -315,7 +339,7 @@ with gr.Blocks() as ui:
                 )
             with gr.Column(scale=3):
                 file_upload = gr.File(label="Upload Files", file_count="multiple")
-                url_input = gr.Textbox(label="Media to be ingested (Append URL's in a new line)")
+                url_input = gr.Textbox(label="Media to be ingested. Append URL's in a new line (Shift + Enter)")
                 upload_button = gr.Button("Upload", variant="primary")
                 upload_status = gr.Textbox(label="Upload Status")
                 file_upload.change(get_file_names, inputs=file_upload, outputs=url_input)

CodeTrans/Dockerfile

@@ -1,8 +1,9 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+ARG IMAGE_REPO=opea
 ARG BASE_TAG=latest
-FROM opea/comps-base:$BASE_TAG
+FROM $IMAGE_REPO/comps-base:$BASE_TAG
 
 COPY ./code_translation.py $HOME/code_translation.py
 

CodeTrans/docker_compose/amd/gpu/rocm/README.md

@@ -1,8 +1,10 @@
-# Deploy CodeTrans on AMD GPU (ROCm)
+# Deploying CodeTrans on AMD ROCm GPU
 
-This document outlines the single node deployment process for a CodeTrans application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservices on AMD GPU (ROCm) server. The steps include pulling Docker images, container deployment via Docker Compose, and service execution using microservices `llm`.
+This document outlines the single node deployment process for a CodeTrans application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservices on Intel Xeon server and AMD GPU. The steps include pulling Docker images, container deployment via Docker Compose, and service execution using microservices `llm`.
 
-# Table of Contents
+Note: The default LLM is `Qwen/Qwen2.5-Coder-7B-Instruct`. Before deploying the application, please make sure either you've requested and been granted the access to it on [Huggingface](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) or you've downloaded the model locally from [ModelScope](https://www.modelscope.cn/models).
+
+## Table of Contents
 
 1. [CodeTrans Quick Start Deployment](#codetrans-quick-start-deployment)
 2. [CodeTrans Docker Compose Files](#codetrans-docker-compose-files)
@@ -11,7 +13,7 @@ This document outlines the single node deployment process for a CodeTrans applic
 
 ## CodeTrans Quick Start Deployment
 
-This section describes how to quickly deploy and test the CodeTrans service manually on an AMD GPU (ROCm) processor. The basic steps are:
+This section describes how to quickly deploy and test the CodeTrans service manually on an AMD ROCm GPU. The basic steps are:
 
 1. [Access the Code](#access-the-code)
 2. [Configure the Deployment Environment](#configure-the-deployment-environment)
@@ -22,7 +24,7 @@ This section describes how to quickly deploy and test the CodeTrans service manu
 
 ### Access the Code
 
-Clone the GenAIExample repository and access the CodeTrans AMD GPU (ROCm) platform Docker Compose files and supporting scripts:
+Clone the GenAIExample repository and access the CodeTrans AMD ROCm GPU platform Docker Compose files and supporting scripts:
 
 ```bash
 git clone https://github.com/opea-project/GenAIExamples.git
@@ -37,29 +39,84 @@ git checkout v1.2
 
 ### Configure the Deployment Environment
 
-To set up environment variables for deploying CodeTrans services, set up some parameters specific to the deployment environment and source the `set_env.sh` script in this directory:
+To set up environment variables for deploying CodeTrans services, set up some parameters specific to the deployment environment and source the `set_env_*.sh` script in this directory:
+
+- if used vLLM - set_env_vllm.sh
+- if used TGI - set_env.sh
+
+Set the values of the variables:
+
+- **HOST_IP, HOST_IP_EXTERNAL** - These variables are used to configure the name/address of the service in the operating system environment for the application services to interact with each other and with the outside world.
+
+  If your server uses only an internal address and is not accessible from the Internet, then the values for these two variables will be the same and the value will be equal to the server's internal name/address.
+
+  If your server uses only an external, Internet-accessible address, then the values for these two variables will be the same and the value will be equal to the server's external name/address.
+
+  If your server is located on an internal network, has an internal address, but is accessible from the Internet via a proxy/firewall/load balancer, then the HOST_IP variable will have a value equal to the internal name/address of the server, and the EXTERNAL_HOST_IP variable will have a value equal to the external name/address of the proxy/firewall/load balancer behind which the server is located.
+
+  We set these values in the file set_env\*\*\*\*.sh
+
+- **Variables with names like "**\*\*\*\*\*\*\_PORT"\*\* - These variables set the IP port numbers for establishing network connections to the application services.
+  The values shown in the file set_env.sh or set_env_vllm they are the values used for the development and testing of the application, as well as configured for the environment in which the development is performed. These values must be configured in accordance with the rules of network access to your environment's server, and must not overlap with the IP ports of other applications that are already in use.
+
+Setting variables in the operating system environment:
 
 ```bash
-export host_ip="External_Public_IP"           # ip address of the node
 export HUGGINGFACEHUB_API_TOKEN="Your_HuggingFace_API_Token"
-export http_proxy="Your_HTTP_Proxy"           # http proxy if any
-export https_proxy="Your_HTTPs_Proxy"         # https proxy if any
-export no_proxy=localhost,127.0.0.1,$host_ip  # additional no proxies if needed
-export NGINX_PORT=${your_nginx_port}          # your usable port for nginx, 80 for example
-source ./set_env.sh
+source ./set_env_*.sh # replace the script name with the appropriate one
 ```
 
 Consult the section on [CodeTrans Service configuration](#codetrans-configuration) for information on how service specific configuration parameters affect deployments.
 
 ### Deploy the Services Using Docker Compose
 
-To deploy the CodeTrans services, execute the `docker compose up` command with the appropriate arguments. For a default deployment, execute the command below. It uses the 'compose.yaml' file.
+To deploy the CodeTrans services, execute the `docker compose up` command with the appropriate arguments. For a default deployment with TGI, execute the command below. It uses the 'compose.yaml' file.
 
 ```bash
 cd docker_compose/amd/gpu/rocm
+# if used TGI
 docker compose -f compose.yaml up -d
+# if used vLLM
+# docker compose -f compose_vllm.yaml up -d
 ```
 
+To enable GPU support for AMD GPUs, the following configuration is added to the Docker Compose file:
+
+- compose_vllm.yaml - for vLLM-based application
+- compose.yaml - for TGI-based
+
+```yaml
+shm_size: 1g
+devices:
+  - /dev/kfd:/dev/kfd
+  - /dev/dri:/dev/dri
+cap_add:
+  - SYS_PTRACE
+group_add:
+  - video
+security_opt:
+  - seccomp:unconfined
+```
+
+This configuration forwards all available GPUs to the container. To use a specific GPU, specify its `cardN` and `renderN` device IDs. For example:
+
+```yaml
+shm_size: 1g
+devices:
+  - /dev/kfd:/dev/kfd
+  - /dev/dri/card0:/dev/dri/card0
+  - /dev/dri/render128:/dev/dri/render128
+cap_add:
+  - SYS_PTRACE
+group_add:
+  - video
+security_opt:
+  - seccomp:unconfined
+```
+
+**How to Identify GPU Device IDs:**
+Use AMD GPU driver utilities to determine the correct `cardN` and `renderN` IDs for your GPU.
+
 > **Note**: developers should build docker image from source when:
 >
 > - Developing off the git main branch (as the container's ports in the repo may be different > from the published docker image).
@@ -71,9 +128,11 @@ Please refer to the table below to build different microservices from source:
 | Microservice | Deployment Guide                                                                                               |
 | ------------ | -------------------------------------------------------------------------------------------------------------- |
 | vLLM         | [vLLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/vllm#build-docker) |
+| TGI          | [TGI project](https://github.com/huggingface/text-generation-inference.git)                                    |
 | LLM          | [LLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/llms)                             |
-| MegaService  | [MegaService build guide](../../../../README_miscellaneous.md#build-megaservice-docker-image)                  |
-| UI           | [Basic UI build guide](../../../../README_miscellaneous.md#build-ui-docker-image)                              |
+| MegaService  | [MegaService guide](../../../../README.md)                                                                     |
+| UI           | [UI guide](../../../../ui/svelte/README.md)                                                                    |
+| Nginx        | [Nginx guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/nginx)                  |
 
 ### Check the Deployment Status
 
@@ -83,15 +142,26 @@ After running docker compose, check if all the containers launched via docker co
 docker ps -a
 ```
 
-For the default deployment, the following 5 containers should have started:
+For the default deployment with TGI, the following 9 containers should have started:
 
 ```
-CONTAINER ID   IMAGE                                                   COMMAND                  CREATED        STATUS        PORTS                                                                                  NAMES
-b3e1388fa2ca   opea/nginx:${RELEASE_VERSION}                           "/usr/local/bin/star…"   32 hours ago   Up 2 hours   0.0.0.0:80->80/tcp, :::80->80/tcp                                                      codetrans-nginx-server
-3b5fa9a722da   opea/codetrans-ui:${RELEASE_VERSION}                    "docker-entrypoint.s…"   32 hours ago   Up 2 hours   0.0.0.0:5173->5173/tcp, :::5173->5173/tcp                                              codetrans-ui-server
-d3b37f3d1faa   opea/codetrans:${RELEASE_VERSION}                       "python codetrans.py"    32 hours ago   Up 2 hours   0.0.0.0:7777->7777/tcp, :::7777->7777/tcp                                              codetrans-backend-server
-24cae0db1a70   opea/llm-textgen:${RELEASE_VERSION}                     "bash entrypoint.sh"     32 hours ago   Up 2 hours   0.0.0.0:9000->9000/tcp, :::9000->9000/tcp                                              codetrans-llm-server
-b98fa07a4f5c   opea/vllm:${RELEASE_VERSION}                            "python3 -m vllm.ent…"   32 hours ago   Up 2 hours   0.0.0.0:9009->80/tcp, :::9009->80/tcp                                                  codetrans-tgi-service
+CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
+eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chaqna-nginx-server
+2fce48a4c0f4   opea/codetrans-ui:latest                                  "docker-entrypoint.s…"   37 seconds ago   Up 5 seconds              0.0.0.0:18101->5173/tcp, [::]:18101->5173/tcp                                              codetrans-ui-server
+613c384979f4   opea/codetrans:latest                                     "bash entrypoint.sh"     37 seconds ago   Up 5 seconds              0.0.0.0:18102->8888/tcp, [::]:18102->8888/tcp                                              codetrans-backend-server
+e0ef1ea67640   opea/llm-textgen:latest                                  "bash entrypoint.sh"     37 seconds ago   Up 36 seconds             0.0.0.0:18011->9000/tcp, [::]:18011->9000/tcp                                              codetrans-llm-server
+342f01bfdbb2   ghcr.io/huggingface/text-generation-inference:2.3.1-rocm"python3 /workspace/…"   37 seconds ago   Up 36 seconds             0.0.0.0:18008->8011/tcp, [::]:18008->8011/tcp                                              codetrans-tgi-service
+```
+
+if used vLLM:
+
+```
+CONTAINER ID   IMAGE                                                   COMMAND                  CREATED          STATUS                    PORTS                                                                                      NAMES
+eaf24161aca8   opea/nginx:latest                                       "/docker-entrypoint.…"   37 seconds ago   Up 5 seconds              0.0.0.0:18104->80/tcp, [::]:18104->80/tcp                                                  chaqna-nginx-server
+2fce48a4c0f4   opea/codetrans-ui:latest                                  "docker-entrypoint.s…"   37 seconds ago   Up 5 seconds              0.0.0.0:18101->5173/tcp, [::]:18101->5173/tcp                                              codetrans-ui-server
+613c384979f4   opea/codetrans:latest                                     "bash entrypoint.sh"     37 seconds ago   Up 5 seconds              0.0.0.0:18102->8888/tcp, [::]:18102->8888/tcp                                              codetrans-backend-server
+e0ef1ea67640   opea/llm-textgen:latest                                  "bash entrypoint.sh"     37 seconds ago   Up 36 seconds             0.0.0.0:18011->9000/tcp, [::]:18011->9000/tcp                                              codetrans-llm-server
+342f01bfdbb2   opea/vllm-rocm:latest                                   "python3 /workspace/…"   37 seconds ago   Up 36 seconds             0.0.0.0:18008->8011/tcp, [::]:18008->8011/tcp                                              codetrans-vllm-service
 ```
 
 If any issues are encountered during deployment, refer to the [Troubleshooting](../../../../README_miscellaneous.md#troubleshooting) section.
@@ -109,65 +179,68 @@ curl http://${HOST_IP}:${CODETRANS_BACKEND_SERVICE_PORT}/v1/codetrans \
   -d "$DATA"
 ```
 
-**Note** : Access the CodeTrans UI by web browser through this URL: `http://${host_ip}:80`. Please confirm the `80` port is opened in the firewall. To validate each microservie used in the pipeline refer to the [Validate Microservices](#validate-microservices) section.
+**Note** : Access the CodeTrans UI by web browser through this URL: `http://${HOST_IP_EXTERNAL}:${CODETRANS_NGINX_PORT}`
 
 ### Cleanup the Deployment
 
 To stop the containers associated with the deployment, execute the following command:
 
 ```bash
+# if used TGI
 docker compose -f compose.yaml down
+# if used vLLM
+# docker compose -f compose_vllm.yaml down
 ```
 
 ## CodeTrans Docker Compose Files
 
-In the context of deploying a CodeTrans pipeline on an AMD GPU (ROCm) platform, we can pick and choose different large language model serving frameworks. The table below outlines the various configurations that are available as part of the application. These configurations can be used as templates and can be extended to different components available in [GenAIComps](https://github.com/opea-project/GenAIComps.git).
+In the context of deploying an ChatQnA pipeline on an Intel® Xeon® platform, we can pick and choose different large language model serving frameworks, or single English TTS/multi-language TTS component. The table below outlines the various configurations that are available as part of the application. These configurations can be used as templates and can be extended to different components available in [GenAIComps](https://github.com/opea-project/GenAIComps.git).
 
-| File                                     | Description                                                                                |
-| ---------------------------------------- | ------------------------------------------------------------------------------------------ |
-| [compose.yaml](./compose.yaml)           | Default compose file using TGI as serving framework                                        |
-| [compose_vllm.yaml](./compose_vllm.yaml) | The LLM serving framework is vLLM. All other configurations remain the same as the default |
+| File                                     | Description                                                                           |
+| ---------------------------------------- | ------------------------------------------------------------------------------------- |
+| [compose.yaml](./compose.yaml)           | The LLM serving framework is TGI. Default compose file using TGI as serving framework |
+| [compose_vllm.yaml](./compose_vllm.yaml) | The LLM serving framework is vLLM. Compose file using vllm as serving framework       |
 
-## Validate Microservices
+## Validate MicroServices
 
-1. LLM backend Service
+LLM backend Service
 
-   In the first startup, this service will take more time to download, load and warm up the model. After it's finished, the service will be ready.
+In the first startup, this service will take more time to download, load and warm up the model. After it's finished, the service will be ready.
 
-   Try the command below to check whether the LLM serving is ready.
+Try the command below to check whether the LLM serving is ready.
 
-   ```bash
-   # vLLM service
-   docker logs codetrans-vllm-service 2>&1 | grep complete
-   # If the service is ready, you will get the response like below.
-   INFO:     Application startup complete.
-   ```
+```bash
+# vLLM service
+docker logs codetrans-vllm-service 2>&1 | grep complete
+# If the service is ready, you will get the response like below.
+INFO:     Application startup complete.
+```
 
-   ```bash
-   # TGI service
-   docker logs codetrans-tgi-service | grep Connected
-   # If the service is ready, you will get the response like below.
-   2024-09-03T02:47:53.402023Z  INFO text_generation_router::server: router/src/server.rs:2311: Connected
-   ```
+```bash
+# TGI service
+docker logs codetrans-tgi-service | grep Connected
+# If the service is ready, you will get the response like below.
+2024-09-03T02:47:53.402023Z  INFO text_generation_router::server: router/src/server.rs:2311: Connected
+```
 
-   Then try the `cURL` command below to validate services.
+Then try the `cURL` command below to validate services.
 
-   ```bash
-   # either vLLM or TGI service
-   # for vllm service
-   export port=${CODETRANS_VLLM_SERVICE_PORT}
-   # for tgi service
-   export port=${CODETRANS_TGI_SERVICE_PORT}
-   curl http://${HOST_IP}:${port}/v1/chat/completions \
-     -X POST \
-     -d '{"inputs":"    ### System: Please translate the following Golang codes into  Python codes.    ### Original codes:    '\'''\'''\''Golang    \npackage main\n\nimport \"fmt\"\nfunc main() {\n    fmt.Println(\"Hello, World!\");\n    '\'''\'''\''    ### Translated codes:","parameters":{"max_new_tokens":17, "do_sample": true}}' \
-     -H 'Content-Type: application/json'
-   ```
+```bash
+# either vLLM or TGI service
+# for vllm service
+export port=${CODETRANS_VLLM_SERVICE_PORT}
+# for tgi service
+export port=${CODETRANS_TGI_SERVICE_PORT}
+curl http://${HOST_IP}:${port}/v1/chat/completions \
+  -X POST \
+  -d '{"inputs":"    ### System: Please translate the following Golang codes into  Python codes.    ### Original codes:    '\'''\'''\''Golang    \npackage main\n\nimport \"fmt\"\nfunc main() {\n    fmt.Println(\"Hello, World!\");\n    '\'''\'''\''    ### Translated codes:","parameters":{"max_new_tokens":17, "do_sample": true}}' \
+  -H 'Content-Type: application/json'
+```
 
 2. LLM Microservice
 
    ```bash
-   curl http://${HOST_IP}:${CODETRANS_LLM_SERVICE_PORT}/v1/chat/completions\
+   curl http://${HOST_IP}:${CODETRANS_LLM_SERVICE_PORT}/v1/chat/completions \
      -X POST \
      -d '{"query":"    ### System: Please translate the following Golang codes into  Python codes.    ### Original codes:    '\'''\'''\''Golang    \npackage main\n\nimport \"fmt\"\nfunc main() {\n    fmt.Println(\"Hello, World!\");\n    '\'''\'''\''    ### Translated codes:"}' \
      -H 'Content-Type: application/json'

CodeTrans/docker_compose/amd/gpu/rocm/set_env.sh

@@ -8,14 +8,14 @@
 # which can be used to connect to the server from the Internet. It must be specified in the EXTERNAL_HOST_IP variable.
 # If the server is used only on the internal network or has a direct external address,
 # specify it in HOST_IP and in EXTERNAL_HOST_IP.
-export HOST_IP=''
-export EXTERNAL_HOST_IP=''
+export HOST_IP=${ip_address}
+export EXTERNAL_HOST_IP=${ip_address}
 
 ### Model ID
 export CODETRANS_LLM_MODEL_ID="Qwen/Qwen2.5-Coder-7B-Instruct"
 
 ### The port of the TGI service. On this port, the TGI service will accept connections
-export CODETRANS_TGI_SERVICE_PORT=18156
+export CODETRANS_TGI_SERVICE_PORT=8008
 
 ### The endpoint of the TGI service to which requests to this service will be sent (formed from previously set variables)
 export CODETRANS_TGI_LLM_ENDPOINT="http://${HOST_IP}:${CODETRANS_TGI_SERVICE_PORT}"
@@ -24,7 +24,7 @@ export CODETRANS_TGI_LLM_ENDPOINT="http://${HOST_IP}:${CODETRANS_TGI_SERVICE_POR
 export CODETRANS_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 
 ### The port of the LLM service. On this port, the LLM service will accept connections
-export CODETRANS_LLM_SERVICE_PORT=18157
+export CODETRANS_LLM_SERVICE_PORT=9000
 
 ### The IP address or domain name of the server for CodeTrans MegaService
 export CODETRANS_MEGA_SERVICE_HOST_IP=${HOST_IP}
@@ -36,7 +36,7 @@ export CODETRANS_LLM_SERVICE_HOST_IP=${HOST_IP}
 export CODETRANS_FRONTEND_SERVICE_IP=${HOST_IP}
 
 ### The port of the frontend service
-export CODETRANS_FRONTEND_SERVICE_PORT=18155
+export CODETRANS_FRONTEND_SERVICE_PORT=5173
 
 ### Name of GenAI service for route requests to application
 export CODETRANS_BACKEND_SERVICE_NAME=codetrans
@@ -45,10 +45,10 @@ export CODETRANS_BACKEND_SERVICE_NAME=codetrans
 export CODETRANS_BACKEND_SERVICE_IP=${HOST_IP}
 
 ### The port of the backend service
-export CODETRANS_BACKEND_SERVICE_PORT=18154
+export CODETRANS_BACKEND_SERVICE_PORT=7777
 
 ### The port of the Nginx reverse proxy for application
-export CODETRANS_NGINX_PORT=18153
+export CODETRANS_NGINX_PORT=8088
 
 ### Endpoint of the backend service
 export CODETRANS_BACKEND_SERVICE_URL="http://${EXTERNAL_HOST_IP}:${CODETRANS_BACKEND_SERVICE_PORT}/v1/codetrans"

CodeTrans/docker_compose/amd/gpu/rocm/set_env_vllm.sh

@@ -8,14 +8,14 @@
 # which can be used to connect to the server from the Internet. It must be specified in the EXTERNAL_HOST_IP variable.
 # If the server is used only on the internal network or has a direct external address,
 # specify it in HOST_IP and in EXTERNAL_HOST_IP.
-export HOST_IP=''
-export EXTERNAL_HOST_IP=''
+export HOST_IP=${ip_address}
+export EXTERNAL_HOST_IP=${ip_address}
 
 ### Model ID
 export CODETRANS_LLM_MODEL_ID="Qwen/Qwen2.5-Coder-7B-Instruct"
 
 ### The port of the TGI service. On this port, the TGI service will accept connections
-export CODETRANS_VLLM_SERVICE_PORT=18156
+export CODETRANS_VLLM_SERVICE_PORT=8008
 
 ### The endpoint of the TGI service to which requests to this service will be sent (formed from previously set variables)
 export CODETRANS_LLM_ENDPOINT="http://${HOST_IP}:${CODETRANS_VLLM_SERVICE_PORT}"
@@ -24,7 +24,7 @@ export CODETRANS_LLM_ENDPOINT="http://${HOST_IP}:${CODETRANS_VLLM_SERVICE_PORT}"
 export CODETRANS_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 
 ### The port of the LLM service. On this port, the LLM service will accept connections
-export CODETRANS_LLM_SERVICE_PORT=18157
+export CODETRANS_LLM_SERVICE_PORT=9000
 
 ### The IP address or domain name of the server for CodeTrans MegaService
 export CODETRANS_MEGA_SERVICE_HOST_IP=${HOST_IP}
@@ -36,7 +36,7 @@ export CODETRANS_LLM_SERVICE_HOST_IP=${HOST_IP}
 export CODETRANS_FRONTEND_SERVICE_IP=${HOST_IP}
 
 ### The port of the frontend service
-export CODETRANS_FRONTEND_SERVICE_PORT=18155
+export CODETRANS_FRONTEND_SERVICE_PORT=5173
 
 ### Name of GenAI service for route requests to application
 export CODETRANS_BACKEND_SERVICE_NAME=codetrans
@@ -45,10 +45,10 @@ export CODETRANS_BACKEND_SERVICE_NAME=codetrans
 export CODETRANS_BACKEND_SERVICE_IP=${HOST_IP}
 
 ### The port of the backend service
-export CODETRANS_BACKEND_SERVICE_PORT=18154
+export CODETRANS_BACKEND_SERVICE_PORT=7777
 
 ### The port of the Nginx reverse proxy for application
-export CODETRANS_NGINX_PORT=18153
+export CODETRANS_NGINX_PORT=8088
 
 ### Endpoint of the backend service
 export CODETRANS_BACKEND_SERVICE_URL="http://${EXTERNAL_HOST_IP}:${CODETRANS_BACKEND_SERVICE_PORT}/v1/codetrans"