Fix ui dockerfile. (#1909)

Signed-off-by: lkk <33276950+lkk12014402@users.noreply.github.com>
(cherry picked from commit ff66600ab4)

.github/CODEOWNERS

@@ -1,3 +1,4 @@
+* liang1.lv@intel.com feng.tian@intel.com suyue.chen@intel.com
 /.github/ suyue.chen@intel.com ze.pan@intel.com
 /AgentQnA/ kaokao.lv@intel.com minmin.hou@intel.com
 /AudioQnA/ sihan.chen@intel.com wenjiao.yue@intel.com
@@ -19,5 +20,4 @@
 /Text2Image/ wenjiao.yue@intel.com xinyu.ye@intel.com
 /Translation/ liang1.lv@intel.com sihan.chen@intel.com
 /VideoQnA/ huiling.bao@intel.com xinyao.wang@intel.com
-/VisualQnA/ liang1.lv@intel.com sihan.chen@intel.com
-/*/ liang1.lv@intel.com feng.tian@intel.com suyue.chen@intel.com
+/VisualQnA/ liang1.lv@intel.com sihan.chen@intel.com

.github/ISSUE_TEMPLATE/1_bug_template.yml

@@ -32,6 +32,7 @@ body:
         - Mac
         - BSD
         - Other (Please let us know in description)
+        - N/A
     validations:
       required: true
 
@@ -56,6 +57,7 @@ body:
         - GPU-Nvidia
         - GPU-AMD
         - GPU-other (Please let us know in description)
+        - N/A
     validations:
       required: true
 
@@ -66,6 +68,8 @@ body:
       options:
         - label: Pull docker images from hub.docker.com
         - label: Build docker images from source
+        - label: Other
+        - label: N/A
     validations:
       required: true
 
@@ -74,10 +78,12 @@ body:
     attributes:
       label: Deploy method
       options:
-        - label: Docker compose
         - label: Docker
-        - label: Kubernetes
-        - label: Helm
+        - label: Docker Compose
+        - label: Kubernetes Helm Charts
+        - label: Kubernetes GMC
+        - label: Other
+        - label: N/A
     validations:
       required: true
 
@@ -88,6 +94,8 @@ body:
       options:
         - Single Node
         - Multiple Nodes
+        - Other
+        - N/A
       default: 0
     validations:
       required: true
@@ -127,3 +135,12 @@ body:
       render: shell
     validations:
       required: false
+
+
+  - type: textarea
+    id: attachments
+    attributes:
+      label: Attachments
+      description: Attach any relevant files or screenshots.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/2_feature_template.yml

@@ -32,6 +32,7 @@ body:
         - Mac
         - BSD
         - Other (Please let us know in description)
+        - N/A
     validations:
       required: true
 
@@ -56,6 +57,7 @@ body:
         - GPU-Nvidia
         - GPU-AMD
         - GPU-other (Please let us know in description)
+        - N/A
     validations:
       required: true
 
@@ -66,6 +68,8 @@ body:
       options:
         - Single Node
         - Multiple Nodes
+        - Other
+        - N/A
       default: 0
     validations:
       required: true

.github/code_spell_ignore.txt

@@ -1,2 +1,3 @@
 ModelIn
 modelin
+pressEnter

.github/license_template.txt

@@ -1,2 +1,2 @@
-Copyright (C) 2024 Intel Corporation
+Copyright (C) 2025 Intel Corporation
 SPDX-License-Identifier: Apache-2.0

.github/workflows/_build_comps_base_image.yml

@@ -0,0 +1,65 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Build Comps Base Image
+permissions: read-all
+on:
+  workflow_call:
+    inputs:
+      node:
+        required: true
+        type: string
+      build:
+        default: true
+        required: false
+        type: boolean
+      tag:
+        default: "latest"
+        required: false
+        type: string
+      opea_branch:
+        default: "main"
+        required: false
+        type: string
+      inject_commit:
+        default: false
+        required: false
+        type: boolean
+
+jobs:
+  pre-build-image-check:
+    runs-on: ubuntu-latest
+    outputs:
+      should_skip: ${{ steps.check-skip.outputs.should_skip }}
+    steps:
+      - name: Check if job should be skipped
+        id: check-skip
+        run: |
+          should_skip=true
+          if [[ "${{ inputs.node }}" == "gaudi" || "${{ inputs.node }}" == "xeon" ]]; then
+            should_skip=false
+          fi
+          echo "should_skip=$should_skip"
+          echo "should_skip=$should_skip" >> $GITHUB_OUTPUT
+
+  build-images:
+    needs: [ pre-build-image-check ]
+    if: ${{ needs.pre-build-image-check.outputs.should_skip == 'false' && fromJSON(inputs.build) }}
+    runs-on: "docker-build-${{ inputs.node }}"
+    steps:
+      - name: Clean Up Working Directory
+        run: sudo rm -rf ${{github.workspace}}/*
+
+      - name: Clone Required Repo
+        run: |
+          git clone --depth 1 --branch ${{ inputs.opea_branch }} https://github.com/opea-project/GenAIComps.git
+          cd GenAIComps && git rev-parse HEAD && cd ../ && ls -l
+
+      - name: Build Image
+        uses: opea-project/validation/actions/image-build@main
+        with:
+          work_dir: ${{ github.workspace }}/GenAIComps
+          docker_compose_path: ${{ github.workspace }}/GenAIComps/.github/workflows/docker/compose/base-compose.yaml
+          registry: ${OPEA_IMAGE_REPO}opea
+          inject_commit: ${{ inputs.inject_commit }}
+          tag: ${{ inputs.tag }}

.github/workflows/_build_image.yml

@@ -0,0 +1,101 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Build Images
+permissions: read-all
+on:
+  workflow_call:
+    inputs:
+      node:
+        required: true
+        type: string
+      build:
+        default: true
+        required: false
+        type: boolean
+      example:
+        required: true
+        type: string
+      services:
+        default: ""
+        required: false
+        type: string
+      tag:
+        default: "latest"
+        required: false
+        type: string
+      opea_branch:
+        default: "main"
+        required: false
+        type: string
+      inject_commit:
+        default: false
+        required: false
+        type: boolean
+
+jobs:
+  pre-build-image-check:
+    runs-on: ubuntu-latest
+    outputs:
+      should_skip: ${{ steps.check-skip.outputs.should_skip }}
+    steps:
+      - name: Check if job should be skipped
+        id: check-skip
+        run: |
+          should_skip=true
+          if [[ "${{ inputs.node }}" == "gaudi" || "${{ inputs.node }}" == "xeon" ]]; then
+            should_skip=false
+          fi
+          echo "should_skip=$should_skip"
+          echo "should_skip=$should_skip" >> $GITHUB_OUTPUT
+
+  build-images:
+    needs: [ pre-build-image-check ]
+    if: ${{ needs.pre-build-image-check.outputs.should_skip == 'false' && fromJSON(inputs.build) }}
+    runs-on: "docker-build-${{ inputs.node }}"
+    steps:
+      - name: Clean Up Working Directory
+        run: sudo rm -rf ${{github.workspace}}/*
+
+      - name: Get Checkout Ref
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
+            echo "CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge" >> $GITHUB_ENV
+          else
+            echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV
+          fi
+
+      - name: Checkout out GenAIExamples
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+          fetch-depth: 0
+
+      - name: Clone Required Repo
+        run: |
+          cd ${{ github.workspace }}/${{ inputs.example }}/docker_image_build
+          docker_compose_path=${{ github.workspace }}/${{ inputs.example }}/docker_image_build/build.yaml
+          if [[ $(grep -c "vllm:" ${docker_compose_path}) != 0 ]]; then
+              git clone https://github.com/vllm-project/vllm.git && cd vllm
+              VLLM_VER=v0.8.3
+              echo "Check out vLLM tag ${VLLM_VER}"
+              git checkout ${VLLM_VER} &> /dev/null && cd ../
+          fi
+          if [[ $(grep -c "vllm-gaudi:" ${docker_compose_path}) != 0 ]]; then
+              git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
+              VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
+              echo "Check out vLLM tag ${VLLM_VER}"
+              git checkout ${VLLM_VER} &> /dev/null && cd ../
+          fi
+          git clone --depth 1 --branch ${{ inputs.opea_branch }} https://github.com/opea-project/GenAIComps.git
+          cd GenAIComps && git rev-parse HEAD && cd ../
+
+      - name: Build Image
+        uses: opea-project/validation/actions/image-build@main
+        with:
+          work_dir: ${{ github.workspace }}/${{ inputs.example }}/docker_image_build
+          docker_compose_path: ${{ github.workspace }}/${{ inputs.example }}/docker_image_build/build.yaml
+          service_list: ${{ inputs.services }}
+          registry: ${OPEA_IMAGE_REPO}opea
+          inject_commit: ${{ inputs.inject_commit }}
+          tag: ${{ inputs.tag }}

.github/workflows/_example-workflow.yml

@@ -28,7 +28,7 @@ on:
         default: false
         required: false
         type: boolean
-      test_k8s:
+      test_helmchart:
         default: false
         required: false
         type: boolean
@@ -43,83 +43,54 @@ on:
       inject_commit:
         default: false
         required: false
-        type: string
+        type: boolean
+      use_model_cache:
+        default: false
+        required: false
+        type: boolean
 
 jobs:
 ####################################################################################################
 # Image Build
 ####################################################################################################
   build-images:
-    runs-on: "docker-build-${{ inputs.node }}"
-    steps:
-      - name: Clean Up Working Directory
-        run: sudo rm -rf ${{github.workspace}}/*
-
-      - name: Get Checkout Ref
-        run: |
-          if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
-            echo "CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge" >> $GITHUB_ENV
-          else
-            echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV
-          fi
-
-      - name: Checkout out GenAIExamples
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ env.CHECKOUT_REF }}
-          fetch-depth: 0
-
-      - name: Clone Required Repo
-        run: |
-          cd ${{ github.workspace }}/${{ inputs.example }}/docker_image_build
-          docker_compose_path=${{ github.workspace }}/${{ inputs.example }}/docker_image_build/build.yaml
-          if [[ $(grep -c "vllm:" ${docker_compose_path}) != 0 ]]; then
-              git clone https://github.com/vllm-project/vllm.git
-              cd vllm && git rev-parse HEAD && cd ../
-          fi
-          if [[ $(grep -c "vllm-gaudi:" ${docker_compose_path}) != 0 ]]; then
-               git clone https://github.com/HabanaAI/vllm-fork.git
-               cd vllm-fork && git checkout 3c39626 && cd ../
-          fi
-          git clone https://github.com/opea-project/GenAIComps.git
-          cd GenAIComps && git checkout ${{ inputs.opea_branch }} && git rev-parse HEAD && cd ../
-
-      - name: Build Image
-        if: ${{ fromJSON(inputs.build) }}
-        uses: opea-project/validation/actions/image-build@main
-        with:
-          work_dir: ${{ github.workspace }}/${{ inputs.example }}/docker_image_build
-          docker_compose_path: ${{ github.workspace }}/${{ inputs.example }}/docker_image_build/build.yaml
-          service_list: ${{ inputs.services }}
-          registry: ${OPEA_IMAGE_REPO}opea
-          inject_commit: ${{ inputs.inject_commit }}
-          tag: ${{ inputs.tag }}
+    uses: ./.github/workflows/_build_image.yml
+    with:
+      node: ${{ inputs.node }}
+      build: ${{ fromJSON(inputs.build) }}
+      example: ${{ inputs.example }}
+      services: ${{ inputs.services }}
+      tag: ${{ inputs.tag }}
+      opea_branch: ${{ inputs.opea_branch }}
+      inject_commit: ${{ inputs.inject_commit }}
 
 ####################################################################################################
 # Docker Compose Test
 ####################################################################################################
   test-example-compose:
     needs: [build-images]
-    if: ${{ fromJSON(inputs.test_compose) }}
+    if: ${{ inputs.test_compose }}
     uses: ./.github/workflows/_run-docker-compose.yml
     with:
       tag: ${{ inputs.tag }}
       example: ${{ inputs.example }}
       hardware: ${{ inputs.node }}
+      use_model_cache: ${{ inputs.use_model_cache }}
+      opea_branch: ${{ inputs.opea_branch }}
     secrets: inherit
 
 
 ####################################################################################################
-# K8S Test
+# helmchart Test
 ####################################################################################################
-  test-k8s-manifest:
-    needs: [build-images]
-    if: ${{ fromJSON(inputs.test_k8s) }}
-    uses: ./.github/workflows/_manifest-e2e.yml
+  test-helmchart:
+    if: ${{ fromJSON(inputs.test_helmchart) }}
+    uses: ./.github/workflows/_helm-e2e.yml
     with:
       example: ${{ inputs.example }}
       hardware: ${{ inputs.node }}
       tag: ${{ inputs.tag }}
+      mode: "CD"
     secrets: inherit
 
 ####################################################################################################
@@ -127,7 +98,7 @@ jobs:
 ####################################################################################################
   test-gmc-pipeline:
     needs: [build-images]
-    if: ${{ fromJSON(inputs.test_gmc) }}
+    if: false # ${{ fromJSON(inputs.test_gmc) }}
     uses: ./.github/workflows/_gmc-e2e.yml
     with:
       example: ${{ inputs.example }}

.github/workflows/_get-test-matrix.yml

@@ -60,9 +60,11 @@ jobs:
             base_commit=$(git rev-parse HEAD~1) # push event
           fi
           merged_commit=$(git log -1 --format='%H')
+          echo "print all changed files..."
+          git diff --name-only ${base_commit} ${merged_commit}
           changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | \
           grep -vE '${{ inputs.diff_excluded_files }}')" || true
-          echo "changed_files=$changed_files"
+          echo "filtered changed_files=$changed_files"
           export changed_files=$changed_files
           export test_mode=${{ inputs.test_mode }}
           export WORKSPACE=${{ github.workspace }}

.github/workflows/_helm-e2e.yml

@@ -0,0 +1,238 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Helm Chart E2e Test For Call
+permissions: read-all
+on:
+  workflow_call:
+    inputs:
+      example:
+        default: "chatqna"
+        required: true
+        type: string
+        description: "example to test, chatqna or common/asr"
+      hardware:
+        default: "xeon"
+        required: true
+        type: string
+      dockerhub:
+        default: "false"
+        required: false
+        type: string
+        description: "Set to true if you want to use released docker images at dockerhub. By default using internal docker registry."
+      mode:
+        default: "CD"
+        description: "Whether the test range is CI, CD or CICD"
+        required: false
+        type: string
+      tag:
+        default: "latest"
+        required: false
+        type: string
+      version:
+        default: "0-latest"
+        required: false
+        type: string
+
+jobs:
+  get-test-case:
+    runs-on: ubuntu-latest
+    outputs:
+      value_files: ${{ steps.get-test-files.outputs.value_files }}
+      CHECKOUT_REF: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
+    steps:
+      - name: Get checkout ref
+        id: get-checkout-ref
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
+            CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge
+          else
+            CHECKOUT_REF=${{ github.ref }}
+          fi
+          echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT
+          echo "checkout ref ${CHECKOUT_REF}"
+
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
+          fetch-depth: 0
+
+      - name: Get test Services
+        id: get-test-files
+        run: |
+          set -x
+          if [ "${{ inputs.mode }}" = "CI" ]; then
+            base_commit=${{ github.event.pull_request.base.sha }}
+            merged_commit=$(git log -1 --format='%H')
+            values_files=$(git diff --name-only ${base_commit} ${merged_commit} | \
+            grep "${{ inputs.example }}/kubernetes/helm" | \
+            grep "values.yaml" |\
+            sort -u)
+            echo $values_files
+          elif [ "${{ inputs.mode }}" = "CD" ]; then
+            values_files=$(ls ${{ inputs.example }}/kubernetes/helm/*values.yaml || true)
+          fi
+          value_files="["
+          for file in ${values_files}; do
+            if [ -f "$file" ]; then
+              filename=$(basename "$file")
+              if [[ "$filename" == *"gaudi"* ]]; then
+                if [[ "${{ inputs.hardware }}" == "gaudi" ]]; then
+                  value_files="${value_files}\"${filename}\","
+                fi
+              elif [[ "$filename" == *"rocm"* ]]; then
+                if [[ "${{ inputs.hardware }}" == "rocm" ]]; then
+                  value_files="${value_files}\"${filename}\","
+                fi
+              elif [[ "$filename" == *"nv"* ]]; then
+                continue
+              else
+                if [[ "${{ inputs.hardware }}" == "xeon" ]]; then
+                  value_files="${value_files}\"${filename}\","
+                fi
+              fi
+            fi
+          done
+          value_files="${value_files%,}]"
+
+          echo "value_files=${value_files}"
+          echo "value_files=${value_files}" >> $GITHUB_OUTPUT
+
+  helm-test:
+    needs: [get-test-case]
+    if: ${{ needs.get-test-case.outputs.value_files != '[]' }}
+    strategy:
+      matrix:
+        value_file: ${{ fromJSON(needs.get-test-case.outputs.value_files) }}
+      fail-fast: false
+    runs-on: k8s-${{ inputs.hardware }}
+    continue-on-error: true
+    steps:
+      - name: Clean Up Working Directory
+        run: |
+          echo "value_file=${{ matrix.value_file }}"
+          sudo rm -rf ${{github.workspace}}/*
+
+      - name: Get checkout ref
+        id: get-checkout-ref
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
+            CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge
+          else
+            CHECKOUT_REF=${{ github.ref }}
+          fi
+          echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT
+          echo "checkout ref ${CHECKOUT_REF}"
+
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
+          fetch-depth: 0
+
+      - name: Set variables
+        env:
+          example: ${{ inputs.example }}
+        run: |
+          CHART_NAME="${example,,}"  # CodeGen
+          echo "CHART_NAME=$CHART_NAME" >> $GITHUB_ENV
+          echo "RELEASE_NAME=${CHART_NAME}$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
+          echo "NAMESPACE=${CHART_NAME}-$(head -c 4 /dev/urandom | xxd -p)" >> $GITHUB_ENV
+          echo "ROLLOUT_TIMEOUT_SECONDS=600s" >> $GITHUB_ENV
+          echo "TEST_TIMEOUT_SECONDS=600s" >> $GITHUB_ENV
+          echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV
+          echo "should_cleanup=false" >> $GITHUB_ENV
+          echo "skip_validate=false" >> $GITHUB_ENV
+          echo "CHART_FOLDER=${example}/kubernetes/helm" >> $GITHUB_ENV
+
+      - name: Helm install
+        id: install
+        env:
+          GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
+          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+          HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
+          HFTOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
+          value_file: ${{ matrix.value_file }}
+        run: |
+          set -xe
+          echo "should_cleanup=true" >> $GITHUB_ENV
+          if [[ ! -f ${{ github.workspace }}/${{ env.CHART_FOLDER }}/${value_file} ]]; then
+            echo "No value file found, exiting test!"
+            echo "skip_validate=true" >> $GITHUB_ENV
+            echo "should_cleanup=false" >> $GITHUB_ENV
+            exit 0
+          fi
+
+          for img in `helm template -n $NAMESPACE $RELEASE_NAME oci://ghcr.io/opea-project/charts/${CHART_NAME} -f ${{ inputs.example }}/kubernetes/helm/${value_file} --version ${{ inputs.version }} | grep 'image:' | grep 'opea/' | awk '{print $2}' | xargs`;
+          do
+            # increase helm install wait for for vllm-gaudi case
+            if [[ $img == *"vllm-gaudi"* ]]; then
+              ROLLOUT_TIMEOUT_SECONDS=900s
+            fi
+          done
+          if ! helm install \
+            --create-namespace \
+            --namespace $NAMESPACE \
+            $RELEASE_NAME \
+            oci://ghcr.io/opea-project/charts/${CHART_NAME} \
+            --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} \
+            --set global.modelUseHostPath=/data2/hf_model \
+            --set GOOGLE_API_KEY=${{ env.GOOGLE_API_KEY}} \
+            --set GOOGLE_CSE_ID=${{ env.GOOGLE_CSE_ID}} \
+            --set web-retriever.GOOGLE_API_KEY=${{ env.GOOGLE_API_KEY}} \
+            --set web-retriever.GOOGLE_CSE_ID=${{ env.GOOGLE_CSE_ID}} \
+            -f ${{ inputs.example }}/kubernetes/helm/${value_file} \
+            --version ${{ inputs.version }} \
+            --wait --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then
+            echo "Failed to install chart ${{ inputs.example }}"
+            echo "skip_validate=true" >> $GITHUB_ENV
+            .github/workflows/scripts/k8s-utils.sh dump_pods_status $NAMESPACE
+            exit 1
+          fi
+
+      - name: Validate e2e test
+        if: always()
+        run: |
+          set -xe
+          if $skip_validate; then
+            echo "Skip validate"
+          else
+            LOG_PATH=/home/$(whoami)/helm-logs
+            chart=${{ env.CHART_NAME }}
+            helm test -n $NAMESPACE $RELEASE_NAME --logs --timeout "$TEST_TIMEOUT_SECONDS" | tee ${LOG_PATH}/charts-${chart}.log
+            exit_code=$?
+            if [ $exit_code -ne 0 ]; then
+                echo "Chart ${chart} test failed, please check the logs in ${LOG_PATH}!"
+                exit 1
+            fi
+
+            echo "Checking response results, make sure the output is reasonable. "
+            teststatus=false
+            if [[ -f $LOG_PATH/charts-${chart}.log ]] && \
+            [[ $(grep -c "^Phase:.*Failed" $LOG_PATH/charts-${chart}.log) != 0 ]]; then
+                teststatus=false
+                ${{ github.workspace }}/.github/workflows/scripts/k8s-utils.sh dump_all_pod_logs $NAMESPACE
+            else
+                teststatus=true
+            fi
+
+            if [ $teststatus == false ]; then
+                echo "Response check failed, please check the logs in artifacts!"
+                exit 1
+            else
+                echo "Response check succeeded!"
+                exit 0
+            fi
+          fi
+
+      - name: Helm uninstall
+        if: always()
+        run: |
+          if $should_cleanup; then
+            helm uninstall $RELEASE_NAME --namespace $NAMESPACE
+            if ! kubectl delete ns $NAMESPACE --timeout=$KUBECTL_TIMEOUT_SECONDS; then
+              kubectl delete pods --namespace $NAMESPACE --force --grace-period=0 --all
+              kubectl delete ns $NAMESPACE --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
+            fi
+          fi

.github/workflows/_manifest-e2e.yml

@@ -1,185 +0,0 @@
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-
-name: Single Kubernetes Manifest E2e Test For Call
-on:
-  workflow_call:
-    inputs:
-      example:
-        default: "ChatQnA"
-        description: "The example to test on K8s"
-        required: true
-        type: string
-      hardware:
-        default: "xeon"
-        description: "Nodes to run the test, xeon or gaudi"
-        required: true
-        type: string
-      tag:
-        default: "latest"
-        description: "Tag to apply to images, default is latest"
-        required: false
-        type: string
-
-jobs:
-  get-test-case:
-    runs-on: ubuntu-latest
-    outputs:
-      test_cases: ${{ steps.test-case-matrix.outputs.test_cases }}
-      CHECKOUT_REF: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
-    steps:
-      - name: Get checkout ref
-        id: get-checkout-ref
-        run: |
-          if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
-            CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge
-          else
-            CHECKOUT_REF=${{ github.ref }}
-          fi
-          echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT
-          echo "checkout ref ${CHECKOUT_REF}"
-
-      - name: Checkout out Repo
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
-          fetch-depth: 0
-
-      - name: Get test matrix
-        shell: bash
-        id: test-case-matrix
-        run: |
-          example_l=$(echo ${{ inputs.example }} | tr '[:upper:]' '[:lower:]')
-          cd ${{ github.workspace }}/${{ inputs.example }}/tests
-          run_test_cases=""
-
-          default_test_case=$(find . -type f -name "test_manifest_on_${{ inputs.hardware }}.sh" | cut -d/ -f2)
-          if [ "$default_test_case" ]; then run_test_cases="$default_test_case"; fi
-          other_test_cases=$(find . -type f -name "test_manifest_*_on_${{ inputs.hardware }}.sh" | cut -d/ -f2)
-          echo "default_test_case=$default_test_case"
-          echo "other_test_cases=$other_test_cases"
-
-          if [ "${{ inputs.tag }}" == "ci" ]; then
-            base_commit=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-            "https://api.github.com/repos/opea-project/GenAIExamples/commits?sha=${{ github.event.pull_request.base.ref }}" | jq -r '.[0].sha')
-            merged_commit=$(git log -1 --format='%H')
-            changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | grep -vE '${{ inputs.diff_excluded_files }}')" || true
-          fi
-
-          for test_case in $other_test_cases; do
-            if [ "${{ inputs.tag }}" == "ci" ]; then
-              flag=${test_case%_on_*}
-              flag=${flag#test_compose_}
-              if [[ $(printf '%s\n' "${changed_files[@]}" | grep ${{ inputs.example }} | grep ${flag}) ]]; then
-                run_test_cases="$run_test_cases $test_case"
-              fi
-            else
-              run_test_cases="$run_test_cases $test_case"
-            fi
-          done
-
-          test_cases=$(echo $run_test_cases | tr ' ' '\n' | sort -u | jq -R '.' | jq -sc '.')
-          echo "test_cases=$test_cases"
-          echo "test_cases=$test_cases" >> $GITHUB_OUTPUT
-
-  manifest-test:
-    needs: [get-test-case]
-    strategy:
-      matrix:
-        test_case: ${{ fromJSON(needs.get-test-case.outputs.test_cases) }}
-      fail-fast: false
-    runs-on: "k8s-${{ inputs.hardware }}"
-    continue-on-error: true
-    steps:
-      - name: Clean Up Working Directory
-        run: sudo rm -rf ${{github.workspace}}/*
-
-      - name: Get checkout ref
-        run: |
-          if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
-            echo "CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge" >> $GITHUB_ENV
-          else
-            echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV
-          fi
-          echo "checkout ref ${{ env.CHECKOUT_REF }}"
-
-      - name: Checkout out Repo
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ env.CHECKOUT_REF }}
-          fetch-depth: 0
-
-      - name: Set variables
-        env:
-          test_case: ${{ matrix.test_case }}
-        run: |
-          echo "IMAGE_REPO=${OPEA_IMAGE_REPO}opea" >> $GITHUB_ENV
-          echo "IMAGE_TAG=${{ inputs.tag }}" >> $GITHUB_ENV
-          lower_example=$(echo "${{ inputs.example }}" | tr '[:upper:]' '[:lower:]')
-          name=$(echo "$test_case" | cut -d/ -f2 | cut -d'_' -f3- |cut -d'_' -f1 | grep -v 'on' | sed 's/^/-/')
-          echo "NAMESPACE=$lower_example$name-$(tr -dc a-z0-9 </dev/urandom | head -c 16)" >> $GITHUB_ENV
-          echo "ROLLOUT_TIMEOUT_SECONDS=1800s" >> $GITHUB_ENV
-          echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV
-          echo "continue_test=true" >> $GITHUB_ENV
-          echo "should_cleanup=false" >> $GITHUB_ENV
-          echo "skip_validate=true" >> $GITHUB_ENV
-          echo "NAMESPACE=$NAMESPACE"
-
-      - name: Kubectl install
-        id: install
-        env:
-          test_case: ${{ matrix.test_case }}
-        run: |
-          set -x
-          echo "test_case=$test_case"
-          if [[ ! -f ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} ]]; then
-            echo "No test script found, exist test!"
-            exit 0
-          else
-            ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} init_${{ inputs.example }}
-            echo "should_cleanup=true" >> $GITHUB_ENV
-            kubectl create ns $NAMESPACE
-            ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} install_${{ inputs.example }} $NAMESPACE
-            echo "Testing ${{ inputs.example }}, waiting for pod ready..."
-            if kubectl rollout status deployment --namespace "$NAMESPACE" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then
-              echo "Testing manifests ${{ inputs.example }}, waiting for pod ready done!"
-              echo "skip_validate=false" >> $GITHUB_ENV
-            else
-              echo "Timeout waiting for pods in namespace $NAMESPACE to be ready!"
-              .github/workflows/scripts/k8s-utils.sh dump_pods_status $NAMESPACE
-              exit 1
-            fi
-            sleep 60
-          fi
-
-      - name: Validate e2e test
-        if: always()
-        env:
-          test_case: ${{ matrix.test_case }}
-        run: |
-          if $skip_validate; then
-            echo "Skip validate"
-          else
-            if ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} validate_${{ inputs.example }} $NAMESPACE ; then
-              echo "Validate ${test_case} successful!"
-            else
-              echo "Validate ${test_case} failure!!!"
-              echo "Check the logs in 'Dump logs when e2e test failed' step!!!"
-              exit 1
-            fi
-          fi
-
-      - name: Dump logs when e2e test failed
-        if: failure()
-        run: |
-          .github/workflows/scripts/k8s-utils.sh dump_all_pod_logs $NAMESPACE
-
-      - name: Kubectl uninstall
-        if: always()
-        run: |
-          if $should_cleanup; then
-            if ! kubectl delete ns $NAMESPACE --timeout=$KUBECTL_TIMEOUT_SECONDS; then
-              kubectl delete pods --namespace $NAMESPACE --force --grace-period=0 --all
-              kubectl delete ns $NAMESPACE --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
-            fi
-          fi

.github/workflows/_run-docker-compose.yml

@@ -28,6 +28,14 @@ on:
         required: false
         type: string
         default: ""
+      use_model_cache:
+        required: false
+        type: boolean
+        default: false
+      opea_branch:
+        default: "main"
+        required: false
+        type: string
 jobs:
   get-test-case:
     runs-on: ubuntu-latest
@@ -60,9 +68,16 @@ jobs:
           cd ${{ github.workspace }}/${{ inputs.example }}/tests
           run_test_cases=""
 
-          default_test_case=$(find . -type f -name "test_compose_on_${{ inputs.hardware }}.sh" | cut -d/ -f2)
+          if [[ "${{ inputs.hardware }}" == "gaudi"* ]]; then
+            hardware="gaudi"
+          elif [[ "${{ inputs.hardware }}" == "xeon"* ]]; then
+            hardware="xeon"
+          else
+            hardware="${{ inputs.hardware }}"
+          fi
+          default_test_case=$(find . -type f -name "test_compose_on_$hardware.sh" | cut -d/ -f2)
           if [ "$default_test_case" ]; then run_test_cases="$default_test_case"; fi
-          other_test_cases=$(find . -type f -name "test_compose_*_on_${{ inputs.hardware }}.sh" | cut -d/ -f2)
+          other_test_cases=$(find . -type f -name "test_compose_*_on_$hardware.sh" | cut -d/ -f2)
           echo "default_test_case=$default_test_case"
           echo "other_test_cases=$other_test_cases"
 
@@ -85,12 +100,17 @@ jobs:
               fi
           done
 
+          if [ -z "$run_test_cases" ] && [[ $(printf '%s\n' "${changed_files[@]}" | grep ${{ inputs.example }} | grep /tests/) ]]; then
+              run_test_cases=$other_test_cases
+          fi
+
           test_cases=$(echo $run_test_cases | tr ' ' '\n' | sort -u | jq -R '.' | jq -sc '.')
           echo "test_cases=$test_cases"
           echo "test_cases=$test_cases" >> $GITHUB_OUTPUT
 
-  run-test:
+  compose-test:
     needs: [get-test-case]
+    if: ${{ needs.get-test-case.outputs.test_cases != '[""]' }}
     strategy:
       matrix:
         test_case: ${{ fromJSON(needs.get-test-case.outputs.test_cases) }}
@@ -101,9 +121,18 @@ jobs:
       - name: Clean up Working Directory
         run: |
           sudo rm -rf ${{github.workspace}}/* || true
+
+          echo "Cleaning up containers using ports..."
+          cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 0.0.0.0:5000 | awk -F' : ' '{print $1}')
+          if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
           docker system prune -f
-          docker rmi $(docker images --filter reference="*/*/*:latest" -q) || true
-          docker rmi $(docker images --filter reference="*/*:ci" -q) || true
+
+          echo "Cleaning up images ..."
+          docker images --filter reference="*/*/*:latest" -q | xargs -r docker rmi && sleep 1s
+          docker images --filter reference="*/*:ci" -q | xargs -r docker rmi && sleep 1s
+          docker images --filter reference="*:5000/*/*" -q | xargs -r docker rmi && sleep 1s
+          docker images --filter reference="opea/comps-base" -q | xargs -r docker rmi && sleep 1s
+          docker images
 
       - name: Checkout out Repo
         uses: actions/checkout@v4
@@ -122,40 +151,73 @@ jobs:
           bash ${{ github.workspace }}/.github/workflows/scripts/docker_compose_clean_up.sh "ports"
           docker ps
 
+      - name: Log in DockerHub
+        uses: docker/login-action@v3.2.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
       - name: Run test
         shell: bash
         env:
           HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
+          HF_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
           GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
           GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
           PINECONE_KEY: ${{ secrets.PINECONE_KEY }}
           PINECONE_KEY_LANGCHAIN_TEST: ${{ secrets.PINECONE_KEY_LANGCHAIN_TEST }}
           SDK_BASE_URL: ${{ secrets.SDK_BASE_URL }}
           SERVING_TOKEN: ${{ secrets.SERVING_TOKEN }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          FINNHUB_API_KEY: ${{ secrets.FINNHUB_API_KEY }}
+          FINANCIAL_DATASETS_API_KEY: ${{ secrets.FINANCIAL_DATASETS_API_KEY }}
           IMAGE_REPO: ${{ inputs.registry }}
           IMAGE_TAG: ${{ inputs.tag }}
+          opea_branch: ${{ inputs.opea_branch }}
           example: ${{ inputs.example }}
           hardware: ${{ inputs.hardware }}
           test_case: ${{ matrix.test_case }}
+          use_model_cache: ${{ inputs.use_model_cache }}
         run: |
           cd ${{ github.workspace }}/$example/tests
           if [[ "$IMAGE_REPO" == "" ]]; then export IMAGE_REPO="${OPEA_IMAGE_REPO}opea"; fi
-          if [ -f ${test_case} ]; then timeout 30m bash ${test_case}; else echo "Test script {${test_case}} not found, skip test!"; fi
+          if [[ "$use_model_cache" == "true" ]]; then
+            if [ -d "/data2/hf_model" ]; then
+              export model_cache="/data2/hf_model"
+            else
+              echo "Model cache directory /data2/hf_model does not exist"
+              export model_cache="$HOME/.cache/huggingface/hub"
+            fi
+            if [[ "$test_case" == *"rocm"* ]]; then
+              export model_cache="/var/lib/GenAI/data"
+            fi
+          fi
+          if [ -f "${test_case}" ]; then timeout 60m bash "${test_case}"; else echo "Test script {${test_case}} not found, skip test!"; fi
 
       - name: Clean up container after test
-        shell: bash
-        if: cancelled() || failure()
+        if: always()
         run: |
-          cd ${{ github.workspace }}/${{ inputs.example }}
-          export test_case=${{ matrix.test_case }}
-          export hardware=${{ inputs.hardware }}
-          bash ${{ github.workspace }}/.github/workflows/scripts/docker_compose_clean_up.sh "containers"
-          docker system prune -f
-          docker rmi $(docker images --filter reference="*:5000/*/*" -q) || true
+          set -x
+
+          echo "Cleaning up containers using ports..."
+          cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 0.0.0.0:5000 | awk -F' : ' '{print $1}')
+          if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
+
+          echo "Cleaning up images ..."
+          if [[ "${{ inputs.hardware }}" == "xeon"* ]]; then
+              docker system prune -a -f
+          else
+              docker images --filter reference="*/*/*:latest" -q | xargs -r docker rmi && sleep 1s
+              docker images --filter reference="*/*:ci" -q | xargs -r docker rmi && sleep 1s
+              docker images --filter reference="*:5000/*/*" -q | xargs -r docker rmi && sleep 1s
+              docker images --filter reference="opea/comps-base" -q | xargs -r docker rmi && sleep 1s
+              docker system prune -f
+          fi
+          docker images
 
       - name: Publish pipeline artifact
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ inputs.example }}_${{ matrix.test_case }}
+          name: ${{ inputs.hardware }}_${{ inputs.example }}_${{ matrix.test_case }}
           path: ${{ github.workspace }}/${{ inputs.example }}/tests/*.log

.github/workflows/check-online-doc-build.yml

@@ -13,7 +13,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
 
     - name: Checkout

.github/workflows/daily_check_issue_and_pr.yml

@@ -0,0 +1,28 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Check stale issue and pr
+
+on:
+  schedule:
+    - cron: "30 22 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v9
+        with:
+          days-before-issue-stale: 30
+          days-before-pr-stale: 30
+          days-before-issue-close: 7
+          days-before-pr-close: 7
+          stale-issue-message: "This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days."
+          stale-pr-message: "This PR is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days."
+          close-issue-message: "This issue was closed because it has been stalled for 7 days with no activity."
+          close-pr-message: "This PR was closed because it has been stalled for 7 days with no activity."
+          repo-token: ${{ secrets.ACTION_TOKEN }}
+          start-date: "2025-03-01T00:00:00Z"

.github/workflows/dockerhub-description.yml

@@ -0,0 +1,117 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Update Docker Hub Description
+on:
+  schedule:
+    - cron: "0 0 * * 0"
+  workflow_dispatch:
+
+jobs:
+  get-images-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      examples_json: ${{ steps.extract.outputs.examples_json }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Extract images info and generate JSON matrix
+        id: extract
+        run: |
+          #!/bin/bash
+          set -e
+          images=$(awk -F'|' '/^\| *\[opea\// {
+            gsub(/^ +| +$/, "", $2);
+            gsub(/^ +| +$/, "", $4);
+            gsub(/^ +| +$/, "", $5);
+
+            # Extract the path portion of the dockerHub link from the Example Images column
+            match($2, /\(https:\/\/hub\.docker\.com\/r\/[^)]*\)/);
+            repository = substr($2, RSTART, RLENGTH);
+            # Remove the prefix and the trailing right bracket
+            sub(/^\(https:\/\/hub\.docker\.com\/r\//, "", repository);
+            sub(/\)$/, "", repository);
+
+            # Description Direct assignment
+            description = $4;
+
+            # Extract the content of the github link from the Readme column
+            match($5, /\(https:\/\/github\.com\/[^)]*\)/);
+            readme_url = substr($5, RSTART, RLENGTH);
+            # Remove the prefix and the trailing right bracket
+            sub(/^\(https:\/\/github\.com\//, "", readme_url);
+            sub(/\)$/, "", readme_url);
+            # Remove blob information, such as "blob/main/" or "blob/habana_main/"
+            gsub(/blob\/[^/]+\//, "", readme_url);
+            # Remove the organization name and keep only the file path, such as changing "opea-project/GenAIExamples/AudioQnA/README.md" to "GenAIExamples/AudioQnA/README.md"
+            sub(/^[^\/]+\//, "", readme_url);
+
+            # Generate JSON object string
+            printf "{\"repository\":\"%s\",\"short-description\":\"%s\",\"readme-filepath\":\"%s\"}\n", repository, description, readme_url;
+          }' docker_images_list.md)
+
+          # Concatenate all JSON objects into a JSON array, using paste to separate them with commas
+          json="[$(echo "$images" | paste -sd, -)]"
+          echo "$json"
+          # Set as output variable for subsequent jobs to use
+          echo "::set-output name=examples_json::$json"
+
+  check-images-matrix:
+    runs-on: ubuntu-latest
+    needs: get-images-matrix
+    if: ${{ needs.get-images-matrix.outputs.examples_json != '' }}
+    strategy:
+      matrix:
+        image: ${{ fromJSON(needs.get-images-matrix.outputs.examples_json) }}
+      fail-fast: false
+    steps:
+    - name: Check dockerhub description
+      run: |
+        echo "dockerhub description for ${{ matrix.image.repository }}"
+        echo "short-description: ${{ matrix.image.short-description }}"
+        echo "readme-filepath: ${{ matrix.image.readme-filepath }}"
+
+  dockerHubDescription:
+    runs-on: ubuntu-latest
+    needs: get-images-matrix
+    if: ${{ needs.get-images-matrix.outputs.examples_json != '' }}
+    strategy:
+      matrix:
+        image: ${{ fromJSON(needs.get-images-matrix.outputs.examples_json) }}
+      fail-fast: false
+    steps:
+    - name: Checkout GenAIExamples
+      uses: actions/checkout@v4
+      with:
+        repository: opea-project/GenAIExamples
+        path: GenAIExamples
+
+    - name: Checkout GenAIComps
+      uses: actions/checkout@v4
+      with:
+        repository: opea-project/GenAIComps
+        path: GenAIComps
+
+    - name: Checkout vllm-openvino
+      uses: actions/checkout@v4
+      with:
+        repository: vllm-project/vllm
+        path: vllm
+
+    - name: Checkout vllm-gaudi
+      uses: actions/checkout@v4
+      with:
+        repository: HabanaAI/vllm-fork
+        ref: habana_main
+        path: vllm-fork
+
+    - name: add dockerhub description
+      uses: peter-evans/dockerhub-description@v4
+      with:
+        username: ${{ secrets.DOCKERHUB_USER }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+        repository: ${{ matrix.image.repository }}
+        short-description: ${{ matrix.image.short-description }}
+        readme-filepath: ${{ matrix.image.readme-filepath }}
+        enable-url-completion: false

.github/workflows/manual-docker-publish.yml

@@ -41,9 +41,11 @@ jobs:
 
   publish:
     needs: [get-image-list]
+    if: ${{ needs.get-image-list.outputs.matrix != '' }}
     strategy:
       matrix:
         image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }}
+      fail-fast: false
     runs-on: "docker-build-${{ inputs.node }}"
     steps:
       - uses: docker/login-action@v3.2.0

.github/workflows/manual-docker-scan.yml

@@ -12,7 +12,7 @@ on:
         type: string
       examples:
         default: ""
-        description: 'List of examples to publish "AgentQnA,AudioQnA,ChatQnA,CodeGen,CodeTrans,DocIndexRetriever,DocSum,FaqGen,InstructionTuning,MultimodalQnA,ProductivitySuite,RerankFinetuning,SearchQnA,Translation,VideoQnA,VisualQnA"'
+        description: 'List of examples to publish "AgentQnA,AudioQnA,ChatQnA,CodeGen,CodeTrans,DocIndexRetriever,DocSum,InstructionTuning,MultimodalQnA,ProductivitySuite,RerankFinetuning,SearchQnA,Translation,VideoQnA,VisualQnA"'
         required: false
         type: string
       images:
@@ -47,6 +47,7 @@ jobs:
   scan-docker:
     needs: get-image-list
     runs-on: "docker-build-${{ inputs.node }}"
+    if: ${{ needs.get-image-list.outputs.matrix != '' }}
     strategy:
       matrix:
         image: ${{ fromJson(needs.get-image-list.outputs.matrix) }}

.github/workflows/manual-example-workflow.yml

@@ -7,7 +7,7 @@ on:
     inputs:
       nodes:
         default: "gaudi,xeon"
-        description: "Hardware to run test"
+        description: "Hardware to run test gaudi,xeon,rocm,arc,gaudi3,xeon-gnr"
         required: true
         type: string
       examples:
@@ -20,11 +20,6 @@ on:
         description: "Tag to apply to images"
         required: true
         type: string
-      deploy_gmc:
-        default: false
-        description: 'Whether to deploy gmc'
-        required: true
-        type: boolean
       build:
         default: true
         description: 'Build test required images for Examples'
@@ -35,14 +30,9 @@ on:
         description: 'Test examples with docker compose'
         required: false
         type: boolean
-      test_k8s:
-        default: false
-        description: 'Test examples with k8s'
-        required: false
-        type: boolean
-      test_gmc:
-        default: false
-        description: 'Test examples with gmc'
+      test_helmchart:
+        default: true
+        description: 'Test examples with helm charts'
         required: false
         type: boolean
       opea_branch:
@@ -52,9 +42,14 @@ on:
         type: string
       inject_commit:
         default: false
-        description: "inject commit to docker images true or false"
+        description: "inject commit to docker images"
         required: false
-        type: string
+        type: boolean
+      use_model_cache:
+        default: false
+        description: "use model cache"
+        required: false
+        type: boolean
 
 permissions: read-all
 jobs:
@@ -74,23 +69,20 @@ jobs:
         nodes_json=$(printf '%s\n' "${nodes[@]}" | sort -u | jq -R '.' | jq -sc '.')
         echo "nodes=$nodes_json" >> $GITHUB_OUTPUT
 
-  build-deploy-gmc:
+  build-comps-base:
     needs: [get-test-matrix]
-    if: ${{ fromJSON(inputs.deploy_gmc) }}
     strategy:
       matrix:
         node: ${{ fromJson(needs.get-test-matrix.outputs.nodes) }}
-      fail-fast: false
-    uses: ./.github/workflows/_gmc-workflow.yml
+    uses: ./.github/workflows/_build_comps_base_image.yml
     with:
       node: ${{ matrix.node }}
+      build: ${{ fromJSON(inputs.build) }}
       tag: ${{ inputs.tag }}
       opea_branch: ${{ inputs.opea_branch }}
-    secrets: inherit
 
   run-examples:
-    needs: [get-test-matrix, build-deploy-gmc]
-    if: always()
+    needs: [get-test-matrix, build-comps-base]
     strategy:
       matrix:
         example: ${{ fromJson(needs.get-test-matrix.outputs.examples) }}
@@ -103,8 +95,8 @@ jobs:
       tag: ${{ inputs.tag }}
       build: ${{ fromJSON(inputs.build) }}
       test_compose: ${{ fromJSON(inputs.test_compose) }}
-      test_k8s: ${{ fromJSON(inputs.test_k8s) }}
-      test_gmc: ${{ fromJSON(inputs.test_gmc) }}
+      test_helmchart: ${{ fromJSON(inputs.test_helmchart) }}
       opea_branch: ${{ inputs.opea_branch }}
       inject_commit: ${{ inputs.inject_commit }}
+      use_model_cache: ${{ inputs.use_model_cache }}
     secrets: inherit

.github/workflows/manual-freeze-tag.yml

@@ -25,9 +25,9 @@ jobs:
 
       - name: Set up Git
         run: |
-          git config --global user.name "NeuralChatBot"
-          git config --global user.email "grp_neural_chat_bot@intel.com"
-          git remote set-url origin https://NeuralChatBot:"${{ secrets.ACTION_TOKEN }}"@github.com/opea-project/GenAIExamples.git
+          git config --global user.name "CICD-at-OPEA"
+          git config --global user.email "CICD@opea.dev"
+          git remote set-url origin https://CICD-at-OPEA:"${{ secrets.ACTION_TOKEN }}"@github.com/opea-project/GenAIExamples.git
 
       - name: Run script
         run: |

.github/workflows/manual-image-build.yml

@@ -32,9 +32,9 @@ on:
         type: string
       inject_commit:
         default: false
-        description: "inject commit to docker images true or false"
+        description: "inject commit to docker images"
         required: false
-        type: string
+        type: boolean
 
 jobs:
   get-test-matrix:
@@ -51,6 +51,7 @@ jobs:
 
   image-build:
     needs: get-test-matrix
+    if: ${{ needs.get-test-matrix.outputs.nodes != '' }}
     strategy:
       matrix:
         node: ${{ fromJson(needs.get-test-matrix.outputs.nodes) }}

.github/workflows/manual-reset-local-registry.yml

@@ -33,6 +33,7 @@ jobs:
 
   clean-up:
     needs: get-build-matrix
+    if: ${{ needs.get-image-list.outputs.matrix != '' }}
     strategy:
       matrix:
         node: ${{ fromJson(needs.get-build-matrix.outputs.nodes) }}
@@ -47,6 +48,7 @@ jobs:
 
   build:
     needs: [get-build-matrix, clean-up]
+    if: ${{ needs.get-image-list.outputs.matrix != '' }}
     strategy:
       matrix:
         example: ${{ fromJson(needs.get-build-matrix.outputs.examples) }}

.github/workflows/nightly-docker-build-publish.yml

@@ -5,7 +5,7 @@ name: Nightly build/publish latest docker images
 
 on:
   schedule:
-    - cron: "30 14 * * *" # UTC time
+    - cron: "30 14 * * 1-5" # UTC time
   workflow_dispatch:
 
 env:
@@ -32,30 +32,54 @@ jobs:
           echo "TAG=$TAG" >> $GITHUB_OUTPUT
           echo "PUBLISH_TAGS=$PUBLISH_TAGS" >> $GITHUB_OUTPUT
 
-  build-and-test:
-    needs: get-build-matrix
+  build-comps-base:
+    needs: [get-build-matrix]
+    uses: ./.github/workflows/_build_comps_base_image.yml
+    with:
+      node: gaudi
+
+  build-images:
+    needs: [get-build-matrix, build-comps-base]
+    strategy:
+      matrix:
+        example: ${{ fromJSON(needs.get-build-matrix.outputs.examples_json) }}
+      fail-fast: false
+    uses: ./.github/workflows/_build_image.yml
+    with:
+      node: gaudi
+      example: ${{ matrix.example }}
+      inject_commit: true
+    secrets: inherit
+
+  test-example:
+    needs: [get-build-matrix]
+    if: ${{ needs.get-build-matrix.outputs.examples_json != '' }}
     strategy:
       matrix:
         example: ${{ fromJSON(needs.get-build-matrix.outputs.examples_json) }}
       fail-fast: false
     uses: ./.github/workflows/_example-workflow.yml
     with:
-      node: gaudi
+      node: xeon
+      build: false
       example: ${{ matrix.example }}
       test_compose: true
+      inject_commit: true
     secrets: inherit
 
   get-image-list:
-    needs: get-build-matrix
+    needs: [get-build-matrix]
     uses: ./.github/workflows/_get-image-list.yml
     with:
       examples: ${{ needs.get-build-matrix.outputs.EXAMPLES }}
 
   publish:
-    needs: [get-build-matrix, get-image-list, build-and-test]
+    needs: [get-build-matrix, get-image-list, build-images]
+    if: always()
     strategy:
       matrix:
         image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }}
+      fail-fast: false
     runs-on: "docker-build-gaudi"
     steps:
       - uses: docker/login-action@v3.2.0

.github/workflows/pr-chart-e2e.yml

@@ -0,0 +1,78 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: E2E Test with Helm Charts
+
+on:
+  pull_request_target:
+    branches: [main]
+    types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
+    paths:
+      - "!**.md"
+      - "**/helm/**"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  job1:
+    name: Get-Test-Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      run_matrix: ${{ steps.get-test-matrix.outputs.run_matrix }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          ref: "refs/pull/${{ github.event.number }}/merge"
+          fetch-depth: 0
+
+      - name: Get Test Matrix
+        id: get-test-matrix
+        run: |
+          set -x
+          echo "base_commit=${{ github.event.pull_request.base.sha }}"
+          base_commit=${{ github.event.pull_request.base.sha }}
+          merged_commit=$(git log -1 --format='%H')
+          values_files=$(git diff --name-only ${base_commit} ${merged_commit} | \
+          grep "values.yaml" | \
+          sort -u ) #CodeGen/kubernetes/helm/cpu-values.yaml
+          run_matrix="{\"include\":["
+          for values_file in ${values_files}; do
+            if [ -f "$values_file" ]; then
+              valuefile=$(basename "$values_file")   # cpu-values.yaml
+              example=$(echo "$values_file" | cut -d'/' -f1) # CodeGen
+              if [[ "$valuefile" == *"gaudi"* ]]; then
+                hardware="gaudi"
+              elif [[ "$valuefile" == *"rocm"* ]]; then
+                hardware="rocm"
+              elif [[ "$valuefile" == *"nv"* ]]; then
+                continue
+              else
+                hardware="xeon"
+              fi
+              echo "example=${example}, hardware=${hardware}, valuefile=${valuefile}"
+              if [[ $(echo ${run_matrix} | grep -c "{\"example\":\"${example}\",\"hardware\":\"${hardware}\"},") == 0 ]]; then
+                run_matrix="${run_matrix}{\"example\":\"${example}\",\"hardware\":\"${hardware}\"},"
+                echo "------------------ add one values file ------------------"
+              fi
+            fi
+          done
+          run_matrix="${run_matrix%,}"
+          run_matrix=$run_matrix"]}"
+          echo "run_matrix="${run_matrix}""
+          echo "run_matrix="${run_matrix}"" >> $GITHUB_OUTPUT
+
+  helm-chart-test:
+    needs: [job1]
+    if: always() && ${{ fromJSON(needs.job1.outputs.run_matrix).length != 0 }}
+    uses: ./.github/workflows/_helm-e2e.yml
+    strategy:
+      matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
+    with:
+      example: ${{ matrix.example }}
+      hardware: ${{ matrix.hardware }}
+      mode: "CI"
+    secrets: inherit

.github/workflows/pr-docker-compose-e2e.yml

@@ -28,19 +28,20 @@ jobs:
     if: ${{ !github.event.pull_request.draft }}
     uses: ./.github/workflows/_get-test-matrix.yml
     with:
-      diff_excluded_files: '.github|*.md|*.txt|kubernetes|manifest|gmc|assets|benchmark'
+      diff_excluded_files: '\.github|\.md|\.txt|kubernetes|gmc|assets|benchmark'
 
   example-test:
     needs: [get-test-matrix]
+    if: ${{ needs.get-test-matrix.outputs.run_matrix != '' }}
     strategy:
       matrix: ${{ fromJSON(needs.get-test-matrix.outputs.run_matrix) }}
       fail-fast: false
-    if: ${{ !github.event.pull_request.draft }}
     uses: ./.github/workflows/_run-docker-compose.yml
     with:
       registry: "opea"
       tag: "ci"
       example: ${{ matrix.example }}
       hardware: ${{ matrix.hardware }}
-      diff_excluded_files: '.github|*.md|*.txt|kubernetes|manifest|gmc|assets|benchmark'
+      use_model_cache: true
+      diff_excluded_files: '\.github|\.md|\.txt|kubernetes|gmc|assets|benchmark'
     secrets: inherit

.github/workflows/pr-dockerfile-path-and-build-yaml-scan.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Clone Repo GenAIComps
         run: |
           cd ..
-          git clone https://github.com/opea-project/GenAIComps.git
+          git clone --depth 1 https://github.com/opea-project/GenAIComps.git
 
       - name: Check for Missing Dockerfile Paths in GenAIComps
         run: |
@@ -60,7 +60,7 @@ jobs:
           shopt -s globstar
           no_add="FALSE"
           cd ${{github.workspace}}
-          Dockerfiles=$(realpath $(find ./ -name '*Dockerfile*'))
+          Dockerfiles=$(realpath $(find ./ -name '*Dockerfile*' ! -path '*/tests/*'))
           if [ -n "$Dockerfiles" ]; then
             for dockerfile in $Dockerfiles; do
               service=$(echo "$dockerfile" | awk -F '/GenAIExamples/' '{print $2}' | awk -F '/' '{print $2}')

.github/workflows/pr-gmc-e2e.yaml.disabled

@@ -8,11 +8,10 @@ on:
     branches: ["main", "*rc"]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
-      - "**/kubernetes/**/gmc/**"
+      - "**/kubernetes/gmc/**"
       - "**/tests/test_gmc**"
       - "!**.md"
       - "!**.txt"
-      - "!**/kubernetes/**/manifest/**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -22,7 +21,7 @@ jobs:
   job1:
     uses: ./.github/workflows/_get-test-matrix.yml
     with:
-      diff_excluded_files: '.github|docker_compose|manifest|assets|*.md|*.txt'
+      diff_excluded_files: '\.github|docker_compose|assets|\.md|\.txt'
       test_mode: "gmc"
 
   gmc-test:

.github/workflows/pr-link-path-scan.yml

@@ -76,13 +76,7 @@ jobs:
           cd ${{github.workspace}}
           fail="FALSE"
           repo_name=${{ github.event.pull_request.head.repo.full_name }}
-          if [ "$(echo "$repo_name"|cut -d'/' -f1)" != "opea-project" ]; then
-            owner=$(echo "${{ github.event.pull_request.head.repo.full_name }}" |cut -d'/' -f1)
-            branch="https://github.com/$owner/GenAIExamples/tree/${{ github.event.pull_request.head.ref }}"
-          else
-            branch="https://github.com/opea-project/GenAIExamples/blob/${{ github.event.pull_request.head.ref }}"
-          fi
-          link_head="https://github.com/opea-project/GenAIExamples/blob/main"
+          branch="https://github.com/$repo_name/blob/${{ github.event.pull_request.head.ref }}"
 
           merged_commit=$(git log -1 --format='%H')
           changed_files="$(git diff --name-status --diff-filter=ARM ${{ github.event.pull_request.base.sha }} ${merged_commit} | awk '/\.md$/ {print $NF}')"

.github/workflows/pr-manifest-e2e.yml

@@ -1,42 +0,0 @@
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-
-name: E2E test with manifests
-
-on:
-  pull_request_target:
-    branches: ["main", "*rc"]
-    types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
-    paths:
-      - "**/Dockerfile**"
-      - "**.py"
-      - "**/kubernetes/**/manifest/**"
-      - "**/tests/test_manifest**"
-      - "!**.md"
-      - "!**.txt"
-      - "!**/kubernetes/**/gmc/**"
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  job1:
-    uses: ./.github/workflows/_get-test-matrix.yml
-    with:
-      diff_excluded_files: '.github|docker_compose|gmc|assets|*.md|*.txt|benchmark'
-      test_mode: "manifest"
-
-  run-example:
-    needs: job1
-    strategy:
-      matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
-      fail-fast: false
-    uses: ./.github/workflows/_example-workflow.yml
-    with:
-      node: ${{ matrix.hardware }}
-      example: ${{ matrix.example }}
-      tag: ${{ github.event.pull_request.head.sha }}
-      test_k8s: true
-    secrets: inherit

.github/workflows/push-image-build.yml

@@ -24,6 +24,7 @@ jobs:
 
   image-build:
     needs: job1
+    if: ${{ needs.job1.outputs.run_matrix != '{"include":[]}' }}
     strategy:
       matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
       fail-fast: false

.github/workflows/push-infra-issue-creation.yml

@@ -40,7 +40,7 @@ jobs:
       - name: Create Issue
         uses: daisy-ycguo/create-issue-action@stable
         with:
-          token: ${{ secrets.Infra_Issue_Token }}
+          token: ${{ secrets.ACTION_TOKEN }}
           owner: opea-project
           repo: GenAIInfra
           title: |
@@ -54,6 +54,6 @@ jobs:
 
             ${{ env.changed_files }}
 
-            Please verify if the helm charts and manifests need to be changed accordingly.
+            Please verify if the helm charts need to be changed accordingly.
 
             > This issue was created automatically by CI.

.github/workflows/scripts/check_duplicated_image.py

@@ -9,6 +9,8 @@ import sys
 import yaml
 
 images = {}
+dockerfiles = {}
+errors = []
 
 
 def check_docker_compose_build_definition(file_path):
@@ -30,18 +32,26 @@ def check_docker_compose_build_definition(file_path):
                 if not os.path.isfile(dockerfile):
                     # dockerfile not exists in the current repo context, assume it's in 3rd party context
                     dockerfile = os.path.normpath(os.path.join(context, build.get("dockerfile", "")))
-                item = {"file_path": file_path, "service": service, "dockerfile": dockerfile}
+                item = {"file_path": file_path, "service": service, "dockerfile": dockerfile, "image": image}
                 if image in images and dockerfile != images[image]["dockerfile"]:
-                    print("ERROR: !!! Found Conflicts !!!")
-                    print(f"Image: {image}, Dockerfile: {dockerfile}, defined in Service: {service}, File: {file_path}")
-                    print(
+                    errors.append(
+                        f"ERROR: !!! Found Conflicts !!!\n"
+                        f"Image: {image}, Dockerfile: {dockerfile}, defined in Service: {service}, File: {file_path}\n"
                         f"Image: {image}, Dockerfile: {images[image]['dockerfile']}, defined in Service: {images[image]['service']}, File: {images[image]['file_path']}"
                     )
-                    sys.exit(1)
                 else:
                     # print(f"Add Image: {image} Dockerfile: {dockerfile}")
                     images[image] = item
 
+                if dockerfile in dockerfiles and image != dockerfiles[dockerfile]["image"]:
+                    errors.append(
+                        f"WARNING: Different images using the same Dockerfile\n"
+                        f"Dockerfile: {dockerfile}, Image: {image}, defined in Service: {service}, File: {file_path}\n"
+                        f"Dockerfile: {dockerfile}, Image: {dockerfiles[dockerfile]['image']}, defined in Service: {dockerfiles[dockerfile]['service']}, File: {dockerfiles[dockerfile]['file_path']}"
+                    )
+                else:
+                    dockerfiles[dockerfile] = item
+
 
 def parse_arg():
     parser = argparse.ArgumentParser(
@@ -56,6 +66,12 @@ def main():
     for file_path in args.files:
         check_docker_compose_build_definition(file_path)
     print("SUCCESS: No Conlicts Found.")
+    if errors:
+        for error in errors:
+            print(error)
+        sys.exit(1)
+    else:
+        print("SUCCESS: No Conflicts Found.")
     return 0
 
 

.github/workflows/scripts/codeScan/hadolint.sh

@@ -7,7 +7,7 @@ source /GenAIExamples/.github/workflows/scripts/change_color
 log_dir=/GenAIExamples/.github/workflows/scripts/codeScan
 ERROR_WARN=false
 
-find . -type f \( -name "Dockerfile*" \) -print -exec hadolint --ignore DL3006 --ignore DL3007 --ignore DL3008 --ignore DL3013 {} \; > ${log_dir}/hadolint.log
+find . -type f \( -name "Dockerfile*" \) -print -exec hadolint --ignore DL3006 --ignore DL3007 --ignore DL3008 --ignore DL3013 --ignore DL3018 --ignore DL3016 {} \; > ${log_dir}/hadolint.log
 
 if [[ $(grep -c "error" ${log_dir}/hadolint.log) != 0 ]]; then
     $BOLD_RED && echo "Error!! Please Click on the artifact button to download and check error details." && $RESET

.github/workflows/scripts/docker_compose_clean_up.sh

@@ -26,14 +26,27 @@ case "$1" in
         echo "Release all ports used by the services in $yaml_file ..."
         pip install jq yq
         ports=$(yq '.services[].ports[] | split(":")[0]' $yaml_file | grep -o '[0-9a-zA-Z_-]\+')
+        echo "All ports list..."
         echo "$ports"
         for port in $ports; do
           if [[ $port =~ [a-zA-Z_-] ]]; then
-            port=$(grep -E "export $port=" tests/$test_case | cut -d'=' -f2)
+            echo "Search port value $port from the test case..."
+            port_fix=$(grep -E "export $port=" tests/$test_case | cut -d'=' -f2)
+            if [[ "$port_fix" == "" ]]; then
+              echo "Can't find the port value from the test case, use the default value in yaml..."
+              port_fix=$(yq '.services[].ports[]' $yaml_file | grep $port | cut -d':' -f2 |  grep -o '[0-9a-zA-Z]\+')
+            fi
+            port=$port_fix
+          fi
+          if [[ $port =~ [0-9] ]]; then
+            if [[ $port == 5000 ]]; then
+              echo "Error: Port 5000 is used by local docker registry, please DO NOT use it in docker compose deployment!!!"
+              exit 1
+            fi
+            echo "Check port $port..."
+            cid=$(docker ps --filter "publish=${port}" --format "{{.ID}}")
+            if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && echo "release $port"; fi
           fi
-          echo $port
-          cid=$(docker ps --filter "publish=${port}" --format "{{.ID}}")
-          if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
         done
         ;;
     *)

.github/workflows/scripts/get_test_matrix.sh

@@ -12,21 +12,25 @@ run_matrix="{\"include\":["
 
 examples=$(printf '%s\n' "${changed_files[@]}" | grep '/' | cut -d'/' -f1 | sort -u)
 for example in ${examples}; do
+    if [[ ! -d $WORKSPACE/$example ]]; then continue; fi
     cd $WORKSPACE/$example
     if [[ ! $(find . -type f | grep ${test_mode}) ]]; then continue; fi
     cd tests
     ls -l
     if [[ "$test_mode" == "docker_image_build" ]]; then
-        find_name="test_manifest_on_*.sh"
+        hardware_list="gaudi xeon"
     else
         find_name="test_${test_mode}*_on_*.sh"
+        hardware_list=$(find . -type f -name "${find_name}" | cut -d/ -f2 | cut -d. -f1 | awk -F'_on_' '{print $2}'| sort -u)
     fi
-    hardware_list=$(find . -type f -name "${find_name}" | cut -d/ -f2 | cut -d. -f1 | awk -F'_on_' '{print $2}'| sort -u)
     echo -e "Test supported hardware list: \n${hardware_list}"
 
     run_hardware=""
-    if [[ $(printf '%s\n' "${changed_files[@]}" | grep ${example} | cut -d'/' -f2 | grep -E '*.py|Dockerfile*|ui|docker_image_build' ) ]]; then
-        # run test on all hardware if megaservice or ui code change
+    if [[ $(printf '%s\n' "${changed_files[@]}" | grep ${example} | cut -d'/' -f2 | grep -E '\.py|Dockerfile*|ui|docker_image_build' ) ]]; then
+        echo "run test on all hardware if megaservice or ui code change..."
+        run_hardware=$hardware_list
+    elif [[ $(printf '%s\n' "${changed_files[@]}" | grep ${example} | grep 'tests'| cut -d'/' -f3 | grep -vE '^test_|^_test' ) ]]; then
+        echo "run test on all hardware if common test scripts change..."
         run_hardware=$hardware_list
     else
         for hardware in ${hardware_list}; do

.github/workflows/scripts/k8s-utils.sh

@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-#set -xe
+set -e
 
 function dump_pod_log() {
     pod_name=$1
@@ -12,7 +12,7 @@ function dump_pod_log() {
     kubectl describe pod $pod_name -n $namespace
     echo "-----------------------------------"
     echo "#kubectl logs $pod_name -n $namespace"
-    kubectl logs $pod_name -n $namespace
+    kubectl logs $pod_name -n $namespace --all-containers --prefix=true
     echo "-----------------------------------"
 }
 
@@ -44,8 +44,13 @@ function dump_pods_status() {
 
 function dump_all_pod_logs() {
     namespace=$1
+    echo "------SUMMARY of POD STATUS in NS $namespace------"
+    kubectl get pods -n $namespace -o wide
+    echo "------SUMMARY of SVC STATUS in NS $namespace------"
+    kubectl get services -n $namespace -o wide
+    echo "------SUMMARY of endpoint STATUS in NS $namespace------"
+    kubectl get endpoints -n $namespace -o wide
     echo "-----DUMP POD STATUS AND LOG in NS $namespace------"
-
     pods=$(kubectl get pods -n $namespace -o jsonpath='{.items[*].metadata.name}')
     for pod_name in $pods
     do

.github/workflows/weekly-example-test.yml

@@ -0,0 +1,55 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Weekly test all examples on multiple HWs
+
+on:
+  schedule:
+    - cron: "30 2 * * 6" # UTC time
+  workflow_dispatch:
+
+env:
+  EXAMPLES: ${{ vars.NIGHTLY_RELEASE_EXAMPLES }}
+  NODES: "gaudi,xeon,rocm,arc"
+
+jobs:
+  get-test-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      examples: ${{ steps.get-matrix.outputs.examples }}
+      nodes: ${{ steps.get-matrix.outputs.nodes }}
+    steps:
+      - name: Create Matrix
+        id: get-matrix
+        run: |
+          examples=($(echo ${EXAMPLES} | tr ',' ' '))
+          examples_json=$(printf '%s\n' "${examples[@]}" | sort -u | jq -R '.' | jq -sc '.')
+          echo "examples=$examples_json" >> $GITHUB_OUTPUT
+          nodes=($(echo ${NODES} | tr ',' ' '))
+          nodes_json=$(printf '%s\n' "${nodes[@]}" | sort -u | jq -R '.' | jq -sc '.')
+          echo "nodes=$nodes_json" >> $GITHUB_OUTPUT
+
+  build-comps-base:
+    needs: [get-test-matrix]
+    strategy:
+      matrix:
+        node: ${{ fromJson(needs.get-test-matrix.outputs.nodes) }}
+    uses: ./.github/workflows/_build_comps_base_image.yml
+    with:
+      node: ${{ matrix.node }}
+
+  run-examples:
+    needs: [get-test-matrix, build-comps-base]
+    strategy:
+      matrix:
+        example: ${{ fromJson(needs.get-test-matrix.outputs.examples) }}
+        node: ${{ fromJson(needs.get-test-matrix.outputs.nodes) }}
+      fail-fast: false
+    uses: ./.github/workflows/_example-workflow.yml
+    with:
+      node: ${{ matrix.node }}
+      example: ${{ matrix.example }}
+      build: true
+      test_compose: true
+      test_helmchart: true
+    secrets: inherit

.github/workflows/weekly-update-images.yml

@@ -1,11 +1,9 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-name: Weekly update base images and 3rd party images
+name: Weekly update 3rd party images
 
 on:
-  schedule:
-    - cron: "0 0 * * 0"
   workflow_dispatch:
 
 permissions:
@@ -16,8 +14,8 @@ jobs:
   freeze-images:
     runs-on: ubuntu-latest
     env:
-      USER_NAME: "NeuralChatBot"
-      USER_EMAIL: "grp_neural_chat_bot@intel.com"
+      USER_NAME: "CICD-at-OPEA"
+      USER_EMAIL: "CICD@opea.dev"
       BRANCH_NAME: "update_images_tag"
     steps:
       - name: Checkout repository

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ ci:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: end-of-file-fixer
         files: (.*\.(py|md|rst|yaml|yml|json|ts|js|html|svelte|sh))$
@@ -74,7 +74,7 @@ repos:
         name: Unused noqa
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.13.2
+    rev: 6.0.1
     hooks:
       - id: isort
 
@@ -100,21 +100,21 @@ repos:
           - prettier@3.2.5
 
   - repo: https://github.com/psf/black.git
-    rev: 24.4.2
+    rev: 25.1.0
     hooks:
       - id: black
         files: (.*\.py)$
 
   - repo: https://github.com/asottile/blacken-docs
-    rev: 1.18.0
+    rev: 1.19.1
     hooks:
       - id: blacken-docs
         args: [--line-length=120, --skip-errors]
         additional_dependencies:
-          - black==24.4.2
+          - black==24.10.0
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.3.0
+    rev: v2.4.1
     hooks:
       - id: codespell
         args: [-w]
@@ -122,7 +122,7 @@ repos:
           - tomli
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.0
+    rev: v0.11.4
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix, --no-cache]

AgentQnA/README.md

@@ -1,9 +1,18 @@
 # Agents for Question Answering
 
+## Table of contents
+
+1. [Overview](#overview)
+2. [Deploy with Docker](#deploy-with-docker)
+3. [How to interact with the agent system with UI](#how-to-interact-with-the-agent-system-with-ui)
+4. [Validate Services](#validate-services)
+5. [Register Tools](#how-to-register-other-tools-with-the-ai-agent)
+6. [Monitoring and Tracing](#monitor-and-tracing)
+
 ## Overview
 
-This example showcases a hierarchical multi-agent system for question-answering applications. The architecture diagram is shown below. The supervisor agent interfaces with the user and dispatch tasks to the worker agent and other tools to gather information and come up with answers. The worker agent uses the retrieval tool to generate answers to the queries posted by the supervisor agent. Other tools used by the supervisor agent may include APIs to interface knowledge graphs, SQL databases, external knowledge bases, etc.
-![Architecture Overview](assets/agent_qna_arch.png)
+This example showcases a hierarchical multi-agent system for question-answering applications. The architecture diagram below shows a supervisor agent that interfaces with the user and dispatches tasks to two worker agents to gather information and come up with answers. The worker RAG agent uses the retrieval tool to retrieve relevant documents from a knowledge base - a vector database. The worker SQL agent retrieves relevant data from a SQL database. Although not included in this example by default, other tools such as a web search tool or a knowledge graph query tool can be used by the supervisor agent to gather information from additional sources.
+![Architecture Overview](assets/img/agent_qna_arch.png)
 
 The AgentQnA example is implemented using the component-level microservices defined in [GenAIComps](https://github.com/opea-project/GenAIComps). The flow chart below shows the information flow between different microservices for this example.
 
@@ -38,6 +47,7 @@ flowchart LR
     end
     AG_REACT([Agent MicroService - react]):::blue
     AG_RAG([Agent MicroService - rag]):::blue
+    AG_SQL([Agent MicroService - sql]):::blue
     LLM_gen{{LLM Service <br>}}
     DP([Data Preparation MicroService]):::blue
     TEI_RER{{Reranking service<br>}}
@@ -51,6 +61,7 @@ flowchart LR
     direction LR
     a[User Input Query] --> AG_REACT
     AG_REACT --> AG_RAG
+    AG_REACT --> AG_SQL
     AG_RAG --> DocIndexRetriever-MegaService
     EM ==> RET
     RET ==> RER
@@ -59,6 +70,7 @@ flowchart LR
     %% Embedding service flow
     direction LR
     AG_RAG <-.-> LLM_gen
+    AG_SQL <-.-> LLM_gen
     AG_REACT <-.-> LLM_gen
     EM <-.-> TEI_EM
     RET <-.-> R_RET
@@ -72,152 +84,178 @@ flowchart LR
 
 ```
 
-### Why Agent for question answering?
+### Why should AI Agents be used for question-answering?
 
-1. Improve relevancy of retrieved context.
-   Agent can rephrase user queries, decompose user queries, and iterate to get the most relevant context for answering user's questions. Compared to conventional RAG, RAG agent can significantly improve the correctness and relevancy of the answer.
-2. Use tools to get additional knowledge.
-   For example, knowledge graphs and SQL databases can be exposed as APIs for Agents to gather knowledge that may be missing in the retrieval vector database.
-3. Hierarchical agent can further improve performance.
-   Expert worker agents, such as retrieval agent, knowledge graph agent, SQL agent, etc., can provide high-quality output for different aspects of a complex query, and the supervisor agent can aggregate the information together to provide a comprehensive answer.
+1. **Improve relevancy of retrieved context.**
+   RAG agents can rephrase user queries, decompose user queries, and iterate to get the most relevant context for answering a user's question. Compared to conventional RAG, RAG agents significantly improve the correctness and relevancy of the answer because of the iterations it goes through.
+2. **Expand scope of skills.**
+   The supervisor agent interacts with multiple worker agents that specialize in different skills (e.g., retrieve documents, write SQL queries, etc.). Thus, it can answer questions with different methods.
+3. **Hierarchical multi-agents improve performance.**
+   Expert worker agents, such as RAG agents and SQL agents, can provide high-quality output for different aspects of a complex query, and the supervisor agent can aggregate the information to provide a comprehensive answer. If only one agent is used and all tools are provided to this single agent, it can lead to large overhead or not use the best tool to provide accurate answers.
 
-## Deployment with docker
+## Deploy with docker
 
-1. Build agent docker image [Optional]
+### 1. Set up environment </br>
 
-> [!NOTE]
-> the step is optional. The docker images will be automatically pulled when running the docker compose commands. This step is only needed if pulling images failed.
-
-First, clone the opea GenAIComps repo.
+#### First, clone the `GenAIExamples` repo.
 
 ```
 export WORKDIR=<your-work-directory>
 cd $WORKDIR
-git clone https://github.com/opea-project/GenAIComps.git
+git clone https://github.com/opea-project/GenAIExamples.git
 ```
 
-Then build the agent docker image. Both the supervisor agent and the worker agent will use the same docker image, but when we launch the two agents we will specify different strategies and register different tools.
+#### Second, set up environment variables.
+
+##### For proxy environments only
 
 ```
-cd GenAIComps
-docker build -t opea/agent-langchain:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/agent/langchain/Dockerfile .
+export http_proxy="Your_HTTP_Proxy"
+export https_proxy="Your_HTTPs_Proxy"
+# Example: no_proxy="localhost, 127.0.0.1, 192.168.1.1"
+export no_proxy="Your_No_Proxy"
 ```
 
-2. Set up environment for this example </br>
-
-   First, clone this repo.
-
-   ```
-   export WORKDIR=<your-work-directory>
-   cd $WORKDIR
-   git clone https://github.com/opea-project/GenAIExamples.git
-   ```
-
-   Second, set up env vars.
-
-   ```
-   # Example: host_ip="192.168.1.1" or export host_ip="External_Public_IP"
-   export host_ip=$(hostname -I | awk '{print $1}')
-   # if you are in a proxy environment, also set the proxy-related environment variables
-   export http_proxy="Your_HTTP_Proxy"
-   export https_proxy="Your_HTTPs_Proxy"
-   # Example: no_proxy="localhost, 127.0.0.1, 192.168.1.1"
-   export no_proxy="Your_No_Proxy"
-
-   export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
-   # for using open-source llms
-   export HUGGINGFACEHUB_API_TOKEN=<your-HF-token>
-   export HF_CACHE_DIR=<directory-where-llms-are-downloaded> #so that no need to redownload every time
-
-   # optional: OPANAI_API_KEY if you want to use OpenAI models
-   export OPENAI_API_KEY=<your-openai-key>
-   ```
-
-3. Deploy the retrieval tool (i.e., DocIndexRetriever mega-service)
-
-   First, launch the mega-service.
-
-   ```
-   cd $WORKDIR/GenAIExamples/AgentQnA/retrieval_tool
-   bash launch_retrieval_tool.sh
-   ```
-
-   Then, ingest data into the vector database. Here we provide an example. You can ingest your own data.
-
-   ```
-   bash run_ingest_data.sh
-   ```
-
-4. Launch other tools. </br>
-   In this example, we will use some of the mock APIs provided in the Meta CRAG KDD Challenge to demonstrate the benefits of gaining additional context from mock knowledge graphs.
-
-   ```
-   docker run -d -p=8080:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0
-   ```
-
-5. Launch agent services</br>
-   We provide two options for `llm_engine` of the agents: 1. open-source LLMs, 2. OpenAI models via API calls.
-
-   Deploy it on Gaudi or Xeon respectively
-
-   ::::{tab-set}
-   :::{tab-item} Gaudi
-   :sync: Gaudi
-
-   To use open-source LLMs on Gaudi2, run commands below.
-
-   ```
-   cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/hpu/gaudi
-   bash launch_tgi_gaudi.sh
-   bash launch_agent_service_tgi_gaudi.sh
-   ```
-
-   :::
-   :::{tab-item} Xeon
-   :sync: Xeon
-
-   To use OpenAI models, run commands below.
-
-   ```
-   cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon
-   bash launch_agent_service_openai.sh
-   ```
-
-   :::
-   ::::
-
-## Validate services
-
-First look at logs of the agent docker containers:
+##### For using open-source llms
 
 ```
-# worker agent
+export HUGGINGFACEHUB_API_TOKEN=<your-HF-token>
+export HF_CACHE_DIR=<directory-where-llms-are-downloaded> #so that no need to redownload every time
+```
+
+##### [Optional] OPANAI_API_KEY to use OpenAI models
+
+```
+export OPENAI_API_KEY=<your-openai-key>
+```
+
+#### Third, set up environment variables for the selected hardware using the corresponding `set_env.sh`
+
+##### Gaudi
+
+```
+source $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/hpu/gaudi/set_env.sh
+```
+
+##### Xeon
+
+```
+source $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon/set_env.sh
+```
+
+### 2. Launch the multi-agent system. </br>
+
+We make it convenient to launch the whole system with docker compose, which includes microservices for LLM, agents, UI, retrieval tool, vector database, dataprep, and telemetry. There are 3 docker compose files, which make it easy for users to pick and choose. Users can choose a different retrieval tool other than the `DocIndexRetriever` example provided in our GenAIExamples repo. Users can choose not to launch the telemetry containers.
+
+#### Launch on Gaudi
+
+On Gaudi, `meta-llama/Meta-Llama-3.3-70B-Instruct` will be served using vllm. The command below will launch the multi-agent system with the `DocIndexRetriever` as the retrieval tool for the Worker RAG agent.
+
+```bash
+cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/hpu/gaudi/
+docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml -f compose.yaml up -d
+```
+
+> **Note**: To enable the web search tool, skip this step and proceed to the "[Optional] Web Search Tool Support" section.
+
+To enable Open Telemetry Tracing, compose.telemetry.yaml file need to be merged along with default compose.yaml file.
+Gaudi example with Open Telemetry feature:
+
+```bash
+cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/hpu/gaudi/
+docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml -f compose.yaml -f compose.telemetry.yaml up -d
+```
+
+##### [Optional] Web Search Tool Support
+
+<details>
+<summary> Instructions </summary>
+A web search tool is supported in this example and can be enabled by running docker compose with the `compose.webtool.yaml` file.
+The Google Search API is used. Follow the [instructions](https://python.langchain.com/docs/integrations/tools/google_search) to create an API key and enable the Custom Search API on a Google account. The environment variables `GOOGLE_CSE_ID` and `GOOGLE_API_KEY` need to be set.
+
+```bash
+cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/hpu/gaudi/
+export GOOGLE_CSE_ID="YOUR_ID"
+export GOOGLE_API_KEY="YOUR_API_KEY"
+docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml -f compose.yaml -f compose.webtool.yaml up -d
+```
+
+</details>
+
+#### Launch on Xeon
+
+On Xeon, only OpenAI models are supported. The command below will launch the multi-agent system with the `DocIndexRetriever` as the retrieval tool for the Worker RAG agent.
+
+```bash
+export OPENAI_API_KEY=<your-openai-key>
+cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon
+docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml -f compose_openai.yaml up -d
+```
+
+### 3. Ingest Data into the vector database
+
+The `run_ingest_data.sh` script will use an example jsonl file to ingest example documents into a vector database. Other ways to ingest data and other types of documents supported can be found in the OPEA dataprep microservice located in the opea-project/GenAIComps repo.
+
+```bash
+cd  $WORKDIR/GenAIExamples/AgentQnA/retrieval_tool/
+bash run_ingest_data.sh
+```
+
+> **Note**: This is a one-time operation.
+
+## How to interact with the agent system with UI
+
+The UI microservice is launched in the previous step with the other microservices.
+To see the UI, open a web browser to `http://${ip_address}:5173` to access the UI. Note the `ip_address` here is the host IP of the UI microservice.
+
+1. `create Admin Account` with a random value
+2. add opea agent endpoint `http://$ip_address:9090/v1` which is a openai compatible api
+
+![opea-agent-setting](assets/img/opea-agent-setting.png)
+
+3. test opea agent with ui
+
+![opea-agent-test](assets/img/opea-agent-test.png)
+
+## [Optional] Deploy using Helm Charts
+
+Refer to the [AgentQnA helm chart](./kubernetes/helm/README.md) for instructions on deploying AgentQnA on Kubernetes.
+
+## Validate Services
+
+1. First look at logs for each of the agent docker containers:
+
+```bash
+# worker RAG agent
 docker logs rag-agent-endpoint
-```
 
-```
+# worker SQL agent
+docker logs sql-agent-endpoint
+
 # supervisor agent
 docker logs react-agent-endpoint
 ```
 
-You should see something like "HTTP server setup successful" if the docker containers are started successfully.</p>
+Look for the message "HTTP server setup successful" to confirm the agent docker container has started successfully.</p>
 
-Second, validate worker agent:
+2. Use python to validate each agent is working properly:
 
-```
-curl http://${host_ip}:9095/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
-     "query": "Most recent album by Taylor Swift"
-    }'
+```bash
+# RAG worker agent
+python $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "Tell me about Michael Jackson song Thriller" --agent_role "worker" --ext_port 9095
+
+# SQL agent
+python $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "How many employees in company" --agent_role "worker" --ext_port 9096
+
+# supervisor agent: this will test a two-turn conversation
+python $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --agent_role "supervisor" --ext_port 9090
 ```
 
-Third, validate supervisor agent:
+## How to register other tools with the AI agent
 
-```
-curl http://${host_ip}:9090/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
-     "query": "Most recent album by Taylor Swift"
-    }'
-```
+The [tools](./tools) folder contains YAML and Python files for additional tools for the supervisor and worker agents. Refer to the "Provide your own tools" section in the instructions [here](https://github.com/opea-project/GenAIComps/tree/main/comps/agent/src/README.md) to add tools and customize the AI agents.
 
-## How to register your own tools with agent
+## Monitor and Tracing
 
-You can take a look at the tools yaml and python files in this example. For more details, please refer to the "Provide your own tools" section in the instructions [here](https://github.com/opea-project/GenAIComps/tree/main/comps/agent/langchain/README.md).
+Follow [OpenTelemetry OPEA Guide](https://opea-project.github.io/latest/tutorial/OpenTelemetry/OpenTelemetry_OPEA_Guide.html) to understand how to use OpenTelemetry tracing and metrics in OPEA.  
+For AgentQnA specific tracing and metrics monitoring, follow [OpenTelemetry on AgentQnA](https://opea-project.github.io/latest/tutorial/OpenTelemetry/deploy/AgentQnA.html) section.

AgentQnA/docker_compose/amd/gpu/rocm/README.md

@@ -0,0 +1,342 @@
+# Build Mega Service of AgentQnA on AMD ROCm GPU
+
+## Build Docker Images
+
+### 1. Build Docker Image
+
+- #### Create application install directory and go to it:
+
+  ```bash
+  mkdir ~/agentqna-install && cd agentqna-install
+  ```
+
+- #### Clone the repository GenAIExamples (the default repository branch "main" is used here):
+
+  ```bash
+  git clone https://github.com/opea-project/GenAIExamples.git
+  ```
+
+  If you need to use a specific branch/tag of the GenAIExamples repository, then (v1.3 replace with its own value):
+
+  ```bash
+  git clone https://github.com/opea-project/GenAIExamples.git && cd GenAIExamples && git checkout v1.3
+  ```
+
+  We remind you that when using a specific version of the code, you need to use the README from this version:
+
+- #### Go to build directory:
+
+  ```bash
+  cd ~/agentqna-install/GenAIExamples/AgentQnA/docker_image_build
+  ```
+
+- Cleaning up the GenAIComps repository if it was previously cloned in this directory.
+  This is necessary if the build was performed earlier and the GenAIComps folder exists and is not empty:
+
+  ```bash
+  echo Y | rm -R GenAIComps
+  ```
+
+- #### Clone the repository GenAIComps (the default repository branch "main" is used here):
+
+```bash
+git clone https://github.com/opea-project/GenAIComps.git
+```
+
+We remind you that when using a specific version of the code, you need to use the README from this version.
+
+- #### Setting the list of images for the build (from the build file.yaml)
+
+  If you want to deploy a vLLM-based or TGI-based application, then the set of services is installed as follows:
+
+  #### vLLM-based application
+
+  ```bash
+  service_list="vllm-rocm agent agent-ui"
+  ```
+
+  #### TGI-based application
+
+  ```bash
+  service_list="agent agent-ui"
+  ```
+
+- #### Optional. Pull TGI Docker Image (Do this if you want to use TGI)
+
+  ```bash
+  docker pull ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
+  ```
+
+- #### Build Docker Images
+
+  ```bash
+  docker compose -f build.yaml build ${service_list} --no-cache
+  ```
+
+- #### Build DocIndexRetriever Docker Images
+
+  ```bash
+  cd ~/agentqna-install/GenAIExamples/DocIndexRetriever/docker_image_build/
+  git clone https://github.com/opea-project/GenAIComps.git
+  service_list="doc-index-retriever dataprep embedding retriever reranking"
+  docker compose -f build.yaml build ${service_list} --no-cache
+  ```
+
+- #### Pull DocIndexRetriever Docker Images
+
+  ```bash
+  docker pull redis/redis-stack:7.2.0-v9
+  docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+  ```
+
+  After the build, we check the list of images with the command:
+
+  ```bash
+  docker image ls
+  ```
+
+  The list of images should include:
+
+  ##### vLLM-based application:
+
+  - opea/vllm-rocm:latest
+  - opea/agent:latest
+  - redis/redis-stack:7.2.0-v9
+  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+  - opea/embedding:latest
+  - opea/retriever:latest
+  - opea/reranking:latest
+  - opea/doc-index-retriever:latest
+
+  ##### TGI-based application:
+
+  - ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
+  - opea/agent:latest
+  - redis/redis-stack:7.2.0-v9
+  - ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
+  - opea/embedding:latest
+  - opea/retriever:latest
+  - opea/reranking:latest
+  - opea/doc-index-retriever:latest
+
+---
+
+## Deploy the AgentQnA Application
+
+### Docker Compose Configuration for AMD GPUs
+
+To enable GPU support for AMD GPUs, the following configuration is added to the Docker Compose file:
+
+- compose_vllm.yaml - for vLLM-based application
+- compose.yaml - for TGI-based
+
+```yaml
+shm_size: 1g
+devices:
+  - /dev/kfd:/dev/kfd
+  - /dev/dri:/dev/dri
+cap_add:
+  - SYS_PTRACE
+group_add:
+  - video
+security_opt:
+  - seccomp:unconfined
+```
+
+This configuration forwards all available GPUs to the container. To use a specific GPU, specify its `cardN` and `renderN` device IDs. For example:
+
+```yaml
+shm_size: 1g
+devices:
+  - /dev/kfd:/dev/kfd
+  - /dev/dri/card0:/dev/dri/card0
+  - /dev/dri/render128:/dev/dri/render128
+cap_add:
+  - SYS_PTRACE
+group_add:
+  - video
+security_opt:
+  - seccomp:unconfined
+```
+
+**How to Identify GPU Device IDs:**
+Use AMD GPU driver utilities to determine the correct `cardN` and `renderN` IDs for your GPU.
+
+### Set deploy environment variables
+
+#### Setting variables in the operating system environment:
+
+```bash
+### Replace the string 'server_address' with your local server IP address
+export host_ip='server_address'
+### Replace the string 'your_huggingfacehub_token' with your HuggingFacehub repository access token.
+export HUGGINGFACEHUB_API_TOKEN='your_huggingfacehub_token'
+### Replace the string 'your_langchain_api_key' with your LANGCHAIN API KEY.
+export LANGCHAIN_API_KEY='your_langchain_api_key'
+export LANGCHAIN_TRACING_V2=""
+```
+
+### Start the services:
+
+#### If you use vLLM
+
+```bash
+cd ~/agentqna-install/GenAIExamples/AgentQnA/docker_compose/amd/gpu/rocm
+bash launch_agent_service_vllm_rocm.sh
+```
+
+#### If you use TGI
+
+```bash
+cd ~/agentqna-install/GenAIExamples/AgentQnA/docker_compose/amd/gpu/rocm
+bash launch_agent_service_tgi_rocm.sh
+```
+
+All containers should be running and should not restart:
+
+##### If you use vLLM:
+
+- dataprep-redis-server
+- doc-index-retriever-server
+- embedding-server
+- rag-agent-endpoint
+- react-agent-endpoint
+- redis-vector-db
+- reranking-tei-xeon-server
+- retriever-redis-server
+- sql-agent-endpoint
+- tei-embedding-server
+- tei-reranking-server
+- vllm-service
+
+##### If you use TGI:
+
+- dataprep-redis-server
+- doc-index-retriever-server
+- embedding-server
+- rag-agent-endpoint
+- react-agent-endpoint
+- redis-vector-db
+- reranking-tei-xeon-server
+- retriever-redis-server
+- sql-agent-endpoint
+- tei-embedding-server
+- tei-reranking-server
+- tgi-service
+
+---
+
+## Validate the Services
+
+### 1. Validate the vLLM/TGI Service
+
+#### If you use vLLM:
+
+```bash
+DATA='{"model": "Intel/neural-chat-7b-v3-3t", '\
+'"messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 256}'
+
+curl http://${HOST_IP}:${VLLM_SERVICE_PORT}/v1/chat/completions \
+  -X POST \
+  -d "$DATA" \
+  -H 'Content-Type: application/json'
+```
+
+Checking the response from the service. The response should be similar to JSON:
+
+```json
+{
+  "id": "chatcmpl-142f34ef35b64a8db3deedd170fed951",
+  "object": "chat.completion",
+  "created": 1742270316,
+  "model": "Intel/neural-chat-7b-v3-3",
+  "choices": [
+    {
+      "index": 0,
+      "message": {
+        "role": "assistant",
+        "content": "",
+        "tool_calls": []
+      },
+      "logprobs": null,
+      "finish_reason": "length",
+      "stop_reason": null
+    }
+  ],
+  "usage": { "prompt_tokens": 66, "total_tokens": 322, "completion_tokens": 256, "prompt_tokens_details": null },
+  "prompt_logprobs": null
+}
+```
+
+If the service response has a meaningful response in the value of the "choices.message.content" key,
+then we consider the vLLM service to be successfully launched
+
+#### If you use TGI:
+
+```bash
+DATA='{"inputs":"What is Deep Learning?",'\
+'"parameters":{"max_new_tokens":256,"do_sample": true}}'
+
+curl http://${HOST_IP}:${TGI_SERVICE_PORT}/generate \
+  -X POST \
+  -d "$DATA" \
+  -H 'Content-Type: application/json'
+```
+
+Checking the response from the service. The response should be similar to JSON:
+
+```json
+{
+  "generated_text": " "
+}
+```
+
+If the service response has a meaningful response in the value of the "generated_text" key,
+then we consider the TGI service to be successfully launched
+
+### 2. Validate Agent Services
+
+#### Validate Rag Agent Service
+
+```bash
+export agent_port=${WORKER_RAG_AGENT_PORT}
+prompt="Tell me about Michael Jackson song Thriller"
+python3 ~/agentqna-install/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt" --agent_role "worker" --ext_port $agent_port
+```
+
+The response must contain the meaningful text of the response to the request from the "prompt" variable
+
+#### Validate Sql Agent Service
+
+```bash
+export agent_port=${WORKER_SQL_AGENT_PORT}
+prompt="How many employees are there in the company?"
+python3 ~/agentqna-install/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt" --agent_role "worker" --ext_port $agent_port
+```
+
+The answer should make sense - "8 employees in the company"
+
+#### Validate React (Supervisor) Agent Service
+
+```bash
+export agent_port=${SUPERVISOR_REACT_AGENT_PORT}
+python3 ~/agentqna-install/GenAIExamples/AgentQnA/tests/test.py --agent_role "supervisor" --ext_port $agent_port --stream
+```
+
+The response should contain "Iron Maiden"
+
+### 3. Stop application
+
+#### If you use vLLM
+
+```bash
+cd ~/agentqna-install/GenAIExamples/AgentQnA/docker_compose/amd/gpu/rocm
+bash stop_agent_service_vllm_rocm.sh
+```
+
+#### If you use TGI
+
+```bash
+cd ~/agentqna-install/GenAIExamples/AgentQnA/docker_compose/amd/gpu/rocm
+bash stop_agent_service_tgi_rocm.sh
+```

AgentQnA/docker_compose/amd/gpu/rocm/compose.yaml

@@ -1,26 +1,24 @@
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2025 Advanced Micro Devices, Inc.
 
 services:
-  agent-tgi-server:
-    image: ${AGENTQNA_TGI_IMAGE}
-    container_name: agent-tgi-server
+  tgi-service:
+    image: ghcr.io/huggingface/text-generation-inference:3.0.0-rocm
+    container_name: tgi-service
     ports:
-      - "${AGENTQNA_TGI_SERVICE_PORT-8085}:80"
+      - "${TGI_SERVICE_PORT-8085}:80"
     volumes:
-      - /var/opea/agent-service/:/data
+      - "${MODEL_CACHE:-./data}:/data"
     environment:
       no_proxy: ${no_proxy}
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
-      TGI_LLM_ENDPOINT: "http://${HOST_IP}:${AGENTQNA_TGI_SERVICE_PORT}"
+      TGI_LLM_ENDPOINT: "http://${ip_address}:${TGI_SERVICE_PORT}"
       HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
-    shm_size: 1g
+    shm_size: 32g
     devices:
       - /dev/kfd:/dev/kfd
-      - /dev/dri/${AGENTQNA_CARD_ID}:/dev/dri/${AGENTQNA_CARD_ID}
-      - /dev/dri/${AGENTQNA_RENDER_ID}:/dev/dri/${AGENTQNA_RENDER_ID}
+      - /dev/dri:/dev/dri
     cap_add:
       - SYS_PTRACE
     group_add:
@@ -31,17 +29,17 @@ services:
     command: --model-id ${LLM_MODEL_ID} --max-input-length 4096 --max-total-tokens 8192
 
   worker-rag-agent:
-    image: opea/agent-langchain:latest
+    image: ${REGISTRY:-opea}/agent:${TAG:-latest}
     container_name: rag-agent-endpoint
     volumes:
-      # - ${WORKDIR}/GenAIExamples/AgentQnA/docker_image_build/GenAIComps/comps/agent/langchain/:/home/user/comps/agent/langchain/
-      - ${TOOLSET_PATH}:/home/user/tools/
+      - "${TOOLSET_PATH}:/home/user/tools/"
     ports:
-      - "9095:9095"
+      - "${WORKER_RAG_AGENT_PORT:-9095}:9095"
     ipc: host
     environment:
       ip_address: ${ip_address}
       strategy: rag_agent_llama
+      with_memory: false
       recursion_limit: ${recursion_limit_worker}
       llm_engine: tgi
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
@@ -49,7 +47,7 @@ services:
       model: ${LLM_MODEL_ID}
       temperature: ${temperature}
       max_new_tokens: ${max_new_tokens}
-      streaming: false
+      stream: false
       tools: /home/user/tools/worker_agent_tools.yaml
       require_human_feedback: false
       RETRIEVAL_TOOL_URL: ${RETRIEVAL_TOOL_URL}
@@ -61,21 +59,49 @@ services:
       LANGCHAIN_PROJECT: "opea-worker-agent-service"
       port: 9095
 
-  supervisor-react-agent:
-    image: opea/agent-langchain:latest
-    container_name: react-agent-endpoint
-    depends_on:
-      - agent-tgi-server
-      - worker-rag-agent
+  worker-sql-agent:
+    image: ${REGISTRY:-opea}/agent:${TAG:-latest}
+    container_name: sql-agent-endpoint
     volumes:
-      # - ${WORKDIR}/GenAIExamples/AgentQnA/docker_image_build/GenAIComps/comps/agent/langchain/:/home/user/comps/agent/langchain/
-      - ${TOOLSET_PATH}:/home/user/tools/
+      - "${WORKDIR}/tests/Chinook_Sqlite.sqlite:/home/user/chinook-db/Chinook_Sqlite.sqlite:rw"
     ports:
-      - "${AGENTQNA_FRONTEND_PORT}:9090"
+      - "${WORKER_SQL_AGENT_PORT:-9096}:9096"
     ipc: host
     environment:
       ip_address: ${ip_address}
-      strategy: react_langgraph
+      strategy: sql_agent_llama
+      with_memory: false
+      db_name: ${db_name}
+      db_path: ${db_path}
+      use_hints: false
+      recursion_limit: ${recursion_limit_worker}
+      llm_engine: vllm
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      llm_endpoint_url: ${LLM_ENDPOINT_URL}
+      model: ${LLM_MODEL_ID}
+      temperature: ${temperature}
+      max_new_tokens: ${max_new_tokens}
+      stream: false
+      require_human_feedback: false
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      port: 9096
+
+  supervisor-react-agent:
+    image: ${REGISTRY:-opea}/agent:${TAG:-latest}
+    container_name: react-agent-endpoint
+    depends_on:
+      - worker-rag-agent
+    volumes:
+      - "${TOOLSET_PATH}:/home/user/tools/"
+    ports:
+      - "${SUPERVISOR_REACT_AGENT_PORT:-9090}:9090"
+    ipc: host
+    environment:
+      ip_address: ${ip_address}
+      strategy: react_llama
+      with_memory: true
       recursion_limit: ${recursion_limit_supervisor}
       llm_engine: tgi
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
@@ -83,7 +109,7 @@ services:
       model: ${LLM_MODEL_ID}
       temperature: ${temperature}
       max_new_tokens: ${max_new_tokens}
-      streaming: false
+      stream: true
       tools: /home/user/tools/supervisor_agent_tools.yaml
       require_human_feedback: false
       no_proxy: ${no_proxy}
@@ -92,6 +118,7 @@ services:
       LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
       LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2}
       LANGCHAIN_PROJECT: "opea-supervisor-agent-service"
-      CRAG_SERVER: $CRAG_SERVER
-      WORKER_AGENT_URL: $WORKER_AGENT_URL
+      CRAG_SERVER: ${CRAG_SERVER}
+      WORKER_AGENT_URL: ${WORKER_AGENT_URL}
+      SQL_AGENT_URL: ${SQL_AGENT_URL}
       port: 9090

AgentQnA/docker_compose/amd/gpu/rocm/compose_vllm.yaml

@@ -0,0 +1,128 @@
+# Copyright (C) 2025 Advanced Micro Devices, Inc.
+
+services:
+  vllm-service:
+    image: ${REGISTRY:-opea}/vllm-rocm:${TAG:-latest}
+    container_name: vllm-service
+    ports:
+      - "${VLLM_SERVICE_PORT:-8081}:8011"
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      HF_HUB_DISABLE_PROGRESS_BARS: 1
+      HF_HUB_ENABLE_HF_TRANSFER: 0
+      WILM_USE_TRITON_FLASH_ATTENTION: 0
+      PYTORCH_JIT: 0
+    volumes:
+      - "${MODEL_CACHE:-./data}:/data"
+    shm_size: 20G
+    devices:
+      - /dev/kfd:/dev/kfd
+      - /dev/dri/:/dev/dri/
+    cap_add:
+      - SYS_PTRACE
+    group_add:
+      - video
+    security_opt:
+      - seccomp:unconfined
+      - apparmor=unconfined
+    command: "--model ${VLLM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\""
+    ipc: host
+
+  worker-rag-agent:
+    image: ${REGISTRY:-opea}/agent:${TAG:-latest}
+    container_name: rag-agent-endpoint
+    volumes:
+      - ${TOOLSET_PATH}:/home/user/tools/
+    ports:
+      - "${WORKER_RAG_AGENT_PORT:-9095}:9095"
+    ipc: host
+    environment:
+      ip_address: ${ip_address}
+      strategy: rag_agent_llama
+      with_memory: false
+      recursion_limit: ${recursion_limit_worker}
+      llm_engine: vllm
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      llm_endpoint_url: ${LLM_ENDPOINT_URL}
+      model: ${LLM_MODEL_ID}
+      temperature: ${temperature}
+      max_new_tokens: ${max_new_tokens}
+      stream: false
+      tools: /home/user/tools/worker_agent_tools.yaml
+      require_human_feedback: false
+      RETRIEVAL_TOOL_URL: ${RETRIEVAL_TOOL_URL}
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
+      LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2}
+      LANGCHAIN_PROJECT: "opea-worker-agent-service"
+      port: 9095
+
+  worker-sql-agent:
+    image: ${REGISTRY:-opea}/agent:${TAG:-latest}
+    container_name: sql-agent-endpoint
+    volumes:
+      - "${WORKDIR}/tests/Chinook_Sqlite.sqlite:/home/user/chinook-db/Chinook_Sqlite.sqlite:rw"
+    ports:
+      - "${WORKER_SQL_AGENT_PORT:-9096}:9096"
+    ipc: host
+    environment:
+      ip_address: ${ip_address}
+      strategy: sql_agent_llama
+      with_memory: false
+      db_name: ${db_name}
+      db_path: ${db_path}
+      use_hints: false
+      recursion_limit: ${recursion_limit_worker}
+      llm_engine: vllm
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      llm_endpoint_url: ${LLM_ENDPOINT_URL}
+      model: ${LLM_MODEL_ID}
+      temperature: ${temperature}
+      max_new_tokens: ${max_new_tokens}
+      stream: false
+      require_human_feedback: false
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      port: 9096
+
+  supervisor-react-agent:
+    image: ${REGISTRY:-opea}/agent:${TAG:-latest}
+    container_name: react-agent-endpoint
+    depends_on:
+      - worker-rag-agent
+    volumes:
+      - ${TOOLSET_PATH}:/home/user/tools/
+    ports:
+      - "${SUPERVISOR_REACT_AGENT_PORT:-9090}:9090"
+    ipc: host
+    environment:
+      ip_address: ${ip_address}
+      strategy: react_llama
+      with_memory: true
+      recursion_limit: ${recursion_limit_supervisor}
+      llm_engine: vllm
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      llm_endpoint_url: ${LLM_ENDPOINT_URL}
+      model: ${LLM_MODEL_ID}
+      temperature: ${temperature}
+      max_new_tokens: ${max_new_tokens}
+      stream: true
+      tools: /home/user/tools/supervisor_agent_tools.yaml
+      require_human_feedback: false
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
+      LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2}
+      LANGCHAIN_PROJECT: "opea-supervisor-agent-service"
+      CRAG_SERVER: ${CRAG_SERVER}
+      WORKER_AGENT_URL: ${WORKER_AGENT_URL}
+      SQL_AGENT_URL: ${SQL_AGENT_URL}
+      port: 9090

AgentQnA/docker_compose/amd/gpu/rocm/launch_agent_service_tgi_rocm.sh

@@ -1,47 +1,87 @@
 # Copyright (C) 2024 Advanced Micro Devices, Inc.
 # SPDX-License-Identifier: Apache-2.0
 
-WORKPATH=$(dirname "$PWD")/..
-export ip_address=${host_ip}
-export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
-export AGENTQNA_TGI_IMAGE=ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
-export AGENTQNA_TGI_SERVICE_PORT="8085"
+# Before start script:
+# export host_ip="your_host_ip_or_host_name"
+# export HUGGINGFACEHUB_API_TOKEN="your_huggingface_api_token"
+# export LANGCHAIN_API_KEY="your_langchain_api_key"
+# export LANGCHAIN_TRACING_V2=""
 
-# LLM related environment variables
-export AGENTQNA_CARD_ID="card1"
-export AGENTQNA_RENDER_ID="renderD136"
-export HF_CACHE_DIR=${HF_CACHE_DIR}
-ls $HF_CACHE_DIR
-export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
-#export NUM_SHARDS=4
-export LLM_ENDPOINT_URL="http://${ip_address}:${AGENTQNA_TGI_SERVICE_PORT}"
+# Set server hostname or IP address
+export ip_address=${host_ip}
+
+# Set services IP ports
+export TGI_SERVICE_PORT="18110"
+export WORKER_RAG_AGENT_PORT="18111"
+export WORKER_SQL_AGENT_PORT="18112"
+export SUPERVISOR_REACT_AGENT_PORT="18113"
+export CRAG_SERVER_PORT="18114"
+
+export WORKPATH=$(dirname "$PWD")
+export WORKDIR=${WORKPATH}/../../../
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export HF_CACHE_DIR="./data"
+export MODEL_CACHE="./data"
+export TOOLSET_PATH=${WORKPATH}/../../../tools/
+export recursion_limit_worker=12
+export LLM_ENDPOINT_URL=http://${ip_address}:${TGI_SERVICE_PORT}
 export temperature=0.01
 export max_new_tokens=512
-
-# agent related environment variables
-export AGENTQNA_WORKER_AGENT_SERVICE_PORT="9095"
-export TOOLSET_PATH=/home/huggingface/datamonsters/amd-opea/GenAIExamples/AgentQnA/tools/
-echo "TOOLSET_PATH=${TOOLSET_PATH}"
+export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool"
+export LANGCHAIN_API_KEY=${LANGCHAIN_API_KEY}
+export LANGCHAIN_TRACING_V2=${LANGCHAIN_TRACING_V2}
+export db_name=Chinook
+export db_path="sqlite:////home/user/chinook-db/Chinook_Sqlite.sqlite"
 export recursion_limit_worker=12
 export recursion_limit_supervisor=10
-export WORKER_AGENT_URL="http://${ip_address}:${AGENTQNA_WORKER_AGENT_SERVICE_PORT}/v1/chat/completions"
-export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool"
-export CRAG_SERVER=http://${ip_address}:18881
-
-export AGENTQNA_FRONTEND_PORT="9090"
-
-#retrieval_tool
+export CRAG_SERVER=http://${ip_address}:${CRAG_SERVER_PORT}
+export WORKER_AGENT_URL="http://${ip_address}:${WORKER_RAG_AGENT_PORT}/v1/chat/completions"
+export SQL_AGENT_URL="http://${ip_address}:${WORKER_SQL_AGENT_PORT}/v1/chat/completions"
+export HF_CACHE_DIR=${HF_CACHE_DIR}
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export no_proxy=${no_proxy}
+export http_proxy=${http_proxy}
+export https_proxy=${https_proxy}
+export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
+export RERANK_MODEL_ID="BAAI/bge-reranker-base"
 export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:6006"
 export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808"
-export REDIS_URL="redis://${host_ip}:26379"
+export REDIS_URL="redis://${host_ip}:6379"
 export INDEX_NAME="rag-redis"
+export RERANK_TYPE="tei"
 export MEGA_SERVICE_HOST_IP=${host_ip}
 export EMBEDDING_SERVICE_HOST_IP=${host_ip}
 export RETRIEVER_SERVICE_HOST_IP=${host_ip}
 export RERANK_SERVICE_HOST_IP=${host_ip}
 export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8889/v1/retrievaltool"
-export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep"
-export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/get_file"
-export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/delete_file"
+export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/ingest"
+export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6008/v1/dataprep/get"
+export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6009/v1/dataprep/delete"
 
+echo ${WORKER_RAG_AGENT_PORT} > ${WORKPATH}/WORKER_RAG_AGENT_PORT_tmp
+echo ${WORKER_SQL_AGENT_PORT} > ${WORKPATH}/WORKER_SQL_AGENT_PORT_tmp
+echo ${SUPERVISOR_REACT_AGENT_PORT} > ${WORKPATH}/SUPERVISOR_REACT_AGENT_PORT_tmp
+echo ${CRAG_SERVER_PORT} > ${WORKPATH}/CRAG_SERVER_PORT_tmp
+
+echo "Downloading chinook data..."
+echo Y | rm -R chinook-database
+git clone https://github.com/lerocha/chinook-database.git
+echo Y | rm -R ../../../../../AgentQnA/tests/Chinook_Sqlite.sqlite
+cp chinook-database/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite ../../../../../AgentQnA/tests
+
+docker compose -f ../../../../../DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml up -d
 docker compose -f compose.yaml up -d
+
+n=0
+until [[ "$n" -ge 100 ]]; do
+    docker logs tgi-service > ${WORKPATH}/tgi_service_start.log
+    if grep -q Connected ${WORKPATH}/tgi_service_start.log; then
+        break
+    fi
+    sleep 10s
+    n=$((n+1))
+done
+
+echo "Starting CRAG server"
+docker run -d --runtime=runc --name=kdd-cup-24-crag-service -p=${CRAG_SERVER_PORT}:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0

AgentQnA/docker_compose/amd/gpu/rocm/launch_agent_service_vllm_rocm.sh

@@ -0,0 +1,88 @@
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Before start script:
+# export host_ip="your_host_ip_or_host_name"
+# export HUGGINGFACEHUB_API_TOKEN="your_huggingface_api_token"
+# export LANGCHAIN_API_KEY="your_langchain_api_key"
+# export LANGCHAIN_TRACING_V2=""
+
+# Set server hostname or IP address
+export ip_address=${host_ip}
+
+# Set services IP ports
+export VLLM_SERVICE_PORT="18110"
+export WORKER_RAG_AGENT_PORT="18111"
+export WORKER_SQL_AGENT_PORT="18112"
+export SUPERVISOR_REACT_AGENT_PORT="18113"
+export CRAG_SERVER_PORT="18114"
+
+export WORKPATH=$(dirname "$PWD")
+export WORKDIR=${WORKPATH}/../../../
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export VLLM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export HF_CACHE_DIR="./data"
+export MODEL_CACHE="./data"
+export TOOLSET_PATH=${WORKPATH}/../../../tools/
+export recursion_limit_worker=12
+export LLM_ENDPOINT_URL=http://${ip_address}:${VLLM_SERVICE_PORT}
+export LLM_MODEL_ID=${VLLM_LLM_MODEL_ID}
+export temperature=0.01
+export max_new_tokens=512
+export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool"
+export LANGCHAIN_API_KEY=${LANGCHAIN_API_KEY}
+export LANGCHAIN_TRACING_V2=${LANGCHAIN_TRACING_V2}
+export db_name=Chinook
+export db_path="sqlite:////home/user/chinook-db/Chinook_Sqlite.sqlite"
+export recursion_limit_worker=12
+export recursion_limit_supervisor=10
+export CRAG_SERVER=http://${ip_address}:${CRAG_SERVER_PORT}
+export WORKER_AGENT_URL="http://${ip_address}:${WORKER_RAG_AGENT_PORT}/v1/chat/completions"
+export SQL_AGENT_URL="http://${ip_address}:${WORKER_SQL_AGENT_PORT}/v1/chat/completions"
+export HF_CACHE_DIR=${HF_CACHE_DIR}
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export no_proxy=${no_proxy}
+export http_proxy=${http_proxy}
+export https_proxy=${https_proxy}
+export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
+export RERANK_MODEL_ID="BAAI/bge-reranker-base"
+export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:6006"
+export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808"
+export REDIS_URL="redis://${host_ip}:6379"
+export INDEX_NAME="rag-redis"
+export RERANK_TYPE="tei"
+export MEGA_SERVICE_HOST_IP=${host_ip}
+export EMBEDDING_SERVICE_HOST_IP=${host_ip}
+export RETRIEVER_SERVICE_HOST_IP=${host_ip}
+export RERANK_SERVICE_HOST_IP=${host_ip}
+export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8889/v1/retrievaltool"
+export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/ingest"
+export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6008/v1/dataprep/get"
+export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6009/v1/dataprep/delete"
+
+echo ${WORKER_RAG_AGENT_PORT} > ${WORKPATH}/WORKER_RAG_AGENT_PORT_tmp
+echo ${WORKER_SQL_AGENT_PORT} > ${WORKPATH}/WORKER_SQL_AGENT_PORT_tmp
+echo ${SUPERVISOR_REACT_AGENT_PORT} > ${WORKPATH}/SUPERVISOR_REACT_AGENT_PORT_tmp
+echo ${CRAG_SERVER_PORT} > ${WORKPATH}/CRAG_SERVER_PORT_tmp
+
+echo "Downloading chinook data..."
+echo Y | rm -R chinook-database
+git clone https://github.com/lerocha/chinook-database.git
+echo Y | rm -R ../../../../../AgentQnA/tests/Chinook_Sqlite.sqlite
+cp chinook-database/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite ../../../../../AgentQnA/tests
+
+docker compose -f ../../../../../DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml up -d
+docker compose -f compose_vllm.yaml up -d
+
+n=0
+until [[ "$n" -ge 500 ]]; do
+    docker logs vllm-service >& "${WORKPATH}"/vllm-service_start.log
+    if grep -q "Application startup complete" "${WORKPATH}"/vllm-service_start.log; then
+        break
+    fi
+    sleep 20s
+    n=$((n+1))
+done
+
+echo "Starting CRAG server"
+docker run -d --runtime=runc --name=kdd-cup-24-crag-service -p=${CRAG_SERVER_PORT}:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0

AgentQnA/docker_compose/amd/gpu/rocm/stop_agent_service_tgi_rocm.sh

@@ -6,7 +6,7 @@
 WORKPATH=$(dirname "$PWD")/..
 export ip_address=${host_ip}
 export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
-export AGENTQNA_TGI_IMAGE=ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
+export AGENTQNA_TGI_IMAGE=ghcr.io/huggingface/text-generation-inference:2.4.1-rocm
 export AGENTQNA_TGI_SERVICE_PORT="19001"
 
 # LLM related environment variables
@@ -14,7 +14,7 @@ export AGENTQNA_CARD_ID="card1"
 export AGENTQNA_RENDER_ID="renderD136"
 export HF_CACHE_DIR=${HF_CACHE_DIR}
 ls $HF_CACHE_DIR
-export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
+export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
 export NUM_SHARDS=4
 export LLM_ENDPOINT_URL="http://${ip_address}:${AGENTQNA_TGI_SERVICE_PORT}"
 export temperature=0.01
@@ -41,6 +41,22 @@ export EMBEDDING_SERVICE_HOST_IP=${host_ip}
 export RETRIEVER_SERVICE_HOST_IP=${host_ip}
 export RERANK_SERVICE_HOST_IP=${host_ip}
 export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8889/v1/retrievaltool"
-export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep"
-export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/get_file"
-export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/delete_file"
+export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/ingest"
+export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/get"
+export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/delete"
+
+echo "Removing chinook data..."
+echo Y | rm -R chinook-database
+if [ -d "chinook-database" ]; then
+    rm -rf chinook-database
+fi
+echo "Chinook data removed!"
+
+echo "Stopping CRAG server"
+docker rm kdd-cup-24-crag-service --force
+
+echo "Stopping Agent services"
+docker compose -f compose.yaml down
+
+echo "Stopping Retrieval services"
+docker compose -f ../../../../../DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml down

AgentQnA/docker_compose/amd/gpu/rocm/stop_agent_service_vllm_rocm.sh

@@ -0,0 +1,84 @@
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+
+# Before start script:
+# export host_ip="your_host_ip_or_host_name"
+# export HUGGINGFACEHUB_API_TOKEN="your_huggingface_api_token"
+# export LANGCHAIN_API_KEY="your_langchain_api_key"
+# export LANGCHAIN_TRACING_V2=""
+
+# Set server hostname or IP address
+export ip_address=${host_ip}
+
+# Set services IP ports
+export VLLM_SERVICE_PORT="18110"
+export WORKER_RAG_AGENT_PORT="18111"
+export WORKER_SQL_AGENT_PORT="18112"
+export SUPERVISOR_REACT_AGENT_PORT="18113"
+export CRAG_SERVER_PORT="18114"
+
+export WORKPATH=$(dirname "$PWD")
+export WORKDIR=${WORKPATH}/../../../
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export VLLM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
+export HF_CACHE_DIR="./data"
+export MODEL_CACHE="./data"
+export TOOLSET_PATH=${WORKPATH}/../../../tools/
+export recursion_limit_worker=12
+export LLM_ENDPOINT_URL=http://${ip_address}:${VLLM_SERVICE_PORT}
+export LLM_MODEL_ID=${VLLM_LLM_MODEL_ID}
+export temperature=0.01
+export max_new_tokens=512
+export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool"
+export LANGCHAIN_API_KEY=${LANGCHAIN_API_KEY}
+export LANGCHAIN_TRACING_V2=${LANGCHAIN_TRACING_V2}
+export db_name=Chinook
+export db_path="sqlite:////home/user/chinook-db/Chinook_Sqlite.sqlite"
+export recursion_limit_worker=12
+export recursion_limit_supervisor=10
+export CRAG_SERVER=http://${ip_address}:${CRAG_SERVER_PORT}
+export WORKER_AGENT_URL="http://${ip_address}:${WORKER_RAG_AGENT_PORT}/v1/chat/completions"
+export SQL_AGENT_URL="http://${ip_address}:${WORKER_SQL_AGENT_PORT}/v1/chat/completions"
+export HF_CACHE_DIR=${HF_CACHE_DIR}
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export no_proxy=${no_proxy}
+export http_proxy=${http_proxy}
+export https_proxy=${https_proxy}
+export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
+export RERANK_MODEL_ID="BAAI/bge-reranker-base"
+export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:6006"
+export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808"
+export REDIS_URL="redis://${host_ip}:6379"
+export INDEX_NAME="rag-redis"
+export RERANK_TYPE="tei"
+export MEGA_SERVICE_HOST_IP=${host_ip}
+export EMBEDDING_SERVICE_HOST_IP=${host_ip}
+export RETRIEVER_SERVICE_HOST_IP=${host_ip}
+export RERANK_SERVICE_HOST_IP=${host_ip}
+export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8889/v1/retrievaltool"
+export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/ingest"
+export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6008/v1/dataprep/get"
+export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6009/v1/dataprep/delete"
+
+echo ${WORKER_RAG_AGENT_PORT} > ${WORKPATH}/WORKER_RAG_AGENT_PORT_tmp
+echo ${WORKER_SQL_AGENT_PORT} > ${WORKPATH}/WORKER_SQL_AGENT_PORT_tmp
+echo ${SUPERVISOR_REACT_AGENT_PORT} > ${WORKPATH}/SUPERVISOR_REACT_AGENT_PORT_tmp
+echo ${CRAG_SERVER_PORT} > ${WORKPATH}/CRAG_SERVER_PORT_tmp
+
+echo "Removing chinook data..."
+echo Y | rm -R chinook-database
+if [ -d "chinook-database" ]; then
+    rm -rf chinook-database
+fi
+echo "Chinook data removed!"
+
+echo "Stopping CRAG server"
+docker rm kdd-cup-24-crag-service --force
+
+echo "Stopping Agent services"
+docker compose -f compose_vllm.yaml down
+
+echo "Stopping Retrieval services"
+docker compose -f ../../../../../DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml down

AgentQnA/docker_compose/intel/cpu/xeon/README.md

@@ -1,100 +1,3 @@
 # Single node on-prem deployment with Docker Compose on Xeon Scalable processors
 
-This example showcases a hierarchical multi-agent system for question-answering applications. We deploy the example on Xeon. For LLMs, we use OpenAI models via API calls. For instructions on using open-source LLMs, please refer to the deployment guide [here](../../../../README.md).
-
-## Deployment with docker
-
-1. First, clone this repo.
-   ```
-   export WORKDIR=<your-work-directory>
-   cd $WORKDIR
-   git clone https://github.com/opea-project/GenAIExamples.git
-   ```
-2. Set up environment for this example </br>
-
-   ```
-   # Example: host_ip="192.168.1.1" or export host_ip="External_Public_IP"
-   export host_ip=$(hostname -I | awk '{print $1}')
-   # if you are in a proxy environment, also set the proxy-related environment variables
-   export http_proxy="Your_HTTP_Proxy"
-   export https_proxy="Your_HTTPs_Proxy"
-   # Example: no_proxy="localhost, 127.0.0.1, 192.168.1.1"
-   export no_proxy="Your_No_Proxy"
-
-   export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
-   #OPANAI_API_KEY if you want to use OpenAI models
-   export OPENAI_API_KEY=<your-openai-key>
-   ```
-
-3. Deploy the retrieval tool (i.e., DocIndexRetriever mega-service)
-
-   First, launch the mega-service.
-
-   ```
-   cd $WORKDIR/GenAIExamples/AgentQnA/retrieval_tool
-   bash launch_retrieval_tool.sh
-   ```
-
-   Then, ingest data into the vector database. Here we provide an example. You can ingest your own data.
-
-   ```
-   bash run_ingest_data.sh
-   ```
-
-4. Launch Tool service
-   In this example, we will use some of the mock APIs provided in the Meta CRAG KDD Challenge to demonstrate the benefits of gaining additional context from mock knowledge graphs.
-   ```
-   docker run -d -p=8080:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0
-   ```
-5. Launch `Agent` service
-
-   The configurations of the supervisor agent and the worker agent are defined in the docker-compose yaml file. We currently use openAI GPT-4o-mini as LLM, and llama3.1-70B-instruct (served by TGI-Gaudi) in Gaudi example. To use openai llm, run command below.
-
-   ```
-   cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon
-   bash launch_agent_service_openai.sh
-   ```
-
-6. [Optional] Build `Agent` docker image if pulling images failed.
-
-   ```
-   git clone https://github.com/opea-project/GenAIComps.git
-   cd GenAIComps
-   docker build -t opea/agent-langchain:latest -f comps/agent/langchain/Dockerfile .
-   ```
-
-## Validate services
-
-First look at logs of the agent docker containers:
-
-```
-# worker agent
-docker logs rag-agent-endpoint
-```
-
-```
-# supervisor agent
-docker logs react-agent-endpoint
-```
-
-You should see something like "HTTP server setup successful" if the docker containers are started successfully.</p>
-
-Second, validate worker agent:
-
-```
-curl http://${host_ip}:9095/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
-     "query": "Most recent album by Taylor Swift"
-    }'
-```
-
-Third, validate supervisor agent:
-
-```
-curl http://${host_ip}:9090/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
-     "query": "Most recent album by Taylor Swift"
-    }'
-```
-
-## How to register your own tools with agent
-
-You can take a look at the tools yaml and python files in this example. For more details, please refer to the "Provide your own tools" section in the instructions [here](https://github.com/opea-project/GenAIComps/tree/main/comps/agent/langchain/README.md).
+This example showcases a hierarchical multi-agent system for question-answering applications. To deploy the example on Xeon, OpenAI LLM models via API calls are used. For instructions, refer to the deployment guide [here](../../../../README.md).

AgentQnA/docker_compose/intel/cpu/xeon/compose_openai.yaml

@@ -3,7 +3,7 @@
 
 services:
   worker-rag-agent:
-    image: opea/agent-langchain:latest
+    image: opea/agent:latest
     container_name: rag-agent-endpoint
     volumes:
       - ${TOOLSET_PATH}:/home/user/tools/
@@ -13,13 +13,14 @@ services:
     environment:
       ip_address: ${ip_address}
       strategy: rag_agent
+      with_memory: false
       recursion_limit: ${recursion_limit_worker}
       llm_engine: openai
       OPENAI_API_KEY: ${OPENAI_API_KEY}
       model: ${model}
       temperature: ${temperature}
       max_new_tokens: ${max_new_tokens}
-      streaming: false
+      stream: false
       tools: /home/user/tools/worker_agent_tools.yaml
       require_human_feedback: false
       RETRIEVAL_TOOL_URL: ${RETRIEVAL_TOOL_URL}
@@ -31,12 +32,40 @@ services:
       LANGCHAIN_PROJECT: "opea-worker-agent-service"
       port: 9095
 
+  worker-sql-agent:
+    image: opea/agent:latest
+    container_name: sql-agent-endpoint
+    volumes:
+      - ${WORKDIR}/GenAIExamples/AgentQnA/tests:/home/user/chinook-db # SQL database
+    ports:
+      - "9096:9096"
+    ipc: host
+    environment:
+      ip_address: ${ip_address}
+      strategy: sql_agent
+      with_memory: false
+      db_name: ${db_name}
+      db_path: ${db_path}
+      use_hints: false
+      recursion_limit: ${recursion_limit_worker}
+      llm_engine: openai
+      OPENAI_API_KEY: ${OPENAI_API_KEY}
+      model: ${model}
+      temperature: 0
+      max_new_tokens: ${max_new_tokens}
+      stream: false
+      require_human_feedback: false
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      port: 9096
 
   supervisor-react-agent:
-    image: opea/agent-langchain:latest
+    image: opea/agent:latest
     container_name: react-agent-endpoint
     depends_on:
       - worker-rag-agent
+      - worker-sql-agent
     volumes:
       - ${TOOLSET_PATH}:/home/user/tools/
     ports:
@@ -44,14 +73,15 @@ services:
     ipc: host
     environment:
       ip_address: ${ip_address}
-      strategy: react_langgraph
+      strategy: react_llama
+      with_memory: true
       recursion_limit: ${recursion_limit_supervisor}
       llm_engine: openai
       OPENAI_API_KEY: ${OPENAI_API_KEY}
       model: ${model}
       temperature: ${temperature}
       max_new_tokens: ${max_new_tokens}
-      streaming: false
+      stream: true
       tools: /home/user/tools/supervisor_agent_tools.yaml
       require_human_feedback: false
       no_proxy: ${no_proxy}
@@ -62,4 +92,21 @@ services:
       LANGCHAIN_PROJECT: "opea-supervisor-agent-service"
       CRAG_SERVER: $CRAG_SERVER
       WORKER_AGENT_URL: $WORKER_AGENT_URL
+      SQL_AGENT_URL: $SQL_AGENT_URL
       port: 9090
+  mock-api:
+    image: docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0
+    container_name: mock-api
+    ports:
+      - "8080:8000"
+    ipc: host
+  agent-ui:
+    image: opea/agent-ui
+    container_name: agent-ui
+    ports:
+      - "5173:8080"
+    ipc: host
+
+networks:
+  default:
+    driver: bridge

AgentQnA/docker_compose/intel/cpu/xeon/launch_agent_service_openai.sh

@@ -1,19 +0,0 @@
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-
-pushd "../../../../../" > /dev/null
-source .set_env.sh
-popd > /dev/null
-export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
-export ip_address=$(hostname -I | awk '{print $1}')
-export recursion_limit_worker=12
-export recursion_limit_supervisor=10
-export model="gpt-4o-mini-2024-07-18"
-export temperature=0
-export max_new_tokens=4096
-export OPENAI_API_KEY=${OPENAI_API_KEY}
-export WORKER_AGENT_URL="http://${ip_address}:9095/v1/chat/completions"
-export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool"
-export CRAG_SERVER=http://${ip_address}:8080
-
-docker compose -f compose_openai.yaml up -d

AgentQnA/docker_compose/intel/cpu/xeon/set_env.sh

@@ -0,0 +1,57 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+pushd "../../../../../" > /dev/null
+source .set_env.sh
+popd > /dev/null
+
+if [[ -z "${WORKDIR}" ]]; then
+	echo "Please set WORKDIR environment variable"
+	exit 0
+fi
+echo "WORKDIR=${WORKDIR}"
+export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
+export ip_address=$(hostname -I | awk '{print $1}')
+export recursion_limit_worker=12
+export recursion_limit_supervisor=10
+export model="gpt-4o-mini-2024-07-18"
+export temperature=0
+export max_new_tokens=4096
+export OPENAI_API_KEY=${OPENAI_API_KEY}
+export WORKER_AGENT_URL="http://${ip_address}:9095/v1/chat/completions"
+export SQL_AGENT_URL="http://${ip_address}:9096/v1/chat/completions"
+export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool"
+export CRAG_SERVER=http://${ip_address}:8080
+export db_name=Chinook
+export db_path="sqlite:////home/user/chinook-db/Chinook_Sqlite.sqlite"
+
+if [ ! -f $WORKDIR/GenAIExamples/AgentQnA/tests/Chinook_Sqlite.sqlite ]; then
+    echo "Download Chinook_Sqlite!"
+    wget  -O $WORKDIR/GenAIExamples/AgentQnA/tests/Chinook_Sqlite.sqlite  https://github.com/lerocha/chinook-database/releases/download/v1.4.5/Chinook_Sqlite.sqlite
+fi
+
+# retriever
+export host_ip=$(hostname -I | awk '{print $1}')
+export HF_CACHE_DIR=${HF_CACHE_DIR}
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export no_proxy=${no_proxy}
+export http_proxy=${http_proxy}
+export https_proxy=${https_proxy}
+export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
+export RERANK_MODEL_ID="BAAI/bge-reranker-base"
+export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:6006"
+export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808"
+export REDIS_URL="redis://${host_ip}:6379"
+export INDEX_NAME="rag-redis"
+export RERANK_TYPE="tei"
+export MEGA_SERVICE_HOST_IP=${host_ip}
+export EMBEDDING_SERVICE_HOST_IP=${host_ip}
+export RETRIEVER_SERVICE_HOST_IP=${host_ip}
+export RERANK_SERVICE_HOST_IP=${host_ip}
+export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8889/v1/retrievaltool"
+export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/ingest"
+export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6008/v1/dataprep/get"
+export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6009/v1/dataprep/delete"
+
+
+export no_proxy="$no_proxy,rag-agent-endpoint,sql-agent-endpoint,react-agent-endpoint,agent-ui"

AgentQnA/docker_compose/intel/hpu/gaudi/README.md

@@ -1,105 +1,3 @@
 # Single node on-prem deployment AgentQnA on Gaudi
 
-This example showcases a hierarchical multi-agent system for question-answering applications. We deploy the example on Gaudi using open-source LLMs,
-For more details, please refer to the deployment guide [here](../../../../README.md).
-
-## Deployment with docker
-
-1. First, clone this repo.
-   ```
-   export WORKDIR=<your-work-directory>
-   cd $WORKDIR
-   git clone https://github.com/opea-project/GenAIExamples.git
-   ```
-2. Set up environment for this example </br>
-
-   ```
-   # Example: host_ip="192.168.1.1" or export host_ip="External_Public_IP"
-   export host_ip=$(hostname -I | awk '{print $1}')
-   # if you are in a proxy environment, also set the proxy-related environment variables
-   export http_proxy="Your_HTTP_Proxy"
-   export https_proxy="Your_HTTPs_Proxy"
-   # Example: no_proxy="localhost, 127.0.0.1, 192.168.1.1"
-   export no_proxy="Your_No_Proxy"
-
-   export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
-   # for using open-source llms
-   export HUGGINGFACEHUB_API_TOKEN=<your-HF-token>
-   # Example export HF_CACHE_DIR=$WORKDIR so that no need to redownload every time
-   export HF_CACHE_DIR=<directory-where-llms-are-downloaded>
-
-   ```
-
-3. Deploy the retrieval tool (i.e., DocIndexRetriever mega-service)
-
-   First, launch the mega-service.
-
-   ```
-   cd $WORKDIR/GenAIExamples/AgentQnA/retrieval_tool
-   bash launch_retrieval_tool.sh
-   ```
-
-   Then, ingest data into the vector database. Here we provide an example. You can ingest your own data.
-
-   ```
-   bash run_ingest_data.sh
-   ```
-
-4. Launch Tool service
-   In this example, we will use some of the mock APIs provided in the Meta CRAG KDD Challenge to demonstrate the benefits of gaining additional context from mock knowledge graphs.
-   ```
-   docker run -d -p=8080:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0
-   ```
-5. Launch `Agent` service
-
-   To use open-source LLMs on Gaudi2, run commands below.
-
-   ```
-   cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/hpu/gaudi
-   bash launch_tgi_gaudi.sh
-   bash launch_agent_service_tgi_gaudi.sh
-   ```
-
-6. [Optional] Build `Agent` docker image if pulling images failed.
-
-   ```
-   git clone https://github.com/opea-project/GenAIComps.git
-   cd GenAIComps
-   docker build -t opea/agent-langchain:latest -f comps/agent/langchain/Dockerfile .
-   ```
-
-## Validate services
-
-First look at logs of the agent docker containers:
-
-```
-# worker agent
-docker logs rag-agent-endpoint
-```
-
-```
-# supervisor agent
-docker logs react-agent-endpoint
-```
-
-You should see something like "HTTP server setup successful" if the docker containers are started successfully.</p>
-
-Second, validate worker agent:
-
-```
-curl http://${host_ip}:9095/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
-     "query": "Most recent album by Taylor Swift"
-    }'
-```
-
-Third, validate supervisor agent:
-
-```
-curl http://${host_ip}:9090/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
-     "query": "Most recent album by Taylor Swift"
-    }'
-```
-
-## How to register your own tools with agent
-
-You can take a look at the tools yaml and python files in this example. For more details, please refer to the "Provide your own tools" section in the instructions [here](https://github.com/opea-project/GenAIComps/tree/main/comps/agent/langchain/README.md).
+This example showcases a hierarchical multi-agent system for question-answering applications. To deploy the example on Gaudi using open-source LLMs, refer to the deployment guide [here](../../../../README.md).

AgentQnA/docker_compose/intel/hpu/gaudi/compose.telemetry.yaml

@@ -0,0 +1,93 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  tei-embedding-service:
+    command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
+  tei-reranking-service:
+    command: --model-id ${RERANK_MODEL_ID} --auto-truncate --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
+  jaeger:
+    image: jaegertracing/all-in-one:1.67.0
+    container_name: jaeger
+    ports:
+      - "16686:16686"
+      - "4317:4317"
+      - "4318:4318"
+      - "9411:9411"
+    ipc: host
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      COLLECTOR_ZIPKIN_HOST_PORT: 9411
+    restart: unless-stopped
+  prometheus:
+    image: prom/prometheus:v2.52.0
+    container_name: prometheus
+    user: root
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus/prometheus.yaml
+      - ./prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yaml'
+    ports:
+      - '9091:9090'
+    ipc: host
+    restart: unless-stopped
+  grafana:
+    image: grafana/grafana:11.0.0
+    container_name: grafana
+    volumes:
+      - ./grafana_data:/var/lib/grafana
+      - ./grafana/dashboards:/var/lib/grafana/dashboards
+      - ./grafana/provisioning:/etc/grafana/provisioning
+    user: root
+    environment:
+      GF_SECURITY_ADMIN_PASSWORD: admin
+      GF_RENDERING_CALLBACK_URL: http://grafana:3000/
+      GF_LOG_FILTERS: rendering:debug
+    depends_on:
+      - prometheus
+    ports:
+      - '3000:3000'
+    ipc: host
+    restart: unless-stopped
+  node-exporter:
+    image: prom/node-exporter
+    container_name: node-exporter
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.sysfs=/host/sys'
+      - --collector.filesystem.ignored-mount-points
+      - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)"
+    ports:
+      - 9100:9100
+    restart: always
+    deploy:
+      mode: global
+  gaudi-exporter:
+    image: vault.habana.ai/gaudi-metric-exporter/metric-exporter:1.19.2-32
+    container_name: gaudi-exporter
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+      - /dev:/dev
+    ports:
+      - 41612:41611
+    restart: always
+    deploy:
+      mode: global
+  worker-rag-agent:
+    environment:
+      - TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT}
+  worker-sql-agent:
+    environment:
+      - TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT}
+  supervisor-react-agent:
+    environment:
+      - TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT}

AgentQnA/docker_compose/intel/hpu/gaudi/compose.webtool.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  supervisor-react-agent:
+    environment:
+      - tools=/home/user/tools/supervisor_agent_webtools.yaml
+      - GOOGLE_CSE_ID=${GOOGLE_CSE_ID}
+      - GOOGLE_API_KEY=${GOOGLE_API_KEY}

AgentQnA/docker_compose/intel/hpu/gaudi/compose.yaml

@@ -3,10 +3,9 @@
 
 services:
   worker-rag-agent:
-    image: opea/agent-langchain:latest
+    image: ${REGISTRY:-opea}/agent:${TAG:-latest}
     container_name: rag-agent-endpoint
     volumes:
-      # - ${WORKDIR}/GenAIExamples/AgentQnA/docker_image_build/GenAIComps/comps/agent/langchain/:/home/user/comps/agent/langchain/
       - ${TOOLSET_PATH}:/home/user/tools/
     ports:
       - "9095:9095"
@@ -14,14 +13,15 @@ services:
     environment:
       ip_address: ${ip_address}
       strategy: rag_agent_llama
+      with_memory: false
       recursion_limit: ${recursion_limit_worker}
-      llm_engine: tgi
+      llm_engine: vllm
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       llm_endpoint_url: ${LLM_ENDPOINT_URL}
       model: ${LLM_MODEL_ID}
       temperature: ${temperature}
       max_new_tokens: ${max_new_tokens}
-      streaming: false
+      stream: false
       tools: /home/user/tools/worker_agent_tools.yaml
       require_human_feedback: false
       RETRIEVAL_TOOL_URL: ${RETRIEVAL_TOOL_URL}
@@ -33,14 +33,42 @@ services:
       LANGCHAIN_PROJECT: "opea-worker-agent-service"
       port: 9095
 
+  worker-sql-agent:
+    image: ${REGISTRY:-opea}/agent:${TAG:-latest}
+    container_name: sql-agent-endpoint
+    volumes:
+      - ${WORKDIR}/GenAIExamples/AgentQnA/tests:/home/user/chinook-db # test db
+    ports:
+      - "9096:9096"
+    ipc: host
+    environment:
+      ip_address: ${ip_address}
+      strategy: sql_agent_llama
+      with_memory: false
+      db_name: ${db_name}
+      db_path: ${db_path}
+      use_hints: false
+      recursion_limit: ${recursion_limit_worker}
+      llm_engine: vllm
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      llm_endpoint_url: ${LLM_ENDPOINT_URL}
+      model: ${LLM_MODEL_ID}
+      temperature: ${temperature}
+      max_new_tokens: ${max_new_tokens}
+      stream: false
+      require_human_feedback: false
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      port: 9096
 
   supervisor-react-agent:
-    image: opea/agent-langchain:latest
+    image: ${REGISTRY:-opea}/agent:${TAG:-latest}
     container_name: react-agent-endpoint
     depends_on:
       - worker-rag-agent
+      - worker-sql-agent
     volumes:
-      # - ${WORKDIR}/GenAIExamples/AgentQnA/docker_image_build/GenAIComps/comps/agent/langchain/:/home/user/comps/agent/langchain/
       - ${TOOLSET_PATH}:/home/user/tools/
     ports:
       - "9090:9090"
@@ -48,14 +76,15 @@ services:
     environment:
       ip_address: ${ip_address}
       strategy: react_llama
+      with_memory: true
       recursion_limit: ${recursion_limit_supervisor}
-      llm_engine: tgi
+      llm_engine: vllm
       HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
       llm_endpoint_url: ${LLM_ENDPOINT_URL}
       model: ${LLM_MODEL_ID}
       temperature: ${temperature}
       max_new_tokens: ${max_new_tokens}
-      streaming: false
+      stream: true
       tools: /home/user/tools/supervisor_agent_tools.yaml
       require_human_feedback: false
       no_proxy: ${no_proxy}
@@ -66,4 +95,47 @@ services:
       LANGCHAIN_PROJECT: "opea-supervisor-agent-service"
       CRAG_SERVER: $CRAG_SERVER
       WORKER_AGENT_URL: $WORKER_AGENT_URL
+      SQL_AGENT_URL: $SQL_AGENT_URL
       port: 9090
+  mock-api:
+    image: docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0
+    container_name: mock-api
+    ports:
+      - "8080:8000"
+    ipc: host
+  agent-ui:
+    image: ${REGISTRY:-opea}/agent-ui:${TAG:-latest}
+    container_name: agent-ui
+    environment:
+      host_ip: ${host_ip}
+    ports:
+      - "5173:8080"
+    ipc: host
+  vllm-service:
+    image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest}
+    container_name: vllm-gaudi-server
+    ports:
+      - "8086:8000"
+    volumes:
+      - "${MODEL_CACHE:-./data}:/data"
+    environment:
+      no_proxy: ${no_proxy}
+      http_proxy: ${http_proxy}
+      https_proxy: ${https_proxy}
+      HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      HABANA_VISIBLE_DEVICES: all
+      OMPI_MCA_btl_vader_single_copy_mechanism: none
+      LLM_MODEL_ID: ${LLM_MODEL_ID}
+      VLLM_TORCH_PROFILER_DIR: "/mnt"
+      VLLM_SKIP_WARMUP: true
+      PT_HPU_ENABLE_LAZY_COLLECTIVES: true
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://$host_ip:8086/health || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 100
+    runtime: habana
+    cap_add:
+      - SYS_NICE
+    ipc: host
+    command: --model $LLM_MODEL_ID --tensor-parallel-size 4 --host 0.0.0.0 --port 8000 --block-size 128 --max-num-seqs 256 --max-seq-len-to-capture 16384

AgentQnA/docker_compose/intel/hpu/gaudi/grafana/dashboards/download_opea_dashboard.sh

@@ -0,0 +1,10 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+rm *.json
+wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/chatqna_megaservice_grafana.json
+mv chatqna_megaservice_grafana.json agentqna_microervices_grafana.json
+wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/vllm_grafana.json
+wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/tgi_grafana.json
+wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/node_grafana.json
+wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/gaudi_grafana.json

AgentQnA/docker_compose/intel/hpu/gaudi/grafana/provisioning/dashboards/local.yaml

@@ -0,0 +1,14 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: 1
+
+providers:
+- name: 'default'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: false
+  updateIntervalSeconds: 10 #how often Grafana will scan for changed dashboards
+  options:
+    path: /var/lib/grafana/dashboards

AgentQnA/docker_compose/intel/hpu/gaudi/grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,54 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# config file version
+apiVersion: 1
+
+# list of datasources that should be deleted from the database
+deleteDatasources:
+  - name: Prometheus
+    orgId: 1
+
+# list of datasources to insert/update depending
+# what's available in the database
+datasources:
+  # <string, required> name of the datasource. Required
+- name: Prometheus
+  # <string, required> datasource type. Required
+  type: prometheus
+  # <string, required> access mode. direct or proxy. Required
+  access: proxy
+  # <int> org id. will default to orgId 1 if not specified
+  orgId: 1
+  # <string> url
+  url: http://prometheus:9090
+  # <string> database password, if used
+  password:
+  # <string> database user, if used
+  user:
+  # <string> database name, if used
+  database:
+  # <bool> enable/disable basic auth
+  basicAuth: false
+  # <string> basic auth username, if used
+  basicAuthUser:
+  # <string> basic auth password, if used
+  basicAuthPassword:
+  # <bool> enable/disable with credentials headers
+  withCredentials:
+  # <bool> mark as default datasource. Max one per org
+  isDefault: true
+  # <map> fields that will be converted to json and stored in json_data
+  jsonData:
+     httpMethod: GET
+     graphiteVersion: "1.1"
+     tlsAuth: false
+     tlsAuthWithCACert: false
+  # <string> json object of data that will be encrypted.
+  secureJsonData:
+    tlsCACert: "..."
+    tlsClientCert: "..."
+    tlsClientKey: "..."
+  version: 1
+  # <bool> allow users to edit datasources from the UI.
+  editable: true

AgentQnA/docker_compose/intel/hpu/gaudi/launch_agent_service_tgi_gaudi.sh

@@ -1,32 +0,0 @@
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-
-pushd "../../../../../" > /dev/null
-source .set_env.sh
-popd > /dev/null
-WORKPATH=$(dirname "$PWD")/..
-# export WORKDIR=$WORKPATH/../../
-echo "WORKDIR=${WORKDIR}"
-export ip_address=$(hostname -I | awk '{print $1}')
-export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-
-# LLM related environment variables
-export HF_CACHE_DIR=${HF_CACHE_DIR}
-ls $HF_CACHE_DIR
-export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-export LLM_MODEL_ID="meta-llama/Meta-Llama-3.1-70B-Instruct"
-export NUM_SHARDS=4
-export LLM_ENDPOINT_URL="http://${ip_address}:8085"
-export temperature=0.01
-export max_new_tokens=4096
-
-# agent related environment variables
-export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
-echo "TOOLSET_PATH=${TOOLSET_PATH}"
-export recursion_limit_worker=12
-export recursion_limit_supervisor=10
-export WORKER_AGENT_URL="http://${ip_address}:9095/v1/chat/completions"
-export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool"
-export CRAG_SERVER=http://${ip_address}:8080
-
-docker compose -f compose.yaml up -d

AgentQnA/docker_compose/intel/hpu/gaudi/launch_tgi_gaudi.sh

@@ -1,25 +0,0 @@
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-
-# LLM related environment variables
-export HF_CACHE_DIR=${HF_CACHE_DIR}
-ls $HF_CACHE_DIR
-export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-export LLM_MODEL_ID="meta-llama/Meta-Llama-3.1-70B-Instruct"
-export NUM_SHARDS=4
-
-docker compose -f tgi_gaudi.yaml up -d
-
-sleep 5s
-echo "Waiting tgi gaudi ready"
-n=0
-until [[ "$n" -ge 100 ]] || [[ $ready == true ]]; do
-    docker logs tgi-server &> tgi-gaudi-service.log
-    n=$((n+1))
-    if grep -q Connected tgi-gaudi-service.log; then
-        break
-    fi
-    sleep 5s
-done
-sleep 5s
-echo "Service started successfully"

AgentQnA/docker_compose/intel/hpu/gaudi/prometheus.yaml

@@ -0,0 +1,55 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+global:
+  scrape_interval: 5s
+  external_labels:
+    monitor: "my-monitor"
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["prometheus:9090"]
+  - job_name: "vllm"
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["vllm-gaudi-server:8000"]
+  - job_name: "tgi"
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["tgi-gaudi-server:80"]
+  - job_name: "tei-embedding"
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["tei-embedding-server:80"]
+  - job_name: "tei-reranking"
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["tei-reranking-server:80"]
+  - job_name: "retriever"
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["retriever:7000"]
+  - job_name: "dataprep-redis-service"
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["dataprep-redis-service:5000"]
+  - job_name: "prometheus-node-exporter"
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["node-exporter:9100"]
+  - job_name: "prometheus-gaudi-exporter"
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["gaudi-exporter:41611"]
+  - job_name: "supervisor-react-agent"
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["react-agent-endpoint:9090"]
+  - job_name: "worker-rag-agent"
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["rag-agent-endpoint:9095"]
+  - job_name: "worker-sql-agent"
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["sql-agent-endpoint:9096"]

AgentQnA/docker_compose/intel/hpu/gaudi/set_env.sh

@@ -0,0 +1,72 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+pushd "../../../../../" > /dev/null
+source .set_env.sh
+popd > /dev/null
+WORKPATH=$(dirname "$PWD")/..
+# export WORKDIR=$WORKPATH/../../
+if [[ -z "${WORKDIR}" ]]; then
+	echo "Please set WORKDIR environment variable"
+	exit 0
+fi
+echo "WORKDIR=${WORKDIR}"
+export ip_address=$(hostname -I | awk '{print $1}')
+
+# LLM related environment variables
+export HF_CACHE_DIR=${HF_CACHE_DIR}
+ls $HF_CACHE_DIR
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export HF_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export LLM_MODEL_ID="meta-llama/Llama-3.3-70B-Instruct"
+export NUM_SHARDS=4
+export LLM_ENDPOINT_URL="http://${ip_address}:8086"
+export temperature=0
+export max_new_tokens=4096
+
+# agent related environment variables
+export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
+echo "TOOLSET_PATH=${TOOLSET_PATH}"
+export recursion_limit_worker=12
+export recursion_limit_supervisor=10
+export WORKER_AGENT_URL="http://${ip_address}:9095/v1/chat/completions"
+export SQL_AGENT_URL="http://${ip_address}:9096/v1/chat/completions"
+export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool"
+export CRAG_SERVER=http://${ip_address}:8080
+
+export db_name=Chinook
+export db_path="sqlite:////home/user/chinook-db/Chinook_Sqlite.sqlite"
+if [ ! -f $WORKDIR/GenAIExamples/AgentQnA/tests/Chinook_Sqlite.sqlite ]; then
+    echo "Download Chinook_Sqlite!"
+    wget  -O $WORKDIR/GenAIExamples/AgentQnA/tests/Chinook_Sqlite.sqlite  https://github.com/lerocha/chinook-database/releases/download/v1.4.5/Chinook_Sqlite.sqlite
+fi
+
+# configure agent ui
+# echo "AGENT_URL = 'http://$ip_address:9090/v1/chat/completions'" | tee ${WORKDIR}/GenAIExamples/AgentQnA/ui/svelte/.env
+
+# retriever
+export host_ip=$(hostname -I | awk '{print $1}')
+export no_proxy=${no_proxy}
+export http_proxy=${http_proxy}
+export https_proxy=${https_proxy}
+export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
+export RERANK_MODEL_ID="BAAI/bge-reranker-base"
+export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:6006"
+export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808"
+export REDIS_URL="redis://${host_ip}:6379"
+export INDEX_NAME="rag-redis"
+export RERANK_TYPE="tei"
+export MEGA_SERVICE_HOST_IP=${host_ip}
+export EMBEDDING_SERVICE_HOST_IP=${host_ip}
+export RETRIEVER_SERVICE_HOST_IP=${host_ip}
+export RERANK_SERVICE_HOST_IP=${host_ip}
+export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8889/v1/retrievaltool"
+export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/ingest"
+export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6008/v1/dataprep/get"
+export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6009/v1/dataprep/delete"
+# Set OpenTelemetry Tracing Endpoint
+export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+')
+export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317
+export TELEMETRY_ENDPOINT=http://$JAEGER_IP:4318/v1/traces
+
+export no_proxy="$no_proxy,rag-agent-endpoint,sql-agent-endpoint,react-agent-endpoint,agent-ui,vllm-gaudi-server,jaeger,grafana,prometheus,node-exporter,gaudi-exporter,127.0.0.1,localhost,0.0.0.0,$host_ip,,$JAEGER_IP"

AgentQnA/docker_compose/intel/hpu/gaudi/tgi_gaudi.yaml

@@ -3,7 +3,7 @@
 
 services:
   tgi-server:
-    image: ghcr.io/huggingface/tgi-gaudi:2.0.6
+    image: ghcr.io/huggingface/tgi-gaudi:2.3.1
     container_name: tgi-server
     ports:
       - "8085:80"

AgentQnA/docker_image_build/build.yaml

@@ -2,12 +2,30 @@
 # SPDX-License-Identifier: Apache-2.0
 
 services:
-  agent-langchain:
+  agent:
     build:
       context: GenAIComps
-      dockerfile: comps/agent/langchain/Dockerfile
+      dockerfile: comps/agent/src/Dockerfile
       args:
         http_proxy: ${http_proxy}
         https_proxy: ${https_proxy}
         no_proxy: ${no_proxy}
-    image: ${REGISTRY:-opea}/agent-langchain:${TAG:-latest}
+    image: ${REGISTRY:-opea}/agent:${TAG:-latest}
+  agent-ui:
+    build:
+      context: ../ui
+      dockerfile: ./docker/Dockerfile
+    extends: agent
+    image: ${REGISTRY:-opea}/agent-ui:${TAG:-latest}
+  vllm-gaudi:
+    build:
+      context: vllm-fork
+      dockerfile: Dockerfile.hpu
+    extends: agent
+    image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest}
+  vllm-rocm:
+    build:
+      context: GenAIComps
+      dockerfile: comps/third_parties/vllm/src/Dockerfile.amd_gpu
+    extends: agent
+    image: ${REGISTRY:-opea}/vllm-rocm:${TAG:-latest}

AgentQnA/kubernetes/helm/README.md

@@ -0,0 +1,11 @@
+# Deploy AgentQnA on Kubernetes cluster
+
+- You should have Helm (version >= 3.15) installed. Refer to the [Helm Installation Guide](https://helm.sh/docs/intro/install/) for more information.
+- For more deploy options, refer to [helm charts README](https://github.com/opea-project/GenAIInfra/tree/main/helm-charts#readme).
+
+## Deploy on Gaudi
+
+```
+export HFTOKEN="insert-your-huggingface-token-here"
+helm install agentqna oci://ghcr.io/opea-project/charts/agentqna  --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} -f gaudi-values.yaml
+```

AgentQnA/kubernetes/helm/cpu-values.yaml

@@ -0,0 +1,22 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+tgi:
+  enabled: false
+vllm:
+  enabled: true
+  LLM_MODEL_ID: "meta-llama/Meta-Llama-3-8B-Instruct"
+  extraCmdArgs: ["--max-seq-len-to-capture", "16384", "--enable-auto-tool-choice", "--tool-call-parser", "llama3_json"]
+
+supervisor:
+  llm_endpoint_url: http://{{ .Release.Name }}-vllm
+  llm_engine: vllm
+  model: "meta-llama/Meta-Llama-3-8B-Instruct"
+ragagent:
+  llm_endpoint_url: http://{{ .Release.Name }}-vllm
+  llm_engine: vllm
+  model: "meta-llama/Meta-Llama-3-8B-Instruct"
+sqlagent:
+  llm_endpoint_url: http://{{ .Release.Name }}-vllm
+  llm_engine: vllm
+  model: "meta-llama/Meta-Llama-3-8B-Instruct"

AgentQnA/kubernetes/helm/gaudi-values.yaml

@@ -0,0 +1,35 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Accelerate inferencing in heaviest components to improve performance
+# by overriding their subchart values
+
+tgi:
+  enabled: false
+vllm:
+  enabled: true
+  accelDevice: "gaudi"
+  image:
+    repository: opea/vllm-gaudi
+  resources:
+    limits:
+      habana.ai/gaudi: 4
+  LLM_MODEL_ID: "meta-llama/Llama-3.3-70B-Instruct"
+  OMPI_MCA_btl_vader_single_copy_mechanism: none
+  PT_HPU_ENABLE_LAZY_COLLECTIVES: true
+  VLLM_SKIP_WARMUP: true
+  shmSize: 16Gi
+  extraCmdArgs: ["--tensor-parallel-size", "4", "--max-seq-len-to-capture", "16384", "--enable-auto-tool-choice", "--tool-call-parser", "llama3_json"]
+
+supervisor:
+  llm_endpoint_url: http://{{ .Release.Name }}-vllm
+  llm_engine: vllm
+  model: "meta-llama/Llama-3.3-70B-Instruct"
+ragagent:
+  llm_endpoint_url: http://{{ .Release.Name }}-vllm
+  llm_engine: vllm
+  model: "meta-llama/Llama-3.3-70B-Instruct"
+sqlagent:
+  llm_endpoint_url: http://{{ .Release.Name }}-vllm
+  llm_engine: vllm
+  model: "meta-llama/Llama-3.3-70B-Instruct"

AgentQnA/retrieval_tool/index_data.py

@@ -53,7 +53,7 @@ def main():
     host_ip = args.host_ip
     port = args.port
     proxies = {"http": ""}
-    url = "http://{host_ip}:{port}/v1/dataprep".format(host_ip=host_ip, port=port)
+    url = "http://{host_ip}:{port}/v1/dataprep/ingest".format(host_ip=host_ip, port=port)
 
     # Split jsonl file into json files
     files = split_jsonl_into_txts(os.path.join(args.filedir, args.filename))

AgentQnA/retrieval_tool/launch_retrieval_tool.sh

@@ -13,13 +13,14 @@ export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:6006"
 export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808"
 export REDIS_URL="redis://${host_ip}:6379"
 export INDEX_NAME="rag-redis"
+export RERANK_TYPE="tei"
 export MEGA_SERVICE_HOST_IP=${host_ip}
 export EMBEDDING_SERVICE_HOST_IP=${host_ip}
 export RETRIEVER_SERVICE_HOST_IP=${host_ip}
 export RERANK_SERVICE_HOST_IP=${host_ip}
 export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8889/v1/retrievaltool"
-export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep"
-export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6008/v1/dataprep/get_file"
-export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6009/v1/dataprep/delete_file"
+export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/ingest"
+export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6008/v1/dataprep/get"
+export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6009/v1/dataprep/delete"
 
 docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml up -d

AgentQnA/retrieval_tool/run_ingest_data.sh

@@ -1,7 +1,22 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+host_ip=$(hostname -I | awk '{print $1}')
+port=6007
 FILEDIR=${WORKDIR}/GenAIExamples/AgentQnA/example_data/
 FILENAME=test_docs_music.jsonl
 
-python3 index_data.py --filedir ${FILEDIR} --filename ${FILENAME} --host_ip $host_ip
+# AgentQnA ingestion script requires following packages
+packages=("requests" "tqdm")
+
+# Check if packages are installed
+for package in "${packages[@]}"; do
+  if pip freeze | grep -q "$package="; then
+    echo "$package is installed"
+  else
+    echo "$package is not installed"
+    pip install --no-cache-dir "$package"
+  fi
+done
+
+python3 index_data.py --filedir ${FILEDIR} --filename ${FILENAME} --host_ip $host_ip --port $port

AgentQnA/tests/_test_compose_openai_on_xeon.sh

@@ -15,28 +15,35 @@ function stop_agent_and_api_server() {
     echo "Stopping CRAG server"
     docker stop $(docker ps -q --filter ancestor=docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0)
     echo "Stopping Agent services"
-    docker stop $(docker ps -q --filter ancestor=opea/agent-langchain:latest)
+    docker stop $(docker ps -q --filter ancestor=opea/agent:latest)
 }
 
 function stop_retrieval_tool() {
     echo "Stopping Retrieval tool"
-    docker compose -f $WORKDIR/GenAIExamples/AgentQnA/retrieval_tool/docker/docker-compose-retrieval-tool.yaml down
+    local RETRIEVAL_TOOL_PATH=$WORKPATH/../DocIndexRetriever
+    cd $RETRIEVAL_TOOL_PATH/docker_compose/intel/cpu/xeon/
+    container_list=$(cat compose.yaml | grep container_name | cut -d':' -f2)
+    for container_name in $container_list; do
+        cid=$(docker ps -aq --filter "name=$container_name")
+        echo "Stopping container $container_name"
+        if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi
+    done
 }
 
 echo "=================== #1 Building docker images===================="
-bash 1_build_images.sh
+bash step1_build_images.sh xeon
 echo "=================== #1 Building docker images completed===================="
 
 echo "=================== #2 Start retrieval tool===================="
-bash 2_start_retrieval_tool.sh
+bash step2_start_retrieval_tool.sh
 echo "=================== #2 Retrieval tool started===================="
 
 echo "=================== #3 Ingest data and validate retrieval===================="
-bash 3_ingest_data_and_validate_retrieval.sh
+bash step3_ingest_data_and_validate_retrieval.sh
 echo "=================== #3 Data ingestion and validation completed===================="
 
 echo "=================== #4 Start agent and API server===================="
-bash 4_launch_and_validate_agent_openai.sh
+bash step4_launch_and_validate_agent_openai.sh
 echo "=================== #4 Agent test passed ===================="
 
 echo "=================== #5 Stop agent and API server===================="

AgentQnA/tests/sql_agent_test/run_data_split.sh

@@ -0,0 +1,6 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+DATAPATH=$WORKDIR/TAG-Bench/tag_queries.csv
+OUTFOLDER=$WORKDIR/TAG-Bench/query_by_db
+python3 split_data.py --path $DATAPATH --output $OUTFOLDER

AgentQnA/tests/sql_agent_test/split_data.py

@@ -0,0 +1,27 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import os
+
+import pandas as pd
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--path", type=str, required=True)
+    parser.add_argument("--output", type=str, required=True)
+    args = parser.parse_args()
+
+    # if output folder does not exist, create it
+    if not os.path.exists(args.output):
+        os.makedirs(args.output)
+
+    # Load the data
+    data = pd.read_csv(args.path)
+
+    # Split the data by domain
+    domains = data["DB used"].unique()
+    for domain in domains:
+        domain_data = data[data["DB used"] == domain]
+        out = os.path.join(args.output, f"query_{domain}.csv")
+        domain_data.to_csv(out, index=False)

AgentQnA/tests/step1_build_images.sh

@@ -11,38 +11,90 @@ export ip_address=$(hostname -I | awk '{print $1}')
 
 function get_genai_comps() {
     if [ ! -d "GenAIComps" ] ; then
-        git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
+        git clone --depth 1 --branch ${opea_branch:-"main"} https://github.com/opea-project/GenAIComps.git
     fi
 }
 
-
 function build_docker_images_for_retrieval_tool(){
     cd $WORKDIR/GenAIExamples/DocIndexRetriever/docker_image_build/
-    # git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
     get_genai_comps
     echo "Build all the images with --no-cache..."
-    service_list="doc-index-retriever dataprep-redis embedding-tei retriever-redis reranking-tei"
-    docker compose -f build.yaml build ${service_list} --no-cache
-    docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
-
+    docker compose -f build.yaml build --no-cache
     docker images && sleep 1s
 }
 
-function build_agent_docker_image() {
+function build_agent_docker_image_xeon() {
     cd $WORKDIR/GenAIExamples/AgentQnA/docker_image_build/
     get_genai_comps
+
     echo "Build agent image with --no-cache..."
-    docker compose -f build.yaml build --no-cache
+    service_list="agent agent-ui"
+    docker compose -f build.yaml build ${service_list} --no-cache
 }
 
+function build_agent_docker_image_gaudi_vllm() {
+    cd $WORKDIR/GenAIExamples/AgentQnA/docker_image_build/
+    get_genai_comps
+
+    git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
+    VLLM_VER=v0.6.6.post1+Gaudi-1.20.0
+    git checkout ${VLLM_VER} &> /dev/null && cd ../
+
+    echo "Build agent image with --no-cache..."
+    service_list="agent agent-ui vllm-gaudi"
+    docker compose -f build.yaml build ${service_list} --no-cache
+}
+
+function build_agent_docker_image_rocm() {
+    cd $WORKDIR/GenAIExamples/AgentQnA/docker_image_build/
+    get_genai_comps
+
+    echo "Build agent image with --no-cache..."
+    service_list="agent agent-ui"
+    docker compose -f build.yaml build ${service_list} --no-cache
+}
+
+function build_agent_docker_image_rocm_vllm() {
+    cd $WORKDIR/GenAIExamples/AgentQnA/docker_image_build/
+    get_genai_comps
+
+    echo "Build agent image with --no-cache..."
+    service_list="agent agent-ui vllm-rocm"
+    docker compose -f build.yaml build ${service_list} --no-cache
+}
+
+
 function main() {
     echo "==================== Build docker images for retrieval tool ===================="
     build_docker_images_for_retrieval_tool
     echo "==================== Build docker images for retrieval tool completed ===================="
 
-    echo "==================== Build agent docker image ===================="
-    build_agent_docker_image
-    echo "==================== Build agent docker image completed ===================="
+    sleep 3s
+
+    case $1 in
+        "rocm")
+            echo "==================== Build agent docker image for ROCm ===================="
+            build_agent_docker_image_rocm
+            ;;
+        "rocm_vllm")
+            echo "==================== Build agent docker image for ROCm VLLM ===================="
+            build_agent_docker_image_rocm_vllm
+            ;;
+        "gaudi_vllm")
+            echo "==================== Build agent docker image for Gaudi ===================="
+            build_agent_docker_image_gaudi_vllm
+            ;;
+        "xeon")
+            echo "==================== Build agent docker image for Xeon ===================="
+            build_agent_docker_image_xeon
+            ;;
+        *)
+            echo "Invalid argument"
+            exit 1
+            ;;
+    esac
+
+    docker image ls | grep vllm
 }
 
-main
+main $1

AgentQnA/tests/step2_start_retrieval_tool.sh

@@ -7,8 +7,9 @@ WORKPATH=$(dirname "$PWD")
 export WORKDIR=$WORKPATH/../../
 echo "WORKDIR=${WORKDIR}"
 export ip_address=$(hostname -I | awk '{print $1}')
+export host_ip=${ip_address}
 
-export HF_CACHE_DIR=$WORKDIR/hf_cache
+export HF_CACHE_DIR=${model_cache:-"$WORKDIR/hf_cache"}
 if [ ! -d "$HF_CACHE_DIR" ]; then
     echo "Creating HF_CACHE directory"
     mkdir -p "$HF_CACHE_DIR"

AgentQnA/tests/step2_start_retrieval_tool_rocm_vllm.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+WORKPATH=$(dirname "$PWD")
+export WORKDIR=$WORKPATH/../
+echo "WORKDIR=${WORKDIR}"
+export ip_address=$(hostname -I | awk '{print $1}')
+export host_ip=${ip_address}
+
+export HF_CACHE_DIR=$WORKPATH/hf_cache
+if [ ! -d "$HF_CACHE_DIR" ]; then
+    echo "Creating HF_CACHE directory"
+    mkdir -p "$HF_CACHE_DIR"
+fi
+
+function start_retrieval_tool() {
+    echo "Starting Retrieval tool"
+    cd $WORKPATH/../DocIndexRetriever/docker_compose/intel/cpu/xeon
+    host_ip=$(hostname -I | awk '{print $1}')
+    export HF_CACHE_DIR=${HF_CACHE_DIR}
+    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+    export no_proxy=${no_proxy}
+    export http_proxy=${http_proxy}
+    export https_proxy=${https_proxy}
+    export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"
+    export RERANK_MODEL_ID="BAAI/bge-reranker-base"
+    export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:6006"
+    export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808"
+    export REDIS_URL="redis://${host_ip}:6379"
+    export INDEX_NAME="rag-redis"
+    export RERANK_TYPE="tei"
+    export MEGA_SERVICE_HOST_IP=${host_ip}
+    export EMBEDDING_SERVICE_HOST_IP=${host_ip}
+    export RETRIEVER_SERVICE_HOST_IP=${host_ip}
+    export RERANK_SERVICE_HOST_IP=${host_ip}
+    export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8889/v1/retrievaltool"
+    export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/ingest"
+    export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6008/v1/dataprep/get"
+    export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6009/v1/dataprep/delete"
+
+    docker compose -f compose.yaml up -d
+}
+
+echo "==================== Start retrieval tool ===================="
+start_retrieval_tool
+sleep 20 # needed for downloading the models
+echo "==================== Retrieval tool started ===================="

AgentQnA/tests/step3_ingest_data_and_validate_retrieval_rocm_vllm.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+WORKPATH=$(dirname "$PWD")
+export WORKDIR=$WORKPATH/../../
+echo "WORKDIR=${WORKDIR}"
+export ip_address=$(hostname -I | awk '{print $1}')
+export host_ip=$ip_address
+echo "ip_address=${ip_address}"
+
+
+function validate() {
+    local CONTENT="$1"
+    local EXPECTED_RESULT="$2"
+    local SERVICE_NAME="$3"
+
+    if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
+        echo "[ $SERVICE_NAME ] Content is as expected: $CONTENT"
+        echo 0
+    else
+        echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
+        echo 1
+    fi
+}
+
+function ingest_data_and_validate() {
+    echo "Ingesting data"
+    cd $WORKPATH/retrieval_tool/
+    echo $PWD
+    local CONTENT=$(bash run_ingest_data.sh)
+    local EXIT_CODE=$(validate "$CONTENT" "Data preparation succeeded" "dataprep-redis-server")
+    echo "$EXIT_CODE"
+    local EXIT_CODE="${EXIT_CODE:0-1}"
+    echo "return value is $EXIT_CODE"
+    if [ "$EXIT_CODE" == "1" ]; then
+        docker logs dataprep-redis-server
+        return 1
+    fi
+}
+
+function validate_retrieval_tool() {
+    echo "----------------Test retrieval tool ----------------"
+    local CONTENT=$(http_proxy="" curl http://${ip_address}:8889/v1/retrievaltool -X POST -H "Content-Type: application/json" -d '{
+     "text": "Who sang Thriller"
+    }')
+    local EXIT_CODE=$(validate "$CONTENT" "Thriller" "retrieval-tool")
+
+    if [ "$EXIT_CODE" == "1" ]; then
+        docker logs retrievaltool-xeon-backend-server
+        exit 1
+    fi
+}
+
+function main(){
+
+    echo "==================== Ingest data ===================="
+    ingest_data_and_validate
+    echo "==================== Data ingestion completed ===================="
+
+    echo "==================== Validate retrieval tool ===================="
+    validate_retrieval_tool
+    echo "==================== Retrieval tool validated ===================="
+}
+
+main

AgentQnA/tests/step4_launch_and_validate_agent_gaudi.sh

@@ -0,0 +1,197 @@
+#!/bin/bash
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+WORKPATH=$(dirname "$PWD")
+export WORKDIR=$WORKPATH/../../
+echo "WORKDIR=${WORKDIR}"
+export ip_address=$(hostname -I | awk '{print $1}')
+export host_ip=$ip_address
+echo "ip_address=${ip_address}"
+export TOOLSET_PATH=$WORKPATH/tools/
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+HF_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+model="meta-llama/Llama-3.3-70B-Instruct" #"meta-llama/Meta-Llama-3.1-70B-Instruct"
+
+export HF_CACHE_DIR=${model_cache:-"/data2/huggingface"}
+if [ ! -d "$HF_CACHE_DIR" ]; then
+    HF_CACHE_DIR=$WORKDIR/hf_cache
+    mkdir -p "$HF_CACHE_DIR"
+fi
+echo  "HF_CACHE_DIR=$HF_CACHE_DIR"
+ls $HF_CACHE_DIR
+
+vllm_port=8086
+vllm_volume=${HF_CACHE_DIR}
+
+
+function start_agent_service() {
+    echo "Starting agent service"
+    cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/hpu/gaudi
+    source set_env.sh
+    docker compose -f compose.yaml up -d
+}
+
+function start_all_services() {
+
+    echo "token is ${HF_TOKEN}"
+
+    echo "start vllm gaudi service"
+    echo "**************model is $model**************"
+    cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/hpu/gaudi
+    source set_env.sh
+    docker compose -f $WORKDIR/GenAIExamples/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml -f compose.yaml -f compose.telemetry.yaml up -d
+    sleep 5s
+    echo "Waiting vllm gaudi ready"
+    n=0
+    LOG_PATH=$PWD
+    until [[ "$n" -ge 100 ]] || [[ $ready == true ]]; do
+        docker logs vllm-gaudi-server
+        docker logs vllm-gaudi-server &> ${LOG_PATH}/vllm-gaudi-service.log
+        n=$((n+1))
+        if grep -q "Uvicorn running on" ${LOG_PATH}/vllm-gaudi-service.log; then
+            break
+        fi
+        if grep -q "No such container" ${LOG_PATH}/vllm-gaudi-service.log; then
+            echo "container vllm-gaudi-server not found"
+            exit 1
+        fi
+        sleep 5s
+    done
+    sleep 5s
+    echo "Service started successfully"
+}
+
+function download_chinook_data(){
+    echo "Downloading chinook data..."
+    cd $WORKDIR
+    git clone https://github.com/lerocha/chinook-database.git
+    cp chinook-database/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite $WORKDIR/GenAIExamples/AgentQnA/tests/
+}
+
+function validate() {
+    local CONTENT="$1"
+    local EXPECTED_RESULT="$2"
+    local SERVICE_NAME="$3"
+
+    if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
+        echo "[ $SERVICE_NAME ] Content is as expected: $CONTENT"
+        echo 0
+    else
+        echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
+        echo 1
+    fi
+}
+
+function validate_agent_service() {
+    # # test worker rag agent
+    echo "======================Testing worker rag agent======================"
+    export agent_port="9095"
+    export agent_ip="127.0.0.1"
+    prompt="Tell me about Michael Jackson song Thriller"
+    local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt" --agent_role "worker" --ip_addr $agent_ip  --ext_port $agent_port)
+    # echo $CONTENT
+    local EXIT_CODE=$(validate "$CONTENT" "Thriller" "rag-agent-endpoint")
+    echo $EXIT_CODE
+    local EXIT_CODE="${EXIT_CODE:0-1}"
+    if [ "$EXIT_CODE" == "1" ]; then
+        docker logs rag-agent-endpoint
+        exit 1
+    fi
+
+    # # test worker sql agent
+    echo "======================Testing worker sql agent======================"
+    export agent_port="9096"
+    prompt="How many employees are there in the company?"
+    local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt" --agent_role "worker" --ip_addr $agent_ip --ext_port $agent_port)
+    local EXIT_CODE=$(validate "$CONTENT" "8" "sql-agent-endpoint")
+    echo $CONTENT
+    # echo $EXIT_CODE
+    local EXIT_CODE="${EXIT_CODE:0-1}"
+    if [ "$EXIT_CODE" == "1" ]; then
+        docker logs sql-agent-endpoint
+        exit 1
+    fi
+
+    # test supervisor react agent
+    echo "======================Testing supervisor react agent======================"
+    export agent_port="9090"
+    local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --agent_role "supervisor" --ip_addr $agent_ip  --ext_port $agent_port --stream)
+    local EXIT_CODE=$(validate "$CONTENT" "Iron" "react-agent-endpoint")
+    # echo $CONTENT
+    echo $EXIT_CODE
+    local EXIT_CODE="${EXIT_CODE:0-1}"
+    if [ "$EXIT_CODE" == "1" ]; then
+        docker logs react-agent-endpoint
+        exit 1
+    fi
+
+}
+
+function remove_chinook_data(){
+    echo "Removing chinook data..."
+    cd $WORKDIR
+    if [ -d "chinook-database" ]; then
+        rm -rf chinook-database
+    fi
+    echo "Chinook data removed!"
+}
+
+function ingest_data_and_validate() {
+    echo "Ingesting data"
+    cd $WORKDIR/GenAIExamples/AgentQnA/retrieval_tool/
+    echo $PWD
+    local CONTENT=$(bash run_ingest_data.sh)
+    local EXIT_CODE=$(validate "$CONTENT" "Data preparation succeeded" "dataprep-redis-server")
+    echo "$EXIT_CODE"
+    local EXIT_CODE="${EXIT_CODE:0-1}"
+    echo "return value is $EXIT_CODE"
+    if [ "$EXIT_CODE" == "1" ]; then
+        docker logs dataprep-redis-server
+        return 1
+    fi
+}
+
+function validate_retrieval_tool() {
+    echo "----------------Test retrieval tool ----------------"
+    local CONTENT=$(http_proxy="" curl http://${ip_address}:8889/v1/retrievaltool -X POST -H "Content-Type: application/json" -d '{
+     "text": "Who sang Thriller"
+    }')
+    local EXIT_CODE=$(validate "$CONTENT" "Thriller" "retrieval-tool")
+
+    if [ "$EXIT_CODE" == "1" ]; then
+        docker logs retrievaltool-xeon-backend-server
+        exit 1
+    fi
+}
+
+function main() {
+    echo "==================== Prepare data ===================="
+    download_chinook_data
+    echo "==================== Data prepare done ===================="
+
+    echo "==================== Start all services ===================="
+    start_all_services
+    echo "==================== all services started ===================="
+
+    echo "==================== Ingest data ===================="
+    ingest_data_and_validate
+    echo "==================== Data ingestion completed ===================="
+
+    echo "==================== Validate retrieval tool ===================="
+    validate_retrieval_tool
+    echo "==================== Retrieval tool validated ===================="
+
+    echo "==================== Validate agent service ===================="
+    validate_agent_service
+    echo "==================== Agent service validated ===================="
+}
+
+
+remove_chinook_data
+
+main
+
+remove_chinook_data

AgentQnA/tests/step4_launch_and_validate_agent_openai.sh

@@ -11,13 +11,22 @@ echo "WORKDIR=${WORKDIR}"
 export ip_address=$(hostname -I | awk '{print $1}')
 export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
 
+
+function download_chinook_data(){
+    echo "Downloading chinook data..."
+    cd $WORKDIR
+    git clone https://github.com/lerocha/chinook-database.git
+    cp chinook-database/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite $WORKDIR/GenAIExamples/AgentQnA/tests/
+}
+
 function start_agent_and_api_server() {
     echo "Starting CRAG server"
     docker run -d --runtime=runc --name=kdd-cup-24-crag-service -p=8080:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0
 
     echo "Starting Agent services"
-    cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon
+    cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon/
     bash launch_agent_service_openai.sh
+    sleep 2m
 }
 
 function validate() {
@@ -35,19 +44,64 @@ function validate() {
 }
 
 function validate_agent_service() {
-    echo "----------------Test agent ----------------"
-    local CONTENT=$(http_proxy="" curl http://${ip_address}:9090/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
-     "query": "Tell me about Michael Jackson song thriller"
-    }')
-    local EXIT_CODE=$(validate "$CONTENT" "Thriller" "react-agent-endpoint")
-    docker logs react-agent-endpoint
+    # # test worker rag agent
+    echo "======================Testing worker rag agent======================"
+    export agent_port="9095"
+    prompt="Tell me about Michael Jackson song Thriller"
+    local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt" --agent_role "worker" --ext_port $agent_port)
+    # echo $CONTENT
+    local EXIT_CODE=$(validate "$CONTENT" "Thriller" "rag-agent-endpoint")
+    echo $EXIT_CODE
+    local EXIT_CODE="${EXIT_CODE:0-1}"
     if [ "$EXIT_CODE" == "1" ]; then
+        docker logs rag-agent-endpoint
+        exit 1
+    fi
+
+    # # test worker sql agent
+    echo "======================Testing worker sql agent======================"
+    export agent_port="9096"
+    prompt="How many employees are there in the company?"
+    local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt" --agent_role "worker" --ext_port $agent_port)
+    local EXIT_CODE=$(validate "$CONTENT" "8" "sql-agent-endpoint")
+    echo $CONTENT
+    # echo $EXIT_CODE
+    local EXIT_CODE="${EXIT_CODE:0-1}"
+    if [ "$EXIT_CODE" == "1" ]; then
+        docker logs sql-agent-endpoint
+        exit 1
+    fi
+
+    # test supervisor react agent
+    echo "======================Testing supervisor react agent======================"
+    export agent_port="9090"
+    local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --agent_role "supervisor" --ext_port $agent_port --stream)
+    local EXIT_CODE=$(validate "$CONTENT" "Iron" "react-agent-endpoint")
+    # echo $CONTENT
+    echo $EXIT_CODE
+    local EXIT_CODE="${EXIT_CODE:0-1}"
+    if [ "$EXIT_CODE" == "1" ]; then
+        docker logs react-agent-endpoint
         exit 1
     fi
 
 }
 
+function remove_chinook_data(){
+    echo "Removing chinook data..."
+    cd $WORKDIR
+    if [ -d "chinook-database" ]; then
+        rm -rf chinook-database
+    fi
+    echo "Chinook data removed!"
+}
+
+
 function main() {
+    echo "==================== Prepare data ===================="
+    download_chinook_data
+    echo "==================== Data prepare done ===================="
+
     echo "==================== Start agent ===================="
     start_agent_and_api_server
     echo "==================== Agent started ===================="
@@ -57,4 +111,9 @@ function main() {
     echo "==================== Agent service validated ===================="
 }
 
+
+remove_chinook_data
+
 main
+
+remove_chinook_data

AgentQnA/tests/step4_launch_and_validate_agent_rocm_vllm.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+WORKPATH=$(dirname "$PWD")
+export LOG_PATH=${WORKPATH}
+export WORKDIR=$WORKPATH/../../
+echo "WORKDIR=${WORKDIR}"
+export ip_address=$(hostname -I | awk '{print $1}')
+export host_ip=${ip_address}
+export TOOLSET_PATH=$WORKPATH/tools/
+
+export HF_CACHE_DIR=$WORKPATH/data2/huggingface
+if [ ! -d "$HF_CACHE_DIR" ]; then
+    HF_CACHE_DIR=$WORKDIR/hf_cache
+    mkdir -p "$HF_CACHE_DIR"
+fi
+
+function download_chinook_data(){
+    echo "Downloading chinook data..."
+    cd $WORKDIR
+    git clone https://github.com/lerocha/chinook-database.git
+    cp chinook-database/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite ${WORKPATH}/tests/
+}
+
+function start_agent_and_api_server() {
+    echo "Starting Agent services"
+    cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/amd/gpu/rocm
+    bash launch_agent_service_vllm_rocm.sh
+}
+
+function validate() {
+    local CONTENT="$1"
+    local EXPECTED_RESULT="$2"
+    local SERVICE_NAME="$3"
+
+    if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
+        echo "[ $SERVICE_NAME ] Content is as expected: $CONTENT"
+        echo 0
+    else
+        echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
+        echo 1
+    fi
+}
+
+function validate_agent_service() {
+    # # test worker rag agent
+    echo "======================Testing worker rag agent======================"
+    export agent_port=$(cat ${WORKPATH}/docker_compose/amd/gpu/WORKER_RAG_AGENT_PORT_tmp)
+    prompt="Tell me about Michael Jackson song Thriller"
+    local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt" --agent_role "worker" --ext_port $agent_port)
+    # echo $CONTENT
+    local EXIT_CODE=$(validate "$CONTENT" "Thriller" "rag-agent-endpoint")
+    echo $EXIT_CODE
+    local EXIT_CODE="${EXIT_CODE:0-1}"
+    if [ "$EXIT_CODE" == "1" ]; then
+        docker logs rag-agent-endpoint
+        exit 1
+    fi
+
+     # test worker sql agent
+    echo "======================Testing worker sql agent======================"
+    export agent_port=$(cat ${WORKPATH}/docker_compose/amd/gpu/WORKER_SQL_AGENT_PORT_tmp)
+    prompt="How many employees are there in the company?"
+    local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt" --agent_role "worker" --ext_port $agent_port)
+    local EXIT_CODE=$(validate "$CONTENT" "8" "sql-agent-endpoint")
+    echo $CONTENT
+    # echo $EXIT_CODE
+    local EXIT_CODE="${EXIT_CODE:0-1}"
+    if [ "$EXIT_CODE" == "1" ]; then
+        docker logs sql-agent-endpoint
+        exit 1
+    fi
+
+    # test supervisor react agent
+    echo "======================Testing supervisor react agent======================"
+    export agent_port=$(cat ${WORKPATH}/docker_compose/amd/gpu/SUPERVISOR_REACT_AGENT_PORT_tmp)
+    local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --agent_role "supervisor" --ext_port $agent_port --stream)
+    local EXIT_CODE=$(validate "$CONTENT" "Iron" "react-agent-endpoint")
+    # echo $CONTENT
+    echo $EXIT_CODE
+    local EXIT_CODE="${EXIT_CODE:0-1}"
+    if [ "$EXIT_CODE" == "1" ]; then
+        docker logs react-agent-endpoint
+        exit 1
+    fi
+
+}
+
+function remove_chinook_data(){
+    echo "Removing chinook data..."
+    cd $WORKDIR
+    if [ -d "chinook-database" ]; then
+        rm -rf chinook-database
+    fi
+    echo "Chinook data removed!"
+}
+
+function main() {
+    echo "==================== Prepare data ===================="
+    download_chinook_data
+    echo "==================== Data prepare done ===================="
+
+    echo "==================== Start agent ===================="
+    start_agent_and_api_server
+    echo "==================== Agent started ===================="
+
+    echo "==================== Validate agent service ===================="
+    validate_agent_service
+    echo "==================== Agent service validated ===================="
+}
+
+
+remove_chinook_data
+
+main
+
+remove_chinook_data

AgentQnA/tests/step4_launch_and_validate_agent_tgi.sh

@@ -1,91 +0,0 @@
-#!/bin/bash
-# Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-
-set -e
-
-WORKPATH=$(dirname "$PWD")
-export WORKDIR=$WORKPATH/../../
-echo "WORKDIR=${WORKDIR}"
-export ip_address=$(hostname -I | awk '{print $1}')
-export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
-export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-
-export HF_CACHE_DIR=$WORKDIR/hf_cache
-if [ ! -d "$HF_CACHE_DIR" ]; then
-    mkdir -p "$HF_CACHE_DIR"
-fi
-ls $HF_CACHE_DIR
-
-function start_tgi(){
-    echo "Starting tgi-gaudi server"
-    cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/hpu/gaudi
-    bash launch_tgi_gaudi.sh
-
-}
-
-function start_agent_and_api_server() {
-    echo "Starting CRAG server"
-    docker run -d --runtime=runc --name=kdd-cup-24-crag-service -p=8080:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0
-
-    echo "Starting Agent services"
-    cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/hpu/gaudi
-    bash launch_agent_service_tgi_gaudi.sh
-    sleep 10
-}
-
-function validate() {
-    local CONTENT="$1"
-    local EXPECTED_RESULT="$2"
-    local SERVICE_NAME="$3"
-
-    if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
-        echo "[ $SERVICE_NAME ] Content is as expected: $CONTENT"
-        echo 0
-    else
-        echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
-        echo 1
-    fi
-}
-
-function validate_agent_service() {
-    echo "----------------Test agent ----------------"
-    # local CONTENT=$(http_proxy="" curl http://${ip_address}:9095/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
-    #  "query": "Tell me about Michael Jackson song thriller"
-    # }')
-    export agent_port="9095"
-    local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py)
-    local EXIT_CODE=$(validate "$CONTENT" "Thriller" "rag-agent-endpoint")
-    docker logs rag-agent-endpoint
-    if [ "$EXIT_CODE" == "1" ]; then
-        exit 1
-    fi
-
-    # local CONTENT=$(http_proxy="" curl http://${ip_address}:9090/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
-    #  "query": "Tell me about Michael Jackson song thriller"
-    # }')
-    export agent_port="9090"
-    local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py)
-    local EXIT_CODE=$(validate "$CONTENT" "Thriller" "react-agent-endpoint")
-    docker logs react-agent-endpoint
-    if [ "$EXIT_CODE" == "1" ]; then
-        exit 1
-    fi
-
-}
-
-function main() {
-    echo "==================== Start TGI ===================="
-    start_tgi
-    echo "==================== TGI started ===================="
-
-    echo "==================== Start agent ===================="
-    start_agent_and_api_server
-    echo "==================== Agent started ===================="
-
-    echo "==================== Validate agent service ===================="
-    validate_agent_service
-    echo "==================== Agent service validated ===================="
-}
-
-main

AgentQnA/tests/step4a_launch_and_validate_agent_tgi_on_rocm.sh

@@ -2,26 +2,30 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -ex
+set -e
 
 WORKPATH=$(dirname "$PWD")
+export LOG_PATH=${WORKPATH}
 export WORKDIR=$WORKPATH/../../
 echo "WORKDIR=${WORKDIR}"
 export ip_address=$(hostname -I | awk '{print $1}')
-export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
-export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export host_ip=${ip_address}
+export TOOLSET_PATH=$WORKPATH/tools/
 
-export HF_CACHE_DIR=$WORKDIR/hf_cache
+export HF_CACHE_DIR=$WORKPATH/data2/huggingface
 if [ ! -d "$HF_CACHE_DIR" ]; then
+    HF_CACHE_DIR=$WORKDIR/hf_cache
     mkdir -p "$HF_CACHE_DIR"
 fi
-ls $HF_CACHE_DIR
 
+function download_chinook_data(){
+    echo "Downloading chinook data..."
+    cd $WORKDIR
+    git clone https://github.com/lerocha/chinook-database.git
+    cp chinook-database/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite ${WORKPATH}/tests/
+}
 
 function start_agent_and_api_server() {
-    echo "Starting CRAG server"
-    docker run -d --runtime=runc --name=kdd-cup-24-crag-service -p=8080:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0
-
     echo "Starting Agent services"
     cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/amd/gpu/rocm
     bash launch_agent_service_tgi_rocm.sh
@@ -42,28 +46,63 @@ function validate() {
 }
 
 function validate_agent_service() {
-    echo "----------------Test agent ----------------"
-    local CONTENT=$(http_proxy="" curl http://${ip_address}:9095/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
-     "query": "Tell me about Michael Jackson song thriller"
-    }')
-    local EXIT_CODE=$(validate "$CONTENT" "Thriller" "react-agent-endpoint")
-    docker logs rag-agent-endpoint
+    # # test worker rag agent
+    echo "======================Testing worker rag agent======================"
+    export agent_port=$(cat ${WORKPATH}/docker_compose/amd/gpu/WORKER_RAG_AGENT_PORT_tmp)
+    prompt="Tell me about Michael Jackson song Thriller"
+    local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt" --agent_role "worker" --ext_port $agent_port)
+    # echo $CONTENT
+    local EXIT_CODE=$(validate "$CONTENT" "Thriller" "rag-agent-endpoint")
+    echo $EXIT_CODE
+    local EXIT_CODE="${EXIT_CODE:0-1}"
     if [ "$EXIT_CODE" == "1" ]; then
+        docker logs rag-agent-endpoint
         exit 1
     fi
 
-    local CONTENT=$(http_proxy="" curl http://${ip_address}:9090/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
-     "query": "Tell me about Michael Jackson song thriller"
-    }')
-    local EXIT_CODE=$(validate "$CONTENT" "Thriller" "react-agent-endpoint")
-    docker logs react-agent-endpoint
+     # test worker sql agent
+    echo "======================Testing worker sql agent======================"
+    export agent_port=$(cat ${WORKPATH}/docker_compose/amd/gpu/WORKER_SQL_AGENT_PORT_tmp)
+    prompt="How many employees are there in the company?"
+    local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt" --agent_role "worker" --ext_port $agent_port)
+    local EXIT_CODE=$(validate "$CONTENT" "8" "sql-agent-endpoint")
+    echo $CONTENT
+    # echo $EXIT_CODE
+    local EXIT_CODE="${EXIT_CODE:0-1}"
     if [ "$EXIT_CODE" == "1" ]; then
+        docker logs sql-agent-endpoint
+        exit 1
+    fi
+
+    # test supervisor react agent
+    echo "======================Testing supervisor react agent======================"
+    export agent_port=$(cat ${WORKPATH}/docker_compose/amd/gpu/SUPERVISOR_REACT_AGENT_PORT_tmp)
+    local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --agent_role "supervisor" --ext_port $agent_port --stream)
+    local EXIT_CODE=$(validate "$CONTENT" "Iron" "react-agent-endpoint")
+    # echo $CONTENT
+    echo $EXIT_CODE
+    local EXIT_CODE="${EXIT_CODE:0-1}"
+    if [ "$EXIT_CODE" == "1" ]; then
+        docker logs react-agent-endpoint
         exit 1
     fi
 
 }
 
+function remove_chinook_data(){
+    echo "Removing chinook data..."
+    cd $WORKDIR
+    if [ -d "chinook-database" ]; then
+        rm -rf chinook-database
+    fi
+    echo "Chinook data removed!"
+}
+
 function main() {
+    echo "==================== Prepare data ===================="
+    download_chinook_data
+    echo "==================== Data prepare done ===================="
+
     echo "==================== Start agent ===================="
     start_agent_and_api_server
     echo "==================== Agent started ===================="
@@ -73,4 +112,9 @@ function main() {
     echo "==================== Agent service validated ===================="
 }
 
+
+remove_chinook_data
+
 main
+
+remove_chinook_data

AgentQnA/tests/test.py

@@ -1,25 +1,77 @@
-# Copyright (C) 2024 Intel Corporation
+# Copyright (C) 2025 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-import os
+import argparse
+import json
+import uuid
 
 import requests
 
 
-def generate_answer_agent_api(url, prompt):
+def process_request(url, query, is_stream=False):
     proxies = {"http": ""}
-    payload = {
-        "query": prompt,
-    }
-    response = requests.post(url, json=payload, proxies=proxies)
-    answer = response.json()["text"]
-    return answer
+    content = json.dumps(query) if query is not None else None
+    try:
+        resp = requests.post(url=url, data=content, proxies=proxies, stream=is_stream)
+        if not is_stream:
+            ret = resp.json()["text"]
+        else:
+            for line in resp.iter_lines(decode_unicode=True):
+                print(line)
+            ret = None
+
+        resp.raise_for_status()  # Raise an exception for unsuccessful HTTP status codes
+        return ret
+    except requests.exceptions.RequestException as e:
+        ret = f"An error occurred:{e}"
+        return None
+
+
+def test_worker_agent(args):
+    url = f"http://{args.ip_addr}:{args.ext_port}/v1/chat/completions"
+    query = {"role": "user", "messages": args.prompt, "stream": "false"}
+    ret = process_request(url, query)
+    print("Response: ", ret)
+
+
+def add_message_and_run(url, user_message, thread_id, stream=False):
+    print("User message: ", user_message)
+    query = {"role": "user", "messages": user_message, "thread_id": thread_id, "stream": stream}
+    ret = process_request(url, query, is_stream=stream)
+    print("Response: ", ret)
+
+
+def test_chat_completion_multi_turn(args):
+    url = f"http://{args.ip_addr}:{args.ext_port}/v1/chat/completions"
+    thread_id = f"{uuid.uuid4()}"
+
+    # first turn
+    print("===============First turn==================")
+    user_message = "Which artist has the most albums in the database?"
+    add_message_and_run(url, user_message, thread_id, stream=args.stream)
+    print("===============End of first turn==================")
+
+    # second turn
+    print("===============Second turn==================")
+    user_message = "Give me a few examples of the artist's albums?"
+    add_message_and_run(url, user_message, thread_id, stream=args.stream)
+    print("===============End of second turn==================")
 
 
 if __name__ == "__main__":
-    ip_address = os.getenv("ip_address", "localhost")
-    agent_port = os.getenv("agent_port", "9095")
-    url = f"http://{ip_address}:{agent_port}/v1/chat/completions"
-    prompt = "Tell me about Michael Jackson song thriller"
-    answer = generate_answer_agent_api(url, prompt)
-    print(answer)
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--ip_addr", type=str, default="127.0.0.1", help="endpoint ip address")
+    parser.add_argument("--ext_port", type=str, default="9090", help="endpoint port")
+    parser.add_argument("--stream", action="store_true", help="streaming mode")
+    parser.add_argument("--prompt", type=str, help="prompt message")
+    parser.add_argument("--agent_role", type=str, default="supervisor", help="supervisor or worker")
+    args, _ = parser.parse_known_args()
+
+    print(args)
+
+    if args.agent_role == "supervisor":
+        test_chat_completion_multi_turn(args)
+    elif args.agent_role == "worker":
+        test_worker_agent(args)
+    else:
+        raise ValueError("Invalid agent role")

AgentQnA/tests/test_compose_on_gaudi.sh

@@ -1,8 +1,7 @@
 #!/bin/bash
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
-
-set -e
+set -xe
 
 WORKPATH=$(dirname "$PWD")
 export WORKDIR=$WORKPATH/../../
@@ -10,6 +9,15 @@ echo "WORKDIR=${WORKDIR}"
 export ip_address=$(hostname -I | awk '{print $1}')
 export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
 export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
+export no_proxy="$no_proxy,rag-agent-endpoint,sql-agent-endpoint,react-agent-endpoint,agent-ui,vllm-gaudi-server,jaeger,grafana,prometheus,127.0.0.1,localhost,0.0.0.0,$ip_address"
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+export REGISTRY=${IMAGE_REPO}
+export TAG=${IMAGE_TAG}
+export MODEL_CACHE=${model_cache:-"./data"}
+
 
 function stop_crag() {
     cid=$(docker ps -aq --filter "name=kdd-cup-24-crag-service")
@@ -17,7 +25,7 @@ function stop_crag() {
     if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi
 }
 
-function stop_agent_docker() {
+function stop_agent_containers() {
     cd $WORKPATH/docker_compose/intel/hpu/gaudi/
     container_list=$(cat compose.yaml | grep container_name | cut -d':' -f2)
     for container_name in $container_list; do
@@ -27,7 +35,19 @@ function stop_agent_docker() {
     done
 }
 
-function stop_tgi(){
+function stop_telemetry_containers(){
+    cd $WORKPATH/docker_compose/intel/hpu/gaudi/
+    container_list=$(cat compose.telemetry.yaml | grep container_name | cut -d':' -f2)
+    for container_name in $container_list; do
+        cid=$(docker ps -aq --filter "name=$container_name")
+        echo "Stopping container $container_name"
+        if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi
+    done
+    container_list=$(cat compose.telemetry.yaml | grep container_name | cut -d':' -f2)
+
+}
+
+function stop_llm(){
     cd $WORKPATH/docker_compose/intel/hpu/gaudi/
     container_list=$(cat tgi_gaudi.yaml | grep container_name | cut -d':' -f2)
     for container_name in $container_list; do
@@ -36,6 +56,14 @@ function stop_tgi(){
         if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi
     done
 
+    cid=$(docker ps -aq --filter "name=vllm-gaudi-server")
+    echo "Stopping container $cid"
+    if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi
+
+    cid=$(docker ps -aq --filter "name=test-comps-vllm-gaudi-service")
+    echo "Stopping container $cid"
+    if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi
+
 }
 
 function stop_retrieval_tool() {
@@ -50,36 +78,31 @@ function stop_retrieval_tool() {
     done
 }
 echo "workpath: $WORKPATH"
-echo "=================== Stop containers ===================="
+echo "::group::=================== Stop containers ===================="
+stop_llm
 stop_crag
-stop_tgi
-stop_agent_docker
+stop_agent_containers
 stop_retrieval_tool
+stop_telemetry_containers
+echo "::endgroup::"
 
 cd $WORKPATH/tests
 
-echo "=================== #1 Building docker images===================="
-bash step1_build_images.sh
-echo "=================== #1 Building docker images completed===================="
+echo "::group::=================== Building docker images===================="
+bash step1_build_images.sh gaudi_vllm > docker_image_build.log
+echo "::endgroup::"
 
-echo "=================== #2 Start retrieval tool===================="
-bash step2_start_retrieval_tool.sh
-echo "=================== #2 Retrieval tool started===================="
+echo "::group::=================== Start agent, API server, retrieval, and ingest data===================="
+bash step4_launch_and_validate_agent_gaudi.sh
+echo "::endgroup::"
 
-echo "=================== #3 Ingest data and validate retrieval===================="
-bash step3_ingest_data_and_validate_retrieval.sh
-echo "=================== #3 Data ingestion and validation completed===================="
-
-echo "=================== #4 Start agent and API server===================="
-bash step4_launch_and_validate_agent_tgi.sh
-echo "=================== #4 Agent test passed ===================="
-
-echo "=================== #5 Stop agent and API server===================="
+echo "::group::=================== Stop agent and API server===================="
+stop_llm
 stop_crag
-stop_agent_docker
+stop_agent_containers
 stop_retrieval_tool
-echo "=================== #5 Agent and API server stopped===================="
-
+stop_telemetry_containers
 echo y | docker system prune
+echo "::endgroup::"
 
-echo "ALL DONE!"
+echo "ALL DONE!!"

AgentQnA/tests/test_compose_on_rocm.sh

@@ -2,14 +2,22 @@
 # Copyright (C) 2024 Advanced Micro Devices, Inc.
 # SPDX-License-Identifier: Apache-2.0
 
-set -e
+set -xe
 
 WORKPATH=$(dirname "$PWD")
+ls $WORKPATH
 export WORKDIR=$WORKPATH/../../
 echo "WORKDIR=${WORKDIR}"
 export ip_address=$(hostname -I | awk '{print $1}')
 export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
-export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
+export TOOLSET_PATH=$WORKPATH/tools/
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+export REGISTRY=${IMAGE_REPO}
+export TAG=${IMAGE_TAG}
+export MODEL_CACHE=${model_cache:-"./data"}
 
 function stop_crag() {
     cid=$(docker ps -aq --filter "name=kdd-cup-24-crag-service")
@@ -19,13 +27,7 @@ function stop_crag() {
 
 function stop_agent_docker() {
     cd $WORKPATH/docker_compose/amd/gpu/rocm
-    # docker compose -f compose.yaml down
-    container_list=$(cat compose.yaml | grep container_name | cut -d':' -f2)
-    for container_name in $container_list; do
-        cid=$(docker ps -aq --filter "name=$container_name")
-        echo "Stopping container $container_name"
-        if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi
-    done
+    bash stop_agent_service_tgi_rocm.sh
 }
 
 function stop_retrieval_tool() {
@@ -41,35 +43,36 @@ function stop_retrieval_tool() {
     done
 }
 echo "workpath: $WORKPATH"
-echo "=================== Stop containers ===================="
+echo "::group::=================== Stop containers ===================="
 stop_crag
 stop_agent_docker
 stop_retrieval_tool
+echo "::endgroup::=================== Stop containers completed ===================="
 
 cd $WORKPATH/tests
 
-echo "=================== #1 Building docker images===================="
-bash step1_build_images.sh
-echo "=================== #1 Building docker images completed===================="
+echo "::group::=================== #1 Building docker images===================="
+bash step1_build_images.sh rocm > docker_image_build.log
+echo "::endgroup::=================== #1 Building docker images completed===================="
 
-echo "=================== #2 Start retrieval tool===================="
+echo "::group::=================== #2 Start retrieval tool===================="
 bash step2_start_retrieval_tool.sh
-echo "=================== #2 Retrieval tool started===================="
+echo "::endgroup::=================== #2 Retrieval tool started===================="
 
-echo "=================== #3 Ingest data and validate retrieval===================="
+echo "::group::=================== #3 Ingest data and validate retrieval===================="
 bash step3_ingest_data_and_validate_retrieval.sh
-echo "=================== #3 Data ingestion and validation completed===================="
+echo "::endgroup::=================== #3 Data ingestion and validation completed===================="
 
-echo "=================== #4 Start agent and API server===================="
+echo "::group::=================== #4 Start agent and API server===================="
 bash step4a_launch_and_validate_agent_tgi_on_rocm.sh
-echo "=================== #4 Agent test passed ===================="
+echo "::endgroup::=================== #4 Agent test passed ===================="
 
-echo "=================== #5 Stop agent and API server===================="
+echo "::group::=================== #5 Stop agent and API server===================="
 stop_crag
 stop_agent_docker
 stop_retrieval_tool
-echo "=================== #5 Agent and API server stopped===================="
+echo "::endgroup::=================== #5 Agent and API server stopped===================="
 
 echo y | docker system prune
 
-echo "ALL DONE!"
+echo "ALL DONE!!"

AgentQnA/tests/test_compose_vllm_on_rocm.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# Copyright (C) 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+WORKPATH=$(dirname "$PWD")
+export WORKDIR=${WORKPATH}/../../
+echo "WORKDIR=${WORKDIR}"
+export ip_address=$(hostname -I | awk '{print $1}')
+export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
+export TOOLSET_PATH=$WORKPATH/tools/
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+export REGISTRY=${IMAGE_REPO}
+export TAG=${IMAGE_TAG}
+export MODEL_CACHE=${model_cache:-"./data"}
+
+function stop_crag() {
+    cid=$(docker ps -aq --filter "name=kdd-cup-24-crag-service")
+    echo "Stopping container kdd-cup-24-crag-service with cid $cid"
+    if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi
+}
+
+function stop_agent_docker() {
+    cd $WORKPATH/docker_compose/amd/gpu/rocm
+    bash stop_agent_service_vllm_rocm.sh
+}
+
+function stop_retrieval_tool() {
+    echo "Stopping Retrieval tool"
+    local RETRIEVAL_TOOL_PATH=$WORKDIR/GenAIExamples/DocIndexRetriever
+    cd $RETRIEVAL_TOOL_PATH/docker_compose/intel/cpu/xeon/
+    docker compose -f compose.yaml down
+}
+
+echo "workpath: $WORKPATH"
+echo "::group::=================== Stop containers ===================="
+stop_crag
+stop_agent_docker
+stop_retrieval_tool
+echo "::endgroup::"
+
+cd $WORKPATH/tests
+
+echo "::group::=================== #1 Building docker images===================="
+bash step1_build_images.sh rocm_vllm > docker_image_build.log
+echo "::endgroup::=================== #1 Building docker images completed===================="
+
+echo "::group::=================== #2 Start retrieval tool===================="
+bash step2_start_retrieval_tool_rocm_vllm.sh
+echo "::endgroup::=================== #2 Retrieval tool started===================="
+
+echo "::group::=================== #3 Ingest data and validate retrieval===================="
+bash step3_ingest_data_and_validate_retrieval_rocm_vllm.sh
+echo "::endgroup::=================== #3 Data ingestion and validation completed===================="
+
+echo "::group::=================== #4 Start agent and API server===================="
+bash step4_launch_and_validate_agent_rocm_vllm.sh
+echo "::endgroup::=================== #4 Agent test passed ===================="
+
+echo "::group::=================== #5 Stop agent and API server===================="
+stop_crag
+stop_agent_docker
+stop_retrieval_tool
+echo "::endgroup::=================== #5 Agent and API server stopped===================="
+
+echo y | docker system prune
+
+echo "ALL DONE!!"

AgentQnA/tools/supervisor_agent_tools.yaml

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 search_knowledge_base:
-  description: Search knowledge base for a given query. Returns text related to the query.
+  description: Search a knowledge base for a given query. Returns text related to the query.
   callable_api: tools.py:search_knowledge_base
   args_schema:
     query:
@@ -10,6 +10,15 @@ search_knowledge_base:
       description: query
   return_output: retrieved_data
 
+search_artist_database:
+  description: Search a SQL database on artists and their music with a natural language query. Returns text related to the query.
+  callable_api: tools.py:search_sql_database
+  args_schema:
+    query:
+      type: str
+      description: natural language query
+  return_output: retrieved_data
+
 get_artist_birth_place:
   description: Get the birth place of an artist.
   callable_api: tools.py:get_artist_birth_place

AgentQnA/tools/supervisor_agent_webtools.yaml

@@ -0,0 +1,77 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+search_web_base:
+  description: Search a web base for a given query. Returns text related to the query.
+  callable_api: tools.py:search_web_base
+  args_schema:
+    query:
+      type: str
+      description: query
+  return_output: retrieved_data
+
+search_knowledge_base:
+  description: Search a knowledge base for a given query. Returns text related to the query.
+  callable_api: tools.py:search_knowledge_base
+  args_schema:
+    query:
+      type: str
+      description: query
+  return_output: retrieved_data
+
+search_artist_database:
+  description: Search a SQL database on artists and their music with a natural language query. Returns text related to the query.
+  callable_api: tools.py:search_sql_database
+  args_schema:
+    query:
+      type: str
+      description: natural language query
+  return_output: retrieved_data
+
+get_artist_birth_place:
+  description: Get the birth place of an artist.
+  callable_api: tools.py:get_artist_birth_place
+  args_schema:
+    artist_name:
+      type: str
+      description: artist name
+  return_output: birth_place
+
+get_billboard_rank_date:
+  description: Get Billboard ranking for a specific rank and date.
+  callable_api: tools.py:get_billboard_rank_date
+  args_schema:
+    rank:
+      type: int
+      description: the rank of interest, for example 1 for top 1
+    date:
+      type: str
+      description: date
+  return_output: billboard_info
+
+get_song_release_date:
+  description: Get the release date of a song.
+  callable_api: tools.py:get_song_release_date
+  args_schema:
+    song_name:
+      type: str
+      description: song name
+  return_output: release_date
+
+get_members:
+  description: Get the member list of a band.
+  callable_api: tools.py:get_members
+  args_schema:
+    band_name:
+      type: str
+      description: band name
+  return_output: members
+
+get_grammy_best_artist_by_year:
+  description: Get the Grammy Best New Artist for a specific year.
+  callable_api: tools.py:get_grammy_best_artist_by_year
+  args_schema:
+    year:
+      type: int
+      description: year
+  return_output: grammy_best_new_artist

AgentQnA/tools/tools.py

@@ -4,22 +4,62 @@
 import os
 
 import requests
+from comps.cores.telemetry.opea_telemetry import opea_telemetry, tracer
 from tools.pycragapi import CRAG
 
 
+@opea_telemetry
+def search_web_base(query: str) -> str:
+    import os
+
+    from langchain_core.tools import Tool
+    from langchain_google_community import GoogleSearchAPIWrapper
+
+    search = GoogleSearchAPIWrapper()
+
+    tool = Tool(
+        name="google_search",
+        description="Search Google for recent results.",
+        func=search.run,
+    )
+
+    response = tool.run(query)
+    return response
+
+
+@opea_telemetry
 def search_knowledge_base(query: str) -> str:
-    """Search the knowledge base for a specific query."""
-    # use worker agent (DocGrader) to search the knowledge base
+    """Search a knowledge base about music and singers for a given query.
+
+    Returns text related to the query.
+    """
     url = os.environ.get("WORKER_AGENT_URL")
     print(url)
     proxies = {"http": ""}
     payload = {
-        "query": query,
+        "messages": query,
     }
     response = requests.post(url, json=payload, proxies=proxies)
     return response.json()["text"]
 
 
+@opea_telemetry
+def search_sql_database(query: str) -> str:
+    """Search a SQL database on artists and their music with a natural language query.
+
+    Returns text related to the query.
+    """
+    url = os.environ.get("SQL_AGENT_URL")
+    print(url)
+    proxies = {"http": ""}
+    payload = {
+        "messages": query,
+    }
+    response = requests.post(url, json=payload, proxies=proxies)
+    return response.json()["text"]
+
+
+@opea_telemetry
 def get_grammy_best_artist_by_year(year: int) -> dict:
     """Get the Grammy Best New Artist for a specific year."""
     api = CRAG()
@@ -27,18 +67,21 @@ def get_grammy_best_artist_by_year(year: int) -> dict:
     return api.music_grammy_get_best_artist_by_year(year)
 
 
+@opea_telemetry
 def get_members(band_name: str) -> dict:
     """Get the member list of a band."""
     api = CRAG()
     return api.music_get_members(band_name)
 
 
+@opea_telemetry
 def get_artist_birth_place(artist_name: str) -> dict:
     """Get the birthplace of an artist."""
     api = CRAG()
     return api.music_get_artist_birth_place(artist_name)
 
 
+@opea_telemetry
 def get_billboard_rank_date(rank: int, date: str = None) -> dict:
     """Get Billboard ranking for a specific rank and date."""
     api = CRAG()
@@ -46,6 +89,7 @@ def get_billboard_rank_date(rank: int, date: str = None) -> dict:
     return api.music_get_billboard_rank_date(rank, date)
 
 
+@opea_telemetry
 def get_song_release_date(song_name: str) -> dict:
     """Get the release date of a song."""
     api = CRAG()

AgentQnA/ui/docker/Dockerfile

@@ -0,0 +1,203 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# syntax=docker/dockerfile:1
+# Initialize device type args
+# use build args in the docker build command with --build-arg="BUILDARG=true"
+ARG USE_CUDA=false
+ARG USE_OLLAMA=false
+# Tested with cu117 for CUDA 11 and cu121 for CUDA 12 (default)
+ARG USE_CUDA_VER=cu121
+# any sentence transformer model; models to use can be found at https://huggingface.co/models?library=sentence-transformers
+# Leaderboard: https://huggingface.co/spaces/mteb/leaderboard
+# for better performance and multilangauge support use "intfloat/multilingual-e5-large" (~2.5GB) or "intfloat/multilingual-e5-base" (~1.5GB)
+# IMPORTANT: If you change the embedding model (sentence-transformers/all-MiniLM-L6-v2) and vice versa, you aren't able to use RAG Chat with your previous documents loaded in the WebUI! You need to re-embed them.
+ARG USE_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+ARG USE_RERANKING_MODEL=""
+
+# Tiktoken encoding name; models to use can be found at https://huggingface.co/models?library=tiktoken
+ARG USE_TIKTOKEN_ENCODING_NAME="cl100k_base"
+
+ARG BUILD_HASH=dev-build
+# Override at your own risk - non-root configurations are untested
+ARG UID=0
+ARG GID=0
+
+######## WebUI frontend ########
+FROM --platform=$BUILDPLATFORM node:22-alpine3.20 AS build
+ARG BUILD_HASH
+
+WORKDIR /app
+
+COPY open_webui_patches /app/patches
+ARG WEBUI_VERSION=v0.5.20
+RUN apk add --no-cache git
+
+# Clone code and use patch
+RUN git config --global user.name "opea" && \
+    git config --global user.email "" && \
+    git clone https://github.com/open-webui/open-webui.git
+
+WORKDIR /app/open-webui
+
+RUN git checkout ${WEBUI_VERSION} && git am /app/patches/*.patch
+
+WORKDIR /app
+
+RUN mv open-webui/* . && rm -fr open-webui && ls -lrth /app/backend/
+
+RUN npm install onnxruntime-node --onnxruntime-node-install-cuda=skip
+RUN apk update && \
+    apk add --no-cache wget && \
+    wget https://github.com/microsoft/onnxruntime/releases/download/v1.20.1/onnxruntime-linux-x64-gpu-1.20.1.tgz
+
+ENV APP_BUILD_HASH=${BUILD_HASH}
+RUN npm run build
+
+######## WebUI backend ########
+FROM python:3.11-slim-bookworm AS base
+
+# Use args
+ARG USE_CUDA
+ARG USE_OLLAMA
+ARG USE_CUDA_VER
+ARG USE_EMBEDDING_MODEL
+ARG USE_RERANKING_MODEL
+ARG UID
+ARG GID
+
+## Basis ##
+ENV ENV=prod \
+    PORT=8080 \
+    # pass build args to the build
+    USE_OLLAMA_DOCKER=${USE_OLLAMA} \
+    USE_CUDA_DOCKER=${USE_CUDA} \
+    USE_CUDA_DOCKER_VER=${USE_CUDA_VER} \
+    USE_EMBEDDING_MODEL_DOCKER=${USE_EMBEDDING_MODEL} \
+    USE_RERANKING_MODEL_DOCKER=${USE_RERANKING_MODEL}
+
+## Basis URL Config ##
+ENV OLLAMA_BASE_URL="/ollama" \
+    OPENAI_API_BASE_URL=""
+
+## API Key and Security Config ##
+ENV OPENAI_API_KEY="" \
+    WEBUI_SECRET_KEY="" \
+    SCARF_NO_ANALYTICS=true \
+    DO_NOT_TRACK=true \
+    ANONYMIZED_TELEMETRY=false
+
+#### Other models #########################################################
+## whisper TTS model settings ##
+ENV WHISPER_MODEL="base" \
+    WHISPER_MODEL_DIR="/app/backend/data/cache/whisper/models"
+
+## RAG Embedding model settings ##
+ENV RAG_EMBEDDING_MODEL="$USE_EMBEDDING_MODEL_DOCKER" \
+    RAG_RERANKING_MODEL="$USE_RERANKING_MODEL_DOCKER" \
+    SENTENCE_TRANSFORMERS_HOME="/app/backend/data/cache/embedding/models"
+
+## Tiktoken model settings ##
+ENV TIKTOKEN_ENCODING_NAME="cl100k_base" \
+    TIKTOKEN_CACHE_DIR="/app/backend/data/cache/tiktoken"
+
+## Hugging Face download cache ##
+ENV HF_HOME="/app/backend/data/cache/embedding/models"
+
+## Torch Extensions ##
+# ENV TORCH_EXTENSIONS_DIR="/.cache/torch_extensions"
+
+#### Other models ##########################################################
+
+COPY --from=build /app/backend /app/backend
+
+WORKDIR /app/backend
+
+
+ENV HOME=/root
+# Create user and group if not root
+RUN if [ $UID -ne 0 ]; then \
+    if [ $GID -ne 0 ]; then \
+    addgroup --gid $GID app; \
+    fi; \
+    adduser --uid $UID --gid $GID --home $HOME --disabled-password --no-create-home app; \
+    fi
+
+RUN mkdir -p $HOME/.cache/chroma
+RUN printf 00000000-0000-0000-0000-000000000000 > $HOME/.cache/chroma/telemetry_user_id
+
+# Make sure the user has access to the app and root directory
+RUN chown -R $UID:$GID /app $HOME
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN if [ "$USE_OLLAMA" = "true" ]; then \
+    apt-get update && \
+    # Install pandoc and netcat
+    apt-get install -y --no-install-recommends git build-essential pandoc netcat-openbsd curl && \
+    apt-get install -y --no-install-recommends gcc python3-dev && \
+    # for RAG OCR
+    apt-get install -y --no-install-recommends ffmpeg libsm6 libxext6 && \
+    # install helper tools
+    apt-get install -y --no-install-recommends curl jq && \
+    # install ollama
+    curl -fsSL https://ollama.com/install.sh | sh && \
+    # cleanup
+    rm -rf /var/lib/apt/lists/*; \
+    else \
+    apt-get update && \
+    # Install pandoc, netcat and gcc
+    apt-get install -y --no-install-recommends git build-essential pandoc gcc netcat-openbsd curl jq && \
+    apt-get install -y --no-install-recommends gcc python3-dev && \
+    # for RAG OCR
+    apt-get install -y --no-install-recommends ffmpeg libsm6 libxext6 && \
+    # cleanup
+    rm -rf /var/lib/apt/lists/*; \
+    fi
+
+# install python dependencies
+# COPY --chown=$UID:$GID ./backend/requirements.txt ./requirements.txt
+# RUN cp /app/backend/requirements.txt ./requirements.txt
+
+RUN pip3 install --no-cache-dir uv && \
+    if [ "$USE_CUDA" = "true" ]; then \
+    # If you use CUDA the whisper and embedding model will be downloaded on first use
+    pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/$USE_CUDA_DOCKER_VER --no-cache-dir && \
+    uv pip install --system -r requirements.txt --no-cache-dir && \
+    python -c "import os; from sentence_transformers import SentenceTransformer; SentenceTransformer(os.environ['RAG_EMBEDDING_MODEL'], device='cpu')" && \
+    python -c "import os; from faster_whisper import WhisperModel; WhisperModel(os.environ['WHISPER_MODEL'], device='cpu', compute_type='int8', download_root=os.environ['WHISPER_MODEL_DIR'])"; \
+    python -c "import os; import tiktoken; tiktoken.get_encoding(os.environ['TIKTOKEN_ENCODING_NAME'])"; \
+    else \
+    pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu --no-cache-dir && \
+    uv pip install --system -r requirements.txt --no-cache-dir && \
+    python -c "import os; from sentence_transformers import SentenceTransformer; SentenceTransformer(os.environ['RAG_EMBEDDING_MODEL'], device='cpu')" && \
+    python -c "import os; from faster_whisper import WhisperModel; WhisperModel(os.environ['WHISPER_MODEL'], device='cpu', compute_type='int8', download_root=os.environ['WHISPER_MODEL_DIR'])"; \
+    python -c "import os; import tiktoken; tiktoken.get_encoding(os.environ['TIKTOKEN_ENCODING_NAME'])"; \
+    fi; \
+    chown -R $UID:$GID /app/backend/data/
+
+
+
+# copy embedding weight from build
+# RUN mkdir -p /root/.cache/chroma/onnx_models/all-MiniLM-L6-v2
+# COPY --from=build /app/onnx /root/.cache/chroma/onnx_models/all-MiniLM-L6-v2/onnx
+
+# copy built frontend files
+COPY --chown=$UID:$GID --from=build /app/build /app/build
+COPY --chown=$UID:$GID --from=build /app/CHANGELOG.md /app/CHANGELOG.md
+COPY --chown=$UID:$GID --from=build /app/package.json /app/package.json
+
+# copy backend files
+# COPY --chown=$UID:$GID ./backend .
+
+EXPOSE 8080
+
+HEALTHCHECK CMD curl --silent --fail http://localhost:${PORT:-8080}/health | jq -ne 'input.status == true' || exit 1
+
+USER $UID:$GID
+
+ARG BUILD_HASH
+ENV WEBUI_BUILD_VERSION=${BUILD_HASH}
+ENV DOCKER=true
+
+CMD [ "bash", "start.sh"]

AgentQnA/ui/open_webui_patches/0001-compatible-opea-agent-tool-content.patch

@@ -0,0 +1,86 @@
+From d90ba418f866bc11848d7d6507aabc6b5e8cc3e2 Mon Sep 17 00:00:00 2001
+From: lkk12014402 <kaokao.lv@intel.com>
+Date: Mon, 7 Apr 2025 07:22:53 +0000
+Subject: [PATCH] compatible opea agent tool content
+
+---
+ backend/open_webui/utils/middleware.py | 56 ++++++++++++++++++++++++++
+ 1 file changed, 56 insertions(+)
+
+diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py
+index 289d887df..fddbe8ee1 100644
+--- a/backend/open_webui/utils/middleware.py
++++ b/backend/open_webui/utils/middleware.py
+@@ -1465,6 +1465,8 @@ async def process_chat_response(
+                 async def stream_body_handler(response):
+                     nonlocal content
+                     nonlocal content_blocks
++                    nonlocal events
++                    sources = []
+ 
+                     response_tool_calls = []
+ 
+@@ -1486,6 +1488,60 @@ async def process_chat_response(
+                         try:
+                             data = json.loads(data)
+ 
++                            tool_content_block = []
++                            if data.get("tool_name"):
++                                sources.append(
++                                    {
++                                        "source": {
++                                            "name": f"TOOL:{data.get('tool_name')}"},
++                                        "document": [data.get("tool_content")],
++                                        "metadata": [{
++                                            "source": f"TOOL:{data.get('tool_name')}"}],
++                                    }
++                                )
++                                events.append({"sources": sources})
++
++                                await event_emitter(
++                                    {
++                                        "type": "chat:completion",
++                                        "data": {"sources": sources},
++                                    }
++                                )
++                                tool_content_block = [
++                                    {
++                                        "type": "tool_calls",
++                                        "content": [
++                                            {"id": data.get('tool_name'), "function": {"name": data.get('tool_name')}}
++                                        ]
++                                    }
++                                ]
++
++                                await event_emitter(
++                                    {
++                                        "type": "chat:completion",
++                                        "data": {
++                                            "content": serialize_content_blocks(tool_content_block),
++                                        },
++                                    }
++                                )
++
++                                tool_content_block = [
++                                        {
++                                            "type": "tool_calls",
++                                            "content": [
++                                                {"id": data.get('tool_name'), "function": {"name": data.get('tool_name')}}
++                                            ],
++                                            "results": [
++                                                {"tool_call_id": data.get('tool_name'), "content": data.get("tool_content")}
++                                            ]
++                                        },
++                                        {
++                                            "type": "text",
++                                            "content": "",
++                                        }
++                                ]
++                                content_blocks.extend(tool_content_block)
++
+                             data, _ = await process_filter_functions(
+                                 request=request,
+                                 filter_functions=filter_functions,
+-- 
+2.34.1
+

AgentQnA/ui/open_webui_patches/0002-update-agent-icloud-upload-feature.patch

@@ -0,0 +1,531 @@
+From 8ad31e50644eab3c9e698d7828b1857919887841 Mon Sep 17 00:00:00 2001
+From: lkk12014402 <kaokao.lv@intel.com>
+Date: Tue, 8 Apr 2025 03:38:09 +0000
+Subject: [PATCH 2/2] update agent icloud upload feature
+
+---
+ src/lib/apis/knowledge/index.ts               |  60 +++++++
+ .../admin/Settings/Connections.svelte         |  50 +++++-
+ .../components/icons/UploadCloudIcon.svelte   |  18 ++
+ src/lib/components/workspace/Knowledge.svelte |  57 +++++-
+ .../KnowledgeBase/AddIcloudContentMenu.svelte | 164 ++++++++++++++++++
+ .../KnowledgeBase/IcloudFiles.svelte          |  37 ++++
+ src/lib/i18n/locales/zh-CN/translation.json   |  15 +-
+ 7 files changed, 396 insertions(+), 5 deletions(-)
+ create mode 100644 src/lib/components/icons/UploadCloudIcon.svelte
+ create mode 100644 src/lib/components/workspace/Knowledge/KnowledgeBase/AddIcloudContentMenu.svelte
+ create mode 100644 src/lib/components/workspace/Knowledge/KnowledgeBase/IcloudFiles.svelte
+
+diff --git a/src/lib/apis/knowledge/index.ts b/src/lib/apis/knowledge/index.ts
+index c5fad1323..32be528a7 100644
+--- a/src/lib/apis/knowledge/index.ts
++++ b/src/lib/apis/knowledge/index.ts
+@@ -345,3 +345,63 @@ export const deleteKnowledgeById = async (token: string, id: string) => {
+ 
+ 	return res;
+ };
++
++export const getIcloudFiles = async (ICLOUD_BASE_URLS: string) => {
++	let error = null;
++
++	const res = await fetch(`${ICLOUD_BASE_URLS}/dataprep/get`, {
++		method: 'POST',
++		headers: {
++			Accept: 'application/json',
++			'Content-Type': 'application/json',
++		}
++	})
++		.then(async (res) => {
++			if (!res.ok) throw await res.json();
++			return res.json();
++		})
++		.then((json) => {
++			return json;
++		})
++		.catch((err) => {
++			error = err.detail;
++
++			console.log(err);
++			return null;
++		});
++
++	if (error) {
++		throw error;
++	}
++
++	return res;
++};
++
++export const updateIcloudFiles = async (ICLOUD_BASE_URLS: string, formData: any) => {
++	let error = null;
++
++	const res = await fetch(`${ICLOUD_BASE_URLS}/dataprep/ingest`, {
++		method: 'POST',
++		body: formData
++	})
++		.then(async (res) => {
++			if (!res.ok) throw await res.json();
++			return res.json();
++		})
++		.then((json) => {
++			return json;
++		})
++		.catch((err) => {
++			error = err.detail;
++
++			console.log(err);
++			return null;
++		});
++
++	if (error) {
++		throw error;
++	}
++
++	return res;
++};
++
+diff --git a/src/lib/components/admin/Settings/Connections.svelte b/src/lib/components/admin/Settings/Connections.svelte
+index 2fcfadaec..3237744d5 100644
+--- a/src/lib/components/admin/Settings/Connections.svelte
++++ b/src/lib/components/admin/Settings/Connections.svelte
+@@ -47,6 +47,9 @@
+ 	let showAddOpenAIConnectionModal = false;
+ 	let showAddOllamaConnectionModal = false;
+ 
++	let ENABLE_ICLOUD_API: null | boolean = (localStorage.getItem('ENABLE_ICLOUD_API') === "enable");
++	let ICLOUD_BASE_URL = localStorage.getItem('ICLOUD_BASE_URL') || '';
++
+ 	const updateOpenAIHandler = async () => {
+ 		if (ENABLE_OPENAI_API !== null) {
+ 			// Remove trailing slashes
+@@ -193,10 +196,22 @@
+ 		}
+ 	});
+ 
++	const updateIcloudHandler = async () => {
++		if (ENABLE_ICLOUD_API) {
++			localStorage.setItem('ICLOUD_BASE_URL', ICLOUD_BASE_URL);
++			localStorage.setItem('ENABLE_ICLOUD_API', "enable");
++		} else {
++			localStorage.setItem('ICLOUD_BASE_URL', '');
++			localStorage.setItem('ENABLE_ICLOUD_API', "");
++		}
++		toast.success($i18n.t('Icloud API settings updated'));
++	};
++
+ 	const submitHandler = async () => {
+ 		updateOpenAIHandler();
+ 		updateOllamaHandler();
+ 		updateDirectConnectionsHandler();
++		updateIcloudHandler();
+ 
+ 		dispatch('save');
+ 	};
+@@ -301,7 +316,7 @@
+ 				</div>
+ 
+ 				{#if ENABLE_OLLAMA_API}
+-					<hr class=" border-gray-100 dark:border-gray-850 my-2" />
++					<hr class=" border-gray-100 dark:border-gray-850" />
+ 
+ 					<div class="">
+ 						<div class="flex justify-between items-center">
+@@ -358,6 +373,39 @@
+ 				{/if}
+ 			</div>
+ 
++			<hr class=" border-gray-50 dark:border-gray-850" />
++
++			<div class="pr-1.5 my-2">
++				<div class="flex justify-between items-center text-sm">
++					<div class="font-medium">{$i18n.t('Icloud File API')}</div>
++
++					<div class="mt-1">
++						<Switch 
++							bind:state={ENABLE_ICLOUD_API} 
++							on:change={async () => {
++								updateIcloudHandler();
++							}}
++						/>
++					</div>
++				</div>
++
++				{#if ENABLE_ICLOUD_API}
++					<hr class=" border-gray-50 dark:border-gray-850 my-2" />
++
++					<div class="">
++						<div class="flex w-full gap-1.5">
++							<div class="flex-1 flex flex-col gap-1.5">
++								<input
++									class="w-full text-sm bg-transparent outline-none"
++									placeholder={$i18n.t('Enter Icloud URL(e.g.') + 'http://localhost:6007/v1)'}
++									bind:value={ICLOUD_BASE_URL}
++								/>
++							</div>
++						</div>
++					</div>
++				{/if}
++			</div>
++
+ 			<hr class=" border-gray-100 dark:border-gray-850" />
+ 
+ 			<div class="pr-1.5 my-2">
+diff --git a/src/lib/components/icons/UploadCloudIcon.svelte b/src/lib/components/icons/UploadCloudIcon.svelte
+new file mode 100644
+index 000000000..eed3bd582
+--- /dev/null
++++ b/src/lib/components/icons/UploadCloudIcon.svelte
+@@ -0,0 +1,18 @@
++<script lang="ts">
++	export let className = 'w-4 h-4';
++</script>
++
++<svg
++	t="1744007283647"
++	viewBox="0 0 1491 1024"
++	version="1.1"
++	xmlns="http://www.w3.org/2000/svg"
++	p-id="1630"
++	class = {className}
++	><path
++		d="M546.047379 263.651842s-90.221363-91.423424-212.63125-16.762074c-109.521121 71.300031-90.154581 201.768179-90.154582 201.76818S0 498.498962 0 759.902727c5.431535 261.003078 264.186314 263.674325 264.186314 263.674326l388.443814 0.422947V744.565318H466.355181l279.434681-279.412421 279.390161 279.412421h-186.297208V1024l377.157796-0.422947s240.812904 0.222604 274.648698-248.092052c16.094262-271.576764-232.754643-325.113003-232.754643-325.113003S1286.205362 48.327085 936.761752 2.470681C637.181417-29.740104 546.047379 263.651842 546.047379 263.651842z"
++		fill="#507BFC"
++		p-id="1631"
++	></path></svg
++>
++
+diff --git a/src/lib/components/workspace/Knowledge.svelte b/src/lib/components/workspace/Knowledge.svelte
+index 57d45312d..43a1f305e 100644
+--- a/src/lib/components/workspace/Knowledge.svelte
++++ b/src/lib/components/workspace/Knowledge.svelte
+@@ -13,7 +13,8 @@
+ 	import {
+ 		getKnowledgeBases,
+ 		deleteKnowledgeById,
+-		getKnowledgeBaseList
++		getKnowledgeBaseList,
++		getIcloudFiles
+ 	} from '$lib/apis/knowledge';
+ 
+ 	import { goto } from '$app/navigation';
+@@ -26,6 +27,11 @@
+ 	import Spinner from '../common/Spinner.svelte';
+ 	import { capitalizeFirstLetter } from '$lib/utils';
+ 	import Tooltip from '../common/Tooltip.svelte';
++	import AddIcloudConnectionModal from '$lib/components/workspace/Knowledge/KnowledgeBase/AddIcloudContentMenu.svelte';
++	import IcloudFiles from '$lib/components/workspace/Knowledge/KnowledgeBase/IcloudFiles.svelte';
++
++	let showAddTextContentModal = false;
++	let IcloudFile = [];
+ 
+ 	let loaded = false;
+ 
+@@ -65,9 +71,26 @@
+ 	};
+ 
+ 	onMount(async () => {
++		await updateIcloudFiles();
++
+ 		knowledgeBases = await getKnowledgeBaseList(localStorage.token);
+ 		loaded = true;
+ 	});
++
++	async function updateIcloudFiles() {
++		let ICLOUD_BASE_URL = localStorage.getItem('ICLOUD_BASE_URL') || '';
++		console.log('ICLOUD_BASE_URL', ICLOUD_BASE_URL);
++		
++		if (ICLOUD_BASE_URL !== '') {
++			const res = await getIcloudFiles(ICLOUD_BASE_URL).catch((e) => {
++				toast.error(`${e}`);
++			});
++
++			if (res) {
++				IcloudFile = res;
++			}
++		}
++	}
+ </script>
+ 
+ <svelte:head>
+@@ -187,11 +210,39 @@
+ 		{/each}
+ 	</div>
+ 
+-	<div class=" text-gray-500 text-xs mt-1 mb-2">
+-		ⓘ {$i18n.t("Use '#' in the prompt input to load and include your knowledge.")}
++	<div class="flex justify-between items-center">
++		<div class="flex md:self-center text-xl font-medium px-0.5 items-center">
++			{$i18n.t('Icloud Knowledge')}
++			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-50 dark:bg-gray-850" />
++			<span class="text-lg font-medium text-gray-500 dark:text-gray-300">{IcloudFile.length}</span>
++		</div>
++		<div>
++			<button
++				class=" px-2 py-2 rounded-xl hover:bg-gray-700/10 dark:hover:bg-gray-100/10 dark:text-gray-300 dark:hover:text-white transition font-medium text-sm flex items-center space-x-1"
++				aria-label={$i18n.t('Upload to Icloud')}
++				on:click={() => {
++					showAddTextContentModal = !showAddTextContentModal;
++				}}
++			>
++				<Plus className="size-3.5" />
++			</button>
++		</div>
++	</div>
++	<hr class="border-gray-100 dark:border-gray-850 my-2" />
++	<div class=" flex overflow-y-auto w-full h-[15rem] scrollbar-hidden text-xs">
++		<IcloudFiles files={IcloudFile} />
+ 	</div>
+ {:else}
+ 	<div class="w-full h-full flex justify-center items-center">
+ 		<Spinner />
+ 	</div>
+ {/if}
++
++<AddIcloudConnectionModal
++	bind:show={showAddTextContentModal}
++	on:updateIcloudFile={async (e) => {
++		if (e.detail.status) {
++			await updateIcloudFiles();
++		}
++	}}
++/>
+diff --git a/src/lib/components/workspace/Knowledge/KnowledgeBase/AddIcloudContentMenu.svelte b/src/lib/components/workspace/Knowledge/KnowledgeBase/AddIcloudContentMenu.svelte
+new file mode 100644
+index 000000000..fb906a0d3
+--- /dev/null
++++ b/src/lib/components/workspace/Knowledge/KnowledgeBase/AddIcloudContentMenu.svelte
+@@ -0,0 +1,164 @@
++<script lang="ts">
++	import { toast } from 'svelte-sonner';
++	import { getContext, onMount, createEventDispatcher } from 'svelte';
++	import Modal from '$lib/components/common/Modal.svelte';
++	import UploadCloudIcon from '$lib/components/icons/UploadCloudIcon.svelte';
++	import Spinner from '$lib/components/common/Spinner.svelte';
++	import { updateIcloudFiles } from '$lib/apis/knowledge';
++
++	const i18n = getContext('i18n');
++	const dispatch = createEventDispatcher();
++
++	export let show = false;
++
++	let url = '';
++
++	let loading = false;
++
++	let selectedFile = null;
++
++	function handleFileSelect(event) {
++		selectedFile = event.target.files[0];
++	}
++
++	function parseAndValidateUrls(normalizedInput: string): string[] {
++		return normalizedInput
++			.split(',')
++			.map((candidate) => {
++				const processed = candidate.replace(/^["']+|["']+$/g, '').trim();
++
++				try {
++					new URL(processed);
++					return processed;
++				} catch {
++					return null;
++				}
++			})
++			.filter((url): url is string => url !== null);
++	}
++
++	async function submitHandler() {
++		loading = true;
++
++		if (!url && !selectedFile) {
++			loading = false;
++			show = false;
++
++			toast.error($i18n.t('URL or File are required'));
++			return;
++		}
++		if (url && selectedFile) {
++			loading = false;
++			show = false;
++
++			toast.error($i18n.t('Upload file or enter URL'));
++			url = '';
++			selectedFile = null;
++			return;
++		}
++
++		const formData = new FormData();
++		if (url) {
++			formData.append('link_list', JSON.stringify(parseAndValidateUrls(url)));
++		}
++		if (selectedFile) {
++			formData.append('files', selectedFile, selectedFile.name);
++		}
++		let ICLOUD_BASE_URL = localStorage.getItem('ICLOUD_BASE_URL') || '';
++		console.log('ICLOUD_BASE_URL', ICLOUD_BASE_URL);
++
++		if (ICLOUD_BASE_URL !== '') {
++			const res = await updateIcloudFiles(ICLOUD_BASE_URL, formData).catch((e) => {
++				toast.error(`${e}`);
++
++				return;
++			});
++
++			if (res) {
++				toast.success($i18n.t('Upload Succeed'));
++				dispatch('updateIcloudFile', { status: true });
++			}
++
++			url = '';
++			selectedFile = null;
++			loading = false;
++			show = false;
++		}
++	}
++</script>
++
++<Modal size="sm" bind:show>
++	<div class="flex flex-col justify-end">
++		<div class=" flex justify-between dark:text-gray-100 px-5 pt-4 pb-2">
++			<div class="flex-col text-lg font-medium self-center font-primary">
++				{$i18n.t('Upload Icloud file')}
++				<span class="text-sm text-gray-500">- {$i18n.t('choose URL or local file')}</span>
++			</div>
++
++			<button
++				class="self-center"
++				on:click={() => {
++					show = false;
++				}}
++			>
++				<svg
++					xmlns="http://www.w3.org/2000/svg"
++					viewBox="0 0 20 20"
++					fill="currentColor"
++					class="w-5 h-5"
++				>
++					<path
++						d="M6.28 5.22a.75.75 0 00-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 101.06 1.06L10 11.06l3.72 3.72a.75.75 0 101.06-1.06L11.06 10l3.72-3.72a.75.75 0 00-1.06-1.06L10 8.94 6.28 5.22z"
++					/>
++				</svg>
++			</button>
++		</div>
++
++		<div class="flex flex-col md:flex-row w-full px-4 pb-4 md:space-x-4 dark:text-gray-200">
++			<div class=" flex flex-col w-full sm:flex-row sm:justify-center sm:space-x-6">
++				<div class="flex items-center w-full">
++					<div class="flex-1 min-w-0 mr-2">
++						<div class="flex flex-col w-full my-8 mx-2">
++							<input
++								class="w-full text-sm bg-transparent placeholder:text-gray-300 outline-none border-b-solid border-b-2 border-blue-500 rounded p-2"
++								type="text"
++								bind:value={url}
++								placeholder={$i18n.t('Upload from URL')}
++							/>
++						</div>
++					</div>
++
++					<div class="flex-none w-[1px] h-[60%] mx-2.5 bg-gray-300"></div>
++
++					<div class="flex-1 min-w-0">
++						<input type="file" id="fileInput" hidden on:change={handleFileSelect} />
++
++						<label
++							for="fileInput"
++							class="cursor-pointer flex flex-col items-center hover:bg-gray-100 rounded-lg p-2 transition-colors"
++						>
++							<UploadCloudIcon className="w-12 h-12 text-gray-500" />
++							<div class="text-xs text-gray-500 pt-2">
++								{selectedFile ? selectedFile.name : '点击上传文件'}
++							</div>
++						</label>
++					</div>
++				</div>
++			</div>
++		</div>
++		{#if loading}
++			<Spinner className="my-4 size-4" />
++		{:else}
++			<button
++				class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-3 px-4 rounded text-sm"
++				on:click={(e) => {
++					e.preventDefault();
++					submitHandler();
++				}}
++			>
++				{$i18n.t('Upload Confirm')}
++			</button>
++		{/if}
++	</div>
++</Modal>
++
+diff --git a/src/lib/components/workspace/Knowledge/KnowledgeBase/IcloudFiles.svelte b/src/lib/components/workspace/Knowledge/KnowledgeBase/IcloudFiles.svelte
+new file mode 100644
+index 000000000..d6490dce2
+--- /dev/null
++++ b/src/lib/components/workspace/Knowledge/KnowledgeBase/IcloudFiles.svelte
+@@ -0,0 +1,37 @@
++<script lang="ts">
++	export let selectedFileId = null;
++	export let files = [];
++
++	export let small = false;
++</script>
++
++<div class="max-h-full flex flex-col w-full">
++	{#each files as file}
++		<div class="mt-1 px-2 flex hover:bg-gray-50 transition">
++			<div class="p-3 bg-black/20 dark:bg-white/10 text-white rounded-xl my-2">
++				<svg
++					xmlns="http://www.w3.org/2000/svg"
++					viewBox="0 0 24 24"
++					fill="currentColor"
++					class=" size-3"
++				>
++					<path
++						fill-rule="evenodd"
++						d="M5.625 1.5c-1.036 0-1.875.84-1.875 1.875v17.25c0 1.035.84 1.875 1.875 1.875h12.75c1.035 0 1.875-.84 1.875-1.875V12.75A3.75 3.75 0 0 0 16.5 9h-1.875a1.875 1.875 0 0 1-1.875-1.875V5.25A3.75 3.75 0 0 0 9 1.5H5.625ZM7.5 15a.75.75 0 0 1 .75-.75h7.5a.75.75 0 0 1 0 1.5h-7.5A.75.75 0 0 1 7.5 15Zm.75 2.25a.75.75 0 0 0 0 1.5H12a.75.75 0 0 0 0-1.5H8.25Z"
++						clip-rule="evenodd"
++					/>
++					<path
++						d="M12.971 1.816A5.23 5.23 0 0 1 14.25 5.25v1.875c0 .207.168.375.375.375H16.5a5.23 5.23 0 0 1 3.434 1.279 9.768 9.768 0 0 0-6.963-6.963Z"
++					/>
++				</svg>
++			</div>
++
++			<div class="flex flex-col justify-center -space-y-0.5 px-2.5 w-full">
++				<div class=" dark:text-gray-100 text-sm font-medium line-clamp-1 mb-1">
++					{file.name}
++				</div>
++			</div>
++		</div>
++	{/each}
++</div>
++
+diff --git a/src/lib/i18n/locales/zh-CN/translation.json b/src/lib/i18n/locales/zh-CN/translation.json
+index ebb53a1b5..d6b72e04d 100644
+--- a/src/lib/i18n/locales/zh-CN/translation.json
++++ b/src/lib/i18n/locales/zh-CN/translation.json
+@@ -1174,5 +1174,18 @@
+ 	"Your entire contribution will go directly to the plugin developer; Open WebUI does not take any percentage. However, the chosen funding platform might have its own fees.": "您的全部捐款将直接给到插件开发者，Open WebUI 不会收取任何比例。但众筹平台可能会有服务费、抽成。",
+ 	"Youtube": "YouTube",
+ 	"Youtube Language": "Youtube 语言",
+-	"Youtube Proxy URL": "Youtube 代理 URL"
++	"Youtube Proxy URL": "Youtube 代理 URL",
++	"Upload Icloud file": "上传到云端",
++	"choose URL or local file": "选择URL或本地文件",
++	"Upload from URL": "从URL上传",
++	"Upload Confirm": "确认上传",
++	"URL or File are required": "未上传文件",
++	"Upload file or enter URL": "文件与URL不能同时提交",
++	"Icloud File": "云端文件",
++	"Icloud File API": "云端存储API",
++	"Enter Icloud URL(e.g.": "输入云端存储URL（例如.",
++	"Upload to Icloud": "上传到云端",
++	"Icloud Knowledge": "云端数据库",
++	"Upload Succeed": "上传文件成功",
++	"Icloud API settings updated": "云端存储API设置已更新"
+ }
+-- 
+2.34.1
+

AgentQnA/ui/open_webui_patches/0003-update-build-script.patch

@@ -0,0 +1,56 @@
+From ebf3218eef81897b536521e2140bdd9176f3ace3 Mon Sep 17 00:00:00 2001
+From: lkk12014402 <kaokao.lv@intel.com>
+Date: Tue, 8 Apr 2025 07:13:20 +0000
+Subject: [PATCH 3/3] update build script
+
+---
+ hatch_build.py | 23 ++++++++++++++++++-----
+ 1 file changed, 18 insertions(+), 5 deletions(-)
+
+diff --git a/hatch_build.py b/hatch_build.py
+index 8ddaf0749..e15d6e99d 100644
+--- a/hatch_build.py
++++ b/hatch_build.py
+@@ -3,21 +3,34 @@ import os
+ import shutil
+ import subprocess
+ from sys import stderr
+-
++ 
+ from hatchling.builders.hooks.plugin.interface import BuildHookInterface
+-
+-
++ 
++ 
+ class CustomBuildHook(BuildHookInterface):
+     def initialize(self, version, build_data):
+         super().initialize(version, build_data)
+-        stderr.write(">>> Building Open Webui frontend\n")
++        stderr.write(">>> Building DCAI小智 frontend\n")
+         npm = shutil.which("npm")
+         if npm is None:
+             raise RuntimeError(
+-                "NodeJS `npm` is required for building Open Webui but it was not found"
++                "NodeJS `npm` is required for building DCAI小智 but it was not found"
+             )
++        stderr.write("### Installing onnxruntime-node\n")
++        subprocess.run([npm, "install", "onnxruntime-node", "--onnxruntime-node-install-cuda=skip"], check=True)  # noqa: S603
++       
++        stderr.write("### Installing huggingface/transformers.js\n")
++        subprocess.run([npm, "i", "@huggingface/transformers"], check=True)  # noqa: S603
++       
++        ort_version = "1.20.1"
++        ort_url = f"https://github.com/microsoft/onnxruntime/releases/download/v{ort_version}/onnxruntime-linux-x64-gpu-{ort_version}.tgz"
++       
++        stderr.write(f"### Downloading onnxruntime binaries from {ort_url}\n")
++        subprocess.run(["curl", "-L", ort_url, "-o", f"onnxruntime-linux-x64-gpu-{ort_version}.tgz"], check=True)  # noqa: S603
++       
+         stderr.write("### npm install\n")
+         subprocess.run([npm, "install"], check=True)  # noqa: S603
++ 
+         stderr.write("\n### npm run build\n")
+         os.environ["APP_BUILD_HASH"] = version
+         subprocess.run([npm, "run", "build"], check=True)  # noqa: S603
+-- 
+2.34.1
+

AgentQnA/ui/open_webui_patches/0004-enhance-tool-formating.patch

@@ -0,0 +1,31 @@
+From 36d61dab9306cb8f12c4497a32781d84f8cfb2e7 Mon Sep 17 00:00:00 2001
+From: lkk12014402 <kaokao.lv@intel.com>
+Date: Tue, 8 Apr 2025 07:22:36 +0000
+Subject: [PATCH 4/4] enhance tool formatting
+
+---
+ backend/open_webui/utils/middleware.py | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py
+index fddbe8ee1..9e44ed91a 100644
+--- a/backend/open_webui/utils/middleware.py
++++ b/backend/open_webui/utils/middleware.py
+@@ -1142,12 +1142,12 @@ async def process_chat_response(
+                                 result_display_content = f"{result_display_content}\n> {tool_name}: {result.get('content', '')}"
+ 
+                             if not raw:
+-                                content = f'{content}\n<details type="tool_calls" done="true" content="{html.escape(json.dumps(block_content))}" results="{html.escape(json.dumps(results))}">\n<summary>Tool Executed</summary>\n{result_display_content}\n</details>\n'
++                                content = f'{content}\n<details type="tool_calls" done="true" content="{html.escape(json.dumps(block_content))}" results="{html.escape(json.dumps(results))}">\n<summary> Tool: {tool_call.get('function', {}).get('name', '')} Executed</summary>\n{result_display_content}\n</details>\n'
+                         else:
+                             tool_calls_display_content = ""
+ 
+                             for tool_call in block_content:
+-                                tool_calls_display_content = f"{tool_calls_display_content}\n> Executing {tool_call.get('function', {}).get('name', '')}"
++                                tool_calls_display_content = f"{tool_calls_display_content}\n> Executing Tool: {tool_call.get('function', {}).get('name', '')}"
+ 
+                             if not raw:
+                                 content = f'{content}\n<details type="tool_calls" done="false" content="{html.escape(json.dumps(block_content))}">\n<summary>Tool Executing...</summary>\n{tool_calls_display_content}\n</details>\n'
+-- 
+2.34.1
+