Split ChatQnA manifest test (#1190)

Signed-off-by: ZePan110 <ze.pan@intel.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/_manifest-e2e.yml

@@ -22,7 +22,72 @@ on:
         type: string
 
 jobs:
+  get-test-case:
+    runs-on: ubuntu-latest
+    outputs:
+      test_cases: ${{ steps.test-case-matrix.outputs.test_cases }}
+      CHECKOUT_REF: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
+    steps:
+      - name: Get checkout ref
+        id: get-checkout-ref
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
+            CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge
+          else
+            CHECKOUT_REF=${{ github.ref }}
+          fi
+          echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT
+          echo "checkout ref ${CHECKOUT_REF}"
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
+          fetch-depth: 0
+
+      - name: Get test matrix
+        shell: bash
+        id: test-case-matrix
+        run: |
+          example_l=$(echo ${{ inputs.example }} | tr '[:upper:]' '[:lower:]')
+          cd ${{ github.workspace }}/${{ inputs.example }}/tests
+          run_test_cases=""
+
+          default_test_case=$(find . -type f -name "test_manifest_on_${{ inputs.hardware }}.sh" | cut -d/ -f2)
+          if [ "$default_test_case" ]; then run_test_cases="$default_test_case"; fi
+          other_test_cases=$(find . -type f -name "test_manifest_*_on_${{ inputs.hardware }}.sh" | cut -d/ -f2)
+          echo "default_test_case=$default_test_case"
+          echo "other_test_cases=$other_test_cases"
+
+          if [ "${{ inputs.tag }}" == "ci" ]; then
+            base_commit=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            "https://api.github.com/repos/opea-project/GenAIExamples/commits?sha=${{ github.event.pull_request.base.ref }}" | jq -r '.[0].sha')
+            merged_commit=$(git log -1 --format='%H')
+            changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | grep -vE '${{ inputs.diff_excluded_files }}')" || true
+          fi
+
+          for test_case in $other_test_cases; do
+            if [ "${{ inputs.tag }}" == "ci" ]; then
+              flag=${test_case%_on_*}
+              flag=${flag#test_compose_}
+              if [[ $(printf '%s\n' "${changed_files[@]}" | grep ${{ inputs.example }} | grep ${flag}) ]]; then
+                run_test_cases="$run_test_cases $test_case"
+              fi
+            else
+              run_test_cases="$run_test_cases $test_case"
+            fi
+          done
+
+          test_cases=$(echo $run_test_cases | tr ' ' '\n' | sort -u | jq -R '.' | jq -sc '.')
+          echo "test_cases=$test_cases"
+          echo "test_cases=$test_cases" >> $GITHUB_OUTPUT
+
   manifest-test:
+    needs: [get-test-case]
+    strategy:
+      matrix:
+        test_case: ${{ fromJSON(needs.get-test-case.outputs.test_cases) }}
+      fail-fast: false
     runs-on: "k8s-${{ inputs.hardware }}"
     continue-on-error: true
     steps:
@@ -45,11 +110,14 @@ jobs:
           fetch-depth: 0
 
       - name: Set variables
+        env:
+          test_case: ${{ matrix.test_case }}
         run: |
           echo "IMAGE_REPO=${OPEA_IMAGE_REPO}opea" >> $GITHUB_ENV
           echo "IMAGE_TAG=${{ inputs.tag }}" >> $GITHUB_ENV
           lower_example=$(echo "${{ inputs.example }}" | tr '[:upper:]' '[:lower:]')
-          echo "NAMESPACE=$lower_example-$(tr -dc a-z0-9 </dev/urandom | head -c 16)" >> $GITHUB_ENV
+          name=$(echo "$test_case" | cut -d/ -f2 | cut -d'_' -f3- |cut -d'_' -f1 | grep -v 'on' | sed 's/^/-/')
+          echo "NAMESPACE=$lower_example$name-$(tr -dc a-z0-9 </dev/urandom | head -c 16)" >> $GITHUB_ENV
           echo "ROLLOUT_TIMEOUT_SECONDS=1800s" >> $GITHUB_ENV
           echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV
           echo "continue_test=true" >> $GITHUB_ENV
@@ -59,15 +127,19 @@ jobs:
 
       - name: Kubectl install
         id: install
+        env:
+          test_case: ${{ matrix.test_case }}
         run: |
-          if [[ ! -f ${{ github.workspace }}/${{ inputs.example }}/tests/test_manifest_on_${{ inputs.hardware }}.sh ]]; then
+          set -x
+          echo "test_case=$test_case"
+          if [[ ! -f ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} ]]; then
             echo "No test script found, exist test!"
             exit 0
           else
-            ${{ github.workspace }}/${{ inputs.example }}/tests/test_manifest_on_${{ inputs.hardware }}.sh init_${{ inputs.example }}
+            ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} init_${{ inputs.example }}
             echo "should_cleanup=true" >> $GITHUB_ENV
             kubectl create ns $NAMESPACE
-            ${{ github.workspace }}/${{ inputs.example }}/tests/test_manifest_on_${{ inputs.hardware }}.sh install_${{ inputs.example }} $NAMESPACE
+            ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} install_${{ inputs.example }} $NAMESPACE
             echo "Testing ${{ inputs.example }}, waiting for pod ready..."
             if kubectl rollout status deployment --namespace "$NAMESPACE" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then
               echo "Testing manifests ${{ inputs.example }}, waiting for pod ready done!"
@@ -82,14 +154,16 @@ jobs:
 
       - name: Validate e2e test
         if: always()
+        env:
+          test_case: ${{ matrix.test_case }}
         run: |
           if $skip_validate; then
             echo "Skip validate"
           else
-            if ${{ github.workspace }}/${{ inputs.example }}/tests/test_manifest_on_${{ inputs.hardware }}.sh validate_${{ inputs.example }} $NAMESPACE ; then
+            if ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} validate_${{ inputs.example }} $NAMESPACE ; then
-              echo "Validate ${{ inputs.example }} successful!"
+              echo "Validate ${test_case} successful!"
             else
-              echo "Validate ${{ inputs.example }} failure!!!"
+              echo "Validate ${test_case} failure!!!"
               echo "Check the logs in 'Dump logs when e2e test failed' step!!!"
               exit 1
             fi

ChatQnA/tests/common/_test_manifest_utils.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+USER_ID=$(whoami)
+MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
+IMAGE_REPO=${IMAGE_REPO:-opea}
+IMAGE_TAG=${IMAGE_TAG:-latest}
+
+ROLLOUT_TIMEOUT_SECONDS="1800s"
+KUBECTL_TIMEOUT_SECONDS="60s"
+
+function init_chatqna() {
+    # replace the mount dir "path: /mnt/opea-models" with "path: $CHART_MOUNT"
+    find ../../kubernetes/intel/*/*/manifest -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \;
+    # replace microservice image tag
+    find ../../kubernetes/intel/*/*/manifest -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
+    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/"
+    find ../../kubernetes/intel/*/*/manifest -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \;
+    # set huggingface token
+    find ../../kubernetes/intel/*/*/manifest -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
+}
+
+function get_end_point() {
+    # $1 is service name, $2 is namespace
+    ip_address=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.clusterIP}')
+    port=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.ports[0].port}')
+    echo "$ip_address:$port"
+}
+
+function _cleanup_ns() {
+    local ns=$1
+    if kubectl get ns $ns; then
+      if ! kubectl delete ns $ns --timeout=$KUBECTL_TIMEOUT_SECONDS; then
+        kubectl delete pods --namespace $ns --force --grace-period=0 --all
+        kubectl delete ns $ns --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
+      fi
+    fi
+}
+
+
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <function_name>"
+    exit 1
+fi
+
+case "$1" in
+    init_ChatQnA)
+        init_chatqna
+        ;;
+    get_end_point)
+        service=$2
+        NAMESPACE=$3
+        get_end_point $service $NAMESPACE
+        ;;
+    _cleanup_ns)
+        NAMESPACE=$2
+        _cleanup_ns $NAMESPACE
+        ;;
+    *)
+        echo "Unknown function: $1"
+        ;;
+esac

ChatQnA/tests/test_manifest_guardrails_on_gaudi.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+USER_ID=$(whoami)
+LOG_PATH=/home/$(whoami)/logs
+MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
+IMAGE_REPO=${IMAGE_REPO:-opea}
+IMAGE_TAG=${IMAGE_TAG:-latest}
+
+ROLLOUT_TIMEOUT_SECONDS="1800s"
+KUBECTL_TIMEOUT_SECONDS="60s"
+
+function validate_chatqna() {
+    local ns=$1
+    local log=$2
+    max_retry=20
+    # make sure microservice retriever-usvc is ready
+    # try to curl retriever-svc for max_retry times
+    test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
+    for ((i=1; i<=max_retry; i++))
+    do
+        endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-retriever-usvc" $ns)
+        curl http://$endpoint_url/v1/retrieval -X POST \
+            -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \
+            -H 'Content-Type: application/json' && break
+        sleep 30
+    done
+    # if i is bigger than max_retry, then exit with error
+    if [ $i -gt $max_retry ]; then
+        echo "Microservice retriever failed, exit with error."
+        return 1
+    fi
+    # make sure microservice tgi-svc is ready
+    for ((i=1; i<=max_retry; i++))
+    do
+        endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-tgi" $ns)
+        curl http://$endpoint_url/generate -X POST \
+            -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \
+            -H 'Content-Type: application/json' && break
+        sleep 30
+    done
+    # if i is bigger than max_retry, then exit with error
+    if [ $i -gt $max_retry ]; then
+        echo "Microservice tgi failed, exit with error."
+        return 1
+    fi
+
+    # check megaservice works
+    # generate a random logfile name to avoid conflict among multiple runners
+    LOGFILE=$LOG_PATH/curlmega_$log.log
+    endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna" $ns)
+    curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE
+    exit_code=$?
+    if [ $exit_code -ne 0 ]; then
+        echo "Megaservice failed, please check the logs in $LOGFILE!"
+        return ${exit_code}
+    fi
+
+    echo "Checking response results, make sure the output is reasonable. "
+    local status=false
+    if [[ -f $LOGFILE ]] &&
+        [[ $(grep -c "\[DONE\]" $LOGFILE) != 0 ]]; then
+        status=true
+    fi
+    if [[ $status == false ]]; then
+        echo "Response check failed, please check the logs in artifacts!"
+        return 1
+    else
+        echo "Response check succeed!"
+    fi
+    return 0
+}
+
+function install_chatqna() {
+    echo "Testing manifests chatqna_guardrails"
+    local ns=$1
+    bash ChatQnA/tests/common/_test_manifest_utils.sh _cleanup_ns $ns
+    pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest
+    kubectl create namespace $ns
+    # install guardrails
+    kubectl apply -f chatqna-guardrails.yaml -n $ns
+    # Sleep enough time for chatqna_guardrails to be ready
+    sleep 60
+}
+
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <function_name>"
+    exit 1
+fi
+
+case "$1" in
+    init_ChatQnA)
+        pushd ChatQnA/tests/common
+        bash _test_manifest_utils.sh init_ChatQnA
+        popd
+        ;;
+    install_ChatQnA)
+        NAMESPACE=$2
+        install_chatqna $NAMESPACE
+        popd
+        ;;
+    validate_ChatQnA)
+        NAMESPACE=$2
+        SERVICE_NAME=chatqna-guardrails
+        validate_chatqna $NAMESPACE chatqna-guardrails
+        ret=$?
+        if [ $ret -ne 0 ]; then
+            exit $ret
+        fi
+        ;;
+
+    *)
+        echo "Unknown function: $1"
+        ;;
+esac

ChatQnA/tests/test_manifest_guardrails_on_xeon.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+USER_ID=$(whoami)
+LOG_PATH=/home/$(whoami)/logs
+MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
+IMAGE_REPO=${IMAGE_REPO:-opea}
+IMAGE_TAG=${IMAGE_TAG:-latest}
+
+ROLLOUT_TIMEOUT_SECONDS="1800s"
+KUBECTL_TIMEOUT_SECONDS="60s"
+
+function validate_chatqna() {
+    local ns=$1
+    local log=$2
+    max_retry=10
+    # make sure microservice retriever-usvc is ready
+    # try to curl retriever-svc for max_retry times
+    test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
+    for ((i=1; i<=max_retry; i++))
+    do
+        endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-retriever-usvc" $ns)
+        curl http://$endpoint_url/v1/retrieval -X POST \
+            -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \
+            -H 'Content-Type: application/json' && break
+        sleep 30
+    done
+    # if i is bigger than max_retry, then exit with error
+    if [ $i -gt $max_retry ]; then
+        echo "Microservice retriever failed, exit with error."
+        return 1
+    fi
+    # make sure microservice tgi-svc is ready
+    for ((i=1; i<=max_retry; i++))
+    do
+        endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-tgi" $ns)
+        curl http://$endpoint_url/generate -X POST \
+            -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \
+            -H 'Content-Type: application/json' && break
+        sleep 30
+    done
+    # if i is bigger than max_retry, then exit with error
+    if [ $i -gt $max_retry ]; then
+        echo "Microservice tgi failed, exit with error."
+        return 1
+    fi
+
+    # check megaservice works
+    # generate a random logfile name to avoid conflict among multiple runners
+    LOGFILE=$LOG_PATH/curlmega_$log.log
+    endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna" $ns)
+    curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE
+    exit_code=$?
+    if [ $exit_code -ne 0 ]; then
+        echo "Megaservice failed, please check the logs in $LOGFILE!"
+        return ${exit_code}
+    fi
+
+    echo "Checking response results, make sure the output is reasonable. "
+    local status=false
+    if [[ -f $LOGFILE ]] &&
+        [[ $(grep -c "\[DONE\]" $LOGFILE) != 0 ]]; then
+        status=true
+    fi
+    if [ $status == false ]; then
+        echo "Response check failed, please check the logs in artifacts!"
+        return 1
+    else
+        echo "Response check succeed!"
+    fi
+    return 0
+}
+
+function install_chatqna() {
+    echo "Testing manifests chatqna_guardrails"
+    local ns=$1
+    bash ChatQnA/tests/common/_test_manifest_utils.sh _cleanup_ns $ns
+    pushd ChatQnA/kubernetes/intel/cpu/xeon/manifest
+    kubectl create namespace $ns
+    # install guardrail
+    kubectl apply -f chatqna-guardrails.yaml -n $ns
+    # Sleep enough time for chatqna_guardrails to be ready
+    sleep 60
+}
+
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <function_name>"
+    exit 1
+fi
+
+case "$1" in
+    init_ChatQnA)
+        pushd ChatQnA/tests/common
+        bash _test_manifest_utils.sh init_ChatQnA
+        popd
+        ;;
+    install_ChatQnA)
+        NAMESPACE=$2
+        install_chatqna $NAMESPACE
+        popd
+        ;;
+    validate_ChatQnA)
+        NAMESPACE=$2
+        SERVICE_NAME=chatqna-guardrails
+        validate_chatqna $NAMESPACE chatqna-guardrails
+        ret=$?
+        if [ $ret -ne 0 ]; then
+            exit $ret
+        fi
+        ;;
+
+    *)
+        echo "Unknown function: $1"
+        ;;
+esac

ChatQnA/tests/test_manifest_on_gaudi.sh

@@ -12,17 +12,6 @@ IMAGE_TAG=${IMAGE_TAG:-latest}
 ROLLOUT_TIMEOUT_SECONDS="1800s"
 KUBECTL_TIMEOUT_SECONDS="60s"
 
-function init_chatqna() {
-    # replace the mount dir "path: /mnt/opea-models" with "path: $CHART_MOUNT"
-    find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \;
-    # replace microservice image tag
-    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
-    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/"
-    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \;
-    # set huggingface token
-    find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
-}
-
 function install_chatqna {
     echo "namespace is $NAMESPACE"
     kubectl apply -f chatqna.yaml -n $NAMESPACE
@@ -30,13 +19,6 @@ function install_chatqna {
     sleep 60
 }
 
-function get_end_point() {
-    # $1 is service name, $2 is namespace
-    ip_address=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.clusterIP}')
-    port=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.ports[0].port}')
-    echo "$ip_address:$port"
-}
-
 function validate_chatqna() {
     local ns=$1
     local log=$2
@@ -46,7 +28,7 @@ function validate_chatqna() {
     test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
     for ((i=1; i<=max_retry; i++))
     do
-        endpoint_url=$(get_end_point "chatqna-retriever-usvc" $ns)
+        endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-retriever-usvc" $ns)
         curl http://$endpoint_url/v1/retrieval -X POST \
             -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \
             -H 'Content-Type: application/json' && break
@@ -60,7 +42,7 @@ function validate_chatqna() {
     # make sure microservice tgi-svc is ready
     for ((i=1; i<=max_retry; i++))
     do
-        endpoint_url=$(get_end_point "chatqna-tgi" $ns)
+        endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-tgi" $ns)
         curl http://$endpoint_url/generate -X POST \
             -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \
             -H 'Content-Type: application/json' && break
@@ -75,7 +57,7 @@ function validate_chatqna() {
     # check megaservice works
     # generate a random logfile name to avoid conflict among multiple runners
     LOGFILE=$LOG_PATH/curlmega_$log.log
-    endpoint_url=$(get_end_point "chatqna" $ns)
+    endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna" $ns)
     curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE
     exit_code=$?
     if [ $exit_code -ne 0 ]; then
@@ -98,128 +80,6 @@ function validate_chatqna() {
     return 0
 }
 
-
-function validate_chatqna_vllm() {
-    local ns=$1
-    local log=$2
-    max_retry=20
-    # make sure microservice retriever-usvc is ready
-    # try to curl retriever-svc for max_retry times
-    test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
-    for ((i=1; i<=max_retry; i++))
-    do
-        endpoint_url=$(get_end_point "chatqna-retriever-usvc" $ns)
-        curl http://$endpoint_url/v1/retrieval -X POST \
-            -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \
-            -H 'Content-Type: application/json' && break
-        sleep 30
-    done
-    # if i is bigger than max_retry, then exit with error
-    if [ $i -gt $max_retry ]; then
-        echo "Microservice retriever failed, exit with error."
-        return 1
-    fi
-
-    # make sure microservice vllm-svc is ready
-    for ((i=1; i<=max_retry; i++))
-    do
-        endpoint_url=$(get_end_point "chatqna-vllm" $ns)
-        curl http://$endpoint_url/v1/chat/completions -X POST \
-            -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}' \
-            -H 'Content-Type: application/json' && break
-        sleep 30
-    done
-    # if i is bigger than max_retry, then exit with error
-    if [ $i -gt $max_retry ]; then
-        echo "Microservice vllm failed, exit with error."
-        return 1
-    fi
-
-    # check megaservice works
-    # generate a random logfile name to avoid conflict among multiple runners
-    LOGFILE=$LOG_PATH/curlmega_$log.log
-    endpoint_url=$(get_end_point "chatqna" $ns)
-    curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE
-    exit_code=$?
-    if [ $exit_code -ne 0 ]; then
-        echo "Megaservice failed, please check the logs in $LOGFILE!"
-        return ${exit_code}
-    fi
-
-    echo "Checking response results, make sure the output is reasonable. "
-    local status=false
-    if [[ -f $LOGFILE ]] &&
-        [[ $(grep -c "\[DONE\]" $LOGFILE) != 0 ]]; then
-        status=true
-    fi
-    if [ $status == false ]; then
-        echo "Response check failed, please check the logs in artifacts!"
-        return 1
-    else
-        echo "Response check succeed!"
-    fi
-    return 0
-}
-
-
-function _cleanup_ns() {
-    local ns=$1
-    if kubectl get ns $ns; then
-      if ! kubectl delete ns $ns --timeout=$KUBECTL_TIMEOUT_SECONDS; then
-        kubectl delete pods --namespace $ns --force --grace-period=0 --all
-        kubectl delete ns $ns --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
-      fi
-    fi
-}
-
-function install_and_validate_chatqna_guardrail() {
-    echo "Testing manifests chatqna_guardrils"
-    local ns=${NAMESPACE}
-    _cleanup_ns $ns
-    kubectl create namespace $ns
-    # install guardrail
-    kubectl apply -f chatqna-guardrails.yaml -n $ns
-    # Sleep enough time for chatqna_guardrail to be ready
-    sleep 60
-    if kubectl rollout status deployment -n "$ns" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then
-        echo "Waiting for chatqna_guardrail pod ready done!"
-    else
-        echo "Timeout waiting for chatqna_guardrail pod ready!"
-        exit 1
-    fi
-
-    # validate guardrail
-    validate_chatqna $ns chatqna-guardrails
-    local ret=$?
-    if [ $ret -ne 0 ]; then
-        exit 1
-    fi
-}
-
-function install_and_validate_chatqna_vllm() {
-    echo "Testing manifests chatqna_vllm"
-    local ns=${NAMESPACE}
-    _cleanup_ns $ns
-    kubectl create namespace $ns
-    # install guardrail
-    kubectl apply -f chatqna-vllm.yaml -n $ns
-    # Sleep enough time for chatqna_vllm to be ready, vllm warmup takes about 5 minutes
-    sleep 280
-    if kubectl rollout status deployment -n "$ns" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then
-        echo "Waiting for chatqna_vllm pod ready done!"
-    else
-        echo "Timeout waiting for chatqna_vllm pod ready!"
-        exit 1
-    fi
-
-    # validate guardrail
-    validate_chatqna_vllm $ns chatqna-vllm
-    local ret=$?
-    if [ $ret -ne 0 ]; then
-        exit 1
-    fi
-}
-
 if [ $# -eq 0 ]; then
     echo "Usage: $0 <function_name>"
     exit 1
@@ -227,8 +87,8 @@ fi
 
 case "$1" in
     init_ChatQnA)
-        pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest
+        pushd ChatQnA/tests/common
-        init_chatqna
+        bash _test_manifest_utils.sh init_ChatQnA
         popd
         ;;
     install_ChatQnA)
@@ -245,12 +105,6 @@ case "$1" in
         if [ $ret -ne 0 ]; then
             exit $ret
         fi
-        pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest
-        install_and_validate_chatqna_guardrail
-        popd
-        pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest
-        install_and_validate_chatqna_vllm
-        popd
         ;;
 
     *)

ChatQnA/tests/test_manifest_on_xeon.sh

@@ -12,17 +12,6 @@ IMAGE_TAG=${IMAGE_TAG:-latest}
 ROLLOUT_TIMEOUT_SECONDS="1800s"
 KUBECTL_TIMEOUT_SECONDS="60s"
 
-function init_chatqna() {
-    # replace the mount dir "path: /mnt/opea-models" with "path: $CHART_MOUNT"
-    find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \;
-    # replace microservice image tag
-    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
-    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/"
-    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \;
-    # set huggingface token
-    find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
-}
-
 function install_chatqna {
     echo "namespace is $NAMESPACE"
     kubectl apply -f chatqna.yaml -n $NAMESPACE
@@ -30,13 +19,6 @@ function install_chatqna {
     sleep 60
 }
 
-function get_end_point() {
-    # $1 is service name, $2 is namespace
-    ip_address=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.clusterIP}')
-    port=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.ports[0].port}')
-    echo "$ip_address:$port"
-}
-
 function validate_chatqna() {
     local ns=$1
     local log=$2
@@ -46,7 +28,7 @@ function validate_chatqna() {
     test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
     for ((i=1; i<=max_retry; i++))
     do
-        endpoint_url=$(get_end_point "chatqna-retriever-usvc" $ns)
+        endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-retriever-usvc" $ns)
         curl http://$endpoint_url/v1/retrieval -X POST \
             -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \
             -H 'Content-Type: application/json' && break
@@ -60,7 +42,7 @@ function validate_chatqna() {
     # make sure microservice tgi-svc is ready
     for ((i=1; i<=max_retry; i++))
     do
-        endpoint_url=$(get_end_point "chatqna-tgi" $ns)
+        endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-tgi" $ns)
         curl http://$endpoint_url/generate -X POST \
             -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \
             -H 'Content-Type: application/json' && break
@@ -75,7 +57,7 @@ function validate_chatqna() {
     # check megaservice works
     # generate a random logfile name to avoid conflict among multiple runners
     LOGFILE=$LOG_PATH/curlmega_$log.log
-    endpoint_url=$(get_end_point "chatqna" $ns)
+    endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna" $ns)
     curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE
     exit_code=$?
     if [ $exit_code -ne 0 ]; then
@@ -98,41 +80,6 @@ function validate_chatqna() {
     return 0
 }
 
-
-function _cleanup_ns() {
-    local ns=$1
-    if kubectl get ns $ns; then
-      if ! kubectl delete ns $ns --timeout=$KUBECTL_TIMEOUT_SECONDS; then
-        kubectl delete pods --namespace $ns --force --grace-period=0 --all
-        kubectl delete ns $ns --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
-      fi
-    fi
-}
-
-function install_and_validate_chatqna_guardrail() {
-    echo "Testing manifests chatqna_guardrils"
-    local ns=${NAMESPACE}
-    _cleanup_ns $ns
-    kubectl create namespace $ns
-    # install guardrail
-    kubectl apply -f chatqna-guardrails.yaml -n $ns
-    # Sleep enough time for chatqna_guardrail to be ready
-    sleep 60
-    if kubectl rollout status deployment -n "$ns" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then
-        echo "Waiting for chatqna_guardrail pod ready done!"
-    else
-        echo "Timeout waiting for chatqna_guardrail pod ready!"
-        exit 1
-    fi
-
-    # validate guardrail
-    validate_chatqna $ns chatqna-guardrails
-    local ret=$?
-    if [ $ret -ne 0 ]; then
-        exit 1
-    fi
-}
-
 if [ $# -eq 0 ]; then
     echo "Usage: $0 <function_name>"
     exit 1
@@ -140,8 +87,8 @@ fi
 
 case "$1" in
     init_ChatQnA)
-        pushd ChatQnA/kubernetes/intel/cpu/xeon/manifest
+        pushd ChatQnA/tests/common
-        init_chatqna
+        bash _test_manifest_utils.sh init_ChatQnA
         popd
         ;;
     install_ChatQnA)
@@ -158,9 +105,6 @@ case "$1" in
         if [ $ret -ne 0 ]; then
             exit $ret
         fi
-        pushd ChatQnA/kubernetes/intel/cpu/xeon/manifest
-        install_and_validate_chatqna_guardrail
-        popd
         ;;
     *)
         echo "Unknown function: $1"

ChatQnA/tests/test_manifest_vllm_on_gaudi.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+USER_ID=$(whoami)
+LOG_PATH=/home/$(whoami)/logs
+MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
+IMAGE_REPO=${IMAGE_REPO:-opea}
+IMAGE_TAG=${IMAGE_TAG:-latest}
+
+ROLLOUT_TIMEOUT_SECONDS="1800s"
+KUBECTL_TIMEOUT_SECONDS="60s"
+
+function validate_chatqna() {
+    local ns=$1
+    local log=$2
+    max_retry=20
+    # make sure microservice retriever-usvc is ready
+    # try to curl retriever-svc for max_retry times
+    test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
+    for ((i=1; i<=max_retry; i++))
+    do
+        endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point  "chatqna-retriever-usvc" $ns)
+        curl http://$endpoint_url/v1/retrieval -X POST \
+            -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \
+            -H 'Content-Type: application/json' && break
+        sleep 30
+    done
+    # if i is bigger than max_retry, then exit with error
+    if [ $i -gt $max_retry ]; then
+        echo "Microservice retriever failed, exit with error."
+        return 1
+    fi
+
+    # make sure microservice vllm-svc is ready
+    for ((i=1; i<=max_retry; i++))
+    do
+        endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point  "chatqna-vllm" $ns)
+        curl http://$endpoint_url/v1/chat/completions -X POST \
+            -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}' \
+            -H 'Content-Type: application/json' && break
+        sleep 30
+    done
+    # if i is bigger than max_retry, then exit with error
+    if [ $i -gt $max_retry ]; then
+        echo "Microservice vllm failed, exit with error."
+        return 1
+    fi
+
+    # check megaservice works
+    # generate a random logfile name to avoid conflict among multiple runners
+    LOGFILE=$LOG_PATH/curlmega_$log.log
+    endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point  "chatqna" $ns)
+    curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE
+    exit_code=$?
+    if [ $exit_code -ne 0 ]; then
+        echo "Megaservice failed, please check the logs in $LOGFILE!"
+        return ${exit_code}
+    fi
+
+    echo "Checking response results, make sure the output is reasonable. "
+    local status=false
+    if [[ -f $LOGFILE ]] &&
+        [[ $(grep -c "\[DONE\]" $LOGFILE) != 0 ]]; then
+        status=true
+    fi
+    if [ $status == false ]; then
+        echo "Response check failed, please check the logs in artifacts!"
+        return 1
+    else
+        echo "Response check succeed!"
+    fi
+    return 0
+}
+
+function install_chatqna() {
+    echo "Testing manifests chatqna_vllm"
+    local ns=$1
+    bash ChatQnA/tests/common/_test_manifest_utils.sh _cleanup_ns $ns
+    kubectl create namespace $ns
+    # install guardrail
+    pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest
+    kubectl apply -f chatqna-vllm.yaml -n $ns
+    # Sleep enough time for chatqna_vllm to be ready, vllm warmup takes about 5 minutes
+    sleep 280
+}
+
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <function_name>"
+    exit 1
+fi
+
+case "$1" in
+    init_ChatQnA)
+        pushd ChatQnA/tests/common
+        bash _test_manifest_utils.sh init_ChatQnA
+        popd
+        ;;
+    install_ChatQnA)
+        NAMESPACE=$2
+        install_chatqna $NAMESPACE
+        popd
+        ;;
+    validate_ChatQnA)
+        NAMESPACE=$2
+        SERVICE_NAME=chatqna-vllm
+        validate_chatqna $NAMESPACE chatqna-vllm
+        ret=$?
+        if [ $ret -ne 0 ]; then
+            exit $ret
+        fi
+        ;;
+
+    *)
+        echo "Unknown function: $1"
+        ;;
+esac