From 340796bbae61d38a6aae4d244eef8d583101c04e Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 4 Dec 2024 15:17:46 +0800 Subject: [PATCH] Split ChatQnA manifest test (#1190) Signed-off-by: ZePan110 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/_manifest-e2e.yml | 88 +++++++++- ChatQnA/tests/common/_test_manifest_utils.sh | 64 +++++++ .../test_manifest_guardrails_on_gaudi.sh | 117 +++++++++++++ .../tests/test_manifest_guardrails_on_xeon.sh | 117 +++++++++++++ ChatQnA/tests/test_manifest_on_gaudi.sh | 156 +----------------- ChatQnA/tests/test_manifest_on_xeon.sh | 66 +------- ChatQnA/tests/test_manifest_vllm_on_gaudi.sh | 118 +++++++++++++ 7 files changed, 507 insertions(+), 219 deletions(-) create mode 100644 ChatQnA/tests/common/_test_manifest_utils.sh create mode 100755 ChatQnA/tests/test_manifest_guardrails_on_gaudi.sh create mode 100755 ChatQnA/tests/test_manifest_guardrails_on_xeon.sh create mode 100755 ChatQnA/tests/test_manifest_vllm_on_gaudi.sh diff --git a/.github/workflows/_manifest-e2e.yml b/.github/workflows/_manifest-e2e.yml index fc414490d..83ccc7714 100644 --- a/.github/workflows/_manifest-e2e.yml +++ b/.github/workflows/_manifest-e2e.yml @@ -22,7 +22,72 @@ on: type: string jobs: + get-test-case: + runs-on: ubuntu-latest + outputs: + test_cases: ${{ steps.test-case-matrix.outputs.test_cases }} + CHECKOUT_REF: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }} + steps: + - name: Get checkout ref + id: get-checkout-ref + run: | + if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then + CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge + else + CHECKOUT_REF=${{ github.ref }} + fi + echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT + echo "checkout ref ${CHECKOUT_REF}" + + - name: Checkout out Repo + uses: actions/checkout@v4 + with: + ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }} + fetch-depth: 0 + + - name: Get test matrix + shell: bash + id: test-case-matrix + run: | + example_l=$(echo ${{ inputs.example }} | tr '[:upper:]' '[:lower:]') + cd ${{ github.workspace }}/${{ inputs.example }}/tests + run_test_cases="" + + default_test_case=$(find . -type f -name "test_manifest_on_${{ inputs.hardware }}.sh" | cut -d/ -f2) + if [ "$default_test_case" ]; then run_test_cases="$default_test_case"; fi + other_test_cases=$(find . -type f -name "test_manifest_*_on_${{ inputs.hardware }}.sh" | cut -d/ -f2) + echo "default_test_case=$default_test_case" + echo "other_test_cases=$other_test_cases" + + if [ "${{ inputs.tag }}" == "ci" ]; then + base_commit=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/opea-project/GenAIExamples/commits?sha=${{ github.event.pull_request.base.ref }}" | jq -r '.[0].sha') + merged_commit=$(git log -1 --format='%H') + changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | grep -vE '${{ inputs.diff_excluded_files }}')" || true + fi + + for test_case in $other_test_cases; do + if [ "${{ inputs.tag }}" == "ci" ]; then + flag=${test_case%_on_*} + flag=${flag#test_compose_} + if [[ $(printf '%s\n' "${changed_files[@]}" | grep ${{ inputs.example }} | grep ${flag}) ]]; then + run_test_cases="$run_test_cases $test_case" + fi + else + run_test_cases="$run_test_cases $test_case" + fi + done + + test_cases=$(echo $run_test_cases | tr ' ' '\n' | sort -u | jq -R '.' | jq -sc '.') + echo "test_cases=$test_cases" + echo "test_cases=$test_cases" >> $GITHUB_OUTPUT + manifest-test: + needs: [get-test-case] + strategy: + matrix: + test_case: ${{ fromJSON(needs.get-test-case.outputs.test_cases) }} + fail-fast: false runs-on: "k8s-${{ inputs.hardware }}" continue-on-error: true steps: @@ -45,11 +110,14 @@ jobs: fetch-depth: 0 - name: Set variables + env: + test_case: ${{ matrix.test_case }} run: | echo "IMAGE_REPO=${OPEA_IMAGE_REPO}opea" >> $GITHUB_ENV echo "IMAGE_TAG=${{ inputs.tag }}" >> $GITHUB_ENV lower_example=$(echo "${{ inputs.example }}" | tr '[:upper:]' '[:lower:]') - echo "NAMESPACE=$lower_example-$(tr -dc a-z0-9 > $GITHUB_ENV + name=$(echo "$test_case" | cut -d/ -f2 | cut -d'_' -f3- |cut -d'_' -f1 | grep -v 'on' | sed 's/^/-/') + echo "NAMESPACE=$lower_example$name-$(tr -dc a-z0-9 > $GITHUB_ENV echo "ROLLOUT_TIMEOUT_SECONDS=1800s" >> $GITHUB_ENV echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV echo "continue_test=true" >> $GITHUB_ENV @@ -59,15 +127,19 @@ jobs: - name: Kubectl install id: install + env: + test_case: ${{ matrix.test_case }} run: | - if [[ ! -f ${{ github.workspace }}/${{ inputs.example }}/tests/test_manifest_on_${{ inputs.hardware }}.sh ]]; then + set -x + echo "test_case=$test_case" + if [[ ! -f ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} ]]; then echo "No test script found, exist test!" exit 0 else - ${{ github.workspace }}/${{ inputs.example }}/tests/test_manifest_on_${{ inputs.hardware }}.sh init_${{ inputs.example }} + ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} init_${{ inputs.example }} echo "should_cleanup=true" >> $GITHUB_ENV kubectl create ns $NAMESPACE - ${{ github.workspace }}/${{ inputs.example }}/tests/test_manifest_on_${{ inputs.hardware }}.sh install_${{ inputs.example }} $NAMESPACE + ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} install_${{ inputs.example }} $NAMESPACE echo "Testing ${{ inputs.example }}, waiting for pod ready..." if kubectl rollout status deployment --namespace "$NAMESPACE" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then echo "Testing manifests ${{ inputs.example }}, waiting for pod ready done!" @@ -82,14 +154,16 @@ jobs: - name: Validate e2e test if: always() + env: + test_case: ${{ matrix.test_case }} run: | if $skip_validate; then echo "Skip validate" else - if ${{ github.workspace }}/${{ inputs.example }}/tests/test_manifest_on_${{ inputs.hardware }}.sh validate_${{ inputs.example }} $NAMESPACE ; then - echo "Validate ${{ inputs.example }} successful!" + if ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} validate_${{ inputs.example }} $NAMESPACE ; then + echo "Validate ${test_case} successful!" else - echo "Validate ${{ inputs.example }} failure!!!" + echo "Validate ${test_case} failure!!!" echo "Check the logs in 'Dump logs when e2e test failed' step!!!" exit 1 fi diff --git a/ChatQnA/tests/common/_test_manifest_utils.sh b/ChatQnA/tests/common/_test_manifest_utils.sh new file mode 100644 index 000000000..a70b983c8 --- /dev/null +++ b/ChatQnA/tests/common/_test_manifest_utils.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +USER_ID=$(whoami) +MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub +IMAGE_REPO=${IMAGE_REPO:-opea} +IMAGE_TAG=${IMAGE_TAG:-latest} + +ROLLOUT_TIMEOUT_SECONDS="1800s" +KUBECTL_TIMEOUT_SECONDS="60s" + +function init_chatqna() { + # replace the mount dir "path: /mnt/opea-models" with "path: $CHART_MOUNT" + find ../../kubernetes/intel/*/*/manifest -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \; + # replace microservice image tag + find ../../kubernetes/intel/*/*/manifest -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \; + # replace the repository "image: opea/*" with "image: $IMAGE_REPO/" + find ../../kubernetes/intel/*/*/manifest -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \; + # set huggingface token + find ../../kubernetes/intel/*/*/manifest -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \; +} + +function get_end_point() { + # $1 is service name, $2 is namespace + ip_address=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.clusterIP}') + port=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.ports[0].port}') + echo "$ip_address:$port" +} + +function _cleanup_ns() { + local ns=$1 + if kubectl get ns $ns; then + if ! kubectl delete ns $ns --timeout=$KUBECTL_TIMEOUT_SECONDS; then + kubectl delete pods --namespace $ns --force --grace-period=0 --all + kubectl delete ns $ns --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS + fi + fi +} + + +if [ $# -eq 0 ]; then + echo "Usage: $0 " + exit 1 +fi + +case "$1" in + init_ChatQnA) + init_chatqna + ;; + get_end_point) + service=$2 + NAMESPACE=$3 + get_end_point $service $NAMESPACE + ;; + _cleanup_ns) + NAMESPACE=$2 + _cleanup_ns $NAMESPACE + ;; + *) + echo "Unknown function: $1" + ;; +esac diff --git a/ChatQnA/tests/test_manifest_guardrails_on_gaudi.sh b/ChatQnA/tests/test_manifest_guardrails_on_gaudi.sh new file mode 100755 index 000000000..274cc5209 --- /dev/null +++ b/ChatQnA/tests/test_manifest_guardrails_on_gaudi.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +USER_ID=$(whoami) +LOG_PATH=/home/$(whoami)/logs +MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub +IMAGE_REPO=${IMAGE_REPO:-opea} +IMAGE_TAG=${IMAGE_TAG:-latest} + +ROLLOUT_TIMEOUT_SECONDS="1800s" +KUBECTL_TIMEOUT_SECONDS="60s" + +function validate_chatqna() { + local ns=$1 + local log=$2 + max_retry=20 + # make sure microservice retriever-usvc is ready + # try to curl retriever-svc for max_retry times + test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") + for ((i=1; i<=max_retry; i++)) + do + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-retriever-usvc" $ns) + curl http://$endpoint_url/v1/retrieval -X POST \ + -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \ + -H 'Content-Type: application/json' && break + sleep 30 + done + # if i is bigger than max_retry, then exit with error + if [ $i -gt $max_retry ]; then + echo "Microservice retriever failed, exit with error." + return 1 + fi + # make sure microservice tgi-svc is ready + for ((i=1; i<=max_retry; i++)) + do + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-tgi" $ns) + curl http://$endpoint_url/generate -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \ + -H 'Content-Type: application/json' && break + sleep 30 + done + # if i is bigger than max_retry, then exit with error + if [ $i -gt $max_retry ]; then + echo "Microservice tgi failed, exit with error." + return 1 + fi + + # check megaservice works + # generate a random logfile name to avoid conflict among multiple runners + LOGFILE=$LOG_PATH/curlmega_$log.log + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna" $ns) + curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "Megaservice failed, please check the logs in $LOGFILE!" + return ${exit_code} + fi + + echo "Checking response results, make sure the output is reasonable. " + local status=false + if [[ -f $LOGFILE ]] && + [[ $(grep -c "\[DONE\]" $LOGFILE) != 0 ]]; then + status=true + fi + if [[ $status == false ]]; then + echo "Response check failed, please check the logs in artifacts!" + return 1 + else + echo "Response check succeed!" + fi + return 0 +} + +function install_chatqna() { + echo "Testing manifests chatqna_guardrails" + local ns=$1 + bash ChatQnA/tests/common/_test_manifest_utils.sh _cleanup_ns $ns + pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest + kubectl create namespace $ns + # install guardrails + kubectl apply -f chatqna-guardrails.yaml -n $ns + # Sleep enough time for chatqna_guardrails to be ready + sleep 60 +} + +if [ $# -eq 0 ]; then + echo "Usage: $0 " + exit 1 +fi + +case "$1" in + init_ChatQnA) + pushd ChatQnA/tests/common + bash _test_manifest_utils.sh init_ChatQnA + popd + ;; + install_ChatQnA) + NAMESPACE=$2 + install_chatqna $NAMESPACE + popd + ;; + validate_ChatQnA) + NAMESPACE=$2 + SERVICE_NAME=chatqna-guardrails + validate_chatqna $NAMESPACE chatqna-guardrails + ret=$? + if [ $ret -ne 0 ]; then + exit $ret + fi + ;; + + *) + echo "Unknown function: $1" + ;; +esac diff --git a/ChatQnA/tests/test_manifest_guardrails_on_xeon.sh b/ChatQnA/tests/test_manifest_guardrails_on_xeon.sh new file mode 100755 index 000000000..63d494c9f --- /dev/null +++ b/ChatQnA/tests/test_manifest_guardrails_on_xeon.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +USER_ID=$(whoami) +LOG_PATH=/home/$(whoami)/logs +MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub +IMAGE_REPO=${IMAGE_REPO:-opea} +IMAGE_TAG=${IMAGE_TAG:-latest} + +ROLLOUT_TIMEOUT_SECONDS="1800s" +KUBECTL_TIMEOUT_SECONDS="60s" + +function validate_chatqna() { + local ns=$1 + local log=$2 + max_retry=10 + # make sure microservice retriever-usvc is ready + # try to curl retriever-svc for max_retry times + test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") + for ((i=1; i<=max_retry; i++)) + do + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-retriever-usvc" $ns) + curl http://$endpoint_url/v1/retrieval -X POST \ + -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \ + -H 'Content-Type: application/json' && break + sleep 30 + done + # if i is bigger than max_retry, then exit with error + if [ $i -gt $max_retry ]; then + echo "Microservice retriever failed, exit with error." + return 1 + fi + # make sure microservice tgi-svc is ready + for ((i=1; i<=max_retry; i++)) + do + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-tgi" $ns) + curl http://$endpoint_url/generate -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \ + -H 'Content-Type: application/json' && break + sleep 30 + done + # if i is bigger than max_retry, then exit with error + if [ $i -gt $max_retry ]; then + echo "Microservice tgi failed, exit with error." + return 1 + fi + + # check megaservice works + # generate a random logfile name to avoid conflict among multiple runners + LOGFILE=$LOG_PATH/curlmega_$log.log + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna" $ns) + curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "Megaservice failed, please check the logs in $LOGFILE!" + return ${exit_code} + fi + + echo "Checking response results, make sure the output is reasonable. " + local status=false + if [[ -f $LOGFILE ]] && + [[ $(grep -c "\[DONE\]" $LOGFILE) != 0 ]]; then + status=true + fi + if [ $status == false ]; then + echo "Response check failed, please check the logs in artifacts!" + return 1 + else + echo "Response check succeed!" + fi + return 0 +} + +function install_chatqna() { + echo "Testing manifests chatqna_guardrails" + local ns=$1 + bash ChatQnA/tests/common/_test_manifest_utils.sh _cleanup_ns $ns + pushd ChatQnA/kubernetes/intel/cpu/xeon/manifest + kubectl create namespace $ns + # install guardrail + kubectl apply -f chatqna-guardrails.yaml -n $ns + # Sleep enough time for chatqna_guardrails to be ready + sleep 60 +} + +if [ $# -eq 0 ]; then + echo "Usage: $0 " + exit 1 +fi + +case "$1" in + init_ChatQnA) + pushd ChatQnA/tests/common + bash _test_manifest_utils.sh init_ChatQnA + popd + ;; + install_ChatQnA) + NAMESPACE=$2 + install_chatqna $NAMESPACE + popd + ;; + validate_ChatQnA) + NAMESPACE=$2 + SERVICE_NAME=chatqna-guardrails + validate_chatqna $NAMESPACE chatqna-guardrails + ret=$? + if [ $ret -ne 0 ]; then + exit $ret + fi + ;; + + *) + echo "Unknown function: $1" + ;; +esac diff --git a/ChatQnA/tests/test_manifest_on_gaudi.sh b/ChatQnA/tests/test_manifest_on_gaudi.sh index 170f04541..d1764401f 100755 --- a/ChatQnA/tests/test_manifest_on_gaudi.sh +++ b/ChatQnA/tests/test_manifest_on_gaudi.sh @@ -12,17 +12,6 @@ IMAGE_TAG=${IMAGE_TAG:-latest} ROLLOUT_TIMEOUT_SECONDS="1800s" KUBECTL_TIMEOUT_SECONDS="60s" -function init_chatqna() { - # replace the mount dir "path: /mnt/opea-models" with "path: $CHART_MOUNT" - find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \; - # replace microservice image tag - find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \; - # replace the repository "image: opea/*" with "image: $IMAGE_REPO/" - find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \; - # set huggingface token - find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \; -} - function install_chatqna { echo "namespace is $NAMESPACE" kubectl apply -f chatqna.yaml -n $NAMESPACE @@ -30,13 +19,6 @@ function install_chatqna { sleep 60 } -function get_end_point() { - # $1 is service name, $2 is namespace - ip_address=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.clusterIP}') - port=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.ports[0].port}') - echo "$ip_address:$port" -} - function validate_chatqna() { local ns=$1 local log=$2 @@ -46,7 +28,7 @@ function validate_chatqna() { test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") for ((i=1; i<=max_retry; i++)) do - endpoint_url=$(get_end_point "chatqna-retriever-usvc" $ns) + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-retriever-usvc" $ns) curl http://$endpoint_url/v1/retrieval -X POST \ -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \ -H 'Content-Type: application/json' && break @@ -60,7 +42,7 @@ function validate_chatqna() { # make sure microservice tgi-svc is ready for ((i=1; i<=max_retry; i++)) do - endpoint_url=$(get_end_point "chatqna-tgi" $ns) + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-tgi" $ns) curl http://$endpoint_url/generate -X POST \ -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \ -H 'Content-Type: application/json' && break @@ -75,7 +57,7 @@ function validate_chatqna() { # check megaservice works # generate a random logfile name to avoid conflict among multiple runners LOGFILE=$LOG_PATH/curlmega_$log.log - endpoint_url=$(get_end_point "chatqna" $ns) + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna" $ns) curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE exit_code=$? if [ $exit_code -ne 0 ]; then @@ -98,128 +80,6 @@ function validate_chatqna() { return 0 } - -function validate_chatqna_vllm() { - local ns=$1 - local log=$2 - max_retry=20 - # make sure microservice retriever-usvc is ready - # try to curl retriever-svc for max_retry times - test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") - for ((i=1; i<=max_retry; i++)) - do - endpoint_url=$(get_end_point "chatqna-retriever-usvc" $ns) - curl http://$endpoint_url/v1/retrieval -X POST \ - -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \ - -H 'Content-Type: application/json' && break - sleep 30 - done - # if i is bigger than max_retry, then exit with error - if [ $i -gt $max_retry ]; then - echo "Microservice retriever failed, exit with error." - return 1 - fi - - # make sure microservice vllm-svc is ready - for ((i=1; i<=max_retry; i++)) - do - endpoint_url=$(get_end_point "chatqna-vllm" $ns) - curl http://$endpoint_url/v1/chat/completions -X POST \ - -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}' \ - -H 'Content-Type: application/json' && break - sleep 30 - done - # if i is bigger than max_retry, then exit with error - if [ $i -gt $max_retry ]; then - echo "Microservice vllm failed, exit with error." - return 1 - fi - - # check megaservice works - # generate a random logfile name to avoid conflict among multiple runners - LOGFILE=$LOG_PATH/curlmega_$log.log - endpoint_url=$(get_end_point "chatqna" $ns) - curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE - exit_code=$? - if [ $exit_code -ne 0 ]; then - echo "Megaservice failed, please check the logs in $LOGFILE!" - return ${exit_code} - fi - - echo "Checking response results, make sure the output is reasonable. " - local status=false - if [[ -f $LOGFILE ]] && - [[ $(grep -c "\[DONE\]" $LOGFILE) != 0 ]]; then - status=true - fi - if [ $status == false ]; then - echo "Response check failed, please check the logs in artifacts!" - return 1 - else - echo "Response check succeed!" - fi - return 0 -} - - -function _cleanup_ns() { - local ns=$1 - if kubectl get ns $ns; then - if ! kubectl delete ns $ns --timeout=$KUBECTL_TIMEOUT_SECONDS; then - kubectl delete pods --namespace $ns --force --grace-period=0 --all - kubectl delete ns $ns --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS - fi - fi -} - -function install_and_validate_chatqna_guardrail() { - echo "Testing manifests chatqna_guardrils" - local ns=${NAMESPACE} - _cleanup_ns $ns - kubectl create namespace $ns - # install guardrail - kubectl apply -f chatqna-guardrails.yaml -n $ns - # Sleep enough time for chatqna_guardrail to be ready - sleep 60 - if kubectl rollout status deployment -n "$ns" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then - echo "Waiting for chatqna_guardrail pod ready done!" - else - echo "Timeout waiting for chatqna_guardrail pod ready!" - exit 1 - fi - - # validate guardrail - validate_chatqna $ns chatqna-guardrails - local ret=$? - if [ $ret -ne 0 ]; then - exit 1 - fi -} - -function install_and_validate_chatqna_vllm() { - echo "Testing manifests chatqna_vllm" - local ns=${NAMESPACE} - _cleanup_ns $ns - kubectl create namespace $ns - # install guardrail - kubectl apply -f chatqna-vllm.yaml -n $ns - # Sleep enough time for chatqna_vllm to be ready, vllm warmup takes about 5 minutes - sleep 280 - if kubectl rollout status deployment -n "$ns" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then - echo "Waiting for chatqna_vllm pod ready done!" - else - echo "Timeout waiting for chatqna_vllm pod ready!" - exit 1 - fi - - # validate guardrail - validate_chatqna_vllm $ns chatqna-vllm - local ret=$? - if [ $ret -ne 0 ]; then - exit 1 - fi -} - if [ $# -eq 0 ]; then echo "Usage: $0 " exit 1 @@ -227,8 +87,8 @@ fi case "$1" in init_ChatQnA) - pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest - init_chatqna + pushd ChatQnA/tests/common + bash _test_manifest_utils.sh init_ChatQnA popd ;; install_ChatQnA) @@ -245,12 +105,6 @@ case "$1" in if [ $ret -ne 0 ]; then exit $ret fi - pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest - install_and_validate_chatqna_guardrail - popd - pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest - install_and_validate_chatqna_vllm - popd ;; *) diff --git a/ChatQnA/tests/test_manifest_on_xeon.sh b/ChatQnA/tests/test_manifest_on_xeon.sh index d405df977..4c93a8958 100755 --- a/ChatQnA/tests/test_manifest_on_xeon.sh +++ b/ChatQnA/tests/test_manifest_on_xeon.sh @@ -12,17 +12,6 @@ IMAGE_TAG=${IMAGE_TAG:-latest} ROLLOUT_TIMEOUT_SECONDS="1800s" KUBECTL_TIMEOUT_SECONDS="60s" -function init_chatqna() { - # replace the mount dir "path: /mnt/opea-models" with "path: $CHART_MOUNT" - find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \; - # replace microservice image tag - find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \; - # replace the repository "image: opea/*" with "image: $IMAGE_REPO/" - find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \; - # set huggingface token - find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \; -} - function install_chatqna { echo "namespace is $NAMESPACE" kubectl apply -f chatqna.yaml -n $NAMESPACE @@ -30,13 +19,6 @@ function install_chatqna { sleep 60 } -function get_end_point() { - # $1 is service name, $2 is namespace - ip_address=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.clusterIP}') - port=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.ports[0].port}') - echo "$ip_address:$port" -} - function validate_chatqna() { local ns=$1 local log=$2 @@ -46,7 +28,7 @@ function validate_chatqna() { test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") for ((i=1; i<=max_retry; i++)) do - endpoint_url=$(get_end_point "chatqna-retriever-usvc" $ns) + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-retriever-usvc" $ns) curl http://$endpoint_url/v1/retrieval -X POST \ -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \ -H 'Content-Type: application/json' && break @@ -60,7 +42,7 @@ function validate_chatqna() { # make sure microservice tgi-svc is ready for ((i=1; i<=max_retry; i++)) do - endpoint_url=$(get_end_point "chatqna-tgi" $ns) + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-tgi" $ns) curl http://$endpoint_url/generate -X POST \ -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \ -H 'Content-Type: application/json' && break @@ -75,7 +57,7 @@ function validate_chatqna() { # check megaservice works # generate a random logfile name to avoid conflict among multiple runners LOGFILE=$LOG_PATH/curlmega_$log.log - endpoint_url=$(get_end_point "chatqna" $ns) + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna" $ns) curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE exit_code=$? if [ $exit_code -ne 0 ]; then @@ -98,41 +80,6 @@ function validate_chatqna() { return 0 } - -function _cleanup_ns() { - local ns=$1 - if kubectl get ns $ns; then - if ! kubectl delete ns $ns --timeout=$KUBECTL_TIMEOUT_SECONDS; then - kubectl delete pods --namespace $ns --force --grace-period=0 --all - kubectl delete ns $ns --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS - fi - fi -} - -function install_and_validate_chatqna_guardrail() { - echo "Testing manifests chatqna_guardrils" - local ns=${NAMESPACE} - _cleanup_ns $ns - kubectl create namespace $ns - # install guardrail - kubectl apply -f chatqna-guardrails.yaml -n $ns - # Sleep enough time for chatqna_guardrail to be ready - sleep 60 - if kubectl rollout status deployment -n "$ns" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then - echo "Waiting for chatqna_guardrail pod ready done!" - else - echo "Timeout waiting for chatqna_guardrail pod ready!" - exit 1 - fi - - # validate guardrail - validate_chatqna $ns chatqna-guardrails - local ret=$? - if [ $ret -ne 0 ]; then - exit 1 - fi -} - if [ $# -eq 0 ]; then echo "Usage: $0 " exit 1 @@ -140,8 +87,8 @@ fi case "$1" in init_ChatQnA) - pushd ChatQnA/kubernetes/intel/cpu/xeon/manifest - init_chatqna + pushd ChatQnA/tests/common + bash _test_manifest_utils.sh init_ChatQnA popd ;; install_ChatQnA) @@ -158,9 +105,6 @@ case "$1" in if [ $ret -ne 0 ]; then exit $ret fi - pushd ChatQnA/kubernetes/intel/cpu/xeon/manifest - install_and_validate_chatqna_guardrail - popd ;; *) echo "Unknown function: $1" diff --git a/ChatQnA/tests/test_manifest_vllm_on_gaudi.sh b/ChatQnA/tests/test_manifest_vllm_on_gaudi.sh new file mode 100755 index 000000000..c1ab58460 --- /dev/null +++ b/ChatQnA/tests/test_manifest_vllm_on_gaudi.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +USER_ID=$(whoami) +LOG_PATH=/home/$(whoami)/logs +MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub +IMAGE_REPO=${IMAGE_REPO:-opea} +IMAGE_TAG=${IMAGE_TAG:-latest} + +ROLLOUT_TIMEOUT_SECONDS="1800s" +KUBECTL_TIMEOUT_SECONDS="60s" + +function validate_chatqna() { + local ns=$1 + local log=$2 + max_retry=20 + # make sure microservice retriever-usvc is ready + # try to curl retriever-svc for max_retry times + test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") + for ((i=1; i<=max_retry; i++)) + do + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-retriever-usvc" $ns) + curl http://$endpoint_url/v1/retrieval -X POST \ + -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \ + -H 'Content-Type: application/json' && break + sleep 30 + done + # if i is bigger than max_retry, then exit with error + if [ $i -gt $max_retry ]; then + echo "Microservice retriever failed, exit with error." + return 1 + fi + + # make sure microservice vllm-svc is ready + for ((i=1; i<=max_retry; i++)) + do + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-vllm" $ns) + curl http://$endpoint_url/v1/chat/completions -X POST \ + -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}' \ + -H 'Content-Type: application/json' && break + sleep 30 + done + # if i is bigger than max_retry, then exit with error + if [ $i -gt $max_retry ]; then + echo "Microservice vllm failed, exit with error." + return 1 + fi + + # check megaservice works + # generate a random logfile name to avoid conflict among multiple runners + LOGFILE=$LOG_PATH/curlmega_$log.log + endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna" $ns) + curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "Megaservice failed, please check the logs in $LOGFILE!" + return ${exit_code} + fi + + echo "Checking response results, make sure the output is reasonable. " + local status=false + if [[ -f $LOGFILE ]] && + [[ $(grep -c "\[DONE\]" $LOGFILE) != 0 ]]; then + status=true + fi + if [ $status == false ]; then + echo "Response check failed, please check the logs in artifacts!" + return 1 + else + echo "Response check succeed!" + fi + return 0 +} + +function install_chatqna() { + echo "Testing manifests chatqna_vllm" + local ns=$1 + bash ChatQnA/tests/common/_test_manifest_utils.sh _cleanup_ns $ns + kubectl create namespace $ns + # install guardrail + pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest + kubectl apply -f chatqna-vllm.yaml -n $ns + # Sleep enough time for chatqna_vllm to be ready, vllm warmup takes about 5 minutes + sleep 280 +} + +if [ $# -eq 0 ]; then + echo "Usage: $0 " + exit 1 +fi + +case "$1" in + init_ChatQnA) + pushd ChatQnA/tests/common + bash _test_manifest_utils.sh init_ChatQnA + popd + ;; + install_ChatQnA) + NAMESPACE=$2 + install_chatqna $NAMESPACE + popd + ;; + validate_ChatQnA) + NAMESPACE=$2 + SERVICE_NAME=chatqna-vllm + validate_chatqna $NAMESPACE chatqna-vllm + ret=$? + if [ $ret -ne 0 ]; then + exit $ret + fi + ;; + + *) + echo "Unknown function: $1" + ;; +esac