jandres/GenAIExamples
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

Split ChatQnA manifest test (#1190)

Browse Source 
Signed-off-by: ZePan110 <ze.pan@intel.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
ZePan110
2024-12-04 15:17:46 +08:00
committed by GitHub
parent 8182a83382
commit 340796bbae
7 changed files with 507 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
.github/workflows/_manifest-e2e.yml vendored
View File

@@ -22,7 +22,72 @@ on:
type: string
jobs:
get-test-case:
runs-on: ubuntu-latest
outputs:
test_cases: ${{ steps.test-case-matrix.outputs.test_cases }}
CHECKOUT_REF: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
steps:
- name: Get checkout ref
id: get-checkout-ref
run: |
if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge
else
CHECKOUT_REF=${{ github.ref }}
fi
echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT
echo "checkout ref ${CHECKOUT_REF}"
- name: Checkout out Repo
uses: actions/checkout@v4
with:
ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
fetch-depth: 0
- name: Get test matrix
shell: bash
id: test-case-matrix
run: |
example_l=$(echo ${{ inputs.example }} | tr '[:upper:]' '[:lower:]')
cd ${{ github.workspace }}/${{ inputs.example }}/tests
run_test_cases=""
default_test_case=$(find . -type f -name "test_manifest_on_${{ inputs.hardware }}.sh" | cut -d/ -f2)
if [ "$default_test_case" ]; then run_test_cases="$default_test_case"; fi
other_test_cases=$(find . -type f -name "test_manifest_*_on_${{ inputs.hardware }}.sh" | cut -d/ -f2)
echo "default_test_case=$default_test_case"
echo "other_test_cases=$other_test_cases"
if [ "${{ inputs.tag }}" == "ci" ]; then
base_commit=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
"https://api.github.com/repos/opea-project/GenAIExamples/commits?sha=${{ github.event.pull_request.base.ref }}" | jq -r '.[0].sha')
merged_commit=$(git log -1 --format='%H')
changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | grep -vE '${{ inputs.diff_excluded_files }}')" || true
fi
for test_case in $other_test_cases; do
if [ "${{ inputs.tag }}" == "ci" ]; then
flag=${test_case%_on_*}
flag=${flag#test_compose_}
if [[ $(printf '%s\n' "${changed_files[@]}" | grep ${{ inputs.example }} | grep ${flag}) ]]; then
run_test_cases="$run_test_cases $test_case"
fi
else
run_test_cases="$run_test_cases $test_case"
fi
done
test_cases=$(echo $run_test_cases | tr ' ' '\n' | sort -u | jq -R '.' | jq -sc '.')
echo "test_cases=$test_cases"
echo "test_cases=$test_cases" >> $GITHUB_OUTPUT
manifest-test:
needs: [get-test-case]
strategy:
matrix:
test_case: ${{ fromJSON(needs.get-test-case.outputs.test_cases) }}
fail-fast: false
runs-on: "k8s-${{ inputs.hardware }}"
continue-on-error: true
steps:
@@ -45,11 +110,14 @@ jobs:
fetch-depth: 0
- name: Set variables
env:
test_case: ${{ matrix.test_case }}
run: |
echo "IMAGE_REPO=${OPEA_IMAGE_REPO}opea" >> $GITHUB_ENV
echo "IMAGE_TAG=${{ inputs.tag }}" >> $GITHUB_ENV
lower_example=$(echo "${{ inputs.example }}" | tr '[:upper:]' '[:lower:]')
echo "NAMESPACE=$lower_example-$(tr -dc a-z0-9 </dev/urandom | head -c 16)" >> $GITHUB_ENV
name=$(echo "$test_case" | cut -d/ -f2 | cut -d'_' -f3- |cut -d'_' -f1 | grep -v 'on' | sed 's/^/-/')
echo "NAMESPACE=$lower_example$name-$(tr -dc a-z0-9 </dev/urandom | head -c 16)" >> $GITHUB_ENV
echo "ROLLOUT_TIMEOUT_SECONDS=1800s" >> $GITHUB_ENV
echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV
echo "continue_test=true" >> $GITHUB_ENV
@@ -59,15 +127,19 @@ jobs:
- name: Kubectl install
id: install
env:
test_case: ${{ matrix.test_case }}
run: |
if [[ ! -f ${{ github.workspace }}/${{ inputs.example }}/tests/test_manifest_on_${{ inputs.hardware }}.sh ]]; then
set -x
echo "test_case=$test_case"
if [[ ! -f ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} ]]; then
echo "No test script found, exist test!"
exit 0
else
${{ github.workspace }}/${{ inputs.example }}/tests/test_manifest_on_${{ inputs.hardware }}.sh init_${{ inputs.example }}
${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} init_${{ inputs.example }}
echo "should_cleanup=true" >> $GITHUB_ENV
kubectl create ns $NAMESPACE
${{ github.workspace }}/${{ inputs.example }}/tests/test_manifest_on_${{ inputs.hardware }}.sh install_${{ inputs.example }} $NAMESPACE
${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} install_${{ inputs.example }} $NAMESPACE
echo "Testing ${{ inputs.example }}, waiting for pod ready..."
if kubectl rollout status deployment --namespace "$NAMESPACE" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then
echo "Testing manifests ${{ inputs.example }}, waiting for pod ready done!"
@@ -82,14 +154,16 @@ jobs:
- name: Validate e2e test
if: always()
env:
test_case: ${{ matrix.test_case }}
run: |
if $skip_validate; then
echo "Skip validate"
else
if ${{ github.workspace }}/${{ inputs.example }}/tests/test_manifest_on_${{ inputs.hardware }}.sh validate_${{ inputs.example }} $NAMESPACE ; then
echo "Validate ${{ inputs.example }} successful!"
if ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} validate_${{ inputs.example }} $NAMESPACE ; then
echo "Validate ${test_case} successful!"
else
echo "Validate ${{ inputs.example }} failure!!!"
echo "Validate ${test_case} failure!!!"
echo "Check the logs in 'Dump logs when e2e test failed' step!!!"
exit 1
fi

64
ChatQnA/tests/common/_test_manifest_utils.sh Normal file
View File

@@ -0,0 +1,64 @@
#!/bin/bash
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
set -xe
USER_ID=$(whoami)
MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
IMAGE_REPO=${IMAGE_REPO:-opea}
IMAGE_TAG=${IMAGE_TAG:-latest}
ROLLOUT_TIMEOUT_SECONDS="1800s"
KUBECTL_TIMEOUT_SECONDS="60s"
function init_chatqna() {
# replace the mount dir "path: /mnt/opea-models" with "path: $CHART_MOUNT"
find ../../kubernetes/intel/*/*/manifest -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \;
# replace microservice image tag
find ../../kubernetes/intel/*/*/manifest -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
# replace the repository "image: opea/*" with "image: $IMAGE_REPO/"
find ../../kubernetes/intel/*/*/manifest -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \;
# set huggingface token
find ../../kubernetes/intel/*/*/manifest -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
}
function get_end_point() {
# $1 is service name, $2 is namespace
ip_address=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.clusterIP}')
port=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.ports[0].port}')
echo "$ip_address:$port"
}
function _cleanup_ns() {
local ns=$1
if kubectl get ns $ns; then
if ! kubectl delete ns $ns --timeout=$KUBECTL_TIMEOUT_SECONDS; then
kubectl delete pods --namespace $ns --force --grace-period=0 --all
kubectl delete ns $ns --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
fi
fi
}
if [ $# -eq 0 ]; then
echo "Usage: $0 <function_name>"
exit 1
fi
case "$1" in
init_ChatQnA)
init_chatqna
;;
get_end_point)
service=$2
NAMESPACE=$3
get_end_point $service $NAMESPACE
;;
_cleanup_ns)
NAMESPACE=$2
_cleanup_ns $NAMESPACE
;;
*)
echo "Unknown function: $1"
;;
esac

117
ChatQnA/tests/test_manifest_guardrails_on_gaudi.sh Executable file
View File

@@ -0,0 +1,117 @@
#!/bin/bash
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
set -xe
USER_ID=$(whoami)
LOG_PATH=/home/$(whoami)/logs
MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
IMAGE_REPO=${IMAGE_REPO:-opea}
IMAGE_TAG=${IMAGE_TAG:-latest}
ROLLOUT_TIMEOUT_SECONDS="1800s"
KUBECTL_TIMEOUT_SECONDS="60s"
function validate_chatqna() {
local ns=$1
local log=$2
max_retry=20
# make sure microservice retriever-usvc is ready
# try to curl retriever-svc for max_retry times
test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
for ((i=1; i<=max_retry; i++))
do
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-retriever-usvc" $ns)
curl http://$endpoint_url/v1/retrieval -X POST \
-d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \
-H 'Content-Type: application/json' && break
sleep 30
done
# if i is bigger than max_retry, then exit with error
if [ $i -gt $max_retry ]; then
echo "Microservice retriever failed, exit with error."
return 1
fi
# make sure microservice tgi-svc is ready
for ((i=1; i<=max_retry; i++))
do
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-tgi" $ns)
curl http://$endpoint_url/generate -X POST \
-d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \
-H 'Content-Type: application/json' && break
sleep 30
done
# if i is bigger than max_retry, then exit with error
if [ $i -gt $max_retry ]; then
echo "Microservice tgi failed, exit with error."
return 1
fi
# check megaservice works
# generate a random logfile name to avoid conflict among multiple runners
LOGFILE=$LOG_PATH/curlmega_$log.log
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna" $ns)
curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo "Megaservice failed, please check the logs in $LOGFILE!"
return ${exit_code}
fi
echo "Checking response results, make sure the output is reasonable. "
local status=false
if [[ -f $LOGFILE ]] &&
[[ $(grep -c "\[DONE\]" $LOGFILE) != 0 ]]; then
status=true
fi
if [[ $status == false ]]; then
echo "Response check failed, please check the logs in artifacts!"
return 1
else
echo "Response check succeed!"
fi
return 0
}
function install_chatqna() {
echo "Testing manifests chatqna_guardrails"
local ns=$1
bash ChatQnA/tests/common/_test_manifest_utils.sh _cleanup_ns $ns
pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest
kubectl create namespace $ns
# install guardrails
kubectl apply -f chatqna-guardrails.yaml -n $ns
# Sleep enough time for chatqna_guardrails to be ready
sleep 60
}
if [ $# -eq 0 ]; then
echo "Usage: $0 <function_name>"
exit 1
fi
case "$1" in
init_ChatQnA)
pushd ChatQnA/tests/common
bash _test_manifest_utils.sh init_ChatQnA
popd
;;
install_ChatQnA)
NAMESPACE=$2
install_chatqna $NAMESPACE
popd
;;
validate_ChatQnA)
NAMESPACE=$2
SERVICE_NAME=chatqna-guardrails
validate_chatqna $NAMESPACE chatqna-guardrails
ret=$?
if [ $ret -ne 0 ]; then
exit $ret
fi
;;
*)
echo "Unknown function: $1"
;;
esac

117
ChatQnA/tests/test_manifest_guardrails_on_xeon.sh Executable file
View File

@@ -0,0 +1,117 @@
#!/bin/bash
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
set -xe
USER_ID=$(whoami)
LOG_PATH=/home/$(whoami)/logs
MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
IMAGE_REPO=${IMAGE_REPO:-opea}
IMAGE_TAG=${IMAGE_TAG:-latest}
ROLLOUT_TIMEOUT_SECONDS="1800s"
KUBECTL_TIMEOUT_SECONDS="60s"
function validate_chatqna() {
local ns=$1
local log=$2
max_retry=10
# make sure microservice retriever-usvc is ready
# try to curl retriever-svc for max_retry times
test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
for ((i=1; i<=max_retry; i++))
do
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-retriever-usvc" $ns)
curl http://$endpoint_url/v1/retrieval -X POST \
-d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \
-H 'Content-Type: application/json' && break
sleep 30
done
# if i is bigger than max_retry, then exit with error
if [ $i -gt $max_retry ]; then
echo "Microservice retriever failed, exit with error."
return 1
fi
# make sure microservice tgi-svc is ready
for ((i=1; i<=max_retry; i++))
do
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-tgi" $ns)
curl http://$endpoint_url/generate -X POST \
-d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \
-H 'Content-Type: application/json' && break
sleep 30
done
# if i is bigger than max_retry, then exit with error
if [ $i -gt $max_retry ]; then
echo "Microservice tgi failed, exit with error."
return 1
fi
# check megaservice works
# generate a random logfile name to avoid conflict among multiple runners
LOGFILE=$LOG_PATH/curlmega_$log.log
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna" $ns)
curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo "Megaservice failed, please check the logs in $LOGFILE!"
return ${exit_code}
fi
echo "Checking response results, make sure the output is reasonable. "
local status=false
if [[ -f $LOGFILE ]] &&
[[ $(grep -c "\[DONE\]" $LOGFILE) != 0 ]]; then
status=true
fi
if [ $status == false ]; then
echo "Response check failed, please check the logs in artifacts!"
return 1
else
echo "Response check succeed!"
fi
return 0
}
function install_chatqna() {
echo "Testing manifests chatqna_guardrails"
local ns=$1
bash ChatQnA/tests/common/_test_manifest_utils.sh _cleanup_ns $ns
pushd ChatQnA/kubernetes/intel/cpu/xeon/manifest
kubectl create namespace $ns
# install guardrail
kubectl apply -f chatqna-guardrails.yaml -n $ns
# Sleep enough time for chatqna_guardrails to be ready
sleep 60
}
if [ $# -eq 0 ]; then
echo "Usage: $0 <function_name>"
exit 1
fi
case "$1" in
init_ChatQnA)
pushd ChatQnA/tests/common
bash _test_manifest_utils.sh init_ChatQnA
popd
;;
install_ChatQnA)
NAMESPACE=$2
install_chatqna $NAMESPACE
popd
;;
validate_ChatQnA)
NAMESPACE=$2
SERVICE_NAME=chatqna-guardrails
validate_chatqna $NAMESPACE chatqna-guardrails
ret=$?
if [ $ret -ne 0 ]; then
exit $ret
fi
;;
*)
echo "Unknown function: $1"
;;
esac

156
ChatQnA/tests/test_manifest_on_gaudi.sh
View File

@@ -12,17 +12,6 @@ IMAGE_TAG=${IMAGE_TAG:-latest}
ROLLOUT_TIMEOUT_SECONDS="1800s"
KUBECTL_TIMEOUT_SECONDS="60s"
function init_chatqna() {
# replace the mount dir "path: /mnt/opea-models" with "path: $CHART_MOUNT"
find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \;
# replace microservice image tag
find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
# replace the repository "image: opea/*" with "image: $IMAGE_REPO/"
find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \;
# set huggingface token
find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
}
function install_chatqna {
echo "namespace is $NAMESPACE"
kubectl apply -f chatqna.yaml -n $NAMESPACE
@@ -30,13 +19,6 @@ function install_chatqna {
sleep 60
}
function get_end_point() {
# $1 is service name, $2 is namespace
ip_address=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.clusterIP}')
port=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.ports[0].port}')
echo "$ip_address:$port"
}
function validate_chatqna() {
local ns=$1
local log=$2
@@ -46,7 +28,7 @@ function validate_chatqna() {
test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
for ((i=1; i<=max_retry; i++))
do
endpoint_url=$(get_end_point "chatqna-retriever-usvc" $ns)
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-retriever-usvc" $ns)
curl http://$endpoint_url/v1/retrieval -X POST \
-d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \
-H 'Content-Type: application/json' && break
@@ -60,7 +42,7 @@ function validate_chatqna() {
# make sure microservice tgi-svc is ready
for ((i=1; i<=max_retry; i++))
do
endpoint_url=$(get_end_point "chatqna-tgi" $ns)
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-tgi" $ns)
curl http://$endpoint_url/generate -X POST \
-d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \
-H 'Content-Type: application/json' && break
@@ -75,7 +57,7 @@ function validate_chatqna() {
# check megaservice works
# generate a random logfile name to avoid conflict among multiple runners
LOGFILE=$LOG_PATH/curlmega_$log.log
endpoint_url=$(get_end_point "chatqna" $ns)
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna" $ns)
curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE
exit_code=$?
if [ $exit_code -ne 0 ]; then
@@ -98,128 +80,6 @@ function validate_chatqna() {
return 0
}
function validate_chatqna_vllm() {
local ns=$1
local log=$2
max_retry=20
# make sure microservice retriever-usvc is ready
# try to curl retriever-svc for max_retry times
test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
for ((i=1; i<=max_retry; i++))
do
endpoint_url=$(get_end_point "chatqna-retriever-usvc" $ns)
curl http://$endpoint_url/v1/retrieval -X POST \
-d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \
-H 'Content-Type: application/json' && break
sleep 30
done
# if i is bigger than max_retry, then exit with error
if [ $i -gt $max_retry ]; then
echo "Microservice retriever failed, exit with error."
return 1
fi
# make sure microservice vllm-svc is ready
for ((i=1; i<=max_retry; i++))
do
endpoint_url=$(get_end_point "chatqna-vllm" $ns)
curl http://$endpoint_url/v1/chat/completions -X POST \
-d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}' \
-H 'Content-Type: application/json' && break
sleep 30
done
# if i is bigger than max_retry, then exit with error
if [ $i -gt $max_retry ]; then
echo "Microservice vllm failed, exit with error."
return 1
fi
# check megaservice works
# generate a random logfile name to avoid conflict among multiple runners
LOGFILE=$LOG_PATH/curlmega_$log.log
endpoint_url=$(get_end_point "chatqna" $ns)
curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo "Megaservice failed, please check the logs in $LOGFILE!"
return ${exit_code}
fi
echo "Checking response results, make sure the output is reasonable. "
local status=false
if [[ -f $LOGFILE ]] &&
[[ $(grep -c "\[DONE\]" $LOGFILE) != 0 ]]; then
status=true
fi
if [ $status == false ]; then
echo "Response check failed, please check the logs in artifacts!"
return 1
else
echo "Response check succeed!"
fi
return 0
}
function _cleanup_ns() {
local ns=$1
if kubectl get ns $ns; then
if ! kubectl delete ns $ns --timeout=$KUBECTL_TIMEOUT_SECONDS; then
kubectl delete pods --namespace $ns --force --grace-period=0 --all
kubectl delete ns $ns --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
fi
fi
}
function install_and_validate_chatqna_guardrail() {
echo "Testing manifests chatqna_guardrils"
local ns=${NAMESPACE}
_cleanup_ns $ns
kubectl create namespace $ns
# install guardrail
kubectl apply -f chatqna-guardrails.yaml -n $ns
# Sleep enough time for chatqna_guardrail to be ready
sleep 60
if kubectl rollout status deployment -n "$ns" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then
echo "Waiting for chatqna_guardrail pod ready done!"
else
echo "Timeout waiting for chatqna_guardrail pod ready!"
exit 1
fi
# validate guardrail
validate_chatqna $ns chatqna-guardrails
local ret=$?
if [ $ret -ne 0 ]; then
exit 1
fi
}
function install_and_validate_chatqna_vllm() {
echo "Testing manifests chatqna_vllm"
local ns=${NAMESPACE}
_cleanup_ns $ns
kubectl create namespace $ns
# install guardrail
kubectl apply -f chatqna-vllm.yaml -n $ns
# Sleep enough time for chatqna_vllm to be ready, vllm warmup takes about 5 minutes
sleep 280
if kubectl rollout status deployment -n "$ns" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then
echo "Waiting for chatqna_vllm pod ready done!"
else
echo "Timeout waiting for chatqna_vllm pod ready!"
exit 1
fi
# validate guardrail
validate_chatqna_vllm $ns chatqna-vllm
local ret=$?
if [ $ret -ne 0 ]; then
exit 1
fi
}
if [ $# -eq 0 ]; then
echo "Usage: $0 <function_name>"
exit 1
@@ -227,8 +87,8 @@ fi
case "$1" in
init_ChatQnA)
pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest
init_chatqna
pushd ChatQnA/tests/common
bash _test_manifest_utils.sh init_ChatQnA
popd
;;
install_ChatQnA)
@@ -245,12 +105,6 @@ case "$1" in
if [ $ret -ne 0 ]; then
exit $ret
fi
pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest
install_and_validate_chatqna_guardrail
popd
pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest
install_and_validate_chatqna_vllm
popd
;;
*)

66
ChatQnA/tests/test_manifest_on_xeon.sh
View File

@@ -12,17 +12,6 @@ IMAGE_TAG=${IMAGE_TAG:-latest}
ROLLOUT_TIMEOUT_SECONDS="1800s"
KUBECTL_TIMEOUT_SECONDS="60s"
function init_chatqna() {
# replace the mount dir "path: /mnt/opea-models" with "path: $CHART_MOUNT"
find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \;
# replace microservice image tag
find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
# replace the repository "image: opea/*" with "image: $IMAGE_REPO/"
find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \;
# set huggingface token
find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
}
function install_chatqna {
echo "namespace is $NAMESPACE"
kubectl apply -f chatqna.yaml -n $NAMESPACE
@@ -30,13 +19,6 @@ function install_chatqna {
sleep 60
}
function get_end_point() {
# $1 is service name, $2 is namespace
ip_address=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.clusterIP}')
port=$(kubectl get svc $1 -n $2 -o jsonpath='{.spec.ports[0].port}')
echo "$ip_address:$port"
}
function validate_chatqna() {
local ns=$1
local log=$2
@@ -46,7 +28,7 @@ function validate_chatqna() {
test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
for ((i=1; i<=max_retry; i++))
do
endpoint_url=$(get_end_point "chatqna-retriever-usvc" $ns)
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-retriever-usvc" $ns)
curl http://$endpoint_url/v1/retrieval -X POST \
-d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \
-H 'Content-Type: application/json' && break
@@ -60,7 +42,7 @@ function validate_chatqna() {
# make sure microservice tgi-svc is ready
for ((i=1; i<=max_retry; i++))
do
endpoint_url=$(get_end_point "chatqna-tgi" $ns)
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-tgi" $ns)
curl http://$endpoint_url/generate -X POST \
-d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \
-H 'Content-Type: application/json' && break
@@ -75,7 +57,7 @@ function validate_chatqna() {
# check megaservice works
# generate a random logfile name to avoid conflict among multiple runners
LOGFILE=$LOG_PATH/curlmega_$log.log
endpoint_url=$(get_end_point "chatqna" $ns)
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna" $ns)
curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE
exit_code=$?
if [ $exit_code -ne 0 ]; then
@@ -98,41 +80,6 @@ function validate_chatqna() {
return 0
}
function _cleanup_ns() {
local ns=$1
if kubectl get ns $ns; then
if ! kubectl delete ns $ns --timeout=$KUBECTL_TIMEOUT_SECONDS; then
kubectl delete pods --namespace $ns --force --grace-period=0 --all
kubectl delete ns $ns --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
fi
fi
}
function install_and_validate_chatqna_guardrail() {
echo "Testing manifests chatqna_guardrils"
local ns=${NAMESPACE}
_cleanup_ns $ns
kubectl create namespace $ns
# install guardrail
kubectl apply -f chatqna-guardrails.yaml -n $ns
# Sleep enough time for chatqna_guardrail to be ready
sleep 60
if kubectl rollout status deployment -n "$ns" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then
echo "Waiting for chatqna_guardrail pod ready done!"
else
echo "Timeout waiting for chatqna_guardrail pod ready!"
exit 1
fi
# validate guardrail
validate_chatqna $ns chatqna-guardrails
local ret=$?
if [ $ret -ne 0 ]; then
exit 1
fi
}
if [ $# -eq 0 ]; then
echo "Usage: $0 <function_name>"
exit 1
@@ -140,8 +87,8 @@ fi
case "$1" in
init_ChatQnA)
pushd ChatQnA/kubernetes/intel/cpu/xeon/manifest
init_chatqna
pushd ChatQnA/tests/common
bash _test_manifest_utils.sh init_ChatQnA
popd
;;
install_ChatQnA)
@@ -158,9 +105,6 @@ case "$1" in
if [ $ret -ne 0 ]; then
exit $ret
fi
pushd ChatQnA/kubernetes/intel/cpu/xeon/manifest
install_and_validate_chatqna_guardrail
popd
;;
*)
echo "Unknown function: $1"

118
ChatQnA/tests/test_manifest_vllm_on_gaudi.sh Executable file
View File

@@ -0,0 +1,118 @@
#!/bin/bash
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
set -xe
USER_ID=$(whoami)
LOG_PATH=/home/$(whoami)/logs
MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
IMAGE_REPO=${IMAGE_REPO:-opea}
IMAGE_TAG=${IMAGE_TAG:-latest}
ROLLOUT_TIMEOUT_SECONDS="1800s"
KUBECTL_TIMEOUT_SECONDS="60s"
function validate_chatqna() {
local ns=$1
local log=$2
max_retry=20
# make sure microservice retriever-usvc is ready
# try to curl retriever-svc for max_retry times
test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
for ((i=1; i<=max_retry; i++))
do
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-retriever-usvc" $ns)
curl http://$endpoint_url/v1/retrieval -X POST \
-d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" \
-H 'Content-Type: application/json' && break
sleep 30
done
# if i is bigger than max_retry, then exit with error
if [ $i -gt $max_retry ]; then
echo "Microservice retriever failed, exit with error."
return 1
fi
# make sure microservice vllm-svc is ready
for ((i=1; i<=max_retry; i++))
do
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna-vllm" $ns)
curl http://$endpoint_url/v1/chat/completions -X POST \
-d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}' \
-H 'Content-Type: application/json' && break
sleep 30
done
# if i is bigger than max_retry, then exit with error
if [ $i -gt $max_retry ]; then
echo "Microservice vllm failed, exit with error."
return 1
fi
# check megaservice works
# generate a random logfile name to avoid conflict among multiple runners
LOGFILE=$LOG_PATH/curlmega_$log.log
endpoint_url=$(bash ChatQnA/tests/common/_test_manifest_utils.sh get_end_point "chatqna" $ns)
curl http://$endpoint_url/v1/chatqna -H "Content-Type: application/json" -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": "What is the revenue of Nike in 2023?"}' > $LOGFILE
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo "Megaservice failed, please check the logs in $LOGFILE!"
return ${exit_code}
fi
echo "Checking response results, make sure the output is reasonable. "
local status=false
if [[ -f $LOGFILE ]] &&
[[ $(grep -c "\[DONE\]" $LOGFILE) != 0 ]]; then
status=true
fi
if [ $status == false ]; then
echo "Response check failed, please check the logs in artifacts!"
return 1
else
echo "Response check succeed!"
fi
return 0
}
function install_chatqna() {
echo "Testing manifests chatqna_vllm"
local ns=$1
bash ChatQnA/tests/common/_test_manifest_utils.sh _cleanup_ns $ns
kubectl create namespace $ns
# install guardrail
pushd ChatQnA/kubernetes/intel/hpu/gaudi/manifest
kubectl apply -f chatqna-vllm.yaml -n $ns
# Sleep enough time for chatqna_vllm to be ready, vllm warmup takes about 5 minutes
sleep 280
}
if [ $# -eq 0 ]; then
echo "Usage: $0 <function_name>"
exit 1
fi
case "$1" in
init_ChatQnA)
pushd ChatQnA/tests/common
bash _test_manifest_utils.sh init_ChatQnA
popd
;;
install_ChatQnA)
NAMESPACE=$2
install_chatqna $NAMESPACE
popd
;;
validate_ChatQnA)
NAMESPACE=$2
SERVICE_NAME=chatqna-vllm
validate_chatqna $NAMESPACE chatqna-vllm
ret=$?
if [ $ret -ne 0 ]; then
exit $ret
fi
;;
*)
echo "Unknown function: $1"
;;
esac