Enable manifest integration test in CI (#220)
Signed-off-by: Yingchun Guo <yingchun.guo@intel.com>
This commit is contained in:
Ying Chun Guo
127
.github/workflows/manifest-e2e.yaml
vendored
Normal file
127
.github/workflows/manifest-e2e.yaml
vendored
Normal file
@@ -0,0 +1,127 @@
|
||||
# Copyright (C) 2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
name: E2E test with manifests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
|
||||
paths:
|
||||
- "**/kubernetes/manifests/**"
|
||||
- "**/tests/**"
|
||||
- "!**.md"
|
||||
- "!**.txt"
|
||||
- .github/workflows/manifest-e2e.yml
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
job1:
|
||||
name: Get-test-matrix
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
run_matrix: ${{ steps.get-test-matrix.outputs.run_matrix }}
|
||||
steps:
|
||||
- name: Checkout out Repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Get test matrix
|
||||
id: get-test-matrix
|
||||
run: |
|
||||
set -xe
|
||||
changed_files="$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | \
|
||||
grep "/kubernetes/manifests/" | \
|
||||
grep -vE '.github|deprecated|docker')" || true
|
||||
examples=$(printf '%s\n' "${changed_files[@]}" | grep '/' | cut -d'/' -f1 | sort -u)
|
||||
run_matrix="{\"include\":["
|
||||
for example in ${examples}; do
|
||||
run_hardware=""
|
||||
if [ $(printf '%s\n' "${changed_files[@]}" | grep ${example} | grep -c gaudi) != 0 ]; then run_hardware="gaudi"; fi
|
||||
if [ $(printf '%s\n' "${changed_files[@]}" | grep ${example} | grep -c xeon) != 0 ]; then run_hardware="xeon ${run_hardware}"; fi
|
||||
if [[ -z "$run_hardware" ]]; then run_hardware="xeon"; fi
|
||||
for hw in ${run_hardware}; do
|
||||
if [ $hw = "gaudi" ]; then
|
||||
continue # skip gaudi for K8s test temporarily
|
||||
else
|
||||
#lower_example=$(echo "${example}" | tr '[:upper:]' '[:lower:]')
|
||||
run_matrix="${run_matrix}{\"example\":\"${example}\",\"hardware\":\"inspur-icx-1\"},"
|
||||
fi
|
||||
done
|
||||
done
|
||||
run_matrix=$run_matrix"]}"
|
||||
echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT
|
||||
|
||||
manifest-test:
|
||||
needs: job1
|
||||
if: always() && ${{ needs.job1.outputs.run_matrix.include.length }} > 0
|
||||
strategy:
|
||||
matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
|
||||
runs-on: ${{ matrix.hardware }}
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: E2e test manifest
|
||||
run: |
|
||||
echo "Matrix - manifest: ${{ matrix.example }}"
|
||||
|
||||
- name: Clean Up Working Directory
|
||||
run: sudo rm -rf ${{github.workspace}}/*
|
||||
|
||||
- name: Checkout out Repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set variables
|
||||
run: |
|
||||
lower_example=$(echo "${{ matrix.example }}" | tr '[:upper:]' '[:lower:]')
|
||||
echo "NAMESPACE=$lower_example-$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
|
||||
echo "ROLLOUT_TIMEOUT_SECONDS=1800s" >> $GITHUB_ENV
|
||||
echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV
|
||||
echo "should_cleanup=false" >> $GITHUB_ENV
|
||||
echo "skip_validate=false" >> $GITHUB_ENV
|
||||
echo "NAMESPACE=$NAMESPACE"
|
||||
|
||||
- name: Initialize manifest testing
|
||||
run: |
|
||||
${{ github.workspace }}/${{ matrix.example }}/tests/test_manifest_on_xeon.sh init_${{ matrix.example }}
|
||||
|
||||
- name: Kubectl install
|
||||
id: install
|
||||
run: |
|
||||
echo "should_cleanup=true" >> $GITHUB_ENV
|
||||
kubectl create ns $NAMESPACE
|
||||
${{ github.workspace }}/${{ matrix.example }}/tests/test_manifest_on_xeon.sh install_${{ matrix.example }} $NAMESPACE
|
||||
echo "Testing ${{ matrix.example }}, waiting for pod ready..."
|
||||
if kubectl rollout status deployment --namespace "$NAMESPACE" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then
|
||||
echo "Testing manifests ${{ matrix.example }}, waiting for pod ready done!"
|
||||
else
|
||||
echo "Timeout waiting for pods in namespace $NAMESPACE to be ready!"
|
||||
echo "skip_validate=true" >> $GITHUB_ENV
|
||||
exit 1
|
||||
fi
|
||||
sleep 60
|
||||
|
||||
- name: Validate e2e test
|
||||
if: always()
|
||||
run: |
|
||||
if $skip_validate; then
|
||||
echo "Skip validate"
|
||||
else
|
||||
${{ github.workspace }}/${{ matrix.example }}/tests/test_manifest_on_xeon.sh validate_${{ matrix.example }} $NAMESPACE
|
||||
fi
|
||||
|
||||
- name: Kubectl uninstall
|
||||
if: always()
|
||||
run: |
|
||||
if $should_cleanup; then
|
||||
if ! kubectl delete ns $NAMESPACE --timeout=$KUBECTL_TIMEOUT_SECONDS; then
|
||||
kubectl delete pods --namespace $NAMESPACE --force --grace-period=0 --all
|
||||
kubectl delete ns $NAMESPACE --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
|
||||
fi
|
||||
fi
|
||||
95
ChatQnA/tests/test_manifest_on_xeon.sh
Executable file
95
ChatQnA/tests/test_manifest_on_xeon.sh
Executable file
@@ -0,0 +1,95 @@
|
||||
#!/bin/bash
|
||||
# Copyright (C) 2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
set -xe
|
||||
USER_ID=$(whoami)
|
||||
LOG_PATH=/home/$(whoami)/logs
|
||||
MOUNT_DIR=/home/$USER_ID/charts-mnt
|
||||
# IMAGE_REPO is $OPEA_IMAGE_REPO, or else ""
|
||||
IMAGE_REPO=${OPEA_IMAGE_REPO:-amr-registry.caas.intel.com/aiops}
|
||||
|
||||
function init_chatqna() {
|
||||
# executed under path manifest/chatqna/xeon
|
||||
# replace the mount dir "path: /mnt" with "path: $CHART_MOUNT"
|
||||
find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/models#path: $MOUNT_DIR#g" {} \;
|
||||
# replace the repository "image: opea/*" with "image: $IMAGE_REPO/opea/"
|
||||
find . -name '*.yaml' -type f -exec sed -i "s#image: opea/*#image: $IMAGE_REPO/opea/#g" {} \;
|
||||
# set huggingface token
|
||||
find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
|
||||
}
|
||||
|
||||
function install_chatqna {
|
||||
# replace namespace "default" with real namespace
|
||||
find . -name '*.yaml' -type f -exec sed -i "s#svc.default#svc.$NAMESPACE#g" {} \;
|
||||
# for very yaml file in yaml_files, apply it to the k8s cluster
|
||||
yaml_files=("qna_configmap_xeon" "redis-vector-db" "tei_embedding_service" "tei_reranking_service" "tgi_service" "retriever" "embedding" "reranking" "llm")
|
||||
for yaml_file in ${yaml_files[@]}; do
|
||||
kubectl apply -f $yaml_file.yaml -n $NAMESPACE
|
||||
done
|
||||
sleep 60
|
||||
kubectl apply -f chaqna-xeon-backend-server.yaml -n $NAMESPACE
|
||||
}
|
||||
|
||||
function validate_chatqna() {
|
||||
# make sure microservice retriever is ready
|
||||
until curl http://retriever-svc.$NAMESPACE:7000/v1/retrieval -X POST \
|
||||
-d '{"text":"What is the revenue of Nike in 2023?","embedding":"'"${your_embedding}"'"}' \
|
||||
-H 'Content-Type: application/json'; do sleep 10; done
|
||||
|
||||
# make sure microservice tgi-svc is ready
|
||||
until curl http://tgi-svc.$NAMESPACE:9009/generate -X POST \
|
||||
-d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \
|
||||
-H 'Content-Type: application/json'; do sleep 10; done
|
||||
|
||||
# check megaservice works
|
||||
curl http://chaqna-xeon-backend-server-svc.$NAMESPACE:8888/v1/chatqna -H "Content-Type: application/json" -d '{
|
||||
"messages": "What is the revenue of Nike in 2023?"}' > ${LOG_PATH}/curl_megaservice.log
|
||||
exit_code=$?
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "Megaservice failed, please check the logs in ${LOG_PATH}!"
|
||||
exit 1
|
||||
fi
|
||||
echo "Response check succeed!"
|
||||
|
||||
# Temporarily disable response check
|
||||
# echo "Checking response results, make sure the output is reasonable. "
|
||||
# local status=false
|
||||
# if [[ -f $LOG_PATH/curl_megaservice.log ]] &&
|
||||
# [[ $(grep -c "algorithms" $LOG_PATH/curl_megaservice.log) != 0 ]]; then
|
||||
# status=true
|
||||
# fi
|
||||
# if [ $status == false ]; then
|
||||
# echo "Response check failed, please check the logs in artifacts!"
|
||||
# exit 1
|
||||
# else
|
||||
# echo "Response check succeed!"
|
||||
# fi
|
||||
}
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "Usage: $0 <function_name>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$1" in
|
||||
init_ChatQnA)
|
||||
pushd ChatQnA/kubernetes/manifests
|
||||
init_chatqna
|
||||
popd
|
||||
;;
|
||||
install_ChatQnA)
|
||||
pushd ChatQnA/kubernetes/manifests
|
||||
NAMESPACE=$2
|
||||
install_chatqna
|
||||
popd
|
||||
;;
|
||||
validate_ChatQnA)
|
||||
NAMESPACE=$2
|
||||
SERVICE_NAME=chaqna-xeon-backend-server-svc
|
||||
validate_chatqna
|
||||
;;
|
||||
*)
|
||||
echo "Unknown function: $1"
|
||||
;;
|
||||
esac
|
||||
75
CodeGen/tests/test_manifest_on_xeon.sh
Executable file
75
CodeGen/tests/test_manifest_on_xeon.sh
Executable file
@@ -0,0 +1,75 @@
|
||||
#!/bin/bash
|
||||
# Copyright (C) 2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
set -xe
|
||||
USER_ID=$(whoami)
|
||||
LOG_PATH=/home/$(whoami)/logs
|
||||
MOUNT_DIR=/home/$USER_ID/charts-mnt
|
||||
# IMAGE_REPO is $OPEA_IMAGE_REPO, or else ""
|
||||
IMAGE_REPO=${OPEA_IMAGE_REPO:-amr-registry.caas.intel.com/aiops}
|
||||
|
||||
function init_codegen() {
|
||||
# executed under path manifest/codegen/xeon
|
||||
# replace the mount dir "path: /mnt/model" with "path: $CHART_MOUNT"
|
||||
find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt#path: $MOUNT_DIR#g" {} \;
|
||||
# replace the repository "image: opea/*" with "image: $IMAGE_REPO/opea/"
|
||||
find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"$IMAGE_REPO/opea/#g" {} \;
|
||||
# set huggingface token
|
||||
find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
|
||||
}
|
||||
|
||||
function install_codegen {
|
||||
echo "namespace is $NAMESPACE"
|
||||
kubectl apply -f . -n $NAMESPACE
|
||||
}
|
||||
|
||||
function validate_codegen() {
|
||||
ip_address=$(kubectl get svc $SERVICE_NAME -n $NAMESPACE -o jsonpath='{.spec.clusterIP}')
|
||||
port=$(kubectl get svc $SERVICE_NAME -n $NAMESPACE -o jsonpath='{.spec.ports[0].port}')
|
||||
echo "try to curl http://${ip_address}:${port}/v1/codegen..."
|
||||
# Curl the Mega Service
|
||||
curl http://${ip_address}:${port}/v1/codegen -H "Content-Type: application/json" -d '{
|
||||
"model": "ise-uiuc/Magicoder-S-DS-6.7B",
|
||||
"messages": "Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception."}' > curl_megaservice.log
|
||||
|
||||
echo "Checking response results, make sure the output is reasonable. "
|
||||
local status=true
|
||||
if [[ -f curl_megaservice.log ]] && \
|
||||
[[ $(grep -c "billion" curl_megaservice.log) != 0 ]]; then
|
||||
status=true
|
||||
fi
|
||||
|
||||
if [ $status == false ]; then
|
||||
echo "Response check failed, please check the logs in artifacts!"
|
||||
else
|
||||
echo "Response check succeed!"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "Usage: $0 <function_name>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$1" in
|
||||
init_CodeGen)
|
||||
pushd CodeGen/kubernetes/manifests/xeon
|
||||
init_codegen
|
||||
popd
|
||||
;;
|
||||
install_CodeGen)
|
||||
pushd CodeGen/kubernetes/manifests/xeon
|
||||
NAMESPACE=$2
|
||||
install_codegen
|
||||
popd
|
||||
;;
|
||||
validate_CodeGen)
|
||||
NAMESPACE=$2
|
||||
SERVICE_NAME=codegen
|
||||
validate_codegen
|
||||
;;
|
||||
*)
|
||||
echo "Unknown function: $1"
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user