253 lines
9.3 KiB
YAML
253 lines
9.3 KiB
YAML
# Copyright (C) 2024 Intel Corporation
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
name: Helm Chart E2e Test For Call
|
|
permissions:
|
|
contents: read
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
example:
|
|
default: "chatqna"
|
|
required: true
|
|
type: string
|
|
description: "example to test, chatqna or common/asr"
|
|
hardware:
|
|
default: "xeon"
|
|
required: true
|
|
type: string
|
|
dockerhub:
|
|
default: "false"
|
|
required: false
|
|
type: string
|
|
description: "Set to true if you want to use released docker images at dockerhub. By default using internal docker registry."
|
|
mode:
|
|
default: "CD"
|
|
description: "Whether the test range is CI, CD or CICD"
|
|
required: false
|
|
type: string
|
|
tag:
|
|
default: "latest"
|
|
required: false
|
|
type: string
|
|
version:
|
|
default: "0-latest"
|
|
required: false
|
|
type: string
|
|
|
|
jobs:
|
|
get-test-case:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
value_files: ${{ steps.get-test-files.outputs.value_files }}
|
|
CHECKOUT_REF: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
|
|
steps:
|
|
- name: Get checkout ref
|
|
id: get-checkout-ref
|
|
run: |
|
|
if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
|
|
CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge
|
|
else
|
|
CHECKOUT_REF=${{ github.ref }}
|
|
fi
|
|
echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT
|
|
echo "checkout ref ${CHECKOUT_REF}"
|
|
|
|
- name: Checkout Repo
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
|
|
fetch-depth: 0
|
|
|
|
- name: Get test Services
|
|
id: get-test-files
|
|
run: |
|
|
set -x
|
|
if [ "${{ inputs.mode }}" = "CI" ]; then
|
|
base_commit=${{ github.event.pull_request.base.sha }}
|
|
merged_commit=$(git log -1 --format='%H')
|
|
values_files=$(git diff --name-only ${base_commit} ${merged_commit} | \
|
|
grep "${{ inputs.example }}/kubernetes/helm" | \
|
|
grep "values.yaml" |\
|
|
sort -u)
|
|
echo $values_files
|
|
elif [ "${{ inputs.mode }}" = "CD" ]; then
|
|
values_files=$(ls ${{ inputs.example }}/kubernetes/helm/*values.yaml || true)
|
|
fi
|
|
value_files="["
|
|
for file in ${values_files}; do
|
|
if [ -f "$file" ]; then
|
|
filename=$(basename "$file")
|
|
if [[ "$filename" == *"gaudi"* ]]; then
|
|
if [[ "${{ inputs.hardware }}" == "gaudi" ]]; then
|
|
value_files="${value_files}\"${filename}\","
|
|
fi
|
|
elif [[ "$filename" == *"rocm"* ]]; then
|
|
if [[ "${{ inputs.hardware }}" == "rocm" ]]; then
|
|
value_files="${value_files}\"${filename}\","
|
|
fi
|
|
elif [[ "$filename" == *"nv"* ]]; then
|
|
continue
|
|
else
|
|
if [[ "${{ inputs.hardware }}" == "xeon" ]]; then
|
|
value_files="${value_files}\"${filename}\","
|
|
fi
|
|
fi
|
|
fi
|
|
done
|
|
value_files="${value_files%,}]"
|
|
|
|
echo "value_files=${value_files}"
|
|
echo "value_files=${value_files}" >> $GITHUB_OUTPUT
|
|
|
|
helm-test:
|
|
needs: [get-test-case]
|
|
if: ${{ needs.get-test-case.outputs.value_files != '[]' }}
|
|
strategy:
|
|
matrix:
|
|
value_file: ${{ fromJSON(needs.get-test-case.outputs.value_files) }}
|
|
fail-fast: false
|
|
runs-on: k8s-${{ inputs.hardware }}
|
|
continue-on-error: true
|
|
steps:
|
|
- name: Clean Up Working Directory
|
|
run: |
|
|
echo "value_file=${{ matrix.value_file }}"
|
|
sudo rm -rf ${{github.workspace}}/*
|
|
|
|
- name: Get checkout ref
|
|
id: get-checkout-ref
|
|
run: |
|
|
if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
|
|
CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge
|
|
else
|
|
CHECKOUT_REF=${{ github.ref }}
|
|
fi
|
|
echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT
|
|
echo "checkout ref ${CHECKOUT_REF}"
|
|
|
|
- name: Checkout Repo
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
|
|
fetch-depth: 0
|
|
|
|
- name: Set variables
|
|
env:
|
|
example: ${{ inputs.example }}
|
|
run: |
|
|
if [[ ! "$example" =~ ^[a-zA-Z0-9]{1,20}$ ]] || [[ "$example" =~ \.\. ]] || [[ "$example" == -* || "$example" == *- ]]; then
|
|
echo "Error: Invalid input - only lowercase alphanumeric and internal hyphens allowed"
|
|
exit 1
|
|
fi
|
|
# SAFE_PREFIX="kb-"
|
|
CHART_NAME="${SAFE_PREFIX}$(echo "$example" | tr '[:upper:]' '[:lower:]')"
|
|
RAND_SUFFIX=$(openssl rand -hex 2 | tr -dc 'a-f0-9')
|
|
|
|
cat <<EOF >> $GITHUB_ENV
|
|
CHART_NAME=${CHART_NAME}
|
|
RELEASE_NAME=${CHART_NAME}-$(date +%s)
|
|
NAMESPACE=ns-${CHART_NAME}-${RAND_SUFFIX}
|
|
ROLLOUT_TIMEOUT_SECONDS=600s
|
|
TEST_TIMEOUT_SECONDS=600s
|
|
KUBECTL_TIMEOUT_SECONDS=60s
|
|
should_cleanup=false
|
|
skip_validate=false
|
|
CHART_FOLDER=${example}/kubernetes/helm
|
|
EOF
|
|
|
|
echo "Generated safe variables:" >> $GITHUB_STEP_SUMMARY
|
|
echo "- CHART_NAME: ${CHART_NAME}" >> $GITHUB_STEP_SUMMARY
|
|
|
|
- name: Helm install
|
|
id: install
|
|
env:
|
|
GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
|
|
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
|
HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
|
|
HFTOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
|
|
value_file: ${{ matrix.value_file }}
|
|
run: |
|
|
set -xe
|
|
echo "should_cleanup=true" >> $GITHUB_ENV
|
|
if [[ ! -f ${{ github.workspace }}/${{ env.CHART_FOLDER }}/${value_file} ]]; then
|
|
echo "No value file found, exiting test!"
|
|
echo "skip_validate=true" >> $GITHUB_ENV
|
|
echo "should_cleanup=false" >> $GITHUB_ENV
|
|
exit 0
|
|
fi
|
|
|
|
for img in `helm template -n $NAMESPACE $RELEASE_NAME oci://ghcr.io/opea-project/charts/${CHART_NAME} -f ${{ inputs.example }}/kubernetes/helm/${value_file} --version ${{ inputs.version }} | grep 'image:' | grep 'opea/' | awk '{print $2}' | xargs`;
|
|
do
|
|
# increase helm install wait for for vllm-gaudi case
|
|
if [[ $img == *"vllm-gaudi"* ]]; then
|
|
ROLLOUT_TIMEOUT_SECONDS=900s
|
|
fi
|
|
done
|
|
if ! helm install \
|
|
--create-namespace \
|
|
--namespace $NAMESPACE \
|
|
$RELEASE_NAME \
|
|
oci://ghcr.io/opea-project/charts/${CHART_NAME} \
|
|
--set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} \
|
|
--set global.modelUseHostPath=/data2/hf_model \
|
|
--set GOOGLE_API_KEY=${{ env.GOOGLE_API_KEY}} \
|
|
--set GOOGLE_CSE_ID=${{ env.GOOGLE_CSE_ID}} \
|
|
--set web-retriever.GOOGLE_API_KEY=${{ env.GOOGLE_API_KEY}} \
|
|
--set web-retriever.GOOGLE_CSE_ID=${{ env.GOOGLE_CSE_ID}} \
|
|
-f ${{ inputs.example }}/kubernetes/helm/${value_file} \
|
|
--version ${{ inputs.version }} \
|
|
--wait --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then
|
|
echo "Failed to install chart ${{ inputs.example }}"
|
|
echo "skip_validate=true" >> $GITHUB_ENV
|
|
.github/workflows/scripts/k8s-utils.sh dump_pods_status $NAMESPACE
|
|
exit 1
|
|
fi
|
|
|
|
- name: Validate e2e test
|
|
if: always()
|
|
run: |
|
|
set -xe
|
|
if $skip_validate; then
|
|
echo "Skip validate"
|
|
else
|
|
LOG_PATH=/home/$(whoami)/helm-logs
|
|
chart=${{ env.CHART_NAME }}
|
|
helm test -n $NAMESPACE $RELEASE_NAME --logs --timeout "$TEST_TIMEOUT_SECONDS" | tee ${LOG_PATH}/charts-${chart}.log
|
|
exit_code=$?
|
|
if [ $exit_code -ne 0 ]; then
|
|
echo "Chart ${chart} test failed, please check the logs in ${LOG_PATH}!"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Checking response results, make sure the output is reasonable. "
|
|
teststatus=false
|
|
if [[ -f $LOG_PATH/charts-${chart}.log ]] && \
|
|
[[ $(grep -c "^Phase:.*Failed" $LOG_PATH/charts-${chart}.log) != 0 ]]; then
|
|
teststatus=false
|
|
${{ github.workspace }}/.github/workflows/scripts/k8s-utils.sh dump_all_pod_logs $NAMESPACE
|
|
else
|
|
teststatus=true
|
|
fi
|
|
|
|
if [ $teststatus == false ]; then
|
|
echo "Response check failed, please check the logs in artifacts!"
|
|
exit 1
|
|
else
|
|
echo "Response check succeeded!"
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
- name: Helm uninstall
|
|
if: always()
|
|
run: |
|
|
if $should_cleanup; then
|
|
helm uninstall $RELEASE_NAME --namespace $NAMESPACE
|
|
if ! kubectl delete ns $NAMESPACE --timeout=$KUBECTL_TIMEOUT_SECONDS; then
|
|
kubectl delete pods --namespace $NAMESPACE --force --grace-period=0 --all
|
|
kubectl delete ns $NAMESPACE --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
|
|
fi
|
|
fi
|