# Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 name: Single Kubernetes Manifest E2e Test For Call on: workflow_call: inputs: example: default: "ChatQnA" description: "The example to test on K8s" required: true type: string hardware: default: "xeon" description: "Nodes to run the test, xeon or gaudi" required: true type: string tag: default: "latest" description: "Tag to apply to images, default is latest" required: false type: string jobs: get-test-case: runs-on: ubuntu-latest outputs: test_cases: ${{ steps.test-case-matrix.outputs.test_cases }} CHECKOUT_REF: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }} steps: - name: Get checkout ref id: get-checkout-ref run: | if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge else CHECKOUT_REF=${{ github.ref }} fi echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT echo "checkout ref ${CHECKOUT_REF}" - name: Checkout out Repo uses: actions/checkout@v4 with: ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }} fetch-depth: 0 - name: Get test matrix shell: bash id: test-case-matrix run: | example_l=$(echo ${{ inputs.example }} | tr '[:upper:]' '[:lower:]') cd ${{ github.workspace }}/${{ inputs.example }}/tests run_test_cases="" default_test_case=$(find . -type f -name "test_manifest_on_${{ inputs.hardware }}.sh" | cut -d/ -f2) if [ "$default_test_case" ]; then run_test_cases="$default_test_case"; fi other_test_cases=$(find . -type f -name "test_manifest_*_on_${{ inputs.hardware }}.sh" | cut -d/ -f2) echo "default_test_case=$default_test_case" echo "other_test_cases=$other_test_cases" if [ "${{ inputs.tag }}" == "ci" ]; then base_commit=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ "https://api.github.com/repos/opea-project/GenAIExamples/commits?sha=${{ github.event.pull_request.base.ref }}" | jq -r '.[0].sha') merged_commit=$(git log -1 --format='%H') changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | grep -vE '${{ inputs.diff_excluded_files }}')" || true fi for test_case in $other_test_cases; do if [ "${{ inputs.tag }}" == "ci" ]; then flag=${test_case%_on_*} flag=${flag#test_compose_} if [[ $(printf '%s\n' "${changed_files[@]}" | grep ${{ inputs.example }} | grep ${flag}) ]]; then run_test_cases="$run_test_cases $test_case" fi else run_test_cases="$run_test_cases $test_case" fi done test_cases=$(echo $run_test_cases | tr ' ' '\n' | sort -u | jq -R '.' | jq -sc '.') echo "test_cases=$test_cases" echo "test_cases=$test_cases" >> $GITHUB_OUTPUT manifest-test: needs: [get-test-case] strategy: matrix: test_case: ${{ fromJSON(needs.get-test-case.outputs.test_cases) }} fail-fast: false runs-on: "k8s-${{ inputs.hardware }}" continue-on-error: true steps: - name: Clean Up Working Directory run: sudo rm -rf ${{github.workspace}}/* - name: Get checkout ref run: | if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then echo "CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge" >> $GITHUB_ENV else echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV fi echo "checkout ref ${{ env.CHECKOUT_REF }}" - name: Checkout out Repo uses: actions/checkout@v4 with: ref: ${{ env.CHECKOUT_REF }} fetch-depth: 0 - name: Set variables env: test_case: ${{ matrix.test_case }} run: | echo "IMAGE_REPO=${OPEA_IMAGE_REPO}opea" >> $GITHUB_ENV echo "IMAGE_TAG=${{ inputs.tag }}" >> $GITHUB_ENV lower_example=$(echo "${{ inputs.example }}" | tr '[:upper:]' '[:lower:]') name=$(echo "$test_case" | cut -d/ -f2 | cut -d'_' -f3- |cut -d'_' -f1 | grep -v 'on' | sed 's/^/-/') echo "NAMESPACE=$lower_example$name-$(tr -dc a-z0-9 > $GITHUB_ENV echo "ROLLOUT_TIMEOUT_SECONDS=1800s" >> $GITHUB_ENV echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV echo "continue_test=true" >> $GITHUB_ENV echo "should_cleanup=false" >> $GITHUB_ENV echo "skip_validate=true" >> $GITHUB_ENV echo "NAMESPACE=$NAMESPACE" - name: Kubectl install id: install env: test_case: ${{ matrix.test_case }} run: | set -x echo "test_case=$test_case" if [[ ! -f ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} ]]; then echo "No test script found, exist test!" exit 0 else ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} init_${{ inputs.example }} echo "should_cleanup=true" >> $GITHUB_ENV kubectl create ns $NAMESPACE ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} install_${{ inputs.example }} $NAMESPACE echo "Testing ${{ inputs.example }}, waiting for pod ready..." if kubectl rollout status deployment --namespace "$NAMESPACE" --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then echo "Testing manifests ${{ inputs.example }}, waiting for pod ready done!" echo "skip_validate=false" >> $GITHUB_ENV else echo "Timeout waiting for pods in namespace $NAMESPACE to be ready!" .github/workflows/scripts/k8s-utils.sh dump_pods_status $NAMESPACE exit 1 fi sleep 60 fi - name: Validate e2e test if: always() env: test_case: ${{ matrix.test_case }} run: | if $skip_validate; then echo "Skip validate" else if ${{ github.workspace }}/${{ inputs.example }}/tests/${test_case} validate_${{ inputs.example }} $NAMESPACE ; then echo "Validate ${test_case} successful!" else echo "Validate ${test_case} failure!!!" echo "Check the logs in 'Dump logs when e2e test failed' step!!!" exit 1 fi fi - name: Dump logs when e2e test failed if: failure() run: | .github/workflows/scripts/k8s-utils.sh dump_all_pod_logs $NAMESPACE - name: Kubectl uninstall if: always() run: | if $should_cleanup; then if ! kubectl delete ns $NAMESPACE --timeout=$KUBECTL_TIMEOUT_SECONDS; then kubectl delete pods --namespace $NAMESPACE --force --grace-period=0 --all kubectl delete ns $NAMESPACE --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS fi fi