add docker container clean up in CI (#131)

Signed-off-by: chensuyue <suyue.chen@intel.com>

.github/workflows/ChatQnA_gaudi.yml

@@ -4,7 +4,7 @@
 name: ChatQnA E2E test on Gaudi
 
 on:
-  pull_request:
+  pull_request_target:
     branches: [main]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
@@ -37,7 +37,17 @@ jobs:
           AISE_CLUSTER_01_2_IP: ${{ secrets.AISE_CLUSTER_01_2_IP }}
         run: |
           cd ${{ github.workspace }}/ChatQnA/tests
-          bash test_chatqna_on_gaudi.sh
+          timeout 20m bash test_chatqna_on_gaudi.sh
+
+      - name: Clean Up Container
+        if: cancelled() || failure()
+        run: |
+          cd ${{ github.workspace }}/ChatQnA/microservice/gaudi
+          container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
+          for container_name in $container_list; do
+              cid=$(docker ps -aq --filter "name=$container_name")
+              if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
+          done
 
       - name: Publish pipeline artifact
         if: ${{ !cancelled() }}

.github/workflows/ChatQnA_xeon.yml

@@ -4,7 +4,7 @@
 name: ChatQnA E2E test on Xeon
 
 on:
-  pull_request:
+  pull_request_target:
     branches: [main]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
@@ -37,7 +37,17 @@ jobs:
           AISE_CLUSTER_01_2_IP: ${{ secrets.AISE_CLUSTER_01_2_IP }}
         run: |
           cd ${{ github.workspace }}/ChatQnA/tests
-          bash test_chatqna_on_xeon.sh
+          timeout 20m bash test_chatqna_on_xeon.sh
+
+      - name: Clean Up Container
+        if: cancelled() || failure()
+        run: |
+          cd ${{ github.workspace }}/ChatQnA/microservice/xeon
+          container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
+          for container_name in $container_list; do
+              cid=$(docker ps -aq --filter "name=$container_name")
+              if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
+          done
 
       - name: Publish pipeline artifact
         if: ${{ !cancelled() }}

ChatQnA/tests/test_chatqna_on_gaudi.sh

@@ -81,7 +81,7 @@ function check_microservices() {
         -X POST \
         -d '{"text":"hello"}' \
         -H 'Content-Type: application/json' > ${LOG_PATH}/embeddings.log
-    sleep 5s
+    sleep 10s
 
     curl http://${ip_address}:7000/v1/retrieval \
         -X POST \

ChatQnA/tests/test_chatqna_on_xeon.sh

@@ -73,7 +73,7 @@ function check_microservices() {
         -X POST \
         -d '{"text":"hello"}' \
         -H 'Content-Type: application/json' > ${LOG_PATH}/embeddings.log
-    sleep 5s
+    sleep 10s
 
     test_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
     curl http://${ip_address}:7000/v1/retrieval \