Enhance CI/CD infrastructure (#593)

Signed-off-by: chensuyue <suyue.chen@intel.com> Signed-off-by: Sun, Xuehao <xuehao.sun@intel.com>
2024-08-15 22:39:21 +08:00
parent e71aba0080
commit c26d0f62b8
61 changed files with 989 additions and 956 deletions
--- a/FaqGen/docker/docker_build_compose.yaml
+++ b/FaqGen/docker/docker_build_compose.yaml
@@ -0,0 +1,30 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  faqgen:
+    build:
+      args:
+        http_proxy: ${http_proxy}
+        https_proxy: ${https_proxy}
+        no_proxy: ${no_proxy}
+      dockerfile: ./Dockerfile
+    image: ${REGISTRY:-opea}/faqgen:${TAG:-latest}
+  faqgen-ui:
+    build:
+      context: ui
+      dockerfile: ./docker/Dockerfile
+    extends: faqgen
+    image: ${REGISTRY:-opea}/faqgen-ui:${TAG:-latest}
+  faqgen-react-ui:
+    build:
+      context: ui
+      dockerfile: ./docker/Dockerfile.react
+    extends: faqgen
+    image: ${REGISTRY:-opea}/faqgen-react-ui:${TAG:-latest}
+  llm-faqgen-tgi:
+    build:
+      context: GenAIComps
+      dockerfile: comps/llms/faq-generation/tgi/Dockerfile
+    extends: faqgen
+    image: ${REGISTRY:-opea}/llm-faqgen-tgi:${TAG:-latest}
--- a/FaqGen/docker/gaudi/compose.yaml
+++ b/FaqGen/docker/gaudi/compose.yaml
@@ -1,8 +1,6 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0

-version: "3.8"
-
 services:
  tgi-service:
    image: ghcr.io/huggingface/tgi-gaudi:2.0.1
@@ -26,7 +24,7 @@ services:
    ipc: host
    command: --model-id ${LLM_MODEL_ID} --max-input-length 2048 --max-total-tokens 4096 --max-batch-total-tokens 65536 --max-batch-prefill-tokens 4096
  llm_faqgen:
-    image: opea/llm-faqgen-tgi:latest
+    image: ${REGISTRY:-opea}/llm-faqgen-tgi:${TAG:-latest}
    container_name: llm-faqgen-server
    depends_on:
      - tgi-service
@@ -41,7 +39,7 @@ services:
      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
    restart: unless-stopped
  faqgen-gaudi-backend-server:
-    image: opea/faqgen:latest
+    image: ${REGISTRY:-opea}/faqgen:${TAG:-latest}
    container_name: faqgen-gaudi-backend-server
    depends_on:
      - tgi-service
@@ -57,7 +55,7 @@ services:
    ipc: host
    restart: always
  faqgen-gaudi-ui-server:
-    image: opea/faqgen-ui:latest
+    image: ${REGISTRY:-opea}/faqgen-ui:${TAG:-latest}
    container_name: faqgen-gaudi-ui-server
    depends_on:
      - faqgen-gaudi-backend-server
--- a/FaqGen/docker/xeon/compose.yaml
+++ b/FaqGen/docker/xeon/compose.yaml
@@ -1,8 +1,6 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0

-version: "3.8"
-
 services:
  tgi-service:
    image: ghcr.io/huggingface/text-generation-inference:1.4
@@ -20,7 +18,7 @@ services:
    shm_size: 1g
    command: --model-id ${LLM_MODEL_ID}
  llm_faqgen:
-    image: opea/llm-faqgen-tgi:latest
+    image: ${REGISTRY:-opea}/llm-faqgen-tgi:${TAG:-latest}
    container_name: llm-faqgen-server
    depends_on:
      - tgi-service
@@ -35,7 +33,7 @@ services:
      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
    restart: unless-stopped
  faqgen-xeon-backend-server:
-    image: opea/faqgen:latest
+    image: ${REGISTRY:-opea}/faqgen:${TAG:-latest}
    container_name: faqgen-xeon-backend-server
    depends_on:
      - tgi-service
@@ -51,7 +49,7 @@ services:
    ipc: host
    restart: always
  faqgen-xeon-ui-server:
-    image: opea/faqgen-ui:latest
+    image: ${REGISTRY:-opea}/faqgen-ui:${TAG:-latest}
    container_name: faqgen-xeon-ui-server
    depends_on:
      - faqgen-xeon-backend-server
--- a/FaqGen/tests/test_faqgen_on_gaudi.sh
+++ b/FaqGen/tests/test_faqgen_on_gaudi.sh
@@ -3,26 +3,26 @@
 # SPDX-License-Identifier: Apache-2.0

 set -x
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+export REGISTRY=${IMAGE_REPO}
+export TAG=${IMAGE_TAG}

 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')

 function build_docker_images() {
-    cd $WORKPATH
-
+    cd $WORKPATH/docker
    git clone https://github.com/opea-project/GenAIComps.git
-    cd GenAIComps
-    docker build --no-cache -t opea/llm-faqgen-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/faq-generation/tgi/Dockerfile .
+
+    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
+    service_list="faqgen faqgen-ui llm-faqgen-tgi"
+    docker compose -f docker_build_compose.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log

    docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1
-
-    cd $WORKPATH/docker
-    docker build --no-cache -t opea/faqgen:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
-
-    cd $WORKPATH/docker/ui
-    docker build --no-cache -t opea/faqgen-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
-
    docker images
 }

@@ -38,14 +38,6 @@ function start_services() {

    sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env

-    # if [[ "$IMAGE_REPO" != "" ]]; then
-    #     # Replace the container name with a test-specific name
-    #     echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
-    #     sed -i "s#image: opea/faqgen:latest#image: opea/faqgen:${IMAGE_TAG}#g" compose.yaml
-    #     sed -i "s#image: opea/faqgen-ui:latest#image: opea/faqgen-ui:${IMAGE_TAG}#g" compose.yaml
-    #     sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" compose.yaml
-    # fi
-
    # Start Docker Containers
    docker compose up -d

@@ -122,13 +114,16 @@ function validate_frontend() {
    cd $WORKPATH/docker/ui/svelte
    local conda_env_name="OPEA_e2e"
    export PATH=${HOME}/miniforge3/bin/:$PATH
-#    conda remove -n ${conda_env_name} --all -y
-#    conda create -n ${conda_env_name} python=3.12 -y
+    if conda info --envs | grep -q "$conda_env_name"; then
+        echo "$conda_env_name exist!"
+    else
+        conda create -n ${conda_env_name} python=3.12 -y
+    fi
    source activate ${conda_env_name}

    sed -i "s/localhost/$ip_address/g" playwright.config.ts

-#    conda install -c conda-forge nodejs -y
+    conda install -c conda-forge nodejs -y
    npm install && npm ci && npx playwright install --with-deps
    node -v && npm -v && pip list

@@ -152,7 +147,7 @@ function main() {

    stop_docker

-    build_docker_images
+    if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
    start_services

    validate_microservices
--- a/FaqGen/tests/test_faqgen_on_xeon.sh
+++ b/FaqGen/tests/test_faqgen_on_xeon.sh
@@ -3,25 +3,26 @@
 # SPDX-License-Identifier: Apache-2.0

 set -x
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+export REGISTRY=${IMAGE_REPO}
+export TAG=${IMAGE_TAG}

 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')

 function build_docker_images() {
-    cd $WORKPATH
-
-    git clone https://github.com/opea-project/GenAIComps.git
-    cd GenAIComps
-
-    docker build --no-cache -t opea/llm-faqgen-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/faq-generation/tgi/Dockerfile .
-
    cd $WORKPATH/docker
-    docker build --no-cache -t opea/faqgen:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
+    git clone https://github.com/opea-project/GenAIComps.git

-    cd $WORKPATH/docker/ui
-    docker build --no-cache -t opea/faqgen-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
+    service_list="faqgen faqgen-ui llm-faqgen-tgi"
+    docker compose -f docker_build_compose.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log

+    docker pull ghcr.io/huggingface/text-generation-inference:1.4
    docker images
 }

@@ -37,14 +38,6 @@ function start_services() {

    sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env

-    # if [[ "$IMAGE_REPO" != "" ]]; then
-    #     # Replace the container name with a test-specific name
-    #     echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
-    #     sed -i "s#image: opea/faqgen:latest#image: opea/faqgen:${IMAGE_TAG}#g" compose.yaml
-    #     sed -i "s#image: opea/faqgen-ui:latest#image: opea/faqgen-ui:${IMAGE_TAG}#g" compose.yaml
-    #     sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" compose.yaml
-    # fi
-
    # Start Docker Containers
    docker compose up -d

@@ -121,13 +114,16 @@ function validate_frontend() {
    cd $WORKPATH/docker/ui/svelte
    local conda_env_name="OPEA_e2e"
    export PATH=${HOME}/miniforge3/bin/:$PATH
-#    conda remove -n ${conda_env_name} --all -y
-#    conda create -n ${conda_env_name} python=3.12 -y
+    if conda info --envs | grep -q "$conda_env_name"; then
+        echo "$conda_env_name exist!"
+    else
+        conda create -n ${conda_env_name} python=3.12 -y
+    fi
    source activate ${conda_env_name}

    sed -i "s/localhost/$ip_address/g" playwright.config.ts

-#    conda install -c conda-forge nodejs -y
+    conda install -c conda-forge nodejs -y
    npm install && npm ci && npx playwright install --with-deps
    node -v && npm -v && pip list

@@ -151,7 +147,7 @@ function main() {

    stop_docker

-    build_docker_images
+    if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
    start_services

    validate_microservices