Enhance CI/CD infrastructure (#593)

Signed-off-by: chensuyue <suyue.chen@intel.com> Signed-off-by: Sun, Xuehao <xuehao.sun@intel.com>
2024-08-15 22:39:21 +08:00
parent e71aba0080
commit c26d0f62b8
61 changed files with 989 additions and 956 deletions
--- a/CodeTrans/docker/docker_build_compose.yaml
+++ b/CodeTrans/docker/docker_build_compose.yaml
@@ -0,0 +1,24 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  codetrans:
+    build:
+      args:
+        http_proxy: ${http_proxy}
+        https_proxy: ${https_proxy}
+        no_proxy: ${no_proxy}
+      dockerfile: ./Dockerfile
+    image: ${REGISTRY:-opea}/codetrans:${TAG:-latest}
+  codetrans-ui:
+    build:
+      context: ui
+      dockerfile: ./docker/Dockerfile
+    extends: codetrans
+    image: ${REGISTRY:-opea}/codetrans-ui:${TAG:-latest}
+  llm-tgi:
+    build:
+      context: GenAIComps
+      dockerfile: comps/llms/text-generation/tgi/Dockerfile
+    extends: codetrans
+    image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
--- a/CodeTrans/docker/gaudi/compose.yaml
+++ b/CodeTrans/docker/gaudi/compose.yaml
@@ -1,9 +1,6 @@
-
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0

-version: "3.8"
-
 services:
  tgi-service:
    image: ghcr.io/huggingface/tgi-gaudi:2.0.1
@@ -25,7 +22,7 @@ services:
    ipc: host
    command: --model-id ${LLM_MODEL_ID} --max-input-length 1024 --max-total-tokens 2048
  llm:
-    image: opea/llm-tgi:latest
+    image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
    container_name: llm-tgi-gaudi-server
    ports:
      - "9000:9000"
@@ -38,7 +35,7 @@ services:
      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
    restart: unless-stopped
  codetrans-gaudi-backend-server:
-    image: opea/codetrans:latest
+    image: ${REGISTRY:-opea}/codetrans:${TAG:-latest}
    container_name: codetrans-gaudi-backend-server
    depends_on:
      - tgi-service
@@ -54,7 +51,7 @@ services:
    ipc: host
    restart: always
  codetrans-gaudi-ui-server:
-    image: opea/codetrans-ui:latest
+    image: ${REGISTRY:-opea}/codetrans-ui:${TAG:-latest}
    container_name: codetrans-gaudi-ui-server
    depends_on:
      - codetrans-gaudi-backend-server
--- a/CodeTrans/docker/xeon/compose.yaml
+++ b/CodeTrans/docker/xeon/compose.yaml
@@ -1,9 +1,6 @@
-
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0

-version: "3.8"
-
 services:
  tgi-service:
    image: ghcr.io/huggingface/text-generation-inference:1.4
@@ -20,7 +17,7 @@ services:
      HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
    command: --model-id ${LLM_MODEL_ID}
  llm:
-    image: opea/llm-tgi:latest
+    image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
    container_name: llm-tgi-server
    ports:
      - "9000:9000"
@@ -33,7 +30,7 @@ services:
      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
    restart: unless-stopped
  codetrans-xeon-backend-server:
-    image: opea/codetrans:latest
+    image: ${REGISTRY:-opea}/codetrans:${TAG:-latest}
    container_name: codetrans-xeon-backend-server
    depends_on:
      - tgi-service
@@ -49,7 +46,7 @@ services:
    ipc: host
    restart: always
  codetrans-xeon-ui-server:
-    image: opea/codetrans-ui:latest
+    image: ${REGISTRY:-opea}/codetrans-ui:${TAG:-latest}
    container_name: codetrans-xeon-ui-server
    depends_on:
      - codetrans-xeon-backend-server
--- a/CodeTrans/tests/test_codetrans_on_gaudi.sh
+++ b/CodeTrans/tests/test_codetrans_on_gaudi.sh
@@ -2,27 +2,27 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0

-set -x
+set -xe
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+export REGISTRY=${IMAGE_REPO}
+export TAG=${IMAGE_TAG}

 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')

 function build_docker_images() {
-    cd $WORKPATH
+    cd $WORKPATH/docker
    git clone https://github.com/opea-project/GenAIComps.git
-    cd GenAIComps

-    docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
+    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
+    service_list="codetrans codetrans-ui llm-tgi"
+    docker compose -f docker_build_compose.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log

    docker pull ghcr.io/huggingface/tgi-gaudi:2.0.1
-
-    cd $WORKPATH/docker
-    docker build --no-cache -t opea/codetrans:latest -f Dockerfile .
-
-    cd $WORKPATH/docker/ui
-    docker build --no-cache -t opea/codetrans-ui:latest -f docker/Dockerfile .
-
    docker images
 }

@@ -40,16 +40,6 @@ function start_services() {

    sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env

-    if [[ "$IMAGE_REPO" != "" ]]; then
-        # Replace the container name with a test-specific name
-        echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
-        sed -i "s#image: opea/codetrans:latest#image: opea/codetrans:${IMAGE_TAG}#g" compose.yaml
-        sed -i "s#image: opea/codetrans-ui:latest#image: opea/codetrans-ui:${IMAGE_TAG}#g" compose.yaml
-        sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" compose.yaml
-        echo "cat compose.yaml"
-        cat compose.yaml
-    fi
-
    # Start Docker Containers
    docker compose up -d

@@ -124,13 +114,16 @@ function validate_frontend() {
    cd $WORKPATH/docker/ui/svelte
    local conda_env_name="OPEA_e2e"
    export PATH=${HOME}/miniforge3/bin/:$PATH
-#    conda remove -n ${conda_env_name} --all -y
-#    conda create -n ${conda_env_name} python=3.12 -y
+    if conda info --envs | grep -q "$conda_env_name"; then
+        echo "$conda_env_name exist!"
+    else
+        conda create -n ${conda_env_name} python=3.12 -y
+    fi
    source activate ${conda_env_name}

    sed -i "s/localhost/$ip_address/g" playwright.config.ts

-#    conda install -c conda-forge nodejs -y
+    conda install -c conda-forge nodejs -y
    npm install && npm ci && npx playwright install --with-deps
    node -v && npm -v && pip list

@@ -155,7 +148,7 @@ function main() {

    stop_docker

-    if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
+    if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
    start_services

    validate_microservices
--- a/CodeTrans/tests/test_codetrans_on_xeon.sh
+++ b/CodeTrans/tests/test_codetrans_on_xeon.sh
@@ -3,24 +3,26 @@
 # SPDX-License-Identifier: Apache-2.0

 set -xe
+IMAGE_REPO=${IMAGE_REPO:-"opea"}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
+echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}"
+echo "TAG=IMAGE_TAG=${IMAGE_TAG}"
+export REGISTRY=${IMAGE_REPO}
+export TAG=${IMAGE_TAG}

 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
 ip_address=$(hostname -I | awk '{print $1}')

 function build_docker_images() {
-    cd $WORKPATH
-    git clone https://github.com/opea-project/GenAIComps.git
-    cd GenAIComps
-
-    docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile .
-
    cd $WORKPATH/docker
-    docker build --no-cache -t opea/codetrans:latest -f Dockerfile .
+    git clone https://github.com/opea-project/GenAIComps.git

-    cd $WORKPATH/docker/ui
-    docker build --no-cache -t opea/codetrans-ui:latest -f docker/Dockerfile .
+    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
+    service_list="codetrans codetrans-ui llm-tgi"
+    docker compose -f docker_build_compose.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log

+    docker pull ghcr.io/huggingface/text-generation-inference:1.4
    docker images
 }

@@ -37,16 +39,6 @@ function start_services() {

    sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env

-    if [[ "$IMAGE_REPO" != "" ]]; then
-        # Replace the container name with a test-specific name
-        echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG"
-        sed -i "s#image: opea/codetrans:latest#image: opea/codetrans:${IMAGE_TAG}#g" compose.yaml
-        sed -i "s#image: opea/codetrans-ui:latest#image: opea/codetrans-ui:${IMAGE_TAG}#g" compose.yaml
-        sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" compose.yaml
-        echo "cat compose.yaml"
-        cat compose.yaml
-    fi
-
    # Start Docker Containers
    docker compose up -d

@@ -123,13 +115,16 @@ function validate_frontend() {
    cd $WORKPATH/docker/ui/svelte
    local conda_env_name="OPEA_e2e"
    export PATH=${HOME}/miniforge3/bin/:$PATH
-#    conda remove -n ${conda_env_name} --all -y
-#    conda create -n ${conda_env_name} python=3.12 -y
+    if conda info --envs | grep -q "$conda_env_name"; then
+        echo "$conda_env_name exist!"
+    else
+        conda create -n ${conda_env_name} python=3.12 -y
+    fi
    source activate ${conda_env_name}

    sed -i "s/localhost/$ip_address/g" playwright.config.ts

-#    conda install -c conda-forge nodejs -y
+    conda install -c conda-forge nodejs -y
    npm install && npm ci && npx playwright install --with-deps
    node -v && npm -v && pip list

@@ -153,7 +148,7 @@ function main() {

    stop_docker

-    if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi
+    if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
    start_services

    validate_microservices