Build up docker images CD workflow (#576)

Signed-off-by: chensuyue <suyue.chen@intel.com>

.github/workflows/_example-workflow.yml

@@ -0,0 +1,164 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Example jobs
+permissions: read-all
+on:
+  workflow_call:
+    inputs:
+      example:
+        required: true
+        type: string
+      tag:
+        default: "latest"
+        required: false
+        type: string
+      build:
+        default: true
+        required: false
+        type: boolean
+      scan:
+        default: true
+        required: false
+        type: boolean
+      test_compose:
+        default: false
+        required: false
+        type: boolean
+      test_k8s:
+        default: false
+        required: false
+        type: boolean
+      publish:
+        default: false
+        required: false
+        type: boolean
+      publish_tags:
+        default: "latest"
+        required: false
+        type: string
+jobs:
+####################################################################################################
+# Image Build
+####################################################################################################
+  build-images:
+    if: ${{ fromJSON(inputs.build) }}
+    strategy:
+      matrix:
+        node: ["docker-build-xeon", "docker-build-gaudi"]
+    runs-on: ${{ matrix.node }}
+    continue-on-error: true
+    steps:
+      - name: Clean Up Working Directory
+        run: |
+          sudo rm -rf ${{github.workspace}}/*
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Build Image
+        uses: opea-project/validation/actions/image-build@main
+        with:
+          work_dir: ${{ github.workspace }}/${{ inputs.example }}
+          docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.example }}-compose.yaml
+          registry: ${OPEA_IMAGE_REPO}opea
+          tag: ${{ inputs.tag }}
+
+####################################################################################################
+# Trivy Scan
+####################################################################################################
+  image-list:
+    needs: [ build-images ]
+    if: ${{ fromJSON(inputs.scan) }}
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.scan-matrix.outputs.matrix }}
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@v2.8.1
+        with:
+          egress-policy: audit
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Set Matrix
+        id: scan-matrix
+        run: |
+          pip install yq
+          compose_path=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.example }}-compose.yaml
+          echo "matrix=$(cat ${compose_path} | yq -r '.[]' | jq 'keys' | jq -c '.')" >> $GITHUB_OUTPUT
+
+  scan-images:
+    needs: [image-list]
+    if: ${{ fromJSON(inputs.scan) }}
+    runs-on: "docker-build-gaudi"
+    strategy:
+      matrix:
+        image: ${{ fromJSON(needs.image-list.outputs.matrix) }}
+      fail-fast: false
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@v2.8.1
+        with:
+          egress-policy: audit
+
+      - name: Pull Image
+        run: docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+
+      - name: Scan Container
+        uses: opea-project/validation/actions/trivy-scan@main
+        with:
+          image-ref: ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+          output: ${{ inputs.example }}-${{ matrix.image }}-scan.txt
+
+      - name: Cleanup
+        if: always()
+        run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+      - uses: actions/upload-artifact@v4.3.4
+        with:
+          name: ${{ inputs.example }}-scan
+          path: ${{ inputs.example }}-${{ matrix.image }}-scan.txt
+          overwrite: true
+
+####################################################################################################
+# Docker Compose Test
+####################################################################################################
+  test-example-compose:
+    needs: [build-images]
+    if: ${{ fromJSON(inputs.test_compose) }}
+    strategy:
+      matrix:
+        hardware: ["xeon", "gaudi"]
+      fail-fast: false
+    uses: ./.github/workflows/_run-docker-compose.yml
+    with:
+      tag: ${{ inputs.tag }}
+      example: ${{ inputs.example }}
+      hardware: ${{ matrix.hardware }}
+    secrets: inherit
+
+
+####################################################################################################
+# K8S Test
+####################################################################################################
+# TODO
+
+
+####################################################################################################
+# Publish
+####################################################################################################
+  publish:
+    needs: [image-list, build-images, scan-images, test-example-compose]
+    if: ${{ fromJSON(inputs.publish) }}
+    strategy:
+      matrix:
+        image: ${{ fromJSON(needs.image-list.outputs.matrix) }}
+    runs-on: "docker-build-gaudi"
+    steps:
+      - name: Image Publish
+        uses: opea-project/validation/actions/image-publish@main
+        with:
+          local_image_ref: ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+          image_name: opea/${{ matrix.image }}
+          publish_tags: ${{ inputs.publish_tags }}

.github/workflows/docker/compose/AudioQnA-compose.yaml

@@ -7,4 +7,4 @@ services:
     build:
       context: docker
       dockerfile: ./Dockerfile
-    image: ${REGISTRY}opea/audioqna:${TAG:-latest}
+    image: ${REGISTRY:-opea}/audioqna:${TAG:-latest}

.github/workflows/docker/compose/ChatQnA-compose.yaml

@@ -7,14 +7,14 @@ services:
     build:
       context: docker
       dockerfile: ./Dockerfile
-    image: ${REGISTRY}opea/chatqna:${TAG:-latest}
+    image: ${REGISTRY:-opea}/chatqna:${TAG:-latest}
   chatqna-ui:
     build:
       context: docker/ui
       dockerfile: ./docker/Dockerfile
-    image: ${REGISTRY}opea/chatqna-ui:${TAG:-latest}
+    image: ${REGISTRY:-opea}/chatqna-ui:${TAG:-latest}
   chatqna-conversation-ui:
     build:
       context: docker/ui
       dockerfile: ./docker/Dockerfile.react
-    image: ${REGISTRY}opea/chatqna-conversation-ui:${TAG:-latest}
+    image: ${REGISTRY:-opea}/chatqna-conversation-ui:${TAG:-latest}

.github/workflows/docker/compose/CodeGen-compose.yaml

@@ -7,14 +7,14 @@ services:
     build:
       context: docker
       dockerfile: ./Dockerfile
-    image: ${REGISTRY}opea/codegen:${TAG:-latest}
+    image: ${REGISTRY:-opea}/codegen:${TAG:-latest}
   codegen-ui:
     build:
       context: docker/ui
       dockerfile: ./docker/Dockerfile
-    image: ${REGISTRY}opea/codegen-ui:${TAG:-latest}
+    image: ${REGISTRY:-opea}/codegen-ui:${TAG:-latest}
   codegen-react-ui:
     build:
       context: docker/ui
       dockerfile: ./docker/Dockerfile.react
-    image: ${REGISTRY}opea/codegen-conversation-ui:${TAG:-latest}
+    image: ${REGISTRY:-opea}/codegen-conversation-ui:${TAG:-latest}

.github/workflows/docker/compose/CodeTrans-compose.yaml

@@ -7,9 +7,9 @@ services:
     build:
       context: docker
       dockerfile: ./Dockerfile
-    image: ${REGISTRY}opea/codetrans:${TAG:-latest}
+    image: ${REGISTRY:-opea}/codetrans:${TAG:-latest}
   codetrans-ui:
     build:
       context: docker/ui
       dockerfile: ./docker/Dockerfile
-    image: ${REGISTRY}opea/codetrans-ui:${TAG:-latest}
+    image: ${REGISTRY:-opea}/codetrans-ui:${TAG:-latest}

.github/workflows/docker/compose/DocSum-compose.yaml

@@ -7,14 +7,14 @@ services:
     build:
       context: docker
       dockerfile: ./Dockerfile
-    image: ${REGISTRY}opea/docsum:${TAG:-latest}
+    image: ${REGISTRY:-opea}/docsum:${TAG:-latest}
   docsum-ui:
     build:
       context: docker/ui
       dockerfile: ./docker/Dockerfile
-    image: ${REGISTRY}opea/docsum-ui:${TAG:-latest}
+    image: ${REGISTRY:-opea}/docsum-ui:${TAG:-latest}
   docsum-react-ui:
     build:
       context: docker/ui
       dockerfile: ./docker/Dockerfile.react
-    image: ${REGISTRY}opea/docsum-react-ui:${TAG:-latest}
+    image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest}

.github/workflows/docker/compose/FaqGen-compose.yaml

@@ -7,14 +7,14 @@ services:
     build:
       context: docker
       dockerfile: ./Dockerfile
-    image: ${REGISTRY}opea/faqgen:${TAG:-latest}
+    image: ${REGISTRY:-opea}/faqgen:${TAG:-latest}
   faqgen-ui:
     build:
       context: docker/ui
       dockerfile: ./docker/Dockerfile
-    image: ${REGISTRY}opea/faqgen-ui:${TAG:-latest}
+    image: ${REGISTRY:-opea}/faqgen-ui:${TAG:-latest}
   faqgen-react-ui:
     build:
       context: docker/ui
       dockerfile: ./docker/Dockerfile.react
-    image: ${REGISTRY}opea/faqgen-react-ui:${TAG:-latest}
+    image: ${REGISTRY:-opea}/faqgen-react-ui:${TAG:-latest}

.github/workflows/docker/compose/SearchQnA-compose.yaml

@@ -7,9 +7,9 @@ services:
     build:
       context: docker
       dockerfile: ./Dockerfile
-    image: ${REGISTRY}opea/searchqna:${TAG:-latest}
+    image: ${REGISTRY:-opea}/searchqna:${TAG:-latest}
   searchqna-ui:
     build:
       context: docker/ui
       dockerfile: ./docker/Dockerfile
-    image: ${REGISTRY}opea/searchqna-ui:${TAG:-latest}
+    image: ${REGISTRY:-opea}/searchqna-ui:${TAG:-latest}

.github/workflows/docker/compose/Translation-compose.yaml

@@ -7,9 +7,9 @@ services:
     build:
       context: docker
       dockerfile: ./Dockerfile
-    image: ${REGISTRY}opea/translation:${TAG:-latest}
+    image: ${REGISTRY:-opea}/translation:${TAG:-latest}
   translation-ui:
     build:
       context: docker/ui
       dockerfile: ./docker/Dockerfile
-    image: ${REGISTRY}opea/translation-ui:${TAG:-latest}
+    image: ${REGISTRY:-opea}/translation-ui:${TAG:-latest}

.github/workflows/manual-cd-workflow.yml

@@ -0,0 +1,79 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Examples CD workflow on manual event
+on:
+  workflow_dispatch:
+    inputs:
+      examples:
+        default: "AudioQnA,ChatQnA,CodeGen,CodeTrans,DocSum,FaqGen,SearchQnA,Translation"
+        description: 'List of examples to test'
+        required: true
+        type: string
+      tag:
+        default: "latest"
+        description: "Tag to apply to images"
+        required: true
+        type: string
+      build:
+        default: true
+        description: 'Build test required images for Examples'
+        required: false
+        type: boolean
+      scan:
+        default: true
+        description: 'Scan all images with Trivy'
+        required: false
+        type: boolean
+      test_compose:
+        default: true
+        description: 'Test examples with docker compose'
+        required: false
+        type: boolean
+      test_k8s:
+        default: false
+        description: 'Test examples with k8s'
+        required: false
+        type: boolean
+      publish:
+        default: false
+        description: 'Publish images to docker hub'
+        required: false
+        type: boolean
+      publish_tags:
+        default: "latest,v0.9"
+        description: 'Tag list apply to publish images'
+        required: false
+        type: string
+
+permissions: read-all
+jobs:
+  get-test-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.get-matrix.outputs.matrix }}
+    steps:
+    - name: Create Matrix
+      id: get-matrix
+      run: |
+        examples=($(echo ${{ github.event.inputs.examples }} | tr ',' ' '))
+        examples_json=$(printf '%s\n' "${examples[@]}" | sort -u | jq -R '.' | jq -sc '.')
+        echo "matrix=$examples_json" >> $GITHUB_OUTPUT
+
+  run-examples:
+    needs: [get-test-matrix]
+    strategy:
+      matrix:
+        example: ${{ fromJson(needs.get-test-matrix.outputs.matrix) }}
+      fail-fast: false
+    uses: ./.github/workflows/_example-workflow.yml
+    with:
+      example: ${{ matrix.example }}
+      tag: ${{ inputs.tag }}
+      build: ${{ fromJSON(inputs.build) }}
+      scan: ${{ fromJSON(inputs.scan) }}
+      test_compose: ${{ fromJSON(inputs.test_compose) }}
+      test_k8s: ${{ fromJSON(inputs.test_k8s) }}
+      publish: ${{ fromJSON(inputs.publish) }}
+      publish_tags: ${{ fromJSON(inputs.publish_tags) }}
+    secrets: inherit

.github/workflows/manual-image-build.yml

@@ -74,5 +74,5 @@ jobs:
         with:
           work_dir: ${{ github.workspace }}/${{ matrix.service }}
           docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${{ matrix.service }}-compose.yaml
-          registry: ${{ env.image_repo }}
+          registry: ${{ env.image_repo }}opea
           tag: ${{ github.event.inputs.tag }}

.github/workflows/push-image-build.yml

@@ -54,4 +54,4 @@ jobs:
         with:
           work_dir: ${{ github.workspace }}/${{ matrix.workload }}
           docker_compose_path: ${{ env.docker_compose_path }}
-          registry: ${OPEA_IMAGE_REPO}
+          registry: ${OPEA_IMAGE_REPO}opea