From 341f97af96fb5cf79e28e1bf12e2e1c04eb3dfcb Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Wed, 4 Sep 2024 10:01:17 +0800 Subject: [PATCH] image scan and publish cd enhance (#604) Signed-off-by: chensuyue --- .github/workflows/_get-image-list.yml | 65 ++++++++++++++++++ .github/workflows/manual-docker-publish.yml | 55 +++++++++++++++ ...al-bom-scan.yml => manual-docker-scan.yml} | 67 ++++++++++--------- 3 files changed, 157 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/_get-image-list.yml create mode 100644 .github/workflows/manual-docker-publish.yml rename .github/workflows/{manual-bom-scan.yml => manual-docker-scan.yml} (63%) diff --git a/.github/workflows/_get-image-list.yml b/.github/workflows/_get-image-list.yml new file mode 100644 index 000000000..aa6192df0 --- /dev/null +++ b/.github/workflows/_get-image-list.yml @@ -0,0 +1,65 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: Get Image List +permissions: read-all +on: + workflow_call: + inputs: + services: + default: "" + required: false + type: string + images: + default: "" + required: false + type: string + mode: + default: "CD" + required: false + type: string + outputs: + matrix: + description: "Image List" + value: ${{ jobs.get-image-list.outputs.matrix }} + +jobs: + get-image-list: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.get-matrix.outputs.matrix }} + steps: + - name: Checkout out Repo + uses: actions/checkout@v4 + + - name: Set Matrix + id: get-matrix + run: | + image_list=[] + if [[ ! -z "${{ inputs.services }}" ]]; then + pip install yq + services=($(echo ${{ inputs.services }} | tr ',' ' ')) + for service in ${services[@]} + do + if [[ "${{ inputs.mode }}" == "CD" ]]; then + docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose-cd.yaml + else + docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml + fi + if [ -f "$docker_compose_yml" ]; then + images=$(cat $docker_compose_yml | yq -r '.[]' | jq 'keys' | jq -c '.') + image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images})) + fi + done + fi + + if [[ ! -z "${{ inputs.images }}" ]]; then + images=($(echo ${{ inputs.images }} | tr ',' ' ')) + input_image_list=$(printf '%s\n' "${images[@]}" | sort -u | jq -R '.' | jq -sc '.') + image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${input_image_list})) + fi + + echo "print image list..." + echo "$image_list" | jq . | jq -r '.[]' + echo "end of image list..." + echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT diff --git a/.github/workflows/manual-docker-publish.yml b/.github/workflows/manual-docker-publish.yml new file mode 100644 index 000000000..b1124f28f --- /dev/null +++ b/.github/workflows/manual-docker-publish.yml @@ -0,0 +1,55 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: Services publish docker image on manual event +on: + workflow_dispatch: + inputs: + services: + default: "asr" + description: "List of services to test [agent_langchain,asr,chathistory_mongo,dataprep_milvus,embeddings,guardrails,knowledgegraphs,llms,lvms,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]" + required: false + type: string + images: + default: "" + description: "List of images to scan [dataprep-pgvector,dataprep-pinecone...]" + required: false + type: string + tag: + default: "v0.9" + description: "Tag to publish" + required: true + type: string + publish_tags: + default: "latest,v0.9" + description: "Tag list apply to publish images" + required: false + type: string + +permissions: read-all +jobs: + get-image-list: + uses: ./.github/workflows/_get-image-list.yml + with: + services: ${{ inputs.services }} + images: ${{ inputs.images }} + mode: ${{ inputs.mode }} + + publish: + needs: [get-image-list] + strategy: + matrix: + image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }} + runs-on: "docker-build-gaudi" + steps: + - uses: docker/login-action@v3.2.0 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Image Publish + uses: opea-project/validation/actions/image-publish@main + with: + local_image_ref: ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }} + image_name: opea/${{ matrix.image }} + publish_tags: ${{ inputs.publish_tags }} diff --git a/.github/workflows/manual-bom-scan.yml b/.github/workflows/manual-docker-scan.yml similarity index 63% rename from .github/workflows/manual-bom-scan.yml rename to .github/workflows/manual-docker-scan.yml index 4a781b254..c10999645 100644 --- a/.github/workflows/manual-bom-scan.yml +++ b/.github/workflows/manual-docker-scan.yml @@ -1,14 +1,19 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -name: Comps docker images BoM scan on manual event +name: Services docker images BoM/CVE scan on manual event on: workflow_dispatch: inputs: services: default: "asr" description: "List of services to test [agent_langchain,asr,chathistory_mongo,dataprep_milvus...]" #,embeddings,guardrails,knowledgegraphs,llms,lvms,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]" - required: true + required: false + type: string + images: + default: "" + description: "List of images to scan [dataprep-pgvector,dataprep-pinecone...]" + required: false type: string tag: default: "comps" @@ -25,31 +30,22 @@ on: description: "Enable trivy-scan" required: false type: boolean + mode: + default: "CD" + description: "Whether the test range is CI or CD" + required: false + type: string permissions: read-all jobs: get-image-list: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.scan-matrix.outputs.matrix }} - steps: - - name: Checkout out Repo - uses: actions/checkout@v4 + uses: ./.github/workflows/_get-image-list.yml + with: + services: ${{ inputs.services }} + images: ${{ inputs.images }} + mode: ${{ inputs.mode }} - - name: Set Matrix - id: scan-matrix - run: | - pip install yq - services=($(echo ${{ inputs.services }} | tr ',' ' ')) - image_list=[] - for service in ${services[@]} - do - images=$(cat ${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml | yq -r '.[]' | jq 'keys' | jq -c '.') - image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images})) - done - echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT - - scan-license: + scan-docker: needs: get-image-list runs-on: "docker-build-gaudi" strategy: @@ -59,8 +55,7 @@ jobs: steps: - name: Pull Image run: | - docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:latest - # ${{ inputs.tag }} + docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }} echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV - name: SBOM Scan Container @@ -87,16 +82,28 @@ jobs: if: always() run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }} - - uses: actions/upload-artifact@v4.3.4 + - name: Collect Logs if: always() + run: | + mkdir -p /tmp/scan-${{ inputs.tag }}-${{ github.run_number }} + mv ${{ matrix.image }}-*-scan.txt /tmp/scan-${{ inputs.tag }}-${{ github.run_number }} + + upload-artifacts: + needs: scan-docker + runs-on: "docker-build-gaudi" + if: always() + steps: + - uses: actions/upload-artifact@v4.3.4 with: - name: sbom-scan - path: ${{ matrix.image }}-sbom-scan.txt + name: sbom-scan-${{ inputs.tag }}-${{ github.run_number }} + path: /tmp/scan-${{ inputs.tag }}-${{ github.run_number }}/*-sbom-scan.txt overwrite: true - uses: actions/upload-artifact@v4.3.4 - if: always() with: - name: trivy-scan - path: ${{ matrix.image }}-trivy-scan.txt + name: trivy-scan-${{ inputs.tag }}-${{ github.run_number }} + path: /tmp/scan-${{ inputs.tag }}-${{ github.run_number }}/*-trivy-scan.txt overwrite: true + + - name: Remove Logs + run: rm -rf /tmp/scan-${{ inputs.tag }}-${{ github.run_number }} && rm -rf /tmp/sbom-action-*