jandres/GenAIExamples
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[CICD enhance] DocIndexRetriever run CI with latest base image, group logs in GHA outputs. (#1932)

Browse Source 
Signed-off-by: chensuyue <suyue.chen@intel.com>
This commit is contained in:
chen, suyue
2025-05-16 15:16:02 +08:00
committed by GitHub
parent 11b04b38db
commit 7ee6f3657c
7 changed files with 115 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
DocIndexRetriever/Dockerfile
View File

@@ -1,8 +1,9 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
ARG IMAGE_REPO=opea
ARG BASE_TAG=latest
FROM opea/comps-base:$BASE_TAG
FROM $IMAGE_REPO/comps-base:$BASE_TAG
COPY ./retrieval_tool.py $HOME/retrieval_tool.py

2
DocIndexRetriever/docker_image_build/build.yaml
View File

@@ -5,6 +5,8 @@ services:
doc-index-retriever:
build:
args:
IMAGE_REPO: ${REGISTRY:-opea}
BASE_TAG: ${TAG:-latest}
http_proxy: ${http_proxy}
https_proxy: ${https_proxy}
no_proxy: ${no_proxy}

40
DocIndexRetriever/tests/test_compose_milvus_on_gaudi.sh
View File

@@ -20,17 +20,15 @@ function build_docker_images() {
if [ ! -d "GenAIComps" ] ; then
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
fi
pushd GenAIComps
echo "GenAIComps test commit is $(git rev-parse HEAD)"
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
popd && sleep 1s
service_list="dataprep embedding retriever reranking doc-index-retriever"
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
docker pull ghcr.io/huggingface/tei-gaudi:1.5.0
docker pull quay.io/coreos/etcd:v3.5.5
docker pull minio/minio:RELEASE.2023-03-20T20-16-18Z
docker pull milvusdb/milvus:v2.4.6
docker images && sleep 1s
echo "Docker images built!"
}
function start_services() {
@@ -112,19 +110,27 @@ function stop_docker() {
function main() {
echo "::group::stop_docker"
stop_docker
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
echo "Dump current docker ps"
docker ps
start_time=$(date +%s)
start_services
end_time=$(date +%s)
duration=$((end_time-start_time))
echo "Mega service start duration is $duration s"
validate_megaservice
echo "::endgroup::"
echo "::group::build_docker_images"
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
echo "::endgroup::"
echo "::group::start_services"
start_services
echo "::endgroup::"
echo "::group::validate_megaservice"
validate_megaservice
echo "::endgroup::"
echo "::group::stop_docker"
stop_docker
echo y | docker system prune
echo "::endgroup::"
docker system prune -f
}

39
DocIndexRetriever/tests/test_compose_milvus_on_xeon.sh
View File

@@ -20,16 +20,15 @@ function build_docker_images() {
if [ ! -d "GenAIComps" ] ; then
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
fi
pushd GenAIComps
echo "GenAIComps test commit is $(git rev-parse HEAD)"
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
popd && sleep 1s
service_list="dataprep embedding retriever reranking doc-index-retriever"
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
docker pull quay.io/coreos/etcd:v3.5.5
docker pull minio/minio:RELEASE.2023-03-20T20-16-18Z
docker pull milvusdb/milvus:v2.4.6
docker images && sleep 1s
echo "Docker images built!"
}
function start_services() {
@@ -111,19 +110,27 @@ function stop_docker() {
function main() {
echo "::group::stop_docker"
stop_docker
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
echo "Dump current docker ps"
docker ps
start_time=$(date +%s)
start_services
end_time=$(date +%s)
duration=$((end_time-start_time))
echo "Mega service start duration is $duration s"
validate_megaservice
echo "::endgroup::"
echo "::group::build_docker_images"
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
echo "::endgroup::"
echo "::group::start_services"
start_services
echo "::endgroup::"
echo "::group::validate_megaservice"
validate_megaservice
echo "::endgroup::"
echo "::group::stop_docker"
stop_docker
echo y | docker system prune
echo "::endgroup::"
docker system prune -f
}

35
DocIndexRetriever/tests/test_compose_on_gaudi.sh
View File

@@ -21,14 +21,15 @@ function build_docker_images() {
if [ ! -d "GenAIComps" ] ; then
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
fi
pushd GenAIComps
echo "GenAIComps test commit is $(git rev-parse HEAD)"
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
popd && sleep 1s
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log
docker pull redis/redis-stack:7.2.0-v9
docker pull ghcr.io/huggingface/tei-gaudi:1.5.0
docker images && sleep 1s
echo "Docker images built!"
}
function start_services() {
@@ -103,19 +104,27 @@ function stop_docker() {
function main() {
echo "::group::stop_docker"
stop_docker
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
echo "Dump current docker ps"
docker ps
start_time=$(date +%s)
start_services
end_time=$(date +%s)
duration=$((end_time-start_time))
echo "Mega service start duration is $duration s"
validate_megaservice
echo "::endgroup::"
echo "::group::build_docker_images"
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
echo "::endgroup::"
echo "::group::start_services"
start_services
echo "::endgroup::"
echo "::group::validate_megaservice"
validate_megaservice
echo "::endgroup::"
echo "::group::stop_docker"
stop_docker
echo y | docker system prune
echo "::endgroup::"
docker system prune -f
}

32
DocIndexRetriever/tests/test_compose_on_xeon.sh
View File

@@ -21,14 +21,15 @@ function build_docker_images() {
if [ ! -d "GenAIComps" ] ; then
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
fi
pushd GenAIComps
echo "GenAIComps test commit is $(git rev-parse HEAD)"
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
popd && sleep 1s
service_list="dataprep embedding retriever reranking doc-index-retriever"
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.6
docker pull redis/redis-stack:7.2.0-v9
docker images && sleep 1s
echo "Docker images built!"
}
function start_services() {
@@ -110,20 +111,27 @@ function stop_docker() {
function main() {
echo "::group::stop_docker"
stop_docker
echo "::endgroup::"
echo "::group::build_docker_images"
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
echo "Dump current docker ps"
docker ps
echo "::endgroup::"
start_time=$(date +%s)
echo "::group::start_services"
start_services
end_time=$(date +%s)
duration=$((end_time-start_time))
echo "Mega service start duration is $duration s"
validate_megaservice
echo "::endgroup::"
echo "::group::validate_megaservice"
validate_megaservice
echo "::endgroup::"
echo "::group::stop_docker"
stop_docker
echo y | docker system prune
echo "::endgroup::"
docker system prune -f
}

33
DocIndexRetriever/tests/test_compose_without_rerank_on_xeon.sh
View File

@@ -21,6 +21,11 @@ function build_docker_images() {
if [ ! -d "GenAIComps" ] ; then
git clone --single-branch --branch "${opea_branch:-"main"}" https://github.com/opea-project/GenAIComps.git
fi
pushd GenAIComps
echo "GenAIComps test commit is $(git rev-parse HEAD)"
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
popd && sleep 1s
service_list="dataprep embedding retriever doc-index-retriever"
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
@@ -114,19 +119,27 @@ function stop_docker() {
function main() {
echo "::group::stop_docker"
stop_docker
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
echo "Dump current docker ps"
docker ps
start_time=$(date +%s)
start_services
end_time=$(date +%s)
duration=$((end_time-start_time))
echo "Mega service start duration is $duration s"
validate_megaservice
echo "::endgroup::"
echo "::group::build_docker_images"
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
echo "::endgroup::"
echo "::group::start_services"
start_services
echo "::endgroup::"
echo "::group::validate_megaservice"
validate_megaservice
echo "::endgroup::"
echo "::group::stop_docker"
stop_docker
echo y | docker system prune
echo "::endgroup::"
docker system prune -f
}