Refactor Retrievers related Examples (#1387)

Delete redundant retrievers docker image in docker_images_list.md. Refactor Retrievers related Examples READMEs. Change all of the comps/retrievers/xxx/xxx/Dockerfile path into comps/retrievers/src/Dockerfile. Fix the Examples CI issues of PR opea-project/GenAIComps#1138. Signed-off-by: letonghan <letong.han@intel.com>
2025-01-16 14:21:48 +08:00
parent 698a06edbf
commit 4cabd55778
61 changed files with 138 additions and 124 deletions
--- a/MultimodalQnA/docker_compose/amd/gpu/rocm/README.md
+++ b/MultimodalQnA/docker_compose/amd/gpu/rocm/README.md
@@ -45,7 +45,7 @@ docker build --no-cache -t opea/lvm-llava:latest --build-arg https_proxy=$https_
 ### 3. Build retriever-multimodal-redis Image

 ```bash
-docker build --no-cache -t opea/retriever-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/redis/langchain/Dockerfile .
+docker build --no-cache -t opea/retriever:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/Dockerfile .
 ```

 ### 4. Build dataprep-multimodal-redis Image
@@ -86,7 +86,7 @@ Then run the command `docker images`, you will have the following 8 Docker Image
 1. `opea/dataprep-multimodal-redis:latest`
 2. `ghcr.io/huggingface/text-generation-inference:2.4.1-rocm`
 3. `opea/lvm:latest`
-4. `opea/retriever-multimodal-redis:latest`
+4. `opea/retriever:latest`
 5. `opea/embedding:latest`
 6. `opea/embedding-multimodal-bridgetower:latest`
 7. `opea/multimodalqna:latest`
--- a/MultimodalQnA/docker_compose/amd/gpu/rocm/compose.yaml
+++ b/MultimodalQnA/docker_compose/amd/gpu/rocm/compose.yaml
@@ -73,7 +73,7 @@ services:
      MULTIMODAL_EMBEDDING: true
    restart: unless-stopped
  retriever-redis:
-    image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest}
+    image: ${REGISTRY:-opea}/retriever:${TAG:-latest}
    container_name: retriever-redis
    depends_on:
      - redis-vector-db
@@ -87,7 +87,8 @@ services:
      REDIS_URL: ${REDIS_URL}
      INDEX_NAME: ${INDEX_NAME}
      BRIDGE_TOWER_EMBEDDING: ${BRIDGE_TOWER_EMBEDDING}
-      RETRIEVER_TYPE: "redis"
+      LOGFLAG: ${LOGFLAG}
+      RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_REDIS"
    restart: unless-stopped
  tgi-rocm:
    image: ghcr.io/huggingface/text-generation-inference:3.0.1-rocm
--- a/MultimodalQnA/docker_compose/intel/cpu/xeon/README.md
+++ b/MultimodalQnA/docker_compose/intel/cpu/xeon/README.md
@@ -124,7 +124,7 @@ docker build --no-cache -t opea/embedding:latest --build-arg https_proxy=$https_
 ### 2. Build retriever-multimodal-redis Image

 ```bash
-docker build --no-cache -t opea/retriever-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/redis/langchain/Dockerfile .
+docker build --no-cache -t opea/retriever:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/Dockerfile .
 ```

 ### 3. Build LVM Images
@@ -181,7 +181,7 @@ Then run the command `docker images`, you will have the following 11 Docker Imag
 1. `opea/dataprep-multimodal-redis:latest`
 2. `opea/lvm:latest`
 3. `opea/lvm-llava:latest`
-4. `opea/retriever-multimodal-redis:latest`
+4. `opea/retriever:latest`
 5. `opea/whisper:latest`
 6. `opea/redis-vector-db`
 7. `opea/embedding:latest`
--- a/MultimodalQnA/docker_compose/intel/cpu/xeon/compose.yaml
+++ b/MultimodalQnA/docker_compose/intel/cpu/xeon/compose.yaml
@@ -73,7 +73,7 @@ services:
      MULTIMODAL_EMBEDDING: true
    restart: unless-stopped
  retriever-redis:
-    image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest}
+    image: ${REGISTRY:-opea}/retriever:${TAG:-latest}
    container_name: retriever-redis
    depends_on:
      - redis-vector-db
@@ -87,7 +87,8 @@ services:
      REDIS_URL: ${REDIS_URL}
      INDEX_NAME: ${INDEX_NAME}
      BRIDGE_TOWER_EMBEDDING: ${BRIDGE_TOWER_EMBEDDING}
-      RETRIEVER_TYPE: "redis"
+      LOGFLAG: ${LOGFLAG}
+      RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_REDIS"
    restart: unless-stopped
  lvm-llava:
    image: ${REGISTRY:-opea}/lvm-llava:${TAG:-latest}
--- a/MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md
+++ b/MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md
@@ -75,7 +75,7 @@ docker build --no-cache -t opea/embedding:latest --build-arg https_proxy=$https_
 ### 2. Build retriever-multimodal-redis Image

 ```bash
-docker build --no-cache -t opea/retriever-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/redis/langchain/Dockerfile .
+docker build --no-cache -t opea/retriever:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/Dockerfile .
 ```

 ### 3. Build LVM Images
@@ -130,7 +130,7 @@ Then run the command `docker images`, you will have the following 11 Docker Imag
 1. `opea/dataprep-multimodal-redis:latest`
 2. `opea/lvm:latest`
 3. `ghcr.io/huggingface/tgi-gaudi:2.0.6`
-4. `opea/retriever-multimodal-redis:latest`
+4. `opea/retriever:latest`
 5. `opea/whisper:latest`
 6. `opea/redis-vector-db`
 7. `opea/embedding:latest`
--- a/MultimodalQnA/docker_compose/intel/hpu/gaudi/compose.yaml
+++ b/MultimodalQnA/docker_compose/intel/hpu/gaudi/compose.yaml
@@ -73,7 +73,7 @@ services:
      MULTIMODAL_EMBEDDING: true
    restart: unless-stopped
  retriever-redis:
-    image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest}
+    image: ${REGISTRY:-opea}/retriever:${TAG:-latest}
    container_name: retriever-redis
    depends_on:
      - redis-vector-db
@@ -87,7 +87,8 @@ services:
      REDIS_URL: ${REDIS_URL}
      INDEX_NAME: ${INDEX_NAME}
      BRIDGE_TOWER_EMBEDDING: ${BRIDGE_TOWER_EMBEDDING}
-      RETRIEVER_TYPE: "redis"
+      LOGFLAG: ${LOGFLAG}
+      RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_REDIS"
    restart: unless-stopped
  tgi-gaudi:
    image: ghcr.io/huggingface/tgi-gaudi:2.0.6
--- a/MultimodalQnA/docker_image_build/build.yaml
+++ b/MultimodalQnA/docker_image_build/build.yaml
@@ -29,12 +29,12 @@ services:
      dockerfile: comps/embeddings/src/Dockerfile
    extends: multimodalqna
    image: ${REGISTRY:-opea}/embedding:${TAG:-latest}
-  retriever-redis:
+  retriever:
    build:
      context: GenAIComps
-      dockerfile: comps/retrievers/redis/langchain/Dockerfile
+      dockerfile: comps/retrievers/src/Dockerfile
    extends: multimodalqna
-    image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest}
+    image: ${REGISTRY:-opea}/retriever:${TAG:-latest}
  lvm-llava:
    build:
      context: GenAIComps
--- a/MultimodalQnA/tests/test_compose_on_gaudi.sh
+++ b/MultimodalQnA/tests/test_compose_on_gaudi.sh
@@ -22,7 +22,7 @@ function build_docker_images() {
    cd $WORKPATH/docker_image_build
    git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
-    service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower embedding retriever-redis lvm dataprep-multimodal-redis whisper"
+    service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower embedding retriever lvm dataprep-multimodal-redis whisper"
    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log

    docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6
--- a/MultimodalQnA/tests/test_compose_on_rocm.sh
+++ b/MultimodalQnA/tests/test_compose_on_rocm.sh
@@ -23,7 +23,7 @@ function build_docker_images() {
    git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../

    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
-    service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower embedding retriever-redis lvm dataprep-multimodal-redis whisper"
+    service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower embedding retriever lvm dataprep-multimodal-redis whisper"
    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log

    docker images && sleep 1m
@@ -196,7 +196,7 @@ function validate_microservices() {
        "dataprep_get" \
        "dataprep-multimodal-redis"

-    sleep 1m
+    sleep 2m

    # multimodal retrieval microservice
    echo "Validating retriever-redis"
@@ -208,7 +208,7 @@ function validate_microservices() {
        "retriever-redis" \
        "{\"text\":\"test\",\"embedding\":${your_embedding}}"

-    sleep 3m
+    sleep 5m

    # llava server
    echo "Evaluating lvm-llava"
--- a/MultimodalQnA/tests/test_compose_on_xeon.sh
+++ b/MultimodalQnA/tests/test_compose_on_xeon.sh
@@ -22,7 +22,7 @@ function build_docker_images() {
    cd $WORKPATH/docker_image_build
    git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
    echo "Build all the images with --no-cache, check docker_image_build.log for details..."
-    service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower embedding retriever-redis lvm-llava lvm dataprep-multimodal-redis whisper"
+    service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower embedding retriever lvm-llava lvm dataprep-multimodal-redis whisper"
    docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log

    docker images && sleep 1m