Refactor AudioQnA/MultiModalQnA/AvatarChatbot (#1310)

Signed-off-by: chensuyue <suyue.chen@intel.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: chensuyue <suyue.chen@intel.com>
2024-12-31 12:47:30 +08:00
parent 250ffb8b66
commit cc1d97f816
43 changed files with 482 additions and 1102 deletions
--- a/MultimodalQnA/docker_compose/amd/gpu/rocm/compose.yaml
+++ b/MultimodalQnA/docker_compose/amd/gpu/rocm/compose.yaml
@@ -13,16 +13,6 @@ services:
      http_proxy: ${http_proxy}
      https_proxy: ${https_proxy}
    restart: unless-stopped
-  asr:
-    image: ${REGISTRY:-opea}/asr:${TAG:-latest}
-    container_name: asr-service
-    ports:
-      - "${ASR_SERVICE_PORT}:9099"
-    ipc: host
-    environment:
-      ASR_ENDPOINT: ${ASR_ENDPOINT}
-      ASR_SERVICE_PORT: ${ASR_SERVICE_PORT}
-      ASR_SERVICE_ENDPOINT: ${ASR_SERVICE_ENDPOINT}
  redis-vector-db:
    image: redis/redis-stack:7.2.0-v9
    container_name: redis-vector-db
@@ -153,8 +143,8 @@ services:
      MM_EMBEDDING_PORT_MICROSERVICE: ${MM_EMBEDDING_PORT_MICROSERVICE}
      MM_RETRIEVER_SERVICE_HOST_IP: ${MM_RETRIEVER_SERVICE_HOST_IP}
      LVM_SERVICE_HOST_IP: ${LVM_SERVICE_HOST_IP}
-      ASR_SERVICE_PORT: ${ASR_SERVICE_PORT}
-      ASR_SERVICE_ENDPOINT: ${ASR_SERVICE_ENDPOINT}
+      WHISPER_SERVER_PORT: ${WHISPER_SERVER_PORT}
+      WHISPER_SERVER_ENDPOINT: ${WHISPER_SERVER_ENDPOINT}
    ipc: host
    restart: always
  multimodalqna-ui:
--- a/MultimodalQnA/docker_compose/intel/cpu/xeon/README.md
+++ b/MultimodalQnA/docker_compose/intel/cpu/xeon/README.md
@@ -78,9 +78,8 @@ export https_proxy=${your_http_proxy}
 export EMBEDDER_PORT=6006
 export MMEI_EMBEDDING_ENDPOINT="http://${host_ip}:$EMBEDDER_PORT/v1/encode"
 export MM_EMBEDDING_PORT_MICROSERVICE=6000
-export ASR_ENDPOINT=http://$host_ip:7066
-export ASR_SERVICE_PORT=3001
-export ASR_SERVICE_ENDPOINT="http://${host_ip}:${ASR_SERVICE_PORT}/v1/audio/transcriptions"
+export WHISPER_SERVER_PORT=7066
+export WHISPER_SERVER_ENDPOINT="http://${host_ip}:${WHISPER_SERVER_PORT}/v1/asr"
 export REDIS_URL="redis://${host_ip}:6379"
 export REDIS_HOST=${host_ip}
 export INDEX_NAME="mm-rag-redis"
@@ -153,13 +152,7 @@ docker build --no-cache -t opea/dataprep-multimodal-redis:latest --build-arg htt
 Build whisper server image

 ```bash
-docker build --no-cache -t opea/whisper:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/whisper/dependency/Dockerfile .
-```
-
-Build asr image
-
-```bash
-docker build --no-cache -t opea/asr:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/whisper/Dockerfile .
+docker build --no-cache -t opea/whisper:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/src/integrations/dependency/whisper/Dockerfile .
 ```

 ### 6. Build MegaService Docker Image
@@ -190,12 +183,11 @@ Then run the command `docker images`, you will have the following 11 Docker Imag
 3. `opea/lvm-llava:latest`
 4. `opea/retriever-multimodal-redis:latest`
 5. `opea/whisper:latest`
-6. `opea/asr:latest`
-7. `opea/redis-vector-db`
-8. `opea/embedding-multimodal:latest`
-9. `opea/embedding-multimodal-bridgetower:latest`
-10. `opea/multimodalqna:latest`
-11. `opea/multimodalqna-ui:latest`
+6. `opea/redis-vector-db`
+7. `opea/embedding-multimodal:latest`
+8. `opea/embedding-multimodal-bridgetower:latest`
+9. `opea/multimodalqna:latest`
+10. `opea/multimodalqna-ui:latest`

 ## 🚀 Start Microservices

@@ -264,10 +256,10 @@ curl http://${host_ip}:7000/v1/multimodal_retrieval \
 4. asr

 ```bash
-curl ${ASR_SERVICE_ENDPOINT} \
+curl ${WHISPER_SERVER_ENDPOINT} \
    -X POST \
    -H "Content-Type: application/json" \
-    -d '{"byte_str" : "UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA"}'
+    -d '{"audio" : "UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA"}'
 ```

 5. lvm-llava
--- a/MultimodalQnA/docker_compose/intel/cpu/xeon/compose.yaml
+++ b/MultimodalQnA/docker_compose/intel/cpu/xeon/compose.yaml
@@ -13,16 +13,6 @@ services:
      http_proxy: ${http_proxy}
      https_proxy: ${https_proxy}
    restart: unless-stopped
-  asr:
-    image: ${REGISTRY:-opea}/asr:${TAG:-latest}
-    container_name: asr-service
-    ports:
-      - "${ASR_SERVICE_PORT}:9099"
-    ipc: host
-    environment:
-      ASR_ENDPOINT: ${ASR_ENDPOINT}
-      ASR_SERVICE_PORT: ${ASR_SERVICE_PORT}
-      ASR_SERVICE_ENDPOINT: ${ASR_SERVICE_ENDPOINT}
  redis-vector-db:
    image: redis/redis-stack:7.2.0-v9
    container_name: redis-vector-db
@@ -124,7 +114,6 @@ services:
      - embedding-multimodal
      - retriever-redis
      - lvm-llava-svc
-      - asr
    ports:
      - "8888:8888"
    environment:
@@ -136,8 +125,8 @@ services:
      MM_EMBEDDING_PORT_MICROSERVICE: ${MM_EMBEDDING_PORT_MICROSERVICE}
      MM_RETRIEVER_SERVICE_HOST_IP: ${MM_RETRIEVER_SERVICE_HOST_IP}
      LVM_SERVICE_HOST_IP: ${LVM_SERVICE_HOST_IP}
-      ASR_SERVICE_PORT: ${ASR_SERVICE_PORT}
-      ASR_SERVICE_ENDPOINT: ${ASR_SERVICE_ENDPOINT}
+      WHISPER_SERVER_PORT: ${WHISPER_SERVER_PORT}
+      WHISPER_SERVER_ENDPOINT: ${WHISPER_SERVER_ENDPOINT}
    ipc: host
    restart: always
  multimodalqna-ui:
--- a/MultimodalQnA/docker_compose/intel/cpu/xeon/set_env.sh
+++ b/MultimodalQnA/docker_compose/intel/cpu/xeon/set_env.sh
@@ -12,9 +12,8 @@ export https_proxy=${your_http_proxy}
 export EMBEDDER_PORT=6006
 export MMEI_EMBEDDING_ENDPOINT="http://${host_ip}:$EMBEDDER_PORT/v1/encode"
 export MM_EMBEDDING_PORT_MICROSERVICE=6000
-export ASR_ENDPOINT=http://$host_ip:7066
-export ASR_SERVICE_PORT=3001
-export ASR_SERVICE_ENDPOINT="http://${host_ip}:${ASR_SERVICE_PORT}/v1/audio/transcriptions"
+export WHISPER_SERVER_PORT=7066
+export WHISPER_SERVER_ENDPOINT="http://${host_ip}:${WHISPER_SERVER_PORT}/v1/asr"
 export REDIS_URL="redis://${host_ip}:6379"
 export REDIS_HOST=${host_ip}
 export INDEX_NAME="mm-rag-redis"
--- a/MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md
+++ b/MultimodalQnA/docker_compose/intel/hpu/gaudi/README.md
@@ -38,9 +38,8 @@ export LVM_MODEL_ID="llava-hf/llava-v1.6-vicuna-13b-hf"
 export WHISPER_MODEL="base"
 export MM_EMBEDDING_SERVICE_HOST_IP=${host_ip}
 export MM_RETRIEVER_SERVICE_HOST_IP=${host_ip}
-export ASR_ENDPOINT=http://$host_ip:7066
-export ASR_SERVICE_PORT=3001
-export ASR_SERVICE_ENDPOINT="http://${host_ip}:${ASR_SERVICE_PORT}/v1/audio/transcriptions"
+export WHISPER_SERVER_PORT=7066
+export WHISPER_SERVER_ENDPOINT="http://${host_ip}:${WHISPER_SERVER_PORT}/v1/asr"v1/audio/transcriptions"
 export LVM_SERVICE_HOST_IP=${host_ip}
 export MEGA_SERVICE_HOST_IP=${host_ip}
 export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/multimodalqna"
@@ -104,13 +103,7 @@ docker build --no-cache -t opea/dataprep-multimodal-redis:latest --build-arg htt
 Build whisper server image

 ```bash
-docker build --no-cache -t opea/whisper:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/whisper/dependency/Dockerfile .
-```
-
-Build asr image
-
-```bash
-docker build --no-cache -t opea/asr:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/whisper/Dockerfile .
+docker build --no-cache -t opea/whisper:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/src/integrations/dependency/whisper/Dockerfile .
 ```

 ### 6. Build MegaService Docker Image
@@ -139,12 +132,11 @@ Then run the command `docker images`, you will have the following 11 Docker Imag
 3. `ghcr.io/huggingface/tgi-gaudi:2.0.6`
 4. `opea/retriever-multimodal-redis:latest`
 5. `opea/whisper:latest`
-6. `opea/asr:latest`
-7. `opea/redis-vector-db`
-8. `opea/embedding-multimodal:latest`
-9. `opea/embedding-multimodal-bridgetower:latest`
-10. `opea/multimodalqna:latest`
-11. `opea/multimodalqna-ui:latest`
+6. `opea/redis-vector-db`
+7. `opea/embedding-multimodal:latest`
+8. `opea/embedding-multimodal-bridgetower:latest`
+9. `opea/multimodalqna:latest`
+10. `opea/multimodalqna-ui:latest`

 ## 🚀 Start Microservices

@@ -213,10 +205,10 @@ curl http://${host_ip}:7000/v1/multimodal_retrieval \
 4. asr

 ```bash
-curl ${ASR_SERVICE_ENDPOINT} \
+curl ${WHISPER_SERVER_ENDPOINT} \
    -X POST \
    -H "Content-Type: application/json" \
-    -d '{"byte_str" : "UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA"}'
+    -d '{"audio" : "UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA"}'
 ```

 5. TGI LLaVA Gaudi Server
--- a/MultimodalQnA/docker_compose/intel/hpu/gaudi/compose.yaml
+++ b/MultimodalQnA/docker_compose/intel/hpu/gaudi/compose.yaml
@@ -19,16 +19,6 @@ services:
      http_proxy: ${http_proxy}
      https_proxy: ${https_proxy}
    restart: unless-stopped
-  asr:
-    image: ${REGISTRY:-opea}/asr:${TAG:-latest}
-    container_name: asr-service
-    ports:
-      - "${ASR_SERVICE_PORT}:9099"
-    ipc: host
-    environment:
-      ASR_ENDPOINT: ${ASR_ENDPOINT}
-      ASR_SERVICE_PORT: ${ASR_SERVICE_PORT}
-      ASR_SERVICE_ENDPOINT: ${ASR_SERVICE_ENDPOINT}
  dataprep-multimodal-redis:
    image: ${REGISTRY:-opea}/dataprep-multimodal-redis:${TAG:-latest}
    container_name: dataprep-multimodal-redis
@@ -141,7 +131,6 @@ services:
      - embedding-multimodal
      - retriever-redis
      - lvm-tgi
-      - asr
    ports:
      - "8888:8888"
    environment:
@@ -153,8 +142,8 @@ services:
      MM_EMBEDDING_PORT_MICROSERVICE: ${MM_EMBEDDING_PORT_MICROSERVICE}
      MM_RETRIEVER_SERVICE_HOST_IP: ${MM_RETRIEVER_SERVICE_HOST_IP}
      LVM_SERVICE_HOST_IP: ${LVM_SERVICE_HOST_IP}
-      ASR_SERVICE_PORT: ${ASR_SERVICE_PORT}
-      ASR_SERVICE_ENDPOINT: ${ASR_SERVICE_ENDPOINT}
+      WHISPER_SERVER_PORT: ${WHISPER_SERVER_PORT}
+      WHISPER_SERVER_ENDPOINT: ${WHISPER_SERVER_ENDPOINT}
    ipc: host
    restart: always
  multimodalqna-ui:
--- a/MultimodalQnA/docker_compose/intel/hpu/gaudi/set_env.sh
+++ b/MultimodalQnA/docker_compose/intel/hpu/gaudi/set_env.sh
@@ -12,9 +12,8 @@ export https_proxy=${your_http_proxy}
 export EMBEDDER_PORT=6006
 export MMEI_EMBEDDING_ENDPOINT="http://${host_ip}:$EMBEDDER_PORT/v1/encode"
 export MM_EMBEDDING_PORT_MICROSERVICE=6000
-export ASR_ENDPOINT=http://$host_ip:7066
-export ASR_SERVICE_PORT=3001
-export ASR_SERVICE_ENDPOINT="http://${host_ip}:${ASR_SERVICE_PORT}/v1/audio/transcriptions"
+export WHISPER_SERVER_PORT=7066
+export WHISPER_SERVER_ENDPOINT="http://${host_ip}:${WHISPER_SERVER_PORT}/v1/asr"
 export REDIS_URL="redis://${host_ip}:6379"
 export REDIS_HOST=${host_ip}
 export INDEX_NAME="mm-rag-redis"