jandres/GenAIExamples
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

add workflow test for microservice (#97)

Browse Source 
Signed-off-by: chensuyue <suyue.chen@intel.com>
Signed-off-by: Spycsh <sihan.chen@intel.com>
Signed-off-by: letonghan <letong.han@intel.com>
This commit is contained in:
chen, suyue
2024-05-29 20:50:35 +08:00
committed by GitHub
parent 0dbd26506d
commit cae346dca8
16 changed files with 557 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
.github/workflows/embeddings-comp-test.yml vendored
View File

@@ -1,50 +0,0 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
name: EmbeddingsMicroService-test
on:
# pull_request:
# branches: [main]
# types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
# paths:
# - .github/workflows/mega-test.yml
# - comps/mega/**
workflow_dispatch:
# If there is a new commit, the previous jobs will be canceled
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
EmbeddingsMicroService:
runs-on: aise-cluster
steps:
- name: Clean Up Working Directory
run: sudo rm -rf ${{github.workspace}}/*
- name: Checkout out Repo
uses: actions/checkout@v4
with:
ref: "refs/pull/${{ github.event.number }}/merge"
- name: Run UT
env:
HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
run: |
cd .github/workflows/scripts
bash test_ut.sh --test_name embeddings
- name: Run Workflow
env:
HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
run: |
cd tests/workflows
bash test_embeddings_comps.sh
- name: Publish pipeline artifact
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
path: ${{ github.workspace }}/.github/workflows/scripts/*.log

50
.github/workflows/llms-comp-test.yml vendored
View File

@@ -1,50 +0,0 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
name: LLMsMicroService-test
on:
# pull_request:
# branches: [main]
# types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
# paths:
# - .github/workflows/llms-comp-test.yml
# - comps/llms/**
workflow_dispatch:
# If there is a new commit, the previous jobs will be canceled
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
LLMsMicroService:
runs-on: aise-cluster
steps:
- name: Clean Up Working Directory
run: sudo rm -rf ${{github.workspace}}/*
- name: Checkout out Repo
uses: actions/checkout@v4
with:
ref: "refs/pull/${{ github.event.number }}/merge"
- name: Run UT
env:
HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
run: |
cd .github/workflows/scripts
bash test_ut.sh --test_name llms
- name: Run Workflow
env:
HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
run: |
cd tests/workflows
bash test_llms_comps.sh
- name: Publish pipeline artifact
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
path: ${{ github.workspace }}/.github/workflows/scripts/*.log

87
.github/workflows/microservice-test.yml vendored Normal file
View File

@@ -0,0 +1,87 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
name: MicroService-test
on:
pull_request_target:
branches: [main]
types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
paths:
- comps/**
- "!**.md"
- "!**.txt"
- .github/workflows/microservice.yml
# If there is a new commit, the previous jobs will be canceled
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
job1:
name: Get-test-matrix
runs-on: ubuntu-latest
outputs:
run_matrix: ${{ steps.get-test-matrix.outputs.run_matrix }}
steps:
- name: Checkout out Repo
uses: actions/checkout@v4
with:
ref: "refs/pull/${{ github.event.number }}/merge"
fetch-depth: 0
- name: Get test matrix
id: get-test-matrix
run: |
set -xe
changed_files=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} \
| grep 'comps/' | grep -vE '*.md|*.txt')
services=$(printf '%s\n' "${changed_files[@]}" | grep '/' | cut -d'/' -f2 | sort -u)
run_matrix="{\"include\":["
for service in ${services}; do
hardware="gaudi" # default hardware, set based on the changed files
run_matrix="${run_matrix}{\"service\":\"${service}\",\"hardware\":\"${hardware}\"},"
done
run_matrix=$run_matrix"]}"
echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT
Microservice-test:
needs: job1
strategy:
matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
runs-on: ${{ matrix.hardware }}
continue-on-error: true
steps:
- name: Clean Up Working Directory
run: sudo rm -rf ${{github.workspace}}/*
- name: Checkout out Repo
uses: actions/checkout@v4
with:
ref: "refs/pull/${{ github.event.number }}/merge"
- name: Run microservice test
env:
HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
service: ${{ matrix.service }}
hardware: ${{ matrix.hardware }}
run: |
cd tests
if [ -f test_${service}.sh ]; then timeout 10m bash test_${service}.sh; else echo "Test script not found, skip test!"; fi
- name: Clean up container
env:
service: ${{ matrix.service }}
hardware: ${{ matrix.hardware }}
if: cancelled() || failure()
run: |
cid=$(docker ps -aq --filter "name=test-comps-*")
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
echo y | docker system prune
- name: Publish pipeline artifact
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.service }}-${{ matrix.hardware }}
path: ${{ github.workspace }}/tests/*.log

22
comps/asr/README.md
View File

@@ -2,43 +2,45 @@
ASR (Audio-Speech-Recognition) microservice helps users convert speech to text. When building a talking bot with LLM, users will need to convert their audio inputs (What they talk, or Input audio from other sources) to text, so the LLM is able to tokenize the text and generate an answer. This microservice is built for that conversion stage.
# 🚀Start Microservice with Python
# 🚀1. Start Microservice with Python (Option 1)
To start the ASR microservice with Python, you need to first install python packages.
## Install Requirements
## 1.1 Install Requirements
```bash
pip install -r requirements.txt
```
## Start ASR Service with Python Script
## 1.2 Start ASR Service with Python Script
```bash
python asr.py
```
# 🚀Start Microservice with Docker
# 🚀2. Start Microservice with Docker (Option 2)
Alternatively, you can also start the ASR microservice with Docker.
## Build Docker Image
## 2.1 Build Docker Image
```bash
cd ../../
docker build -t opea/gen-ai-comps:asr --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/Dockerfile .
docker build -t opea/asr:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/Dockerfile .
```
## Run Docker with CLI
## 2.2 Run Docker with CLI
```bash
docker run -p 9099:9099 --network=host --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy opea/gen-ai-comps:asr
docker run -p 9099:9099 --network=host --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy opea/asr:latest
```
# Test
# 🚀3. Consume ASR Service
You can use the following `curl` command to test whether the service is up. Notice that the first request can be slow because it needs to download the models.
```bash
curl http://localhost:9099/v1/audio/transcriptions -H "Content-Type: application/json" -d '{"url": "https://github.com/intel/intel-extension-for-transformers/raw/main/intel_extension_for_transformers/neural_chat/assets/audio/sample_2.wav"}'
curl http://localhost:9099/v1/audio/transcriptions \
-H "Content-Type: application/json" \
-d '{"url": "https://github.com/intel/intel-extension-for-transformers/raw/main/intel_extension_for_transformers/neural_chat/assets/audio/sample_2.wav"}'
```

47
comps/dataprep/redis/README.md
View File

@@ -1,18 +1,18 @@
# Dataprep Microservice with Redis
# 🚀Start Microservice with Python
# 🚀1. Start Microservice with Python（Option 1）
## Install Requirements
## 1.1 Install Requirements
```bash
pip install -r requirements.txt
```
## Start Redis Stack Server
## 1.2 Start Redis Stack Server
Please refer to this [readme](../../../vectorstores/langchain/redis/README.md).
## Setup Environment Variables
## 1.3 Setup Environment Variables
```bash
export REDIS_URL="redis://${your_ip}:6379"
@@ -22,7 +22,7 @@ export LANGCHAIN_API_KEY=${your_langchain_api_key}
export LANGCHAIN_PROJECT="opea/gen-ai-comps:dataprep"
```
## Start Document Preparation Microservice for Redis with Python Script
## 1.4 Start Document Preparation Microservice for Redis with Python Script
Start document preparation microservice for Redis with below command.
@@ -30,38 +30,49 @@ Start document preparation microservice for Redis with below command.
python prepare_doc_redis.py
```
# 🚀Start Microservice with Docker
# 🚀2. Start Microservice with Docker (Option 2)
## Build Docker Image
## 2.1 Start Redis Stack Server
```bash
cd ../../../../
docker build -t opea/dataprep-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/redis/docker/Dockerfile .
```
Please refer to this [readme](../../../vectorstores/langchain/redis/README.md).
## Run Docker with CLI
## 2.2 Setup Environment Variables
```bash
export REDIS_URL="redis://${your_ip}:6379"
export INDEX_NAME=${your_index_name}
export LANGCHAIN_TRACING_V2=true
export LANGCHAIN_API_KEY=${your_langchain_api_key}
export LANGCHAIN_PROJECT="opea/gen-ai-comps:dataprep"
docker run -d --name="dataprep-redis-server" -p 6007:6007 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -e INDEX_NAME=$INDEX_NAME opea/dataprep-redis:latest
export LANGCHAIN_PROJECT="opea/dataprep"
```
## Run Docker with Docker Compose
## 2.3 Build Docker Image
```bash
cd ../../../../
docker build -t opea/dataprep-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/redis/docker/Dockerfile .
```
## 2.4 Run Docker with CLI (Option A)
```bash
docker run -d --name="dataprep-redis-server" -p 6007:6007 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -e INDEX_NAME=$INDEX_NAME -e TEI_ENDPOINT=$TEI_ENDPOINT opea/dataprep-redis:latest
```
## 2.5 Run with Docker Compose (Option B)
```bash
cd comps/dataprep/redis/docker
docker compose -f docker-compose-dataprep-redis.yaml up -d
```
# Invoke Microservice
# 🚀3. Consume Microservice
Once document preparation microservice for Redis is started, user can use below command to invoke the microservice to convert the document to embedding and save to the database.
```bash
curl -X POST -H "Content-Type: application/json" -d '{"path":"/path/to/document"}' http://localhost:6007/v1/dataprep
curl -X POST \
-H "Content-Type: application/json" \
-d '{"path":"/path/to/document"}' \
http://localhost:6007/v1/dataprep
```

61
comps/embeddings/README.md
View File

@@ -14,7 +14,7 @@ Key Features:
Users are albe to configure and build embedding-related services according to their actual needs.
# 🚀Start Microservice with Python
# 🚀1. Start Microservice with Python (Option 1)
Currently, we provide two ways to implement the embedding service:
@@ -24,19 +24,17 @@ Currently, we provide two ways to implement the embedding service:
For both of the implementations, you need to install requirements first.
## Install Requirements
## 1.1 Install Requirements
```bash
pip install -r langchain/requirements.txt
```
## Start Embedding Service with Local Model
## 1.2 Start Embedding Service
```bash
python local_embedding.py
```
You can select one of following ways to start the embedding service:
## Start Embedding Service with TEI
### Start Embedding Service with TEI
First, you need to start a TEI service.
@@ -67,31 +65,64 @@ export LANGCHAIN_PROJECT="opea/gen-ai-comps:embeddings"
python embedding_tei_gaudi.py
```
# 🚀Start Microservice with Docker
### Start Embedding Service with Local Model
## Build Docker Image
```bash
cd langchain
python local_embedding.py
```
# 🚀2. Start Microservice with Docker (Optional 2)
## 2.1 Start Embedding Service with TEI
First, you need to start a TEI service.
```bash
your_port=8090
model="BAAI/bge-large-en-v1.5"
revision="refs/pr/5"
docker run -p $your_port:80 -v ./data:/data --name tei_server -e http_proxy=$http_proxy -e https_proxy=$https_proxy --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 --model-id $model --revision $revision
```
Then you need to test your TEI service using the following commands:
```bash
curl localhost:$your_port/embed \
-X POST \
-d '{"inputs":"What is Deep Learning?"}' \
-H 'Content-Type: application/json'
```
Export the `TEI_EMBEDDING_ENDPOINT` for later usage:
```bash
export TEI_EMBEDDING_ENDPOINT="http://localhost:$yourport"
```
## 2.2 Build Docker Image
```bash
cd ../../
docker build -t opea/embedding-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/langchain/docker/Dockerfile .
```
## Run Docker with CLI
## 2.3 Run Docker with CLI
```bash
docker run -d --name="embedding-tei-server" -p 6000:6000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TEI_EMBEDDING_ENDPOINT=$TEI_EMBEDDING_ENDPOINT opea/embedding-tei:latest
```
## Run Docker with Docker Compose
## 2.4 Run Docker with Docker Compose
```bash
cd docker
docker compose -f docker_compose.yaml up -d
docker compose -f docker_compose_embedding.yaml up -d
```
# 🚀Consume Embedding Service
# 🚀3. Consume Embedding Service
## Check Service Status
## 3.1 Check Service Status
```bash
curl http://localhost:6000/v1/health_check\
@@ -99,7 +130,7 @@ curl http://localhost:6000/v1/health_check\
-H 'Content-Type: application/json'
```
## Consume Embedding Service
## 3.2 Consume Embedding Service
```bash
curl http://localhost:6000/v1/embeddings\

28
comps/guardrails/README.md
View File

@@ -20,30 +20,30 @@ Any content that is detected in the following categories is determined as unsafe
- Regulated or Controlled Substances
- Suicide & Self Harm
# 🚀Start Microservice with Python
# 🚀1. Start Microservice with Python (Option 1)
To start the Guardrails microservice, you need to install python packages first.
## Install Requirements
## 1.1 Install Requirements
```bash
pip install -r requirements.txt
```
## Start TGI Gaudi Service Manually
## 1.2 Start TGI Gaudi Service
```bash
export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
export LANGCHAIN_TRACING_V2=true
export LANGCHAIN_API_KEY=${your_langchain_api_key}
export LANGCHAIN_PROJECT="opea/gen-ai-comps:gaurdrails"
export LANGCHAIN_PROJECT="opea/gaurdrails"
volume=$PWD/data
model_id="meta-llama/LlamaGuard-7b"
docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1
docker run -p 8088:80 -v $volume:/data --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy ghcr.io/huggingface/tgi-gaudi:1.2.1 --model-id $model_id
```
## Verify the TGI Gaudi Service
## 1.3 Verify the TGI Gaudi Service
```bash
curl 127.0.0.1:8088/generate \
@@ -52,18 +52,18 @@ curl 127.0.0.1:8088/generate \
-H 'Content-Type: application/json'
```
## Start Guardrails Service with Python Script
## 1.4 Start Guardrails Service
```bash
export SAFETY_GUARD_ENDPOINT="http://${your_ip}:8088"
python langchain/guardrails_tgi_gaudi.py
```
# 🚀Start Microservice with Docker
# 🚀2. Start Microservice with Docker (Option 2)
If you start an Guardrails microservice with docker, the `docker_compose_guardrails.yaml` file will automatically start a TGI gaudi service with docker.
## Setup Environment Variables
## 2.1 Setup Environment Variables
In order to start TGI and LLM services, you need to setup the following environment variables first.
@@ -76,29 +76,29 @@ export LANGCHAIN_API_KEY=${your_langchain_api_key}
export LANGCHAIN_PROJECT="opea/gen-ai-comps:gaurdrails"
```
## Build Docker Image
## 2.2 Build Docker Image
```bash
cd ../../
docker build -t opea/guardrails-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/guardrails/langchain/docker/Dockerfile .
```
## Run Docker with CLI
## 2.3 Run Docker with CLI
```bash
docker run -d --name="guardrails-tgi-server" -p 9090:9090 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e SAFETY_GUARD_ENDPOINT=$SAFETY_GUARD_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/guardrails-tgi:latest
```
## Run Docker with Docker Compose
## 2.4 Run Docker with Docker Compose
```bash
cd langchain/docker
docker compose -f docker_compose_guardrails.yaml up -d
```
# 🚀Consume Guardrails Service
# 🚀3. Consume Guardrails Service
## Check Service Status
## 3.1 Check Service Status
```bash
curl http://localhost:9090/v1/health_check\
@@ -106,7 +106,7 @@ curl http://localhost:9090/v1/health_check\
-H 'Content-Type: application/json'
```
## Consume Guardrails Service
## 3.2 Consume Guardrails Service
```bash
curl http://localhost:9090/v1/guardrails\

35
comps/llms/README.md
View File

@@ -6,17 +6,17 @@ A prerequisite for using this microservice is that users must have a Text Genera
Overall, this microservice offers a streamlined way to integrate large language model inference into applications, requiring minimal setup from the user beyond initiating a TGI service and configuring the necessary environment variables. This allows for the seamless processing of queries and documents to generate intelligent, context-aware responses.
# 🚀Start Microservice with Python
# 🚀1. Start Microservice with Python (Option 1)
To start the LLM microservice, you need to install python packages first.
## Install Requirements
## 1.1 Install Requirements
```bash
pip install -r requirements.txt
```
## Start TGI Service Manually
## 1.2 Start TGI Service
```bash
export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
@@ -26,7 +26,7 @@ export LANGCHAIN_PROJECT="opea/gen-ai-comps:llms"
docker run -p 8008:80 -v ./data:/data --name tgi_service --shm-size 1g ghcr.io/huggingface/text-generation-inference:1.4 --model-id ${your_hf_llm_model}
```
## Verify the TGI Service
## 1.3 Verify the TGI Service
```bash
curl http://${your_ip}:8008/generate \
@@ -35,18 +35,18 @@ curl http://${your_ip}:8008/generate \
-H 'Content-Type: application/json'
```
## Start LLM Service with Python Script
## 1.4 Start LLM Service
```bash
export TGI_LLM_ENDPOINT="http://${your_ip}:8008"
python text-generation/tgi/llm.py
```
# 🚀Start Microservice with Docker
# 🚀2. Start Microservice with Docker (Option 2)
If you start an LLM microservice with docker, the `docker_compose_llm.yaml` file will automatically start a TGI service with docker.
## Setup Environment Variables
## 2.1 Setup Environment Variables
In order to start TGI and LLM services, you need to setup the following environment variables first.
@@ -56,32 +56,39 @@ export TGI_LLM_ENDPOINT="http://${your_ip}:8008"
export LLM_MODEL_ID=${your_hf_llm_model}
export LANGCHAIN_TRACING_V2=true
export LANGCHAIN_API_KEY=${your_langchain_api_key}
export LANGCHAIN_PROJECT="opea/gen-ai-comps:llms"
export LANGCHAIN_PROJECT="opea/llms"
```
## Build Docker Image
## 2.2 Build Docker Image
```bash
cd ../../
docker build -t opea/llm-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
```
## Run Docker with CLI
To start a docker container, you have two options:
- A. Run Docker with CLI
- B. Run Docker with Docker Compose
You can choose one as needed.
## 2.3 Run Docker with CLI (Option A)
```bash
docker run -d --name="llm-tgi-server" -p 9000:9000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TGI_LLM_ENDPOINT=$TGI_LLM_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/llm-tgi:latest
```
## Run Docker with Docker Compose
## 2.4 Run Docker with Docker Compose (Option B)
```bash
cd text-generation/tgi
docker compose -f docker_compose_llm.yaml up -d
```
# 🚀Consume LLM Service
# 🚀3. Consume LLM Service
## Check Service Status
## 3.1 Check Service Status
```bash
curl http://${your_ip}:9000/v1/health_check\
@@ -89,7 +96,7 @@ curl http://${your_ip}:9000/v1/health_check\
-H 'Content-Type: application/json'
```
## Consume LLM Service
## 3.2 Consume LLM Service
You can set the following model parameters according to your actual needs, such as `max_new_tokens`, `streaming`.

45
comps/reranks/README.md
View File

@@ -2,30 +2,30 @@
The Reranking Microservice, fueled by reranking models, stands as a straightforward yet immensely potent tool for semantic search. When provided with a query and a collection of documents, reranking swiftly indexes the documents based on their semantic relevance to the query, arranging them from most to least pertinent. This microservice significantly enhances overall accuracy. In a text retrieval system, either a dense embedding model or a sparse lexical search index is often employed to retrieve relevant text documents based on the input. However, a reranking model can further refine this process by rearranging potential candidates into a final, optimized order.
# 🚀Start Microservice with Python
# 🚀1. Start Microservice with Python (Option 1)
To start the Reranking microservice, you must first install the required python packages.
## Install Requirements
## 1.1 Install Requirements
```bash
pip install -r requirements.txt
```
## Start TEI Service Manually
## 1.2 Start TEI Service
```bash
export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
export LANGCHAIN_TRACING_V2=true
export LANGCHAIN_API_KEY=${your_langchain_api_key}
export LANGCHAIN_PROJECT="opea/gen-ai-comps:reranks"
export LANGCHAIN_PROJECT="opea/reranks"
model=BAAI/bge-reranker-large
revision=refs/pr/4
volume=$PWD/data
docker run -d -p 6060:80 -v $volume:/data -e http_proxy=$http_proxy -e https_proxy=$https_proxy --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 --model-id $model --revision $revision
```
## Verify the TEI Service
## 1.3 Verify the TEI Service
```bash
curl 127.0.0.1:6060/rerank \
@@ -34,42 +34,57 @@ curl 127.0.0.1:6060/rerank \
-H 'Content-Type: application/json'
```
## Start Reranking Service with Python Script
## 1.4 Start Reranking Service with Python Script
```bash
export TEI_RERANKING_ENDPOINT="http://${your_ip}:6060"
python reranking_tei_xeon.py
```
# 🚀Start Microservice with Docker
# 🚀2. Start Microservice with Docker (Option 2)
If you start an Reranking microservice with docker, the `docker_compose_reranking.yaml` file will automatically start a TEI service with docker.
## Setup Environment Variables
## 2.1 Setup Environment Variables
## Build Docker Image
```bash
export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
export LANGCHAIN_TRACING_V2=true
export LANGCHAIN_API_KEY=${your_langchain_api_key}
export LANGCHAIN_PROJECT="opea/reranks"
export TEI_RERANKING_ENDPOINT="http://${your_ip}:8808"
```
## 2.2 Build Docker Image
```bash
cd ../../
docker build -t opea/reranking-tei:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/langchain/docker/Dockerfile .
```
## Run Docker with CLI
To start a docker container, you have two options:
- A. Run Docker with CLI
- B. Run Docker with Docker Compose
You can choose one as needed.
## 2.3 Run Docker with CLI (Option A)
```bash
docker run -d --name="reranking-tei-server" -p 8000:8000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TEI_RERANKING_ENDPOINT=$TEI_RERANKING_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/reranking-tei:latest
docker run -d --name="reranking-tei-server" -p 8000:8000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TEI_RERANKING_ENDPOINT=$TEI_RERANKING_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN -e LANGCHAIN_API_KEY=$LANGCHAIN_API_KEY opea/reranking-tei:latest
```
## Run Docker with Docker Compose
## 2.4 Run Docker with Docker Compose (Option B)
```bash
cd langchain/docker
docker compose -f docker_compose_reranking.yaml up -d
```
# 🚀Consume Reranking Service
# 🚀3. Consume Reranking Service
## Check Service Status
## 3.1 Check Service Status
```bash
curl http://localhost:8000/v1/health_check \
@@ -77,7 +92,7 @@ curl http://localhost:8000/v1/health_check \
-H 'Content-Type: application/json'
```
## Consume Reranking Service
## 3.2 Consume Reranking Service
```bash
curl http://localhost:8000/v1/reranking \

76
comps/retrievers/README.md
View File

@@ -6,17 +6,38 @@ The service primarily utilizes similarity measures in vector space to rapidly re
Overall, this microservice provides robust backend support for applications requiring efficient similarity searches, playing a vital role in scenarios such as recommendation systems, information retrieval, or any other context where precise measurement of document similarity is crucial.
# 🚀Start Microservice with Python
# 🚀1. Start Microservice with Python (Option 1)
To start the retriever microservice, you must first install the required python packages.
## Install Requirements
## 1.1 Install Requirements
```bash
pip install -r requirements.txt
```
## Setup VectorDB Service
## 1.2 Start TEI Service
```bash
export LANGCHAIN_TRACING_V2=true
export LANGCHAIN_API_KEY=${your_langchain_api_key}
export LANGCHAIN_PROJECT="opea/retriever"
model=BAAI/bge-base-en-v1.5
revision=refs/pr/4
volume=$PWD/data
docker run -d -p 6060:80 -v $volume:/data -e http_proxy=$http_proxy -e https_proxy=$https_proxy --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 --model-id $model --revision $revision
```
## 1.3 Verify the TEI Service
```bash
curl 127.0.0.1:6060/rerank \
-X POST \
-d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \
-H 'Content-Type: application/json'
```
## 1.4 Setup VectorDB Service
You need to setup your own VectorDB service (Redis in this example), and ingest your knowledge documents into the vector database.
@@ -27,48 +48,57 @@ Remember to ingest data into it manually.
docker run -d --name="redis-vector-db" -p 6379:6379 -p 8001:8001 redis/redis-stack:7.2.0-v9
```
## Setup Environment Variables
```bash
export REDIS_URL="redis://${your_ip}:6379"
export INDEX_NAME=${your_index_name}
export TEI_EMBEDDING_ENDPOINT=${your_embedding_endpoint}
export LANGCHAIN_TRACING_V2=true
export LANGCHAIN_API_KEY=${your_langchain_api_key}
export LANGCHAIN_PROJECT="opea/gen-ai-comps:retrievers"
```
## Start Retriever Service with Local Model
## 1.5 Start Retriever Service
```bash
export TEI_EMBEDDING_ENDPOINT="http://${your_ip}:6060"
python langchain/retriever_redis.py
```
# 🚀Start Microservice with Docker
# 🚀2. Start Microservice with Docker (Option 2)
## Build Docker Image
## 2.1 Setup Environment Variables
```bash
export RETRIEVE_MODEL_ID="BAAI/bge-base-en-v1.5"
export REDIS_URL="redis://${your_ip}:6379"
export INDEX_NAME=${your_index_name}
export TEI_EMBEDDING_ENDPOINT="http://${your_ip}:6060"
export LANGCHAIN_TRACING_V2=true
export LANGCHAIN_API_KEY=${your_langchain_api_key}
export LANGCHAIN_PROJECT="opea/retrievers"
```
## 2.2 Build Docker Image
```bash
cd ../../
docker build -t opea/retriever-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/langchain/docker/Dockerfile .
```
## Run Docker with CLI
To start a docker container, you have two options:
- A. Run Docker with CLI
- B. Run Docker with Docker Compose
You can choose one as needed.
## 2.3 Run Docker with CLI (Option A)
```bash
docker run -d --name="retriever-redis-server" -p 7000:7000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -e INDEX_NAME=$INDEX_NAME opea/retriever-redis:latest
docker run -d --name="retriever-redis-server" -p 7000:7000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -e INDEX_NAME=$INDEX_NAME -e TEI_EMBEDDING_ENDPOINT=$TEI_EMBEDDING_ENDPOINT opea/retriever-redis:latest
```
## Run Docker with Docker Compose
## 2.4 Run Docker with Docker Compose (Option B)
```bash
cd langchain/docker
docker compose -f docker_compose_retriever.yaml up -d
```
# 🚀Consume Retriever Service
# 🚀3. Consume Retriever Service
## Check Service Status
## 3.1 Check Service Status
```bash
curl http://localhost:7000/v1/health_check \
@@ -76,7 +106,7 @@ curl http://localhost:7000/v1/health_check \
-H 'Content-Type: application/json'
```
## Consume Embedding Service
## 3.2 Consume Embedding Service
To consume the Retriever Microservice, you need to generate a mock embedding vector of length 768 in Python script:

10
comps/retrievers/langchain/docker/docker_compose_retriever.yaml
View File

@@ -15,6 +15,15 @@
version: "3.8"
services:
tei_xeon_service:
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.2
container_name: tei-xeon-server
ports:
- "6060:80"
volumes:
- "./data:/data"
shm_size: 1g
command: --model-id ${RETRIEVE_MODEL_ID}
retriever:
image: opea/retriever-redis:latest
container_name: retriever-redis-server
@@ -26,7 +35,6 @@ services:
https_proxy: ${https_proxy}
REDIS_URL: ${REDIS_URL}
INDEX_NAME: ${INDEX_NAME}
TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
restart: unless-stopped

22
comps/tts/README.md
View File

@@ -2,43 +2,45 @@
TTS (Text-To-Speech) microservice helps users convert text to speech. When building a talking bot with LLM, users might need an LLM generated answer in audio format. This microservice is built for that conversion stage.
# 🚀Start Microservice with Python
# 🚀1. Start Microservice with Python (Option 1)
To start the TTS microservice, you need to first install python packages.
## Install Requirements
## 1.1 Install Requirements
```bash
pip install -r requirements.txt
```
## Start TTS Service with Python Script
## 1.2 Start TTS Service with Python Script
```bash
python tts.py
```
# 🚀Start Microservice with Docker
# 🚀2. Start Microservice with Docker (Option 2)
Alternatively, you can start the ASR microservice with Docker.
## Build Docker Image
## 2.1 Build Docker Image
```bash
cd ../../
docker build -t opea/gen-ai-comps:tts --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/tts/Dockerfile .
docker build -t opea/tts:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/tts/Dockerfile .
```
## Run Docker with CLI
## 2.2 Run Docker with CLI
```bash
docker run -p 9999:9999 --network=host --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy opea/gen-ai-comps:tts
docker run -p 9999:9999 --network=host --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy opea/tts:latest
```
# Test
# 🚀3. Consume TTS Service
You can use the following `curl` command to test whether the service is up. Notice that the first request can be slow because it needs to download the models.
```bash
curl http://localhost:9999/v1/audio/speech -H "Content-Type: application/json" -d '{"text":"Hello there."}'
curl http://localhost:9999/v1/audio/speech \
-H "Content-Type: application/json" \
-d '{"text":"Hello there."}'
```

55
tests/test_embeddings.sh Normal file
View File

@@ -0,0 +1,55 @@
#!/bin/bash
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
set -xe
WORKPATH=$(dirname "$PWD")
ip_address=$(hostname -I | awk '{print $1}')
function build_docker_images() {
cd $WORKPATH
echo $(pwd)
docker build --no-cache -t opea/embedding-tei:comps -f comps/embeddings/langchain/docker/Dockerfile .
}
function start_service() {
tei_endpoint=5001
model="BAAI/bge-large-en-v1.5"
revision="refs/pr/5"
unset http_proxy
docker run -d --name="test-comps-embedding-tei-endpoint" -p $tei_endpoint:80 -v ./data:/data --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 --model-id $model --revision $revision
export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:${tei_endpoint}"
tei_service_port=5002
docker run -d --name="test-comps-embedding-tei-server" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p ${tei_service_port}:6000 --ipc=host -e TEI_EMBEDDING_ENDPOINT=$TEI_EMBEDDING_ENDPOINT opea/embedding-tei:comps
sleep 3m
}
function validate_microservice() {
tei_service_port=5002
http_proxy="" curl http://${ip_address}:$tei_service_port/v1/embeddings \
-X POST \
-d '{"text":"What is Deep Learning?"}' \
-H 'Content-Type: application/json'
}
function stop_docker() {
cid=$(docker ps -aq --filter "name=test-comps-embedding-*")
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
}
function main() {
stop_docker
build_docker_images
start_service
validate_microservice
stop_docker
echo y | docker system prune
}
main

68
tests/test_llms.sh Normal file
View File

@@ -0,0 +1,68 @@
#!/bin/bash
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
set -xe
WORKPATH=$(dirname "$PWD")
ip_address=$(hostname -I | awk '{print $1}')
function build_docker_images() {
cd $WORKPATH
docker build --no-cache -t opea/llm-tgi:comps -f comps/llms/text-generation/tgi/Dockerfile .
}
function start_service() {
tgi_endpoint_port=5004
export your_hf_llm_model="Intel/neural-chat-7b-v3-3"
# Remember to set HUGGINGFACEHUB_API_TOKEN before invoking this test!
export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
docker run -d --name="test-comps-llm-tgi-endpoint" -p $tgi_endpoint_port:80 -v ./data:/data --shm-size 1g ghcr.io/huggingface/text-generation-inference:1.4 --model-id ${your_hf_llm_model}
export TGI_LLM_ENDPOINT="http://${ip_address}:${tgi_endpoint_port}"
tei_service_port=5005
unset http_proxy
docker run -d --name="test-comps-llm-tgi-server" -p ${tei_service_port}:9000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TGI_LLM_ENDPOINT=$TGI_LLM_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/llm-tgi:comps
# check whether tgi is fully ready
n=0
until [[ "$n" -ge 100 ]] || [[ $ready == true ]]; do
docker logs test-comps-llm-tgi-endpoint > test-comps-llm-tgi-endpoint.log
n=$((n+1))
if grep -q Connected test-comps-llm-tgi-endpoint.log; then
break
fi
sleep 5s
done
# rm -f test-comps-llm-tgi-endpoint.log
sleep 5s
}
function validate_microservice() {
tei_service_port=5005
http_proxy="" curl http://${ip_address}:${tei_service_port}/v1/chat/completions \
-X POST \
-d '{"query":"What is Deep Learning?"}' \
-H 'Content-Type: application/json'
}
function stop_docker() {
cid=$(docker ps -aq --filter "name=test-comps-llm-*")
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
}
function main() {
stop_docker
build_docker_images
start_service
validate_microservice
stop_docker
echo y | docker system prune
}
main

57
tests/test_reranks.sh Normal file
View File

@@ -0,0 +1,57 @@
#!/bin/bash
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
set -xe
WORKPATH=$(dirname "$PWD")
ip_address=$(hostname -I | awk '{print $1}')
function build_docker_images() {
cd $WORKPATH
docker build --no-cache -t opea/reranking-tei:comps -f comps/reranks/langchain/docker/Dockerfile .
}
function start_service() {
tei_endpoint=5006
# Remember to set HUGGINGFACEHUB_API_TOKEN before invoking this test!
export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
model=BAAI/bge-reranker-large
revision=refs/pr/4
volume=$PWD/data
docker run -d --name="test-comps-reranking-tei-endpoint" -p $tei_endpoint:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 --model-id $model --revision $revision
export TEI_RERANKING_ENDPOINT="http://${ip_address}:${tei_endpoint}"
tei_service_port=5007
unset http_proxy
docker run -d --name="test-comps-reranking-tei-server" -p ${tei_service_port}:8000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TEI_RERANKING_ENDPOINT=$TEI_RERANKING_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/reranking-tei:comps
sleep 1m
}
function validate_microservice() {
tei_service_port=5007
http_proxy="" curl http://${ip_address}:${tei_service_port}/v1/reranking\
-X POST \
-d '{"initial_query":"What is Deep Learning?", "retrieved_docs": [{"text":"Deep Learning is not..."}, {"text":"Deep learning is..."}]}' \
-H 'Content-Type: application/json'
}
function stop_docker() {
cid=$(docker ps -aq --filter "name=test-comps-rerank*")
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
}
function main() {
stop_docker
build_docker_images
start_service
validate_microservice
stop_docker
echo y | docker system prune
}
main

64
tests/test_retrievers.sh Normal file
View File

@@ -0,0 +1,64 @@
#!/bin/bash
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
set -xe
WORKPATH=$(dirname "$PWD")
ip_address=$(hostname -I | awk '{print $1}')
function build_docker_images() {
cd $WORKPATH
docker build --no-cache -t opea/retriever-redis:comps -f comps/retrievers/langchain/docker/Dockerfile .
}
function start_service() {
# tei endpoint
tei_endpoint=5008
model="BAAI/bge-large-en-v1.5"
revision="refs/pr/5"
docker run -d --name="test-comps-retriever-tei-endpoint" -p $tei_endpoint:80 -v ./data:/data --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 --model-id $model --revision $revision
export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:${tei_endpoint}"
# redis retriever
export REDIS_URL="redis://${ip_address}:6379"
export INDEX_NAME="rag-redis"
retriever_port=5009
unset http_proxy
docker run -d --name="test-comps-retriever-redis-server" -p ${retriever_port}:7000 --ipc=host -e TEI_EMBEDDING_ENDPOINT=$TEI_EMBEDDING_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -e INDEX_NAME=$INDEX_NAME opea/retriever-redis:comps
sleep 3m
}
function validate_microservice() {
retriever_port=5009
export PATH="${HOME}/miniforge3/bin:$PATH"
source activate
sleep 1s
test_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
sleep 1s
http_proxy='' curl http://${ip_address}:$retriever_port/v1/retrieval \
-X POST \
-d '{"text":"test","embedding":${test_embedding}}' \
-H 'Content-Type: application/json'
}
function stop_docker() {
cid=$(docker ps -aq --filter "name=test-comps-retrievers*")
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
}
function main() {
stop_docker
build_docker_images
start_service
validate_microservice
stop_docker
echo y | docker system prune
}
main