jandres/GenAIExamples
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Update DBQnA tgi docker image to latest tgi 2.4.0 (#1593)

Browse Source
This commit is contained in:
Ying Hu
2025-03-03 16:17:19 +08:00
committed by GitHub
parent 7b3a125bdf
commit e8b07c28ec
4 changed files with 55 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
DBQnA/docker_compose/intel/cpu/xeon/README.md
View File

@@ -38,25 +38,29 @@ We set default model as "mistralai/Mistral-7B-Instruct-v0.3", change "LLM_MODEL_
If use gated models, you also need to provide [huggingface token](https://huggingface.co/docs/hub/security-tokens) to "HUGGINGFACEHUB_API_TOKEN" environment variable.
```bash
export HUGGINGFACEHUB_API_TOKEN="xxx"
```
### 2.1 Setup Environment Variables
Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below.
```bash
# your_ip should be your external IP address, do not use localhost.
export your_ip=$(hostname -I | awk '{print $1}')
# host_ip should be your external IP address, do not use localhost.
export host_ip=$(hostname -I | awk '{print $1}')
# Example: no_proxy="localhost,127.0.0.1,192.168.1.1"
export no_proxy=${your_no_proxy},${your_ip}
export no_proxy=${no_proxy},${host_ip}
# If you are in a proxy environment, also set the proxy-related environment variables:
export http_proxy=${your_http_proxy}
export https_proxy=${your_http_proxy}
export http_proxy=${http_proxy}
export https_proxy=${https_proxy}
# Set other required variables
export TGI_PORT=8008
export TGI_LLM_ENDPOINT=http://${your_ip}:${TGI_PORT}
export TGI_LLM_ENDPOINT=http://${host_ip}:${TGI_PORT}
export HF_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
export LLM_MODEL_ID="mistralai/Mistral-7B-Instruct-v0.3"
export POSTGRES_USER=postgres
@@ -65,7 +69,14 @@ export POSTGRES_DB=chinook
export text2sql_port=9090
```
Note: Please replace with `your_ip` with your external IP address, do not use localhost.
or
edit the file set_env.sh to set those environment variables,
```bash
source set_env.sh
```
Note: Please replace with `host_ip` with your external IP address, do not use localhost.
### 2.2 Start Microservice Docker Containers
@@ -120,7 +131,7 @@ docker run -d --name="test-dbqna-react-ui-server" --ipc=host -p 5174:80 -e no_pr
```bash
curl http://${your_ip}:$TGI_PORT/generate \
curl http://${host_ip}:$TGI_PORT/generate \
-X POST \
-d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \
-H 'Content-Type: application/json'
@@ -133,17 +144,17 @@ Once Text-to-SQL microservice is started, user can use below command
#### 3.2.1 Test the Database connection
```bash
curl --location http://${your_ip}:9090/v1/postgres/health \
curl --location http://${host_ip}:9090/v1/postgres/health \
--header 'Content-Type: application/json' \
--data '{"user": "'${POSTGRES_USER}'","password": "'${POSTGRES_PASSWORD}'","host": "'${your_ip}'", "port": "5442", "database": "'${POSTGRES_DB}'"}'
--data '{"user": "'${POSTGRES_USER}'","password": "'${POSTGRES_PASSWORD}'","host": "'${host_ip}'", "port": "5442", "database": "'${POSTGRES_DB}'"}'
```
#### 3.2.2 Invoke the microservice.
```bash
curl http://${your_ip}:9090/v1/text2sql\
curl http://${host_ip}:9090/v1/text2sql\
-X POST \
-d '{"input_text": "Find the total number of Albums.","conn_str": {"user": "'${POSTGRES_USER}'","password": "'${POSTGRES_PASSWORD}'","host": "'${your_ip}'", "port": "5442", "database": "'${POSTGRES_DB}'"}}' \
-d '{"input_text": "Find the total number of Albums.","conn_str": {"user": "'${POSTGRES_USER}'","password": "'${POSTGRES_PASSWORD}'","host": "'${host_ip}'", "port": "5442", "database": "'${POSTGRES_DB}'"}}' \
-H 'Content-Type: application/json'
```
@@ -161,7 +172,7 @@ npm run test
## 🚀 Launch the React UI
Open this URL `http://{your_ip}:5174` in your browser to access the frontend.
Open this URL `http://{host_ip}:5174` in your browser to access the frontend.
![project-screenshot](../../../../assets/img/dbQnA_ui_init.png)

4
DBQnA/docker_compose/intel/cpu/xeon/compose.yaml
View File

@@ -1,11 +1,9 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
version: "3.8"
services:
tgi-service:
image: ghcr.io/huggingface/text-generation-inference:2.1.0
image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu
container_name: tgi-service
ports:
- "8008:80"

27
DBQnA/docker_compose/intel/cpu/xeon/set_env.sh Normal file
View File

@@ -0,0 +1,27 @@
#!/usr/bin/env bash
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
pushd "../../../../../" > /dev/null
source .set_env.sh
popd > /dev/null
#export host_ip=$(hostname -I | awk '{print $1}')
if [ -z "${HUGGINGFACEHUB_API_TOKEN}" ]; then
echo "Error: HUGGINGFACEHUB_API_TOKEN is not set. Please set HUGGINGFACEHUB_API_TOKEN."
fi
if [ -z "${host_ip}" ]; then
echo "Error: host_ip is not set. Please set host_ip first."
fi
export no_proxy=$no_proxy,$host_ip,dbqna-xeon-react-ui-server,text2sql-service,tgi-service,postgres-container
export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
export TGI_PORT=8008
export TGI_LLM_ENDPOINT="http://${host_ip}:${TGI_PORT}"
export LLM_MODEL_ID="mistralai/Mistral-7B-Instruct-v0.3"
export POSTGRES_USER=postgres
export POSTGRES_PASSWORD=testpwd
export POSTGRES_DB=chinook
export TEXT2SQL_PORT=9090
"set_env.sh" 27L, 974B

3
DBQnA/tests/test_compose_on_xeon.sh
View File

@@ -22,6 +22,9 @@ function build_docker_images() {
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log
docker pull ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu
docker images && sleep 1s
}
function start_service() {