jandres/GenAIExamples
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Refactor llm predictionguard (#1143)

Browse Source 
* refactor llm predictionguard

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>

* refine predictionguard ut

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>

* remove duplicated dockerfile path

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>

* fix bug

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix bug

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>

---------

Signed-off-by: Xinyao Wang <xinyao.wang@intel.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: ZePan110 <ze.pan@intel.com>
This commit is contained in:
XinyaoWa
2025-01-15 10:13:42 +08:00
committed by GitHub
parent 89dd6285d8
commit 4c21738ad5
12 changed files with 119 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
tests/llms/test_llms_text-generation_predictionguard.sh
View File

@@ -13,22 +13,24 @@ fi
function build_docker_images() {
cd $WORKPATH
echo $(pwd)
docker build --no-cache -t opea/llm-pg:comps -f comps/llms/text-generation/predictionguard/Dockerfile .
docker build --no-cache -t opea/llm-textgen:comps -f comps/llms/src/text-generation/Dockerfile .
if [ $? -ne 0 ]; then
echo "opea/llm-pg built failed"
echo "opea/llm-textgen built failed"
exit 1
else
echo "opea/llm-pg built successfully"
echo "opea/llm-textgen built successfully"
fi
}
function start_service() {
llm_service_port=9000
unset http_proxy
docker run -d --name=test-comps-llm-pg-server \
docker run -d --name=test-comps-llm-textgen-pg-server \
-e http_proxy= -e https_proxy= \
-e PREDICTIONGUARD_API_KEY=${PREDICTIONGUARD_API_KEY} \
-p 9000:9000 --ipc=host opea/llm-pg:comps
-e LLM_COMPONENT_NAME="OPEATextGen_Predictionguard" \
-e LOGFLAG=True \
-p 9000:9000 --ipc=host opea/llm-textgen:comps
sleep 60 # Sleep for 1 minute to allow the service to start
}
@@ -36,20 +38,20 @@ function validate_microservice() {
llm_service_port=9000
result=$(http_proxy="" curl http://${ip_address}:${llm_service_port}/v1/chat/completions \
-X POST \
-d '{"model": "Hermes-2-Pro-Llama-3-8B", "query": "What is AI?", "stream": false, "max_tokens": 100, "temperature": 0.7, "top_p": 1.0, "top_k": 50}' \
-d '{"model": "Hermes-2-Pro-Llama-3-8B", "messages": "What is AI?", "stream": false, "max_tokens": 100, "temperature": 0.7, "top_p": 1.0, "top_k": 50}' \
-H 'Content-Type: application/json')
if [[ $result == *"text"* ]]; then
if [[ $result == *"content"* ]]; then
echo "Service response is correct."
else
echo "Result wrong. Received was $result"
docker logs test-comps-llm-pg-server
docker logs test-comps-llm-textgen-pg-server
exit 1
fi
}
function stop_docker() {
cid=$(docker ps -aq --filter "name=test-comps-llm-pg-*")
cid=$(docker ps -aq --filter "name=test-comps-llm-textgen-pg-*")
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
}