jandres/GenAIExamples
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Adding files to deploy DocSum application on ROCm vLLM (#1572)

Browse Source 
Signed-off-by: Chingis Yundunov <YundunovCN@sibedge.com>
This commit is contained in:
chyundunovDatamonsters
2025-04-03 13:20:23 +07:00
committed by GitHub
parent 1a0c5f03c6
commit 319dbdaa6b
10 changed files with 816 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
DocSum/docker_compose/amd/gpu/rocm/set_env.sh
View File

@@ -3,15 +3,16 @@
# Copyright (C) 2024 Advanced Micro Devices, Inc.
# SPDX-License-Identifier: Apache-2.0
export MAX_INPUT_TOKENS=2048
export MAX_TOTAL_TOKENS=4096
export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.4.1-rocm"
export HOST_IP=''
export DOCSUM_MAX_INPUT_TOKENS="2048"
export DOCSUM_MAX_TOTAL_TOKENS="4096"
export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
export HOST_IP=${host_ip}
export DOCSUM_TGI_SERVICE_PORT="8008"
export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}"
export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
export DOCSUM_WHISPER_PORT="7066"
export ASR_SERVICE_HOST_IP="${HOST_IP}"
export DOCSUM_LLM_SERVER_PORT="9000"
export DOCSUM_BACKEND_SERVER_PORT="8888"
export DOCSUM_FRONTEND_PORT="5173"
export DOCSUM_BACKEND_SERVER_PORT="18072"
export DOCSUM_FRONTEND_PORT="18073"
export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum"