9180f1066d
Set vllm as default llm serving, and add related docker compose files, readmes, and test scripts. Issue: https://github.com/opea-project/GenAIExamples/issues/1436 Signed-off-by: letonghan <letong.han@intel.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
25 lines
739 B
Bash
25 lines
739 B
Bash
#!/usr/bin/env bash
|
|
|
|
# Copyright (C) 2024 Intel Corporation
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
pushd "../../" > /dev/null
|
|
source .set_env.sh
|
|
popd > /dev/null
|
|
|
|
|
|
export LLM_MODEL_ID="mistralai/Mistral-7B-Instruct-v0.3"
|
|
export LLM_ENDPOINT="http://${host_ip}:8008"
|
|
export LLM_COMPONENT_NAME="OpeaTextGenService"
|
|
export NUM_CARDS=1
|
|
export BLOCK_SIZE=128
|
|
export MAX_NUM_SEQS=256
|
|
export MAX_SEQ_LEN_TO_CAPTURE=2048
|
|
export MEGA_SERVICE_HOST_IP=${host_ip}
|
|
export LLM_SERVICE_HOST_IP=${host_ip}
|
|
export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:7777/v1/codetrans"
|
|
export FRONTEND_SERVICE_IP=${host_ip}
|
|
export FRONTEND_SERVICE_PORT=5173
|
|
export BACKEND_SERVICE_NAME=codetrans
|
|
export BACKEND_SERVICE_IP=${host_ip}
|
|
export BACKEND_SERVICE_PORT=7777
|