28 lines
550 B
YAML
28 lines
550 B
YAML
# Copyright (C) 2024 Intel Corporation
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
tgi:
|
|
enabled: false
|
|
|
|
vllm:
|
|
enabled: true
|
|
accelDevice: "gaudi"
|
|
image:
|
|
repository: opea/vllm-gaudi
|
|
PT_HPU_ENABLE_LAZY_COLLECTIVES: "true"
|
|
OMPI_MCA_btl_vader_single_copy_mechanism: "none"
|
|
startupProbe:
|
|
failureThreshold: 360
|
|
resources:
|
|
limits:
|
|
habana.ai/gaudi: 1
|
|
extraCmdArgs: [
|
|
"--tensor-parallel-size", "1",
|
|
"--block-size", "128",
|
|
"--max-num-seqs", "256",
|
|
]
|
|
|
|
llm-uservice:
|
|
TEXTGEN_BACKEND: vLLM
|
|
retryTimeoutSeconds: 720
|