1852e6bcc3
Signed-off-by: minmin-intel <minmin.hou@intel.com> Signed-off-by: Rita Brugarolas <rita.brugarolas.brufau@intel.com> Signed-off-by: rbrugaro <rita.brugarolas.brufau@intel.com> Co-authored-by: rbrugaro <rita.brugarolas.brufau@intel.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: lkk <33276950+lkk12014402@users.noreply.github.com> Co-authored-by: lkk12014402 <kaokao.lv@intel.com>
36 lines
1.0 KiB
YAML
36 lines
1.0 KiB
YAML
|
|
# Copyright (C) 2025 Intel Corporation
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
services:
|
|
vllm-service:
|
|
image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest}
|
|
container_name: vllm-gaudi-server
|
|
ports:
|
|
- "8086:8000"
|
|
volumes:
|
|
- ${HF_CACHE_DIR}:/data
|
|
environment:
|
|
no_proxy: ${no_proxy}
|
|
http_proxy: ${http_proxy}
|
|
https_proxy: ${https_proxy}
|
|
HF_TOKEN: ${HF_TOKEN}
|
|
HUGGING_FACE_HUB_TOKEN: ${HF_TOKEN}
|
|
HF_HOME: /data
|
|
HABANA_VISIBLE_DEVICES: all
|
|
OMPI_MCA_btl_vader_single_copy_mechanism: none
|
|
LLM_MODEL_ID: ${LLM_MODEL_ID}
|
|
VLLM_TORCH_PROFILER_DIR: "/mnt"
|
|
VLLM_SKIP_WARMUP: true
|
|
PT_HPU_ENABLE_LAZY_COLLECTIVES: true
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "curl -f http://$host_ip:8086/health || exit 1"]
|
|
interval: 10s
|
|
timeout: 10s
|
|
retries: 100
|
|
runtime: habana
|
|
cap_add:
|
|
- SYS_NICE
|
|
ipc: host
|
|
command: --model $LLM_MODEL_ID --tensor-parallel-size 4 --host 0.0.0.0 --port 8000 --max-seq-len-to-capture $MAX_LEN
|