jandres/GenAIExamples
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

update manifests for v0.9 (#623)

Browse Source 
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Zhenzhong1
2024-08-19 15:55:04 +08:00
committed by GitHub
parent 5a9c109e35
commit 08f57fa54a
33 changed files with 765 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
ChatQnA/benchmark/four_gaudi/chatqna_config_map.yaml Normal file
View File

@@ -0,0 +1,23 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
apiVersion: v1
kind: ConfigMap
metadata:
name: qna-config
namespace: default
data:
EMBEDDING_MODEL_ID: BAAI/bge-base-en-v1.5
RERANK_MODEL_ID: BAAI/bge-reranker-base
LLM_MODEL_ID: Intel/neural-chat-7b-v3-3
TEI_EMBEDDING_ENDPOINT: http://embedding-dependency-svc.default.svc.cluster.local:6006
TEI_RERANKING_ENDPOINT: http://reranking-dependency-svc.default.svc.cluster.local:8808
TGI_LLM_ENDPOINT: http://llm-dependency-svc.default.svc.cluster.local:9009
REDIS_URL: redis://vector-db.default.svc.cluster.local:6379
INDEX_NAME: rag-redis
HUGGINGFACEHUB_API_TOKEN: {HF_TOKEN}
EMBEDDING_SERVICE_HOST_IP: embedding-svc
RETRIEVER_SERVICE_HOST_IP: retriever-svc
RERANK_SERVICE_HOST_IP: reranking-svc
NODE_SELECTOR: chatqna-opea
LLM_SERVICE_HOST_IP: llm-svc

62
ChatQnA/benchmark/four_gaudi/chatqna_mega_service_run.yaml Normal file
View File

@@ -0,0 +1,62 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
apiVersion: apps/v1
kind: Deployment
metadata:
name: chatqna-backend-server-deploy
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app: chatqna-backend-server-deploy
template:
metadata:
annotations:
sidecar.istio.io/rewriteAppHTTPProbers: 'true'
labels:
app: chatqna-backend-server-deploy
spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: chatqna-backend-server-deploy
hostIPC: true
containers:
- envFrom:
- configMapRef:
name: qna-config
image: opea/chatqna:latest
imagePullPolicy: IfNotPresent
name: chatqna-backend-server-deploy
args: null
ports:
- containerPort: 8888
resources:
limits:
cpu: 8
memory: 4000Mi
requests:
cpu: 8
memory: 4000Mi
serviceAccountName: default
---
kind: Service
apiVersion: v1
metadata:
name: chaqna-backend-server-svc
spec:
type: NodePort
selector:
app: chatqna-backend-server-deploy
ports:
- name: service
port: 8888
targetPort: 8888
nodePort: 30888

70
ChatQnA/benchmark/four_gaudi/dataprep-microservice_run.yaml Normal file
View File

@@ -0,0 +1,70 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
apiVersion: apps/v1
kind: Deployment
metadata:
name: dataprep-deploy
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app: dataprep-deploy
template:
metadata:
annotations:
sidecar.istio.io/rewriteAppHTTPProbers: 'true'
labels:
app: dataprep-deploy
spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: dataprep-deploy
hostIPC: true
containers:
- env:
- name: REDIS_URL
valueFrom:
configMapKeyRef:
name: qna-config
key: REDIS_URL
- name: INDEX_NAME
valueFrom:
configMapKeyRef:
name: qna-config
key: INDEX_NAME
image: opea/dataprep-redis:latest
imagePullPolicy: IfNotPresent
name: dataprep-deploy
args: null
ports:
- containerPort: 6007
- containerPort: 6008
- containerPort: 6009
serviceAccountName: default
---
kind: Service
apiVersion: v1
metadata:
name: dataprep-svc
spec:
type: ClusterIP
selector:
app: dataprep-deploy
ports:
- name: port1
port: 6007
targetPort: 6007
- name: port2
port: 6008
targetPort: 6008
- name: port3
port: 6009
targetPort: 6009

69
ChatQnA/benchmark/four_gaudi/embedding-dependency_run.yaml Normal file
View File

@@ -0,0 +1,69 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
apiVersion: apps/v1
kind: Deployment
metadata:
name: embedding-dependency-deploy
namespace: default
spec:
replicas: 6
selector:
matchLabels:
app: embedding-dependency-deploy
template:
metadata:
annotations:
sidecar.istio.io/rewriteAppHTTPProbers: 'true'
labels:
app: embedding-dependency-deploy
spec:
nodeSelector:
node-type: chatqna-opea
containers:
- envFrom:
- configMapRef:
name: qna-config
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
name: embedding-dependency-deploy
args:
- --model-id
- $(EMBEDDING_MODEL_ID)
- --auto-truncate
volumeMounts:
- mountPath: /data
name: model-volume
- mountPath: /dev/shm
name: shm
ports:
- containerPort: 80
resources:
limits:
cpu: 80
memory: 20000Mi
requests:
cpu: 80
memory: 20000Mi
serviceAccountName: default
volumes:
- name: model-volume
hostPath:
path: /home/sdp/cesg
type: Directory
- name: shm
emptyDir:
medium: Memory
sizeLimit: 1Gi
---
kind: Service
apiVersion: v1
metadata:
name: embedding-dependency-svc
spec:
type: ClusterIP
selector:
app: embedding-dependency-deploy
ports:
- name: service
port: 6006
targetPort: 80

59
ChatQnA/benchmark/four_gaudi/embedding-microservice_run.yaml Normal file
View File

@@ -0,0 +1,59 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
apiVersion: apps/v1
kind: Deployment
metadata:
name: embedding-deploy
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app: embedding-deploy
template:
metadata:
annotations:
sidecar.istio.io/rewriteAppHTTPProbers: 'true'
labels:
app: embedding-deploy
spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: embedding-deploy
hostIPC: true
containers:
- envFrom:
- configMapRef:
name: qna-config
image: opea/embedding-tei:latest
imagePullPolicy: IfNotPresent
name: embedding-deploy
args: null
ports:
- containerPort: 6000
resources:
limits:
cpu: 4
requests:
cpu: 4
serviceAccountName: default
---
kind: Service
apiVersion: v1
metadata:
name: embedding-svc
spec:
type: ClusterIP
selector:
app: embedding-deploy
ports:
- name: service
port: 6000
targetPort: 6000

88
ChatQnA/benchmark/four_gaudi/llm-dependency_run.yaml Normal file
View File

@@ -0,0 +1,88 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
apiVersion: apps/v1
kind: Deployment
metadata:
name: llm-dependency-deploy
namespace: default
spec:
replicas: 31
selector:
matchLabels:
app: llm-dependency-deploy
template:
metadata:
annotations:
sidecar.istio.io/rewriteAppHTTPProbers: 'true'
labels:
app: llm-dependency-deploy
spec:
nodeSelector:
node-type: chatqna-opea
hostIPC: true
containers:
- envFrom:
- configMapRef:
name: qna-config
image: tgi_gaudi:2.0.1
name: llm-dependency-deploy-demo
securityContext:
capabilities:
add:
- SYS_NICE
args:
- --model-id
- $(LLM_MODEL_ID)
- --max-input-length
- '1024'
- --max-total-tokens
- '2048'
- --max-batch-total-tokens
- '65536'
- --max-batch-prefill-tokens
- '4096'
volumeMounts:
- mountPath: /data
name: model-volume
- mountPath: /dev/shm
name: shm
ports:
- containerPort: 80
resources:
limits:
habana.ai/gaudi: 1
env:
- name: OMPI_MCA_btl_vader_single_copy_mechanism
value: none
- name: PT_HPU_ENABLE_LAZY_COLLECTIVES
value: 'true'
- name: runtime
value: habana
- name: HABANA_VISIBLE_DEVICES
value: all
- name: HF_TOKEN
value: $(HF_TOKEN)
serviceAccountName: default
volumes:
- name: model-volume
hostPath:
path: /home/sdp/cesg
type: Directory
- name: shm
emptyDir:
medium: Memory
sizeLimit: 1Gi
---
kind: Service
apiVersion: v1
metadata:
name: llm-dependency-svc
spec:
type: ClusterIP
selector:
app: llm-dependency-deploy
ports:
- name: service
port: 9009
targetPort: 80

59
ChatQnA/benchmark/four_gaudi/llm-microservice_run.yaml Normal file
View File

@@ -0,0 +1,59 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
apiVersion: apps/v1
kind: Deployment
metadata:
name: llm-deploy
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app: llm-deploy
template:
metadata:
annotations:
sidecar.istio.io/rewriteAppHTTPProbers: 'true'
labels:
app: llm-deploy
spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: llm-deploy
hostIPC: true
containers:
- envFrom:
- configMapRef:
name: qna-config
image: opea/llm-tgi:latest
imagePullPolicy: IfNotPresent
name: llm-deploy
args: null
ports:
- containerPort: 9000
resources:
limits:
cpu: 4
requests:
cpu: 4
serviceAccountName: default
---
kind: Service
apiVersion: v1
metadata:
name: llm-svc
spec:
type: ClusterIP
selector:
app: llm-deploy
ports:
- name: service
port: 9000
targetPort: 9000

85
ChatQnA/benchmark/four_gaudi/reranking-dependency_run.yaml Normal file
View File

@@ -0,0 +1,85 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
apiVersion: apps/v1
kind: Deployment
metadata:
name: reranking-dependency-deploy
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app: reranking-dependency-deploy
template:
metadata:
annotations:
sidecar.istio.io/rewriteAppHTTPProbers: 'true'
labels:
app: reranking-dependency-deploy
spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: reranking-dependency-deploy
containers:
- envFrom:
- configMapRef:
name: qna-config
image: tei_gaudi:rerank
name: reranking-dependency-deploy
args:
- --model-id
- $(RERANK_MODEL_ID)
- --auto-truncate
volumeMounts:
- mountPath: /data
name: model-volume
- mountPath: /dev/shm
name: shm
ports:
- containerPort: 80
resources:
limits:
habana.ai/gaudi: 1
env:
- name: OMPI_MCA_btl_vader_single_copy_mechanism
value: none
- name: PT_HPU_ENABLE_LAZY_COLLECTIVES
value: 'true'
- name: runtime
value: habana
- name: HABANA_VISIBLE_DEVICES
value: all
- name: HF_TOKEN
value: $(HF_TOKEN)
- name: MAX_WARMUP_SEQUENCE_LENGTH
value: '512'
serviceAccountName: default
volumes:
- name: model-volume
hostPath:
path: /home/sdp/cesg
type: Directory
- name: shm
emptyDir:
medium: Memory
sizeLimit: 1Gi
---
kind: Service
apiVersion: v1
metadata:
name: reranking-dependency-svc
spec:
type: ClusterIP
selector:
app: reranking-dependency-deploy
ports:
- name: service
port: 8808
targetPort: 80

59
ChatQnA/benchmark/four_gaudi/reranking-microservice_run.yaml Normal file
View File

@@ -0,0 +1,59 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
apiVersion: apps/v1
kind: Deployment
metadata:
name: reranking-deploy
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app: reranking-deploy
template:
metadata:
annotations:
sidecar.istio.io/rewriteAppHTTPProbers: 'true'
labels:
app: reranking-deploy
spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: reranking-deploy
hostIPC: true
containers:
- envFrom:
- configMapRef:
name: qna-config
image: opea/reranking-tei:latest
imagePullPolicy: IfNotPresent
name: reranking-deploy
args: null
ports:
- containerPort: 8000
resources:
limits:
cpu: 4
requests:
cpu: 4
serviceAccountName: default
---
kind: Service
apiVersion: v1
metadata:
name: reranking-svc
spec:
type: ClusterIP
selector:
app: reranking-deploy
ports:
- name: service
port: 8000
targetPort: 8000

69
ChatQnA/benchmark/four_gaudi/retrieval-microservice_run.yaml Normal file
View File

@@ -0,0 +1,69 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
apiVersion: apps/v1
kind: Deployment
metadata:
name: retriever-deploy
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app: retriever-deploy
template:
metadata:
annotations:
sidecar.istio.io/rewriteAppHTTPProbers: 'true'
labels:
app: retriever-deploy
spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: retriever-deploy
hostIPC: true
containers:
- env:
- name: REDIS_URL
valueFrom:
configMapKeyRef:
name: qna-config
key: REDIS_URL
- name: INDEX_NAME
valueFrom:
configMapKeyRef:
name: qna-config
key: INDEX_NAME
image: opea/retriever-redis:latest
imagePullPolicy: IfNotPresent
name: retriever-deploy
args: null
ports:
- containerPort: 7000
resources:
limits:
cpu: 8
memory: 2500Mi
requests:
cpu: 8
memory: 2500Mi
serviceAccountName: default
---
kind: Service
apiVersion: v1
metadata:
name: retriever-svc
spec:
type: ClusterIP
selector:
app: retriever-deploy
ports:
- name: service
port: 7000
targetPort: 7000

48
ChatQnA/benchmark/four_gaudi/vector-db_run.yaml Normal file
View File

@@ -0,0 +1,48 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
apiVersion: apps/v1
kind: Deployment
metadata:
name: vector-db
spec:
replicas: 1
selector:
matchLabels:
app: vector-db
template:
metadata:
labels:
app: vector-db
spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector:
matchLabels:
app: vector-db
containers:
- name: vector-db
image: redis/redis-stack:7.2.0-v9
ports:
- containerPort: 6379
- containerPort: 8001
---
apiVersion: v1
kind: Service
metadata:
name: vector-db
spec:
type: ClusterIP
selector:
app: vector-db
ports:
- name: vector-db-service
port: 6379
targetPort: 6379
- name: vector-db-insight
port: 8001
targetPort: 8001

2
ChatQnA/benchmark/single_gaudi/chatqna_config_map.yaml
View File

@@ -15,7 +15,9 @@ data:
TGI_LLM_ENDPOINT: http://llm-dependency-svc.default.svc.cluster.local:9009 TGI_LLM_ENDPOINT: http://llm-dependency-svc.default.svc.cluster.local:9009
REDIS_URL: redis://vector-db.default.svc.cluster.local:6379 REDIS_URL: redis://vector-db.default.svc.cluster.local:6379
INDEX_NAME: rag-redis INDEX_NAME: rag-redis
HUGGINGFACEHUB_API_TOKEN: {HF_TOKEN}
EMBEDDING_SERVICE_HOST_IP: embedding-svc EMBEDDING_SERVICE_HOST_IP: embedding-svc
RETRIEVER_SERVICE_HOST_IP: retriever-svc RETRIEVER_SERVICE_HOST_IP: retriever-svc
RERANK_SERVICE_HOST_IP: reranking-svc RERANK_SERVICE_HOST_IP: reranking-svc
NODE_SELECTOR: chatqna-opea
LLM_SERVICE_HOST_IP: llm-svc LLM_SERVICE_HOST_IP: llm-svc

2
ChatQnA/benchmark/single_gaudi/chatqna_mega_service_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: chatqna-backend-server-deploy app: chatqna-backend-server-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname

2
ChatQnA/benchmark/single_gaudi/dataprep-microservice_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: dataprep-deploy app: dataprep-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname

14
ChatQnA/benchmark/single_gaudi/embedding-dependency_run.yaml
View File

@@ -7,7 +7,7 @@ metadata:
name: embedding-dependency-deploy name: embedding-dependency-deploy
namespace: default namespace: default
spec: spec:
replicas: 4 replicas: 1
selector: selector:
matchLabels: matchLabels:
app: embedding-dependency-deploy app: embedding-dependency-deploy
@@ -18,11 +18,13 @@ spec:
labels: labels:
app: embedding-dependency-deploy app: embedding-dependency-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
containers: containers:
- envFrom: - envFrom:
- configMapRef: - configMapRef:
name: qna-config name: qna-config
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
name: embedding-dependency-deploy name: embedding-dependency-deploy
args: args:
- --model-id - --model-id
@@ -37,11 +39,11 @@ spec:
- containerPort: 80 - containerPort: 80
resources: resources:
limits: limits:
cpu: 24 cpu: 80
memory: 4000Mi memory: 20000Mi
requests: requests:
cpu: 24 cpu: 80
memory: 4000Mi memory: 20000Mi
serviceAccountName: default serviceAccountName: default
volumes: volumes:
- name: model-volume - name: model-volume

2
ChatQnA/benchmark/single_gaudi/embedding-microservice_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: embedding-deploy app: embedding-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname

8
ChatQnA/benchmark/single_gaudi/llm-dependency_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: llm-dependency-deploy app: llm-dependency-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
hostIPC: true hostIPC: true
containers: containers:
- envFrom: - envFrom:
@@ -33,8 +35,12 @@ spec:
- --model-id - --model-id
- $(LLM_MODEL_ID) - $(LLM_MODEL_ID)
- --max-input-length - --max-input-length
- '2048' - '1024'
- --max-total-tokens - --max-total-tokens
- '2048'
- --max-batch-total-tokens
- '65536'
- --max-batch-prefill-tokens
- '4096' - '4096'
volumeMounts: volumeMounts:
- mountPath: /data - mountPath: /data

2
ChatQnA/benchmark/single_gaudi/llm-microservice_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: llm-deploy app: llm-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname

2
ChatQnA/benchmark/single_gaudi/reranking-dependency_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: reranking-dependency-deploy app: reranking-dependency-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname

2
ChatQnA/benchmark/single_gaudi/reranking-microservice_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: reranking-deploy app: reranking-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname

2
ChatQnA/benchmark/single_gaudi/retrieval-microservice_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: retriever-deploy app: retriever-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname

2
ChatQnA/benchmark/single_gaudi/vector-db_run.yaml
View File

@@ -15,6 +15,8 @@ spec:
labels: labels:
app: vector-db app: vector-db
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname

3
ChatQnA/benchmark/two_gaudi/chatqna_config_map.yaml
View File

@@ -15,8 +15,9 @@ data:
TGI_LLM_ENDPOINT: http://llm-dependency-svc.default.svc.cluster.local:9009 TGI_LLM_ENDPOINT: http://llm-dependency-svc.default.svc.cluster.local:9009
REDIS_URL: redis://vector-db.default.svc.cluster.local:6379 REDIS_URL: redis://vector-db.default.svc.cluster.local:6379
INDEX_NAME: rag-redis INDEX_NAME: rag-redis
HUGGINGFACEHUB_API_TOKEN: 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 HUGGINGFACEHUB_API_TOKEN: {HF_TOKEN}
EMBEDDING_SERVICE_HOST_IP: embedding-svc EMBEDDING_SERVICE_HOST_IP: embedding-svc
RETRIEVER_SERVICE_HOST_IP: retriever-svc RETRIEVER_SERVICE_HOST_IP: retriever-svc
RERANK_SERVICE_HOST_IP: reranking-svc RERANK_SERVICE_HOST_IP: reranking-svc
NODE_SELECTOR: chatqna-opea
LLM_SERVICE_HOST_IP: llm-svc LLM_SERVICE_HOST_IP: llm-svc

2
ChatQnA/benchmark/two_gaudi/chatqna_mega_service_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: chatqna-backend-server-deploy app: chatqna-backend-server-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname

10
ChatQnA/benchmark/two_gaudi/dataprep-microservice_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: dataprep-deploy app: dataprep-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname
@@ -44,6 +46,8 @@ spec:
args: null args: null
ports: ports:
- containerPort: 6007 - containerPort: 6007
- containerPort: 6008
- containerPort: 6009
serviceAccountName: default serviceAccountName: default
--- ---
kind: Service kind: Service
@@ -58,3 +62,9 @@ spec:
- name: port1 - name: port1
port: 6007 port: 6007
targetPort: 6007 targetPort: 6007
- name: port2
port: 6008
targetPort: 6008
- name: port3
port: 6009
targetPort: 6009

14
ChatQnA/benchmark/two_gaudi/embedding-dependency_run.yaml
View File

@@ -7,7 +7,7 @@ metadata:
name: embedding-dependency-deploy name: embedding-dependency-deploy
namespace: default namespace: default
spec: spec:
replicas: 10 replicas: 3
selector: selector:
matchLabels: matchLabels:
app: embedding-dependency-deploy app: embedding-dependency-deploy
@@ -18,11 +18,13 @@ spec:
labels: labels:
app: embedding-dependency-deploy app: embedding-dependency-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
containers: containers:
- envFrom: - envFrom:
- configMapRef: - configMapRef:
name: qna-config name: qna-config
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
name: embedding-dependency-deploy name: embedding-dependency-deploy
args: args:
- --model-id - --model-id
@@ -37,11 +39,11 @@ spec:
- containerPort: 80 - containerPort: 80
resources: resources:
limits: limits:
cpu: 24 cpu: 80
memory: 4000Mi memory: 20000Mi
requests: requests:
cpu: 24 cpu: 80
memory: 4000Mi memory: 20000Mi
serviceAccountName: default serviceAccountName: default
volumes: volumes:
- name: model-volume - name: model-volume

2
ChatQnA/benchmark/two_gaudi/embedding-microservice_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: embedding-deploy app: embedding-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname

8
ChatQnA/benchmark/two_gaudi/llm-dependency_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: llm-dependency-deploy app: llm-dependency-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
hostIPC: true hostIPC: true
containers: containers:
- envFrom: - envFrom:
@@ -33,8 +35,12 @@ spec:
- --model-id - --model-id
- $(LLM_MODEL_ID) - $(LLM_MODEL_ID)
- --max-input-length - --max-input-length
- '2048' - '1024'
- --max-total-tokens - --max-total-tokens
- '2048'
- --max-batch-total-tokens
- '65536'
- --max-batch-prefill-tokens
- '4096' - '4096'
volumeMounts: volumeMounts:
- mountPath: /data - mountPath: /data

2
ChatQnA/benchmark/two_gaudi/llm-microservice_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: llm-deploy app: llm-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname

2
ChatQnA/benchmark/two_gaudi/reranking-dependency_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: reranking-dependency-deploy app: reranking-dependency-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname

2
ChatQnA/benchmark/two_gaudi/reranking-microservice_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: reranking-deploy app: reranking-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname

2
ChatQnA/benchmark/two_gaudi/retrieval-microservice_run.yaml
View File

@@ -18,6 +18,8 @@ spec:
labels: labels:
app: retriever-deploy app: retriever-deploy
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname

2
ChatQnA/benchmark/two_gaudi/vector-db_run.yaml
View File

@@ -15,6 +15,8 @@ spec:
labels: labels:
app: vector-db app: vector-db
spec: spec:
nodeSelector:
node-type: chatqna-opea
topologySpreadConstraints: topologySpreadConstraints:
- maxSkew: 1 - maxSkew: 1
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname