Refactor example code (#183)

Signed-off-by: lvliang-intel <liang1.lv@intel.com> Signed-off-by: Yue, Wenjiao <wenjiao.yue@intel.com> Signed-off-by: chensuyue <suyue.chen@intel.com>
2024-05-24 13:32:14 +08:00
parent b91a9d10af
commit a6b3caf128
73 changed files with 461 additions and 555 deletions
--- a/CodeGen/microservice/gaudi/docker/Dockerfile
+++ b/CodeGen/microservice/gaudi/docker/Dockerfile
@@ -33,7 +33,7 @@ RUN cd /home/user/ && \
 RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \
    pip install -r /home/user/GenAIComps/requirements.txt

-COPY ../codegen.py /home/user/codegen.py
+COPY ./codegen.py /home/user/codegen.py

 ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps

--- a/CodeGen/README.md
+++ b/CodeGen/README.md
@@ -24,8 +24,8 @@ The CodeGen service can be effortlessly deployed on either Intel Gaudi2 or Intel

 ## Deploy CodeGen on Gaudi

-Refer to the [Gaudi Guide](./microservice/gaudi/README.md) for instructions on deploying CodeGen on Gaudi.
+Refer to the [Gaudi Guide](./docker-composer/gaudi/README.md) for instructions on deploying CodeGen on Gaudi.

 ## Deploy CodeGen on Xeon

-Refer to the [Xeon Guide](./microservice/xeon/README.md) for instructions on deploying CodeGen on Xeon.
+Refer to the [Xeon Guide](./docker-composer/xeon/README.md) for instructions on deploying CodeGen on Xeon.
--- a/CodeGen/microservice/xeon/codegen.py
+++ b/CodeGen/microservice/xeon/codegen.py
@@ -17,19 +17,23 @@ import os

 from comps import CodeGenGateway, MicroService, ServiceOrchestrator, ServiceType

-SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
+MEGA_SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
+MEGA_SERVICE_PORT = os.getenv("MEGA_SERVICE_PORT", 6666)
+LLM_SERVICE_HOST_IP = os.getenv("LLM_SERVICE_HOST_IP", "0.0.0.0")
+LLM_SERVICE_PORT = os.getenv("LLM_SERVICE_PORT", 9000)


-class ChatQnAService:
-    def __init__(self, port=8000):
+class CodeGenService:
+    def __init__(self, host="0.0.0.0", port=8000):
+        self.host = host
        self.port = port
        self.megaservice = ServiceOrchestrator()

    def add_remote_service(self):
        llm = MicroService(
            name="llm",
-            host=SERVICE_HOST_IP,
-            port=9000,
+            host=LLM_SERVICE_HOST_IP,
+            port=LLM_SERVICE_PORT,
            endpoint="/v1/chat/completions",
            use_remote_service=True,
            service_type=ServiceType.LLM,
@@ -46,6 +50,6 @@ class ChatQnAService:


 if __name__ == "__main__":
-    chatqna = ChatQnAService(port=6666)
+    chatqna = CodeGenService(host=MEGA_SERVICE_HOST_IP, port=MEGA_SERVICE_PORT)
    chatqna.add_remote_service()
    asyncio.run(chatqna.schedule())
--- a/CodeGen/codegen.yaml
+++ b/CodeGen/codegen.yaml
@@ -0,0 +1,53 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+opea_micro_services:
+  tgi_service:
+    host: ${TGI_SERVICE_IP}
+    ports: ${TGI_SERVICE_PORT}
+    image: ghcr.io/huggingface/tgi-gaudi:1.2.1
+    volumes:
+      - "./data:/data"
+    runtime: habana
+    cap_add:
+      - SYS_NICE
+    ipc: host
+    environment:
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+      HABANA_VISIBLE_DEVICES: all
+      OMPI_MCA_btl_vader_single_copy_mechanism: none
+    model-id: ${LLM_MODEL_ID}
+  llm:
+    host: ${LLM_SERVICE_HOST_IP}
+    ports: ${LLM_SERVICE_PORT}
+    image: opea/gen-ai-comps:llm-tgi-gaudi-server
+    endpoint: /v1/chat/completions
+    environment:
+      TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
+      HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
+  ui:
+    host: ${UI_SERVICE_HOST_IP}
+    ports:
+      - "5173:5173"
+    environment:
+      - CHAT_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
+    image: opea/gen-ai-comps:codegen-ui-server
+
+opea_mega_service:
+  host: ${MEGA_SERVICE_HOST_IP}
+  ports: ${MEGA_SERVICE_PORT}
+  endpoint: /v1/codegen
+  image: opea/gen-ai-comps:codegen-megaservice-server
+  mega_flow:
+    - llm
--- a/CodeGen/docker-composer/gaudi/README.md
+++ b/CodeGen/docker-composer/gaudi/README.md
@@ -16,7 +16,7 @@ cd GenAIComps
 ### 2. Build LLM Image

 ```bash
-docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/langchain/docker/Dockerfile .
+docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
 ```

 ### 3. Build MegaService Docker Image
@@ -25,8 +25,8 @@ To construct the Mega Service, we utilize the [GenAIComps](https://github.com/op

 ```bash
 git clone https://github.com/opea-project/GenAIExamples
-cd GenAIExamples/CodeGen/microservice/gaudi/
-docker build -t opea/gen-ai-comps:codegen-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+cd GenAIExamples/CodeGen
+docker build -t opea/gen-ai-comps:codegen-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
 ```

 ### 4. Build UI Docker Image
@@ -57,6 +57,7 @@ export LLM_MODEL_ID="ise-uiuc/Magicoder-S-DS-6.7B"
 export TGI_LLM_ENDPOINT="http://${host_ip}:8028"
 export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
 export MEGA_SERVICE_HOST_IP=${host_ip}
+export LLM_SERVICE_HOST_IP=${host_ip}
 export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:6666/v1/codegen"
 ```

@@ -65,6 +66,7 @@ Note: Please replace with `host_ip` with you external IP address, do not use loc
 ### Start all the services Docker Containers

 ```bash
+cd GenAIExamples/CodeGen/docker-composer/gaudi
 docker compose -f docker_compose.yaml up -d
 ```

--- a/CodeGen/docker-composer/gaudi/docker_compose.yaml
+++ b/CodeGen/docker-composer/gaudi/docker_compose.yaml
@@ -55,6 +55,7 @@ services:
      - https_proxy=${https_proxy}
      - http_proxy=${http_proxy}
      - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
+      - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
    ipc: host
    restart: always
  codegen-gaudi-ui-server:
--- a/CodeGen/docker-composer/xeon/README.md
+++ b/CodeGen/docker-composer/xeon/README.md
@@ -22,7 +22,7 @@ cd GenAIComps
 ### 1. Build LLM Image

 ```bash
-docker build -t opea/gen-ai-comps:llm-tgi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/langchain/docker/Dockerfile .
+docker build -t opea/gen-ai-comps:llm-tgi-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/text-generation/tgi/Dockerfile .
 ```

 ### 2. Build MegaService Docker Image
@@ -31,8 +31,8 @@ To construct the Mega Service, we utilize the [GenAIComps](https://github.com/op

 ```bash
 git clone https://github.com/opea-project/GenAIExamples
-cd GenAIExamples/CodeGen/microservice/xeon/
-docker build -t opea/gen-ai-comps:codegen-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile .
+cd GenAIExamples/CodeGen
+docker build -t opea/gen-ai-comps:codegen-megaservice-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
 ```

 ### 6. Build UI Docker Image
@@ -63,6 +63,7 @@ export LLM_MODEL_ID="ise-uiuc/Magicoder-S-DS-6.7B"
 export TGI_LLM_ENDPOINT="http://${host_ip}:8028"
 export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
 export MEGA_SERVICE_HOST_IP=${host_ip}
+export LLM_SERVICE_HOST_IP=${host_ip}
 export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:6666/v1/codegen"
 ```

@@ -71,6 +72,7 @@ Note: Please replace with `host_ip` with you external IP address, do not use loc
 ### Start all the services Docker Containers

 ```bash
+cd GenAIExamples/CodeGen/docker-composer/xeon
 docker compose -f docker_compose.yaml up -d
 ```

--- a/CodeGen/docker-composer/xeon/docker_compose.yaml
+++ b/CodeGen/docker-composer/xeon/docker_compose.yaml
@@ -53,6 +53,7 @@ services:
      - https_proxy=${https_proxy}
      - http_proxy=${http_proxy}
      - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
+      - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
    ipc: host
    restart: always
  codegen-xeon-ui-server:
--- a/CodeGen/kubernetes/README.md
+++ b/CodeGen/kubernetes/README.md
--- a/CodeGen/kubernetes/helm-charts/README.md
+++ b/CodeGen/kubernetes/helm-charts/README.md
--- a/CodeGen/kubernetes/manifests/README.md
+++ b/CodeGen/kubernetes/manifests/README.md
--- a/CodeGen/kubernetes/service-mesh/README.md
+++ b/CodeGen/kubernetes/service-mesh/README.md
--- a/CodeGen/microservice/gaudi/codegen.py
+++ b/CodeGen/microservice/gaudi/codegen.py
@@ -1,51 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import asyncio
-import os
-
-from comps import CodeGenGateway, MicroService, ServiceOrchestrator, ServiceType
-
-SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
-
-
-class ChatQnAService:
-    def __init__(self, port=8000):
-        self.port = port
-        self.megaservice = ServiceOrchestrator()
-
-    def add_remote_service(self):
-        llm = MicroService(
-            name="llm",
-            host=SERVICE_HOST_IP,
-            port=9000,
-            endpoint="/v1/chat/completions",
-            use_remote_service=True,
-            service_type=ServiceType.LLM,
-        )
-        self.megaservice.add(llm)
-        self.gateway = CodeGenGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port)
-
-    async def schedule(self):
-        await self.megaservice.schedule(
-            initial_inputs={"text": "Write a function that checks if a year is a leap year in Python."}
-        )
-        result_dict = self.megaservice.result_dict
-        print(result_dict)
-
-
-if __name__ == "__main__":
-    chatqna = ChatQnAService(port=6666)
-    chatqna.add_remote_service()
-    asyncio.run(chatqna.schedule())
--- a/CodeGen/microservice/xeon/docker/Dockerfile
+++ b/CodeGen/microservice/xeon/docker/Dockerfile
@@ -1,44 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-FROM python:3.11-slim
-
-ENV LANG C.UTF-8
-
-RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
-    libgl1-mesa-glx \
-    libjemalloc-dev \
-    vim \
-    git
-
-RUN useradd -m -s /bin/bash user && \
-    mkdir -p /home/user && \
-    chown -R user /home/user/
-
-RUN cd /home/user/ && \
-    git clone https://github.com/opea-project/GenAIComps.git
-
-RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \
-    pip install -r /home/user/GenAIComps/requirements.txt
-
-COPY ../codegen.py /home/user/codegen.py
-
-ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
-
-USER user
-
-WORKDIR /home/user
-
-ENTRYPOINT ["python", "codegen.py"]
--- a/CodeGen/tests/test_codegen_on_gaudi.sh
+++ b/CodeGen/tests/test_codegen_on_gaudi.sh
@@ -24,12 +24,12 @@ function build_docker_images() {
    git clone https://github.com/opea-project/GenAIComps.git
    cd GenAIComps

-    docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server -f comps/llms/langchain/docker/Dockerfile .
+    docker build -t opea/gen-ai-comps:llm-tgi-gaudi-server -f comps/llms/text-generation/tgi/Dockerfile .

    docker pull ghcr.io/huggingface/tgi-gaudi:1.2.1

-    cd $WORKPATH/microservice/gaudi
-    docker build --no-cache -t opea/gen-ai-comps:codegen-megaservice-server -f docker/Dockerfile .
+    cd $WORKPATH
+    docker build --no-cache -t opea/gen-ai-comps:codegen-megaservice-server -f Dockerfile .

    cd $WORKPATH/ui
    docker build --no-cache -t opea/gen-ai-comps:codegen-ui-server -f docker/Dockerfile .
@@ -38,12 +38,13 @@ function build_docker_images() {
 }

 function start_services() {
-    cd $WORKPATH/microservice/gaudi
+    cd $WORKPATH/docker-composer/gaudi

    export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
    export TGI_LLM_ENDPOINT="http://${ip_address}:8028"
    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
    export MEGA_SERVICE_HOST_IP=${ip_address}
+    export LLM_SERVICE_HOST_IP=${ip_address}
    export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:6666/v1/codegen"

    # Start Docker Containers
@@ -61,7 +62,7 @@ function validate_microservices() {

    curl http://${ip_address}:8028/generate \
        -X POST \
-        -d '{"inputs":"Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception.","parameters":{"max_new_tokens":1024, "do_sample": true}}' \
+        -d '{"inputs":"def print_hello_world():","parameters":{"max_new_tokens":1024, "do_sample": true}}' \
        -H 'Content-Type: application/json' > ${LOG_PATH}/generate.log
    exit_code=$?
    if [ $exit_code -ne 0 ]; then
@@ -73,7 +74,7 @@ function validate_microservices() {

    curl http://${ip_address}:9000/v1/chat/completions \
        -X POST \
-        -d '{"text":"Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception."}' \
+        -d '{"text":"def print_hello_world():"}' \
        -H 'Content-Type: application/json' > ${LOG_PATH}/completions.log
    exit_code=$?
    if [ $exit_code -ne 0 ]; then
@@ -88,12 +89,12 @@ function validate_megaservice() {
    # Curl the Mega Service
    curl http://${ip_address}:6666/v1/codegen -H "Content-Type: application/json" -d '{
        "model": "ise-uiuc/Magicoder-S-DS-6.7B",
-        "messages": "Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception."}' > ${LOG_PATH}/curl_megaservice.log
+        "messages": "def print_hello_world():"}' > ${LOG_PATH}/curl_megaservice.log

    echo "Checking response results, make sure the output is reasonable. "
-    local status=true
+    local status=false
    if [[ -f $LOG_PATH/curl_megaservice.log ]] && \
-    [[ $(grep -c "billion" $LOG_PATH/curl_megaservice.log) != 0 ]]; then
+    [[ $(grep -c "Hello" $LOG_PATH/curl_megaservice.log) != 0 ]]; then
        status=true
    fi

@@ -109,7 +110,7 @@ function validate_megaservice() {
 }

 function stop_docker() {
-    cd $WORKPATH/microservice/gaudi
+    cd $WORKPATH/docker-composer/gaudi
    container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
    for container_name in $container_list; do
        cid=$(docker ps -aq --filter "name=$container_name")
--- a/CodeGen/tests/test_codegen_on_xeon.sh
+++ b/CodeGen/tests/test_codegen_on_xeon.sh
@@ -14,10 +14,10 @@ function build_docker_images() {
    git clone https://github.com/opea-project/GenAIComps.git
    cd GenAIComps

-    docker build -t opea/gen-ai-comps:llm-tgi-server -f comps/llms/langchain/docker/Dockerfile .
+    docker build -t opea/gen-ai-comps:llm-tgi-server -f comps/llms/text-generation/tgi/Dockerfile .

-    cd $WORKPATH/microservice/xeon
-    docker build --no-cache -t opea/gen-ai-comps:codegen-megaservice-server -f docker/Dockerfile .
+    cd $WORKPATH
+    docker build --no-cache -t opea/gen-ai-comps:codegen-megaservice-server -f Dockerfile .

    cd $WORKPATH/ui
    docker build --no-cache -t opea/gen-ai-comps:codegen-ui-server -f docker/Dockerfile .
@@ -26,12 +26,13 @@ function build_docker_images() {
 }

 function start_services() {
-    cd $WORKPATH/microservice/xeon
+    cd $WORKPATH/docker-composer/xeon

    export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3"
    export TGI_LLM_ENDPOINT="http://${ip_address}:8028"
    export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
    export MEGA_SERVICE_HOST_IP=${ip_address}
+    export LLM_SERVICE_HOST_IP=${ip_address}
    export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:6666/v1/codegen"

    # Start Docker Containers
@@ -49,13 +50,13 @@ function validate_microservices() {

    curl http://${ip_address}:8028/generate \
        -X POST \
-        -d '{"inputs":"Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception.","parameters":{"max_new_tokens":256, "do_sample": true}}' \
+        -d '{"inputs":"def print_hello_world():","parameters":{"max_new_tokens":256, "do_sample": true}}' \
        -H 'Content-Type: application/json' > ${LOG_PATH}/generate.log
    sleep 5s

    curl http://${ip_address}:9000/v1/chat/completions \
        -X POST \
-        -d '{"text":"Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception."}' \
+        -d '{"text":"def print_hello_world():"}' \
        -H 'Content-Type: application/json' > ${LOG_PATH}/completions.log
    sleep 5s
 }
@@ -64,12 +65,13 @@ function validate_megaservice() {
    # Curl the Mega Service
    curl http://${ip_address}:6666/v1/codegen -H "Content-Type: application/json" -d '{
        "model": "ise-uiuc/Magicoder-S-DS-6.7B",
-        "messages": "Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception."}' > ${LOG_PATH}/curl_megaservice.log
+        "messages": "def print_hello_world():"}' > ${LOG_PATH}/curl_megaservice.log

+    sleep 2s
    echo "Checking response results, make sure the output is reasonable. "
-    local status=true
+    local status=false
    if [[ -f $LOG_PATH/curl_megaservice.log ]] && \
-    [[ $(grep -c "billion" $LOG_PATH/curl_megaservice.log) != 0 ]]; then
+    [[ $(grep -c "Hello" $LOG_PATH/curl_megaservice.log) != 0 ]]; then
        status=true
    fi

@@ -86,7 +88,7 @@ function validate_megaservice() {
 }

 function stop_docker() {
-    cd $WORKPATH/microservice/xeon
+    cd $WORKPATH/docker-composer/xeon
    container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
    for container_name in $container_list; do
        cid=$(docker ps -aq --filter "name=$container_name")