Support megaservice ut in CI test (#13)

Signed-off-by: chensuyue <suyue.chen@intel.com>
Signed-off-by: lvliang-intel <liang1.lv@intel.com>

.github/workflows/docker/ut.dockerfile

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG UBUNTU_VER=22.04
+FROM ubuntu:${UBUNTU_VER} as devel
+
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
+    aspell \
+    aspell-en \
+    build-essential \
+    python3 \
+    python3-pip \
+    python3-dev \
+    python3-distutils \
+    git \
+    vim \
+    wget
+
+RUN ln -sf $(which python3) /usr/bin/python
+RUN python -m pip install --no-cache-dir pytest
+
+WORKDIR /

.github/workflows/embeddings-comp-test.yml

@@ -0,0 +1,61 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: EmbeddingsMicroService-test
+
+on:
+  #  pull_request:
+  #    branches: [main]
+  #    types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
+  #    paths:
+  #      - .github/workflows/mega-test.yml
+  #      - comps/mega/**
+  workflow_dispatch:
+
+# If there is a new commit, the previous jobs will be canceled
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  EmbeddingsMicroService:
+    runs-on: aise-cluster
+    steps:
+      - name: Clean Up Working Directory
+        run: sudo rm -rf ${{github.workspace}}/*
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          ref: "refs/pull/${{ github.event.number }}/merge"
+
+      - name: Run UT
+        env:
+          HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
+        run: |
+          cd .github/workflows/scripts
+          bash test_ut.sh --test_name embeddings
+
+      - name: Run Workflow
+        env:
+          HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
+        run: |
+          cd tests/workflows
+          bash test_embeddings_comps.sh
+
+      - name: Publish pipeline artifact
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          path: ${{ github.workspace }}/.github/workflows/scripts/*.log

.github/workflows/llms-comp-test.yml

@@ -0,0 +1,61 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: LLMsMicroService-test
+
+on:
+  #  pull_request:
+  #    branches: [main]
+  #    types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
+  #    paths:
+  #      - .github/workflows/llms-comp-test.yml
+  #      - comps/llms/**
+  workflow_dispatch:
+
+# If there is a new commit, the previous jobs will be canceled
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  LLMsMicroService:
+    runs-on: aise-cluster
+    steps:
+      - name: Clean Up Working Directory
+        run: sudo rm -rf ${{github.workspace}}/*
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          ref: "refs/pull/${{ github.event.number }}/merge"
+
+      - name: Run UT
+        env:
+          HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
+        run: |
+          cd .github/workflows/scripts
+          bash test_ut.sh --test_name llms
+
+      - name: Run Workflow
+        env:
+          HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
+        run: |
+          cd tests/workflows
+          bash test_llms_comps.sh
+
+      - name: Publish pipeline artifact
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          path: ${{ github.workspace }}/.github/workflows/scripts/*.log

.github/workflows/mega-test.yaml

@@ -0,0 +1,82 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: MegaService-test
+
+on:
+  pull_request:
+    branches: [main]
+    types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
+    paths:
+      - .github/workflows/mega-test.yml
+      - comps/mega/**
+      - requirements.txt
+      - setup.py
+      - tests/mega/**
+  workflow_dispatch:
+
+# If there is a new commit, the previous jobs will be canceled
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  CONTAINER_NAME: "ut-test-container"
+  TEST_NAME: "mega"
+
+jobs:
+  MegaService:
+    runs-on: aise-cluster
+    steps:
+      - name: Clean Up Working Directory
+        run: sudo rm -rf ${{github.workspace}}/*
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          ref: "refs/pull/${{ github.event.number }}/merge"
+
+      - name: Docker Build
+        run: |
+          docker build -f ${{ github.workspace }}/.github/workflows/docker/ut.dockerfile -t ut-test:1.0 .
+
+      - name: Docker Run
+        run: |
+          if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
+            docker stop ${{ env.CONTAINER_NAME }}
+            docker rm -vf ${{ env.CONTAINER_NAME }} || true
+          fi
+          docker run -dit --memory="4g" --memory-reservation="1g" --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} --shm-size="1g" \
+          -v ${{ github.workspace }}:/GenAIComps ut-test:1.0
+
+      - name: Install Dependencies
+        run: |
+          docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /GenAIComps && pip install -r requirements.txt && python setup.py install"
+
+      - name: Run UT
+        run: |
+          docker exec ${{ env.CONTAINER_NAME }} \
+          bash -c "bash /GenAIComps/.github/workflows/scripts/test_ut.sh ${{ env.TEST_NAME }}"
+
+      #      - name: Run Workflow
+      #        env:
+      #          HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
+      #        run: |
+      #          xxx
+
+      - name: Publish pipeline artifact
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          path: ${{ github.workspace }}/.github/workflows/scripts/*.log

.github/workflows/scripts/test_ut.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+test_name=$1
+
+# run test
+ut_log_name=/GenAIComps/.github/workflows/scripts/${test_name}_ut.log
+cd /GenAIComps/tests
+if [ $test_name = 'mega' ]; then
+    echo "run mega test"
+    cd mega
+    find . -name "test*.py" | sed 's,\.\/,python -m pytest -vs --disable-warnings ,g' > run.sh
+    bash run.sh 2>&1 | tee ${ut_log_name}
+else
+    echo "run other test"
+    python -m pytest -vs --disable-warnings ./test_${test_name}*.py 2>&1 | tee ${ut_log_name}
+fi
+
+# clean the pytest cache
+rm -rf /GenAIComps/.pytest_cache
+
+# check test result
+if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS ==' ${ut_log_name}) != 0 ] || [ $(grep -c ' passed' ${ut_log_name}) == 0 ]; then
+    echo "Find errors in pytest case, please check the output..."
+    echo "Please search for '== FAILURES ==' or '== ERRORS =='"
+    exit 1
+fi
+echo "UT finished successfully! "

comps/cores/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

comps/cores/telemetry/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

comps/version.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "0.1"

setup.py

@@ -0,0 +1,86 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import subprocess
+from io import open
+
+from setuptools import find_packages, setup
+
+
+def fetch_requirements(path):
+    with open(path, "r") as fd:
+        return [r.strip() for r in fd.readlines()]
+
+
+def is_commit_on_tag():
+    try:
+        result = subprocess.run(
+            ["git", "describe", "--exact-match", "--tags"], capture_output=True, text=True, check=True
+        )
+        tag_name = result.stdout.strip()
+        return tag_name
+    except subprocess.CalledProcessError:
+        return False
+
+
+def get_build_version():
+    if is_commit_on_tag():
+        return __version__
+    try:
+        result = subprocess.run(["git", "describe", "--tags"], capture_output=True, text=True, check=True)
+        _, distance, commit = result.stdout.strip().split("-")
+        return f"{__version__}.dev{distance}+{commit}"
+    except subprocess.CalledProcessError:
+        return __version__
+
+
+try:
+    filepath = "./comps/version.py"
+    with open(filepath) as version_file:
+        (__version__,) = re.findall('__version__ = "(.*)"', version_file.read())
+except Exception as error:
+    assert False, "Error: Could not open '%s' due %s\n" % (filepath, error)
+
+
+if __name__ == "__main__":
+
+    setup(
+        name="opea-comps",
+        author="Intel DCAI Software",
+        version=get_build_version(),
+        author_email="liang1.lv@intel.com, haihao.shen@intel.com, suyue.chen@intel.com",
+        description="Generative AI components",
+        long_description=open("README.md", "r", encoding="utf-8").read(),
+        long_description_content_type="text/markdown",
+        keywords="GenAI",
+        license="Apache 2.0",
+        url="https://github.com/opea-project/GenAIComps",
+        packages=find_packages(
+            include=[
+                "comps.cores",
+                "comps.cores.*",
+            ],
+        ),
+        package_data={"": ["*.yaml", "../*.py"]},
+        include_package_data=True,
+        install_requires=fetch_requirements("requirements.txt"),
+        python_requires=">=3.8.0",
+        classifiers=[
+            "Intended Audience :: Science/Research",
+            "Programming Language :: Python :: 3",
+            "Topic :: Scientific/Engineering :: Artificial Intelligence",
+            "License :: OSI Approved :: Apache Software License",
+        ],
+    )

tests/mega/test_hybrid_service_orchestrator.py

@@ -18,7 +18,7 @@ import unittest
 from comps import RemoteMicroService, ServiceOrchestrator, TextDoc, opea_microservices, register_microservice
 
 
-@register_microservice(name="s1", port=8081, expose_endpoint="/v1/add")
+@register_microservice(name="s1", host="0.0.0.0", port=8086, expose_endpoint="/v1/add")
 async def s1_add(request: TextDoc) -> TextDoc:
     req = request.model_dump_json()
     req_dict = json.loads(req)

tests/mega/test_hybrid_service_orchestrator_with_yaml.py

@@ -18,7 +18,7 @@ import unittest
 from comps import ServiceOrchestratorWithYaml, TextDoc, opea_microservices, register_microservice
 
 
-@register_microservice(name="s1", port=8081, expose_endpoint="/v1/add")
+@register_microservice(name="s1", host="0.0.0.0", port=8085, expose_endpoint="/v1/add")
 async def s1_add(request: TextDoc) -> TextDoc:
     req = request.model_dump_json()
     req_dict = json.loads(req)

tests/mega/test_microservice.py

@@ -20,7 +20,7 @@ from fastapi.testclient import TestClient
 from comps import TextDoc, opea_microservices, register_microservice
 
 
-@register_microservice(name="s1", port=8080, expose_endpoint="/v1/add")
+@register_microservice(name="s1", host="0.0.0.0", port=8080, expose_endpoint="/v1/add")
 async def add(request: TextDoc) -> TextDoc:
     req = request.model_dump_json()
     req_dict = json.loads(req)

tests/mega/test_service_orchestrator.py

@@ -18,7 +18,7 @@ import unittest
 from comps import ServiceOrchestrator, TextDoc, opea_microservices, register_microservice
 
 
-@register_microservice(name="s1", port=8081, expose_endpoint="/v1/add")
+@register_microservice(name="s1", host="0.0.0.0", port=8083, expose_endpoint="/v1/add")
 async def s1_add(request: TextDoc) -> TextDoc:
     req = request.model_dump_json()
     req_dict = json.loads(req)
@@ -27,7 +27,7 @@ async def s1_add(request: TextDoc) -> TextDoc:
     return {"text": text}
 
 
-@register_microservice(name="s2", port=8082, expose_endpoint="/v1/add")
+@register_microservice(name="s2", host="0.0.0.0", port=8084, expose_endpoint="/v1/add")
 async def s2_add(request: TextDoc) -> TextDoc:
     req = request.model_dump_json()
     req_dict = json.loads(req)

tests/mega/test_service_orchestrator_with_yaml.py

@@ -18,7 +18,7 @@ import unittest
 from comps import ServiceOrchestratorWithYaml, TextDoc, opea_microservices, register_microservice
 
 
-@register_microservice(name="s1", port=8081, expose_endpoint="/v1/add")
+@register_microservice(name="s1", host="0.0.0.0", port=8081, expose_endpoint="/v1/add")
 async def s1_add(request: TextDoc) -> TextDoc:
     req = request.model_dump_json()
     req_dict = json.loads(req)
@@ -27,7 +27,7 @@ async def s1_add(request: TextDoc) -> TextDoc:
     return {"text": text}
 
 
-@register_microservice(name="s2", port=8082, expose_endpoint="/v1/add")
+@register_microservice(name="s2", host="0.0.0.0", port=8082, expose_endpoint="/v1/add")
 async def s2_add(request: TextDoc) -> TextDoc:
     req = request.model_dump_json()
     req_dict = json.loads(req)