diff --git a/.github/code_spell_ignore.txt b/.github/code_spell_ignore.txt new file mode 100644 index 000000000..e69de29bb diff --git a/.github/license_template.txt b/.github/license_template.txt new file mode 100644 index 000000000..498754910 --- /dev/null +++ b/.github/license_template.txt @@ -0,0 +1,13 @@ +Copyright (c) 2024 Intel Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..e79b96869 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,15 @@ +## Type of Change + +feature or bug fix or documentation or validation or others + +## Description + +detail description + +## How has this PR been tested? + +how to reproduce the test (including hardware information) + +## Dependency Change? + +any library dependency introduced or removed diff --git a/.github/workflows/code-scan.yml b/.github/workflows/code-scan.yml new file mode 100644 index 000000000..74b98e536 --- /dev/null +++ b/.github/workflows/code-scan.yml @@ -0,0 +1,72 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Code Scan + +on: + pull_request: + branches: [main] + types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped + paths-ignore: + - "**.md" + workflow_dispatch: + +# If there is a new commit, the previous jobs will be canceled +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + DOCKER_CONFIG_NAME: "commonDockerConfig" + REPO_NAME: "code-scan" + REPO_TAG: "1.0" + DOCKER_FILE_NAME: "code-scan" + CONTAINER_NAME: "code-scan" + +jobs: + code-scan: + runs-on: ubuntu-latest + strategy: + matrix: + job_name: ["bandit"] + fail-fast: false + steps: + - name: Checkout out Repo + uses: actions/checkout@v4 + + - name: Docker Build + run: | + docker build -f ${{ github.workspace }}/.github/workflows/docker/${{ env.DOCKER_FILE_NAME }}.dockerfile -t ${{ env.REPO_NAME }}:${{ env.REPO_TAG }} . + + - name: Docker Run + run: | + if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then + docker stop ${{ env.CONTAINER_NAME }} + docker rm -vf ${{ env.CONTAINER_NAME }} || true + fi + docker run -dit --memory="4g" --memory-reservation="1g" --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} --shm-size="1g" \ + -v ${{ github.workspace }}:/GenAIComps \ + ${{ env.REPO_NAME }}:${{ env.REPO_TAG }} + + - name: Code scan check + run: | + docker exec ${{ env.CONTAINER_NAME }} \ + bash -c "bash /GenAIComps/.github/workflows/scripts/codeScan/${{ matrix.job_name }}.sh" + + - name: Publish pipeline artifact + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.job_name }} + path: ${{ github.workspace }}/.github/workflows/scripts/codeScan/${{ matrix.job_name }}.* diff --git a/.github/workflows/docker/code-scan.dockerfile b/.github/workflows/docker/code-scan.dockerfile new file mode 100644 index 000000000..6922e3a07 --- /dev/null +++ b/.github/workflows/docker/code-scan.dockerfile @@ -0,0 +1,35 @@ +# +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG UBUNTU_VER=22.04 +FROM ubuntu:${UBUNTU_VER} as devel + +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + aspell \ + aspell-en \ + build-essential \ + python3 \ + python3-pip \ + python3-dev \ + python3-distutils \ + wget + +RUN ln -sf $(which python3) /usr/bin/python + +RUN python -m pip install --no-cache-dir bandit + +WORKDIR / diff --git a/.github/workflows/scripts/change_color b/.github/workflows/scripts/change_color new file mode 100644 index 000000000..b2ea724fa --- /dev/null +++ b/.github/workflows/scripts/change_color @@ -0,0 +1,80 @@ +#!/bin/bash + +# -------------- general approach start---------------- + +# 1. import this file: +# source path/change_color.sh +# 2. use COLOR/BG: +# $VARIABLE_NAME && out_put_content && $RESET +# 3. COLOR + BG: +# $COLOR/BG_VARIABLE_NAME && $BG/COLOR_VARIABLE_NAME && out_put_content && $RESET +# 4. custom +# abbreviation(change number) +# txt number range (30, 37) +# bg number range (40, 47) +# special effects number range (1, 7) +# echo -en \\E[number1 + ; + number2 + ; + number3 + m" +# e.g - BG_GRAY+LIGHT_RED = "echo -en \\E[47;31m" + +# -------------- general approach end----------------== + +# general setting +# ------------- light_color start---------------- +# black +LIGHT_BLACK="echo -en \\E[30m" +# red +LIGHT_RED="echo -en \\E[31m" +# green +LIGHT_GREEN="echo -en \\E[32m" +# yellow +LIGHT_YELLOW="echo -en \\E[33m" +# blue +LIGHT_BLUE="echo -en \\E[34m" +# purple +LIGHT_PURPLE="echo -en \\E[35m" +# cyan +LIGHT_CYAN="echo -en \\E[36m" +# gray +LIGHT_GRAY="echo -en \\E[37m" +# ------------- light_color end---------------- + +# ------------- bold_color start---------------- +# black +BOLD_BLACK="echo -en \\E[1;30m" +# red +BOLD_RED="echo -en \\E[1;31m" +# green +BOLD_GREEN="echo -en \\E[1;32m" +# yellow +BOLD_YELLOW="echo -en \\E[1;33m" +# blue +BOLD_BLUE="echo -en \\E[1;34m" +# purple +BOLD_PURPLE="echo -en \\E[1;35m" +# cyan +BOLD_CYAN="echo -en \\E[1;36m" +# gray +BOLD_GRAY="echo -en \\E[1;37m" +# ------------- bold_color end---------------- + +# ------------- background_color start---------------- +# black +BG_BLACK="echo -en \\E[40m" +# red +BG_RED="echo -en \\E[41m" +# green +BG_GREEN="echo -en \\E[42m" +# yellow +BG_YELLOW="echo -en \\E[43m" +# blue +BG_BLUE="echo -en \\E[44m" +# purple +BG_PURPLE="echo -en \\E[45m" +# cyan +BG_CYAN="echo -en \\E[46m" +# gray +BG_GRAY="echo -en \\E[47m" +# ------------- background_color end---------------- + +# close +RESET="echo -en \\E[0m" diff --git a/.github/workflows/scripts/codeScan/bandit.sh b/.github/workflows/scripts/codeScan/bandit.sh new file mode 100644 index 000000000..e680e22f8 --- /dev/null +++ b/.github/workflows/scripts/codeScan/bandit.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /GenAIComps/.github/workflows/scripts/change_color +pip install bandit==1.7.8 +log_dir=/GenAIComps/.github/workflows/scripts/codeScan +python -m bandit -r -lll -iii /GenAIComps >${log_dir}/bandit.log +exit_code=$? + +$BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" +cat ${log_dir}/bandit.log +$BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET + +if [ ${exit_code} -ne 0 ]; then + $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Bandit error details." && $RESET + exit 1 +fi + +$BOLD_PURPLE && echo "Congratulations, Bandit check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET +exit 0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..f0bbc3dd4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,109 @@ +ci: + autofix_prs: true + autoupdate_schedule: quarterly + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: end-of-file-fixer + files: (.*\.(py|md|rst|yaml|yml|json|ts|js|html|svelte|sh))$ + - id: check-json + - id: check-yaml + - id: debug-statements + - id: requirements-txt-fixer + - id: trailing-whitespace + files: (.*\.(py|rst|cmake|yaml|yml|json|ts|js|html|svelte|sh))$ + + - repo: https://github.com/Lucas-C/pre-commit-hooks + rev: v1.5.5 + hooks: + - id: insert-license + files: (.*\.(py|yaml|yml|sh))$ + args: + [ + --license-filepath=.github/license_template.txt, + --use-current-year, + --detect-license-in-X-top-lines=40, + --skip-license-insertion-comment=Copyright, + ] + - id: insert-license + files: (.*\.(ts|js))$ + args: + [ + --license-filepath=.github/license_template.txt, + --use-current-year, + --detect-license-in-X-top-lines=40, + --skip-license-insertion-comment=Copyright, + --comment-style=//, + ] + - id: insert-license + files: (.*\.(html|svelte))$ + args: + [ + --license-filepath=.github/license_template.txt, + --use-current-year, + --detect-license-in-X-top-lines=40, + --skip-license-insertion-comment=Copyright, + --comment-style=, + ] + + - repo: https://github.com/asottile/yesqa + rev: v1.5.0 + hooks: + - id: yesqa + name: Unused noqa + + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + + - repo: https://github.com/PyCQA/docformatter + rev: v1.7.5 + hooks: + - id: docformatter + args: [ + --in-place, + --wrap-summaries=0, # 0 means disable wrap + --wrap-descriptions=0, # 0 means disable wrap + --black, + --style=google, + ] + + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v4.0.0-alpha.8" # Use the sha / tag you want to point at + hooks: + - id: prettier + args: [--print-width=120] + types_or: [yaml, markdown, html, css, scss, javascript, json, ts, shell, sh] + additional_dependencies: + - prettier@3.2.5 + + - repo: https://github.com/psf/black.git + rev: 24.3.0 + hooks: + - id: black + files: (.*\.py)$ + + - repo: https://github.com/asottile/blacken-docs + rev: 1.16.0 + hooks: + - id: blacken-docs + args: [--line-length=120, --skip-errors] + additional_dependencies: + - black==24.3.0 + + - repo: https://github.com/codespell-project/codespell + rev: v2.2.6 + hooks: + - id: codespell + args: [-w] + additional_dependencies: + - tomli + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.5 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix, --no-cache] diff --git a/GenAIComps/__init__.py b/GenAIComps/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/__init__.py +++ b/GenAIComps/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/asr/__init__.py b/GenAIComps/asr/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/asr/__init__.py +++ b/GenAIComps/asr/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/dataprep/langchain/__init__.py b/GenAIComps/dataprep/langchain/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/dataprep/langchain/__init__.py +++ b/GenAIComps/dataprep/langchain/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/dataprep/langchain/ingest.py b/GenAIComps/dataprep/langchain/ingest.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/dataprep/langchain/ingest.py +++ b/GenAIComps/dataprep/langchain/ingest.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/embedding/__init__.py b/GenAIComps/embedding/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/embedding/__init__.py +++ b/GenAIComps/embedding/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/embedding/langchain/__init__.py b/GenAIComps/embedding/langchain/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/embedding/langchain/__init__.py +++ b/GenAIComps/embedding/langchain/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/embedding/langchain/embedding.py b/GenAIComps/embedding/langchain/embedding.py index a4c025748..b0c2b9695 100644 --- a/GenAIComps/embedding/langchain/embedding.py +++ b/GenAIComps/embedding/langchain/embedding.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os from fastapi import APIRouter, FastAPI, Request diff --git a/GenAIComps/guardrails/__init__.py b/GenAIComps/guardrails/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/guardrails/__init__.py +++ b/GenAIComps/guardrails/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/guardrails/guardrails.py b/GenAIComps/guardrails/guardrails.py index d08588d25..76784f11e 100644 --- a/GenAIComps/guardrails/guardrails.py +++ b/GenAIComps/guardrails/guardrails.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os from fastapi import APIRouter, FastAPI, Request diff --git a/GenAIComps/mega/__init__.py b/GenAIComps/mega/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/mega/__init__.py +++ b/GenAIComps/mega/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/mega/api.py b/GenAIComps/mega/api.py index eaa14cd1e..252275da4 100644 --- a/GenAIComps/mega/api.py +++ b/GenAIComps/mega/api.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from typing import Dict, List diff --git a/GenAIComps/mega/async_loop.py b/GenAIComps/mega/async_loop.py index 206e185a2..426e77f97 100644 --- a/GenAIComps/mega/async_loop.py +++ b/GenAIComps/mega/async_loop.py @@ -1,11 +1,25 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import asyncio import signal import time -from typing import Optional, Dict +from typing import Dict, Optional + from logger import CustomLogger from utils import check_ports_availability - # Define the signals that will be handled by the AsyncLoop class HANDLED_SIGNALS = ( signal.SIGINT, # Unix signal 2. Sent by Ctrl+C. @@ -15,38 +29,38 @@ HANDLED_SIGNALS = ( class AsyncLoop: - """ - Async loop to run a microservice asynchronously. + """Async loop to run a microservice asynchronously. + This class is designed to handle the running of a microservice in an asynchronous manner. It sets up an event loop and handles certain signals to gracefully stop the service. """ def __init__(self, args: Optional[Dict] = None) -> None: - """ - Initialize the AsyncLoop class. + """Initialize the AsyncLoop class. + This method sets up the initial state of the AsyncLoop, including setting up the event loop and signal handlers. """ self.args = args - if args.get('name', None): + if args.get("name", None): self.name = f'{args.get("name")}/{self.__class__.__name__}' else: self.name = self.__class__.__name__ - self.protocol = args.get('protocol', 'http') - self.host = args.get('host', 'localhost') - self.port = args.get('port', 8080) - self.quiet_error = args.get('quiet_error', False) + self.protocol = args.get("protocol", "http") + self.host = args.get("host", "localhost") + self.port = args.get("port", 8080) + self.quiet_error = args.get("quiet_error", False) self.logger = CustomLogger(self.name) self._loop = asyncio.new_event_loop() asyncio.set_event_loop(self._loop) self.is_cancel = asyncio.Event() - self.logger.info(f'Setting signal handlers') + self.logger.info("Setting signal handlers") def _cancel(signum, frame): - """ - Signal handler for the AsyncLoop class. + """Signal handler for the AsyncLoop class. + This method is called when a signal is received. It sets the is_cancel event to stop the event loop. """ - self.logger.info(f'Received signal {signum}') + self.logger.info(f"Received signal {signum}") self.is_cancel.set(), for sig in HANDLED_SIGNALS: @@ -56,15 +70,15 @@ class AsyncLoop: self._loop.run_until_complete(self.async_setup()) def run_forever(self): - """ - Running method to block the main thread. + """Running method to block the main thread. + This method runs the event loop until a Future is done. It is designed to be called in the main thread to keep it busy. """ self._loop.run_until_complete(self._loop_body()) def teardown(self): - """ - Call async_teardown() and stop and close the event loop. + """Call async_teardown() and stop and close the event loop. + This method is responsible for tearing down the event loop. It first calls the async_teardown method to perform any necessary cleanup, then it stops and closes the event loop. It also logs the duration of the event loop. """ @@ -75,105 +89,97 @@ class AsyncLoop: self.logger.info(f"Async loop is tore down. Duration: {self._stop_time - self._start_time}") def _get_server(self): - """ - Get the server instance based on the protocol. + """Get the server instance based on the protocol. + This method currently only supports HTTP services. It creates an instance of the HTTPService class with the - necessary arguments. + necessary arguments. In the future, it will also support gRPC services. """ - if self.protocol.lower() == 'http': + if self.protocol.lower() == "http": from http_service import HTTPService - runtime_args = self.args.get('runtime_args', None) - runtime_args['protocol'] = self.protocol - runtime_args['host'] = self.host - runtime_args['port'] = self.port + runtime_args = self.args.get("runtime_args", None) + runtime_args["protocol"] = self.protocol + runtime_args["host"] = self.host + runtime_args["port"] = self.port return HTTPService( - uvicorn_kwargs=self.args.get('uvicorn_kwargs', None), + uvicorn_kwargs=self.args.get("uvicorn_kwargs", None), runtime_args=runtime_args, - cors=self.args.get('cors', None), + cors=self.args.get("cors", None), ) async def async_setup(self): - """ - The async method setup the runtime. + """The async method setup the runtime. + This method is responsible for setting up the server. It first checks if the port is available, then it gets the server instance and initializes it. """ if not (check_ports_availability(self.host, self.port)): - raise RuntimeError(f'port:{self.port}') + raise RuntimeError(f"port:{self.port}") self.server = self._get_server() await self.server.initialize_server() async def async_run_forever(self): - """ - Running method of the server. - """ + """Running method of the server.""" await self.server.execute_server() async def async_teardown(self): - """ - Terminate the server. - """ + """Terminate the server.""" await self.server.terminate_server() async def _wait_for_cancel(self): - """ - Wait for the cancellation event. + """Wait for the cancellation event. + This method waits for the is_cancel event to be set. If the server has a _should_exit attribute, it will also wait for that to be True. Once either of these conditions is met, it will call the async_teardown method. """ - if isinstance(self.is_cancel, asyncio.Event) and not hasattr( - self.server, '_should_exit' - ): + if isinstance(self.is_cancel, asyncio.Event) and not hasattr(self.server, "_should_exit"): await self.is_cancel.wait() else: - while not self.is_cancel.is_set() and not getattr( - self.server, '_should_exit', False - ): + while not self.is_cancel.is_set() and not getattr(self.server, "_should_exit", False): await asyncio.sleep(0.1) await self.async_teardown() async def _loop_body(self): - """ - The main body of the event loop. + """The main body of the event loop. + This method runs the async_run_forever and _wait_for_cancel methods concurrently. If a CancelledError is raised, it logs a warning message. """ try: await asyncio.gather(self.async_run_forever(), self._wait_for_cancel()) except asyncio.CancelledError: - self.logger.warning('received terminate ctrl message from main process') + self.logger.warning("received terminate ctrl message from main process") def __enter__(self): - """ - Enter method for the context manager. + """Enter method for the context manager. + This method simply returns the instance itself. """ return self def __exit__(self, exc_type, exc_val, exc_tb): - """ - Exit method for the context manager. + """Exit method for the context manager. + This method handles any exceptions that occurred within the context. If a KeyboardInterrupt was raised, it logs an info message. If any other exception was raised, it logs an error message. Finally, it attempts to call the teardown method. If an OSError is raised during this, it is ignored. Any other exceptions are logged as errors. """ if exc_type == KeyboardInterrupt: - self.logger.info(f'{self!r} is interrupted by user') + self.logger.info(f"{self!r} is interrupted by user") elif exc_type and issubclass(exc_type, Exception): self.logger.error( ( - f'{exc_val!r} during {self.run_forever!r}' - + f'\n add "--quiet-error" to suppress the exception details' + f"{exc_val!r} during {self.run_forever!r}" + + '\n add "--quiet-error" to suppress the exception details' if not self.quiet_error - else '' + else "" ), exc_info=not self.quiet_error, ) else: - self.logger.info(f'{self!r} is ended') + self.logger.info(f"{self!r} is ended") return True diff --git a/GenAIComps/mega/base_service.py b/GenAIComps/mega/base_service.py index 932a93289..aabd9e412 100644 --- a/GenAIComps/mega/base_service.py +++ b/GenAIComps/mega/base_service.py @@ -1,26 +1,36 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import abc -from logger import CustomLogger from types import SimpleNamespace -from typing import Dict, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, Optional + +from logger import CustomLogger + +__all__ = ["BaseService"] -__all__ = ['BaseServer'] - - -class BaseService(): - """ - BaseService creates a HTTP/gRPC server as a microservice. - """ +class BaseService: + """BaseService creates an HTTP/gRPC server as a microservice.""" def __init__( self, - name: Optional[str] = 'Base service', + name: Optional[str] = "Base service", runtime_args: Optional[Dict] = None, **kwargs, ): - """ - Initialize the BaseService with a name, runtime arguments, and any additional arguments. - """ + """Initialize the BaseService with a name, runtime arguments, and any additional arguments.""" self.name = name self.runtime_args = runtime_args self._process_runtime_args() @@ -30,78 +40,62 @@ class BaseService(): self.server = None def _process_runtime_args(self): - """ - Process the runtime arguments to ensure they are in the correct format. - """ - _runtime_args = ( - self.runtime_args - if isinstance(self.runtime_args, dict) - else vars(self.runtime_args or {}) - ) + """Process the runtime arguments to ensure they are in the correct format.""" + _runtime_args = self.runtime_args if isinstance(self.runtime_args, dict) else vars(self.runtime_args or {}) self.runtime_args = SimpleNamespace(**_runtime_args) @property def primary_port(self): - """ - Gets the first port of the port list argument. + """Gets the first port of the port list argument. + :return: The first port to be exposed """ - return ( - self.runtime_args.port[0] - if isinstance(self.runtime_args.port, list) - else self.runtime_args.port - ) + return self.runtime_args.port[0] if isinstance(self.runtime_args.port, list) else self.runtime_args.port @property def all_ports(self): - """ - Gets all the list of ports from the runtime_args as a list. + """Gets all the list of ports from the runtime_args as a list. + :return: The lists of ports to be exposed """ - return ( - self.runtime_args.port - if isinstance(self.runtime_args.port, list) - else [self.runtime_args.port] - ) + return self.runtime_args.port if isinstance(self.runtime_args.port, list) else [self.runtime_args.port] @property def protocols(self): - """ - Gets all the list of protocols from the runtime_args as a list. + """Gets all the list of protocols from the runtime_args as a list. + :return: The lists of protocols to be exposed """ return ( - self.runtime_args.protocol - if isinstance(self.runtime_args.protocol, list) - else [self.runtime_args.protocol] + self.runtime_args.protocol if isinstance(self.runtime_args.protocol, list) else [self.runtime_args.protocol] ) @property def host_address(self): - """ - Gets the host from the runtime_args - :return: The host where to bind the gateway - """ - return self.runtime_args.host or '127.0.0.1' + """Gets the host from the runtime_args + :return: The host where to bind the gateway.""" + return self.runtime_args.host or "127.0.0.1" @abc.abstractmethod async def initialize_server(self): - """ - Abstract method to setup the server. This should be implemented in the child class. + """Abstract method to setup the server. + + This should be implemented in the child class. """ ... @abc.abstractmethod async def execute_server(self): - """ - Abstract method to run the server indefinitely. This should be implemented in the child class. + """Abstract method to run the server indefinitely. + + This should be implemented in the child class. """ ... @abc.abstractmethod async def terminate_server(self): - """ - Abstract method to shutdown the server and free other allocated resources, e.g, health check service, etc. + """Abstract method to shutdown the server and free other allocated resources, e.g, health check service, etc. + This should be implemented in the child class. """ ... @@ -109,31 +103,31 @@ class BaseService(): @staticmethod def check_server_readiness( ctrl_address: str, - protocol: Optional[str] = 'http', + protocol: Optional[str] = "http", **kwargs, ) -> bool: - """ - Check if server status is ready. + """Check if server status is ready. + :param ctrl_address: the address where the control request needs to be sent. :param protocol: protocol of the service. :param kwargs: extra keyword arguments. :return: True if status is ready else False. """ from http_service import HTTPService + res = False - if protocol is None or protocol == 'http': + if protocol is None or protocol == "http": res = HTTPService.check_readiness(ctrl_address) return res - @staticmethod async def async_check_server_readiness( ctrl_address: str, - protocol: Optional[str] = 'grpc', + protocol: Optional[str] = "grpc", **kwargs, ) -> bool: - """ - Asynchronously check if server status is ready. + """Asynchronously check if server status is ready. + :param ctrl_address: the address where the control request needs to be sent. :param protocol: protocol of the service. :param kwargs: extra keyword arguments. @@ -142,8 +136,6 @@ class BaseService(): if TYPE_CHECKING: from http_service import HTTPService res = False - if protocol is None or protocol == 'http': + if protocol is None or protocol == "http": res = await HTTPService.async_check_readiness(ctrl_address) return res - - diff --git a/GenAIComps/mega/composer.py b/GenAIComps/mega/composer.py index 00d2197a4..8bbfee991 100644 --- a/GenAIComps/mega/composer.py +++ b/GenAIComps/mega/composer.py @@ -1 +1,15 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # compose microservices to build metaservice diff --git a/GenAIComps/mega/config.py b/GenAIComps/mega/config.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/mega/config.py +++ b/GenAIComps/mega/config.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/mega/logger.py b/GenAIComps/mega/logger.py index 3502c69a6..9a13bd0f4 100644 --- a/GenAIComps/mega/logger.py +++ b/GenAIComps/mega/logger.py @@ -1,42 +1,52 @@ -import logging +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import functools +import logging from typing import Callable class CustomLogger: - """ - A custom logger class that adds additional logging levels. - """ + """A custom logger class that adds additional logging levels.""" def __init__(self, name: str = None): - """ - Initialize the logger with a name and custom levels. - """ - name = 'GenAIComps' if not name else name + """Initialize the logger with a name and custom levels.""" + name = "GenAIComps" if not name else name self.logger = logging.getLogger(name) # Define custom log levels log_config = { - 'DEBUG': 10, - 'INFO': 20, - 'TRAIN': 21, - 'EVAL': 22, - 'WARNING': 30, - 'ERROR': 40, - 'CRITICAL': 50, - 'EXCEPTION': 100, + "DEBUG": 10, + "INFO": 20, + "TRAIN": 21, + "EVAL": 22, + "WARNING": 30, + "ERROR": 40, + "CRITICAL": 50, + "EXCEPTION": 100, } # Add custom levels to logger for key, level in log_config.items(): logging.addLevelName(level, key) - if key == 'EXCEPTION': + if key == "EXCEPTION": self.__dict__[key.lower()] = self.logger.exception else: self.__dict__[key.lower()] = functools.partial(self.log_message, level) # Set up log format and handler - self.format = logging.Formatter(fmt='[%(asctime)-15s] [%(levelname)8s] - %(message)s') + self.format = logging.Formatter(fmt="[%(asctime)-15s] [%(levelname)8s] - %(message)s") self.handler = logging.StreamHandler() self.handler.setFormatter(self.format) @@ -46,8 +56,7 @@ class CustomLogger: self.logger.propagate = False def log_message(self, log_level: str, msg: str): - """ - Log a message at a given level. + """Log a message at a given level. :param log_level: The level at which to log the message. :param msg: The message to log. @@ -65,4 +74,4 @@ class CustomLogger: exception: Callable[[str], None] -logger = CustomLogger() \ No newline at end of file +logger = CustomLogger() diff --git a/GenAIComps/mega/service.py b/GenAIComps/mega/service.py index b6f2cafa7..73190800f 100644 --- a/GenAIComps/mega/service.py +++ b/GenAIComps/mega/service.py @@ -1,12 +1,28 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from typing import Optional + from base_service import BaseService from fastapi import FastAPI from uvicorn import Config, Server class HTTPService(BaseService): - """ - FastAPI HTTP service based on BaseService class. This property should return a fastapi app. + """FastAPI HTTP service based on BaseService class. + + This property should return a fastapi app. """ def __init__( @@ -15,10 +31,10 @@ class HTTPService(BaseService): cors: Optional[bool] = False, **kwargs, ): - """ - Initialize the HTTPService + """Initialize the HTTPService :param uvicorn_kwargs: Dictionary of kwargs arguments that will be passed to Uvicorn server when starting the server :param cors: If set, a CORS middleware is added to FastAPI frontend to allow cross-origin access. + :param kwargs: keyword args """ super().__init__(**kwargs) @@ -28,61 +44,50 @@ class HTTPService(BaseService): @property def app(self): - """ - Get the default base API app for Server - :return: Return a FastAPI app for the default HTTPGateway - """ + """Get the default base API app for Server + :return: Return a FastAPI app for the default HTTPGateway.""" return self._app - + def _create_app(self): - """ - Create a FastAPI application. + """Create a FastAPI application. + :return: a FastAPI application. """ app = FastAPI(title=self.title, description=self.description) if self.cors: from fastapi.middleware.cors import CORSMiddleware + app.add_middleware( CORSMiddleware, - allow_origins=['*'], + allow_origins=["*"], allow_credentials=True, - allow_methods=['*'], - allow_headers=['*'], + allow_methods=["*"], + allow_headers=["*"], ) - self.logger.info('CORS is enabled.') + self.logger.info("CORS is enabled.") @app.get( - path='/v1/health_check', - summary='Get the status of GenAI microservice', - tags=['Debug'], + path="/v1/health_check", + summary="Get the status of GenAI microservice", + tags=["Debug"], ) async def _health_check(): - """ - Get the health status of this GenAI microservice. - """ - return { - 'Service Title': self.title, - 'Service Description': self.description - } - + """Get the health status of this GenAI microservice.""" + return {"Service Title": self.title, "Service Description": self.description} + return app async def initialize_server(self): - """ - Initialize and return HTTP server - """ - self.logger.info(f'Setting up HTTP server') - + """Initialize and return HTTP server.""" + self.logger.info("Setting up HTTP server") class UviServer(Server): - """ - The uvicorn server. - """ + """The uvicorn server.""" async def setup_server(self, sockets=None): - """ - Setup uvicorn server. + """Setup uvicorn server. + :param sockets: sockets of server. """ config = self.config @@ -94,52 +99,43 @@ class HTTPService(BaseService): return async def start_server(self, **kwargs): - """ - Start the server. + """Start the server. + :param kwargs: keyword arguments """ await self.main_loop() app = self.app - + self.server = UviServer( config=Config( app=app, host=self.host_address, port=self.primary_port, - log_level='info', + log_level="info", **self.uvicorn_kwargs, ) ) - self.logger.info(f'Uvicorn server setup on port {self.primary_port}') + self.logger.info(f"Uvicorn server setup on port {self.primary_port}") await self.server.setup_server() - self.logger.info(f'HTTP server setup successful') + self.logger.info("HTTP server setup successful") async def execute_server(self): - """ - Run the HTTP server indefinitely. - """ + """Run the HTTP server indefinitely.""" await self.server.start_server() - + async def terminate_server(self): - """ - Terminate the HTTP server and free resources allocated when setting up the server. - """ - self.logger.info('Initiating server termination') + """Terminate the HTTP server and free resources allocated when setting up the server.""" + self.logger.info("Initiating server termination") await super().shutdown() self.server.should_exit = True await self.server.shutdown() - self.logger.info('Server termination completed') + self.logger.info("Server termination completed") @staticmethod - def check_server_readiness( - ctrl_address: str, - timeout: float = 1.0, - logger=None, - **kwargs - ) -> bool: - """ - Check if server status is ready. + def check_server_readiness(ctrl_address: str, timeout: float = 1.0, logger=None, **kwargs) -> bool: + """Check if server status is ready. + :param ctrl_address: the address where the control request needs to be sent :param timeout: timeout of the health check in seconds :param logger: Customized Logger to be used @@ -150,23 +146,18 @@ class HTTPService(BaseService): from http import HTTPStatus try: - conn = urllib.request.urlopen(url=f'http://{ctrl_address}', timeout=timeout) + conn = urllib.request.urlopen(url=f"http://{ctrl_address}", timeout=timeout) return conn.code == HTTPStatus.OK except Exception as exc: if logger: - logger.info(f'Exception: {exc}') + logger.info(f"Exception: {exc}") return False @staticmethod - async def async_check_server_readiness( - ctrl_address: str, - timeout: float = 1.0, - logger=None, - **kwargs - ) -> bool: - """ - Asynchronously check if server status is ready. + async def async_check_server_readiness(ctrl_address: str, timeout: float = 1.0, logger=None, **kwargs) -> bool: + """Asynchronously check if server status is ready. + :param ctrl_address: the address where the control request needs to be sent :param timeout: timeout of the health check in seconds :param logger: Customized Logger to be used @@ -174,4 +165,3 @@ class HTTPService(BaseService): :return: True if status is ready else False. """ return HTTPService.check_server_readiness(ctrl_address, timeout, logger=logger) - diff --git a/GenAIComps/mega/utils.py b/GenAIComps/mega/utils.py index 39baa014c..ab39278b6 100644 --- a/GenAIComps/mega/utils.py +++ b/GenAIComps/mega/utils.py @@ -1,10 +1,23 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from socket import AF_INET, SOCK_STREAM, socket from typing import List, Union def _is_port_free(host: str, port: int) -> bool: - """ - Check if a given port on a host is free. + """Check if a given port on a host is free. :param host: The host to check. :param port: The port to check. @@ -15,8 +28,7 @@ def _is_port_free(host: str, port: int) -> bool: def check_ports_availability(host: Union[str, List[str]], port: Union[int, List[int]]) -> bool: - """ - Check if one or more ports on one or more hosts are free. + """Check if one or more ports on one or more hosts are free. :param host: The host(s) to check. :param port: The port(s) to check. @@ -25,4 +37,4 @@ def check_ports_availability(host: Union[str, List[str]], port: Union[int, List[ hosts = [host] if isinstance(host, str) else host ports = [port] if isinstance(port, int) else port - return all(_is_port_free(h, p) for h in hosts for p in ports) \ No newline at end of file + return all(_is_port_free(h, p) for h in hosts for p in ports) diff --git a/GenAIComps/rerank/__init__.py b/GenAIComps/rerank/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/rerank/__init__.py +++ b/GenAIComps/rerank/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/rerank/langchain/__init__.py b/GenAIComps/rerank/langchain/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/rerank/langchain/__init__.py +++ b/GenAIComps/rerank/langchain/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/retrieval/__init__.py b/GenAIComps/retrieval/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/retrieval/__init__.py +++ b/GenAIComps/retrieval/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/retrieval/langchain/__init__.py b/GenAIComps/retrieval/langchain/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/retrieval/langchain/__init__.py +++ b/GenAIComps/retrieval/langchain/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/tts/__init__.py b/GenAIComps/tts/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/tts/__init__.py +++ b/GenAIComps/tts/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/vectordb/__init__.py b/GenAIComps/vectordb/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/vectordb/__init__.py +++ b/GenAIComps/vectordb/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/vectordb/langchain/chroma/__init__.py b/GenAIComps/vectordb/langchain/chroma/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/vectordb/langchain/chroma/__init__.py +++ b/GenAIComps/vectordb/langchain/chroma/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/vectordb/langchain/qdrant/__init__.py b/GenAIComps/vectordb/langchain/qdrant/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/vectordb/langchain/qdrant/__init__.py +++ b/GenAIComps/vectordb/langchain/qdrant/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/vectordb/langchain/redis/__init__.py b/GenAIComps/vectordb/langchain/redis/__init__.py index e69de29bb..28f108cb6 100644 --- a/GenAIComps/vectordb/langchain/redis/__init__.py +++ b/GenAIComps/vectordb/langchain/redis/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/vectordb/langchain/redis/config.py b/GenAIComps/vectordb/langchain/redis/config.py index bae3fc8ae..f03a98f0d 100644 --- a/GenAIComps/vectordb/langchain/redis/config.py +++ b/GenAIComps/vectordb/langchain/redis/config.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os # Embedding model diff --git a/GenAIComps/vectordb/langchain/redis/docker-compose-redis.yml b/GenAIComps/vectordb/langchain/redis/docker-compose-redis.yml index fc0d3bd46..5b8ec341e 100644 --- a/GenAIComps/vectordb/langchain/redis/docker-compose-redis.yml +++ b/GenAIComps/vectordb/langchain/redis/docker-compose-redis.yml @@ -1,3 +1,17 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + version: "3" services: redis-vector-db: diff --git a/GenAIComps/vectordb/langchain/redis/schema.yml b/GenAIComps/vectordb/langchain/redis/schema.yml index 806c5bed4..9d7254752 100644 --- a/GenAIComps/vectordb/langchain/redis/schema.yml +++ b/GenAIComps/vectordb/langchain/redis/schema.yml @@ -1,3 +1,17 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + text: - name: content - name: source diff --git a/GenAIComps/vectordb/langchain/redis/schema_dim_1024.yml b/GenAIComps/vectordb/langchain/redis/schema_dim_1024.yml index db17662f6..8bbb67d9a 100644 --- a/GenAIComps/vectordb/langchain/redis/schema_dim_1024.yml +++ b/GenAIComps/vectordb/langchain/redis/schema_dim_1024.yml @@ -1,3 +1,17 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + text: - name: content - name: source diff --git a/GenAIComps/vectordb/langchain/redis/schema_dim_768.yml b/GenAIComps/vectordb/langchain/redis/schema_dim_768.yml index 74c0474ea..13484a88f 100644 --- a/GenAIComps/vectordb/langchain/redis/schema_dim_768.yml +++ b/GenAIComps/vectordb/langchain/redis/schema_dim_768.yml @@ -1,3 +1,17 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + text: - name: content - name: source diff --git a/GenAIComps/vectordb/langchain/redis/schema_lcdocs_dim_768.yml b/GenAIComps/vectordb/langchain/redis/schema_lcdocs_dim_768.yml index 81d00f761..f0ec4e030 100644 --- a/GenAIComps/vectordb/langchain/redis/schema_lcdocs_dim_768.yml +++ b/GenAIComps/vectordb/langchain/redis/schema_lcdocs_dim_768.yml @@ -1,3 +1,17 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + text: - name: content - name: changefreq diff --git a/README.md b/README.md index 0cd7239b0..d1e05a460 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ # GenAIComps + GenAI components at micro-service level; GenAI service composer to create mega-service diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..c6087c97f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,108 @@ +[tool.isort] +profile = "black" +line_length = 120 +extend_skip_glob = ["**/__init__.py"] + + +[tool.black] +line-length = 120 + + +[tool.codespell] +skip = '*.po,*.js,*.map,*.js.map,*.css.map,*.json' +count = '' +quiet-level = 3 +ignore-words = ".github/code_spell_ignore.txt" + + +[tool.ruff] +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +# Same as Black. +line-length = 120 +indent-width = 4 + +# Assume Python 3.10 +target-version = "py310" + +[tool.ruff.lint] +# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. +# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or +# McCabe complexity (`C901`) by default. +select = ["E4", "E7", "E9", "F"] +ignore = [ + "E402", # Module level import not at top of file + "E501", # Line too long (121 > 120 characters) + "E721", # Do not compare types, use isinstance() + "E722", # Do not use bare except + "E731", # Do not assign a lambda expression, use a def + "E741", # Do not use variables named ‘l’, ‘O’, or ‘I’ + "F401", # {name} imported but unused + "F403", # from {name} import * used; unable to detect undefined names + "F405", # {name} may be undefined, or defined from star imports + "F841", # Local variable is assigned to but never used{name} +] + +# Allow fix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +ignore-init-module-imports = true + +[tool.ruff.format] +# Like Black, use double quotes for strings. +quote-style = "double" + +# Like Black, indent with spaces, rather than tabs. +indent-style = "space" + +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false + +# Like Black, automatically detect the appropriate line ending. +line-ending = "auto" + +# Enable auto-formatting of code examples in docstrings. Markdown, +# reStructuredText code/literal blocks and doctests are all supported. +# +# This is currently disabled by default, but it is planned for this +# to be opt-out in the future. +docstring-code-format = false + +# Set the line length limit used when formatting code snippets in +# docstrings. +# +# This only has an effect when the `docstring-code-format` setting is +# enabled. +docstring-code-line-length = "dynamic" diff --git a/tests/uts/test_asr.py b/tests/uts/test_asr.py new file mode 100644 index 000000000..28f108cb6 --- /dev/null +++ b/tests/uts/test_asr.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/workflows/test_asr_comps.sh b/tests/workflows/test_asr_comps.sh new file mode 100644 index 000000000..28f108cb6 --- /dev/null +++ b/tests/workflows/test_asr_comps.sh @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.