From 06117077b0b1d1a775a9f66e89fb0afd04fd6015 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Sat, 14 Sep 2024 18:25:01 +0800 Subject: [PATCH] Add hyperlinks picture paths validation. (#800) Signed-off-by: ZePan110 Signed-off-by: chensuyue --- .github/workflows/pr-path-detection.yml | 134 ++++++++++++++++++ .github/workflows/pr-path_detection.yml | 44 ------ .../docker_compose/intel/cpu/xeon/README.md | 2 +- Translation/README.md | 2 +- docker_images_list.md | 19 ++- 5 files changed, 145 insertions(+), 56 deletions(-) create mode 100644 .github/workflows/pr-path-detection.yml delete mode 100644 .github/workflows/pr-path_detection.yml diff --git a/.github/workflows/pr-path-detection.yml b/.github/workflows/pr-path-detection.yml new file mode 100644 index 000000000..6ad53c0aa --- /dev/null +++ b/.github/workflows/pr-path-detection.yml @@ -0,0 +1,134 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: Check Paths and Hyperlinks + +on: + pull_request: + branches: [main] + types: [opened, reopened, ready_for_review, synchronize] + +jobs: + check-dockerfile-paths: + runs-on: ubuntu-latest + steps: + - name: Clean Up Working Directory + run: sudo rm -rf ${{github.workspace}}/* + + - name: Checkout Repo GenAIExamples + uses: actions/checkout@v4 + + - name: Clone Repo GenAIComps + run: | + cd .. + git clone https://github.com/opea-project/GenAIComps.git + + - name: Check for Missing Dockerfile Paths in GenAIComps + run: | + cd ${{github.workspace}} + miss="FALSE" + while IFS=: read -r file line content; do + dockerfile_path=$(echo "$content" | awk -F '-f ' '{print $2}' | awk '{print $1}') + if [[ ! -f "../GenAIComps/${dockerfile_path}" ]]; then + miss="TRUE" + echo "Missing Dockerfile: GenAIComps/${dockerfile_path} (Referenced in GenAIExamples/${file}:${line})" + fi + done < <(grep -Ern 'docker build .* -f comps/.+/Dockerfile' --include='*.md' .) + + + if [[ "$miss" == "TRUE" ]]; then + exit 1 + fi + + shell: bash + + check-the-validity-of-hyperlinks-in-README: + runs-on: ubuntu-latest + steps: + - name: Clean Up Working Directory + run: sudo rm -rf ${{github.workspace}}/* + + - name: Checkout Repo GenAIExamples + uses: actions/checkout@v4 + + - name: Check the Validity of Hyperlinks + run: | + cd ${{github.workspace}} + fail="FALSE" + url_lines=$(grep -Eo '\]\(http[s]?://[^)]+\)' --include='*.md' -r .) + if [ -n "$url_lines" ]; then + for url_line in $url_lines; do + url=$(echo "$url_line"|cut -d '(' -f2 | cut -d ')' -f1|sed 's/\.git$//') + path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-) + response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response" -ne 200 ]; then + echo "**********Validation failed, try again**********" + response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response_retry" -eq 200 ]; then + echo "*****Retry successfully*****" + else + echo "Invalid link from ${{github.workspace}}/$path: $url" + fail="TRUE" + fi + fi + done + fi + + if [[ "$fail" == "TRUE" ]]; then + exit 1 + else + echo "All hyperlinks are valid." + fi + shell: bash + + check-the-validity-of-relative-path: + runs-on: ubuntu-latest + steps: + - name: Clean up Working Directory + run: sudo rm -rf ${{github.workspace}}/* + + - name: Checkout Repo GenAIExamples + uses: actions/checkout@v4 + + - name: Checking Relative Path Validity + run: | + cd ${{github.workspace}} + fail="FALSE" + link_head="https://github.com/opea-project/GenAIExamples/blob/main/" + png_lines=$(grep -Eo '\]\([^)]+\)' -r -I .|grep -Ev 'http') + if [ -n "$png_lines" ]; then + for png_line in $png_lines; do + refer_path=$(echo "$png_line"|cut -d':' -f1 | cut -d'/' -f2-) + png_path=$(echo "$png_line"|cut -d '(' -f2 | cut -d ')' -f1) + if [[ "${png_path:0:1}" == "/" ]]; then + check_path=${{github.workspace}}$png_path + else + check_path=${{github.workspace}}/$(dirname "$refer_path")/$png_path + fi + real_path=$(realpath $check_path) + if [ $? -ne 0 ]; then + echo "Path $png_path in file ${{github.workspace}}/$refer_path does not exist" + fail="TRUE" + else + url=$link_head$(echo "$real_path" | sed 's|.*/GenAIExamples/||') + response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response" -ne 200 ]; then + echo "**********Validation failed, try again**********" + response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url") + if [ "$response_retry" -eq 200 ]; then + echo "*****Retry successfully*****" + else + echo "Invalid link from $check_path: $url" + fail="TRUE" + fi + fi + fi + done + fi + + if [[ "$fail" == "TRUE" ]]; then + exit 1 + else + echo "All hyperlinks are valid." + fi + shell: bash diff --git a/.github/workflows/pr-path_detection.yml b/.github/workflows/pr-path_detection.yml deleted file mode 100644 index 6d68c1935..000000000 --- a/.github/workflows/pr-path_detection.yml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -name: Check for missing Dockerfile paths in repo comps - -on: - pull_request: - branches: [main] - types: [opened, reopened, ready_for_review, synchronize] - -jobs: - check-dockerfile-paths: - runs-on: ubuntu-latest - - steps: - - name: Clean Up Working Directory - run: sudo rm -rf ${{github.workspace}}/* - - - name: Checkout repo GenAIExamples - uses: actions/checkout@v4 - - - name: Clone repo GenAIComps - run: | - cd .. - git clone https://github.com/opea-project/GenAIComps.git - - - name: Check for missing Dockerfile paths in GenAIComps - run: | - cd ${{github.workspace}} - miss="FALSE" - while IFS=: read -r file line content; do - dockerfile_path=$(echo "$content" | awk -F '-f ' '{print $2}' | awk '{print $1}') - if [[ ! -f "../GenAIComps/${dockerfile_path}" ]]; then - miss="TRUE" - echo "Missing Dockerfile: GenAIComps/${dockerfile_path} (Referenced in GenAIExamples/${file}:${line})" - fi - done < <(grep -Ern 'docker build .* -f comps/.+/Dockerfile' --include='*.md' .) - - - if [[ "$miss" == "TRUE" ]]; then - exit 1 - fi - - shell: bash diff --git a/ProductivitySuite/docker_compose/intel/cpu/xeon/README.md b/ProductivitySuite/docker_compose/intel/cpu/xeon/README.md index 68eeb0826..1741b9b94 100644 --- a/ProductivitySuite/docker_compose/intel/cpu/xeon/README.md +++ b/ProductivitySuite/docker_compose/intel/cpu/xeon/README.md @@ -1,6 +1,6 @@ # Build Mega Service of Productivity Suite on Xeon -This document outlines the deployment process for OPEA Productivity Suite utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Xeon server and [GenAIExamples](https://github.com/opea-project/GenAIExamppes.git) solutions. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as `embedding`, `retriever`, `rerank`, and `llm`. We will publish the Docker images to Docker Hub soon, it will simplify the deployment process for this service. +This document outlines the deployment process for OPEA Productivity Suite utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Xeon server and [GenAIExamples](https://github.com/opea-project/GenAIExamples.git) solutions. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as `embedding`, `retriever`, `rerank`, and `llm`. We will publish the Docker images to Docker Hub soon, it will simplify the deployment process for this service. ## 🚀 Build Docker Images diff --git a/Translation/README.md b/Translation/README.md index 4d200e181..37bfdd902 100644 --- a/Translation/README.md +++ b/Translation/README.md @@ -6,7 +6,7 @@ Translation architecture shows below: ![architecture](./assets/img/translation_architecture.png) -This Translation use case performs Language Translation Inference on Intel Gaudi2 or Intel Xeon Scalable Processors. The Intel Gaudi2 accelerator supports both training and inference for deep learning models in particular for LLMs. Visit [Habana AI products](https://habana.ai/products) for more details. +This Translation use case performs Language Translation Inference on Intel Gaudi2 or Intel Xeon Scalable Processors. The Intel Gaudi2 accelerator supports both training and inference for deep learning models in particular for LLMs. Visit [Habana AI products](https://habana.ai/products/) for more details. ## Deploy Translation Service diff --git a/docker_images_list.md b/docker_images_list.md index fa58fcdd3..a25d1e572 100644 --- a/docker_images_list.md +++ b/docker_images_list.md @@ -8,8 +8,8 @@ Take ChatQnA for example. ChatQnA is a chatbot application service based on the | Example Images | Dockerfile | Description | | ----------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [opea/audioqna](https://hub.docker.com/r/opea/audioqna) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/AudioQnA/docker/Dockerfile) | The docker image served as a chatqna gateway and using language modeling to generate answers to user queries by converting audio input to text, and then using text-to-speech (TTS) to convert those answers back to speech for interaction. | -| [opea/chatqna](https://hub.docker.com/r/opea/chatqna) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/ChatQnA/docker/Dockerfile) | The docker image served as a chatqna gateway and interacted with users by understanding their questions and providing relevant answers. | +| [opea/audioqna](https://hub.docker.com/r/opea/audioqna) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/AudioQnA/Dockerfile) | The docker image served as a chatqna gateway and using language modeling to generate answers to user queries by converting audio input to text, and then using text-to-speech (TTS) to convert those answers back to speech for interaction. | +| [opea/chatqna](https://hub.docker.com/r/opea/chatqna) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/ChatQnA/Dockerfile) | The docker image served as a chatqna gateway and interacted with users by understanding their questions and providing relevant answers. | | [opea/chatqna-ui](https://hub.docker.com/r/opea/chatqna-ui) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/ChatQnA/ui/docker/Dockerfile) | The docker image acted as the chatqna UI entry for facilitating interaction with users for question answering | | [opea/chatqna-conversation-ui](https://hub.docker.com/r/opea/chatqna-conversation-ui) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/ChatQnA/ui/docker/Dockerfile.react) | The purpose of the docker image is to provide a user interface for chat-based Q&A using React. It allows for interaction with users and supports continuing conversations with a history that is stored in the browser's local storage. | | [opea/chatqna-guardrails](https://hub.docker.com/r/opea/chatqna-guardrails) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/ChatQnA/Dockerfile.guardrails) | This docker image is used to encapsulate chatqna's LLM service to secure model inputs and outputs. Guardrails proactively prevents models from interacting with insecure content and signals in time to stop insecure behavior. | @@ -17,20 +17,20 @@ Take ChatQnA for example. ChatQnA is a chatbot application service based on the | [opea/chat-conversation-service](https://hub.docker.com/r/opea/chat-conversation-service) | | | | [opea/chat-conversation-ui-service](https://hub.docker.com/r/opea/chat-conversation-ui-service) | | | | [opea/chat-history-service](https://hub.docker.com/r/opea/chat-history-service) | | | -| [opea/codegen](https://hub.docker.com/r/opea/codegen) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/CodeGen/docker/Dockerfile) | The docker image served as the codegen gateway to provide service of the automatic creation of source code from a higher-level representation | +| [opea/codegen](https://hub.docker.com/r/opea/codegen) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/CodeGen/Dockerfile) | The docker image served as the codegen gateway to provide service of the automatic creation of source code from a higher-level representation | | [opea/codegen-ui](https://hub.docker.com/r/opea/codegen-ui) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/CodeGen/ui/docker/Dockerfile) | The docker image acted as the codegen UI entry for facilitating interaction with users for automatically generating code from user's description | | [opea/codegen-react-ui](https://hub.docker.com/r/opea/codegen-react-ui) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/CodeGen/ui/docker/Dockerfile.react) | The purpose of the docker image is to provide a user interface for Codegen using React. It allows generating the appropriate code based on the current user input. | -| [opea/codetrans](https://hub.docker.com/r/opea/codetrans) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/CodeTrans/docker/Dockerfile) | The docker image served as a codetrans gateway to provide service of converting source code written in one programming language into an equivalent version in another programming language | +| [opea/codetrans](https://hub.docker.com/r/opea/codetrans) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/CodeTrans/Dockerfile) | The docker image served as a codetrans gateway to provide service of converting source code written in one programming language into an equivalent version in another programming language | | [opea/codetrans-ui](https://hub.docker.com/r/opea/codetrans-ui) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/CodeTrans/ui/docker/Dockerfile) | The docker image acted as the codetrans UI entry for facilitating interaction with users for translating one programming language to another one | -| [opea/docsum](https://hub.docker.com/r/opea/docsum) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/DocSum/docker/Dockerfile) | The docker image served as a docsum gateway to provide service of capturing the main points and essential details of the original text | +| [opea/docsum](https://hub.docker.com/r/opea/docsum) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/DocSum/Dockerfile) | The docker image served as a docsum gateway to provide service of capturing the main points and essential details of the original text | | [opea/docsum-ui](https://hub.docker.com/r/opea/docsum-ui) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/DocSum/ui/docker/Dockerfile) | The docker image acted as the docsum UI entry for facilitating interaction with users for document summarization | | [opea/docsum-react-ui](https://hub.docker.com/r/opea/docsum-react-ui) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/DocSum/ui/docker/Dockerfile.react) | The purpose of the docker image is to provide a user interface for document summary using React. It allows upload a file or paste text and then click on “Generate Summary” to get a condensed summary of the generated content and automatically scroll to the bottom of the summary. | -| [opea/faqgen](https://hub.docker.com/r/opea/faqgen) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/FaqGen/docker/Dockerfile) | The docker image served as a faqgen gateway and automatically generating comprehensive, natural sounding Frequently Asked Questions (FAQs) from documents, legal texts, customer inquiries and other sources. | +| [opea/faqgen](https://hub.docker.com/r/opea/faqgen) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/FaqGen/Dockerfile) | The docker image served as a faqgen gateway and automatically generating comprehensive, natural sounding Frequently Asked Questions (FAQs) from documents, legal texts, customer inquiries and other sources. | | [opea/faqgen-ui](https://hub.docker.com/r/opea/faqgen-ui) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/FaqGen/ui/docker/Dockerfile) | The docker image serves as the docsum UI entry point for easy interaction with users, generating FAQs by pasting in question text. | | [opea/faqgen-react-ui](https://hub.docker.com/r/opea/faqgen-react-ui) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/FaqGen/ui/docker/Dockerfile.react) | The purpose of the docker image is to provide a user interface for Generate FAQs using React. It allows generating FAQs by uploading files or pasting text. | -| [opea/searchqna](https://hub.docker.com/r/opea/searchqna/tags) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/SearchQnA/docker/Dockerfile) | The docker image served as the searchqna gateway to provide service of retrieving accurate and relevant answers to user queries from a knowledge base or dataset | +| [opea/searchqna](https://hub.docker.com/r/opea/searchqna/tags) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/SearchQnA/Dockerfile) | The docker image served as the searchqna gateway to provide service of retrieving accurate and relevant answers to user queries from a knowledge base or dataset | | [opea/searchqna-ui](https://hub.docker.com/r/opea/searchqna-ui) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/SearchQnA/ui/docker/Dockerfile) | The docker image acted as the searchqna UI entry for facilitating interaction with users for question answering | -| [opea/translation](https://hub.docker.com/r/opea/translation) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/Translation/docker/Dockerfile) | The docker image served as the translation gateway to provide service of language translation | +| [opea/translation](https://hub.docker.com/r/opea/translation) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/Translation/Dockerfile) | The docker image served as the translation gateway to provide service of language translation | | [opea/translation-ui](https://hub.docker.com/r/opea/translation-ui) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/Translation/ui/docker/Dockerfile) | The docker image acted as the translation UI entry for facilitating interaction with users for language translation | ## Microservice images @@ -52,7 +52,6 @@ Take ChatQnA for example. ChatQnA is a chatbot application service based on the | [opea/llm-docsum-tgi](https://hub.docker.com/r/opea/llm-docsum-tgi) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/summarization/tgi/langchain/Dockerfile) | This docker image is designed to build a document summarization microservice using the HuggingFace Text Generation Inference(TGI) framework. The microservice accepts document input and generates a document summary. | | [opea/llm-faqgen-tgi](https://hub.docker.com/r/opea/llm-faqgen-tgi) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/faq-generation/tgi/langchain/Dockerfile) | This docker image is designed to build a frequently asked questions microservice using the HuggingFace Text Generation Inference(TGI) framework. The microservice accepts document input and generates a FAQ. | | [opea/llm-ollama](https://hub.docker.com/r/opea/llm-ollama) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/text-generation/ollama/langchain/Dockerfile) | The docker image exposed the OPEA LLM microservice based on ollama for GenAI application use | -| [opea/llm-ray](https://hub.docker.com/r/opea/llm-ray) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/text-generation/ray_serve/docker/Dockerfile.microservice) | The docker image exposed the OPEA LLM microservice based on Ray for GenAI application use | | [opea/llm-tgi](https://hub.docker.com/r/opea/llm-tgi) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/text-generation/tgi/Dockerfile) | The docker image exposed the OPEA LLM microservice upon TGI docker image for GenAI application use | | [opea/llm-vllm](https://hub.docker.com/r/opea/llm-vllm) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/text-generation/vllm/langchain/Dockerfile) | The docker image exposed the OPEA LLM microservice upon vLLM docker image for GenAI application use | | [opea/llm-vllm-hpu](https://hub.docker.com/r/opea/llm-vllm-hpu) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/llms/text-generation/vllm/langchain/dependency/Dockerfile.intel_hpu) | The docker image exposed the OPEA LLM microservice upon vLLM docker image for use by GenAI apps on the Gaudi | @@ -67,6 +66,6 @@ Take ChatQnA for example. ChatQnA is a chatbot application service based on the | [opea/tei-gaudi](https://hub.docker.com/r/opea/tei-gaudi/tags) | [Link](https://github.com/huggingface/tei-gaudi/blob/habana-main/Dockerfile-hpu) | The docker image powered by HuggingFace Text Embedding Inference (TEI) on Gaudi2 for deploying and serving Embedding Models | | [opea/tts](https://hub.docker.com/r/opea/tts) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/tts/speecht5/Dockerfile) | The docker image exposed the OPEA Text-To-Speech microservice for GenAI application use | | [opea/vllm](https://hub.docker.com/r/opea/vllm) | [Link](https://github.com/vllm-project/vllm/blob/main/Dockerfile.cpu) | The docker image powered by vllm-project for deploying and serving vllm Models | -| [opea/web-retriever-chroma](https://hub.docker.com/r/opea/web-retriever-chroma) | [Link](https://github.com/opea-project/GenAIComps/tree/main/comps/web_retrievers/langchain/chroma/docker) | The docker image exposed the OPEA retrieval microservice based on chroma vectordb for GenAI application use | +| [opea/web-retriever-chroma](https://hub.docker.com/r/opea/web-retriever-chroma) | [Link](https://github.com/opea-project/GenAIComps/tree/main/comps/web_retrievers/chroma/langchain/Dockerfile) | The docker image exposed the OPEA retrieval microservice based on chroma vectordb for GenAI application use | | [opea/whisper](https://hub.docker.com/r/opea/whisper) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/asr/whisper/dependency/Dockerfile) | The docker image exposed the OPEA Whisper service for GenAI application use | | [opea/whisper-gaudi](https://hub.docker.com/r/opea/whisper-gaudi) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/asr/whisper/dependency/Dockerfile.intel_hpu) | The docker image exposed the OPEA Whisper service on Gaudi2 for GenAI application use |