jandres/GenAIExamples
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Add ChatQnA E2E test workflow (#144)

Browse Source 
Signed-off-by: Sun, Xuehao <xuehao.sun@intel.com>
This commit is contained in:
Sun, Xuehao
2024-05-29 22:38:09 +08:00
committed by GitHub
parent c1c6d3387e
commit f3d61d4246
8 changed files with 118 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/E2E_test_with_compose.yml vendored
View File

@@ -88,7 +88,7 @@ jobs:
env:
example: ${{ matrix.example }}
hardware: ${{ matrix.hardware }}
if: cancelled() || failure()
if: always()
run: |
cd ${{ github.workspace }}/$example/docker-composer/$hardware
container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)

57
ChatQnA/tests/test_chatqna_on_gaudi.sh
View File

@@ -56,6 +56,8 @@ function start_services() {
export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/chatqna"
export DATAPREP_SERVICE_ENDPOINT="http://${ip_address}:6007/v1/dataprep"
sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
# Start Docker Containers
# TODO: Replace the container name with a test-specific name
@@ -77,11 +79,11 @@ function validate_microservices() {
curl ${ip_address}:8090/embed \
-X POST \
-d '{"inputs":"What is Deep Learning?"}' \
-H 'Content-Type: application/json' > ${LOG_PATH}/embed.log
-H 'Content-Type: application/json' >${LOG_PATH}/embed.log
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo "Microservice failed, please check the logs in artifacts!"
docker logs tei-embedding-gaudi-server >> ${LOG_PATH}/embed.log
docker logs tei-embedding-gaudi-server >>${LOG_PATH}/embed.log
exit 1
fi
sleep 1s
@@ -89,11 +91,11 @@ function validate_microservices() {
curl http://${ip_address}:6000/v1/embeddings \
-X POST \
-d '{"text":"hello"}' \
-H 'Content-Type: application/json' > ${LOG_PATH}/embeddings.log
-H 'Content-Type: application/json' >${LOG_PATH}/embeddings.log
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo "Microservice failed, please check the logs in artifacts!"
docker logs embedding-tei-server >> ${LOG_PATH}/embeddings.log
docker logs embedding-tei-server >>${LOG_PATH}/embeddings.log
exit 1
fi
sleep 1s
@@ -108,7 +110,7 @@ function validate_microservices() {
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo "Microservice failed, please check the logs in artifacts!"
docker logs retriever-redis-server >> ${LOG_PATH}/retrieval.log
docker logs retriever-redis-server >>${LOG_PATH}/retrieval.log
exit 1
fi
sleep 1s
@@ -116,11 +118,11 @@ function validate_microservices() {
curl http://${ip_address}:8808/rerank \
-X POST \
-d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \
-H 'Content-Type: application/json' > ${LOG_PATH}/rerank.log
-H 'Content-Type: application/json' >${LOG_PATH}/rerank.log
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo "Microservice failed, please check the logs in artifacts!"
docker logs tei-xeon-server >> ${LOG_PATH}/rerank.log
docker logs tei-xeon-server >>${LOG_PATH}/rerank.log
exit 1
fi
sleep 1s
@@ -128,11 +130,11 @@ function validate_microservices() {
curl http://${ip_address}:8000/v1/reranking \
-X POST \
-d '{"initial_query":"What is Deep Learning?", "retrieved_docs": [{"text":"Deep Learning is not..."}, {"text":"Deep learning is..."}]}' \
-H 'Content-Type: application/json' > ${LOG_PATH}/reranking.log
-H 'Content-Type: application/json' >${LOG_PATH}/reranking.log
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo "Microservice failed, please check the logs in artifacts!"
docker logs reranking-tei-gaudi-server >> ${LOG_PATH}/reranking.log
docker logs reranking-tei-gaudi-server >>${LOG_PATH}/reranking.log
exit 1
fi
sleep 1s
@@ -140,11 +142,11 @@ function validate_microservices() {
curl http://${ip_address}:8008/generate \
-X POST \
-d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \
-H 'Content-Type: application/json' > ${LOG_PATH}/generate.log
-H 'Content-Type: application/json' >${LOG_PATH}/generate.log
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo "Microservice failed, please check the logs in artifacts!"
docker logs tgi-gaudi-server >> ${LOG_PATH}/generate.log
docker logs tgi-gaudi-server >>${LOG_PATH}/generate.log
exit 1
fi
sleep 1s
@@ -152,11 +154,11 @@ function validate_microservices() {
curl http://${ip_address}:9000/v1/chat/completions \
-X POST \
-d '{"text":"What is Deep Learning?"}' \
-H 'Content-Type: application/json' > ${LOG_PATH}/completions.log
-H 'Content-Type: application/json' >${LOG_PATH}/completions.log
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo "Microservice failed, please check the logs in artifacts!"
docker logs llm-tgi-gaudi-server >> ${LOG_PATH}/completions.log
docker logs llm-tgi-gaudi-server >>${LOG_PATH}/completions.log
exit 1
fi
sleep 1s
@@ -175,8 +177,8 @@ function validate_megaservice() {
echo "Checking response results, make sure the output is reasonable. "
local status=false
if [[ -f $LOG_PATH/curl_megaservice.log ]] && \
[[ $(grep -c "billion" $LOG_PATH/curl_megaservice.log) != 0 ]]; then
if [[ -f $LOG_PATH/curl_megaservice.log ]] &&
[[ $(grep -c "billion" $LOG_PATH/curl_megaservice.log) != 0 ]]; then
status=true
fi
@@ -191,6 +193,30 @@ function validate_megaservice() {
# TODO
}
function validate_frontend() {
cd $WORKPATH/ui/svelte
local conda_env_name="ChatQnA_e2e"
export PATH=${HOME}/miniconda3/bin/:$PATH
conda remove -n ${conda_env_name} --all -y
conda create -n ${conda_env_name} python=3.12 -y
source activate ${conda_env_name}
sed -i "s/localhost/$ip_address/g" playwright.config.ts
conda install -c conda-forge nodejs -y && npm install && npm ci && npx playwright install --with-deps
node -v && npm -v && pip list
exit_status=0
npx playwright test || exit_status=$?
if [ $exit_status -ne 0 ]; then
echo "[TEST INFO]: ---------frontend test failed---------"
exit $exit_status
else
echo "[TEST INFO]: ---------frontend test passed---------"
fi
}
function stop_docker() {
cd $WORKPATH/docker-composer/gaudi
container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
@@ -214,6 +240,7 @@ function main() {
validate_microservices
validate_megaservice
validate_frontend
stop_docker
echo y | docker system prune

35
ChatQnA/tests/test_chatqna_on_xeon.sh
View File

@@ -48,6 +48,8 @@ function start_services() {
export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/chatqna"
export DATAPREP_SERVICE_ENDPOINT="http://${ip_address}:6007/v1/dataprep"
sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env
# Start Docker Containers
# TODO: Replace the container name with a test-specific name
@@ -116,8 +118,7 @@ function validate_microservices() {
fi
sleep 1s
curl http://${ip_address}:8000/v1/reranking\
-X POST \
curl http://${ip_address}:8000/v1/reranking -X POST \
-d '{"initial_query":"What is Deep Learning?", "retrieved_docs": [{"text":"Deep Learning is not..."}, {"text":"Deep learning is..."}]}' \
-H 'Content-Type: application/json' > ${LOG_PATH}/reranking.log
if [ $exit_code -ne 0 ]; then
@@ -164,8 +165,8 @@ function validate_megaservice() {
echo "Checking response results, make sure the output is reasonable. "
local status=false
if [[ -f $LOG_PATH/curl_megaservice.log ]] && \
[[ $(grep -c "billion" $LOG_PATH/curl_megaservice.log) != 0 ]]; then
if [[ -f $LOG_PATH/curl_megaservice.log ]] &&
[[ $(grep -c "billion" $LOG_PATH/curl_megaservice.log) != 0 ]]; then
status=true
fi
@@ -181,6 +182,31 @@ function validate_megaservice() {
}
function validate_frontend() {
cd $WORKPATH/ui/svelte
local conda_env_name="ChatQnA_e2e"
export PATH=${HOME}/miniconda3/bin/:$PATH
conda remove -n ${conda_env_name} --all -y
conda create -n ${conda_env_name} python=3.12 -y
source activate ${conda_env_name}
sed -i "s/localhost/$ip_address/g" playwright.config.ts
conda install -c conda-forge nodejs -y && npm install && npm ci && npx playwright install --with-deps
node -v && npm -v && pip list
exit_status=0
npx playwright test || exit_status=$?
if [ $exit_status -ne 0 ]; then
echo "[TEST INFO]: ---------frontend test failed---------"
exit $exit_status
else
echo "[TEST INFO]: ---------frontend test passed---------"
fi
}
function stop_docker() {
cd $WORKPATH/docker-composer/xeon
container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2)
@@ -205,6 +231,7 @@ function main() {
validate_microservices
validate_megaservice
validate_frontend
stop_docker
echo y | docker system prune

2
ChatQnA/ui/svelte/.env
View File

@@ -1,3 +1,3 @@
CHAT_BASE_URL = 'http://x.x.x.x'
CHAT_BASE_URL = 'http://backend_address:8888/v1/chatqna'
UPLOAD_FILE_BASE_URL = 'http://x.x.x.x'

12
ChatQnA/ui/svelte/playwright.config.ts
View File

@@ -46,10 +46,10 @@ export default defineConfig({
/* Configure projects for major browsers */
projects: [
{
name: "chromium",
use: { ...devices["Desktop Chrome"] },
},
// {
// name: "chromium",
// use: { ...devices["Desktop Chrome"] },
// },
/* Test against mobile viewports. */
// {
@@ -66,6 +66,10 @@ export default defineConfig({
// name: 'Microsoft Edge',
// use: { channel: 'msedge' },
// },
{
name: "webkit",
use: { ...devices["Desktop Safari"] },
},
// {
// name: 'Google Chrome',
// use: { channel: 'chrome' },

3
ChatQnA/ui/svelte/src/lib/modules/chat/ChatMessage.svelte
View File

@@ -31,6 +31,9 @@
class={msg.role === 0
? "flex w-full gap-3"
: "flex w-full items-center gap-3"}
data-testid={msg.role === 0
? "display-answer"
: "display-question"}
>
<div
class={msg.role === 0

2
ChatQnA/ui/svelte/src/lib/modules/chat/MessageTimer.svelte
View File

@@ -58,7 +58,7 @@
/></svg
>
</div>
<div class="flex items-center space-x-1 text-base text-gray-800">
<div class="flex items-center space-x-1 text-base text-gray-800" data-testid='msg-time'>
<strong>End to End Time: </strong>
<p>{time}s</p>
</div>

64
ChatQnA/ui/svelte/tests/chatQnA.spec.ts
View File

@@ -20,13 +20,11 @@ async function checkNotificationText(page, expectedText) {
// Helper function to enter message to chat
async function enterMessageToChat(page, message) {
const newChat = page.getByTestId("chat-input");
await newChat.fill(message);
await newChat.press("Enter");
// Adding timeout and debug information
const msgTime = await page.waitForSelector("[data-testid='msg-time']", { timeout: 10000 });
await expect(msgTime).toBeVisible;
console.log("Message time is visible.");
await page.getByTestId("chat-input").click();
await page.getByTestId("chat-input").fill(message);
await page.getByTestId("chat-input").press("Enter");
await page.waitForTimeout(10000);
await expect(page.getByTestId("display-answer")).toBeVisible();
}
// Helper function to upload a file
@@ -50,34 +48,34 @@ test.describe("New Chat", () => {
test("should enter message to chat and clear chat", async ({ page }) => {
await enterMessageToChat(page, CHAT_ITEMS[0]);
const clearChat = page.getByTestId("clear-chat");
await clearChat.click();
// Verify the chat is cleared
const chatMessageContent = await page.$eval(
"[data-testid='chat-message']",
(message) => message?.textContent?.trim() || "",
);
expect(chatMessageContent).toBe("");
// const clearChat = page.getByTestId("clear-chat");
// await clearChat.click();
// // Verify the chat is cleared
// const chatMessageContent = await page.$eval(
// "[data-testid='chat-message']",
// (message) => message?.textContent?.trim() || "",
// );
// expect(chatMessageContent).toBe("");
});
});
test.describe("Upload file and create new Chat", () => {
// upload file
test("should upload a file", async ({ page }) => {
const openUpload = page.getByTestId("open-upload");
await openUpload.click();
await uploadFile(page, FILE_PATH);
});
// test.describe("Upload file and create new Chat", () => {
// // upload file
// test("should upload a file", async ({ page }) => {
// const openUpload = page.getByTestId("open-upload");
// await openUpload.click();
// await uploadFile(page, FILE_PATH);
// });
// paste link
test("should paste link", async ({ page }) => {
const openUpload = page.getByTestId("open-upload");
await openUpload.click();
await pasteLink(page, UPLOAD_LINK[0]);
});
// // paste link
// test("should paste link", async ({ page }) => {
// const openUpload = page.getByTestId("open-upload");
// await openUpload.click();
// await pasteLink(page, UPLOAD_LINK[0]);
// });
// chat with uploaded file and link
test("should test uploaded chat", async ({ page }) => {
await enterMessageToChat(page, CHAT_ITEMS[0]);
});
});
// // chat with uploaded file and link
// test("should test uploaded chat", async ({ page }) => {
// await enterMessageToChat(page, CHAT_ITEMS[0]);
// });
// });