Added the function of detecting whether the uploaded content is safe and providing prompts (#867)

Signed-off-by: Yue, Wenjiao <wenjiao.yue@intel.com>

VisualQnA/ui/svelte/.env

@@ -1 +1,2 @@
-BACKEND_BASE_URL = '/v1/visualqna'
+GUARDRAIL_BASE_URL = 'http://backend_address:9399/v1/lvm'
+BACKEND_BASE_URL = 'http://backend_address:9499/v1/lvm'

VisualQnA/ui/svelte/src/lib/modules/chat/ChatMessage.svelte

@@ -17,6 +17,7 @@
 <script lang="ts">
 	import MessageAvatar from "$lib/modules/chat/MessageAvatar.svelte";
 	import type { Message } from "$lib/shared/constant/Interface";
+	import { Alert } from "flowbite-svelte";
 	import MessageTimer from "./MessageTimer.svelte";
 	import { createEventDispatcher } from "svelte";
 
@@ -31,26 +32,42 @@
 	class={msg.role === 0
 		? "flex w-full gap-3"
 		: "flex w-full items-center gap-3"}
-	data-testid={msg.role === 0
+	data-testid={msg.role === 0 ? "display-answer" : "display-question"}
-		? "display-answer"
-		: "display-question"}
 >
 	<div
 		class={msg.role === 0
 			? "flex aspect-square w-[3px]  items-center justify-center rounded bg-[#0597ff] max-sm:hidden"
-			: "flex aspect-square h-10 w-[3px] items-center justify-center rounded bg-[#000] max-sm:hidden"}
+			: "flex aspect-square h-10 w-[3px] items-center justify-center rounded bg-[#acacac] max-sm:hidden mb-4"}
 	>
 		<MessageAvatar role={msg.role} />
 	</div>
-	<div class="group relative flex items-start">
+	<div
+		class={msg.role === 0
+			? "group relative flex items-start border-b-4 border-gray-200 pb-2"
+			: "group relative flex items-start"}
+	>
 		<div class="flex flex-col items-start">
-			<img src={msg.imgSrc} alt="Uploaded Image" class="m-2 max-w-28 max-h-28" />
+			{#if msg.imgSrc}
+				<img
+					src={msg.imgSrc}
+					alt="Uploaded Image"
+					class="max-w-28 m-2 max-h-28"
+				/>
+			{/if}
 
+			{#if msg.content === "unsafe"}
+				<Alert color="red">
+					<span class="font-medium">Danger alert! </span>
+					<span>The uploaded image/question contains potential security risks.</span>
+				</Alert>
+			{:else}
 			<p
-				class="xl:max-w-[65vw] max-w-[60vw] items-start whitespace-pre-line break-keep text-[0.8rem] leading-5 sm:max-w-[50rem]"
+				class="max-w-[60vw] items-start whitespace-pre-line break-keep leading-6 sm:max-w-[50rem] xl:max-w-[65vw]"
 			>
 				{@html msg.content}
 			</p>
+			{/if}
+			
 		</div>
 	</div>
 </div>

VisualQnA/ui/svelte/src/lib/modules/upload/imagePrompt.svelte

@@ -13,17 +13,17 @@
 
   let images = [
     {
-      id: 1,
+      id: 0,
       alt: 'Waterview',
       imgurl: waterview,
       prompt: 'What are the things I should be cautious about when I visit here?'
     },
     {
-      id: 0,
+      id: 1,
       alt: 'Extreme Ironing',
       imgurl: extreme_ironing,
       prompt: 'What is unusual about this image?'
-    }
+    },
   ];
 
   let currentIndex = 0;
@@ -37,12 +37,14 @@
   }
 
 
-  async function handleImageClick() {
+  async function handleImageClick() {    
     const imgUrl = images[currentIndex].imgurl;
     const base64Data = await convertImageToBase64(imgUrl);
     const currentPrompt = images[currentIndex].prompt;
-    dispatch("imagePrompt", { content: currentPrompt });
     base64ImageStore.set(base64Data);
+
+    dispatch("imagePrompt", { content: currentPrompt });
+    
   }
 
   async function convertImageToBase64(url) {

VisualQnA/ui/svelte/src/lib/modules/upload/upload.svelte

@@ -10,23 +10,29 @@
 	import { Range } from "flowbite-svelte";
 	import { FilePasteSolid } from "flowbite-svelte-icons";
 	import { stepValueStore } from "$lib/shared/stores/common/Store";
-	let stepValue = 512;
+	let stepValue = 128;
     let imageUrl = '';
 
 	$: stepValueStore.set(stepValue);
+
 </script>
 
+
+
 <div class="flex w-full flex-col gap-3 rounded-xl bg-white p-5">
 	<p>Upload Images</p>
 	<UploadImg imageUrl={imageUrl}/>
 	<Hr classHr="my-8 w-64">or</Hr>
-	<div class="mb-6">
+	<div class="gap-1">
-		<Label for="input-group-1" class="block mb-2">Import from URL</Label>
+		<div class="">
-		<Input type="text" placeholder=""  bind:value={imageUrl}>
+			<Label for="input-group-1" class="block mb-2">Import from URL</Label>
-		  <FilePasteSolid slot="left" class="w-5 h-5 text-gray-500 dark:text-gray-400" />
+			<Input type="text" placeholder=""  bind:value={imageUrl}>
-		</Input>
+			  <FilePasteSolid slot="left" class="w-5 h-5 text-gray-500 dark:text-gray-400" />
-	  </div>
+			</Input>
-	<p>Parameters</p>
+		  </div>
-	<Range id="range-steps" min="0" max="1024" bind:value={stepValue} step="1" />
+		<p class="text-[0.9rem]">Parameters</p>
-	<p>Max output tokens: {stepValue}</p>
+		<Range id="range-steps" min="0" max="1024" bind:value={stepValue} step="1" />
-</div>
+		<p class="text-xs">Max output tokens: {stepValue}</p>
+	</div>
+
+</div>	

VisualQnA/ui/svelte/src/lib/modules/upload/uploadImg.svelte

@@ -103,6 +103,6 @@
 			SVG, PNG, JPG
 		</p>
 	{:else if imageUrl}
-		<img src={imageUrl} alt="Uploaded Image" class="m-2 mx-auto block" />
+		<img src={imageUrl} alt="Uploaded Image" class="m-2 mx-auto block max-h-[15.5rem]" />
 	{/if}
 </Dropzone>

VisualQnA/ui/svelte/src/lib/network/chat/Network.ts

@@ -13,9 +13,47 @@
 // limitations under the License.
 
 import { env } from "$env/dynamic/public";
-import { SSE } from "sse.js";
 
 const BACKEND_BASE_URL = env.BACKEND_BASE_URL;
+const guardrail_BASE_URL = env.GUARDRAIL_BASE_URL;
+
+
+async function fetchFunc(url, init) {
+	try {
+		const response = await fetch(url, init);
+		if (!response.ok) throw response.status;
+
+		return await response.json();
+	} catch (error) {
+		console.error("network error: ", error);
+
+		return undefined;
+	}
+}
+
+
+export async function fetchGuardRail(query: string, stepValueStore: number, base64ImageStore: string) {	
+	let payload = {};
+	let url = "";
+	base64ImageStore = base64ImageStore.replace(/^data:[a-zA-Z]+\/[a-zA-Z]+;base64,/, "");
+
+	payload = {
+		image: base64ImageStore,
+		prompt: query,
+		max_new_tokens: 1,
+		stream: false,
+	};
+
+	url = `${guardrail_BASE_URL}`;
+
+	const init: RequestInit = {
+		method: "POST",
+		headers: { "Content-Type": "application/json" },
+		body: JSON.stringify(payload),
+	};
+
+	return fetchFunc(url, init);
+}
 
 export async function fetchTextStream(query: string, stepValueStore: number, base64ImageStore: string) {
 	let payload = {};
@@ -23,30 +61,19 @@ export async function fetchTextStream(query: string, stepValueStore: number, bas
 	base64ImageStore = base64ImageStore.replace(/^data:[a-zA-Z]+\/[a-zA-Z]+;base64,/, "");
 
 	payload = {
-		messages: [
+		image: base64ImageStore,
-			{
+		prompt: query,
-				role: "user",
+		max_new_tokens: stepValueStore,
-				content: [
+
-					{
-						type: "text",
-						text: query,
-					},
-					{
-						type: "image_url",
-						image_url: { url: base64ImageStore },
-					},
-				],
-			},
-		],
-		max_tokens: stepValueStore,
-		stream: true,
 	};
-	console.log("payload", payload);
 
 	url = `${BACKEND_BASE_URL}`;
 
-	return new SSE(url, {
+	const init: RequestInit = {
+		method: "POST",
 		headers: { "Content-Type": "application/json" },
-		payload: JSON.stringify(payload),
+		body: JSON.stringify(payload),
-	});
+	};
+
+	return fetchFunc(url, init);
 }

VisualQnA/ui/svelte/src/lib/shared/components/header/header.svelte

@@ -25,7 +25,7 @@
   >
     <div class="mx-auto flex flex-wrap justify-end items-center w-full">
       <span
-        class="self-center py-2 whitespace-nowrap text-2xl font-semibold text-white ml-4"
+        class="self-center py-2 whitespace-nowrap text-[2rem] font-semibold text-white ml-4"
         data-svelte-h="svelte-1hbktnk">VisualQnA</span
       >
     </div>

VisualQnA/ui/svelte/src/lib/shared/stores/common/Store.ts

@@ -42,4 +42,4 @@ export const knowledgeName = writable("");
 
 export const base64ImageStore = writable("");
 
-export const stepValueStore = writable(512);
+export const stepValueStore = writable(256);

VisualQnA/ui/svelte/src/routes/+page.svelte

@@ -33,7 +33,7 @@
 		scrollToBottom,
 		scrollToTop,
 	} from "$lib/shared/Utils";
-	import { fetchTextStream } from "$lib/network/chat/Network";
+	import { fetchGuardRail, fetchTextStream } from "$lib/network/chat/Network";
 	import LoadingAnimation from "$lib/shared/components/loading/Loading.svelte";
 	import "driver.js/dist/driver.css";
 	import "$lib/assets/layout/css/driver.css";
@@ -42,12 +42,14 @@
 	import ChatMessage from "$lib/modules/chat/ChatMessage.svelte";
 	import Upload from "$lib/modules/upload/upload.svelte";
 	import ImagePrompt from "$lib/modules/upload/imagePrompt.svelte";
+	import { Toast } from "flowbite-svelte";
+	import { ExclamationCircleSolid, FireOutline } from "flowbite-svelte-icons";
 
 	let query: string = "";
 	let loading: boolean = false;
 	let scrollToDiv: HTMLDivElement;
 	let chatMessages: Message[] = data.chatMsg ? data.chatMsg : [];
-	console.log("chatMessages", chatMessages);
+	let showToast = false;
 
 	onMount(async () => {
 		scrollToDiv = document
@@ -81,62 +83,54 @@
 	}
 
 	const callTextStream = async (query: string) => {
-		const eventSource = await fetchTextStream(
+		const res = await fetchGuardRail(query, $stepValueStore, $base64ImageStore);
-			query,
+		const lastSegment = res.text.split("[/INST]").pop().trim();
-			$stepValueStore,
-			$base64ImageStore
-		);
 
-		eventSource.addEventListener("message", (e: any) => {
+		if (lastSegment === "unsafe") {
-			let Msg = e.data;
+			loading = false;
-			if (Msg.startsWith("b")) {
-				let trimmedData = Msg.slice(2, -1);
 
-				if (/\\x[\dA-Fa-f]{2}/.test(trimmedData)) {
+			showToast = true;
-					trimmedData = decodeEscapedBytes(trimmedData);
+			setTimeout(() => {
-				} else if (/\\u[\dA-Fa-f]{4}/.test(trimmedData)) {
+				showToast = false;
-					trimmedData = decodeUnicode(trimmedData);
+			}, 3000);
-				}
 
-				if (trimmedData !== "</s>") {
+			chatMessages = [
-					trimmedData = trimmedData.replace(/\\n/g, "\n");
+				...chatMessages,
-				}
+				{
-				if (chatMessages[chatMessages.length - 1].role == MessageRole.User) {
+					role: MessageRole.Assistant,
-					chatMessages = [
+					type: MessageType.Text,
-						...chatMessages,
+					content: "unsafe",
-						{
+					time: getCurrentTimeStamp(),
-							role: MessageRole.Assistant,
+					imgSrc: null, // Add the imgSrc property here
-							type: MessageType.Text,
+				},
-							content: trimmedData,
+			];
-							time: getCurrentTimeStamp(),
-							imgSrc: null, // Add the imgSrc property here
-						},
-					];
-					console.log("? chatMessages", chatMessages);
-				} else {
-					let content = chatMessages[chatMessages.length - 1].content as string;
-					chatMessages[chatMessages.length - 1].content = content + trimmedData;
-				}
-				scrollToBottom(scrollToDiv);
-			} else if (Msg === "[DONE]") {
-				let startTime = chatMessages[chatMessages.length - 1].time;
 
+			return;
+		} else {
+			const chatRes = await fetchTextStream(
+				query,
+				$stepValueStore,
+				$base64ImageStore
+			);
+			if (chatRes.text) {
 				loading = false;
-				let totalTime = parseFloat(
+				chatMessages = [
-					((getCurrentTimeStamp() - startTime) / 1000).toFixed(2)
+					...chatMessages,
-				);
+					{
-				if (chatMessages.length - 1 !== -1) {
+						role: MessageRole.Assistant,
-					chatMessages[chatMessages.length - 1].time = totalTime;
+						type: MessageType.Text,
-				}
+						content: chatRes.text,
+						time: getCurrentTimeStamp(),
+						imgSrc: null, // Add the imgSrc property here
+					},
+				];
 				storeMessages();
 			}
-		});
+		}
-		eventSource.stream();
 	};
 
 	const handleTextSubmit = async () => {
 		loading = true;
-		console.log("handleTextSubmit", $base64ImageStore);
 
 		const newMessage = {
 			role: MessageRole.User,
@@ -171,13 +165,26 @@
 
 <Header />
 <div class="h-full gap-5 bg-white sm:flex sm:pb-2 lg:rounded-tl-3xl">
-	<div class="w-1/5 bg-gray-200 p-4">
+	<div class="w-1/4 bg-gray-200 p-4 pt-[0.08rem]">
-		<Upload />
 		<ImagePrompt on:imagePrompt={handleUpdateQuery} />
+		<Upload />
 	</div>
-	<div class="flex-1 bg-gray-100 p-4">
+
+	<div class="flex-1 p-4">
+		{#if showToast}
+			<div class="fixed right-0 top-0 z-50 mr-4 mt-4">
+				<Toast color="red" class="w-[50rem]">
+					<svelte:fragment slot="icon">
+						<ExclamationCircleSolid class="h-5 w-5" />
+						<span class="sr-only">Warning icon</span>
+					</svelte:fragment>
+					The uploaded image/question contains potential security risks.
+				</Toast>
+			</div>
+		{/if}
+
 		<div
-			class="mx-auto flex h-full w-full flex-col bg-white px-10 sm:mt-0 sm:w-[80%]"
+			class="mx-auto flex h-[93%] w-full flex-col bg-white pb-4 sm:mt-0 sm:w-[95%]"
 		>
 			<div
 				class="fixed relative flex w-full flex-col items-center justify-between bg-white p-2 pb-0"
@@ -185,7 +192,7 @@
 				<div class="relative my-4 flex w-full flex-row justify-center">
 					<div class="relative w-full focus:border-none">
 						<input
-							class="text-md block w-full border-0 border-b-2 border-gray-300 px-1 py-4
+							class="block w-full border-0 border-b-2 border-gray-300 px-1 py-4
 						text-gray-900 focus:border-gray-300 focus:ring-0 dark:border-gray-600 dark:bg-gray-700 dark:text-white dark:placeholder-gray-400 dark:focus:border-blue-500 dark:focus:ring-blue-500"
 							type="text"
 							data-testid="chat-input"
@@ -225,8 +232,8 @@
 							><svg
 								xmlns="http://www.w3.org/2000/svg"
 								viewBox="0 0 20 20"
-								width="24"
+								width="20"
-								height="24"
+								height="20"
 								class="fill-[#0597ff] group-hover:fill-[#0597ff]"
 								><path
 									d="M12.6 12 10 9.4 7.4 12 6 10.6 8.6 8 6 5.4 7.4 4 10 6.6 12.6 4 14 5.4 11.4 8l2.6 2.6zm7.4 8V2q0-.824-.587-1.412A1.93 1.93 0 0 0 18 0H2Q1.176 0 .588.588A1.93 1.93 0 0 0 0 2v12q0 .825.588 1.412Q1.175 16 2 16h14zm-3.15-6H2V2h16v13.125z"
@@ -240,11 +247,9 @@
 
 			<div class="mx-auto flex h-full w-full flex-col">
 				<!-- Loading text -->
-				{#if loading}
+
-					<LoadingAnimation />
-				{/if}
 				<Scrollbar
-					classLayout="flex flex-col gap-1 mr-4"
+					classLayout="flex flex-col gap-2 mr-4"
 					className="chat-scrollbar h-0 w-full grow px-2 pt-2 mt-3 mr-5"
 				>
 					{#each chatMessages as message, i}
@@ -257,7 +262,9 @@
 						/>
 					{/each}
 				</Scrollbar>
-
+				{#if loading}
+					<LoadingAnimation />
+				{/if}
 			</div>
 			<!-- gallery -->
 		</div>

VisualQnA/ui/svelte/vite.config.ts

@@ -17,7 +17,10 @@ import type { UserConfig } from "vite";
 
 const config: UserConfig = {
 	plugins: [sveltekit()],
-	server: {},
+	server: {
+		host: '0.0.0.0', 
+		port: 5173,
+	},
 };
 
 export default config;