Update UI of the three demos: faqGen, VisualQnA, and DocSum. (#1528)

Signed-off-by: WenjiaoYue <wenjiao.yue@intel.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

DocSum/ui/gradio/docsum_ui_gradio.py

@@ -296,8 +296,8 @@ class DocSumUI:
                     audio_ui.render()
                 with gr.TabItem("Upload Video"):
                     video_ui.render()
-                with gr.TabItem("Enter URL"):
-                    url_ui.render()
+                # with gr.TabItem("Enter URL"):
+                #     url_ui.render()
 
         return self.demo
 

FaqGen/ui/svelte/src/lib/shared/Network.ts

@@ -52,6 +52,7 @@ export async function fetchTextStream(query: string | Blob, params: string, file
     }
     const reader = postResponse.body.getReader();
     const decoder = new TextDecoder("utf-8");
+
     let done, value;
 
     let buffer = ""; // Initialize a buffer
@@ -61,6 +62,7 @@ export async function fetchTextStream(query: string | Blob, params: string, file
 
       // Decode chunk and append to buffer
       const chunk = decoder.decode(value, { stream: true });
+
       buffer += chunk;
 
       // Use regex to clean and extract data
@@ -72,6 +74,21 @@ export async function fetchTextStream(query: string | Blob, params: string, file
         })
         .filter((line) => line); // Remove empty lines
 
+      const validJsonChunks = cleanedChunks.filter((item) => {
+        if (item === "[DONE]") {
+          return true;
+        }
+        try {
+          JSON.parse(item);
+          return true;
+        } catch (e) {
+          return false;
+        }
+      });
+
+      cleanedChunks.length = 0;
+      cleanedChunks.push(...validJsonChunks);
+
       for (const cleanedChunk of cleanedChunks) {
         // Further clean to ensure all unnecessary parts are removed
         yield cleanedChunk.replace(/^b'|['"]$/g, ""); // Again clean 'b' and other single or double quotes

FaqGen/ui/svelte/src/routes/+page.svelte

@@ -36,6 +36,7 @@
     urlSuffix: string,
     params: string
   ) => {
+    messages = "";
     // Fetch the stream
     const eventStream = await fetchTextStream(
       query,

VisualQnA/ui/svelte/package.json

@@ -23,6 +23,7 @@
     "@tailwindcss/typography": "0.5.7",
     "@types/debug": "4.1.7",
     "@types/node": "^20.12.13",
+    "@types/pica": "^9.0.5",
     "@typescript-eslint/eslint-plugin": "^5.27.0",
     "@typescript-eslint/parser": "^5.27.0",
     "autoprefixer": "^10.4.16",
@@ -51,6 +52,7 @@
     "flowbite-svelte-icons": "^1.4.0",
     "fuse.js": "^6.6.2",
     "lodash": "^4.17.21",
+    "pica": "^9.0.1",
     "playwright": "^1.44.0",
     "ramda": "^0.29.0",
     "sse.js": "^0.6.1",

VisualQnA/ui/svelte/src/lib/modules/chat/ChatMessage.svelte

@@ -31,9 +31,7 @@
 	class={msg.role === 0
 		? "flex w-full gap-3"
 		: "flex w-full items-center gap-3"}
-	data-testid={msg.role === 0
-		? "display-answer"
-		: "display-question"}
+	data-testid={msg.role === 0 ? "display-answer" : "display-question"}
 >
 	<div
 		class={msg.role === 0
@@ -44,10 +42,15 @@
 	</div>
 	<div class="group relative flex items-start">
 		<div class="flex flex-col items-start">
-			<img src={msg.imgSrc} alt="Uploaded Image" class="m-2 max-w-28 max-h-28" />
-
+			{#if msg.imgSrc}
+				<img
+					src={msg.imgSrc}
+					alt="Uploaded Image"
+					class="max-w-28 m-2 max-h-28"
+				/>
+			{/if}
 			<p
-				class="xl:max-w-[65vw] max-w-[60vw] items-start whitespace-pre-line break-keep text-[0.8rem] leading-5 sm:max-w-[50rem]"
+				class="max-w-[60vw] items-start whitespace-pre-line break-keep text-[0.8rem] leading-5 sm:max-w-[50rem] xl:max-w-[65vw]"
 			>
 				{@html msg.content}
 			</p>

VisualQnA/ui/svelte/src/lib/modules/upload/imagePrompt.svelte

@@ -5,8 +5,8 @@
 
 <script>
 	import { createEventDispatcher } from "svelte";
-  import extreme_ironing from '$lib/assets/imageData/extreme_ironing.jpg';
-  import waterview from '$lib/assets/imageData/waterview.jpg';
+	import extreme_ironing from "$lib/assets/imageData/extreme_ironing.png";
+	import waterview from "$lib/assets/imageData/waterview.png";
 	import { base64ImageStore } from "$lib/shared/stores/common/Store";
 
 	let dispatch = createEventDispatcher();
@@ -14,16 +14,17 @@
 	let images = [
 		{
 			id: 1,
-      alt: 'Waterview',
+			alt: "Waterview",
 			imgurl: waterview,
-      prompt: 'What are the things I should be cautious about when I visit here?'
+			prompt:
+				"What are the things I should be cautious about when I visit here?",
 		},
 		{
 			id: 0,
-      alt: 'Extreme Ironing',
+			alt: "Extreme Ironing",
 			imgurl: extreme_ironing,
-      prompt: 'What is unusual about this image?'
-    }
+			prompt: "What is unusual about this image?",
+		},
 	];
 
 	let currentIndex = 0;
@@ -36,13 +37,15 @@
 		currentIndex = (currentIndex - 1 + images.length) % images.length;
 	}
 
-
 	async function handleImageClick() {
 		const imgUrl = images[currentIndex].imgurl;
+
 		const base64Data = await convertImageToBase64(imgUrl);
+
+		base64ImageStore.set(base64Data);
+
 		const currentPrompt = images[currentIndex].prompt;
 		dispatch("imagePrompt", { content: currentPrompt });
-    base64ImageStore.set(base64Data);
 	}
 
 	async function convertImageToBase64(url) {
@@ -57,11 +60,11 @@
 	}
 </script>
 
-<div class="flex w-full flex-col gap-3 rounded-xl bg-white p-5 my-2">
+<div class="my-2 flex w-full flex-col gap-3 rounded-xl bg-white p-5">
 	<p>Example</p>
-  <div class="relative w-full max-w-4xl mx-auto">
+	<div class="relative mx-auto w-full max-w-4xl">
 		<button
-      class="absolute left-0 top-1/2 transform -translate-y-1/2 z-10  w-8 h-8 rounded-full sm:w-10 sm:h-10 bg-white/30 dark:bg-gray-800/30 group-hover:bg-white/50 dark:group-hover:bg-gray-800/60 group-focus:ring-4 group-focus:ring-white dark:group-focus:ring-gray-800/70 group-focus:outline-none"
+			class="absolute left-0 top-1/2 z-10 h-8 w-8 -translate-y-1/2 transform rounded-full bg-white/30 group-hover:bg-white/50 group-focus:outline-none group-focus:ring-4 group-focus:ring-white dark:bg-gray-800/30 dark:group-hover:bg-gray-800/60 dark:group-focus:ring-gray-800/70 sm:h-10 sm:w-10"
 			on:click={prevImage}
 			aria-label="Previous image"
 		>
@@ -72,16 +75,18 @@
 			<img
 				src={images[currentIndex].imgurl}
 				alt={images[currentIndex].alt}
-        class="carousel-image w-full h-auto cursor-pointer"
+				class="carousel-image h-auto w-full cursor-pointer"
 				on:click={handleImageClick}
 			/>
-      <div class="absolute bottom-0 left-0 bg-opacity-55 bg-black text-white p-3 w-full">
+			<div
+				class="absolute bottom-0 left-0 w-full bg-black bg-opacity-55 p-3 text-white"
+			>
 				<p>{images[currentIndex].prompt}</p>
 			</div>
 		</div>
 
 		<button
-      class="absolute right-0 top-1/2 transform -translate-y-1/2 z-10  w-8 h-8 rounded-full sm:w-10 sm:h-10 bg-white/30 dark:bg-gray-800/30 group-hover:bg-white/50 dark:group-hover:bg-gray-800/60 group-focus:ring-4 group-focus:ring-white dark:group-focus:ring-gray-800/70 group-focus:outline-none"
+			class="absolute right-0 top-1/2 z-10 h-8 w-8 -translate-y-1/2 transform rounded-full bg-white/30 group-hover:bg-white/50 group-focus:outline-none group-focus:ring-4 group-focus:ring-white dark:bg-gray-800/30 dark:group-hover:bg-gray-800/60 dark:group-focus:ring-gray-800/70 sm:h-10 sm:w-10"
 			on:click={nextImage}
 			aria-label="Next image"
 		>

VisualQnA/ui/svelte/src/lib/modules/upload/uploadImg.svelte

@@ -6,11 +6,12 @@
 <script lang="ts">
 	import { base64ImageStore } from "$lib/shared/stores/common/Store";
 	import { Dropzone } from "flowbite-svelte";
+	import Pica from 'pica';
 
 	let value = [];
 	export let imageUrl = "";
 
-	$: if (imageUrl) {
+	$: if (imageUrl !== "") {
 		uploadImage();
 	}
 
@@ -57,13 +58,72 @@
 	const readFileAsBase64 = (file) => {
     const reader = new FileReader();
     reader.onload = () => {
-			const base64Data = reader.result; // Get Base64 data
-			base64ImageStore.set(base64Data); // Store the Base64 string in the store
-			imageUrl = URL.createObjectURL(file); // Keep the object URL for preview
+        const base64Data = reader.result;
+        const fileType = file.type;
+
+        if (!fileType.includes("png")) {
+            convertImageToPNG(base64Data); // Convert if not PNG
+        } else {
+            base64ImageStore.set(base64Data); // Store Base64
+        }
+
+        imageUrl = URL.createObjectURL(file); // Create URL for preview
     };
-		reader.readAsDataURL(file); // Read the file as a Data URL
+    reader.readAsDataURL(file); // Read file as Data URL
+};
+
+const convertImageToPNG = async (base64Data) => {
+    if (!base64Data || !base64Data.startsWith("data:image/")) {
+        console.error("Invalid Base64 data");
+        return;
+    }
+
+    console.log("Starting image conversion...");
+
+    const img = new Image();
+    img.src = base64Data;
+
+    img.onload = async () => {
+        const canvas = document.createElement("canvas");
+        const ctx = canvas.getContext("2d");
+        let width = img.width;
+        let height = img.height;
+
+        // Set resize factor to 1 (no scaling) to keep the original size
+        const scaleFactor = 0.1; // Resize factor (keep original size)
+        width = Math.floor(width * scaleFactor);
+        height = Math.floor(height * scaleFactor);
+
+        canvas.width = width;
+        canvas.height = height;
+
+        ctx.drawImage(img, 0, 0, width, height); // Draw the original image (no resizing)
+
+        const outputCanvas = document.createElement("canvas");
+        outputCanvas.width = width;
+        outputCanvas.height = height;
+
+        const pica = new Pica();
+
+        try {
+            // Resize and compress the image using Pica
+            await pica.resize(canvas, outputCanvas);
+
+            // Convert canvas to PNG format with data URL
+            const pngDataUrl = outputCanvas.toDataURL("image/png", 0.8); // Adjust quality (0.9 is high, between 0-1)
+
+            // Store the Base64 PNG image
+            base64ImageStore.set(pngDataUrl);
+        } catch (err) {
+            console.error("Error during image processing:", err);
+        }
     };
 
+    img.onerror = (err) => {
+        console.error("Error loading image:", err);
+    };
+};
+
 	const showFiles = (files) => {
 		if (files.length === 1) return files[0];
 		let concat = files.join(", ");