Update UI of the three demos: faqGen, VisualQnA, and DocSum. (#1528)

Signed-off-by: WenjiaoYue <wenjiao.yue@intel.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

DocSum/ui/gradio/docsum_ui_gradio.py

@@ -296,8 +296,8 @@ class DocSumUI:
                     audio_ui.render()
                 with gr.TabItem("Upload Video"):
                     video_ui.render()
-                with gr.TabItem("Enter URL"):
+                # with gr.TabItem("Enter URL"):
-                    url_ui.render()
+                #     url_ui.render()
 
         return self.demo
 

FaqGen/ui/svelte/src/lib/shared/Network.ts

@@ -52,6 +52,7 @@ export async function fetchTextStream(query: string | Blob, params: string, file
     }
     const reader = postResponse.body.getReader();
     const decoder = new TextDecoder("utf-8");
+
     let done, value;
 
     let buffer = ""; // Initialize a buffer
@@ -61,6 +62,7 @@ export async function fetchTextStream(query: string | Blob, params: string, file
 
       // Decode chunk and append to buffer
       const chunk = decoder.decode(value, { stream: true });
+
       buffer += chunk;
 
       // Use regex to clean and extract data
@@ -72,6 +74,21 @@ export async function fetchTextStream(query: string | Blob, params: string, file
         })
         .filter((line) => line); // Remove empty lines
 
+      const validJsonChunks = cleanedChunks.filter((item) => {
+        if (item === "[DONE]") {
+          return true;
+        }
+        try {
+          JSON.parse(item);
+          return true;
+        } catch (e) {
+          return false;
+        }
+      });
+
+      cleanedChunks.length = 0;
+      cleanedChunks.push(...validJsonChunks);
+
       for (const cleanedChunk of cleanedChunks) {
         // Further clean to ensure all unnecessary parts are removed
         yield cleanedChunk.replace(/^b'|['"]$/g, ""); // Again clean 'b' and other single or double quotes

FaqGen/ui/svelte/src/routes/+page.svelte

@@ -36,6 +36,7 @@
     urlSuffix: string,
     params: string
   ) => {
+    messages = "";
     // Fetch the stream
     const eventStream = await fetchTextStream(
       query,

VisualQnA/ui/svelte/package.json

@@ -23,6 +23,7 @@
     "@tailwindcss/typography": "0.5.7",
     "@types/debug": "4.1.7",
     "@types/node": "^20.12.13",
+    "@types/pica": "^9.0.5",
     "@typescript-eslint/eslint-plugin": "^5.27.0",
     "@typescript-eslint/parser": "^5.27.0",
     "autoprefixer": "^10.4.16",
@@ -51,6 +52,7 @@
     "flowbite-svelte-icons": "^1.4.0",
     "fuse.js": "^6.6.2",
     "lodash": "^4.17.21",
+    "pica": "^9.0.1",
     "playwright": "^1.44.0",
     "ramda": "^0.29.0",
     "sse.js": "^0.6.1",

VisualQnA/ui/svelte/src/lib/modules/chat/ChatMessage.svelte

@@ -31,9 +31,7 @@
 	class={msg.role === 0
 		? "flex w-full gap-3"
 		: "flex w-full items-center gap-3"}
-	data-testid={msg.role === 0
+	data-testid={msg.role === 0 ? "display-answer" : "display-question"}
-		? "display-answer"
-		: "display-question"}
 >
 	<div
 		class={msg.role === 0
@@ -44,10 +42,15 @@
 	</div>
 	<div class="group relative flex items-start">
 		<div class="flex flex-col items-start">
-			<img src={msg.imgSrc} alt="Uploaded Image" class="m-2 max-w-28 max-h-28" />
+			{#if msg.imgSrc}
-
+				<img
+					src={msg.imgSrc}
+					alt="Uploaded Image"
+					class="max-w-28 m-2 max-h-28"
+				/>
+			{/if}
 			<p
-				class="xl:max-w-[65vw] max-w-[60vw] items-start whitespace-pre-line break-keep text-[0.8rem] leading-5 sm:max-w-[50rem]"
+				class="max-w-[60vw] items-start whitespace-pre-line break-keep text-[0.8rem] leading-5 sm:max-w-[50rem] xl:max-w-[65vw]"
 			>
 				{@html msg.content}
 			</p>

VisualQnA/ui/svelte/src/lib/modules/upload/imagePrompt.svelte

@@ -5,93 +5,98 @@
 
 <script>
 	import { createEventDispatcher } from "svelte";
-  import extreme_ironing from '$lib/assets/imageData/extreme_ironing.jpg';
+	import extreme_ironing from "$lib/assets/imageData/extreme_ironing.png";
-  import waterview from '$lib/assets/imageData/waterview.jpg';
+	import waterview from "$lib/assets/imageData/waterview.png";
 	import { base64ImageStore } from "$lib/shared/stores/common/Store";
 
-  let dispatch = createEventDispatcher();
+	let dispatch = createEventDispatcher();
 
-  let images = [
+	let images = [
-    {
+		{
-      id: 1,
+			id: 1,
-      alt: 'Waterview',
+			alt: "Waterview",
-      imgurl: waterview,
+			imgurl: waterview,
-      prompt: 'What are the things I should be cautious about when I visit here?'
+			prompt:
-    },
+				"What are the things I should be cautious about when I visit here?",
-    {
+		},
-      id: 0,
+		{
-      alt: 'Extreme Ironing',
+			id: 0,
-      imgurl: extreme_ironing,
+			alt: "Extreme Ironing",
-      prompt: 'What is unusual about this image?'
+			imgurl: extreme_ironing,
-    }
+			prompt: "What is unusual about this image?",
-  ];
+		},
+	];
 
-  let currentIndex = 0;
+	let currentIndex = 0;
 
-  function nextImage() {
+	function nextImage() {
-    currentIndex = (currentIndex + 1) % images.length;
+		currentIndex = (currentIndex + 1) % images.length;
-  }
+	}
 
-  function prevImage() {
+	function prevImage() {
-    currentIndex = (currentIndex - 1 + images.length) % images.length;
+		currentIndex = (currentIndex - 1 + images.length) % images.length;
-  }
+	}
 
+	async function handleImageClick() {
+		const imgUrl = images[currentIndex].imgurl;
 
-  async function handleImageClick() {
+		const base64Data = await convertImageToBase64(imgUrl);
-    const imgUrl = images[currentIndex].imgurl;
-    const base64Data = await convertImageToBase64(imgUrl);
-    const currentPrompt = images[currentIndex].prompt;
-    dispatch("imagePrompt", { content: currentPrompt });
-    base64ImageStore.set(base64Data);
-  }
 
-  async function convertImageToBase64(url) {
+		base64ImageStore.set(base64Data);
-    const response = await fetch(url);
+
-    const blob = await response.blob();
+		const currentPrompt = images[currentIndex].prompt;
-    return new Promise((resolve, reject) => {
+		dispatch("imagePrompt", { content: currentPrompt });
-      const reader = new FileReader();
+	}
-      reader.onloadend = () => resolve(reader.result);
+
-      reader.onerror = reject;
+	async function convertImageToBase64(url) {
-      reader.readAsDataURL(blob);
+		const response = await fetch(url);
-    });
+		const blob = await response.blob();
-  }
+		return new Promise((resolve, reject) => {
+			const reader = new FileReader();
+			reader.onloadend = () => resolve(reader.result);
+			reader.onerror = reject;
+			reader.readAsDataURL(blob);
+		});
+	}
 </script>
 
-<div class="flex w-full flex-col gap-3 rounded-xl bg-white p-5 my-2">
+<div class="my-2 flex w-full flex-col gap-3 rounded-xl bg-white p-5">
-  <p>Example</p>
+	<p>Example</p>
-  <div class="relative w-full max-w-4xl mx-auto">
+	<div class="relative mx-auto w-full max-w-4xl">
-    <button
+		<button
-      class="absolute left-0 top-1/2 transform -translate-y-1/2 z-10  w-8 h-8 rounded-full sm:w-10 sm:h-10 bg-white/30 dark:bg-gray-800/30 group-hover:bg-white/50 dark:group-hover:bg-gray-800/60 group-focus:ring-4 group-focus:ring-white dark:group-focus:ring-gray-800/70 group-focus:outline-none"
+			class="absolute left-0 top-1/2 z-10 h-8 w-8 -translate-y-1/2 transform rounded-full bg-white/30 group-hover:bg-white/50 group-focus:outline-none group-focus:ring-4 group-focus:ring-white dark:bg-gray-800/30 dark:group-hover:bg-gray-800/60 dark:group-focus:ring-gray-800/70 sm:h-10 sm:w-10"
-      on:click={prevImage}
+			on:click={prevImage}
-      aria-label="Previous image"
+			aria-label="Previous image"
-    >
+		>
-      &#10094;
+			&#10094;
-    </button>
+		</button>
 
-    <div class="relative">
+		<div class="relative">
-      <img
+			<img
-        src={images[currentIndex].imgurl}
+				src={images[currentIndex].imgurl}
-        alt={images[currentIndex].alt}
+				alt={images[currentIndex].alt}
-        class="carousel-image w-full h-auto cursor-pointer"
+				class="carousel-image h-auto w-full cursor-pointer"
-        on:click={handleImageClick}
+				on:click={handleImageClick}
-      />
+			/>
-      <div class="absolute bottom-0 left-0 bg-opacity-55 bg-black text-white p-3 w-full">
+			<div
-        <p>{images[currentIndex].prompt}</p>
+				class="absolute bottom-0 left-0 w-full bg-black bg-opacity-55 p-3 text-white"
-      </div>
+			>
-    </div>
+				<p>{images[currentIndex].prompt}</p>
+			</div>
+		</div>
 
-    <button
+		<button
-      class="absolute right-0 top-1/2 transform -translate-y-1/2 z-10  w-8 h-8 rounded-full sm:w-10 sm:h-10 bg-white/30 dark:bg-gray-800/30 group-hover:bg-white/50 dark:group-hover:bg-gray-800/60 group-focus:ring-4 group-focus:ring-white dark:group-focus:ring-gray-800/70 group-focus:outline-none"
+			class="absolute right-0 top-1/2 z-10 h-8 w-8 -translate-y-1/2 transform rounded-full bg-white/30 group-hover:bg-white/50 group-focus:outline-none group-focus:ring-4 group-focus:ring-white dark:bg-gray-800/30 dark:group-hover:bg-gray-800/60 dark:group-focus:ring-gray-800/70 sm:h-10 sm:w-10"
-      on:click={nextImage}
+			on:click={nextImage}
-      aria-label="Next image"
+			aria-label="Next image"
-    >
+		>
-      &#10095;
+			&#10095;
-    </button>
+		</button>
-  </div>
+	</div>
 </div>
 
 <style>
-  .relative img {
+	.relative img {
-    object-fit: cover;
+		object-fit: cover;
-  }
+	}
 </style>

VisualQnA/ui/svelte/src/lib/modules/upload/uploadImg.svelte

@@ -6,11 +6,12 @@
 <script lang="ts">
 	import { base64ImageStore } from "$lib/shared/stores/common/Store";
 	import { Dropzone } from "flowbite-svelte";
+	import Pica from 'pica';
 
 	let value = [];
 	export let imageUrl = "";
 
-	$: if (imageUrl) {
+	$: if (imageUrl !== "") {
 		uploadImage();
 	}
 
@@ -47,7 +48,7 @@
 	};
 
 	const handleChange = (event) => {
-		const files = event.target.files;
+				const files = event.target.files;
 		if (files.length > 0) {
 			value = [files[0].name]; // Allow only one file selection
 			readFileAsBase64(files[0]); // Convert to Base64
@@ -55,14 +56,73 @@
 	};
 
 	const readFileAsBase64 = (file) => {
-		const reader = new FileReader();
+    const reader = new FileReader();
-		reader.onload = () => {
+    reader.onload = () => {
-			const base64Data = reader.result; // Get Base64 data
+        const base64Data = reader.result;
-			base64ImageStore.set(base64Data); // Store the Base64 string in the store
+        const fileType = file.type;
-			imageUrl = URL.createObjectURL(file); // Keep the object URL for preview
+
-		};
+        if (!fileType.includes("png")) {
-		reader.readAsDataURL(file); // Read the file as a Data URL
+            convertImageToPNG(base64Data); // Convert if not PNG
-	};
+        } else {
+            base64ImageStore.set(base64Data); // Store Base64
+        }
+
+        imageUrl = URL.createObjectURL(file); // Create URL for preview
+    };
+    reader.readAsDataURL(file); // Read file as Data URL
+};
+
+const convertImageToPNG = async (base64Data) => {
+    if (!base64Data || !base64Data.startsWith("data:image/")) {
+        console.error("Invalid Base64 data");
+        return;
+    }
+
+    console.log("Starting image conversion...");
+
+    const img = new Image();
+    img.src = base64Data;
+
+    img.onload = async () => {
+        const canvas = document.createElement("canvas");
+        const ctx = canvas.getContext("2d");
+        let width = img.width;
+        let height = img.height;
+
+        // Set resize factor to 1 (no scaling) to keep the original size
+        const scaleFactor = 0.1; // Resize factor (keep original size)
+        width = Math.floor(width * scaleFactor);
+        height = Math.floor(height * scaleFactor);
+
+        canvas.width = width;
+        canvas.height = height;
+
+        ctx.drawImage(img, 0, 0, width, height); // Draw the original image (no resizing)
+
+        const outputCanvas = document.createElement("canvas");
+        outputCanvas.width = width;
+        outputCanvas.height = height;
+
+        const pica = new Pica();
+
+        try {
+            // Resize and compress the image using Pica
+            await pica.resize(canvas, outputCanvas);
+
+            // Convert canvas to PNG format with data URL
+            const pngDataUrl = outputCanvas.toDataURL("image/png", 0.8); // Adjust quality (0.9 is high, between 0-1)
+
+            // Store the Base64 PNG image
+            base64ImageStore.set(pngDataUrl);
+        } catch (err) {
+            console.error("Error during image processing:", err);
+        }
+    };
+
+    img.onerror = (err) => {
+        console.error("Error loading image:", err);
+    };
+};
 
 	const showFiles = (files) => {
 		if (files.length === 1) return files[0];