Pyhton server integration

lib/ml/detection-engine.ts

@@ -1,6 +1,7 @@
 import type { DetectionConfig, DetectionResult, DetectionMetrics, DetectionMode } from './types';
 import { DetectionWorkerManager } from './detection-worker-manager';
 import { detectDeviceCapabilities, getRecommendedConfig } from './device-capabilities';
+import { skuIdentificationService } from '../sku-identification';
 
 // Extend window interface for TensorFlow.js
 declare global {
@@ -150,6 +151,35 @@ export class DetectionEngine {
     }
   }
 
+  /**
+   * Identify product SKU from detected shoe
+   * @param videoElement - Video element to capture image from
+   * @returns Promise<string | null> - Product SKU if identified successfully
+   */
+  async identifyProductSKU(videoElement: HTMLVideoElement): Promise<string | null> {
+    try {
+      console.log('🔍 Starting product SKU identification...');
+      
+      // Capture high-quality image for SKU identification
+      const imageData = this.captureVideoFrame(videoElement, true);
+      
+      // Call SKU identification service
+      const sku = await skuIdentificationService.identifySKU(imageData);
+      
+      if (sku) {
+        console.log('✅ Product SKU identified:', sku);
+      } else {
+        console.log('❌ No valid SKU found');
+      }
+      
+      return sku;
+      
+    } catch (error) {
+      console.error('❌ SKU identification failed:', error);
+      return null;
+    }
+  }
+
   /**
    * Continuous detection loop
    */

lib/ml/use-detection.ts

@@ -19,14 +19,18 @@ interface UseDetectionReturn {
   error: string | null;
   
   // Actions
-  initialize: (videoElement: HTMLVideoElement) => Promise<void>;
+  initialize: (videoElement: HTMLVideoElement) => Promise<DetectionEngine>;
   startContinuous: () => void;
   stopContinuous: () => void;
   triggerDetection: () => Promise<DetectionResult | null>;
   updateConfig: (config: Partial<DetectionConfig>) => Promise<void>;
+  setDetectionCallback: (callback: (detection: DetectionResult | null) => void) => void;
   
   // Config
   config: DetectionConfig | null;
+  
+  // Engine reference
+  detectionEngine: DetectionEngine | null;
 }
 
 /**
@@ -41,6 +45,9 @@ export function useDetection(options: UseDetectionOptions = {}): UseDetectionRet
     onError
   } = options;
 
+  // Store the callback in a ref so it can be updated
+  const detectionCallbackRef = useRef<((detection: DetectionResult | null) => void) | undefined>(onDetection);
+
   // State
   const [isLoading, setIsLoading] = useState(false);
   const [isDetecting, setIsDetecting] = useState(false);
@@ -55,7 +62,7 @@ export function useDetection(options: UseDetectionOptions = {}): UseDetectionRet
   const initializationPromiseRef = useRef<Promise<void> | null>(null);
 
   // Initialize detection engine
-  const initialize = useCallback(async (videoElement: HTMLVideoElement): Promise<void> => {
+  const initialize = useCallback(async (videoElement: HTMLVideoElement): Promise<DetectionEngine> => {
     console.log('🚀 useDetection.initialize called:', { videoElement: !!videoElement });
     
     // Prevent multiple initializations
@@ -78,7 +85,7 @@ export function useDetection(options: UseDetectionOptions = {}): UseDetectionRet
         // Set up event listeners
         engine.onDetection((detection) => {
           setCurrentDetection(detection);
-          onDetection?.(detection);
+          detectionCallbackRef.current?.(detection);
         });
 
         engine.onMetrics((newMetrics) => {
@@ -96,6 +103,7 @@ export function useDetection(options: UseDetectionOptions = {}): UseDetectionRet
         setConfig(initialConfig);
 
         console.log('Detection hook initialized successfully');
+        return engine; // Return engine instance
 
       } catch (err) {
         const error = err instanceof Error ? err : new Error('Unknown initialization error');
@@ -175,7 +183,7 @@ export function useDetection(options: UseDetectionOptions = {}): UseDetectionRet
       
       // Update current detection state
       setCurrentDetection(detection);
-      onDetection?.(detection);
+      detectionCallbackRef.current?.(detection);
       
       return detection;
     } catch (err) {
@@ -185,7 +193,7 @@ export function useDetection(options: UseDetectionOptions = {}): UseDetectionRet
       onError?.(error);
       throw error;
     }
-  }, [enableTrigger, onDetection, onError]);
+  }, [enableTrigger, onError]);
 
   // Update configuration
   const updateConfig = useCallback(async (newConfig: Partial<DetectionConfig>): Promise<void> => {
@@ -233,9 +241,15 @@ export function useDetection(options: UseDetectionOptions = {}): UseDetectionRet
     stopContinuous,
     triggerDetection,
     updateConfig,
+    setDetectionCallback: (callback: (detection: DetectionResult | null) => void) => {
+      detectionCallbackRef.current = callback;
+    },
     
     // Config
-    config
+    config,
+    
+    // Engine reference
+    detectionEngine: detectionEngineRef.current
   };
 }
 

lib/shoe-database.ts

@@ -1,10 +1,15 @@
 export type Shoe = {
   id: string;
   name: string;
-  price: number;
-  description: string;
-  imageUrl: string;
-  colorOptions: { color: string; imageUrl: string }[];
+  brand?: string;
+  price: number | string;
+  description?: string;
+  image?: string;
+  imageUrl?: string;
+  colorOptions?: { color: string; imageUrl: string }[];
+  confidence?: number;
+  sku?: string;
+  timestamp?: string;
 };
 
 export const SHOE_DATABASE: Shoe[] = [

lib/sku-identification.ts

@@ -0,0 +1,224 @@
+interface SKUResponse {
+  status: boolean;
+  SKU: string | null;
+}
+
+interface CacheEntry {
+  response: SKUResponse;
+  timestamp: number;
+}
+
+/**
+ * Service for identifying product SKUs from shoe images using external API
+ */
+export class SKUIdentificationService {
+  private cache = new Map<string, CacheEntry>();
+  private readonly API_ENDPOINT = 'https://pegasus-working-bison.ngrok-free.app/predictfile';
+  private readonly CACHE_TTL = 60 * 60 * 1000; // 1 hour cache
+  private readonly MAX_CACHE_SIZE = 100; // Prevent memory leaks
+
+  /**
+   * Convert ImageData to File object for form-data upload
+   */
+  private imageDataToFile(imageData: ImageData, filename: string = 'shoe.jpg'): File {
+    // Create canvas from ImageData
+    const canvas = document.createElement('canvas');
+    const ctx = canvas.getContext('2d')!;
+    canvas.width = imageData.width;
+    canvas.height = imageData.height;
+    
+    // Put image data onto canvas
+    ctx.putImageData(imageData, 0, 0);
+    
+    // Convert canvas to blob then to File
+    return new Promise<File>((resolve) => {
+      canvas.toBlob((blob) => {
+        if (blob) {
+          const file = new File([blob], filename, { type: 'image/jpeg' });
+          resolve(file);
+        }
+      }, 'image/jpeg', 0.8); // 80% quality for optimal size/quality balance
+    }) as any; // TypeScript workaround for sync return
+  }
+
+  /**
+   * Generate a simple hash from image data for caching
+   */
+  private generateImageHash(imageData: ImageData): string {
+    // Sample pixels at regular intervals for hash generation
+    const step = Math.floor(imageData.data.length / 1000); // Sample ~1000 pixels
+    let hash = 0;
+    
+    for (let i = 0; i < imageData.data.length; i += step) {
+      hash = ((hash << 5) - hash + imageData.data[i]) & 0xffffffff;
+    }
+    
+    return hash.toString(36);
+  }
+
+  /**
+   * Clean expired cache entries and maintain size limit
+   */
+  private cleanCache(): void {
+    const now = Date.now();
+    const entries = Array.from(this.cache.entries());
+    
+    // Remove expired entries
+    for (const [key, entry] of entries) {
+      if (now - entry.timestamp > this.CACHE_TTL) {
+        this.cache.delete(key);
+      }
+    }
+    
+    // If still over limit, remove oldest entries
+    if (this.cache.size > this.MAX_CACHE_SIZE) {
+      const sortedEntries = Array.from(this.cache.entries())
+        .sort((a, b) => a[1].timestamp - b[1].timestamp);
+      
+      const entriesToRemove = sortedEntries.slice(0, this.cache.size - this.MAX_CACHE_SIZE);
+      for (const [key] of entriesToRemove) {
+        this.cache.delete(key);
+      }
+    }
+  }
+
+  /**
+   * Identify product SKU from shoe image
+   * @param imageData - Image data captured from video
+   * @returns Promise<string | null> - SKU if found, null otherwise
+   */
+  async identifySKU(imageData: ImageData): Promise<string | null> {
+    try {
+      // Generate hash for caching
+      const imageHash = this.generateImageHash(imageData);
+      
+      // Check cache first
+      const cached = this.cache.get(imageHash);
+      if (cached && (Date.now() - cached.timestamp) < this.CACHE_TTL) {
+        console.log('🎯 SKU cache hit:', cached.response);
+        return cached.response.status && cached.response.SKU ? cached.response.SKU : null;
+      }
+
+      console.log('📡 Calling SKU identification API...');
+      
+      // Convert ImageData to File synchronously (simplified approach)
+      const canvas = document.createElement('canvas');
+      const ctx = canvas.getContext('2d')!;
+      canvas.width = imageData.width;
+      canvas.height = imageData.height;
+      ctx.putImageData(imageData, 0, 0);
+      
+      // Convert canvas to blob and create File
+      const blob = await new Promise<Blob>((resolve) => {
+        canvas.toBlob((blob) => resolve(blob!), 'image/jpeg', 0.9); // Higher quality
+      });
+      
+      const file = new File([blob], 'shoe.jpg', { 
+        type: 'image/jpeg',
+        lastModified: Date.now()
+      });
+      
+      console.log('📸 Created file:', {
+        name: file.name,
+        size: file.size,
+        type: file.type,
+        lastModified: file.lastModified
+      });
+      
+      // Prepare form data
+      const formData = new FormData();
+      formData.append('file', file);
+      
+      console.log('📤 FormData prepared with file key');
+      
+      // Debug FormData contents
+      for (let [key, value] of formData.entries()) {
+        console.log(`📋 FormData entry: ${key} =`, value);
+        if (value instanceof File) {
+          console.log(`  📄 File details: ${value.name}, ${value.size} bytes, ${value.type}`);
+        }
+      }
+
+      console.log('🚀 Making API call to:', this.API_ENDPOINT);
+      console.log('🔧 Request method: POST');
+      console.log('🔧 Request headers:', { 'ngrok-skip-browser-warning': 'true' });
+      console.log('🔧 Request body type:', typeof formData, formData.constructor.name);
+      
+      // Make API call with CORS headers
+      const response = await fetch(this.API_ENDPOINT, {
+        method: 'POST',
+        body: formData,
+        mode: 'cors', // Need CORS headers from server
+        headers: {
+          // Don't set Content-Type, let browser set it with boundary for multipart/form-data
+          'ngrok-skip-browser-warning': 'true', // Skip ngrok browser warning
+          'Accept': 'application/json', // Tell server we expect JSON
+        }
+      });
+
+      console.log('📡 Response status:', response.status, response.statusText);
+      console.log('📡 Response headers:', Object.fromEntries(response.headers.entries()));
+
+      if (!response.ok) {
+        throw new Error(`API call failed: ${response.status} ${response.statusText}`);
+      }
+
+      // Try to get response as text first to debug
+      const responseText = await response.text();
+      console.log('📡 Raw response text:', responseText);
+
+      // Try to parse as JSON
+      let data: SKUResponse;
+      try {
+        data = JSON.parse(responseText);
+        console.log('📦 SKU API response parsed:', data);
+      } catch (parseError) {
+        console.error('❌ Failed to parse JSON response:', parseError);
+        console.error('❌ Response was:', responseText);
+        throw new Error(`Invalid JSON response: ${parseError}`);
+      }
+
+      // Cache the response (both success and failure)
+      this.cleanCache();
+      this.cache.set(imageHash, {
+        response: data,
+        timestamp: Date.now()
+      });
+
+      // Return SKU only if status is true and SKU is not null
+      return data.status && data.SKU ? data.SKU : null;
+      
+    } catch (error) {
+      console.error('❌ SKU identification failed:', error);
+      // Return null on any error (network, parsing, etc.)
+      return null;
+    }
+  }
+
+  /**
+   * Clear all cached results
+   */
+  clearCache(): void {
+    this.cache.clear();
+    console.log('🗑️ SKU cache cleared');
+  }
+
+  /**
+   * Get cache statistics for debugging
+   */
+  getCacheStats() {
+    const now = Date.now();
+    const entries = Array.from(this.cache.values());
+    const validEntries = entries.filter(entry => (now - entry.timestamp) < this.CACHE_TTL);
+    
+    return {
+      totalEntries: this.cache.size,
+      validEntries: validEntries.length,
+      expiredEntries: this.cache.size - validEntries.length,
+      cacheHitRate: validEntries.length / Math.max(1, this.cache.size)
+    };
+  }
+}
+
+// Export singleton instance
+export const skuIdentificationService = new SKUIdentificationService();

lib/sku-prediction-api.ts

@@ -0,0 +1,40 @@
+/**
+ * Communicates with the SKU prediction API to get a product SKU from an image.
+ */
+
+/**
+ * Sends an image file to the prediction endpoint and returns a product SKU.
+ *
+ * @param imageFile The image file to be sent for prediction.
+ * @returns A promise that resolves to the product SKU string if the prediction is successful, otherwise null.
+ */
+export async function getSkuFromImage(imageFile: File): Promise<string | null> {
+  const formData = new FormData();
+  formData.append("file", imageFile);
+
+  try {
+    const response = await fetch(
+      "https://pegasus-working-bison.ngrok-free.app/predictfile",
+      {
+        method: "POST",
+        body: formData,
+      }
+    );
+
+    if (!response.ok) {
+      console.error("SKU prediction API request failed:", response.status, response.statusText);
+      return null;
+    }
+
+    const result = await response.json();
+
+    if (result.status === true && typeof result.SKU === "string" && result.SKU) {
+      return result.SKU;
+    }
+
+    return null;
+  } catch (error) {
+    console.error("Error calling SKU prediction API:", error);
+    return null;
+  }
+}