Pyhton server integration

This commit is contained in:
2025-09-01 11:00:23 -06:00
parent 25ad465bf4
commit 9ab56dfbfe
9 changed files with 449 additions and 64 deletions

View File

@@ -7,13 +7,12 @@ import { Camera, History, VideoOff, Settings, Video } from 'lucide-react';
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select';
import { Slider } from '@/components/ui/slider';
import { Switch } from '@/components/ui/switch';
import { SHOE_DATABASE, type Shoe } from '@/lib/shoe-database';
import { detectShoe } from '@/lib/ml-classification';
import { type Shoe } from '@/lib/shoe-database';
import { addToHistory, getHistory } from '@/lib/history-storage';
import ShoeResultsPopup from '@/components/shoe-results-popup';
import HistorySidebar from '@/components/history-sidebar';
import { useDetection } from '@/lib/ml/use-detection';
import type { DetectionResult, DetectionConfig } from '@/lib/ml/types';
import type { DetectionResult } from '@/lib/ml/types';
type CameraStatus = 'loading' | 'active' | 'denied' | 'no_devices';
@@ -31,6 +30,7 @@ export default function HomePage() {
const [isPopupOpen, setPopupOpen] = useState(false);
const [isHistoryOpen, setHistoryOpen] = useState(false);
const [history, setHistory] = useState<Shoe[]>([]);
const [detectedSKU, setDetectedSKU] = useState<string | null>(null);
const [isSettingsPanelOpen, setSettingsPanelOpen] = useState(false);
// ML Detection state
@@ -40,8 +40,31 @@ export default function HomePage() {
const [lastSoundTime, setLastSoundTime] = useState(0);
// Initialize ML detection system first
const {
isLoading: isMLLoading,
metrics,
error: mlError,
initialize: initializeML,
startContinuous,
stopContinuous,
triggerDetection,
updateConfig,
config,
detectionEngine,
setDetectionCallback
} = useDetection({
modelVariant: 'standard', // Start with standard model
enableContinuous: true,
enableTrigger: true,
onDetection: undefined, // Will be set after handleDetection is defined
onError: (error) => {
console.error('ML Detection Error:', error);
}
});
// Clean detection callback - no canvas drawing needed
const handleDetection = useCallback((detection: DetectionResult | null) => {
const handleDetection = useCallback(async (detection: DetectionResult | null) => {
const callbackId = Math.random().toString(36).substr(2, 9);
console.log(`🔍 Detection callback received [${callbackId}]:`, detection);
setCurrentDetection(detection);
@@ -54,6 +77,37 @@ export default function HomePage() {
// Auto-trigger popup when shoe is detected with high confidence
if (detection && detection.confidence > 0.7) {
console.log(`🎯 HIGH CONFIDENCE SHOE DETECTED! [${callbackId}] Opening popup...`, detection);
// Call SKU identification API
if (videoRef.current && detectionEngine) {
try {
console.log(`🔍 [${callbackId}] Calling SKU identification...`);
const sku = await detectionEngine.identifyProductSKU(videoRef.current);
console.log(`📦 [${callbackId}] SKU result:`, sku);
setDetectedSKU(sku);
if (sku) {
// Create shoe object with SKU for history
const shoeWithSKU: Shoe = {
id: Date.now().toString(),
name: `Producto ${sku}`,
brand: 'Identificado por IA',
price: 'Precio por consultar',
image: '/placeholder.jpg',
confidence: detection.confidence,
sku: sku,
timestamp: new Date().toISOString()
};
const updatedHistory = addToHistory(shoeWithSKU);
setHistory(updatedHistory);
}
} catch (error) {
console.error(`❌ [${callbackId}] SKU identification failed:`, error);
setDetectedSKU(null);
}
}
setPopupOpen(true);
// Play detection sound with debouncing (max once per 2 seconds)
@@ -93,28 +147,14 @@ export default function HomePage() {
}
});
}
}, []);
}, [detectionEngine]);
// Initialize ML detection system
const {
isLoading: isMLLoading,
metrics,
error: mlError,
initialize: initializeML,
startContinuous,
stopContinuous,
triggerDetection,
updateConfig,
config
} = useDetection({
modelVariant: 'standard', // Start with standard model
enableContinuous: true,
enableTrigger: true,
onDetection: handleDetection,
onError: (error) => {
console.error('ML Detection Error:', error);
// Set the detection callback after handleDetection is defined
useEffect(() => {
if (setDetectionCallback && handleDetection) {
setDetectionCallback(handleDetection);
}
});
}, [handleDetection, setDetectionCallback]);
// Effect to clean up the stream when component unmounts or stream changes
useEffect(() => {
@@ -237,38 +277,55 @@ export default function HomePage() {
const handleScan = async () => {
if (detectionEnabled && triggerDetection) {
try {
console.log('Triggering ML detection...');
console.log('🎯 Triggering ML detection...');
const mlResult = await triggerDetection();
if (mlResult) {
// Use the existing detected shoe but with real ML confidence
const detected = detectShoe(SHOE_DATABASE);
if (detected) {
const updatedHistory = addToHistory(detected);
setHistory(updatedHistory);
console.log('✅ Shoe detected by ML, calling SKU identification...');
// Call SKU identification with the detected shoe
if (videoRef.current && detectionEngine) {
try {
const sku = await detectionEngine.identifyProductSKU(videoRef.current);
console.log('📦 Manual scan SKU result:', sku);
setDetectedSKU(sku);
if (sku) {
// Create shoe object with SKU for history
const shoeWithSKU: Shoe = {
id: Date.now().toString(),
name: `Producto ${sku}`,
brand: 'Identificado por IA',
price: 'Precio por consultar',
image: '/placeholder.jpg',
confidence: mlResult.confidence,
sku: sku,
timestamp: new Date().toISOString()
};
const updatedHistory = addToHistory(shoeWithSKU);
setHistory(updatedHistory);
}
setPopupOpen(true);
} catch (skuError) {
console.error('❌ SKU identification failed:', skuError);
// Still show popup even if SKU fails
setDetectedSKU(null);
setPopupOpen(true);
}
} else {
console.warn('⚠️ Video or detection engine not available for SKU call');
setPopupOpen(true);
}
} else {
console.log('No shoe detected by ML');
console.log('No shoe detected by ML');
}
} catch (error) {
console.error('ML detection failed, using fallback:', error);
// Fallback to original random detection
const detected = detectShoe(SHOE_DATABASE);
if (detected) {
const updatedHistory = addToHistory(detected);
setHistory(updatedHistory);
setPopupOpen(true);
}
console.error('ML detection failed:', error);
}
} else {
// Fallback to original random detection when ML is disabled
const detected = detectShoe(SHOE_DATABASE);
if (detected) {
const updatedHistory = addToHistory(detected);
setHistory(updatedHistory);
setPopupOpen(true);
}
console.log('⚠️ ML detection is disabled');
}
};
@@ -790,7 +847,7 @@ export default function HomePage() {
return (
<main className="relative h-screen w-screen bg-black overflow-hidden">
{renderContent()}
<ShoeResultsPopup isOpen={isPopupOpen} onOpenChange={setPopupOpen} />
<ShoeResultsPopup isOpen={isPopupOpen} onOpenChange={setPopupOpen} detectedSKU={detectedSKU} />
<HistorySidebar isOpen={isHistoryOpen} onOpenChange={setHistoryOpen} history={history} onItemClick={handleHistoryItemClick} />
</main>
);

View File

@@ -67,10 +67,10 @@ export default function HistorySidebar({ history, isOpen, onOpenChange, onItemCl
{shoe.name}
</h4>
<div className="flex items-center space-x-2 mt-1">
<p className="text-green-400 font-bold text-sm">${shoe.price.toFixed(2)}</p>
<p className="text-green-400 font-bold text-sm">${shoe.price}</p>
{shoe.promotions?.find(p => p.originalPrice) && (
<p className="text-gray-500 text-xs line-through">
${shoe.promotions.find(p => p.originalPrice)?.originalPrice?.toFixed(2)}
${shoe.promotions.find(p => p.originalPrice)?.originalPrice}
</p>
)}
</div>

View File

@@ -15,9 +15,10 @@ import { fetchProduct, getProductImages, getProductPricing, getProductVariants,
interface ShoeResultsPopupProps {
isOpen: boolean;
onOpenChange: (isOpen: boolean) => void;
detectedSKU?: string | null;
}
export default function ShoeResultsPopup({ isOpen, onOpenChange }: ShoeResultsPopupProps) {
export default function ShoeResultsPopup({ isOpen, onOpenChange, detectedSKU }: ShoeResultsPopupProps) {
const [product, setProduct] = useState<Product | null>(null);
const [loading, setLoading] = useState(false);
const [selectedVariant, setSelectedVariant] = useState<string>('');
@@ -213,16 +214,30 @@ export default function ShoeResultsPopup({ isOpen, onOpenChange }: ShoeResultsPo
{/* Product Name & Price */}
<div className="space-y-2">
{/* SKU Detection Badge */}
{detectedSKU && (
<div className="bg-green-500/20 border border-green-500/30 rounded-lg p-3 mb-4">
<div className="flex items-center gap-2">
<div className="w-2 h-2 bg-green-400 rounded-full animate-pulse"></div>
<span className="text-green-300 font-medium text-sm"> Producto Identificado por IA</span>
</div>
<div className="flex items-center gap-2 mt-2">
<Tag size={16} className="text-green-400" />
<span className="text-green-100 font-bold">SKU: {detectedSKU}</span>
</div>
</div>
)}
<h1 className="text-2xl font-bold tracking-tight text-white md:text-3xl leading-tight">
{product.productName}
</h1>
<div className="flex items-baseline gap-4">
<p className="text-3xl font-bold bg-gradient-to-r from-green-400 to-emerald-400 bg-clip-text text-transparent">
${pricing.price.toFixed(2)}
${pricing.price}
</p>
{pricing.listPrice > pricing.price && (
<p className="text-xl text-gray-500 line-through">
${pricing.listPrice.toFixed(2)}
${pricing.listPrice}
</p>
)}
</div>
@@ -335,7 +350,7 @@ export default function ShoeResultsPopup({ isOpen, onOpenChange }: ShoeResultsPo
<div className="flex items-center gap-2">
{variant.price > 0 && (
<span className="text-sm text-white/60">
${variant.price.toFixed(2)}
${variant.price}
</span>
)}
<Badge

View File

@@ -1,6 +1,7 @@
import type { DetectionConfig, DetectionResult, DetectionMetrics, DetectionMode } from './types';
import { DetectionWorkerManager } from './detection-worker-manager';
import { detectDeviceCapabilities, getRecommendedConfig } from './device-capabilities';
import { skuIdentificationService } from '../sku-identification';
// Extend window interface for TensorFlow.js
declare global {
@@ -150,6 +151,35 @@ export class DetectionEngine {
}
}
/**
* Identify product SKU from detected shoe
* @param videoElement - Video element to capture image from
* @returns Promise<string | null> - Product SKU if identified successfully
*/
async identifyProductSKU(videoElement: HTMLVideoElement): Promise<string | null> {
try {
console.log('🔍 Starting product SKU identification...');
// Capture high-quality image for SKU identification
const imageData = this.captureVideoFrame(videoElement, true);
// Call SKU identification service
const sku = await skuIdentificationService.identifySKU(imageData);
if (sku) {
console.log('✅ Product SKU identified:', sku);
} else {
console.log('❌ No valid SKU found');
}
return sku;
} catch (error) {
console.error('❌ SKU identification failed:', error);
return null;
}
}
/**
* Continuous detection loop
*/

View File

@@ -19,14 +19,18 @@ interface UseDetectionReturn {
error: string | null;
// Actions
initialize: (videoElement: HTMLVideoElement) => Promise<void>;
initialize: (videoElement: HTMLVideoElement) => Promise<DetectionEngine>;
startContinuous: () => void;
stopContinuous: () => void;
triggerDetection: () => Promise<DetectionResult | null>;
updateConfig: (config: Partial<DetectionConfig>) => Promise<void>;
setDetectionCallback: (callback: (detection: DetectionResult | null) => void) => void;
// Config
config: DetectionConfig | null;
// Engine reference
detectionEngine: DetectionEngine | null;
}
/**
@@ -41,6 +45,9 @@ export function useDetection(options: UseDetectionOptions = {}): UseDetectionRet
onError
} = options;
// Store the callback in a ref so it can be updated
const detectionCallbackRef = useRef<((detection: DetectionResult | null) => void) | undefined>(onDetection);
// State
const [isLoading, setIsLoading] = useState(false);
const [isDetecting, setIsDetecting] = useState(false);
@@ -55,7 +62,7 @@ export function useDetection(options: UseDetectionOptions = {}): UseDetectionRet
const initializationPromiseRef = useRef<Promise<void> | null>(null);
// Initialize detection engine
const initialize = useCallback(async (videoElement: HTMLVideoElement): Promise<void> => {
const initialize = useCallback(async (videoElement: HTMLVideoElement): Promise<DetectionEngine> => {
console.log('🚀 useDetection.initialize called:', { videoElement: !!videoElement });
// Prevent multiple initializations
@@ -78,7 +85,7 @@ export function useDetection(options: UseDetectionOptions = {}): UseDetectionRet
// Set up event listeners
engine.onDetection((detection) => {
setCurrentDetection(detection);
onDetection?.(detection);
detectionCallbackRef.current?.(detection);
});
engine.onMetrics((newMetrics) => {
@@ -96,6 +103,7 @@ export function useDetection(options: UseDetectionOptions = {}): UseDetectionRet
setConfig(initialConfig);
console.log('Detection hook initialized successfully');
return engine; // Return engine instance
} catch (err) {
const error = err instanceof Error ? err : new Error('Unknown initialization error');
@@ -175,7 +183,7 @@ export function useDetection(options: UseDetectionOptions = {}): UseDetectionRet
// Update current detection state
setCurrentDetection(detection);
onDetection?.(detection);
detectionCallbackRef.current?.(detection);
return detection;
} catch (err) {
@@ -185,7 +193,7 @@ export function useDetection(options: UseDetectionOptions = {}): UseDetectionRet
onError?.(error);
throw error;
}
}, [enableTrigger, onDetection, onError]);
}, [enableTrigger, onError]);
// Update configuration
const updateConfig = useCallback(async (newConfig: Partial<DetectionConfig>): Promise<void> => {
@@ -233,9 +241,15 @@ export function useDetection(options: UseDetectionOptions = {}): UseDetectionRet
stopContinuous,
triggerDetection,
updateConfig,
setDetectionCallback: (callback: (detection: DetectionResult | null) => void) => {
detectionCallbackRef.current = callback;
},
// Config
config
config,
// Engine reference
detectionEngine: detectionEngineRef.current
};
}

View File

@@ -1,10 +1,15 @@
export type Shoe = {
id: string;
name: string;
price: number;
description: string;
imageUrl: string;
colorOptions: { color: string; imageUrl: string }[];
brand?: string;
price: number | string;
description?: string;
image?: string;
imageUrl?: string;
colorOptions?: { color: string; imageUrl: string }[];
confidence?: number;
sku?: string;
timestamp?: string;
};
export const SHOE_DATABASE: Shoe[] = [

224
lib/sku-identification.ts Normal file
View File

@@ -0,0 +1,224 @@
interface SKUResponse {
status: boolean;
SKU: string | null;
}
interface CacheEntry {
response: SKUResponse;
timestamp: number;
}
/**
* Service for identifying product SKUs from shoe images using external API
*/
export class SKUIdentificationService {
private cache = new Map<string, CacheEntry>();
private readonly API_ENDPOINT = 'https://pegasus-working-bison.ngrok-free.app/predictfile';
private readonly CACHE_TTL = 60 * 60 * 1000; // 1 hour cache
private readonly MAX_CACHE_SIZE = 100; // Prevent memory leaks
/**
* Convert ImageData to File object for form-data upload
*/
private imageDataToFile(imageData: ImageData, filename: string = 'shoe.jpg'): File {
// Create canvas from ImageData
const canvas = document.createElement('canvas');
const ctx = canvas.getContext('2d')!;
canvas.width = imageData.width;
canvas.height = imageData.height;
// Put image data onto canvas
ctx.putImageData(imageData, 0, 0);
// Convert canvas to blob then to File
return new Promise<File>((resolve) => {
canvas.toBlob((blob) => {
if (blob) {
const file = new File([blob], filename, { type: 'image/jpeg' });
resolve(file);
}
}, 'image/jpeg', 0.8); // 80% quality for optimal size/quality balance
}) as any; // TypeScript workaround for sync return
}
/**
* Generate a simple hash from image data for caching
*/
private generateImageHash(imageData: ImageData): string {
// Sample pixels at regular intervals for hash generation
const step = Math.floor(imageData.data.length / 1000); // Sample ~1000 pixels
let hash = 0;
for (let i = 0; i < imageData.data.length; i += step) {
hash = ((hash << 5) - hash + imageData.data[i]) & 0xffffffff;
}
return hash.toString(36);
}
/**
* Clean expired cache entries and maintain size limit
*/
private cleanCache(): void {
const now = Date.now();
const entries = Array.from(this.cache.entries());
// Remove expired entries
for (const [key, entry] of entries) {
if (now - entry.timestamp > this.CACHE_TTL) {
this.cache.delete(key);
}
}
// If still over limit, remove oldest entries
if (this.cache.size > this.MAX_CACHE_SIZE) {
const sortedEntries = Array.from(this.cache.entries())
.sort((a, b) => a[1].timestamp - b[1].timestamp);
const entriesToRemove = sortedEntries.slice(0, this.cache.size - this.MAX_CACHE_SIZE);
for (const [key] of entriesToRemove) {
this.cache.delete(key);
}
}
}
/**
* Identify product SKU from shoe image
* @param imageData - Image data captured from video
* @returns Promise<string | null> - SKU if found, null otherwise
*/
async identifySKU(imageData: ImageData): Promise<string | null> {
try {
// Generate hash for caching
const imageHash = this.generateImageHash(imageData);
// Check cache first
const cached = this.cache.get(imageHash);
if (cached && (Date.now() - cached.timestamp) < this.CACHE_TTL) {
console.log('🎯 SKU cache hit:', cached.response);
return cached.response.status && cached.response.SKU ? cached.response.SKU : null;
}
console.log('📡 Calling SKU identification API...');
// Convert ImageData to File synchronously (simplified approach)
const canvas = document.createElement('canvas');
const ctx = canvas.getContext('2d')!;
canvas.width = imageData.width;
canvas.height = imageData.height;
ctx.putImageData(imageData, 0, 0);
// Convert canvas to blob and create File
const blob = await new Promise<Blob>((resolve) => {
canvas.toBlob((blob) => resolve(blob!), 'image/jpeg', 0.9); // Higher quality
});
const file = new File([blob], 'shoe.jpg', {
type: 'image/jpeg',
lastModified: Date.now()
});
console.log('📸 Created file:', {
name: file.name,
size: file.size,
type: file.type,
lastModified: file.lastModified
});
// Prepare form data
const formData = new FormData();
formData.append('file', file);
console.log('📤 FormData prepared with file key');
// Debug FormData contents
for (let [key, value] of formData.entries()) {
console.log(`📋 FormData entry: ${key} =`, value);
if (value instanceof File) {
console.log(` 📄 File details: ${value.name}, ${value.size} bytes, ${value.type}`);
}
}
console.log('🚀 Making API call to:', this.API_ENDPOINT);
console.log('🔧 Request method: POST');
console.log('🔧 Request headers:', { 'ngrok-skip-browser-warning': 'true' });
console.log('🔧 Request body type:', typeof formData, formData.constructor.name);
// Make API call with CORS headers
const response = await fetch(this.API_ENDPOINT, {
method: 'POST',
body: formData,
mode: 'cors', // Need CORS headers from server
headers: {
// Don't set Content-Type, let browser set it with boundary for multipart/form-data
'ngrok-skip-browser-warning': 'true', // Skip ngrok browser warning
'Accept': 'application/json', // Tell server we expect JSON
}
});
console.log('📡 Response status:', response.status, response.statusText);
console.log('📡 Response headers:', Object.fromEntries(response.headers.entries()));
if (!response.ok) {
throw new Error(`API call failed: ${response.status} ${response.statusText}`);
}
// Try to get response as text first to debug
const responseText = await response.text();
console.log('📡 Raw response text:', responseText);
// Try to parse as JSON
let data: SKUResponse;
try {
data = JSON.parse(responseText);
console.log('📦 SKU API response parsed:', data);
} catch (parseError) {
console.error('❌ Failed to parse JSON response:', parseError);
console.error('❌ Response was:', responseText);
throw new Error(`Invalid JSON response: ${parseError}`);
}
// Cache the response (both success and failure)
this.cleanCache();
this.cache.set(imageHash, {
response: data,
timestamp: Date.now()
});
// Return SKU only if status is true and SKU is not null
return data.status && data.SKU ? data.SKU : null;
} catch (error) {
console.error('❌ SKU identification failed:', error);
// Return null on any error (network, parsing, etc.)
return null;
}
}
/**
* Clear all cached results
*/
clearCache(): void {
this.cache.clear();
console.log('🗑️ SKU cache cleared');
}
/**
* Get cache statistics for debugging
*/
getCacheStats() {
const now = Date.now();
const entries = Array.from(this.cache.values());
const validEntries = entries.filter(entry => (now - entry.timestamp) < this.CACHE_TTL);
return {
totalEntries: this.cache.size,
validEntries: validEntries.length,
expiredEntries: this.cache.size - validEntries.length,
cacheHitRate: validEntries.length / Math.max(1, this.cache.size)
};
}
}
// Export singleton instance
export const skuIdentificationService = new SKUIdentificationService();

40
lib/sku-prediction-api.ts Normal file
View File

@@ -0,0 +1,40 @@
/**
* Communicates with the SKU prediction API to get a product SKU from an image.
*/
/**
* Sends an image file to the prediction endpoint and returns a product SKU.
*
* @param imageFile The image file to be sent for prediction.
* @returns A promise that resolves to the product SKU string if the prediction is successful, otherwise null.
*/
export async function getSkuFromImage(imageFile: File): Promise<string | null> {
const formData = new FormData();
formData.append("file", imageFile);
try {
const response = await fetch(
"https://pegasus-working-bison.ngrok-free.app/predictfile",
{
method: "POST",
body: formData,
}
);
if (!response.ok) {
console.error("SKU prediction API request failed:", response.status, response.statusText);
return null;
}
const result = await response.json();
if (result.status === true && typeof result.SKU === "string" && result.SKU) {
return result.SKU;
}
return null;
} catch (error) {
console.error("Error calling SKU prediction API:", error);
return null;
}
}

View File

@@ -3,7 +3,7 @@
"version": "0.1.0",
"private": true,
"scripts": {
"dev": "next dev --turbopack",
"dev": "next dev --turbopack --port=3011",
"build": "next build",
"start": "next start",
"lint": "next lint"