388 lines
11 KiB
TypeScript
388 lines
11 KiB
TypeScript
import type { DetectionConfig, DetectionResult, DetectionMetrics, DetectionMode } from './types';
|
|
import { DetectionWorkerManager } from './detection-worker-manager';
|
|
import { detectDeviceCapabilities, getRecommendedConfig } from './device-capabilities';
|
|
import { skuIdentificationService } from '../sku-identification';
|
|
|
|
// Extend window interface for TensorFlow.js
|
|
declare global {
|
|
interface Window {
|
|
tf: unknown;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Main detection engine that coordinates continuous and trigger detection
|
|
*/
|
|
export class DetectionEngine {
|
|
private workerManager: DetectionWorkerManager;
|
|
private config: DetectionConfig;
|
|
private model: unknown = null; // TensorFlow.js model instance
|
|
|
|
// Detection state
|
|
private isRunning = false;
|
|
private detectionMode: DetectionMode = 'hybrid';
|
|
private frameSkipCounter = 0;
|
|
private detectionCount = 0;
|
|
|
|
// Temporal filtering
|
|
private detectionHistory: DetectionResult[] = [];
|
|
private lastValidDetection: DetectionResult | null = null;
|
|
|
|
// Performance tracking
|
|
private metrics: DetectionMetrics = {
|
|
fps: 0,
|
|
inferenceTime: 0,
|
|
memoryUsage: 0
|
|
};
|
|
|
|
// Event callbacks
|
|
private onDetectionCallback?: (detection: DetectionResult | null) => void;
|
|
private onMetricsCallback?: (metrics: DetectionMetrics) => void;
|
|
private lastDetectionCallbackTime?: number;
|
|
|
|
constructor() {
|
|
console.log('🏗️ DetectionEngine constructor called');
|
|
this.workerManager = new DetectionWorkerManager();
|
|
|
|
|
|
// Get device-optimized configuration
|
|
const capabilities = detectDeviceCapabilities();
|
|
this.config = getRecommendedConfig(capabilities);
|
|
|
|
console.log('✅ Detection engine initialized', { capabilities, config: this.config });
|
|
}
|
|
|
|
/**
|
|
* Initialize the detection engine with a specific model
|
|
*/
|
|
async initialize(modelVariant?: 'quantized' | 'standard' | 'full', onProgress?: (progress: number) => void): Promise<void> {
|
|
const variant = modelVariant || this.config.modelVariant;
|
|
|
|
console.log(`🔧 Initializing detection engine with ${variant} model...`);
|
|
|
|
try {
|
|
// Load the model into the worker
|
|
console.log('📥 Loading model into worker...');
|
|
await this.workerManager.loadModel(variant, onProgress);
|
|
|
|
// Configure the worker with current settings
|
|
console.log('⚙️ Configuring worker...');
|
|
await this.workerManager.configure(this.config);
|
|
|
|
console.log(`✅ Detection engine initialized with ${variant} model`);
|
|
|
|
} catch (error) {
|
|
console.error('❌ Failed to initialize detection engine:', error);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Start continuous detection
|
|
*/
|
|
startContinuousDetection(videoElement: HTMLVideoElement): void {
|
|
console.log('🚀 startContinuousDetection called:', {
|
|
isRunning: this.isRunning,
|
|
enableContinuous: this.config.enableContinuous,
|
|
videoElement: !!videoElement
|
|
});
|
|
|
|
if (this.isRunning) {
|
|
console.warn('Detection already running');
|
|
return;
|
|
}
|
|
|
|
this.isRunning = true;
|
|
this.detectionMode = this.config.enableContinuous ? 'continuous' : 'trigger';
|
|
|
|
if (this.config.enableContinuous) {
|
|
console.log('🔄 Starting continuous detection loop...');
|
|
this.runContinuousLoop(videoElement);
|
|
}
|
|
|
|
console.log(`✅ Started detection in ${this.detectionMode} mode`);
|
|
}
|
|
|
|
/**
|
|
* Stop continuous detection
|
|
*/
|
|
stopContinuousDetection(): void {
|
|
this.isRunning = false;
|
|
this.frameSkipCounter = 0;
|
|
this.detectionHistory = [];
|
|
console.log('Stopped continuous detection');
|
|
}
|
|
|
|
/**
|
|
* Perform single trigger detection - higher quality/confidence than continuous
|
|
*/
|
|
async triggerDetection(videoElement: HTMLVideoElement): Promise<DetectionResult | null> {
|
|
const startTime = performance.now();
|
|
|
|
try {
|
|
console.log('🎯 Starting trigger detection (high quality)');
|
|
|
|
// Capture image data for trigger detection (high quality)
|
|
const imageData = this.captureVideoFrame(videoElement, true);
|
|
|
|
// Use worker manager for detection
|
|
const detections = await this.workerManager.detect(imageData);
|
|
const detection = detections.length > 0 ? detections[0] : null;
|
|
|
|
// Update metrics
|
|
this.metrics.inferenceTime = performance.now() - startTime;
|
|
|
|
console.log('✅ Trigger detection completed:', detection);
|
|
|
|
// Trigger callbacks for immediate display
|
|
if (this.onDetectionCallback && detection) {
|
|
this.onDetectionCallback(detection);
|
|
}
|
|
|
|
return detection;
|
|
|
|
} catch (error) {
|
|
console.error('❌ Trigger detection failed:', error);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Identify product SKU from detected shoe
|
|
* @param videoElement - Video element to capture image from
|
|
* @returns Promise<string | null> - Product SKU if identified successfully
|
|
*/
|
|
async identifyProductSKU(videoElement: HTMLVideoElement): Promise<string | null> {
|
|
try {
|
|
console.log('🔍 Starting product SKU identification...');
|
|
|
|
// Capture high-quality image for SKU identification
|
|
const imageData = this.captureVideoFrame(videoElement, true);
|
|
|
|
// Call SKU identification service
|
|
const sku = await skuIdentificationService.identifySKU(imageData);
|
|
|
|
if (sku) {
|
|
console.log('✅ Product SKU identified:', sku);
|
|
} else {
|
|
console.log('❌ No valid SKU found');
|
|
}
|
|
|
|
return sku;
|
|
|
|
} catch (error) {
|
|
console.error('❌ SKU identification failed:', error);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Continuous detection loop
|
|
*/
|
|
private async runContinuousLoop(videoElement: HTMLVideoElement): Promise<void> {
|
|
if (!this.isRunning) return;
|
|
|
|
// Frame skipping logic
|
|
this.frameSkipCounter++;
|
|
if (this.frameSkipCounter < this.config.frameSkip) {
|
|
// Skip this frame, schedule next iteration
|
|
requestAnimationFrame(() => this.runContinuousLoop(videoElement));
|
|
return;
|
|
}
|
|
|
|
this.frameSkipCounter = 0;
|
|
// Only log every 10th detection to reduce noise
|
|
if (this.detectionCount % 10 === 0) {
|
|
console.log(`🔄 Continuous detection running... (${this.detectionCount} inferences)`);
|
|
}
|
|
|
|
try {
|
|
const startTime = performance.now();
|
|
|
|
// Capture image data for continuous detection (lower quality)
|
|
const imageData = this.captureVideoFrame(videoElement, false);
|
|
|
|
// Use worker manager for detection
|
|
const detections = await this.workerManager.detect(imageData);
|
|
const detection = detections.length > 0 ? detections[0] : null;
|
|
|
|
const inferenceTime = performance.now() - startTime;
|
|
console.log('⚡ Continuous detection completed:', { time: inferenceTime, detection });
|
|
|
|
// Apply temporal filtering
|
|
const validDetection = this.applyTemporalFiltering(detection);
|
|
|
|
// Update metrics
|
|
this.updateMetrics(inferenceTime);
|
|
|
|
// Trigger callbacks (only if we have a valid detection)
|
|
// Use a debounced approach to avoid too frequent updates
|
|
if (this.onDetectionCallback) {
|
|
// Only update if it's been at least 100ms since last detection callback for continuous
|
|
const now = Date.now();
|
|
if (!this.lastDetectionCallbackTime || now - this.lastDetectionCallbackTime > 100) {
|
|
this.onDetectionCallback(validDetection);
|
|
this.lastDetectionCallbackTime = now;
|
|
}
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error('Continuous detection error:', error);
|
|
}
|
|
|
|
// Schedule next iteration
|
|
if (this.isRunning) {
|
|
requestAnimationFrame(() => this.runContinuousLoop(videoElement));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Capture frame from video element
|
|
*/
|
|
private captureVideoFrame(videoElement: HTMLVideoElement, highQuality: boolean): ImageData {
|
|
const canvas = document.createElement('canvas');
|
|
const ctx = canvas.getContext('2d')!;
|
|
|
|
// Use different resolutions based on detection mode
|
|
const [targetWidth, targetHeight] = highQuality
|
|
? [640, 480] // High quality for trigger detection
|
|
: [320, 240]; // Lower quality for continuous detection
|
|
|
|
canvas.width = targetWidth;
|
|
canvas.height = targetHeight;
|
|
|
|
// Draw video frame to canvas
|
|
ctx.drawImage(videoElement, 0, 0, targetWidth, targetHeight);
|
|
|
|
// Extract image data
|
|
const imageData = ctx.getImageData(0, 0, targetWidth, targetHeight);
|
|
|
|
// Cleanup
|
|
canvas.remove();
|
|
|
|
return imageData;
|
|
}
|
|
|
|
/**
|
|
* Apply temporal consistency filtering to reduce false positives
|
|
*/
|
|
private applyTemporalFiltering(detection: DetectionResult | null): DetectionResult | null {
|
|
if (!detection) {
|
|
// No detection - decay previous detections
|
|
this.detectionHistory = this.detectionHistory.filter(d =>
|
|
Date.now() - d.timestamp < 1000 // Keep detections from last second
|
|
);
|
|
|
|
// If we have recent consistent detections, continue showing them
|
|
if (this.detectionHistory.length >= 2) {
|
|
return this.lastValidDetection;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
// Add current detection to history
|
|
this.detectionHistory.push(detection);
|
|
|
|
// Keep only recent detections (last 3 seconds)
|
|
this.detectionHistory = this.detectionHistory.filter(d =>
|
|
Date.now() - d.timestamp < 3000
|
|
);
|
|
|
|
// Check temporal consistency
|
|
const recentDetections = this.detectionHistory.filter(d =>
|
|
Date.now() - d.timestamp < 500 // Last 500ms
|
|
);
|
|
|
|
if (recentDetections.length >= 2) {
|
|
// We have consistent detections - this is likely valid
|
|
this.lastValidDetection = detection;
|
|
return detection;
|
|
}
|
|
|
|
// Not enough temporal consistency yet
|
|
return this.lastValidDetection;
|
|
}
|
|
|
|
/**
|
|
* Update performance metrics
|
|
*/
|
|
private updateMetrics(inferenceTime: number): void {
|
|
this.detectionCount++;
|
|
this.metrics = {
|
|
fps: 0, // Placeholder, as PerformanceMonitor is removed
|
|
inferenceTime: inferenceTime,
|
|
memoryUsage: this.getMemoryUsage()
|
|
};
|
|
|
|
if (this.onMetricsCallback) {
|
|
this.onMetricsCallback(this.metrics);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
* Get current memory usage (rough estimate)
|
|
*/
|
|
private getMemoryUsage(): number {
|
|
const memInfo = (performance as Performance & { memory?: { usedJSHeapSize: number } }).memory;
|
|
if (memInfo && memInfo.usedJSHeapSize) {
|
|
return memInfo.usedJSHeapSize / (1024 * 1024); // MB
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
|
|
/**
|
|
* Set detection callback
|
|
*/
|
|
onDetection(callback: (detection: DetectionResult | null) => void): void {
|
|
this.onDetectionCallback = callback;
|
|
}
|
|
|
|
/**
|
|
* Set metrics callback
|
|
*/
|
|
onMetrics(callback: (metrics: DetectionMetrics) => void): void {
|
|
this.onMetricsCallback = callback;
|
|
}
|
|
|
|
/**
|
|
* Update configuration
|
|
*/
|
|
async updateConfig(newConfig: Partial<DetectionConfig>): Promise<void> {
|
|
this.config = { ...this.config, ...newConfig };
|
|
await this.workerManager.configure(this.config);
|
|
console.log('Configuration updated:', this.config);
|
|
}
|
|
|
|
/**
|
|
* Get current configuration
|
|
*/
|
|
getConfig(): DetectionConfig {
|
|
return { ...this.config };
|
|
}
|
|
|
|
/**
|
|
* Get current metrics
|
|
*/
|
|
getMetrics(): DetectionMetrics {
|
|
return { ...this.metrics };
|
|
}
|
|
|
|
/**
|
|
* Check if detection is running
|
|
*/
|
|
isDetectionRunning(): boolean {
|
|
return this.isRunning;
|
|
}
|
|
|
|
/**
|
|
* Destroy the detection engine
|
|
*/
|
|
destroy(): void {
|
|
this.stopContinuousDetection();
|
|
this.workerManager.destroy();
|
|
}
|
|
} |