Continuos detection added with 'beep'
This commit is contained in:
@@ -42,6 +42,7 @@ export class DetectionEngine {
|
||||
private lastDetectionCallbackTime?: number;
|
||||
|
||||
constructor() {
|
||||
console.log('🏗️ DetectionEngine constructor called');
|
||||
this.workerManager = new DetectionWorkerManager();
|
||||
|
||||
|
||||
@@ -49,7 +50,7 @@ export class DetectionEngine {
|
||||
const capabilities = detectDeviceCapabilities();
|
||||
this.config = getRecommendedConfig(capabilities);
|
||||
|
||||
console.log('Detection engine initialized', { capabilities, config: this.config });
|
||||
console.log('✅ Detection engine initialized', { capabilities, config: this.config });
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -58,17 +59,21 @@ export class DetectionEngine {
|
||||
async initialize(modelVariant?: 'quantized' | 'standard' | 'full', onProgress?: (progress: number) => void): Promise<void> {
|
||||
const variant = modelVariant || this.config.modelVariant;
|
||||
|
||||
console.log(`🔧 Initializing detection engine with ${variant} model...`);
|
||||
|
||||
try {
|
||||
// Load the model into the worker
|
||||
console.log('📥 Loading model into worker...');
|
||||
await this.workerManager.loadModel(variant, onProgress);
|
||||
|
||||
// Configure the worker with current settings
|
||||
console.log('⚙️ Configuring worker...');
|
||||
await this.workerManager.configure(this.config);
|
||||
|
||||
console.log(`Detection engine initialized with ${variant} model`);
|
||||
console.log(`✅ Detection engine initialized with ${variant} model`);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize detection engine:', error);
|
||||
console.error('❌ Failed to initialize detection engine:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -77,6 +82,12 @@ export class DetectionEngine {
|
||||
* Start continuous detection
|
||||
*/
|
||||
startContinuousDetection(videoElement: HTMLVideoElement): void {
|
||||
console.log('🚀 startContinuousDetection called:', {
|
||||
isRunning: this.isRunning,
|
||||
enableContinuous: this.config.enableContinuous,
|
||||
videoElement: !!videoElement
|
||||
});
|
||||
|
||||
if (this.isRunning) {
|
||||
console.warn('Detection already running');
|
||||
return;
|
||||
@@ -86,10 +97,11 @@ export class DetectionEngine {
|
||||
this.detectionMode = this.config.enableContinuous ? 'continuous' : 'trigger';
|
||||
|
||||
if (this.config.enableContinuous) {
|
||||
console.log('🔄 Starting continuous detection loop...');
|
||||
this.runContinuousLoop(videoElement);
|
||||
}
|
||||
|
||||
console.log(`Started detection in ${this.detectionMode} mode`);
|
||||
console.log(`✅ Started detection in ${this.detectionMode} mode`);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -111,64 +123,26 @@ export class DetectionEngine {
|
||||
try {
|
||||
console.log('🎯 Starting trigger detection (high quality)');
|
||||
|
||||
// Load TensorFlow.js if not already loaded
|
||||
if (!window.tf) {
|
||||
const tf = await import('@tensorflow/tfjs');
|
||||
await import('@tensorflow/tfjs-backend-webgl');
|
||||
await tf.setBackend('webgl');
|
||||
await tf.ready();
|
||||
window.tf = tf;
|
||||
console.log('✅ TensorFlow.js loaded in main thread');
|
||||
}
|
||||
// Capture image data for trigger detection (high quality)
|
||||
const imageData = this.captureVideoFrame(videoElement, true);
|
||||
|
||||
// Load model if not already loaded
|
||||
if (!this.model) {
|
||||
console.log('📥 Loading model in main thread...');
|
||||
this.model = await window.tf.loadGraphModel('/models/model.json');
|
||||
console.log('✅ Model loaded in main thread');
|
||||
}
|
||||
|
||||
// Capture and preprocess image with higher quality
|
||||
const tensor = window.tf.tidy(() => {
|
||||
const canvas = document.createElement('canvas');
|
||||
const ctx = canvas.getContext('2d')!;
|
||||
canvas.width = 300;
|
||||
canvas.height = 300;
|
||||
ctx.drawImage(videoElement, 0, 0, 300, 300);
|
||||
|
||||
const img = window.tf.browser.fromPixels(canvas);
|
||||
return img.expandDims(0);
|
||||
});
|
||||
|
||||
console.log('📸 Trigger detection - Input tensor shape:', tensor.shape);
|
||||
|
||||
// Run model inference
|
||||
const result = await this.model.executeAsync(tensor);
|
||||
tensor.dispose();
|
||||
|
||||
console.log('🔬 Trigger detection - Model output:', result);
|
||||
|
||||
// Return high-confidence detection for manual triggers
|
||||
const triggerDetection: DetectionResult = {
|
||||
bbox: [0.25, 0.25, 0.5, 0.5], // Different position than continuous
|
||||
confidence: 0.92, // Higher confidence for trigger
|
||||
class: 'shoe',
|
||||
timestamp: Date.now()
|
||||
};
|
||||
// Use worker manager for detection
|
||||
const detections = await this.workerManager.detect(imageData);
|
||||
const detection = detections.length > 0 ? detections[0] : null;
|
||||
|
||||
// Update metrics
|
||||
this.metrics.inferenceTime = performance.now() - startTime;
|
||||
this.metrics.detectionCount++;
|
||||
this.metrics.timestamp = Date.now();
|
||||
|
||||
console.log('✅ Trigger detection completed:', triggerDetection);
|
||||
console.log('✅ Trigger detection completed:', detection);
|
||||
|
||||
// Temporarily update the current detection to show trigger result
|
||||
if (this.onDetectionCallback) {
|
||||
this.onDetectionCallback(triggerDetection);
|
||||
// Trigger callbacks for immediate display
|
||||
if (this.onDetectionCallback && detection) {
|
||||
this.onDetectionCallback(detection);
|
||||
}
|
||||
|
||||
return triggerDetection;
|
||||
return detection;
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Trigger detection failed:', error);
|
||||
@@ -194,61 +168,30 @@ export class DetectionEngine {
|
||||
console.log('🔄 Running continuous detection frame...');
|
||||
|
||||
try {
|
||||
// Load TensorFlow.js if not already loaded
|
||||
if (!window.tf) {
|
||||
const tf = await import('@tensorflow/tfjs');
|
||||
await import('@tensorflow/tfjs-backend-webgl');
|
||||
await tf.setBackend('webgl');
|
||||
await tf.ready();
|
||||
window.tf = tf;
|
||||
}
|
||||
|
||||
// Load model if not already loaded
|
||||
if (!this.model) {
|
||||
this.model = await window.tf.loadGraphModel('/models/model.json');
|
||||
}
|
||||
|
||||
const startTime = performance.now();
|
||||
|
||||
// Capture and preprocess image (lower quality for continuous)
|
||||
const tensor = window.tf.tidy(() => {
|
||||
const canvas = document.createElement('canvas');
|
||||
const ctx = canvas.getContext('2d')!;
|
||||
canvas.width = 300;
|
||||
canvas.height = 300;
|
||||
ctx.drawImage(videoElement, 0, 0, 300, 300);
|
||||
|
||||
const img = window.tf.browser.fromPixels(canvas);
|
||||
return img.expandDims(0);
|
||||
});
|
||||
// Capture image data for continuous detection (lower quality)
|
||||
const imageData = this.captureVideoFrame(videoElement, false);
|
||||
|
||||
// Run model inference
|
||||
const result = await this.model.executeAsync(tensor);
|
||||
tensor.dispose();
|
||||
|
||||
// Return low-confidence detection for continuous mode (below popup threshold)
|
||||
const mockDetection: DetectionResult = {
|
||||
bbox: [0.1, 0.1, 0.3, 0.3],
|
||||
confidence: 0.5, // Medium confidence - shows bounding box but won't trigger popup
|
||||
class: 'shoe',
|
||||
timestamp: Date.now()
|
||||
};
|
||||
// Use worker manager for detection
|
||||
const detections = await this.workerManager.detect(imageData);
|
||||
const detection = detections.length > 0 ? detections[0] : null;
|
||||
|
||||
const inferenceTime = performance.now() - startTime;
|
||||
console.log('⚡ Continuous detection completed:', { time: inferenceTime, confidence: mockDetection.confidence });
|
||||
console.log('⚡ Continuous detection completed:', { time: inferenceTime, detection });
|
||||
|
||||
// Apply temporal filtering
|
||||
const validDetection = this.applyTemporalFiltering(mockDetection);
|
||||
const validDetection = this.applyTemporalFiltering(detection);
|
||||
|
||||
// Update metrics
|
||||
this.updateMetrics(inferenceTime);
|
||||
|
||||
// Trigger callbacks (only if we have a valid detection)
|
||||
// Use a debounced approach to avoid too frequent updates
|
||||
if (this.onDetectionCallback && validDetection) {
|
||||
// Only update if it's been at least 500ms since last detection callback
|
||||
if (this.onDetectionCallback) {
|
||||
// Only update if it's been at least 100ms since last detection callback for continuous
|
||||
const now = Date.now();
|
||||
if (!this.lastDetectionCallbackTime || now - this.lastDetectionCallbackTime > 500) {
|
||||
if (!this.lastDetectionCallbackTime || now - this.lastDetectionCallbackTime > 100) {
|
||||
this.onDetectionCallback(validDetection);
|
||||
this.lastDetectionCallbackTime = now;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user