Image Scene Identifier And Name Translator Tool

/**
 * Identifies objects in an image using a pre-trained model, translates their names to a specified language,
 * and draws the bounding boxes and translated labels on the image.
 *
 * This function requires an internet connection to load the machine learning model,
 * the translation API, and the font files on the first run.
 *
 * @param {HTMLImageElement} originalImg The original image element to process.
 * @param {string} [targetLang='es'] The ISO 639-1 code for the language to translate the object names to (e.g., 'es' for Spanish, 'fr' for French).
 * @param {number} [confidenceThreshold=0.5] The minimum confidence score (0-1) for an object detection to be displayed.
 * @returns {Promise<HTMLCanvasElement>} A promise that resolves to a canvas element with the detections drawn on it.
 */
async function processImage(originalImg, targetLang = 'es', confidenceThreshold = 0.5) {

    // Helper functions to dynamically load external scripts and fonts
    const loadScript = (url, globalVar) => {
        return new Promise((resolve, reject) => {
            if (window[globalVar]) {
                return resolve();
            }
            const script = document.createElement('script');
            script.src = url;
            script.onload = () => resolve();
            script.onerror = (err) => reject(new Error(`Failed to load script: ${url}`));
            document.head.appendChild(script);
        });
    };

    const loadFont = async (fontFamily, fontUrl) => {
        const fontFace = `16px ${fontFamily}`;
        if ([...document.fonts].some(f => f.family === fontFamily && f.status === 'loaded')) {
             await document.fonts.load(fontFace);
             return;
        }
        if (!document.querySelector(`link[href="${fontUrl}"]`)) {
            const link = document.createElement('link');
            link.rel = 'stylesheet';
            link.href = fontUrl;
            document.head.appendChild(link);
        }
        await document.fonts.load(fontFace);
    };

    const canvas = document.createElement('canvas');
    const ctx = canvas.getContext('2d');
    canvas.width = originalImg.naturalWidth;
    canvas.height = originalImg.naturalHeight;
    ctx.drawImage(originalImg, 0, 0);

    try {
        // Load necessary libraries: TensorFlow.js, COCO-SSD model, and Roboto font
        await Promise.all([
            loadScript('https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@3.11.0/dist/tf.min.js', 'tf'),
            loadScript('https://cdn.jsdelivr.net/npm/@tensorflow-models/coco-ssd@2.2.2/dist/coco-ssd.min.js', 'cocoSsd'),
            loadFont('Roboto', 'https://fonts.googleapis.com/css2?family=Roboto&display=swap')
        ]);
    } catch (error) {
        console.error("Failed to load required libraries:", error);
        ctx.fillStyle = 'red';
        ctx.font = '20px sans-serif';
        ctx.fillText('Error: Could not load required libraries.', 10, 30);
        return canvas;
    }

    // Load the COCO-SSD model.
    const model = await cocoSsd.load();

    // Detect objects in the image.
    const predictions = await model.detect(originalImg);

    // Filter predictions based on the confidence threshold.
    const filteredPredictions = predictions.filter(p => p.score >= confidenceThreshold);

    if (filteredPredictions.length === 0) {
        return canvas; // Return original image on canvas if nothing is detected
    }

    // Helper function to translate text using the MyMemory API
    const translateText = async (text, lang) => {
        if (lang.toLowerCase() === 'en') {
            return text; // No translation needed for English
        }
        try {
            // This is a free, public API with usage limits.
            const apiUrl = `https://api.mymemory.translated.net/get?q=${encodeURIComponent(text)}&langpair=en|${lang}`;
            const response = await fetch(apiUrl);
            if (!response.ok) {
                console.error('Translation API request failed:', response.statusText);
                return text; // Fallback to original text
            }
            const data = await response.json();
            return data.responseData?.translatedText || text;
        } catch (error) {
            console.error('Error during translation:', error);
            return text; // Fallback on network error
        }
    };

    // Translate all detected object class names.
    const translationPromises = filteredPredictions.map(p => translateText(p.class, targetLang));
    const translatedLabels = await Promise.all(translationPromises);

    // Draw the bounding boxes and labels on the canvas.
    ctx.font = '16px Roboto';
    ctx.lineWidth = 3;

    filteredPredictions.forEach((prediction, index) => {
        const [x, y, width, height] = prediction.bbox;
        const label = translatedLabels[index];
        const confidence = (prediction.score * 100).toFixed(1);
        const labelText = `${label} (${confidence}%)`;

        // Generate a consistent color for each object class
        let hash = 0;
        for (let i = 0; i < prediction.class.length; i++) {
            hash = prediction.class.charCodeAt(i) + ((hash << 5) - hash);
        }
        const color = `hsl(${hash % 360}, 90%, 40%)`;

        ctx.strokeStyle = color;
        ctx.fillStyle = color;

        // Draw bounding box
        ctx.beginPath();
        ctx.rect(x, y, width, height);
        ctx.stroke();

        // Draw label background
        const textMetrics = ctx.measureText(labelText);
        const textHeight = 18;
        ctx.fillRect(x, y > textHeight ? y - textHeight : y, textMetrics.width + 10, textHeight);

        // Draw label text
        ctx.fillStyle = '#FFFFFF';
        ctx.fillText(labelText, x + 5, y > textHeight ? y - 4 : y + 14);
    });

    return canvas;
}