Image Music Track API

/**
 * Interprets an image as a spectrogram to generate a sound track.
 * The image's X-axis represents time, the Y-axis represents frequency,
 * and the pixel brightness represents the amplitude of that frequency.
 *
 * This function uses the Web Audio API to synthesize the sound. It creates
 * a series of complex tones using PeriodicWave for each column of pixels
 * in the image, which is much more performant than creating an oscillator per-pixel.
 *
 * The final output is an HTMLAudioElement with controls, embedded in a div,
 * which can be played directly by the user.
 *
 * @param {HTMLImageElement} originalImg The input image object.
 * @param {number} [duration=10] The desired duration of the audio track in seconds.
 * @param {number} [maxFrequency=8000] The maximum frequency in Hz represented by the top of the image.
 * @returns {Promise<HTMLDivElement>} A promise that resolves to a div element containing the playable audio track.
 */
async function processImage(originalImg, duration = 10, maxFrequency = 8000) {

    /**
     * Encodes an AudioBuffer into a WAV format Blob.
     * @param {AudioBuffer} buffer The audio buffer to convert.
     * @returns {Blob} A Blob containing the WAV file data.
     */
    function bufferToWave(buffer) {
        const numOfChan = buffer.numberOfChannels;
        const length = buffer.length * numOfChan * 2 + 44;
        const bufferArray = new ArrayBuffer(length);
        const view = new DataView(bufferArray);
        const channels = [];
        let i, sample;
        let pos = 0;

        // Helper function to write strings
        const writeString = (view, offset, string) => {
            for (let i = 0; i < string.length; i++) {
                view.setUint8(offset + i, string.charCodeAt(i));
            }
        };

        // RIFF header
        writeString(view, 0, 'RIFF');
        view.setUint32(4, 36 + buffer.length * 2, true);
        writeString(view, 8, 'WAVE');

        // fmt sub-chunk
        writeString(view, 12, 'fmt ');
        view.setUint32(16, 16, true);
        view.setUint16(20, 1, true); // PCM
        view.setUint16(22, numOfChan, true);
        view.setUint32(24, buffer.sampleRate, true);
        view.setUint32(28, buffer.sampleRate * 2 * numOfChan, true); // byte rate
        view.setUint16(32, numOfChan * 2, true); // block align
        view.setUint16(34, 16, true); // 16-bit

        // data sub-chunk
        writeString(view, 36, 'data');
        view.setUint32(40, buffer.length * 2, true);

        // Write the PCM samples
        pos = 44;
        for (i = 0; i < buffer.numberOfChannels; i++) {
            channels.push(buffer.getChannelData(i));
        }

        for (let i = 0; i < buffer.length; i++) {
            for (let j = 0; j < numOfChan; j++) {
                sample = Math.max(-1, Math.min(1, channels[j][i])); // clamp
                sample = (sample < 0 ? sample * 0x8000 : sample * 0x7FFF) | 0; // scale to 16-bit
                view.setInt16(pos, sample, true);
                pos += 2;
            }
        }
        return new Blob([view], {
            type: 'audio/wav'
        });
    }

    // 1. Get pixel data from the image
    const canvas = document.createElement('canvas');
    const ctx = canvas.getContext('2d', {
        willReadFrequently: true
    });
    const width = originalImg.naturalWidth;
    const height = originalImg.naturalHeight;
    canvas.width = width;
    canvas.height = height;
    ctx.drawImage(originalImg, 0, 0, width, height);
    const imageData = ctx.getImageData(0, 0, width, height).data;

    // 2. Set up Web Audio API Offline Context
    const sampleRate = 44100;
    const offlineCtx = new OfflineAudioContext(1, sampleRate * duration, sampleRate);

    // Use a master gain to prevent clipping and provide headroom
    const masterGain = offlineCtx.createGain();
    masterGain.gain.value = 0.5;
    masterGain.connect(offlineCtx.destination);


    // 3. Sonification logic using PeriodicWave
    const timeSliceDuration = duration / width;
    const fundamentalFrequency = maxFrequency / height;

    for (let x = 0; x < width; x++) {
        const currentTime = x * timeSliceDuration;

        // For each column (time slice), create a wave based on the pixel brightnesses
        const real = new Float32Array(height + 1); // for cosine components
        const imag = new Float32Array(height + 1); // for sine components

        for (let y = 0; y < height; y++) {
            const pixelIndex = (y * width + x) * 4;
            const r = imageData[pixelIndex];
            const g = imageData[pixelIndex + 1];
            const b = imageData[pixelIndex + 2];
            // Calculate luminance (brightness)
            const brightness = (0.299 * r + 0.587 * g + 0.114 * b) / 255;

            if (brightness > 0) {
                // Map y-position to harmonic. Invert y so bottom of image is low frequency.
                const harmonicIndex = height - y;
                imag[harmonicIndex] = brightness;
            }
        }

        try {
            const wave = offlineCtx.createPeriodicWave(real, imag, {
                disableNormalization: true
            });
            const osc = offlineCtx.createOscillator();
            osc.setPeriodicWave(wave);
            osc.frequency.value = fundamentalFrequency;
            osc.connect(masterGain);
            osc.start(currentTime);
            osc.stop(currentTime + timeSliceDuration);
        } catch (e) {
            // Some browsers (like Safari) throw an error if the wave arrays are all zero.
            // We can safely ignore these empty/black columns.
        }
    }

    // 4. Render audio and create the output element
    const renderedBuffer = await offlineCtx.startRendering();
    const wavBlob = bufferToWave(renderedBuffer);
    const audioUrl = URL.createObjectURL(wavBlob);

    const container = document.createElement('div');
    const title = document.createElement('p');
    title.textContent = 'Generated Audio Track';
    title.style.fontFamily = 'Arial, sans-serif';
    title.style.fontSize = '14px';
    title.style.color = '#333';
    title.style.margin = '0 0 5px 0';

    const audioElement = document.createElement('audio');
    audioElement.controls = true;
    audioElement.src = audioUrl;
    audioElement.style.width = '100%';

    container.appendChild(title);
    container.appendChild(audioElement);

    return container;
}