From 325806066162fd5778d4ebaa985460b05df77436 Mon Sep 17 00:00:00 2001 From: vadimwit Date: Mon, 9 Mar 2026 22:44:15 +0000 Subject: [PATCH] essentia additions: probably not good enough to add 10x app disk size --- package-lock.json | 23 ++++- package.json | 1 + src/App.jsx | 14 +-- src/components/AudioCapture.jsx | 23 ++++- src/components/BassFretboard.jsx | 149 +++++++++++++++++++++++++++++++ src/components/Settings.jsx | 38 ++++++++ src/lib/essentiaHPCP.js | 98 ++++++++++++++++++++ vite.config.js | 5 ++ 8 files changed, 342 insertions(+), 9 deletions(-) create mode 100644 src/components/BassFretboard.jsx create mode 100644 src/lib/essentiaHPCP.js diff --git a/package-lock.json b/package-lock.json index fbc5e8c..20efc5d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,14 +1,15 @@ { "name": "whattheflat", - "version": "0.1.0", + "version": "0.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "whattheflat", - "version": "0.1.0", + "version": "0.6.0", "dependencies": { "audiomotion-analyzer": "^4.5.4", + "essentia.js": "^0.1.3", "pitchy": "^4.1.0", "react": "^19.2.4", "react-dom": "^19.2.4" @@ -4183,6 +4184,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/essentia.js": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/essentia.js/-/essentia.js-0.1.3.tgz", + "integrity": "sha512-vVEPgeVMEBLRXbM5o5H5Rgu53EPHu25vyFKYg+flWLzI/nEoegJQez9FKRv8GR/KxIBwm+fXDEFL+MkQeoHaLw==", + "license": "AGPL-3.0", + "dependencies": { + "node-wav": "0.0.2" + } + }, "node_modules/exponential-backoff": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.3.tgz", @@ -5865,6 +5875,15 @@ "dev": true, "license": "MIT" }, + "node_modules/node-wav": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/node-wav/-/node-wav-0.0.2.tgz", + "integrity": "sha512-M6Rm/bbG6De/gKGxOpeOobx/dnGuP0dz40adqx38boqHhlWssBJZgLCPBNtb9NkrmnKYiV04xELq+R6PFOnoLA==", + "license": "MIT", + "engines": { + "node": ">=4.4.0" + } + }, "node_modules/nopt": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-8.1.0.tgz", diff --git a/package.json b/package.json index 4dc7417..788abbb 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ }, "dependencies": { "audiomotion-analyzer": "^4.5.4", + "essentia.js": "^0.1.3", "pitchy": "^4.1.0", "react": "^19.2.4", "react-dom": "^19.2.4" diff --git a/src/App.jsx b/src/App.jsx index 5ccf4fe..1bbc207 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -3,6 +3,7 @@ import AudioCapture from './components/AudioCapture' import ProgressionBanner from './components/ProgressionBanner' import ProgressionSuggestions from './components/ProgressionSuggestions' import Fretboard from './components/Fretboard' +import BassFretboard from './components/BassFretboard' import Tuner from './components/Tuner' import Piano from './components/Piano' import Settings from './components/Settings' @@ -24,6 +25,8 @@ const DEFAULTS = { // Audio input minClarity: 0.80, minVolume: 0.01, + // Experimental + useEssentia: false, } export default function App() { @@ -42,7 +45,7 @@ export default function App() { const [isListening, setIsListening] = useState(false) // ── Instrument view + tuner ─────────────────────────────────────────────────── - const [instrument, setInstrument] = useState('guitar') // 'guitar' | 'piano' + const [instrument, setInstrument] = useState('guitar') // 'guitar' | 'bass' | 'piano' const [showTuner, setShowTuner] = useState(false) const [showDebug, setShowDebug] = useState(false) const [monoColor, setMonoColor] = useState(false) @@ -394,6 +397,7 @@ export default function App() { className="appearance-none bg-surface border border-border hover:border-gray-500 focus:border-accent focus:outline-none rounded-lg pl-3 pr-7 py-1 text-sm text-gray-200 cursor-pointer transition-colors" > + ▾ @@ -493,6 +497,7 @@ export default function App() { isListening={isListening} minClarity={config.minClarity} minVolume={config.minVolume} + useEssentia={config.useEssentia} onPermissionError={() => { setMicError(true) setIsListening(false) @@ -517,10 +522,9 @@ export default function App() { {/* ── Instrument + progressions row ── */}
- {instrument === 'guitar' - ? - : - } + {instrument === 'guitar' && } + {instrument === 'bass' && } + {instrument === 'piano' && }
diff --git a/src/components/AudioCapture.jsx b/src/components/AudioCapture.jsx index 3ca6fdf..1a20220 100644 --- a/src/components/AudioCapture.jsx +++ b/src/components/AudioCapture.jsx @@ -1,6 +1,7 @@ import { useEffect, useRef, useCallback } from 'react' import { PitchDetector } from 'pitchy' import { NOTES } from '../lib/theory' +import { initEssentia, getEssentia, computeHPCP } from '../lib/essentiaHPCP' // ─── Why two analysers? ─────────────────────────────────────────────────────── // @@ -81,7 +82,7 @@ function detectBassPC(freqData, sampleRate, fftSize) { return ((bestMidi % 12) + 12) % 12 } -export default function AudioCapture({ onNote, onChroma, onOnset, isListening, minClarity = 0.80, minVolume = 0.01, onPermissionError }) { +export default function AudioCapture({ onNote, onChroma, onOnset, isListening, minClarity = 0.80, minVolume = 0.01, onPermissionError, useEssentia = false }) { const audioCtxRef = useRef(null) const timeBufRef = useRef(null) const freqBufRef = useRef(null) @@ -97,6 +98,7 @@ export default function AudioCapture({ onNote, onChroma, onOnset, isListening, m const onPermissionErrorRef = useRef(onPermissionError) const minClarityRef = useRef(minClarity) const minVolumeRef = useRef(minVolume) + const useEssentiaRef = useRef(useEssentia) const smoothRmsRef = useRef(0) const lastOnsetRef = useRef(0) useEffect(() => { onNoteRef.current = onNote }, [onNote]) @@ -105,6 +107,11 @@ export default function AudioCapture({ onNote, onChroma, onOnset, isListening, m useEffect(() => { onPermissionErrorRef.current = onPermissionError }, [onPermissionError]) useEffect(() => { minClarityRef.current = minClarity }, [minClarity]) useEffect(() => { minVolumeRef.current = minVolume }, [minVolume]) + useEffect(() => { useEssentiaRef.current = useEssentia }, [useEssentia]) + + // Pre-load Essentia WASM as soon as the component mounts — it's a singleton, + // so repeated calls just return the cached instance. + useEffect(() => { initEssentia().catch(console.error) }, []) const stop = useCallback(() => { activeRef.current = false @@ -174,8 +181,20 @@ export default function AudioCapture({ onNote, onChroma, onOnset, isListening, m if (onChromaRef.current) { const freqBuf = freqBufRef.current ca.getFloatFrequencyData(freqBuf) + let chroma + const essentia = useEssentiaRef.current ? getEssentia() : null + if (essentia) { + try { + chroma = computeHPCP(essentia, freqBuf, ctx.sampleRate) + } catch (err) { + console.error('[Essentia] computeHPCP failed, falling back to custom chroma:', err) + chroma = computeChroma(freqBuf, ctx.sampleRate, ca.fftSize) + } + } else { + chroma = computeChroma(freqBuf, ctx.sampleRate, ca.fftSize) + } onChromaRef.current( - computeChroma(freqBuf, ctx.sampleRate, ca.fftSize), + chroma, detectBassPC(freqBuf, ctx.sampleRate, ca.fftSize) ) } diff --git a/src/components/BassFretboard.jsx b/src/components/BassFretboard.jsx new file mode 100644 index 0000000..47319a4 --- /dev/null +++ b/src/components/BassFretboard.jsx @@ -0,0 +1,149 @@ +import { getPentatonicScale, getFullScale, getChordTones, NOTES } from '../lib/theory' + +// Standard bass tuning (top of diagram = highest string) +const STRINGS = [ + { label: 'G', root: 7, thickness: 1.5 }, + { label: 'D', root: 2, thickness: 2 }, + { label: 'A', root: 9, thickness: 2.5 }, + { label: 'E', root: 4, thickness: 3 }, +] + +const NUM_FRETS = 13 +const FRET_MARKERS = [3, 5, 7, 9] +const DOUBLE_MARKER = 12 + +// Layout +const NUT_X = 40 +const OPEN_X = 18 +const FRET_W = 52 +const STRING_H = 36 // wider spacing than guitar — 4 strings feel more spread +const PAD_T = 28 +const PAD_B = 18 +const BOARD_W = NUT_X + (NUM_FRETS - 1) * FRET_W + 10 +const BOARD_H = PAD_T + 3 * STRING_H + PAD_B +const DOT_R = 10 + +const fretX = f => NUT_X + (f - 0.5) * FRET_W +const stringY = si => PAD_T + si * STRING_H + +function noteColor(isChordTone, isPenta, isScale, mono = false) { + if (isChordTone) return { fill: '#a855f7', text: '#fff' } + if (isPenta) return mono ? { fill: '#c084fc', text: '#1e1b4b' } : { fill: '#f59e0b', text: '#000' } + if (isScale) return mono ? { fill: '#e9d5ff', text: '#581c87' } : { fill: '#374151', text: '#d1d5db' } + return null +} + +export default function BassFretboard({ keyInfo, currentChord, monoColor = false }) { + const { root, mode } = keyInfo ?? {} + + if (!root) return null + + const pentaSet = new Set(getPentatonicScale(root, mode).map(n => NOTES.indexOf(n))) + const scaleSet = new Set(getFullScale(root, mode).map(n => NOTES.indexOf(n))) + const chordSet = currentChord + ? new Set(getChordTones(currentChord).map(n => NOTES.indexOf(n))) + : new Set() + + return ( +
+

+ Bass — {root} {mode} + {currentChord && / {currentChord}} +

+ +
+ + {/* Fretboard background */} + + + {/* Position marker dots (centred between strings 1–2) */} + {FRET_MARKERS.map(f => ( + + ))} + {/* Double dot at 12 */} + + + + {/* Fret lines */} + {Array.from({ length: NUM_FRETS - 1 }, (_, i) => i + 1).map(f => ( + + ))} + + {/* Nut */} + + + {/* Strings — thicker as pitch drops */} + {STRINGS.map((s, si) => ( + + ))} + + {/* Fret numbers */} + {[3, 5, 7, 9, 12].map(f => ( + {f} + ))} + + {/* String labels */} + {STRINGS.map((s, si) => ( + {s.label} + ))} + + {/* Note dots */} + {STRINGS.flatMap((str, si) => + Array.from({ length: NUM_FRETS }, (_, fi) => { + const pc = (str.root + fi) % 12 + const color = noteColor(chordSet.has(pc), pentaSet.has(pc), scaleSet.has(pc), monoColor) + if (!color) return null + + const cx = fi === 0 ? OPEN_X : fretX(fi) + const cy = stringY(si) + + return ( + + + + {NOTES[pc]} + + + ) + }) + )} + +
+ +
+ ● Chord tone + ● Pentatonic + ● Scale +
+
+ ) +} diff --git a/src/components/Settings.jsx b/src/components/Settings.jsx index 0ea6e49..cfca135 100644 --- a/src/components/Settings.jsx +++ b/src/components/Settings.jsx @@ -70,6 +70,19 @@ const SETTINGS = [ }, ] +const TOGGLES = [ + { + section: 'Experimental', + items: [ + { + key: 'useEssentia', + label: 'Essentia HPCP', + desc: 'Replace custom chroma with Essentia\'s Harmonic Pitch Class Profile (spectral peaks + 8 harmonics). More accurate on polyphonic input. May be slower.', + }, + ], + }, +] + export default function Settings({ config, onChange, onClose, onReset }) { return (
@@ -97,6 +110,31 @@ export default function Settings({ config, onChange, onClose, onReset }) {
+ {TOGGLES.map(section => ( +
+

+ {section.section} +

+
+ {section.items.map(item => ( + + ))} +
+
+ ))} {SETTINGS.map(section => (

diff --git a/src/lib/essentiaHPCP.js b/src/lib/essentiaHPCP.js new file mode 100644 index 0000000..1de1364 --- /dev/null +++ b/src/lib/essentiaHPCP.js @@ -0,0 +1,98 @@ +// Essentia HPCP pipeline — replaces our custom computeChroma() when enabled. +// +// Pipeline (using Web Audio FFT data directly to skip WASM re-FFT): +// freqBuf (dB from AnalyserNode) → linear magnitude → SpectralPeaks → HPCP +// +// Pitch class ordering: +// Essentia HPCP[0] = A (referenceFrequency=440) +// Our chroma[0] = C +// Rotation applied: ourChroma[(i + 9) % 12] = HPCP[i] + +import { EssentiaWASM } from 'essentia.js/dist/essentia-wasm.es.js' +import Essentia from 'essentia.js/dist/essentia.js-core.es.js' + +let instance = null + +// Call once at startup — safe to call multiple times (returns cached instance). +export async function initEssentia() { + if (instance) return instance + console.log('[Essentia] loading WASM…') + await EssentiaWASM['ready'] + instance = new Essentia(EssentiaWASM) + console.log('[Essentia] ready —', instance.version) + return instance +} + +export function getEssentia() { return instance } + +const NOISE_FLOOR_DB = -65 + +// Compute 12-bin HPCP from Web Audio frequency-domain data. +// +// freqData — Float32Array from AnalyserNode.getFloatFrequencyData() (dB values) +// sampleRate — AudioContext.sampleRate +// +// Returns Float32Array(12) with same [C, C#, D, …, B] ordering as computeChroma(). +export function computeHPCP(essentia, freqData, sampleRate) { + const N = freqData.length + + // Convert dB → linear magnitude. Bins below noise floor stay 0. + const magSpectrum = new Float32Array(N) + for (let i = 0; i < N; i++) { + if (freqData[i] > NOISE_FLOOR_DB) { + magSpectrum[i] = Math.pow(10, freqData[i] / 20) + } + } + + const specVec = essentia.arrayToVector(magSpectrum) + + // SpectralPeaks derives bin → Hz as: freq = binIndex * sampleRate / (2*(N-1)) + // which matches Web Audio's FFT bin spacing (sampleRate / fftSize). + const peaks = essentia.SpectralPeaks( + specVec, + 0, // magnitudeThreshold — noise already zeroed above + 4000, // maxFrequency (Hz) + 60, // maxPeaks + 40, // minFrequency (Hz) + 'byMagnitude', + sampleRate + ) + specVec.delete() + + // If no peaks found (silence / below noise floor), return zeros. + if (peaks.frequencies.size() === 0) { + peaks.frequencies.delete() + peaks.magnitudes.delete() + return new Float32Array(12) + } + + const hpcpResult = essentia.HPCP( + peaks.frequencies, + peaks.magnitudes, + false, // bandPreset + 500, // bandSplitFrequency (unused when bandPreset=false) + 8, // harmonics + 4000, // maxFrequency (Hz) + false, // maxShifted + 40, // minFrequency (Hz) + false, // nonLinear + 'unitMax', // normalized + 440, // referenceFrequency (A4 = 440 Hz → HPCP[0] = A) + sampleRate, + 12, // size (bins per octave) + 'squaredCosine', // weightType + 0.5 // windowSize (octaves) + ) + peaks.frequencies.delete() + peaks.magnitudes.delete() + + // Rotate A-origin → C-origin to match our chroma convention. + // HPCP[0]=A → ourChroma[9]=A, so ourChroma[(i+9)%12] = HPCP[i] + const result = new Float32Array(12) + for (let i = 0; i < 12; i++) { + result[(i + 9) % 12] = hpcpResult.hpcp.get(i) + } + hpcpResult.hpcp.delete() + + return result +} diff --git a/vite.config.js b/vite.config.js index beb4b29..7c6efb9 100644 --- a/vite.config.js +++ b/vite.config.js @@ -11,4 +11,9 @@ export default defineConfig({ build: { outDir: 'dist', }, + optimizeDeps: { + // Essentia uses emscripten output that esbuild can't pre-bundle reliably. + // Exclude it so Vite serves the files directly from node_modules. + exclude: ['essentia.js'], + }, })