essentia additions: probably not good enough to add 10x app disk size

This commit is contained in:
vadimwit
2026-03-09 22:44:15 +00:00
parent 22b8f1fe04
commit 3258060661
8 changed files with 342 additions and 9 deletions
+21 -2
View File
@@ -1,14 +1,15 @@
{
"name": "whattheflat",
"version": "0.1.0",
"version": "0.6.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "whattheflat",
"version": "0.1.0",
"version": "0.6.0",
"dependencies": {
"audiomotion-analyzer": "^4.5.4",
"essentia.js": "^0.1.3",
"pitchy": "^4.1.0",
"react": "^19.2.4",
"react-dom": "^19.2.4"
@@ -4183,6 +4184,15 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/essentia.js": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/essentia.js/-/essentia.js-0.1.3.tgz",
"integrity": "sha512-vVEPgeVMEBLRXbM5o5H5Rgu53EPHu25vyFKYg+flWLzI/nEoegJQez9FKRv8GR/KxIBwm+fXDEFL+MkQeoHaLw==",
"license": "AGPL-3.0",
"dependencies": {
"node-wav": "0.0.2"
}
},
"node_modules/exponential-backoff": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.3.tgz",
@@ -5865,6 +5875,15 @@
"dev": true,
"license": "MIT"
},
"node_modules/node-wav": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/node-wav/-/node-wav-0.0.2.tgz",
"integrity": "sha512-M6Rm/bbG6De/gKGxOpeOobx/dnGuP0dz40adqx38boqHhlWssBJZgLCPBNtb9NkrmnKYiV04xELq+R6PFOnoLA==",
"license": "MIT",
"engines": {
"node": ">=4.4.0"
}
},
"node_modules/nopt": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/nopt/-/nopt-8.1.0.tgz",
+1
View File
@@ -17,6 +17,7 @@
},
"dependencies": {
"audiomotion-analyzer": "^4.5.4",
"essentia.js": "^0.1.3",
"pitchy": "^4.1.0",
"react": "^19.2.4",
"react-dom": "^19.2.4"
+9 -5
View File
@@ -3,6 +3,7 @@ import AudioCapture from './components/AudioCapture'
import ProgressionBanner from './components/ProgressionBanner'
import ProgressionSuggestions from './components/ProgressionSuggestions'
import Fretboard from './components/Fretboard'
import BassFretboard from './components/BassFretboard'
import Tuner from './components/Tuner'
import Piano from './components/Piano'
import Settings from './components/Settings'
@@ -24,6 +25,8 @@ const DEFAULTS = {
// Audio input
minClarity: 0.80,
minVolume: 0.01,
// Experimental
useEssentia: false,
}
export default function App() {
@@ -42,7 +45,7 @@ export default function App() {
const [isListening, setIsListening] = useState(false)
// ── Instrument view + tuner ───────────────────────────────────────────────────
const [instrument, setInstrument] = useState('guitar') // 'guitar' | 'piano'
const [instrument, setInstrument] = useState('guitar') // 'guitar' | 'bass' | 'piano'
const [showTuner, setShowTuner] = useState(false)
const [showDebug, setShowDebug] = useState(false)
const [monoColor, setMonoColor] = useState(false)
@@ -394,6 +397,7 @@ export default function App() {
className="appearance-none bg-surface border border-border hover:border-gray-500 focus:border-accent focus:outline-none rounded-lg pl-3 pr-7 py-1 text-sm text-gray-200 cursor-pointer transition-colors"
>
<option value="guitar">Guitar</option>
<option value="bass">Bass</option>
<option value="piano">Piano</option>
</select>
<span className="pointer-events-none absolute right-2 top-1/2 -translate-y-1/2 text-gray-500 text-xs"></span>
@@ -493,6 +497,7 @@ export default function App() {
isListening={isListening}
minClarity={config.minClarity}
minVolume={config.minVolume}
useEssentia={config.useEssentia}
onPermissionError={() => {
setMicError(true)
setIsListening(false)
@@ -517,10 +522,9 @@ export default function App() {
{/* ── Instrument + progressions row ── */}
<div className="flex gap-3 mb-3 items-stretch">
<div className="w-full lg:w-[70%] min-w-0">
{instrument === 'guitar'
? <Fretboard keyInfo={effectiveKey} currentChord={currentChord} pentatonicOnly={false} monoColor={monoColor} />
: <Piano keyInfo={effectiveKey} currentChord={currentChord} monoColor={monoColor} />
}
{instrument === 'guitar' && <Fretboard keyInfo={effectiveKey} currentChord={currentChord} pentatonicOnly={false} monoColor={monoColor} />}
{instrument === 'bass' && <BassFretboard keyInfo={effectiveKey} currentChord={currentChord} monoColor={monoColor} />}
{instrument === 'piano' && <Piano keyInfo={effectiveKey} currentChord={currentChord} monoColor={monoColor} />}
</div>
<div className="hidden lg:block w-[30%] min-w-0 relative">
+21 -2
View File
@@ -1,6 +1,7 @@
import { useEffect, useRef, useCallback } from 'react'
import { PitchDetector } from 'pitchy'
import { NOTES } from '../lib/theory'
import { initEssentia, getEssentia, computeHPCP } from '../lib/essentiaHPCP'
// Why two analysers?
//
@@ -81,7 +82,7 @@ function detectBassPC(freqData, sampleRate, fftSize) {
return ((bestMidi % 12) + 12) % 12
}
export default function AudioCapture({ onNote, onChroma, onOnset, isListening, minClarity = 0.80, minVolume = 0.01, onPermissionError }) {
export default function AudioCapture({ onNote, onChroma, onOnset, isListening, minClarity = 0.80, minVolume = 0.01, onPermissionError, useEssentia = false }) {
const audioCtxRef = useRef(null)
const timeBufRef = useRef(null)
const freqBufRef = useRef(null)
@@ -97,6 +98,7 @@ export default function AudioCapture({ onNote, onChroma, onOnset, isListening, m
const onPermissionErrorRef = useRef(onPermissionError)
const minClarityRef = useRef(minClarity)
const minVolumeRef = useRef(minVolume)
const useEssentiaRef = useRef(useEssentia)
const smoothRmsRef = useRef(0)
const lastOnsetRef = useRef(0)
useEffect(() => { onNoteRef.current = onNote }, [onNote])
@@ -105,6 +107,11 @@ export default function AudioCapture({ onNote, onChroma, onOnset, isListening, m
useEffect(() => { onPermissionErrorRef.current = onPermissionError }, [onPermissionError])
useEffect(() => { minClarityRef.current = minClarity }, [minClarity])
useEffect(() => { minVolumeRef.current = minVolume }, [minVolume])
useEffect(() => { useEssentiaRef.current = useEssentia }, [useEssentia])
// Pre-load Essentia WASM as soon as the component mounts it's a singleton,
// so repeated calls just return the cached instance.
useEffect(() => { initEssentia().catch(console.error) }, [])
const stop = useCallback(() => {
activeRef.current = false
@@ -174,8 +181,20 @@ export default function AudioCapture({ onNote, onChroma, onOnset, isListening, m
if (onChromaRef.current) {
const freqBuf = freqBufRef.current
ca.getFloatFrequencyData(freqBuf)
let chroma
const essentia = useEssentiaRef.current ? getEssentia() : null
if (essentia) {
try {
chroma = computeHPCP(essentia, freqBuf, ctx.sampleRate)
} catch (err) {
console.error('[Essentia] computeHPCP failed, falling back to custom chroma:', err)
chroma = computeChroma(freqBuf, ctx.sampleRate, ca.fftSize)
}
} else {
chroma = computeChroma(freqBuf, ctx.sampleRate, ca.fftSize)
}
onChromaRef.current(
computeChroma(freqBuf, ctx.sampleRate, ca.fftSize),
chroma,
detectBassPC(freqBuf, ctx.sampleRate, ca.fftSize)
)
}
+149
View File
@@ -0,0 +1,149 @@
import { getPentatonicScale, getFullScale, getChordTones, NOTES } from '../lib/theory'
// Standard bass tuning (top of diagram = highest string)
const STRINGS = [
{ label: 'G', root: 7, thickness: 1.5 },
{ label: 'D', root: 2, thickness: 2 },
{ label: 'A', root: 9, thickness: 2.5 },
{ label: 'E', root: 4, thickness: 3 },
]
const NUM_FRETS = 13
const FRET_MARKERS = [3, 5, 7, 9]
const DOUBLE_MARKER = 12
// Layout
const NUT_X = 40
const OPEN_X = 18
const FRET_W = 52
const STRING_H = 36 // wider spacing than guitar 4 strings feel more spread
const PAD_T = 28
const PAD_B = 18
const BOARD_W = NUT_X + (NUM_FRETS - 1) * FRET_W + 10
const BOARD_H = PAD_T + 3 * STRING_H + PAD_B
const DOT_R = 10
const fretX = f => NUT_X + (f - 0.5) * FRET_W
const stringY = si => PAD_T + si * STRING_H
function noteColor(isChordTone, isPenta, isScale, mono = false) {
if (isChordTone) return { fill: '#a855f7', text: '#fff' }
if (isPenta) return mono ? { fill: '#c084fc', text: '#1e1b4b' } : { fill: '#f59e0b', text: '#000' }
if (isScale) return mono ? { fill: '#e9d5ff', text: '#581c87' } : { fill: '#374151', text: '#d1d5db' }
return null
}
export default function BassFretboard({ keyInfo, currentChord, monoColor = false }) {
const { root, mode } = keyInfo ?? {}
if (!root) return null
const pentaSet = new Set(getPentatonicScale(root, mode).map(n => NOTES.indexOf(n)))
const scaleSet = new Set(getFullScale(root, mode).map(n => NOTES.indexOf(n)))
const chordSet = currentChord
? new Set(getChordTones(currentChord).map(n => NOTES.indexOf(n)))
: new Set()
return (
<div className="bg-panel border border-border rounded-2xl p-6">
<p className="text-sm text-gray-500 uppercase tracking-widest mb-4">
Bass {root} {mode}
{currentChord && <span className="text-amber-400 ml-2">/ {currentChord}</span>}
</p>
<div>
<svg
viewBox={`0 0 ${BOARD_W} ${BOARD_H}`}
width="100%"
height="auto"
style={{ display: 'block' }}
>
{/* Fretboard background */}
<rect x={NUT_X} y={PAD_T - 6} width={BOARD_W - NUT_X - 4} height={3 * STRING_H + 12}
fill="#1a120b" rx={2} />
{/* Position marker dots (centred between strings 12) */}
{FRET_MARKERS.map(f => (
<circle key={f}
cx={fretX(f)} cy={PAD_T + 1.5 * STRING_H}
r={5} fill="#3a2a1a" />
))}
{/* Double dot at 12 */}
<circle cx={fretX(DOUBLE_MARKER)} cy={PAD_T + 0.5 * STRING_H} r={5} fill="#3a2a1a" />
<circle cx={fretX(DOUBLE_MARKER)} cy={PAD_T + 2.5 * STRING_H} r={5} fill="#3a2a1a" />
{/* Fret lines */}
{Array.from({ length: NUM_FRETS - 1 }, (_, i) => i + 1).map(f => (
<line key={f}
x1={NUT_X + f * FRET_W} y1={PAD_T - 6}
x2={NUT_X + f * FRET_W} y2={PAD_T + 3 * STRING_H + 6}
stroke={f === DOUBLE_MARKER ? '#888' : '#4a3a2a'}
strokeWidth={f === DOUBLE_MARKER ? 2 : 1} />
))}
{/* Nut */}
<line x1={NUT_X} y1={PAD_T - 6} x2={NUT_X} y2={PAD_T + 3 * STRING_H + 6}
stroke="#c0b090" strokeWidth={4} />
{/* Strings — thicker as pitch drops */}
{STRINGS.map((s, si) => (
<line key={si}
x1={OPEN_X - DOT_R - 2} y1={stringY(si)}
x2={BOARD_W - 8} y2={stringY(si)}
stroke="#9ca3af"
strokeWidth={s.thickness} />
))}
{/* Fret numbers */}
{[3, 5, 7, 9, 12].map(f => (
<text key={f}
x={fretX(f)} y={PAD_T - 10}
textAnchor="middle" fontSize={10} fill="#6b7280"
>{f}</text>
))}
{/* String labels */}
{STRINGS.map((s, si) => (
<text key={si}
x={6} y={stringY(si) + 4}
textAnchor="middle" fontSize={10} fill="#6b7280"
>{s.label}</text>
))}
{/* Note dots */}
{STRINGS.flatMap((str, si) =>
Array.from({ length: NUM_FRETS }, (_, fi) => {
const pc = (str.root + fi) % 12
const color = noteColor(chordSet.has(pc), pentaSet.has(pc), scaleSet.has(pc), monoColor)
if (!color) return null
const cx = fi === 0 ? OPEN_X : fretX(fi)
const cy = stringY(si)
return (
<g key={`${si}-${fi}`}>
<circle cx={cx} cy={cy} r={DOT_R} fill={color.fill} />
<text
x={cx} y={cy + 4}
textAnchor="middle"
fontSize={9}
fontWeight="600"
fill={color.text}
>
{NOTES[pc]}
</text>
</g>
)
})
)}
</svg>
</div>
<div className="mt-3 flex gap-5 text-xs text-gray-500">
<span><span className="text-accent"></span> Chord tone</span>
<span style={{ color: monoColor ? '#c084fc' : '#f59e0b' }}></span><span> Pentatonic</span>
<span style={{ color: monoColor ? '#e9d5ff' : '#6b7280' }}></span><span> Scale</span>
</div>
</div>
)
}
+38
View File
@@ -70,6 +70,19 @@ const SETTINGS = [
},
]
const TOGGLES = [
{
section: 'Experimental',
items: [
{
key: 'useEssentia',
label: 'Essentia HPCP',
desc: 'Replace custom chroma with Essentia\'s Harmonic Pitch Class Profile (spectral peaks + 8 harmonics). More accurate on polyphonic input. May be slower.',
},
],
},
]
export default function Settings({ config, onChange, onClose, onReset }) {
return (
<div className="fixed inset-0 bg-surface z-50 overflow-y-auto">
@@ -97,6 +110,31 @@ export default function Settings({ config, onChange, onClose, onReset }) {
</div>
<div className="space-y-8">
{TOGGLES.map(section => (
<div key={section.section}>
<h3 className="text-xs uppercase tracking-widest text-gray-500 mb-4 border-b border-border pb-2">
{section.section}
</h3>
<div className="space-y-4">
{section.items.map(item => (
<label key={item.key} className="flex items-start gap-3 cursor-pointer group">
<input
type="checkbox"
checked={!!config[item.key]}
onChange={e => onChange(item.key, e.target.checked)}
className="mt-0.5 accent-purple-500 w-4 h-4 flex-shrink-0"
/>
<div>
<div className="text-sm font-semibold text-gray-200 group-hover:text-white transition-colors">
{item.label}
</div>
<div className="text-xs text-gray-600 mt-0.5">{item.desc}</div>
</div>
</label>
))}
</div>
</div>
))}
{SETTINGS.map(section => (
<div key={section.section}>
<h3 className="text-xs uppercase tracking-widest text-gray-500 mb-4 border-b border-border pb-2">
+98
View File
@@ -0,0 +1,98 @@
// Essentia HPCP pipeline — replaces our custom computeChroma() when enabled.
//
// Pipeline (using Web Audio FFT data directly to skip WASM re-FFT):
// freqBuf (dB from AnalyserNode) → linear magnitude → SpectralPeaks → HPCP
//
// Pitch class ordering:
// Essentia HPCP[0] = A (referenceFrequency=440)
// Our chroma[0] = C
// Rotation applied: ourChroma[(i + 9) % 12] = HPCP[i]
import { EssentiaWASM } from 'essentia.js/dist/essentia-wasm.es.js'
import Essentia from 'essentia.js/dist/essentia.js-core.es.js'
let instance = null
// Call once at startup — safe to call multiple times (returns cached instance).
export async function initEssentia() {
if (instance) return instance
console.log('[Essentia] loading WASM…')
await EssentiaWASM['ready']
instance = new Essentia(EssentiaWASM)
console.log('[Essentia] ready —', instance.version)
return instance
}
export function getEssentia() { return instance }
const NOISE_FLOOR_DB = -65
// Compute 12-bin HPCP from Web Audio frequency-domain data.
//
// freqData — Float32Array from AnalyserNode.getFloatFrequencyData() (dB values)
// sampleRate — AudioContext.sampleRate
//
// Returns Float32Array(12) with same [C, C#, D, …, B] ordering as computeChroma().
export function computeHPCP(essentia, freqData, sampleRate) {
const N = freqData.length
// Convert dB → linear magnitude. Bins below noise floor stay 0.
const magSpectrum = new Float32Array(N)
for (let i = 0; i < N; i++) {
if (freqData[i] > NOISE_FLOOR_DB) {
magSpectrum[i] = Math.pow(10, freqData[i] / 20)
}
}
const specVec = essentia.arrayToVector(magSpectrum)
// SpectralPeaks derives bin → Hz as: freq = binIndex * sampleRate / (2*(N-1))
// which matches Web Audio's FFT bin spacing (sampleRate / fftSize).
const peaks = essentia.SpectralPeaks(
specVec,
0, // magnitudeThreshold — noise already zeroed above
4000, // maxFrequency (Hz)
60, // maxPeaks
40, // minFrequency (Hz)
'byMagnitude',
sampleRate
)
specVec.delete()
// If no peaks found (silence / below noise floor), return zeros.
if (peaks.frequencies.size() === 0) {
peaks.frequencies.delete()
peaks.magnitudes.delete()
return new Float32Array(12)
}
const hpcpResult = essentia.HPCP(
peaks.frequencies,
peaks.magnitudes,
false, // bandPreset
500, // bandSplitFrequency (unused when bandPreset=false)
8, // harmonics
4000, // maxFrequency (Hz)
false, // maxShifted
40, // minFrequency (Hz)
false, // nonLinear
'unitMax', // normalized
440, // referenceFrequency (A4 = 440 Hz → HPCP[0] = A)
sampleRate,
12, // size (bins per octave)
'squaredCosine', // weightType
0.5 // windowSize (octaves)
)
peaks.frequencies.delete()
peaks.magnitudes.delete()
// Rotate A-origin → C-origin to match our chroma convention.
// HPCP[0]=A → ourChroma[9]=A, so ourChroma[(i+9)%12] = HPCP[i]
const result = new Float32Array(12)
for (let i = 0; i < 12; i++) {
result[(i + 9) % 12] = hpcpResult.hpcp.get(i)
}
hpcpResult.hpcp.delete()
return result
}
+5
View File
@@ -11,4 +11,9 @@ export default defineConfig({
build: {
outDir: 'dist',
},
optimizeDeps: {
// Essentia uses emscripten output that esbuild can't pre-bundle reliably.
// Exclude it so Vite serves the files directly from node_modules.
exclude: ['essentia.js'],
},
})