whattheflat/JamBuddy
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

additions and improvements

Browse Source
This commit is contained in:
vadimwit
2026-03-05 17:11:20 +00:00
parent 5e7954ee24
commit 4e6d997b68
5 changed files with 205 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/components/AudioCapture.jsx
+80 -44
View File
@@ -2,48 +2,75 @@ import { useEffect, useRef, useCallback } from 'react'
import { PitchDetector } from 'pitchy'
import { NOTES } from '../lib/theory'
// ─── Why two analysers? ───────────────────────────────────────────────────────
//
// The Web Audio FFT has linearly-spaced bins: bin width = sampleRate / fftSize.
//
// fftSize 4096 → ~10.8 Hz/bin (default we were using)
// fftSize 16384 → ~2.7 Hz/bin (multi-rate chord analyser)
//
// On the low guitar strings the gap between adjacent semitones is only ~5-6 Hz.
// At 10.8 Hz/bin we literally cannot separate A2 (110 Hz) from A#2 (116 Hz).
// That is the single biggest source of wrong chord notes on the low strings.
//
// Solution: run a second, larger analyser just for chord/chroma detection.
// The pitch analyser stays small (4096) so pitchy has a 90ms window — fast
// enough for responsive pitch detection. The chord analyser uses 16384 (~370ms
// window) — slower to respond but with 2.7 Hz bins that can cleanly separate
// every semitone across the guitar's entire range.
//
// This is an approximation of the Constant-Q Transform (CQT) your friend
// mentioned: CQT achieves log-spaced bins mathematically; we approximate it
// by simply using a much larger FFT window.
// ─────────────────────────────────────────────────────────────────────────────
const PITCH_FFT = 4096 // ~90ms window — good temporal resolution for pitch
const CHORD_FFT = 16384 // ~370ms window — 2.7 Hz/bin, separates low semitones
const MIN_CLARITY = 0.85
const MIN_VOLUME = 0.01
const FFT_SIZE = 4096 // larger = better frequency resolution
const NOISE_FLOOR = -60 // dB — ignore bins quieter than this
const NOISE_FLOOR = -65 // dB
// ─── Harmonic summation chroma ────────────────────────────────────────────────
// Each FFT bin votes back toward lower fundamentals that could have generated
// it as an overtone. This undoes the harmonic contamination that makes minor
// chords look like major ones (the 5th harmonic of the root lands on the major
// 3rd, which is NOT in the minor chord).
const HARMONIC_WEIGHTS = [1.0, 0.5, 0.33, 0.25, 0.2] // h = 1…5
// Build 12-bin chroma from FFT power spectrum.
// Restricts to guitar fundamental range and applies log compression.
function computeChroma(freqData, sampleRate, fftSize) {
const chroma = new Float32Array(12)
const binHz = sampleRate / fftSize
const N = freqData.length
for (let bin = 2; bin < freqData.length; bin++) {
for (let bin = 2; bin < N; bin++) {
const freq = bin * binHz
if (freq < 75 || freq > 1400) continue // guitar fundamentals only
if (freq < 80 || freq > 6000) continue
const db = freqData[bin]
if (db < NOISE_FLOOR) continue
// Power (db/10) discriminates harmonics better than amplitude (db/20)
const power = Math.pow(10, db / 10)
const midi = 12 * Math.log2(freq / 440) + 69
const pc = ((Math.round(midi) % 12) + 12) % 12
chroma[pc] += power
const amp = Math.sqrt(Math.pow(10, db / 10)) // amplitude, not power
for (let h = 1; h <= HARMONIC_WEIGHTS.length; h++) {
const fundamental = freq / h
if (fundamental < 40 || fundamental > 2000) continue
const midi = 12 * Math.log2(fundamental / 440) + 69
const pc = ((Math.round(midi) % 12) + 12) % 12
chroma[pc] += amp * HARMONIC_WEIGHTS[h - 1]
}
}
// Log compression reduces dominance of very loud partials
for (let i = 0; i < 12; i++) chroma[i] = Math.log1p(chroma[i] * 100)
for (let i = 0; i < 12; i++) chroma[i] = Math.log1p(chroma[i])
const max = Math.max(...chroma)
if (max > 0) for (let i = 0; i < 12; i++) chroma[i] /= max
return chroma
}
// Find the dominant pitch class in the bass range (guitar lowest notes).
// This gives us a strong root-note hint for chord matching.
function detectBassPC(freqData, sampleRate, fftSize) {
const binHz = sampleRate / fftSize
let maxPower = 0, bestMidi = -1
for (let bin = 2; bin < freqData.length; bin++) {
const freq = bin * binHz
if (freq < 75 || freq > 350) continue
if (freq < 40 || freq > 350) continue
const db = freqData[bin]
if (db < NOISE_FLOOR) continue
const power = Math.pow(10, db / 10)
@@ -57,17 +84,18 @@ function detectBassPC(freqData, sampleRate, fftSize) {
}
export default function AudioCapture({ onNote, onChroma, isListening }) {
const audioCtxRef = useRef(null)
const analyserRef = useRef(null)
const detectorRef = useRef(null)
const timeBufRef = useRef(null)
const freqBufRef = useRef(null)
const rafRef = useRef(null)
const streamRef = useRef(null)
const audioCtxRef = useRef(null)
const pitchAnalyser = useRef(null)
const chordAnalyser = useRef(null)
const timeBufRef = useRef(null)
const freqBufRef = useRef(null)
const detectorRef = useRef(null)
const rafRef = useRef(null)
const streamRef = useRef(null)
const stop = useCallback(() => {
if (rafRef.current) cancelAnimationFrame(rafRef.current)
if (streamRef.current) streamRef.current.getTracks().forEach(t => t.stop())
if (rafRef.current) cancelAnimationFrame(rafRef.current)
if (streamRef.current) streamRef.current.getTracks().forEach(t => t.stop())
if (audioCtxRef.current) audioCtxRef.current.close()
audioCtxRef.current = null
}, [])
@@ -79,25 +107,32 @@ export default function AudioCapture({ onNote, onChroma, isListening }) {
const ctx = new AudioContext()
audioCtxRef.current = ctx
const source = ctx.createMediaStreamSource(stream)
const analyser = ctx.createAnalyser()
analyser.fftSize = FFT_SIZE
analyser.smoothingTimeConstant = 0.6 // smooth FFT over time
analyserRef.current = analyser
// Small analyser — pitch detection needs fast time-domain data
const pa = ctx.createAnalyser()
pa.fftSize = PITCH_FFT
pa.smoothingTimeConstant = 0.0 // no smoothing: pitchy needs clean waveform
pitchAnalyser.current = pa
source.connect(pa)
timeBufRef.current = new Float32Array(pa.fftSize)
detectorRef.current = PitchDetector.forFloat32Array(pa.fftSize)
ctx.createMediaStreamSource(stream).connect(analyser)
timeBufRef.current = new Float32Array(analyser.fftSize)
freqBufRef.current = new Float32Array(analyser.frequencyBinCount)
detectorRef.current = PitchDetector.forFloat32Array(analyser.fftSize)
// Large analyser — chord detection needs fine frequency resolution
const ca = ctx.createAnalyser()
ca.fftSize = CHORD_FFT
ca.smoothingTimeConstant = 0.65 // smooth over time for stable chord reading
chordAnalyser.current = ca
source.connect(ca)
freqBufRef.current = new Float32Array(ca.frequencyBinCount)
function tick() {
const timeBuf = timeBufRef.current
analyser.getFloatTimeDomainData(timeBuf)
pa.getFloatTimeDomainData(timeBuf)
const rms = Math.sqrt(timeBuf.reduce((s, v) => s + v * v, 0) / timeBuf.length)
if (rms >= MIN_VOLUME) {
// Pitch — used for key detection
// Pitch via McLeod (autocorrelation) — unaffected by FFT bin size
const [freq, clarity] = detectorRef.current.findPitch(timeBuf, ctx.sampleRate)
if (clarity >= MIN_CLARITY && freq > 60 && freq < 4200) {
const midi = Math.round(12 * Math.log2(freq / 440) + 69)
@@ -105,13 +140,14 @@ export default function AudioCapture({ onNote, onChroma, isListening }) {
onNote({ noteName: NOTES[pitchClass], pitchClass, freq, midi, clarity })
}
// Chroma + bass — used for chord detection
// Chord chroma from the high-resolution FFT
if (onChroma) {
const freqBuf = freqBufRef.current
analyser.getFloatFrequencyData(freqBuf)
const chroma = computeChroma(freqBuf, ctx.sampleRate, analyser.fftSize)
const bassPC = detectBassPC(freqBuf, ctx.sampleRate, analyser.fftSize)
onChroma(chroma, bassPC)
ca.getFloatFrequencyData(freqBuf)
onChroma(
computeChroma(freqBuf, ctx.sampleRate, ca.fftSize),
detectBassPC(freqBuf, ctx.sampleRate, ca.fftSize)
)
}
}