diff --git a/README.md b/README.md index e69de29..a07bf21 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,48 @@ +# WhatTheFlat + +Real-time key and chord detection for musicians. Play guitar, bass, piano, or any instrument into your microphone and WhatTheFlat will identify the key you're in, the chords you're playing, and suggest progressions. + +## Features + +- Real-time chord detection from live audio +- Automatic key detection (Krumhansl-Schmuckler profiles) +- Chord history and repeating progression detection +- Roman numeral analysis relative to detected key +- Fretboard visualiser showing safe notes and chord tones +- Beginner / Advanced modes +- Manual key lock for jam sessions +- AI chat assistant for music theory questions + +## Tech Stack + +- **Frontend**: React 18, Vite, Tailwind CSS +- **Audio**: Web Audio API, [Pitchy](https://github.com/ianprime0509/pitchy) (McLeod pitch detection) +- **Backend**: Python (Claude API for chat assistant) + +## Getting Started + +### Frontend + +```bash +cd frontend +npm install +npm run dev +``` + +Open `http://localhost:5173` in your browser and click **Start Listening**. Allow microphone access when prompted. + +### Backend (chat assistant) + +```bash +pip install -r requirements.txt +python main.py +``` + +## How It Works + +Audio is processed in two parallel paths: + +1. **Pitch path** — small 4096-sample FFT with McLeod autocorrelation for fast, accurate single-note pitch detection. Feeds the key detection algorithm. +2. **Chord path** — large 16384-sample FFT (2.7 Hz/bin resolution) with harmonic summation chroma extraction. The chroma vector is matched against chord templates (major, minor, dominant 7th, sus4, diminished) to identify the current chord. + +Key detection uses a rolling vote over the last 12 detections and requires 9/12 agreement before committing, keeping the display stable during transitions. diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index ea44935..43e1e4f 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -15,8 +15,8 @@ const KEY_VOTE_WINDOW = 12 const KEY_VOTE_THRESHOLD = 9 // out of 12 — very stable // Chord detection tuning -const CHROMA_SMOOTH = 16 // frames to average (~250ms at 60fps) -const CHORD_VOTE_THRESHOLD = 5 // consecutive agreements before commit +const CHROMA_SMOOTH = 8 // frames to average (~130ms at 60fps) +const CHORD_VOTE_THRESHOLD = 3 // consecutive agreements before commit export default function App() { // ── Listening state ────────────────────────────────────────────────────── @@ -97,10 +97,9 @@ export default function App() { if (prev?.root === root && prev?.mode === mode) { return { root, mode, confidence: result.confidence } } - // Key changed — clear chord history only if not locked + // Key changed — reset chord votes but keep history visible if (!lockedKey) { chordVotesRef.current = [] - setChordHistory([]) } return { root, mode, confidence: result.confidence } }) diff --git a/frontend/src/components/AudioCapture.jsx b/frontend/src/components/AudioCapture.jsx index 578384e..68005b5 100644 --- a/frontend/src/components/AudioCapture.jsx +++ b/frontend/src/components/AudioCapture.jsx @@ -2,48 +2,75 @@ import { useEffect, useRef, useCallback } from 'react' import { PitchDetector } from 'pitchy' import { NOTES } from '../lib/theory' +// ─── Why two analysers? ─────────────────────────────────────────────────────── +// +// The Web Audio FFT has linearly-spaced bins: bin width = sampleRate / fftSize. +// +// fftSize 4096 → ~10.8 Hz/bin (default we were using) +// fftSize 16384 → ~2.7 Hz/bin (multi-rate chord analyser) +// +// On the low guitar strings the gap between adjacent semitones is only ~5-6 Hz. +// At 10.8 Hz/bin we literally cannot separate A2 (110 Hz) from A#2 (116 Hz). +// That is the single biggest source of wrong chord notes on the low strings. +// +// Solution: run a second, larger analyser just for chord/chroma detection. +// The pitch analyser stays small (4096) so pitchy has a 90ms window — fast +// enough for responsive pitch detection. The chord analyser uses 16384 (~370ms +// window) — slower to respond but with 2.7 Hz bins that can cleanly separate +// every semitone across the guitar's entire range. +// +// This is an approximation of the Constant-Q Transform (CQT) your friend +// mentioned: CQT achieves log-spaced bins mathematically; we approximate it +// by simply using a much larger FFT window. +// ───────────────────────────────────────────────────────────────────────────── + +const PITCH_FFT = 4096 // ~90ms window — good temporal resolution for pitch +const CHORD_FFT = 16384 // ~370ms window — 2.7 Hz/bin, separates low semitones const MIN_CLARITY = 0.85 const MIN_VOLUME = 0.01 -const FFT_SIZE = 4096 // larger = better frequency resolution -const NOISE_FLOOR = -60 // dB — ignore bins quieter than this +const NOISE_FLOOR = -65 // dB + +// ─── Harmonic summation chroma ──────────────────────────────────────────────── +// Each FFT bin votes back toward lower fundamentals that could have generated +// it as an overtone. This undoes the harmonic contamination that makes minor +// chords look like major ones (the 5th harmonic of the root lands on the major +// 3rd, which is NOT in the minor chord). +const HARMONIC_WEIGHTS = [1.0, 0.5, 0.33, 0.25, 0.2] // h = 1…5 -// Build 12-bin chroma from FFT power spectrum. -// Restricts to guitar fundamental range and applies log compression. function computeChroma(freqData, sampleRate, fftSize) { const chroma = new Float32Array(12) const binHz = sampleRate / fftSize + const N = freqData.length - for (let bin = 2; bin < freqData.length; bin++) { + for (let bin = 2; bin < N; bin++) { const freq = bin * binHz - if (freq < 75 || freq > 1400) continue // guitar fundamentals only + if (freq < 80 || freq > 6000) continue const db = freqData[bin] if (db < NOISE_FLOOR) continue - // Power (db/10) discriminates harmonics better than amplitude (db/20) - const power = Math.pow(10, db / 10) - const midi = 12 * Math.log2(freq / 440) + 69 - const pc = ((Math.round(midi) % 12) + 12) % 12 - chroma[pc] += power + const amp = Math.sqrt(Math.pow(10, db / 10)) // amplitude, not power + + for (let h = 1; h <= HARMONIC_WEIGHTS.length; h++) { + const fundamental = freq / h + if (fundamental < 40 || fundamental > 2000) continue + const midi = 12 * Math.log2(fundamental / 440) + 69 + const pc = ((Math.round(midi) % 12) + 12) % 12 + chroma[pc] += amp * HARMONIC_WEIGHTS[h - 1] + } } - // Log compression reduces dominance of very loud partials - for (let i = 0; i < 12; i++) chroma[i] = Math.log1p(chroma[i] * 100) - + for (let i = 0; i < 12; i++) chroma[i] = Math.log1p(chroma[i]) const max = Math.max(...chroma) if (max > 0) for (let i = 0; i < 12; i++) chroma[i] /= max - return chroma } -// Find the dominant pitch class in the bass range (guitar lowest notes). -// This gives us a strong root-note hint for chord matching. function detectBassPC(freqData, sampleRate, fftSize) { const binHz = sampleRate / fftSize let maxPower = 0, bestMidi = -1 - for (let bin = 2; bin < freqData.length; bin++) { const freq = bin * binHz - if (freq < 75 || freq > 350) continue + if (freq < 40 || freq > 350) continue const db = freqData[bin] if (db < NOISE_FLOOR) continue const power = Math.pow(10, db / 10) @@ -57,17 +84,18 @@ function detectBassPC(freqData, sampleRate, fftSize) { } export default function AudioCapture({ onNote, onChroma, isListening }) { - const audioCtxRef = useRef(null) - const analyserRef = useRef(null) - const detectorRef = useRef(null) - const timeBufRef = useRef(null) - const freqBufRef = useRef(null) - const rafRef = useRef(null) - const streamRef = useRef(null) + const audioCtxRef = useRef(null) + const pitchAnalyser = useRef(null) + const chordAnalyser = useRef(null) + const timeBufRef = useRef(null) + const freqBufRef = useRef(null) + const detectorRef = useRef(null) + const rafRef = useRef(null) + const streamRef = useRef(null) const stop = useCallback(() => { - if (rafRef.current) cancelAnimationFrame(rafRef.current) - if (streamRef.current) streamRef.current.getTracks().forEach(t => t.stop()) + if (rafRef.current) cancelAnimationFrame(rafRef.current) + if (streamRef.current) streamRef.current.getTracks().forEach(t => t.stop()) if (audioCtxRef.current) audioCtxRef.current.close() audioCtxRef.current = null }, []) @@ -79,25 +107,32 @@ export default function AudioCapture({ onNote, onChroma, isListening }) { const ctx = new AudioContext() audioCtxRef.current = ctx + const source = ctx.createMediaStreamSource(stream) - const analyser = ctx.createAnalyser() - analyser.fftSize = FFT_SIZE - analyser.smoothingTimeConstant = 0.6 // smooth FFT over time - analyserRef.current = analyser + // Small analyser — pitch detection needs fast time-domain data + const pa = ctx.createAnalyser() + pa.fftSize = PITCH_FFT + pa.smoothingTimeConstant = 0.0 // no smoothing: pitchy needs clean waveform + pitchAnalyser.current = pa + source.connect(pa) + timeBufRef.current = new Float32Array(pa.fftSize) + detectorRef.current = PitchDetector.forFloat32Array(pa.fftSize) - ctx.createMediaStreamSource(stream).connect(analyser) - - timeBufRef.current = new Float32Array(analyser.fftSize) - freqBufRef.current = new Float32Array(analyser.frequencyBinCount) - detectorRef.current = PitchDetector.forFloat32Array(analyser.fftSize) + // Large analyser — chord detection needs fine frequency resolution + const ca = ctx.createAnalyser() + ca.fftSize = CHORD_FFT + ca.smoothingTimeConstant = 0.65 // smooth over time for stable chord reading + chordAnalyser.current = ca + source.connect(ca) + freqBufRef.current = new Float32Array(ca.frequencyBinCount) function tick() { const timeBuf = timeBufRef.current - analyser.getFloatTimeDomainData(timeBuf) + pa.getFloatTimeDomainData(timeBuf) const rms = Math.sqrt(timeBuf.reduce((s, v) => s + v * v, 0) / timeBuf.length) if (rms >= MIN_VOLUME) { - // Pitch — used for key detection + // Pitch via McLeod (autocorrelation) — unaffected by FFT bin size const [freq, clarity] = detectorRef.current.findPitch(timeBuf, ctx.sampleRate) if (clarity >= MIN_CLARITY && freq > 60 && freq < 4200) { const midi = Math.round(12 * Math.log2(freq / 440) + 69) @@ -105,13 +140,14 @@ export default function AudioCapture({ onNote, onChroma, isListening }) { onNote({ noteName: NOTES[pitchClass], pitchClass, freq, midi, clarity }) } - // Chroma + bass — used for chord detection + // Chord chroma from the high-resolution FFT if (onChroma) { const freqBuf = freqBufRef.current - analyser.getFloatFrequencyData(freqBuf) - const chroma = computeChroma(freqBuf, ctx.sampleRate, analyser.fftSize) - const bassPC = detectBassPC(freqBuf, ctx.sampleRate, analyser.fftSize) - onChroma(chroma, bassPC) + ca.getFloatFrequencyData(freqBuf) + onChroma( + computeChroma(freqBuf, ctx.sampleRate, ca.fftSize), + detectBassPC(freqBuf, ctx.sampleRate, ca.fftSize) + ) } } diff --git a/frontend/src/components/ProgressionBanner.jsx b/frontend/src/components/ProgressionBanner.jsx index e88302e..1286df2 100644 --- a/frontend/src/components/ProgressionBanner.jsx +++ b/frontend/src/components/ProgressionBanner.jsx @@ -1,49 +1,38 @@ import { useRef, useEffect } from 'react' import { toRomanNumeral } from '../lib/theory' -// Sizes for the chord trail (oldest → current) -const TRAIL_SIZES = [ - 'text-lg opacity-20', - 'text-xl opacity-30', - 'text-2xl opacity-45', - 'text-3xl opacity-60', - 'text-4xl opacity-80', -] -const CURRENT_SIZE = 'text-7xl opacity-100' +const HISTORY_SHOWN = 8 // ~2 bars at 4 chords/bar function findLoopPosition(chordHistory, progression) { if (!progression?.length || !chordHistory.length) return -1 - const len = progression.length - // Walk backwards through the progression to find where current chord sits - for (let p = len - 1; p >= 0; p--) { - if (progression[p] !== chordHistory[chordHistory.length - 1]) continue + const last = chordHistory[chordHistory.length - 1] + for (let p = progression.length - 1; p >= 0; p--) { + if (progression[p] !== last) continue let match = true for (let i = 1; i < Math.min(p + 1, chordHistory.length); i++) { - if (progression[p - i] !== chordHistory[chordHistory.length - 1 - i]) { - match = false; break - } + if (progression[p - i] !== chordHistory[chordHistory.length - 1 - i]) { match = false; break } } if (match) return p } - return progression.indexOf(chordHistory[chordHistory.length - 1]) + return progression.indexOf(last) } export default function ProgressionBanner({ chordHistory, keyInfo, detectedProgression }) { const { root, mode } = keyInfo ?? {} - // Show up to 5 previous chords + current - const trail = chordHistory.slice(-6, -1) // up to 5 previous - const current = chordHistory[chordHistory.length - 1] + // Newest chord is the last entry; we show the most recent HISTORY_SHOWN + const visible = chordHistory.slice(-HISTORY_SHOWN) + const current = visible[visible.length - 1] - // Flash the current chord when it changes + // Animate the current chord slot when it changes const currentRef = useRef(null) const prevChord = useRef(null) useEffect(() => { if (current && current !== prevChord.current && currentRef.current) { currentRef.current.animate( - [{ opacity: 0, transform: 'translateY(8px) scale(0.9)' }, - { opacity: 1, transform: 'translateY(0) scale(1)' }], - { duration: 220, easing: 'ease-out', fill: 'forwards' } + [{ opacity: 0, transform: 'scale(0.85)' }, + { opacity: 1, transform: 'scale(1)' }], + { duration: 200, easing: 'ease-out', fill: 'forwards' } ) prevChord.current = current } @@ -53,49 +42,55 @@ export default function ProgressionBanner({ chordHistory, keyInfo, detectedProgr if (!chordHistory.length) { return ( -
-

Start listening to detect chords…

+
+

Start listening to detect chords…

) } return ( -
- {/* ── Chord trail ── */} -
- {trail.map((chord, i) => { - const sizeClass = TRAIL_SIZES[Math.max(0, i - (trail.length - TRAIL_SIZES.length))] - const rn = root ? toRomanNumeral(chord, root, mode) : '' +
+ + {/* ── Chord history strip: all HISTORY_SHOWN chords at consistent size ── */} +
+ {visible.map((chord, i) => { + const isCurrent = i === visible.length - 1 + const age = visible.length - 1 - i // 0 = current, higher = older + const opacity = Math.max(0.2, 1 - age * 0.1) // fade but stay readable + const rn = root ? toRomanNumeral(chord, root, mode) : '' + return ( -
- {chord} - {rn} +
+ + {chord} + + + {rn || '\u00A0'} +
) })} - - {/* Arrow between trail and current */} - {trail.length > 0 && ( - › - )} - - {/* Current chord — BIG */} - {current && ( -
- {current} - - {root ? toRomanNumeral(current, root, mode) : ''} - -
- )}
{/* ── Detected loop ── */} {detectedProgression && ( -
-

- ♻ Detected loop -

+
+

♻ Detected loop

{detectedProgression.map((chord, i) => { const isActive = i === loopPos @@ -105,7 +100,7 @@ export default function ProgressionBanner({ chordHistory, keyInfo, detectedProgr key={i} className={`flex flex-col items-center px-4 py-2 rounded-xl border transition-all duration-200 ${ isActive - ? 'bg-accent/20 border-accent shadow-[0_0_12px_rgba(168,85,247,0.4)]' + ? 'bg-accent/20 border-accent shadow-[0_0_14px_rgba(168,85,247,0.35)]' : 'bg-border border-border' }`} > @@ -118,9 +113,7 @@ export default function ProgressionBanner({ chordHistory, keyInfo, detectedProgr
) })} -
- → loop -
+ → loop
)} diff --git a/frontend/src/lib/theory.js b/frontend/src/lib/theory.js index aa04c10..75f9f01 100644 --- a/frontend/src/lib/theory.js +++ b/frontend/src/lib/theory.js @@ -177,12 +177,19 @@ export function matchChordFromChroma(chroma, keyInfo, bassPC = null, strictDiato const diatonic = new Set(getChordsInKey(keyInfo.root, keyInfo.mode)) - // Only match triads — more reliable for live guitar than extended chords - const triadTypes = [CHORD_TYPES.maj, CHORD_TYPES.min, CHORD_TYPES.dim] + // Match triads + dominant 7ths (blues/rock/band) + sus chords (rock guitar) + const matchTypes = [ + CHORD_TYPES.maj, + CHORD_TYPES.min, + CHORD_TYPES.dom7, + CHORD_TYPES.min7, + CHORD_TYPES.dim, + CHORD_TYPES.sus4, + ] let best = { name: null, score: -Infinity } for (let r = 0; r < 12; r++) { - for (const type of triadTypes) { + for (const type of matchTypes) { const tones = new Set(type.intervals.map(i => (r + i) % 12)) const chordName = noteName(r) + type.suffix @@ -190,23 +197,28 @@ export function matchChordFromChroma(chroma, keyInfo, bassPC = null, strictDiato let inEnergy = 0, outEnergy = 0 for (let pc = 0; pc < 12; pc++) { - if (tones.has(pc)) inEnergy += chroma[pc] - else outEnergy += chroma[pc] + if (pc === r) { + // Root note is the strongest identity signal — weight it double + inEnergy += chroma[pc] * 2 + } else if (tones.has(pc)) { + inEnergy += chroma[pc] + } else { + outEnergy += chroma[pc] + } } if (inEnergy + outEnergy < 0.05) continue - // Core score: fraction of energy on chord tones, penalise noise - const coverageScore = inEnergy / (inEnergy + outEnergy * 0.6) - // Bass note matching the chord root is a strong harmonic signal - const bassBonus = (bassPC !== null && r === bassPC) ? 0.4 : 0 - const diatonicBonus = diatonic.has(chordName) ? 0.2 : 0 + const coverageScore = inEnergy / (inEnergy + outEnergy * 0.5) + // Bass note matching chord root is a strong harmonic signal + const bassBonus = (bassPC !== null && r === bassPC) ? 0.35 : 0 + const diatonicBonus = diatonic.has(chordName) ? 0.15 : 0 const finalScore = coverageScore + bassBonus + diatonicBonus if (finalScore > best.score) best = { name: chordName, score: finalScore } } } - return best.score > 0.45 ? best.name : null + return best.score > 0.42 ? best.name : null } // ─── Roman numeral notation ───────────────────────────────────────────────────