From d865162c391e860dfb89036ba5edfefbb2baba1c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 09:10:25 +0000 Subject: [PATCH 1/5] Listen mode: keyboard speakers and animated keys In listen mode, two pill switches sit left of the song name above the on-screen keyboard: Keyboard speakers (only when a MIDI output is connected) sends playback to the keyboard, and Animate keys presses each on-screen key as its note sounds and lifts it when the note ends. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HvmoDPbUWx35sRniGQQfBT --- e2e/listen.spec.ts | 62 ++++++++++++++++++++++++ src/core/playback/player.ts | 12 ++++- src/core/settings.ts | 3 ++ src/ui/components/PianoKeyboard.tsx | 2 +- src/ui/live.ts | 24 ++++++++++ src/ui/practice/ListenOptions.tsx | 73 +++++++++++++++++++++++++++++ src/ui/practice/PracticeScreen.tsx | 2 + src/ui/practice/controller.ts | 30 ++++++++++++ src/ui/runtime.ts | 2 + src/ui/styles.css | 62 ++++++++++++++++++++++++ tests/playback/player.test.ts | 23 +++++++++ 11 files changed, 293 insertions(+), 2 deletions(-) create mode 100644 e2e/listen.spec.ts create mode 100644 src/ui/practice/ListenOptions.tsx diff --git a/e2e/listen.spec.ts b/e2e/listen.spec.ts new file mode 100644 index 0000000..1e02096 --- /dev/null +++ b/e2e/listen.spec.ts @@ -0,0 +1,62 @@ +import { test, expect, type Page } from '@playwright/test'; + +type OK = { useApp: { getState: () => { settingsLoaded: boolean; updateSettings: (p: object) => void } } }; + +/** Mocked Web MIDI: one keyboard with an input and an output that records what it is sent. */ +const MOCK = () => { + const input = { id: 'in1', name: 'CASIO USB-MIDI', manufacturer: 'Casio', state: 'connected', type: 'input', onmidimessage: null }; + const sent: number[][] = []; + const output = { id: 'out1', name: 'CASIO USB-MIDI', manufacturer: 'Casio', state: 'connected', type: 'output', send: (b: number[]) => sent.push([...b]) }; + const access = { inputs: new Map([[input.id, input]]), outputs: new Map([[output.id, output]]), onstatechange: null, sysexEnabled: false }; + (navigator as unknown as { requestMIDIAccess: () => Promise }).requestMIDIAccess = async () => access; + (window as unknown as { __midiSent: number[][] }).__midiSent = sent; +}; + +async function openOde(page: Page, settings: object) { + await page.goto('/'); + await page.waitForFunction(() => (globalThis as unknown as { __openkeys: OK }).__openkeys.useApp.getState().settingsLoaded); + await page.evaluate((s) => (globalThis as unknown as { __openkeys: OK }).__openkeys.useApp.getState().updateSettings({ countInBars: 0, metronome: false, onboarded: true, ...s }), settings); + await page.getByRole('button', { name: /Library/ }).first().click(); + await page.locator('.card', { hasText: 'Ode to Joy' }).getByRole('button', { name: 'Right hand' }).click(); + await expect(page.getByRole('button', { name: /Start/ })).toBeVisible(); +} + +test('listen: options appear only in listen mode and the keys play along', async ({ page }) => { + await openOde(page, { inputSource: 'virtual' }); + const animate = page.getByRole('switch', { name: 'Animate keys' }); + await expect(animate).toHaveCount(0); + await page.getByRole('button', { name: 'Listen', exact: true }).click(); + await expect(animate).toHaveAttribute('aria-checked', 'true'); + // No MIDI keyboard: no speaker option. + await expect(page.getByRole('switch', { name: 'Keyboard speakers' })).toHaveCount(0); + + await page.getByRole('button', { name: /Start/ }).click(); + await expect(page.locator('.keyboard .key.down').first()).toBeVisible(); + // Keys come back up between notes, so not every sampled moment has a key down. + await expect.poll(() => page.locator('.keyboard .key.down').count()).toBe(0); + await expect.poll(() => page.locator('.keyboard .key.down').count()).toBeGreaterThan(0); + + await animate.click(); + await expect(animate).toHaveAttribute('aria-checked', 'false'); + await expect(page.locator('.keyboard .key.down')).toHaveCount(0); + await page.waitForTimeout(1500); + await expect(page.locator('.keyboard .key.down')).toHaveCount(0); + await page.getByRole('button', { name: /Stop/ }).click(); +}); + +test('listen: plays through the MIDI keyboard speakers when one is connected', async ({ page }) => { + await page.addInitScript(MOCK); + await openOde(page, { inputSource: 'midi' }); + await page.getByRole('button', { name: 'Connect MIDI keyboard' }).click(); + await expect(page.locator('.meter')).toContainText('CASIO USB-MIDI'); + await page.getByRole('button', { name: 'Listen', exact: true }).click(); + const speakers = page.getByRole('switch', { name: 'Keyboard speakers' }); + await expect(speakers).toHaveAttribute('aria-checked', 'false'); + await speakers.click(); + await expect(speakers).toHaveAttribute('aria-checked', 'true'); + await page.getByRole('button', { name: /Start/ }).click(); + // E4 (64) is the first note of Ode to Joy. + await expect.poll(() => page.evaluate(() => (window as unknown as { __midiSent: number[][] }).__midiSent.some((b) => b[0] === 0x90 && b[1] === 64))).toBe(true); + await expect(page.locator('.keyboard .key.down').first()).toBeVisible(); + await page.getByRole('button', { name: /Stop/ }).click(); +}); diff --git a/src/core/playback/player.ts b/src/core/playback/player.ts index 9f52e7c..76f1669 100644 --- a/src/core/playback/player.ts +++ b/src/core/playback/player.ts @@ -1,9 +1,10 @@ -import type { HandSelection, Score, ScoreNote } from '../types'; +import type { Hand, HandSelection, Score, ScoreNote } from '../types'; import { TempoMap, metronomeBeats, scoreEndBeat, timeSignatureAt, measureLengthBeats } from '../score/tempo'; import { Emitter } from '../emitter'; import { Timeline } from './timeline'; import type { ClickSink, NoteSink, ScheduledAudioLog } from './sink'; import { soundingEndBeat } from './pedal'; +import { tiedLengthBeats } from '../score/events'; export interface PlayerConfig { /** 0.25 - 1.5 */ @@ -49,6 +50,11 @@ export class Player { readonly ended = new Emitter(); /** Visual metronome pulse (fires at schedule time with the click's context time). */ readonly pulse = new Emitter<{ time: number; accent: boolean; sub: boolean; countIn: boolean }>(); + /** + * Each score note as it is scheduled: context times of the key going down and coming back up + * (the written length including ties, not the pedalled sound). Drives the listen-mode keys. + */ + readonly scheduled = new Emitter<{ midi: number; hand: Hand; time: number; upTime: number }>(); state: PlayerState = 'stopped'; config: PlayerConfig = { ...DEFAULT_PLAYER_CONFIG }; @@ -208,6 +214,10 @@ export class Player { const vel = n.velocity ?? this.config.defaultVelocity; this.sink.play(n.midi, vel, t, dur); this.log?.addNote(n.midi, t, t + dur); + if (this.scheduled.size) { + const upBeat = Math.min(this.segEnd, n.startBeat + tiedLengthBeats(this.score.notes, n)); + this.scheduled.emit({ midi: n.midi, hand: n.hand, time: t, upTime: Math.max(t + 0.05, this.segTime(upBeat)) }); + } } // Clicks. while (this.clickIdx < this.clicks.length && this.clicks[this.clickIdx].beat < horizonBeat) { diff --git a/src/core/settings.ts b/src/core/settings.ts index d60fd44..0d515f7 100644 --- a/src/core/settings.ts +++ b/src/core/settings.ts @@ -40,6 +40,8 @@ export interface Settings { /** In mic mode, allow app audio through the speakers anyway (default muted). */ micAllowSpeakerPlayback: boolean; errorSound: boolean; + /** Listen mode: the on-screen keys go down and up with the notes the app plays. */ + listenKeyAnimation: boolean; splitPoint: number; detector: DetectorParams; debug: boolean; @@ -76,6 +78,7 @@ export const DEFAULT_SETTINGS: Settings = { accompaniment: true, micAllowSpeakerPlayback: false, errorSound: false, + listenKeyAnimation: true, splitPoint: 60, detector: DEFAULT_DETECTOR_PARAMS, debug: false, diff --git a/src/ui/components/PianoKeyboard.tsx b/src/ui/components/PianoKeyboard.tsx index 1e714af..d579adf 100644 --- a/src/ui/components/PianoKeyboard.tsx +++ b/src/ui/components/PianoKeyboard.tsx @@ -60,7 +60,7 @@ export function PianoKeyboard({ low, high, height = 120, labels = 'c', keyHintsB useEffect(() => { const apply = () => { for (const [midi, el] of els.current) { - const mark = liveKeys.pressed.get(midi); + const mark = liveKeys.pressed.get(midi) ?? (liveKeys.playing.has(midi) ? 'down' : undefined); const up = liveKeys.upcoming.get(midi); el.classList.toggle('down', mark === 'down'); el.classList.toggle('hit', mark === 'hit'); diff --git a/src/ui/live.ts b/src/ui/live.ts index 23c20a9..19cede5 100644 --- a/src/ui/live.ts +++ b/src/ui/live.ts @@ -11,6 +11,8 @@ class LiveKeys { pressed = new Map(); /** Upcoming keys to play (outlined), with hand. */ upcoming = new Map(); + /** Keys the app is playing in listen mode: midi -> tokens of the notes holding it down. */ + playing = new Map>(); private listeners = new Set<() => void>(); private scheduled = false; private flashTimers = new Map(); @@ -68,6 +70,27 @@ class LiveKeys { ); } + /** Listen mode: a note the app plays holds its key down (tokens keep overlapping notes apart). */ + playOn(midi: number, token: number) { + const held = this.playing.get(midi); + if (held) held.add(token); + else this.playing.set(midi, new Set([token])); + this.changed(); + } + + playOff(midi: number, token: number) { + const held = this.playing.get(midi); + if (!held?.delete(token)) return; + if (!held.size) this.playing.delete(midi); + this.changed(); + } + + clearPlaying() { + if (!this.playing.size) return; + this.playing.clear(); + this.changed(); + } + setUpcoming(next: Map) { let same = next.size === this.upcoming.size; if (same) for (const [k, v] of next) if (this.upcoming.get(k) !== v) same = false; @@ -78,6 +101,7 @@ class LiveKeys { clear() { this.pressed.clear(); + this.playing.clear(); this.upcoming = new Map(); this.changed(); } diff --git a/src/ui/practice/ListenOptions.tsx b/src/ui/practice/ListenOptions.tsx new file mode 100644 index 0000000..1e67d0f --- /dev/null +++ b/src/ui/practice/ListenOptions.tsx @@ -0,0 +1,73 @@ +import { useEffect, useState } from 'react'; +import { useApp } from '../store'; +import { runtime } from '../runtime'; +import { liveKeys } from '../live'; + +/** Names of the connected MIDI outputs (empty until MIDI is in use). */ +function useMidiOutputs(): string[] { + const [names, setNames] = useState([]); + useEffect(() => { + let offPorts = () => {}; + const attach = () => { + offPorts(); + const midi = runtime.midi; + if (!midi) return; + const read = () => setNames(midi.outputs().filter((o) => o.state === 'connected').map((o) => o.name)); + read(); + offPorts = midi.ports.on(read); + }; + attach(); + const offStatus = runtime.midiStatus.on(attach); + return () => { + offPorts(); + offStatus(); + }; + }, []); + return names; +} + +function Switch({ on, label, title, onChange }: { on: boolean; label: string; title: string; onChange: (on: boolean) => void }) { + return ( + + ); +} + +/** Listen-mode options: play through the MIDI keyboard's speakers, and animate the on-screen keys. */ +export function ListenOptions() { + const settings = useApp((s) => s.settings); + const update = useApp((s) => s.updateSettings); + const outputs = useMidiOutputs(); + const speakers = outputs.length > 0; + return ( +
+ {speakers && ( + + update({ + midiOutPlayback: on, + // The remembered output is gone: use the one that is connected. + ...(on && !(settings.midiOutputName && outputs.includes(settings.midiOutputName)) ? { midiOutputName: outputs[0] } : {}), + }) + } + /> + )} + { + update({ listenKeyAnimation: on }); + if (!on) liveKeys.clearPlaying(); + }} + /> +
+ ); +} diff --git a/src/ui/practice/PracticeScreen.tsx b/src/ui/practice/PracticeScreen.tsx index 582916b..25d3709 100644 --- a/src/ui/practice/PracticeScreen.tsx +++ b/src/ui/practice/PracticeScreen.tsx @@ -21,6 +21,7 @@ import { TakeReplay } from './TakeReplay'; import { ShortcutsHelp } from '../components/ShortcutsHelp'; import { kvSet } from '../../core/progress/db'; import { Icon } from '../components/Icon'; +import { ListenOptions } from './ListenOptions'; const MODES: { id: PracticeMode; label: string; hint: string }[] = [ { id: 'listen', label: 'Listen', hint: 'Hear the piece; the cursor follows.' }, @@ -382,6 +383,7 @@ function Practice({ score }: { score: Score }) { {streak >= 3 && {streak} } + {mode === 'listen' && } {score.title} {score.variant ? ` ยท ${score.variant}` : ''} diff --git a/src/ui/practice/controller.ts b/src/ui/practice/controller.ts index d91169b..79970d7 100644 --- a/src/ui/practice/controller.ts +++ b/src/ui/practice/controller.ts @@ -36,6 +36,9 @@ class PracticeController { readonly stateChange = new Emitter(); private unsubs: (() => void)[] = []; private raf = 0; + /** Listen-mode key animation timers. */ + private keyTimers = new Set(); + private keyToken = 0; private score: Score | null = null; private idleBeat = 0; private opts: TakeOptions | null = null; @@ -207,6 +210,7 @@ class PracticeController { practiceLive.changed(); }), ); + if (opts.mode === 'listen') this.unsubs.push(engine.player.scheduled.on((n) => this.animateKey(n.midi, n.time, n.upTime))); session.start(); practiceLive.running = true; this.stateChange.emit(session.state); @@ -221,6 +225,29 @@ class PracticeController { if (opts.mode !== 'listen') this.raf = requestAnimationFrame(loop); } + /** Listen mode: presses the on-screen key when the note is heard and lifts it when it ends. */ + private animateKey(midi: number, time: number, upTime: number) { + const engine = runtime.engine; + if (!engine || !getSettings().listenKeyAnimation) return; + const token = ++this.keyToken; + const now = performance.now(); + const down = Math.max(0, engine.audioToPerf(time) - now); + // Lift a little early so a repeated note visibly strikes again. + const len = (upTime - time) * 1000; + const up = down + Math.max(40, len - Math.min(60, len * 0.25)); + const at = (ms: number, fn: () => void) => { + const id = window.setTimeout(() => { + this.keyTimers.delete(id); + fn(); + }, ms); + this.keyTimers.add(id); + }; + at(down, () => { + if (getSettings().listenKeyAnimation) liveKeys.playOn(midi, token); + }); + at(up, () => liveKeys.playOff(midi, token)); + } + private onFeedback(fb: FollowerFeedback, score: Score) { const s = getSettings(); switch (fb.type) { @@ -297,6 +324,9 @@ class PracticeController { private cleanup() { this.detector = null; cancelAnimationFrame(this.raf); + for (const id of this.keyTimers) window.clearTimeout(id); + this.keyTimers.clear(); + liveKeys.clearPlaying(); for (const u of this.unsubs) u(); this.unsubs = []; const s = this.session; diff --git a/src/ui/runtime.ts b/src/ui/runtime.ts index d7b3df3..bc7b267 100644 --- a/src/ui/runtime.ts +++ b/src/ui/runtime.ts @@ -201,6 +201,8 @@ class Runtime { if (!isInput) return; useApp.getState().toast(connected ? `${name} connected` : `${name} disconnected; plug it back in to continue`, connected ? 'good' : 'warn'); }); + // Keyboard plugged in or out: route playback to it, or back to the app piano. + midi.ports.on(() => this.applyMidiOut()); midi.deviceChanged.on((name) => { meter.set({ ...meter.value, source: 'midi', deviceName: name, active: !!name }); if (name && name !== getSettings().midiInputName) useApp.getState().updateSettings({ midiInputName: name }); diff --git a/src/ui/styles.css b/src/ui/styles.css index f4a2aac..946827f 100644 --- a/src/ui/styles.css +++ b/src/ui/styles.css @@ -1049,6 +1049,68 @@ input[type='range'], text-overflow: ellipsis; } +/* Listen-mode options: pill switches beside the song name; on = ink track (selected = inversion). */ +.listen-options { + display: inline-flex; + flex-wrap: wrap; + align-items: center; + gap: var(--space-sm); +} + +.switch { + display: inline-flex; + align-items: center; + gap: var(--space-sm); + min-height: 32px; + padding: var(--space-xs) var(--space-xs) var(--space-xs) var(--space-md); + border: 0; + border-radius: var(--radius-button); + background: var(--surface-soft); + color: var(--text-primary); + font-size: var(--type-button-sm-size); + line-height: var(--type-button-sm-lh); + font-weight: var(--weight-medium); + white-space: nowrap; + cursor: pointer; + transition: + transform var(--duration-base) var(--ease-standard), + opacity var(--duration-base) var(--ease-standard); +} + +.switch:active { + transform: scale(var(--press-scale)); + opacity: var(--press-opacity); +} + +.switch-track { + position: relative; + flex: none; + width: 36px; + height: 22px; + border-radius: var(--radius-full); + background: var(--border-hairline); + transition: background-color var(--duration-base) var(--ease-standard); +} + +.switch-knob { + position: absolute; + top: 3px; + left: 3px; + width: 16px; + height: 16px; + border-radius: var(--radius-full); + background: var(--surface-canvas); + transition: transform var(--duration-base) var(--ease-standard); +} + +.switch[aria-checked='true'] .switch-track { + background: var(--color-primary); +} + +.switch[aria-checked='true'] .switch-knob { + transform: translateX(14px); +} + .results { position: absolute; inset: var(--space-lg); diff --git a/tests/playback/player.test.ts b/tests/playback/player.test.ts index 77edd4d..915453f 100644 --- a/tests/playback/player.test.ts +++ b/tests/playback/player.test.ts @@ -93,6 +93,29 @@ describe('Player', () => { p.stop(); }); + it('reports key down and up times for each scheduled note (ties held, pedal ignored)', () => { + const { p, advance } = setup(); + const score = makeScore(); + score.notes.push({ id: 'e', midi: 64, startBeat: 8, durationBeats: 2, measure: 1, hand: 'R', tiedFromPrevious: true }); + score.measures.push({ index: 2, startBeat: 8, lengthBeats: 4 }); + score.pedal = [ + { beat: 0, down: true }, + { beat: 4, down: false }, + ]; + p.setScore(score); + p.configure({ countInBars: 0, tempoFactor: 1 }); + const keys: { midi: number; time: number; upTime: number }[] = []; + p.scheduled.on((k) => keys.push(k)); + const start = p.start(0); + advance(start + 7); + const c = keys.find((k) => k.midi === 60)!; + expect(c.time).toBeCloseTo(start); + expect(c.upTime).toBeCloseTo(start + 0.5); // one beat at 120 bpm, not held to the pedal lift + const e = keys.find((k) => k.midi === 64)!; + expect(e.upTime - e.time).toBeCloseTo(3); // 4 beats tied into 2 more + expect(keys.filter((k) => k.midi === 64)).toHaveLength(1); + }); + it('logs clicks for mic gating', () => { const { p, log, advance } = setup(); p.configure({ metronome: true, countInBars: 0 }); From 588f835604d32dfd5372be124e6dc27ac1db00c0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 09:15:29 +0000 Subject: [PATCH 2/5] Listen e2e: count key presses and releases instead of sampling for a gap Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HvmoDPbUWx35sRniGQQfBT --- e2e/listen.spec.ts | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/e2e/listen.spec.ts b/e2e/listen.spec.ts index 1e02096..3626204 100644 --- a/e2e/listen.spec.ts +++ b/e2e/listen.spec.ts @@ -30,11 +30,25 @@ test('listen: options appear only in listen mode and the keys play along', async // No MIDI keyboard: no speaker option. await expect(page.getByRole('switch', { name: 'Keyboard speakers' })).toHaveCount(0); + // Count every key going down and coming back up (the gaps between notes are too short to sample). + await page.evaluate(() => { + const moves = { down: 0, up: 0 }; + (window as unknown as { __keyMoves: typeof moves }).__keyMoves = moves; + new MutationObserver((records) => { + for (const r of records) { + const el = r.target as HTMLElement; + const was = (r.oldValue ?? '').split(' ').includes('down'); + const is = el.classList.contains('down'); + if (!was && is) moves.down++; + if (was && !is) moves.up++; + } + }).observe(document.querySelector('.keyboard')!, { subtree: true, attributeFilter: ['class'], attributeOldValue: true }); + }); await page.getByRole('button', { name: /Start/ }).click(); await expect(page.locator('.keyboard .key.down').first()).toBeVisible(); - // Keys come back up between notes, so not every sampled moment has a key down. - await expect.poll(() => page.locator('.keyboard .key.down').count()).toBe(0); - await expect.poll(() => page.locator('.keyboard .key.down').count()).toBeGreaterThan(0); + const moves = () => page.evaluate(() => (window as unknown as { __keyMoves: { down: number; up: number } }).__keyMoves); + await expect.poll(async () => (await moves()).up, { timeout: 10_000 }).toBeGreaterThanOrEqual(3); + expect((await moves()).down).toBeGreaterThanOrEqual(3); await animate.click(); await expect(animate).toHaveAttribute('aria-checked', 'false'); From e9cae5ab32ea00cf3f33a8363f95dc9417e658f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 09:26:37 +0000 Subject: [PATCH 3/5] Listen options: move the switches into a settings menu that opens upwards A quiet settings button sits left of the song name in listen mode; its square menu lists one row per option so more can be added later. It closes on an outside click or Escape. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HvmoDPbUWx35sRniGQQfBT --- e2e/listen.spec.ts | 10 +++- src/ui/practice/ListenOptions.tsx | 89 ++++++++++++++++++++++--------- src/ui/styles.css | 46 ++++++++++++++-- 3 files changed, 115 insertions(+), 30 deletions(-) diff --git a/e2e/listen.spec.ts b/e2e/listen.spec.ts index 3626204..0ea8e72 100644 --- a/e2e/listen.spec.ts +++ b/e2e/listen.spec.ts @@ -24,8 +24,10 @@ async function openOde(page: Page, settings: object) { test('listen: options appear only in listen mode and the keys play along', async ({ page }) => { await openOde(page, { inputSource: 'virtual' }); const animate = page.getByRole('switch', { name: 'Animate keys' }); - await expect(animate).toHaveCount(0); + const menu = page.getByRole('button', { name: 'Listen options' }); + await expect(menu).toHaveCount(0); await page.getByRole('button', { name: 'Listen', exact: true }).click(); + await menu.click(); await expect(animate).toHaveAttribute('aria-checked', 'true'); // No MIDI keyboard: no speaker option. await expect(page.getByRole('switch', { name: 'Keyboard speakers' })).toHaveCount(0); @@ -50,6 +52,9 @@ test('listen: options appear only in listen mode and the keys play along', async await expect.poll(async () => (await moves()).up, { timeout: 10_000 }).toBeGreaterThanOrEqual(3); expect((await moves()).down).toBeGreaterThanOrEqual(3); + // Clicking Start closed the menu; open it again while the piece plays. + await expect(animate).toHaveCount(0); + await menu.click(); await animate.click(); await expect(animate).toHaveAttribute('aria-checked', 'false'); await expect(page.locator('.keyboard .key.down')).toHaveCount(0); @@ -64,10 +69,13 @@ test('listen: plays through the MIDI keyboard speakers when one is connected', a await page.getByRole('button', { name: 'Connect MIDI keyboard' }).click(); await expect(page.locator('.meter')).toContainText('CASIO USB-MIDI'); await page.getByRole('button', { name: 'Listen', exact: true }).click(); + await page.getByRole('button', { name: 'Listen options' }).click(); const speakers = page.getByRole('switch', { name: 'Keyboard speakers' }); await expect(speakers).toHaveAttribute('aria-checked', 'false'); await speakers.click(); await expect(speakers).toHaveAttribute('aria-checked', 'true'); + await page.keyboard.press('Escape'); + await expect(speakers).toHaveCount(0); await page.getByRole('button', { name: /Start/ }).click(); // E4 (64) is the first note of Ode to Joy. await expect.poll(() => page.evaluate(() => (window as unknown as { __midiSent: number[][] }).__midiSent.some((b) => b[0] === 0x90 && b[1] === 64))).toBe(true); diff --git a/src/ui/practice/ListenOptions.tsx b/src/ui/practice/ListenOptions.tsx index 1e67d0f..e1b5488 100644 --- a/src/ui/practice/ListenOptions.tsx +++ b/src/ui/practice/ListenOptions.tsx @@ -1,7 +1,8 @@ -import { useEffect, useState } from 'react'; +import { useEffect, useRef, useState } from 'react'; import { useApp } from '../store'; import { runtime } from '../runtime'; import { liveKeys } from '../live'; +import { Icon } from '../components/Icon'; /** Names of the connected MIDI outputs (empty until MIDI is in use). */ function useMidiOutputs(): string[] { @@ -37,37 +38,75 @@ function Switch({ on, label, title, onChange }: { on: boolean; label: string; ti ); } -/** Listen-mode options: play through the MIDI keyboard's speakers, and animate the on-screen keys. */ +/** + * Listen-mode options behind a small settings button; the menu opens upwards over the keyboard. + * Each option is a row with a switch, so more can be added as rows. + */ export function ListenOptions() { const settings = useApp((s) => s.settings); const update = useApp((s) => s.updateSettings); const outputs = useMidiOutputs(); const speakers = outputs.length > 0; + const [open, setOpen] = useState(false); + const ref = useRef(null); + + useEffect(() => { + if (!open) return; + const down = (e: PointerEvent) => { + if (!ref.current?.contains(e.target as Node)) setOpen(false); + }; + const key = (e: KeyboardEvent) => { + if (e.key === 'Escape') setOpen(false); + }; + window.addEventListener('pointerdown', down); + window.addEventListener('keydown', key); + return () => { + window.removeEventListener('pointerdown', down); + window.removeEventListener('keydown', key); + }; + }, [open]); + return ( -
- {speakers && ( - - update({ - midiOutPlayback: on, - // The remembered output is gone: use the one that is connected. - ...(on && !(settings.midiOutputName && outputs.includes(settings.midiOutputName)) ? { midiOutputName: outputs[0] } : {}), - }) - } - /> +
+ + {open && ( +
+
Listen options
+ {speakers && ( + + update({ + midiOutPlayback: on, + // The remembered output is gone: use the one that is connected. + ...(on && !(settings.midiOutputName && outputs.includes(settings.midiOutputName)) ? { midiOutputName: outputs[0] } : {}), + }) + } + /> + )} + { + update({ listenKeyAnimation: on }); + if (!on) liveKeys.clearPlaying(); + }} + /> +
)} - { - update({ listenKeyAnimation: on }); - if (!on) liveKeys.clearPlaying(); - }} - />
); } diff --git a/src/ui/styles.css b/src/ui/styles.css index 946827f..dff1af4 100644 --- a/src/ui/styles.css +++ b/src/ui/styles.css @@ -1049,12 +1049,50 @@ input[type='range'], text-overflow: ellipsis; } -/* Listen-mode options: pill switches beside the song name; on = ink track (selected = inversion). */ +/* + * Listen-mode options: a quiet settings button; its menu opens upwards as a square panel with + * one row per option. Switches are pill tracks; on = ink (selected = inversion). + */ .listen-options { + position: relative; display: inline-flex; - flex-wrap: wrap; - align-items: center; - gap: var(--space-sm); +} + +.listen-options .icon-btn { + width: 32px; + height: 32px; + color: var(--text-secondary); +} + +.listen-options .icon-btn[aria-expanded='true'] { + color: var(--text-primary); +} + +.listen-menu { + position: absolute; + right: 0; + /* Clears the hairline above the keyboard panel so the menu reads as floating over the stage. */ + bottom: calc(100% + var(--space-lg)); + z-index: 30; + min-width: 240px; + display: flex; + flex-direction: column; + padding: var(--space-sm); + background: var(--surface-canvas); + border: 1px solid var(--border-hairline); +} + +.listen-menu-title { + padding: var(--space-xs) var(--space-md) var(--space-sm); + font-size: var(--type-caption-sm-size); + line-height: var(--type-caption-sm-lh); + color: var(--text-secondary); +} + +.listen-menu .switch { + justify-content: space-between; + width: 100%; + background: transparent; } .switch { From 9b1bba4b1563cea49b0810e2c1c896456fb260fb Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 09:33:20 +0000 Subject: [PATCH 4/5] Listen options: hand colours on animated keys A Hand colours switch (on by default, off when keys are not animated) lights the keys the app plays blue for the right hand and purple for the left, matching the falling notes. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HvmoDPbUWx35sRniGQQfBT --- e2e/listen.spec.ts | 24 ++++++++++++++++++------ src/core/settings.ts | 3 +++ src/ui/components/PianoKeyboard.tsx | 8 ++++++-- src/ui/live.ts | 22 +++++++++++++++++----- src/ui/practice/ListenOptions.tsx | 11 +++++++++-- src/ui/practice/controller.ts | 7 ++++--- src/ui/styles.css | 29 +++++++++++++++++++++++++++-- 7 files changed, 84 insertions(+), 20 deletions(-) diff --git a/e2e/listen.spec.ts b/e2e/listen.spec.ts index 0ea8e72..5326415 100644 --- a/e2e/listen.spec.ts +++ b/e2e/listen.spec.ts @@ -39,27 +39,39 @@ test('listen: options appear only in listen mode and the keys play along', async new MutationObserver((records) => { for (const r of records) { const el = r.target as HTMLElement; - const was = (r.oldValue ?? '').split(' ').includes('down'); - const is = el.classList.contains('down'); + const lit = (c: string[]) => c.some((x) => x === 'down' || x === 'play-R' || x === 'play-L'); + const was = lit((r.oldValue ?? '').split(' ')); + const is = lit([...el.classList]); if (!was && is) moves.down++; if (was && !is) moves.up++; } }).observe(document.querySelector('.keyboard')!, { subtree: true, attributeFilter: ['class'], attributeOldValue: true }); }); await page.getByRole('button', { name: /Start/ }).click(); - await expect(page.locator('.keyboard .key.down').first()).toBeVisible(); const moves = () => page.evaluate(() => (window as unknown as { __keyMoves: { down: number; up: number } }).__keyMoves); await expect.poll(async () => (await moves()).up, { timeout: 10_000 }).toBeGreaterThanOrEqual(3); expect((await moves()).down).toBeGreaterThanOrEqual(3); + // Hand colours (on by default): Ode to Joy's right hand lights blue, never the neutral pressed look. + await expect(page.locator('.keyboard .key.play-R').first()).toBeVisible(); + expect(await page.locator('.keyboard .key.down, .keyboard .key.play-L').count()).toBe(0); + // Clicking Start closed the menu; open it again while the piece plays. await expect(animate).toHaveCount(0); await menu.click(); + const colours = page.getByRole('switch', { name: 'Hand colours' }); + await colours.click(); + await expect(colours).toHaveAttribute('aria-checked', 'false'); + await expect(page.locator('.keyboard .key.down').first()).toBeVisible(); + await expect(page.locator('.keyboard .key.play-R')).toHaveCount(0); + await animate.click(); + await expect(colours).toBeDisabled(); await expect(animate).toHaveAttribute('aria-checked', 'false'); - await expect(page.locator('.keyboard .key.down')).toHaveCount(0); + const lit = page.locator('.keyboard .key.down, .keyboard .key.play-R, .keyboard .key.play-L'); + await expect(lit).toHaveCount(0); await page.waitForTimeout(1500); - await expect(page.locator('.keyboard .key.down')).toHaveCount(0); + await expect(lit).toHaveCount(0); await page.getByRole('button', { name: /Stop/ }).click(); }); @@ -79,6 +91,6 @@ test('listen: plays through the MIDI keyboard speakers when one is connected', a await page.getByRole('button', { name: /Start/ }).click(); // E4 (64) is the first note of Ode to Joy. await expect.poll(() => page.evaluate(() => (window as unknown as { __midiSent: number[][] }).__midiSent.some((b) => b[0] === 0x90 && b[1] === 64))).toBe(true); - await expect(page.locator('.keyboard .key.down').first()).toBeVisible(); + await expect(page.locator('.keyboard .key.play-R').first()).toBeVisible(); await page.getByRole('button', { name: /Stop/ }).click(); }); diff --git a/src/core/settings.ts b/src/core/settings.ts index 0d515f7..bd96274 100644 --- a/src/core/settings.ts +++ b/src/core/settings.ts @@ -42,6 +42,8 @@ export interface Settings { errorSound: boolean; /** Listen mode: the on-screen keys go down and up with the notes the app plays. */ listenKeyAnimation: boolean; + /** Listen mode: animated keys take their hand's colour (blue right, purple left). */ + listenHandColours: boolean; splitPoint: number; detector: DetectorParams; debug: boolean; @@ -79,6 +81,7 @@ export const DEFAULT_SETTINGS: Settings = { micAllowSpeakerPlayback: false, errorSound: false, listenKeyAnimation: true, + listenHandColours: true, splitPoint: 60, detector: DEFAULT_DETECTOR_PARAMS, debug: false, diff --git a/src/ui/components/PianoKeyboard.tsx b/src/ui/components/PianoKeyboard.tsx index d579adf..f979c74 100644 --- a/src/ui/components/PianoKeyboard.tsx +++ b/src/ui/components/PianoKeyboard.tsx @@ -60,14 +60,18 @@ export function PianoKeyboard({ low, high, height = 120, labels = 'c', keyHintsB useEffect(() => { const apply = () => { for (const [midi, el] of els.current) { - const mark = liveKeys.pressed.get(midi) ?? (liveKeys.playing.has(midi) ? 'down' : undefined); + // Keys the app plays (listen mode) look pressed, in their hand's colour when it has one. + const played = liveKeys.pressed.has(midi) ? undefined : liveKeys.playingHand(midi); + const mark = liveKeys.pressed.get(midi) ?? (played === null ? 'down' : undefined); const up = liveKeys.upcoming.get(midi); el.classList.toggle('down', mark === 'down'); el.classList.toggle('hit', mark === 'hit'); el.classList.toggle('wrong', mark === 'wrong'); el.classList.toggle('up-R', up === 'R' || up === 'unknown'); el.classList.toggle('up-L', up === 'L'); - el.setAttribute('aria-pressed', mark ? 'true' : 'false'); + el.classList.toggle('play-R', played === 'R' || played === 'unknown'); + el.classList.toggle('play-L', played === 'L'); + el.setAttribute('aria-pressed', mark || played ? 'true' : 'false'); } }; apply(); diff --git a/src/ui/live.ts b/src/ui/live.ts index 19cede5..f3abc65 100644 --- a/src/ui/live.ts +++ b/src/ui/live.ts @@ -11,8 +11,11 @@ class LiveKeys { pressed = new Map(); /** Upcoming keys to play (outlined), with hand. */ upcoming = new Map(); - /** Keys the app is playing in listen mode: midi -> tokens of the notes holding it down. */ - playing = new Map>(); + /** + * Keys the app is playing in listen mode: midi -> the notes holding it down (token -> hand, + * or null for the neutral pressed look). + */ + playing = new Map>(); private listeners = new Set<() => void>(); private scheduled = false; private flashTimers = new Map(); @@ -71,10 +74,10 @@ class LiveKeys { } /** Listen mode: a note the app plays holds its key down (tokens keep overlapping notes apart). */ - playOn(midi: number, token: number) { + playOn(midi: number, token: number, hand: Hand | null = null) { const held = this.playing.get(midi); - if (held) held.add(token); - else this.playing.set(midi, new Set([token])); + if (held) held.set(token, hand); + else this.playing.set(midi, new Map([[token, hand]])); this.changed(); } @@ -85,6 +88,15 @@ class LiveKeys { this.changed(); } + /** Hand colour of a key the app is playing (the latest note on it); null = neutral; undefined = not playing. */ + playingHand(midi: number): Hand | null | undefined { + const held = this.playing.get(midi); + if (!held) return undefined; + let hand: Hand | null = null; + for (const h of held.values()) hand = h; + return hand; + } + clearPlaying() { if (!this.playing.size) return; this.playing.clear(); diff --git a/src/ui/practice/ListenOptions.tsx b/src/ui/practice/ListenOptions.tsx index e1b5488..49410f1 100644 --- a/src/ui/practice/ListenOptions.tsx +++ b/src/ui/practice/ListenOptions.tsx @@ -27,9 +27,9 @@ function useMidiOutputs(): string[] { return names; } -function Switch({ on, label, title, onChange }: { on: boolean; label: string; title: string; onChange: (on: boolean) => void }) { +function Switch({ on, label, title, disabled, onChange }: { on: boolean; label: string; title: string; disabled?: boolean; onChange: (on: boolean) => void }) { return ( -
)} diff --git a/src/ui/practice/controller.ts b/src/ui/practice/controller.ts index 79970d7..6c59d5c 100644 --- a/src/ui/practice/controller.ts +++ b/src/ui/practice/controller.ts @@ -210,7 +210,7 @@ class PracticeController { practiceLive.changed(); }), ); - if (opts.mode === 'listen') this.unsubs.push(engine.player.scheduled.on((n) => this.animateKey(n.midi, n.time, n.upTime))); + if (opts.mode === 'listen') this.unsubs.push(engine.player.scheduled.on((n) => this.animateKey(n.midi, n.hand, n.time, n.upTime))); session.start(); practiceLive.running = true; this.stateChange.emit(session.state); @@ -226,7 +226,7 @@ class PracticeController { } /** Listen mode: presses the on-screen key when the note is heard and lifts it when it ends. */ - private animateKey(midi: number, time: number, upTime: number) { + private animateKey(midi: number, hand: Hand, time: number, upTime: number) { const engine = runtime.engine; if (!engine || !getSettings().listenKeyAnimation) return; const token = ++this.keyToken; @@ -243,7 +243,8 @@ class PracticeController { this.keyTimers.add(id); }; at(down, () => { - if (getSettings().listenKeyAnimation) liveKeys.playOn(midi, token); + const s = getSettings(); + if (s.listenKeyAnimation) liveKeys.playOn(midi, token, s.listenHandColours ? hand : null); }); at(up, () => liveKeys.playOff(midi, token)); } diff --git a/src/ui/styles.css b/src/ui/styles.css index dff1af4..1c4359d 100644 --- a/src/ui/styles.css +++ b/src/ui/styles.css @@ -750,6 +750,20 @@ input[type='range'], background: var(--ok-red); color: var(--ok-white); } +/* Listen mode with hand colours: the key the app plays dips in its hand's colour. */ +.key.play-R, +.key.play-L { + transform: scaleY(0.98); + transition-duration: var(--duration-fast); +} +.key.play-R { + background: var(--hand-r); + color: var(--ok-white); +} +.key.play-L { + background: var(--hand-l); + color: var(--ok-ink); +} .key.middle-c::after { content: ''; position: absolute; @@ -1115,7 +1129,16 @@ input[type='range'], opacity var(--duration-base) var(--ease-standard); } -.switch:active { +.switch:disabled { + color: var(--text-tertiary); + cursor: not-allowed; +} + +.switch:disabled .switch-track { + opacity: 0.5; +} + +.switch:active:not(:disabled) { transform: scale(var(--press-scale)); opacity: var(--press-opacity); } @@ -1389,7 +1412,9 @@ kbd { .key.hit, .key.wrong, .key.up-R, - .key.up-L { + .key.up-L, + .key.play-R, + .key.play-L { outline: 3px solid Highlight; } .btn.primary, From 891720565f42d0cc15c09c872242a4a649634596 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 09:48:26 +0000 Subject: [PATCH 5/5] Practice bar: switch the input between computer keys, MIDI and microphone A small icon picker beside the input meter changes the judged input without going to Settings; it is locked while a take runs. The meter shows the new source straight away and calibration reloads for it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HvmoDPbUWx35sRniGQQfBT --- e2e/listen.spec.ts | 27 +++++++++++++++++ src/ui/components/SourceSwitch.tsx | 33 +++++++++++++++++++++ src/ui/practice/PracticeScreen.tsx | 2 ++ src/ui/runtime.ts | 9 ++++++ src/ui/styles.css | 47 +++++++++++++++++++++++++++++- 5 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 src/ui/components/SourceSwitch.tsx diff --git a/e2e/listen.spec.ts b/e2e/listen.spec.ts index 5326415..36233b0 100644 --- a/e2e/listen.spec.ts +++ b/e2e/listen.spec.ts @@ -94,3 +94,30 @@ test('listen: plays through the MIDI keyboard speakers when one is connected', a await expect(page.locator('.keyboard .key.play-R').first()).toBeVisible(); await page.getByRole('button', { name: /Stop/ }).click(); }); + +test('the practice bar switches the input between computer keys, MIDI and microphone', async ({ page }) => { + await openOde(page, { inputSource: 'virtual' }); + const input = page.getByRole('radiogroup', { name: 'Input' }); + const meter = page.locator('.meter .detail'); + await expect(input.getByRole('radio', { name: 'Computer keys' })).toHaveAttribute('aria-checked', 'true'); + await expect(meter).toHaveText('Computer keys / on-screen piano'); + + await input.getByRole('radio', { name: 'MIDI keyboard' }).click(); + await expect(input.getByRole('radio', { name: 'MIDI keyboard' })).toHaveAttribute('aria-checked', 'true'); + await expect(page.getByRole('button', { name: 'Connect MIDI keyboard' })).toBeVisible(); + await expect(meter).not.toHaveText('Computer keys / on-screen piano'); + + await input.getByRole('radio', { name: 'Microphone' }).click(); + await expect(page.getByRole('button', { name: 'Turn on the microphone' })).toBeVisible(); + + await input.getByRole('radio', { name: 'Computer keys' }).click(); + await expect(meter).toHaveText('Computer keys / on-screen piano'); + await expect(page.getByRole('button', { name: 'Turn on the microphone' })).toHaveCount(0); + + // Locked while a take runs. + await page.getByRole('button', { name: 'Listen', exact: true }).click(); + await page.getByRole('button', { name: /Start/ }).click(); + await expect(input.getByRole('radio', { name: 'Microphone' })).toBeDisabled(); + await page.getByRole('button', { name: /Stop/ }).click(); + await expect(input.getByRole('radio', { name: 'Microphone' })).toBeEnabled(); +}); diff --git a/src/ui/components/SourceSwitch.tsx b/src/ui/components/SourceSwitch.tsx new file mode 100644 index 0000000..ef724ad --- /dev/null +++ b/src/ui/components/SourceSwitch.tsx @@ -0,0 +1,33 @@ +import type { InputSource } from '../../core/types'; +import { useApp } from '../store'; +import { runtime } from '../runtime'; +import { Icon } from './Icon'; + +const SOURCES: { id: InputSource; label: string; icon: 'keyboard' | 'piano' | 'mic' }[] = [ + { id: 'virtual', label: 'Computer keys', icon: 'keyboard' }, + { id: 'midi', label: 'MIDI keyboard', icon: 'piano' }, + { id: 'mic', label: 'Microphone', icon: 'mic' }, +]; + +/** Compact input picker beside the input meter: computer keys, MIDI keyboard or microphone. */ +export function SourceSwitch({ disabled = false }: { disabled?: boolean }) { + const source = useApp((s) => s.settings.inputSource); + return ( +
+ {SOURCES.map((s) => ( + + ))} +
+ ); +} diff --git a/src/ui/practice/PracticeScreen.tsx b/src/ui/practice/PracticeScreen.tsx index 25d3709..975012c 100644 --- a/src/ui/practice/PracticeScreen.tsx +++ b/src/ui/practice/PracticeScreen.tsx @@ -22,6 +22,7 @@ import { ShortcutsHelp } from '../components/ShortcutsHelp'; import { kvSet } from '../../core/progress/db'; import { Icon } from '../components/Icon'; import { ListenOptions } from './ListenOptions'; +import { SourceSwitch } from '../components/SourceSwitch'; const MODES: { id: PracticeMode; label: string; hint: string }[] = [ { id: 'listen', label: 'Listen', hint: 'Hear the piece; the cursor follows.' }, @@ -377,6 +378,7 @@ function Practice({ score }: { score: Score }) {
{!compact && } +
{line || (running ? '' : range ? modeHint : `${modeHint} Drag across bars to loop.`)}
diff --git a/src/ui/runtime.ts b/src/ui/runtime.ts index bc7b267..5f788b7 100644 --- a/src/ui/runtime.ts +++ b/src/ui/runtime.ts @@ -216,6 +216,15 @@ class Runtime { return this.midi; } + /** Switches the judged input; the meter shows the new source straight away. */ + setSource(source: InputSource): void { + if (source !== 'mic') this.stopMic(); + useApp.getState().updateSettings({ inputSource: source }); + const device = source === 'midi' ? (this.midi?.current?.name ?? null) : null; + meter.set({ ...meter.value, source, deviceName: device, active: source === 'midi' && !!device, noteName: null, cents: null, confidence: 0, lastVelocity: null, levelDb: -100 }); + void this.reloadCalibration(); + } + /** Routes scheduled playback to the keyboard over MIDI out when enabled. */ applyMidiOut(): void { const eng = getEngine(); diff --git a/src/ui/styles.css b/src/ui/styles.css index 1c4359d..f119488 100644 --- a/src/ui/styles.css +++ b/src/ui/styles.css @@ -882,7 +882,52 @@ input[type='range'], justify-content: center; } -.toolbar .tempo .pulse-dot { +.toolbar .tempo /* Input picker beside the meter: a soft-cloud pill track of round icon buttons; chosen = ink. */ +.source-switch { + display: inline-flex; + align-items: center; + flex: none; + padding: var(--space-xxs); + border-radius: var(--radius-button); + background: var(--surface-soft); +} + +.source-switch button { + width: 28px; + height: 28px; + display: inline-flex; + align-items: center; + justify-content: center; + padding: 0; + border: 0; + border-radius: var(--radius-full); + background: transparent; + color: var(--text-secondary); + cursor: pointer; + transition: + transform var(--duration-base) var(--ease-standard), + opacity var(--duration-base) var(--ease-standard); +} + +.source-switch button[aria-checked='true'] { + background: var(--color-primary); + color: var(--color-on-primary); +} + +.source-switch button:active:not(:disabled) { + transform: scale(var(--press-scale)); + opacity: var(--press-opacity); +} + +.source-switch button:disabled { + cursor: not-allowed; +} + +.source-switch button:disabled:not([aria-checked='true']) { + color: var(--text-tertiary); +} + +.pulse-dot { margin-right: var(--space-sm); }