Listen options, animated keys and an input picker in the practice bar

e2e/listen.spec.ts

@@ -0,0 +1,123 @@
+import { test, expect, type Page } from '@playwright/test';
+
+type OK = { useApp: { getState: () => { settingsLoaded: boolean; updateSettings: (p: object) => void } } };
+
+/** Mocked Web MIDI: one keyboard with an input and an output that records what it is sent. */
+const MOCK = () => {
+  const input = { id: 'in1', name: 'CASIO USB-MIDI', manufacturer: 'Casio', state: 'connected', type: 'input', onmidimessage: null };
+  const sent: number[][] = [];
+  const output = { id: 'out1', name: 'CASIO USB-MIDI', manufacturer: 'Casio', state: 'connected', type: 'output', send: (b: number[]) => sent.push([...b]) };
+  const access = { inputs: new Map([[input.id, input]]), outputs: new Map([[output.id, output]]), onstatechange: null, sysexEnabled: false };
+  (navigator as unknown as { requestMIDIAccess: () => Promise<unknown> }).requestMIDIAccess = async () => access;
+  (window as unknown as { __midiSent: number[][] }).__midiSent = sent;
+};
+
+async function openOde(page: Page, settings: object) {
+  await page.goto('/');
+  await page.waitForFunction(() => (globalThis as unknown as { __openkeys: OK }).__openkeys.useApp.getState().settingsLoaded);
+  await page.evaluate((s) => (globalThis as unknown as { __openkeys: OK }).__openkeys.useApp.getState().updateSettings({ countInBars: 0, metronome: false, onboarded: true, ...s }), settings);
+  await page.getByRole('button', { name: /Library/ }).first().click();
+  await page.locator('.card', { hasText: 'Ode to Joy' }).getByRole('button', { name: 'Right hand' }).click();
+  await expect(page.getByRole('button', { name: /Start/ })).toBeVisible();
+}
+
+test('listen: options appear only in listen mode and the keys play along', async ({ page }) => {
+  await openOde(page, { inputSource: 'virtual' });
+  const animate = page.getByRole('switch', { name: 'Animate keys' });
+  const menu = page.getByRole('button', { name: 'Listen options' });
+  await expect(menu).toHaveCount(0);
+  await page.getByRole('button', { name: 'Listen', exact: true }).click();
+  await menu.click();
+  await expect(animate).toHaveAttribute('aria-checked', 'true');
+  // No MIDI keyboard: no speaker option.
+  await expect(page.getByRole('switch', { name: 'Keyboard speakers' })).toHaveCount(0);
+
+  // Count every key going down and coming back up (the gaps between notes are too short to sample).
+  await page.evaluate(() => {
+    const moves = { down: 0, up: 0 };
+    (window as unknown as { __keyMoves: typeof moves }).__keyMoves = moves;
+    new MutationObserver((records) => {
+      for (const r of records) {
+        const el = r.target as HTMLElement;
+        const lit = (c: string[]) => c.some((x) => x === 'down' || x === 'play-R' || x === 'play-L');
+        const was = lit((r.oldValue ?? '').split(' '));
+        const is = lit([...el.classList]);
+        if (!was && is) moves.down++;
+        if (was && !is) moves.up++;
+      }
+    }).observe(document.querySelector('.keyboard')!, { subtree: true, attributeFilter: ['class'], attributeOldValue: true });
+  });
+  await page.getByRole('button', { name: /Start/ }).click();
+  const moves = () => page.evaluate(() => (window as unknown as { __keyMoves: { down: number; up: number } }).__keyMoves);
+  await expect.poll(async () => (await moves()).up, { timeout: 10_000 }).toBeGreaterThanOrEqual(3);
+  expect((await moves()).down).toBeGreaterThanOrEqual(3);
+
+  // Hand colours (on by default): Ode to Joy's right hand lights blue, never the neutral pressed look.
+  await expect(page.locator('.keyboard .key.play-R').first()).toBeVisible();
+  expect(await page.locator('.keyboard .key.down, .keyboard .key.play-L').count()).toBe(0);
+
+  // Clicking Start closed the menu; open it again while the piece plays.
+  await expect(animate).toHaveCount(0);
+  await menu.click();
+  const colours = page.getByRole('switch', { name: 'Hand colours' });
+  await colours.click();
+  await expect(colours).toHaveAttribute('aria-checked', 'false');
+  await expect(page.locator('.keyboard .key.down').first()).toBeVisible();
+  await expect(page.locator('.keyboard .key.play-R')).toHaveCount(0);
+
+  await animate.click();
+  await expect(colours).toBeDisabled();
+  await expect(animate).toHaveAttribute('aria-checked', 'false');
+  const lit = page.locator('.keyboard .key.down, .keyboard .key.play-R, .keyboard .key.play-L');
+  await expect(lit).toHaveCount(0);
+  await page.waitForTimeout(1500);
+  await expect(lit).toHaveCount(0);
+  await page.getByRole('button', { name: /Stop/ }).click();
+});
+
+test('listen: plays through the MIDI keyboard speakers when one is connected', async ({ page }) => {
+  await page.addInitScript(MOCK);
+  await openOde(page, { inputSource: 'midi' });
+  await page.getByRole('button', { name: 'Connect MIDI keyboard' }).click();
+  await expect(page.locator('.meter')).toContainText('CASIO USB-MIDI');
+  await page.getByRole('button', { name: 'Listen', exact: true }).click();
+  await page.getByRole('button', { name: 'Listen options' }).click();
+  const speakers = page.getByRole('switch', { name: 'Keyboard speakers' });
+  await expect(speakers).toHaveAttribute('aria-checked', 'false');
+  await speakers.click();
+  await expect(speakers).toHaveAttribute('aria-checked', 'true');
+  await page.keyboard.press('Escape');
+  await expect(speakers).toHaveCount(0);
+  await page.getByRole('button', { name: /Start/ }).click();
+  // E4 (64) is the first note of Ode to Joy.
+  await expect.poll(() => page.evaluate(() => (window as unknown as { __midiSent: number[][] }).__midiSent.some((b) => b[0] === 0x90 && b[1] === 64))).toBe(true);
+  await expect(page.locator('.keyboard .key.play-R').first()).toBeVisible();
+  await page.getByRole('button', { name: /Stop/ }).click();
+});
+
+test('the practice bar switches the input between computer keys, MIDI and microphone', async ({ page }) => {
+  await openOde(page, { inputSource: 'virtual' });
+  const input = page.getByRole('radiogroup', { name: 'Input' });
+  const meter = page.locator('.meter .detail');
+  await expect(input.getByRole('radio', { name: 'Computer keys' })).toHaveAttribute('aria-checked', 'true');
+  await expect(meter).toHaveText('Computer keys / on-screen piano');
+
+  await input.getByRole('radio', { name: 'MIDI keyboard' }).click();
+  await expect(input.getByRole('radio', { name: 'MIDI keyboard' })).toHaveAttribute('aria-checked', 'true');
+  await expect(page.getByRole('button', { name: 'Connect MIDI keyboard' })).toBeVisible();
+  await expect(meter).not.toHaveText('Computer keys / on-screen piano');
+
+  await input.getByRole('radio', { name: 'Microphone' }).click();
+  await expect(page.getByRole('button', { name: 'Turn on the microphone' })).toBeVisible();
+
+  await input.getByRole('radio', { name: 'Computer keys' }).click();
+  await expect(meter).toHaveText('Computer keys / on-screen piano');
+  await expect(page.getByRole('button', { name: 'Turn on the microphone' })).toHaveCount(0);
+
+  // Locked while a take runs.
+  await page.getByRole('button', { name: 'Listen', exact: true }).click();
+  await page.getByRole('button', { name: /Start/ }).click();
+  await expect(input.getByRole('radio', { name: 'Microphone' })).toBeDisabled();
+  await page.getByRole('button', { name: /Stop/ }).click();
+  await expect(input.getByRole('radio', { name: 'Microphone' })).toBeEnabled();
+});

src/core/playback/player.ts

@@ -1,9 +1,10 @@
-import type { HandSelection, Score, ScoreNote } from '../types';
+import type { Hand, HandSelection, Score, ScoreNote } from '../types';
 import { TempoMap, metronomeBeats, scoreEndBeat, timeSignatureAt, measureLengthBeats } from '../score/tempo';
 import { Emitter } from '../emitter';
 import { Timeline } from './timeline';
 import type { ClickSink, NoteSink, ScheduledAudioLog } from './sink';
 import { soundingEndBeat } from './pedal';
+import { tiedLengthBeats } from '../score/events';
 
 export interface PlayerConfig {
   /** 0.25 - 1.5 */
@@ -49,6 +50,11 @@ export class Player {
   readonly ended = new Emitter<void>();
   /** Visual metronome pulse (fires at schedule time with the click's context time). */
   readonly pulse = new Emitter<{ time: number; accent: boolean; sub: boolean; countIn: boolean }>();
+  /**
+   * Each score note as it is scheduled: context times of the key going down and coming back up
+   * (the written length including ties, not the pedalled sound). Drives the listen-mode keys.
+   */
+  readonly scheduled = new Emitter<{ midi: number; hand: Hand; time: number; upTime: number }>();
 
   state: PlayerState = 'stopped';
   config: PlayerConfig = { ...DEFAULT_PLAYER_CONFIG };
@@ -208,6 +214,10 @@ export class Player {
         const vel = n.velocity ?? this.config.defaultVelocity;
         this.sink.play(n.midi, vel, t, dur);
         this.log?.addNote(n.midi, t, t + dur);
+        if (this.scheduled.size) {
+          const upBeat = Math.min(this.segEnd, n.startBeat + tiedLengthBeats(this.score.notes, n));
+          this.scheduled.emit({ midi: n.midi, hand: n.hand, time: t, upTime: Math.max(t + 0.05, this.segTime(upBeat)) });
+        }
       }
       // Clicks.
       while (this.clickIdx < this.clicks.length && this.clicks[this.clickIdx].beat < horizonBeat) {

src/core/settings.ts

@@ -40,6 +40,10 @@ export interface Settings {
   /** In mic mode, allow app audio through the speakers anyway (default muted). */
   micAllowSpeakerPlayback: boolean;
   errorSound: boolean;
+  /** Listen mode: the on-screen keys go down and up with the notes the app plays. */
+  listenKeyAnimation: boolean;
+  /** Listen mode: animated keys take their hand's colour (blue right, purple left). */
+  listenHandColours: boolean;
   splitPoint: number;
   detector: DetectorParams;
   debug: boolean;
@@ -76,6 +80,8 @@ export const DEFAULT_SETTINGS: Settings = {
   accompaniment: true,
   micAllowSpeakerPlayback: false,
   errorSound: false,
+  listenKeyAnimation: true,
+  listenHandColours: true,
   splitPoint: 60,
   detector: DEFAULT_DETECTOR_PARAMS,
   debug: false,

src/ui/components/PianoKeyboard.tsx

@@ -60,14 +60,18 @@ export function PianoKeyboard({ low, high, height = 120, labels = 'c', keyHintsB
   useEffect(() => {
     const apply = () => {
       for (const [midi, el] of els.current) {
-        const mark = liveKeys.pressed.get(midi);
+        // Keys the app plays (listen mode) look pressed, in their hand's colour when it has one.
+        const played = liveKeys.pressed.has(midi) ? undefined : liveKeys.playingHand(midi);
+        const mark = liveKeys.pressed.get(midi) ?? (played === null ? 'down' : undefined);
         const up = liveKeys.upcoming.get(midi);
         el.classList.toggle('down', mark === 'down');
         el.classList.toggle('hit', mark === 'hit');
         el.classList.toggle('wrong', mark === 'wrong');
         el.classList.toggle('up-R', up === 'R' || up === 'unknown');
         el.classList.toggle('up-L', up === 'L');
-        el.setAttribute('aria-pressed', mark ? 'true' : 'false');
+        el.classList.toggle('play-R', played === 'R' || played === 'unknown');
+        el.classList.toggle('play-L', played === 'L');
+        el.setAttribute('aria-pressed', mark || played ? 'true' : 'false');
       }
     };
     apply();

src/ui/components/SourceSwitch.tsx

@@ -0,0 +1,33 @@
+import type { InputSource } from '../../core/types';
+import { useApp } from '../store';
+import { runtime } from '../runtime';
+import { Icon } from './Icon';
+
+const SOURCES: { id: InputSource; label: string; icon: 'keyboard' | 'piano' | 'mic' }[] = [
+  { id: 'virtual', label: 'Computer keys', icon: 'keyboard' },
+  { id: 'midi', label: 'MIDI keyboard', icon: 'piano' },
+  { id: 'mic', label: 'Microphone', icon: 'mic' },
+];
+
+/** Compact input picker beside the input meter: computer keys, MIDI keyboard or microphone. */
+export function SourceSwitch({ disabled = false }: { disabled?: boolean }) {
+  const source = useApp((s) => s.settings.inputSource);
+  return (
+    <div className="source-switch" role="radiogroup" aria-label="Input">
+      {SOURCES.map((s) => (
+        <button
+          key={s.id}
+          type="button"
+          role="radio"
+          aria-checked={source === s.id}
+          aria-label={s.label}
+          title={disabled ? `${s.label} (stop first to switch)` : s.label}
+          disabled={disabled}
+          onClick={() => source !== s.id && runtime.setSource(s.id)}
+        >
+          <Icon name={s.icon} size={16} />
+        </button>
+      ))}
+    </div>
+  );
+}

src/ui/live.ts

@@ -11,6 +11,11 @@ class LiveKeys {
   pressed = new Map<number, KeyMark>();
   /** Upcoming keys to play (outlined), with hand. */
   upcoming = new Map<number, Hand>();
+  /**
+   * Keys the app is playing in listen mode: midi -> the notes holding it down (token -> hand,
+   * or null for the neutral pressed look).
+   */
+  playing = new Map<number, Map<number, Hand | null>>();
   private listeners = new Set<() => void>();
   private scheduled = false;
   private flashTimers = new Map<number, number>();
@@ -68,6 +73,36 @@ class LiveKeys {
     );
   }
 
+  /** Listen mode: a note the app plays holds its key down (tokens keep overlapping notes apart). */
+  playOn(midi: number, token: number, hand: Hand | null = null) {
+    const held = this.playing.get(midi);
+    if (held) held.set(token, hand);
+    else this.playing.set(midi, new Map([[token, hand]]));
+    this.changed();
+  }
+
+  playOff(midi: number, token: number) {
+    const held = this.playing.get(midi);
+    if (!held?.delete(token)) return;
+    if (!held.size) this.playing.delete(midi);
+    this.changed();
+  }
+
+  /** Hand colour of a key the app is playing (the latest note on it); null = neutral; undefined = not playing. */
+  playingHand(midi: number): Hand | null | undefined {
+    const held = this.playing.get(midi);
+    if (!held) return undefined;
+    let hand: Hand | null = null;
+    for (const h of held.values()) hand = h;
+    return hand;
+  }
+
+  clearPlaying() {
+    if (!this.playing.size) return;
+    this.playing.clear();
+    this.changed();
+  }
+
   setUpcoming(next: Map<number, Hand>) {
     let same = next.size === this.upcoming.size;
     if (same) for (const [k, v] of next) if (this.upcoming.get(k) !== v) same = false;
@@ -78,6 +113,7 @@ class LiveKeys {
 
   clear() {
     this.pressed.clear();
+    this.playing.clear();
     this.upcoming = new Map();
     this.changed();
   }