diff --git a/.gitignore b/.gitignore index 920c847..53a9755 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,12 @@ yarn-error.log* # OS / Misc *.local + + +# + +*.png +*.playwright-mcp/ + +preview.html +preview-assets.js diff --git a/apps/web/public/mascot/Bust_animated.glb b/apps/web/public/mascot/Bust_animated.glb new file mode 100644 index 0000000..b52e72a Binary files /dev/null and b/apps/web/public/mascot/Bust_animated.glb differ diff --git a/apps/web/public/mascot/Sounds/chime_done.wav b/apps/web/public/mascot/Sounds/chime_done.wav new file mode 100644 index 0000000..1354136 Binary files /dev/null and b/apps/web/public/mascot/Sounds/chime_done.wav differ diff --git a/apps/web/public/mascot/Sounds/dialogue_blip_soft.wav b/apps/web/public/mascot/Sounds/dialogue_blip_soft.wav new file mode 100644 index 0000000..67f1b1c Binary files /dev/null and b/apps/web/public/mascot/Sounds/dialogue_blip_soft.wav differ diff --git a/apps/web/public/mascot/Sounds/dialogue_blip_synth.wav b/apps/web/public/mascot/Sounds/dialogue_blip_synth.wav new file mode 100644 index 0000000..7faebbc Binary files /dev/null and b/apps/web/public/mascot/Sounds/dialogue_blip_synth.wav differ diff --git a/apps/web/public/mascot/Sounds/tech_ui_chirp.wav b/apps/web/public/mascot/Sounds/tech_ui_chirp.wav new file mode 100644 index 0000000..1166921 Binary files /dev/null and b/apps/web/public/mascot/Sounds/tech_ui_chirp.wav differ diff --git a/apps/web/public/mascot/Sounds/text_print_soft.wav b/apps/web/public/mascot/Sounds/text_print_soft.wav new file mode 100644 index 0000000..461b362 Binary files /dev/null and b/apps/web/public/mascot/Sounds/text_print_soft.wav differ diff --git a/apps/web/public/mascot/Sounds/typewriter_key.wav b/apps/web/public/mascot/Sounds/typewriter_key.wav new file mode 100644 index 0000000..275da25 Binary files /dev/null and b/apps/web/public/mascot/Sounds/typewriter_key.wav differ diff --git a/apps/web/public/mascot/Sounds/whisper_murmur.wav b/apps/web/public/mascot/Sounds/whisper_murmur.wav new file mode 100644 index 0000000..915336f Binary files /dev/null and b/apps/web/public/mascot/Sounds/whisper_murmur.wav differ diff --git a/apps/web/public/mascot/textures/presets.json b/apps/web/public/mascot/textures/presets.json new file mode 100644 index 0000000..09fc5c2 --- /dev/null +++ b/apps/web/public/mascot/textures/presets.json @@ -0,0 +1,68 @@ +[ + { + "id": "realistic", + "name": "Realistic Human", + "name_ua": "Реалістична людина", + "icon": "👤", + "desc": "Natural human skin with vascular flush, pores, hazel iris & veined sclera", + "desc_ua": "Анатомічна текстура шкіри з порами, рум'янцем, карими очима та капілярами", + "path": "textures/realistic/", + "metalness": 0.0, + "roughness": 1.0 + }, + { + "id": "cyber", + "name": "Cyber Android", + "name_ua": "Кіберпанк / Андроїд", + "icon": "🤖", + "desc": "Non-human titanium composite shell with glowing cyan circuits & tech iris", + "desc_ua": "Синтетична титанова броня з неоновими бірюзовими мікросхемами та оптичним сенсором", + "path": "textures/cyber/", + "metalness": 0.65, + "roughness": 0.6 + }, + { + "id": "alien", + "name": "Bioluminescent Alien", + "name_ua": "Неземна / Алієн", + "icon": "👽", + "desc": "Monotonic alien violet skin with glowing turquoise bioluminescent markings", + "desc_ua": "Монотонна фіолетова інопланетна шкіра з сяючими біолюмінесцентними точками", + "path": "textures/alien/", + "metalness": 0.08, + "roughness": 0.8 + }, + { + "id": "noir", + "name": "Noir Chiaroscuro", + "name_ua": "Чорно-білий контраст", + "icon": "🎬", + "desc": "High-contrast monochromatic graphic novel aesthetic with deep charcoal shadows", + "desc_ua": "Висококонтрастний монохромний нуар зі стрімкими тінями та світлом", + "path": "textures/noir/", + "metalness": 0.0, + "roughness": 1.0 + }, + { + "id": "stylized", + "name": "Stylized Anime", + "name_ua": "Стилізована / Аніме", + "icon": "🎨", + "desc": "Porcelain cel-shaded skin, soft pastel blush & sparkling emerald anime eyes", + "desc_ua": "Чиста порцелянова шкіра, ніжний рум'янець та виразні смарагдові очі", + "path": "textures/stylized/", + "metalness": 0.0, + "roughness": 0.9 + }, + { + "id": "clay", + "name": "Studio Clay (No Textures)", + "name_ua": "Без текстур (Глина)", + "icon": "🏛️", + "desc": "Clean untextured matte sculpture finish resembling an artist's studio bust", + "desc_ua": "Мінімалістична матова скульптурна глина без текстур, як у художній майстерні", + "path": "textures/clay/", + "metalness": 0.0, + "roughness": 1.0 + } +] \ No newline at end of file diff --git a/apps/web/src/components/ChatRoom.tsx b/apps/web/src/components/ChatRoom.tsx index a64a64b..31f0d89 100644 --- a/apps/web/src/components/ChatRoom.tsx +++ b/apps/web/src/components/ChatRoom.tsx @@ -1,11 +1,33 @@ 'use client'; -import React, { useState, useEffect, useRef } from 'react'; +import React, { useState, useEffect, useRef, useCallback } from 'react'; import { useRouter } from 'next/navigation'; -import { Send, Bot, User, CheckCircle, Sparkles, AlertCircle, Clock } from 'lucide-react'; +import { + Send, + Bot, + User, + CheckCircle, + Sparkles, + AlertCircle, + Clock, + Volume2, + VolumeX, + Smile, + LayoutGrid, + Maximize2, + Columns2, + SlidersHorizontal, + X, + Play, + RotateCcw, + Gauge, + Activity, + Palette, +} from 'lucide-react'; import { InterviewMessage } from '@dev-dialogue/api-contracts'; import { api } from '@/lib/api-client'; import { useTranslation } from '@/lib/useTranslation'; +import { useAuthStore } from '@/store/authStore'; interface ChatRoomProps { interviewId: string; @@ -14,6 +36,477 @@ interface ChatRoomProps { initialLanguage?: string; } +const MASCOT_BASE = '/mascot/'; +const SOUNDS_BASE = '/mascot/Sounds/'; + +interface TexturePresetDef { + id: string; + nameUk: string; + nameEn: string; + icon: string; + descUk: string; + descEn: string; + folder: string | null; + metalness: number; + roughness: number; + skinColor: number; + emissive?: number; + emissiveIntensity?: number; +} + +const TEXTURE_PRESETS: TexturePresetDef[] = [ + { + id: 'realistic', + nameUk: 'Реалістична', + nameEn: 'Realistic', + icon: '👤', + descUk: "Анатомічна текстура шкіри з порами, рум'янцем та карими очима", + descEn: 'Natural anatomical human skin, flush, pores & hazel eyes', + folder: '/mascot/textures/realistic/', + metalness: 0.0, + roughness: 1.0, + skinColor: 0xffffff, + emissive: 0x000000, + emissiveIntensity: 0.0, + }, + { + id: 'cyber', + nameUk: 'Кіберпанк', + nameEn: 'Cyberpunk', + icon: '🤖', + descUk: 'Синтетична титанова броня з неоновими бірюзовими мікросхемами', + descEn: 'Titanium shell with glowing cyan circuits & optical sensor', + folder: '/mascot/textures/cyber/', + metalness: 0.65, + roughness: 0.60, + skinColor: 0xffffff, + emissive: 0x00c8e0, + emissiveIntensity: 0.85, + }, + { + id: 'alien', + nameUk: 'Алієн', + nameEn: 'Alien', + icon: '👽', + descUk: 'Монотонна фіолетова шкіра з біолюмінесцентними точками', + descEn: 'Monotonic violet alien skin with bioluminescent markings', + folder: '/mascot/textures/alien/', + metalness: 0.08, + roughness: 0.85, + skinColor: 0xffffff, + emissive: 0xd08020, + emissiveIntensity: 0.65, + }, + { + id: 'noir', + nameUk: 'Нуар', + nameEn: 'Noir', + icon: '🎬', + descUk: 'Висококонтрастний монохромний нуар зі стрімкими тінями', + descEn: 'High-contrast monochromatic graphic novel style', + folder: '/mascot/textures/noir/', + metalness: 0.0, + roughness: 1.0, + skinColor: 0xffffff, + emissive: 0x000000, + emissiveIntensity: 0.0, + }, + { + id: 'stylized', + nameUk: 'Аніме', + nameEn: 'Anime', + icon: '🎨', + descUk: "Порцелянова шкіра, ніжний рум'янець та виразні смарагдові очі", + descEn: 'Porcelain cel-shaded skin & sparkling emerald eyes', + folder: '/mascot/textures/stylized/', + metalness: 0.0, + roughness: 0.90, + skinColor: 0xffffff, + emissive: 0x000000, + emissiveIntensity: 0.0, + }, + { + id: 'clay', + nameUk: 'Глина', + nameEn: 'Studio Clay', + icon: '🏛️', + descUk: 'Мінімалістична матова скульптурна глина без текстур', + descEn: 'Clean untextured matte sculpture finish', + folder: null, + metalness: 0.0, + roughness: 0.88, + skinColor: 0xbaa898, + }, +]; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function applyPresetToMaterials(THREE: any, mats: Record, preset: TexturePresetDef, textureCache: Map, texLoader: any) { + if (!THREE || !mats || !texLoader) return; + const loadCached = (url: string, isSRGB = true) => { + if (!textureCache.has(url)) { + const tex = texLoader.load(url); + tex.flipY = false; + tex.colorSpace = isSRGB ? THREE.SRGBColorSpace : THREE.NoColorSpace; + textureCache.set(url, tex); + } + return textureCache.get(url); + }; + + const skinMat = mats['basemeshmaterial']; + const irisMat = mats['Mat_Iris']; + const scleraMat = mats['Mat_Sclera']; + const pupilMat = mats['Mat_Pupil']; + const browMat = mats['Mat_Eyebrow']; + const mouthMat = mats['MouthCavity']; + const mouthDeepMat = mats['MouthCavityDeep']; + const teethMat = mats['Mat_Teeth']; + const gumsMat = mats['Mat_Gums']; + const tongueMat = mats['Mat_Tongue']; + + if (!preset.folder) { + if (skinMat) { + skinMat.map = null; + skinMat.roughnessMap = null; + skinMat.color.setHex(preset.skinColor); + skinMat.roughness = preset.roughness; + skinMat.metalness = preset.metalness; + skinMat.needsUpdate = true; + } + if (irisMat) { + irisMat.map = null; + irisMat.color.setHex(0xbaa898); + if (irisMat.emissive) irisMat.emissive.setHex(0x000000); + irisMat.roughness = 0.88; + irisMat.metalness = 0.0; + irisMat.needsUpdate = true; + } + if (scleraMat) { + scleraMat.map = null; + scleraMat.color.setHex(0xbaa898); + scleraMat.roughness = 0.88; + scleraMat.needsUpdate = true; + } + if (pupilMat) { + pupilMat.color.setHex(0x33302c); + pupilMat.needsUpdate = true; + } + if (browMat) { + browMat.map = null; + browMat.color.setHex(0xa89688); + browMat.roughness = 0.88; + browMat.needsUpdate = true; + } + if (mouthMat) { + mouthMat.color.setHex(0x605850); + mouthMat.needsUpdate = true; + } + if (mouthDeepMat) { + mouthDeepMat.color.setHex(0x403830); + mouthDeepMat.needsUpdate = true; + } + if (teethMat) { + teethMat.color.setHex(0xd5c8bc); + teethMat.roughness = 0.88; + teethMat.metalness = 0.0; + teethMat.needsUpdate = true; + } + if (gumsMat) { + gumsMat.color.setHex(0xb09080); + gumsMat.roughness = 0.88; + gumsMat.metalness = 0.0; + gumsMat.needsUpdate = true; + } + if (tongueMat) { + tongueMat.color.setHex(0xa88878); + tongueMat.roughness = 0.88; + tongueMat.metalness = 0.0; + tongueMat.needsUpdate = true; + } + } else { + const skinDiff = loadCached(preset.folder + 'skin_diffuse.png', true); + const skinRough = loadCached(preset.folder + 'skin_roughness.png', false); + const irisDiff = loadCached(preset.folder + 'iris_diffuse.png', true); + const scleraDiff = loadCached(preset.folder + 'sclera_diffuse.png', true); + const browDiff = loadCached(preset.folder + 'eyebrow_diffuse.png', true); + + if (skinMat) { + skinMat.map = skinDiff; + skinMat.roughnessMap = skinRough; + skinMat.color.setHex(preset.skinColor); + skinMat.metalness = preset.metalness; + skinMat.roughness = preset.roughness; + skinMat.needsUpdate = true; + } + if (irisMat) { + irisMat.map = irisDiff; + irisMat.color.setHex(0xffffff); + if (irisMat.emissive) irisMat.emissive.setHex(preset.emissive ?? 0x000000); + if (irisMat.emissiveIntensity !== undefined) irisMat.emissiveIntensity = preset.emissiveIntensity ?? 0.0; + irisMat.roughness = 0.10; + irisMat.needsUpdate = true; + } + if (scleraMat) { + scleraMat.map = scleraDiff; + scleraMat.color.setHex(0xffffff); + scleraMat.roughness = 0.15; + scleraMat.needsUpdate = true; + } + if (pupilMat) { + pupilMat.color.setHex(preset.id === 'cyber' ? 0x051a22 : preset.id === 'noir' ? 0x000000 : 0x030303); + pupilMat.needsUpdate = true; + } + if (browMat) { + browMat.map = browDiff; + browMat.color.setHex(0xffffff); + browMat.roughness = 0.75; + browMat.needsUpdate = true; + } + if (mouthMat) { + mouthMat.color.setHex(preset.id === 'cyber' ? 0x223344 : preset.id === 'alien' ? 0x281242 : preset.id === 'noir' ? 0x202020 : 0x3d0b0b); + mouthMat.needsUpdate = true; + } + if (mouthDeepMat) { + mouthDeepMat.color.setHex(preset.id === 'cyber' ? 0x111c24 : preset.id === 'alien' ? 0x140822 : preset.id === 'noir' ? 0x080808 : 0x150303); + mouthDeepMat.needsUpdate = true; + } + if (teethMat) { + if (preset.id === 'cyber') { + teethMat.color.setHex(0xd8e8f2); + teethMat.roughness = 0.12; + teethMat.metalness = 0.75; + } else if (preset.id === 'alien') { + teethMat.color.setHex(0xc0e0f8); + teethMat.roughness = 0.18; + teethMat.metalness = 0.10; + } else if (preset.id === 'noir') { + teethMat.color.setHex(0xffffff); + teethMat.roughness = 0.20; + teethMat.metalness = 0.0; + } else if (preset.id === 'stylized') { + teethMat.color.setHex(0xfffff8); + teethMat.roughness = 0.18; + teethMat.metalness = 0.0; + } else { + teethMat.color.setHex(0xf2ede2); + teethMat.roughness = 0.22; + teethMat.metalness = 0.0; + } + teethMat.needsUpdate = true; + } + if (gumsMat) { + if (preset.id === 'cyber') { + gumsMat.color.setHex(0x1a2e38); + gumsMat.roughness = 0.45; + gumsMat.metalness = 0.3; + } else if (preset.id === 'alien') { + gumsMat.color.setHex(0x481b66); + gumsMat.roughness = 0.40; + gumsMat.metalness = 0.0; + } else if (preset.id === 'noir') { + gumsMat.color.setHex(0x444444); + gumsMat.roughness = 0.45; + gumsMat.metalness = 0.0; + } else if (preset.id === 'stylized') { + gumsMat.color.setHex(0xe8687a); + gumsMat.roughness = 0.35; + gumsMat.metalness = 0.0; + } else { + gumsMat.color.setHex(0xb24552); + gumsMat.roughness = 0.38; + gumsMat.metalness = 0.0; + } + gumsMat.needsUpdate = true; + } + if (tongueMat) { + if (preset.id === 'cyber') { + tongueMat.color.setHex(0x162c36); + tongueMat.roughness = 0.40; + tongueMat.metalness = 0.2; + } else if (preset.id === 'alien') { + tongueMat.color.setHex(0x602078); + tongueMat.roughness = 0.40; + tongueMat.metalness = 0.0; + } else if (preset.id === 'noir') { + tongueMat.color.setHex(0x383838); + tongueMat.roughness = 0.45; + tongueMat.metalness = 0.0; + } else if (preset.id === 'stylized') { + tongueMat.color.setHex(0xeb6478); + tongueMat.roughness = 0.40; + tongueMat.metalness = 0.0; + } else { + tongueMat.color.setHex(0xad4148); + tongueMat.roughness = 0.45; + tongueMat.metalness = 0.0; + } + tongueMat.needsUpdate = true; + } + } +} + +const VISEME_MAP: Record> = { + а: { jawOpen: 0.48, mouthWide: 0.22, mouthSmileLeft: 0.08, mouthSmileRight: 0.08, cheekSquintLeft: 0.12, cheekSquintRight: 0.12, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, browInnerUp: 0.08, tongueUp: 0 }, + a: { jawOpen: 0.48, mouthWide: 0.22, mouthSmileLeft: 0.08, mouthSmileRight: 0.08, cheekSquintLeft: 0.12, cheekSquintRight: 0.12, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, browInnerUp: 0.08, tongueUp: 0 }, + я: { jawOpen: 0.44, mouthWide: 0.3, mouthSmileLeft: 0.12, mouthSmileRight: 0.12, cheekSquintLeft: 0.14, cheekSquintRight: 0.14, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, browInnerUp: 0.08, tongueUp: 0.2 }, + о: { jawOpen: 0.38, mouthFunnel: 0.46, mouthPucker: 0.22, cheekSquintLeft: 0.08, cheekSquintRight: 0.08, browInnerUp: 0.06, tongueUp: 0 }, + o: { jawOpen: 0.38, mouthFunnel: 0.46, mouthPucker: 0.22, cheekSquintLeft: 0.08, cheekSquintRight: 0.08, browInnerUp: 0.06, tongueUp: 0 }, + у: { jawOpen: 0.2, mouthPucker: 0.55, mouthFunnel: 0.26, browInnerUp: 0.05, tongueUp: 0 }, + u: { jawOpen: 0.2, mouthPucker: 0.55, mouthFunnel: 0.26, browInnerUp: 0.05, tongueUp: 0 }, + ю: { jawOpen: 0.2, mouthPucker: 0.5, mouthFunnel: 0.24, cheekSquintLeft: 0.06, cheekSquintRight: 0.06, tongueUp: 0.25 }, + w: { jawOpen: 0.18, mouthPucker: 0.58, mouthFunnel: 0.25, tongueUp: 0 }, + і: { jawOpen: 0.18, mouthWide: 0.5, mouthSmileLeft: 0.2, mouthSmileRight: 0.2, cheekSquintLeft: 0.16, cheekSquintRight: 0.16, mouthUpperUpLeft: 0.12, mouthUpperUpRight: 0.12, tongueUp: 0.2 }, + i: { jawOpen: 0.18, mouthWide: 0.5, mouthSmileLeft: 0.2, mouthSmileRight: 0.2, cheekSquintLeft: 0.16, cheekSquintRight: 0.16, mouthUpperUpLeft: 0.12, mouthUpperUpRight: 0.12, tongueUp: 0.2 }, + и: { jawOpen: 0.22, mouthWide: 0.42, mouthSmileLeft: 0.14, mouthSmileRight: 0.14, cheekSquintLeft: 0.12, cheekSquintRight: 0.12, mouthUpperUpLeft: 0.08, mouthUpperUpRight: 0.08, tongueUp: 0.15 }, + ї: { jawOpen: 0.2, mouthWide: 0.48, mouthSmileLeft: 0.18, mouthSmileRight: 0.18, cheekSquintLeft: 0.15, cheekSquintRight: 0.15, mouthUpperUpLeft: 0.12, mouthUpperUpRight: 0.12, tongueUp: 0.25 }, + е: { jawOpen: 0.3, mouthWide: 0.4, mouthSmileLeft: 0.1, mouthSmileRight: 0.1, cheekSquintLeft: 0.1, cheekSquintRight: 0.1, mouthUpperUpLeft: 0.08, mouthUpperUpRight: 0.08, browInnerUp: 0.06, tongueUp: 0.12 }, + e: { jawOpen: 0.3, mouthWide: 0.4, mouthSmileLeft: 0.1, mouthSmileRight: 0.1, cheekSquintLeft: 0.1, cheekSquintRight: 0.1, mouthUpperUpLeft: 0.08, mouthUpperUpRight: 0.08, browInnerUp: 0.06, tongueUp: 0.12 }, + є: { jawOpen: 0.28, mouthWide: 0.42, mouthSmileLeft: 0.14, mouthSmileRight: 0.14, cheekSquintLeft: 0.12, cheekSquintRight: 0.12, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, browInnerUp: 0.06, tongueUp: 0.18 }, + м: { mouthClose: 0.95, mouthShrugLower: 0.2, jawOpen: 0, tongueUp: 0 }, + m: { mouthClose: 0.95, mouthShrugLower: 0.2, jawOpen: 0, tongueUp: 0 }, + б: { mouthClose: 0.9, mouthShrugLower: 0.22, jawOpen: 0, tongueUp: 0 }, + b: { mouthClose: 0.9, mouthShrugLower: 0.22, jawOpen: 0, tongueUp: 0 }, + п: { mouthClose: 0.94, mouthShrugLower: 0.2, jawOpen: 0, tongueUp: 0 }, + p: { mouthClose: 0.94, mouthShrugLower: 0.2, jawOpen: 0, tongueUp: 0 }, + ф: { jawOpen: 0.1, mouthWide: 0.22, mouthClose: 0.3, mouthShrugLower: 0.42, mouthUpperUpLeft: 0.12, mouthUpperUpRight: 0.12, tongueUp: 0 }, + f: { jawOpen: 0.1, mouthWide: 0.22, mouthClose: 0.3, mouthShrugLower: 0.42, mouthUpperUpLeft: 0.12, mouthUpperUpRight: 0.12, tongueUp: 0 }, + в: { jawOpen: 0.1, mouthWide: 0.22, mouthClose: 0.28, mouthShrugLower: 0.38, mouthUpperUpLeft: 0.12, mouthUpperUpRight: 0.12, tongueUp: 0 }, + v: { jawOpen: 0.1, mouthWide: 0.22, mouthClose: 0.28, mouthShrugLower: 0.38, mouthUpperUpLeft: 0.12, mouthUpperUpRight: 0.12, tongueUp: 0 }, + л: { jawOpen: 0.2, mouthWide: 0.24, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, tongueUp: 0.88 }, + l: { jawOpen: 0.2, mouthWide: 0.24, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, tongueUp: 0.88 }, + н: { jawOpen: 0.18, mouthWide: 0.26, mouthUpperUpLeft: 0.08, mouthUpperUpRight: 0.08, tongueUp: 0.82 }, + n: { jawOpen: 0.18, mouthWide: 0.26, mouthUpperUpLeft: 0.08, mouthUpperUpRight: 0.08, tongueUp: 0.82 }, + д: { jawOpen: 0.18, mouthWide: 0.24, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, tongueUp: 0.85 }, + d: { jawOpen: 0.18, mouthWide: 0.24, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, tongueUp: 0.85 }, + т: { jawOpen: 0.18, mouthWide: 0.24, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, tongueUp: 0.85 }, + t: { jawOpen: 0.18, mouthWide: 0.24, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, tongueUp: 0.85 }, + с: { jawOpen: 0.12, mouthWide: 0.36, mouthUpperUpLeft: 0.14, mouthUpperUpRight: 0.14, tongueUp: 0.28 }, + s: { jawOpen: 0.12, mouthWide: 0.36, mouthUpperUpLeft: 0.14, mouthUpperUpRight: 0.14, tongueUp: 0.28 }, + з: { jawOpen: 0.14, mouthWide: 0.34, mouthUpperUpLeft: 0.12, mouthUpperUpRight: 0.12, tongueUp: 0.25 }, + z: { jawOpen: 0.14, mouthWide: 0.34, mouthUpperUpLeft: 0.12, mouthUpperUpRight: 0.12, tongueUp: 0.25 }, + ц: { jawOpen: 0.14, mouthWide: 0.36, mouthUpperUpLeft: 0.14, mouthUpperUpRight: 0.14, tongueUp: 0.35 }, + c: { jawOpen: 0.14, mouthWide: 0.36, mouthUpperUpLeft: 0.14, mouthUpperUpRight: 0.14, tongueUp: 0.35 }, + ш: { jawOpen: 0.22, mouthFunnel: 0.36, mouthPucker: 0.26, mouthUpperUpLeft: 0.08, mouthUpperUpRight: 0.08, tongueUp: 0.2 }, + ж: { jawOpen: 0.22, mouthFunnel: 0.36, mouthPucker: 0.24, mouthUpperUpLeft: 0.08, mouthUpperUpRight: 0.08, tongueUp: 0.2 }, + ч: { jawOpen: 0.24, mouthFunnel: 0.38, mouthPucker: 0.24, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, tongueUp: 0.4 }, + щ: { jawOpen: 0.24, mouthFunnel: 0.36, mouthPucker: 0.26, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, tongueUp: 0.3 }, + р: { jawOpen: 0.22, mouthWide: 0.22, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, tongueUp: 0.4 }, + r: { jawOpen: 0.22, mouthWide: 0.22, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1, tongueUp: 0.4 }, + к: { jawOpen: 0.26, mouthWide: 0.2, mouthUpperUpLeft: 0.06, mouthUpperUpRight: 0.06, tongueUp: 0 }, + k: { jawOpen: 0.26, mouthWide: 0.2, mouthUpperUpLeft: 0.06, mouthUpperUpRight: 0.06, tongueUp: 0 }, + г: { jawOpen: 0.26, mouthWide: 0.18, mouthUpperUpLeft: 0.06, mouthUpperUpRight: 0.06, tongueUp: 0 }, + g: { jawOpen: 0.26, mouthWide: 0.18, mouthUpperUpLeft: 0.06, mouthUpperUpRight: 0.06, tongueUp: 0 }, + ґ: { jawOpen: 0.26, mouthWide: 0.18, mouthUpperUpLeft: 0.06, mouthUpperUpRight: 0.06, tongueUp: 0 }, + х: { jawOpen: 0.28, mouthWide: 0.18, tongueUp: 0 }, + h: { jawOpen: 0.28, mouthWide: 0.18, tongueUp: 0 }, + ' ': { jawOpen: 0.02, mouthClose: 0, mouthWide: 0, mouthPucker: 0, mouthFunnel: 0, mouthShrugLower: 0, tongueUp: 0, browInnerUp: 0 }, +}; + +const SHAPES_LIST = [ + 'jawOpen', 'mouthWide', 'mouthSmileLeft', 'mouthSmileRight', 'mouthPucker', + 'mouthFunnel', 'mouthClose', 'eyeBlinkLeft', 'eyeBlinkRight', 'browInnerUp', + 'browDown', 'browOuterUp', 'eyeLookUp', 'eyeLookSide', 'tongueUp', + 'browDownLeft', 'browDownRight', 'browOuterUpLeft', 'browOuterUpRight', + 'browInnerUpLeft', 'browInnerUpRight', 'mouthFrownLeft', 'mouthFrownRight', + 'cheekSquintLeft', 'cheekSquintRight', 'eyeWideLeft', 'eyeWideRight', + 'mouthUpperUpLeft', 'mouthUpperUpRight', 'mouthShrugLower', +]; + +function parseTextToTimeline(text: string, speed = 0.95) { + const speedScale = 1.0 / Math.max(0.3, speed); + const timeline: Array<{ + time: number; + viseme: Record; + wordIdx: number; + char?: string; + isPause?: boolean; + isCue: boolean; + isWordStart?: boolean; + }> = []; + const charTimeline: Array<{ + char: string; + time: number; + isPrintable: boolean; + }> = []; + + let curTime = 0.05 * speedScale; + const tokenRegex = new RegExp('[\\p{L}\\p{N}]+|[^\\s\\p{L}\\p{N}]+|\\s+', 'gu'); + const tokens = text.match(tokenRegex) || [text]; + let wordIndex = 0; + const VOWELS = 'аоуіиеєяюaouiey'; + + tokens.forEach((tok) => { + if (/^\s+$/.test(tok)) { + curTime += 0.08 * speedScale; + timeline.push({ time: curTime, viseme: VISEME_MAP[' '], wordIdx: wordIndex, isPause: true, isCue: false }); + charTimeline.push({ char: ' ', time: curTime, isPrintable: false }); + } else if (/^[.,!?;:—-]+$/.test(tok)) { + const punctDur = (0.16 + tok.length * 0.06) * speedScale; + for (let i = 0; i < tok.length; i++) { + curTime += punctDur / tok.length; + charTimeline.push({ char: tok[i], time: curTime, isPrintable: true }); + } + timeline.push({ time: curTime, viseme: VISEME_MAP[' '], wordIdx: wordIndex, isPause: true, isCue: false }); + } else { + const lower = tok.toLowerCase(); + let hasVowelInWord = false; + for (let i = 0; i < lower.length; i++) { + const char = lower[i]; + const rawChar = tok[i]; + let dur = 0.075; + const isVowel = VOWELS.includes(char); + if (isVowel) { + dur = 0.12; + hasVowelInWord = true; + } else if ('мбпmbp'.includes(char)) { + dur = 0.085; + } else if ('лндтlndth'.includes(char)) { + dur = 0.09; + } + dur *= speedScale; + curTime += dur; + const v = VISEME_MAP[char] || VISEME_MAP[' ']; + const isCue = isVowel || (i === 0 && !hasVowelInWord); + + timeline.push({ + time: curTime - dur * 0.45, + viseme: v, + wordIdx: wordIndex, + char, + isCue, + isWordStart: i === 0, + }); + charTimeline.push({ + char: rawChar, + time: curTime - dur * 0.45, + isPrintable: true, + }); + } + wordIndex++; + curTime += 0.04 * speedScale; + } + }); + + curTime += 0.18 * speedScale; + timeline.push({ time: curTime, viseme: VISEME_MAP[' '], wordIdx: -1, isPause: true, isCue: false }); + return { timeline, charTimeline, totalDuration: curTime }; +} + +const MASCOT_EMOTIONS = [ + { id: 'Idle', uk: 'Нейтральна (Idle)', en: 'Neutral (Idle)' }, + { id: 'Smile', uk: 'Посмішка (Smile)', en: 'Smile' }, + { id: 'Thinking', uk: 'Роздуми (Thinking)', en: 'Thinking' }, + { id: 'Surprise', uk: 'Здивування (Surprise)', en: 'Surprise' }, + { id: 'Skeptical', uk: 'Скептична (Skeptical)', en: 'Skeptical' }, + { id: 'Shy', uk: 'Соромʼязлива (Shy)', en: 'Shy' }, + { id: 'Smirk', uk: 'Усмішка (Smirk)', en: 'Smirk' }, + { id: 'Laugh', uk: 'Сміх (Laugh)', en: 'Laugh' }, + { id: 'Wink', uk: 'Підморгування (Wink)', en: 'Wink' }, + { id: 'Sad', uk: 'Сумна (Sad)', en: 'Sad' }, + { id: 'Anger', uk: 'Сердита (Anger)', en: 'Anger' }, + { id: 'Disgust', uk: 'Огида (Disgust)', en: 'Disgust' }, + { id: 'Fear', uk: 'Переляк (Fear)', en: 'Fear' }, +]; + export const ChatRoom: React.FC = ({ interviewId, initialDirection = 'Frontend', @@ -22,6 +515,7 @@ export const ChatRoom: React.FC = ({ }) => { const router = useRouter(); const { t, lang } = useTranslation(); + const { interviewLayout, setInterviewLayout } = useAuthStore(); const sessionLanguage = initialLanguage || lang; const [messages, setMessages] = useState([]); @@ -31,7 +525,602 @@ export const ChatRoom: React.FC = ({ const [error] = useState(null); const scrollRef = useRef(null); - // Load existing messages or initialize first message + // Minimalist Mascot, Emotion & Sound state + const [modelReady, setModelReady] = useState(false); + const [isSpeaking, setIsSpeaking] = useState(false); + const [typedSubtitle, setTypedSubtitle] = useState(''); + const [activeEmotion, setActiveEmotion] = useState('Idle'); + const activeEmotionRef = useRef('Idle'); + const [activeSound, setActiveSound] = useState('text_print_soft.wav'); + const [soundVolume, setSoundVolume] = useState(0.7); + const [isMuted, setIsMuted] = useState(false); + const prevVolumeRef = useRef(0.7); + const [showDebugMenu, setShowDebugMenu] = useState(false); + + // Voice, Tempo, Rhythm & Animation Controls + const [speechSpeed, setSpeechSpeed] = useState(1.0); + const [soundCadence, setSoundCadence] = useState(0.18); + const [pitchVariation, setPitchVariation] = useState(0.06); + const [mouthAmplitude, setMouthAmplitude] = useState(1.0); + const [headMotionIntensity, setHeadMotionIntensity] = useState(1.0); + const [activeTexturePreset, setActiveTexturePreset] = useState('realistic'); + const activeTexturePresetRef = useRef('realistic'); + + const videoTileRef = useRef(null); + const canvasMountRef = useRef(null); + + const engineRef = useRef<{ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + model?: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + mixer?: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + actions: Record; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + currentAction: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + materialsByName: Record; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + THREE: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + texLoader: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + textureCache: Map; + speechActive: boolean; + speechStartTime: number; + speechDuration: number; + speechTone: 'statement' | 'question' | 'exclamation'; + speechTimeline: ReturnType['timeline']; + speechCharTimeline: ReturnType['charTimeline']; + currentSpeechVisemes: Record; + speechNodPitch: number; + speechRoll: number; + speechYaw: number; + lastTypingSoundTime: number; + lastRevealedCharIdx: number; + activeSound: string; + soundVolume: number; + soundCadence: number; + pitchVariation: number; + speechSpeed: number; + mouthAmplitude: number; + headMotionIntensity: number; + audioCtx: AudioContext | null; + audioBuffers: Record; + lastSpokenMsgCount: number; + }>({ + actions: {}, + currentAction: null, + materialsByName: {}, + THREE: null, + texLoader: null, + textureCache: new Map(), + speechActive: false, + speechStartTime: 0, + speechDuration: 0, + speechTone: 'statement', + speechTimeline: [], + speechCharTimeline: [], + currentSpeechVisemes: Object.fromEntries(SHAPES_LIST.map((s) => [s, 0])), + speechNodPitch: 0, + speechRoll: 0, + speechYaw: 0, + lastTypingSoundTime: 0, + lastRevealedCharIdx: -1, + activeSound: 'text_print_soft.wav', + soundVolume: 0.7, + soundCadence: 0.18, + pitchVariation: 0.06, + speechSpeed: 1.0, + mouthAmplitude: 1.0, + headMotionIntensity: 1.0, + audioCtx: null, + audioBuffers: {}, + lastSpokenMsgCount: 0, + }); + + useEffect(() => { + engineRef.current.activeSound = isMuted ? 'none' : activeSound; + engineRef.current.soundVolume = isMuted ? 0 : soundVolume; + engineRef.current.speechSpeed = speechSpeed; + engineRef.current.soundCadence = soundCadence; + engineRef.current.pitchVariation = pitchVariation; + engineRef.current.mouthAmplitude = mouthAmplitude; + engineRef.current.headMotionIntensity = headMotionIntensity; + }, [ + activeSound, + soundVolume, + isMuted, + speechSpeed, + soundCadence, + pitchVariation, + mouthAmplitude, + headMotionIntensity, + ]); + + const toggleMute = useCallback(() => { + setIsMuted((prev) => { + const next = !prev; + if (next) { + if (soundVolume > 0) prevVolumeRef.current = soundVolume; + } else if (soundVolume === 0) { + setSoundVolume(prevVolumeRef.current || 0.7); + } + return next; + }); + }, [soundVolume]); + + const getAudioCtx = useCallback(() => { + const eng = engineRef.current; + if (!eng.audioCtx && typeof window !== 'undefined') { + const Ctx = + window.AudioContext || + (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext; + if (Ctx) eng.audioCtx = new Ctx(); + } + if (eng.audioCtx && eng.audioCtx.state === 'suspended') { + eng.audioCtx.resume(); + } + return eng.audioCtx; + }, []); + + const playSound = useCallback( + async (filename: string, volumeScale = 1.0, pitchVariation = 0.0) => { + const eng = engineRef.current; + if (!filename || filename === 'none' || eng.soundVolume <= 0) return; + try { + const ctx = getAudioCtx(); + if (!ctx) return; + if (!eng.audioBuffers[filename]) { + const resp = await fetch(SOUNDS_BASE + filename); + if (!resp.ok) return; + const arr = await resp.arrayBuffer(); + eng.audioBuffers[filename] = await ctx.decodeAudioData(arr); + } + const buffer = eng.audioBuffers[filename]; + if (!buffer) return; + + const source = ctx.createBufferSource(); + source.buffer = buffer; + if (pitchVariation > 0) { + source.playbackRate.value = 1.0 + (Math.random() * 2 - 1) * pitchVariation; + } + const gain = ctx.createGain(); + gain.gain.value = Math.max(0, Math.min(1, eng.soundVolume * volumeScale)); + source.connect(gain); + gain.connect(ctx.destination); + source.start(0); + } catch { + // Ignore audio decode/autoplay errors + } + }, + [getAudioCtx] + ); + + const playClip = useCallback((name: string) => { + const eng = engineRef.current; + if (!eng.mixer || !eng.actions[name]) return; + const next = eng.actions[name]; + if (eng.currentAction && eng.currentAction !== next) { + eng.currentAction.fadeOut(0.24); + } + next.reset().fadeIn(0.24).play(); + eng.currentAction = next; + }, []); + + const handleEmotionChange = useCallback( + (emotion: string) => { + setActiveEmotion(emotion); + activeEmotionRef.current = emotion; + playClip(emotion); + }, + [playClip] + ); + + const applyTexturePreset = useCallback((presetId: string) => { + const eng = engineRef.current; + if (!eng.THREE || !eng.materialsByName || !eng.texLoader) return; + const preset = TEXTURE_PRESETS.find((p) => p.id === presetId); + if (!preset) return; + applyPresetToMaterials(eng.THREE, eng.materialsByName, preset, eng.textureCache, eng.texLoader); + }, []); + + const handleTextureChange = useCallback( + (presetId: string) => { + setActiveTexturePreset(presetId); + activeTexturePresetRef.current = presetId; + applyTexturePreset(presetId); + }, + [applyTexturePreset] + ); + + const speakTextWithMascot = useCallback( + (text: string) => { + const eng = engineRef.current; + getAudioCtx(); + + if (/\?\s*$/.test(text)) eng.speechTone = 'question'; + else if (/!\s*$/.test(text)) eng.speechTone = 'exclamation'; + else eng.speechTone = 'statement'; + + const currentSpeed = eng.speechSpeed || 1.0; + const { timeline, charTimeline, totalDuration } = parseTextToTimeline(text, currentSpeed); + eng.speechTimeline = timeline; + eng.speechCharTimeline = charTimeline; + eng.speechDuration = totalDuration; + eng.lastTypingSoundTime = 0; + eng.lastRevealedCharIdx = -1; + + const baseEmotion = activeEmotionRef.current || 'Idle'; + if (eng.actions[baseEmotion]) playClip(baseEmotion); + else if (eng.actions['Idle']) playClip('Idle'); + + setTypedSubtitle(''); + setIsSpeaking(true); + + eng.speechStartTime = performance.now() / 1000.0; + eng.speechActive = true; + }, + [getAudioCtx, playClip] + ); + + // Initialize Three.js Scene & Load v12 Bust_animated.glb + useEffect(() => { + let disposed = false; + let animFrameId = 0; + let resizeObserver: ResizeObserver | null = null; + + async function initThree() { + const mount = canvasMountRef.current; + const tile = videoTileRef.current; + if (!mount || !tile) return; + + const dynamicImport = new Function('u', 'return import(u)'); + const [THREE, { GLTFLoader }, { OrbitControls }] = await Promise.all([ + dynamicImport('https://esm.sh/three@0.160.0'), + dynamicImport('https://esm.sh/three@0.160.0/examples/jsm/loaders/GLTFLoader.js'), + dynamicImport('https://esm.sh/three@0.160.0/examples/jsm/controls/OrbitControls.js'), + ]); + if (disposed) return; + + const eng = engineRef.current; + + const scene = new THREE.Scene(); + scene.background = new THREE.Color(0x0b0b0f); + + const rect = mount.getBoundingClientRect(); + const initW = Math.max(Math.floor(rect.width), 360); + const initH = Math.max(Math.floor(rect.height), 320); + + // Framed close enough so the bottom of the bust is always off-screen and never floating in mid-air + const BASE_VFOV = 38; + const computeAdaptiveFov = (aspect: number) => { + if (aspect < 0.75) { + const rad = (BASE_VFOV * Math.PI) / 180; + const adapted = 2 * Math.atan(Math.tan(rad / 2) * (0.75 / Math.max(0.42, aspect))); + return Math.min(44, (adapted * 180) / Math.PI); + } + return BASE_VFOV; + }; + // Distance clamped so the bottom cut of the bust never lifts into view + const computeCameraZ = (aspect: number) => (aspect < 0.75 ? 4.65 : 4.45); + + const camera = new THREE.PerspectiveCamera( + computeAdaptiveFov(initW / initH), + initW / initH, + 0.1, + 50 + ); + // Camera y is 1.16, Target y is 1.18: positions the mascot higher in frame to eliminate empty top void while keeping bust bottom submerged + camera.position.set(0, 1.16, computeCameraZ(initW / initH)); + + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2)); + renderer.setSize(initW, initH, true); + renderer.shadowMap.enabled = true; + renderer.shadowMap.type = THREE.PCFSoftShadowMap; + renderer.toneMapping = THREE.ACESFilmicToneMapping; + renderer.toneMappingExposure = 1.06; + mount.innerHTML = ''; + mount.appendChild(renderer.domElement); + + // Orbit controls: allows smooth rotation without mouse-wheel zoom detachment + const controls = new OrbitControls(camera, renderer.domElement); + controls.enableDamping = true; + controls.dampingFactor = 0.08; + controls.enableZoom = false; + controls.minDistance = 4.2; + controls.maxDistance = 4.7; + controls.enablePan = false; + controls.enableRotate = true; + controls.rotateSpeed = 0.85; + controls.minPolarAngle = Math.PI * 0.44; + controls.maxPolarAngle = Math.PI * 0.50; // prevents pitching camera so low that the underside shows + controls.minAzimuthAngle = -0.45; + controls.maxAzimuthAngle = 0.45; + // Target positioned at y=1.18, elevating the mascot head to reduce top gap + controls.target.set(0, 1.18, 0.45); + controls.update(); + + eng.THREE = THREE; + eng.texLoader = new THREE.TextureLoader(); + eng.textureCache = new Map(); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (window as any).__mascotDebug = { camera, controls, scene, renderer, playClip, applyTexturePreset }; + + let lastW = initW; + let lastH = initH; + const syncSize = () => { + if (!mount) return; + const w = mount.clientWidth; + const h = mount.clientHeight; + if (w > 10 && h > 10 && (w !== lastW || h !== lastH)) { + lastW = w; + lastH = h; + const aspect = w / h; + camera.aspect = aspect; + camera.fov = computeAdaptiveFov(aspect); + const targetZ = computeCameraZ(aspect); + if (Math.abs(camera.position.z - targetZ) > 0.05) { + camera.position.z = targetZ; + } + camera.updateProjectionMatrix(); + renderer.setSize(w, h, true); + } + }; + resizeObserver = new ResizeObserver(syncSize); + resizeObserver.observe(mount); + + // Studio Three-Point Lighting + scene.add(new THREE.AmbientLight(0xffffff, 0.28)); + const key = new THREE.DirectionalLight(0xffffff, 1.65); + key.position.set(3.5, -5, 5); + key.castShadow = true; + key.shadow.mapSize.set(2048, 2048); + key.shadow.camera.left = -4; + key.shadow.camera.right = 4; + key.shadow.camera.top = 4; + key.shadow.camera.bottom = -4; + key.shadow.camera.near = 0.1; + key.shadow.camera.far = 20; + key.shadow.bias = -0.0008; + scene.add(key); + + const rim = new THREE.DirectionalLight(0x60a5fa, 0.48); + rim.position.set(-4, 2, 2); + scene.add(rim); + + const gltfLoader = new GLTFLoader(); + gltfLoader.load( + MASCOT_BASE + 'Bust_animated.glb', + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (gltf: any) => { + if (disposed) return; + const model = gltf.scene; + scene.add(model); + eng.model = model; + eng.mixer = new THREE.AnimationMixer(model); + eng.actions = {}; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + gltf.animations.forEach((clip: any) => { + eng.actions[clip.name] = eng.mixer.clipAction(clip); + }); + + eng.materialsByName = {}; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + model.traverse((child: any) => { + if (child.isMesh) { + child.castShadow = true; + child.receiveShadow = true; + if (Array.isArray(child.material)) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + child.material.forEach((mat: any) => { + eng.materialsByName[mat.name] = mat; + }); + } else if (child.material) { + eng.materialsByName[child.material.name] = child.material; + } + } + }); + + // Apply initial texture preset (e.g. realistic) + const initPreset = + TEXTURE_PRESETS.find((p) => p.id === activeTexturePresetRef.current) || + TEXTURE_PRESETS[0]; + applyPresetToMaterials( + THREE, + eng.materialsByName, + initPreset, + eng.textureCache, + eng.texLoader + ); + + playClip(activeEmotionRef.current || 'Idle'); + setModelReady(true); + } + ); + + ['text_print_soft.wav', 'typewriter_key.wav', 'dialogue_blip_soft.wav', 'dialogue_blip_synth.wav', 'tech_ui_chirp.wav', 'whisper_murmur.wav', 'chime_done.wav'].forEach((s) => { + fetch(SOUNDS_BASE + s).catch(() => {}); + }); + + const clock = new THREE.Clock(); + const animate = () => { + if (disposed) return; + animFrameId = requestAnimationFrame(animate); + syncSize(); + const delta = clock.getDelta(); + if (eng.mixer) eng.mixer.update(delta); + + if (eng.model) { + const now = performance.now() / 1000.0; + if (eng.speechActive) { + const elapsed = now - eng.speechStartTime; + if (elapsed > eng.speechDuration) { + eng.speechActive = false; + setIsSpeaking(false); + if (eng.speechCharTimeline.length > 0) { + setTypedSubtitle(eng.speechCharTimeline.map((c) => c.char).join('')); + } + playSound('chime_done.wav', 0.5); + const resumeEmotion = activeEmotionRef.current || 'Idle'; + if (eng.actions[resumeEmotion]) playClip(resumeEmotion); + } else { + let revealedCount = 0; + let newLetters = 0; + for (let c = eng.lastRevealedCharIdx + 1; c < eng.speechCharTimeline.length; c++) { + const item = eng.speechCharTimeline[c]; + if (item.time <= elapsed) { + eng.lastRevealedCharIdx = c; + revealedCount++; + if (item.isPrintable && item.char !== ' ') newLetters++; + } else break; + } + if (revealedCount > 0) { + setTypedSubtitle( + eng.speechCharTimeline + .slice(0, eng.lastRevealedCharIdx + 1) + .map((c) => c.char) + .join('') + ); + } + if (newLetters > 0 && eng.activeSound !== 'none') { + if (now - eng.lastTypingSoundTime >= (eng.soundCadence || 0.18) * 0.45) { + playSound(eng.activeSound, 0.44, eng.pitchVariation ?? 0.06); + eng.lastTypingSoundTime = now; + } + } + + // Interpolate current viseme + let targetViseme: Record = { ...VISEME_MAP[' '] }; + if (eng.speechTimeline.length > 0) { + if (elapsed <= eng.speechTimeline[0].time) { + targetViseme = { ...eng.speechTimeline[0].viseme }; + } else if (elapsed < eng.speechTimeline[eng.speechTimeline.length - 1].time) { + for (let i = 0; i < eng.speechTimeline.length - 1; i++) { + if (elapsed >= eng.speechTimeline[i].time && elapsed <= eng.speechTimeline[i + 1].time) { + const segT = + (elapsed - eng.speechTimeline[i].time) / + Math.max(0.001, eng.speechTimeline[i + 1].time - eng.speechTimeline[i].time); + const smoothT = segT * segT * (3.0 - 2.0 * segT); + const vA = eng.speechTimeline[i].viseme; + const vB = eng.speechTimeline[i + 1].viseme; + targetViseme = {}; + SHAPES_LIST.forEach((s) => { + targetViseme[s] = (vA[s] || 0) * (1 - smoothT) + (vB[s] || 0) * smoothT; + }); + break; + } + } + } + } + + // Prosody modulation (? and !) + const speechProgress = Math.min(1.0, elapsed / Math.max(0.4, eng.speechDuration)); + let qArch = 0.0; + if (eng.speechTone === 'question' && speechProgress > 0.35) { + qArch = Math.sin(Math.PI * Math.min(1.0, (speechProgress - 0.35) / 0.65)); + targetViseme['browOuterUpRight'] = (targetViseme['browOuterUpRight'] || 0) + 0.42 * qArch; + targetViseme['browInnerUpLeft'] = (targetViseme['browInnerUpLeft'] || 0) + 0.24 * qArch; + targetViseme['mouthSmileRight'] = (targetViseme['mouthSmileRight'] || 0) + 0.12 * qArch; + } else if (eng.speechTone === 'exclamation') { + const exPulse = 0.5 * (1.0 + Math.sin(elapsed * 4.0)); + targetViseme['browInnerUp'] = (targetViseme['browInnerUp'] || 0) + 0.28; + targetViseme['eyeWideLeft'] = 0.22 * exPulse; + targetViseme['eyeWideRight'] = 0.22 * exPulse; + targetViseme['jawOpen'] = (targetViseme['jawOpen'] || 0) * 1.15; + } + + targetViseme['cheekSquintLeft'] = (targetViseme['cheekSquintLeft'] || 0) + 0.08; + targetViseme['cheekSquintRight'] = (targetViseme['cheekSquintRight'] || 0) + 0.08; + targetViseme['mouthSmileLeft'] = (targetViseme['mouthSmileLeft'] || 0) + 0.04; + targetViseme['mouthSmileRight'] = (targetViseme['mouthSmileRight'] || 0) + 0.04; + + const lerpFactor = 1.0 - Math.exp(-26.0 * delta); + SHAPES_LIST.forEach((s) => { + eng.currentSpeechVisemes[s] += ((targetViseme[s] || 0) - eng.currentSpeechVisemes[s]) * lerpFactor; + }); + + const isMouthOpen = (eng.currentSpeechVisemes['jawOpen'] || 0) > 0.1; + const nodBase = 0.015 * Math.sin(elapsed * 4.8) * (isMouthOpen ? 1.0 : 0.45); + eng.speechNodPitch += (nodBase - eng.speechNodPitch) * lerpFactor; + const targetRoll = (eng.speechTone === 'question' ? -0.024 * qArch : 0) + 0.01 * Math.sin(elapsed * 1.7); + const targetYaw = (eng.speechTone === 'question' ? 0.018 * qArch : 0) + 0.008 * Math.cos(elapsed * 1.3); + eng.speechRoll += (targetRoll - eng.speechRoll) * lerpFactor; + eng.speechYaw += (targetYaw - eng.speechYaw) * lerpFactor; + } + } else { + const fadeFactor = 1.0 - Math.exp(-14.0 * delta); + SHAPES_LIST.forEach((s) => { + eng.currentSpeechVisemes[s] += (0 - eng.currentSpeechVisemes[s]) * fadeFactor; + }); + eng.speechNodPitch += (0 - eng.speechNodPitch) * fadeFactor; + eng.speechRoll += (0 - eng.speechRoll) * fadeFactor; + eng.speechYaw += (0 - eng.speechYaw) * fadeFactor; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + eng.model.traverse((child: any) => { + if (child.isMesh && child.morphTargetDictionary && child.morphTargetInfluences) { + const closeWeight = eng.currentSpeechVisemes['mouthClose'] || 0; + SHAPES_LIST.forEach((s) => { + const idx = child.morphTargetDictionary[s]; + if (idx !== undefined) { + let val = eng.currentSpeechVisemes[s]; + if (s === 'jawOpen') { + val = val * (1.0 - 0.85 * closeWeight); + if (closeWeight > 0.1) { + child.morphTargetInfluences[idx] *= 1.0 - 0.8 * closeWeight; + } + } + if (val > 0.001) { + const scaled = Math.min(1.0, val * (eng.mouthAmplitude ?? 1.0)); + child.morphTargetInfluences[idx] = Math.max(child.morphTargetInfluences[idx], scaled); + } + } + }); + } + }); + + const motionScale = eng.headMotionIntensity ?? 1.0; + eng.model.rotation.set( + eng.speechNodPitch * motionScale, + eng.speechYaw * motionScale, + eng.speechRoll * motionScale + ); + } + + controls.update(); + renderer.render(scene, camera); + }; + + animate(); + } + + initThree(); + + return () => { + disposed = true; + cancelAnimationFrame(animFrameId); + if (resizeObserver) resizeObserver.disconnect(); + }; + }, [playClip, playSound]); + + // Trigger mascot speech whenever a new assistant message arrives and model is ready + useEffect(() => { + if (!modelReady || messages.length === 0) return; + const eng = engineRef.current; + if (messages.length !== eng.lastSpokenMsgCount) { + eng.lastSpokenMsgCount = messages.length; + const lastMsg = messages[messages.length - 1]; + if (lastMsg.role === 'assistant') { + speakTextWithMascot(lastMsg.content); + } + } + }, [messages, modelReady, speakTextWithMascot]); + + // Load existing messages or initialize first message (untouched backend contract) useEffect(() => { let isMounted = true; @@ -75,12 +1164,13 @@ export const ChatRoom: React.FC = ({ if (scrollRef.current) { scrollRef.current.scrollTop = scrollRef.current.scrollHeight; } - }, [messages, loading]); + }, [messages, loading, typedSubtitle]); const handleSend = async (e?: React.FormEvent) => { if (e) e.preventDefault(); if (!input.trim() || loading || completing) return; + getAudioCtx(); const userText = input.trim(); setInput(''); @@ -90,6 +1180,10 @@ export const ChatRoom: React.FC = ({ timestamp: new Date().toISOString(), }; + engineRef.current.speechActive = false; + setIsSpeaking(false); + playClip('Thinking'); + setMessages((prev) => [...prev, userMessage]); setLoading(true); @@ -163,15 +1257,44 @@ export const ChatRoom: React.FC = ({ } }; + const handleTestSpeech = useCallback(() => { + const phrasesUk = [ + 'Вітаю! Я уважно слухаю вашу відповідь на технічній співбесіді.', + 'Чудово! Як саме ви оптимізуєте роботу з компонентами?', + 'Дякую за ґрунтовну відповідь, переходимо до наступного запитання!', + ]; + const phrasesEn = [ + 'Welcome! I am attentively listening to your interview answer.', + 'Great! How do you optimize component rendering in production?', + 'Thank you for the detailed reasoning, let us proceed to the next question!', + ]; + const pool = sessionLanguage === 'uk' ? phrasesUk : phrasesEn; + const sample = pool[Math.floor(Math.random() * pool.length)]; + speakTextWithMascot(sample); + }, [sessionLanguage, speakTextWithMascot]); + + const handleResetSettings = useCallback(() => { + setSpeechSpeed(1.0); + setSoundCadence(0.18); + setPitchVariation(0.06); + setMouthAmplitude(1.0); + setHeadMotionIntensity(1.0); + setSoundVolume(0.7); + setActiveEmotion('Idle'); + handleEmotionChange('Idle'); + handleTextureChange('realistic'); + }, [handleEmotionChange, handleTextureChange]); + const userAnswersCount = messages.filter((m) => m.role === 'user').length; + const lastAssistantIdx = messages.map((m) => m.role).lastIndexOf('assistant'); return ( -
- {/* Top Session Info Bar */} -
+
+ {/* Top Session Minimalist Bar */} +
-
- +
+
@@ -192,109 +1315,665 @@ export const ChatRoom: React.FC = ({
- {/* Complete Session Action Button */} - + + {/* Clean Layout Switcher Pill */} +
+ + + +
+ + {/* Minimalist Debug Menu Toggle Button */} + + + {/* Collapsible Debug Popover Panel */} + {showDebugMenu && ( +
+ {/* Header */} +
+
+ + Керування мовленням та маскотом +
+ +
+ + {/* Action Buttons: Test Voice & Reset Defaults */} +
+ + +
+ + {/* SECTION 1: Темп, ритм та динаміка мовлення */} +
+
+ + Темп і ритм мовлення +
+ + {/* Speech Speed / Темп говоріння */} +
+
+ Темп мовлення (Speed): + {speechSpeed.toFixed(2)}x +
+ setSpeechSpeed(parseFloat(e.target.value))} + className="w-full accent-blue-500 cursor-pointer" + /> +
+ 0.5x (Повільно) + 1.0x (Норма) + 2.0x (Швидко) +
+
+ + {/* Sound Cadence / Ритм клацань друку */} +
+
+ Ритм звуку друку (Cadence): + {soundCadence.toFixed(2)}s +
+ setSoundCadence(parseFloat(e.target.value))} + className="w-full accent-blue-500 cursor-pointer" + /> +
+ Частий (0.06s) + Збалансований + Рідкісний (0.30s) +
+
+ + {/* Pitch Jitter / Варіація висоти тону */} +
+
+ Варіація висоти тону (Pitch Jitter): + {Math.round(pitchVariation * 100)}% +
+ setPitchVariation(parseFloat(e.target.value))} + className="w-full accent-blue-500 cursor-pointer" + /> +
+
+ + {/* SECTION 2: Звукові ефекти друку */} +
+
+ + Звуковий супровід +
+ + {/* Sound Preset Select */} +
+ + +
+ + {/* Sound Volume */} +
+
+ Гучність звуку: + {isMuted || activeSound === 'none' ? '0%' : `${Math.round(soundVolume * 100)}%`} +
+ { + const v = parseFloat(e.target.value); + setSoundVolume(v); + if (v > 0) { + prevVolumeRef.current = v; + setIsMuted(false); + } else { + setIsMuted(true); + } + }} + className="w-full accent-rose-500 cursor-pointer" + /> +
+
+ + {/* SECTION 3: Міміка та Анімація */} +
+
+ + Міміка та рухи маскота +
+ + {/* Emotion Selector */} +
+ + +
+ + {/* Mouth Viseme Amplitude / Амплітуда артикуляції */} +
+
+ Амплітуда міміки рота (Articul.): + {mouthAmplitude.toFixed(2)}x +
+ setMouthAmplitude(parseFloat(e.target.value))} + className="w-full accent-amber-500 cursor-pointer" + /> +
+ 0.5x (Стримано) + 1.0x (Норма) + 1.5x (Експресивно) +
+
+ + {/* Head Motion Intensity / Динаміка кивків */} +
+
+ Динаміка рухів голови (Head Motion): + {headMotionIntensity.toFixed(1)}x +
+ setHeadMotionIntensity(parseFloat(e.target.value))} + className="w-full accent-amber-500 cursor-pointer" + /> +
+ 0.0x (Нерухомо) + 1.0x (Природно) + 2.0x (Активно) +
+
+
+ + {/* SECTION 4: Текстури та стилі маскота (Version 12) */} +
+
+ + Текстура маскота (Version 12) +
+ +
+ {TEXTURE_PRESETS.map((preset) => { + const isActive = activeTexturePreset === preset.id; + return ( + + ); + })} +
+ + {/* Selected Texture Description */} + {(() => { + const currentPreset = TEXTURE_PRESETS.find((p) => p.id === activeTexturePreset); + if (!currentPreset) return null; + return ( +
+ + {currentPreset.icon} {sessionLanguage === 'uk' ? currentPreset.nameUk : currentPreset.nameEn}: + {' '} + {sessionLanguage === 'uk' ? currentPreset.descUk : currentPreset.descEn} +
+ ); + })()} +
+
)} - + + {/* Finish Interview Button */} + +
{error && ( -
+
{error}
)} - {/* Messages Scroll Area */} + {/* Interview Workspace Container */}
- {messages.map((msg, index) => { - const isAssistant = msg.role === 'assistant'; - return ( + {/* Minimalistic Mascot Video Window */} +
+
+ {/* 3D WebGL Canvas Mount */}
- {isAssistant && ( -
- + ref={canvasMountRef} + className="absolute inset-0 w-full h-full overflow-hidden cursor-grab active:cursor-grabbing" + /> + + {/* Subtle soft gradient fade at the very bottom edge so the bust base seamlessly merges into the container */} +
+ + {!modelReady && ( +
+ + Підключення відео... +
+ )} + + {/* Question Bubble inside bottom of Video Window (in 'center' and 'split' layouts) */} + {interviewLayout !== 'sections' && ( +
+
+ {loading ? ( +
+ + {t.chat.aiThinking} +
+ ) : ( +

+ {typedSubtitle || + (lastAssistantIdx >= 0 ? messages[lastAssistantIdx]?.content : '')} + {isSpeaking && ( + + )} +

+ )}
- )} +
+ )} +
+
-
-
{msg.content}
-
- {new Date(msg.timestamp).toLocaleTimeString([], { - hour: '2-digit', - minute: '2-digit', - })} + {/* --- FORMAT 1: 'center' (Answer Form below) --- */} + {interviewLayout === 'center' && ( +
+