From 7b9418e8cbecc14c0c7b9882a9ce2675135021c5 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Sun, 20 Sep 2026 17:42:30 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix(asr):=20=E5=85=81=E8=AE=B8=E6=8C=89?= =?UTF-8?q?=E6=B8=A0=E9=81=93=E6=89=8B=E5=8A=A8=E9=80=89=E6=8B=A9=E7=99=BE?= =?UTF-8?q?=E7=82=BC=E6=A8=A1=E5=9E=8B=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/asr/dashscope_multimodal.rs | 71 ++++++++- .../openless-core/src/cloud_providers.rs | 93 +++++++++-- .../openless-core/src/provider_rules.rs | 125 ++++++++++++++- .../openless-core/src/provider_service.rs | 110 ++++++++++++- .../app/scripts/bailian-protocol-h5.mjs | 144 ++++++++++++++++++ openless-all/app/src/i18n/de.ts | 10 ++ openless-all/app/src/i18n/en.ts | 10 ++ openless-all/app/src/i18n/es.ts | 10 ++ openless-all/app/src/i18n/fr.ts | 10 ++ openless-all/app/src/i18n/ja.ts | 10 ++ openless-all/app/src/i18n/ko.ts | 10 ++ openless-all/app/src/i18n/zh-CN.ts | 10 ++ openless-all/app/src/i18n/zh-TW.ts | 10 ++ .../app/src/lib/bailianProtocol.test.ts | 33 ++++ openless-all/app/src/lib/bailianProtocol.ts | 30 ++++ .../pages/settings/BailianProtocolField.tsx | 124 +++++++++++++++ .../src/pages/settings/ProvidersSection.tsx | 40 ++--- 17 files changed, 797 insertions(+), 53 deletions(-) create mode 100644 openless-all/app/scripts/bailian-protocol-h5.mjs create mode 100644 openless-all/app/src/lib/bailianProtocol.test.ts create mode 100644 openless-all/app/src/lib/bailianProtocol.ts create mode 100644 openless-all/app/src/pages/settings/BailianProtocolField.tsx diff --git a/openless-all/app/crates/openless-core/src/asr/dashscope_multimodal.rs b/openless-all/app/crates/openless-core/src/asr/dashscope_multimodal.rs index 61b85fa53..0c00a75a4 100644 --- a/openless-all/app/crates/openless-core/src/asr/dashscope_multimodal.rs +++ b/openless-all/app/crates/openless-core/src/asr/dashscope_multimodal.rs @@ -47,6 +47,7 @@ pub struct DashScopeMultimodalASR { api_key: String, base_url: String, model: String, + protocol: crate::provider_rules::BailianProtocol, buffer: Mutex>, } @@ -56,16 +57,24 @@ impl DashScopeMultimodalASR { api_key, base_url, model, + protocol: crate::provider_rules::BailianProtocol::Auto, buffer: Mutex::new(Vec::new()), } } + pub fn with_protocol(mut self, protocol: crate::provider_rules::BailianProtocol) -> Self { + self.protocol = protocol; + self + } + pub fn buffer_duration_ms(&self) -> u64 { crate::asr::pcm::pcm_duration_ms(&self.buffer.lock()) } pub fn transcribe_timeout(&self, audio_secs: f64) -> Duration { - if protocol_for_model(&self.model) == Some(DashScopeBatchProtocol::AsyncTranscription) { + if self.protocol.batch_protocol(&self.model) + == Some(DashScopeBatchProtocol::AsyncTranscription) + { let pcm_bytes = (audio_secs.max(0.0) * 32_000.0).ceil() as u64; return async_upload_timeout(pcm_bytes.saturating_add(44)) + Duration::from_secs(ASYNC_TASK_POLL_TIMEOUT_SECS + ASYNC_WORKFLOW_OVERHEAD_SECS); @@ -98,7 +107,9 @@ impl DashScopeMultimodalASR { } let duration_ms = crate::asr::pcm::pcm_duration_ms(pcm); - if protocol_for_model(&self.model) == Some(DashScopeBatchProtocol::AsyncTranscription) { + if self.protocol.batch_protocol(&self.model) + == Some(DashScopeBatchProtocol::AsyncTranscription) + { let samples: Vec = pcm .as_chunks::<2>() .0 @@ -129,7 +140,11 @@ impl DashScopeMultimodalASR { .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]])) .collect(); let wav = encode_wav_16k_mono(&samples); - let body = dashscope_multimodal_body(&self.model, &wav); + let audio_data = format!( + "data:audio/wav;base64,{}", + base64::engine::general_purpose::STANDARD.encode(&wav) + ); + let body = dashscope_multimodal_body_with_protocol(&self.model, &audio_data, self.protocol); let url = generation_url(&self.base_url)?; let request_timeout = self.transcribe_timeout(crate::asr::pcm::pcm_duration_ms(pcm) as f64 / 1000.0); @@ -471,7 +486,19 @@ pub fn dashscope_multimodal_body(model: &str, wav: &[u8]) -> Value { } pub fn dashscope_multimodal_body_from_uri(model: &str, audio_uri: &str) -> Value { - if crate::provider_rules::dashscope_uses_qwen_sync_envelope(model) { + dashscope_multimodal_body_with_protocol( + model, + audio_uri, + crate::provider_rules::BailianProtocol::Auto, + ) +} + +pub fn dashscope_multimodal_body_with_protocol( + model: &str, + audio_uri: &str, + protocol: crate::provider_rules::BailianProtocol, +) -> Value { + if protocol.uses_qwen_envelope(model) { return serde_json::json!({ "model": model, "input": { @@ -623,6 +650,42 @@ pub fn extract_dashscope_text(json: &Value) -> String { #[cfg(test)] mod tests { + #[test] + fn manual_protocol_controls_envelope_and_timeout_without_model_inference() { + use super::*; + use crate::provider_rules::BailianProtocol; + let uri = "data:audio/wav;base64,AAAA"; + let standard = dashscope_multimodal_body_with_protocol( + "qwen3-asr-flash", + uri, + BailianProtocol::Multimodal, + ); + assert_eq!( + standard["input"]["messages"][0]["content"][0]["input_audio"]["data"], + uri + ); + let qwen = dashscope_multimodal_body_with_protocol( + "unknown-model", + uri, + BailianProtocol::QwenMultimodal, + ); + assert_eq!(qwen["input"]["messages"][0]["content"][0]["audio"], uri); + assert!(qwen.get("parameters").is_none()); + let asynchronous = DashScopeMultimodalASR::new( + "key".into(), + "https://example.com".into(), + "unknown-model".into(), + ) + .with_protocol(BailianProtocol::AsyncTranscription); + assert!(asynchronous.transcribe_timeout(1.0) > Duration::from_secs(600)); + let synchronous = DashScopeMultimodalASR::new( + "key".into(), + "https://example.com".into(), + "fun-asr".into(), + ) + .with_protocol(BailianProtocol::Multimodal); + assert_eq!(synchronous.transcribe_timeout(1.0), Duration::from_secs(30)); + } use super::*; use crate::asr::AudioConsumer; use std::io::{Read, Write}; diff --git a/openless-all/app/crates/openless-core/src/cloud_providers.rs b/openless-all/app/crates/openless-core/src/cloud_providers.rs index 1f29141b7..b8ed0c607 100644 --- a/openless-all/app/crates/openless-core/src/cloud_providers.rs +++ b/openless-all/app/crates/openless-core/src/cloud_providers.rs @@ -138,6 +138,7 @@ struct CloudTranscriptionPreparation { context: Arc, provider_type: String, effective_provider: String, + bailian_protocol: crate::provider_rules::BailianProtocol, kind: crate::provider_rules::ActiveAsrProviderKind, model: String, api_key: String, @@ -401,9 +402,14 @@ async fn prepare_cloud_transcription( .clone() .or_else(|| value(ASR_MODEL_ACCOUNT)) .unwrap_or_default(); - let effective_provider = - crate::provider_rules::resolve_effective_asr_provider(&provider_type, &model) - .map_err(|message| BackendError::new(BackendErrorCode::InvalidArgument, message))?; + let bailian_protocol = crate::provider_rules::BailianProtocol::from_config( + &provider_type, + value(ASR_ADVANCED_CONFIG_ACCOUNT).as_deref(), + ) + .map_err(|message| BackendError::new(BackendErrorCode::InvalidArgument, message))?; + let effective_provider = bailian_protocol + .resolve_provider(&provider_type, &model) + .map_err(|message| BackendError::new(BackendErrorCode::InvalidArgument, message))?; let advanced_config = crate::provider_rules::advanced_asr_config_for( &provider_type, value(ASR_ADVANCED_CONFIG_ACCOUNT).as_deref(), @@ -413,6 +419,7 @@ async fn prepare_cloud_transcription( provider_type, kind: crate::provider_rules::active_asr_provider_kind(&effective_provider), effective_provider, + bailian_protocol, api_key: value(ASR_API_KEY_ACCOUNT).unwrap_or_default(), endpoint: value(ASR_ENDPOINT_ACCOUNT).unwrap_or_default(), advanced_config, @@ -622,24 +629,22 @@ async fn build_cloud_transcription_session( let stored_endpoint = non_blank_owned(endpoint) .unwrap_or_else(|| crate::asr::dashscope_multimodal::DEFAULT_ENDPOINT.to_string()); let endpoint = if provider_type == crate::asr::bailian::PROVIDER_ID { - let protocol = - match crate::provider_rules::dashscope_batch_protocol_for_model(&model) { - Some(crate::provider_rules::DashScopeBatchProtocol::AsyncTranscription) => { - BailianEndpointProtocol::AsyncTranscription - } - _ => BailianEndpointProtocol::Multimodal, - }; + let protocol = match preparation.bailian_protocol.batch_protocol(&model) { + Some(crate::provider_rules::DashScopeBatchProtocol::AsyncTranscription) => { + BailianEndpointProtocol::AsyncTranscription + } + _ => BailianEndpointProtocol::Multimodal, + }; crate::provider_rules::derive_bailian_endpoint(&stored_endpoint, protocol) .unwrap_or(stored_endpoint) } else { stored_endpoint }; ( - CloudTranscriptionSessionKind::DashScope(Arc::new(DashScopeMultimodalASR::new( - api_key, - endpoint, - model.clone(), - ))), + CloudTranscriptionSessionKind::DashScope(Arc::new( + DashScopeMultimodalASR::new(api_key, endpoint, model.clone()) + .with_protocol(preparation.bailian_protocol), + )), Some(model), ) } @@ -2129,6 +2134,64 @@ fn build_omni_prompt(context: &DictationContext) -> String { #[cfg(test)] mod tests { + #[tokio::test] + async fn manual_bailian_protocol_is_snapshotted_for_recording() { + use super::*; + use crate::provider_rules::BailianProtocol; + let credentials = crate::InMemoryCredentialStore::default(); + for (name, expected) in [ + ("dashscope-realtime", "bailian"), + ("qwen-realtime", "bailian-qwen3-realtime"), + ("multimodal", "bailian-fun-asr-flash"), + ("qwen-multimodal", "bailian-fun-asr-flash"), + ("async-transcription", "bailian-fun-asr-flash"), + ] { + let raw = format!(r#"{{"bailianProtocol":"{name}"}}"#); + for (account, value) in [ + (ASR_API_KEY_ACCOUNT, "fixture-key"), + (ASR_ADVANCED_CONFIG_ACCOUNT, raw.as_str()), + ] { + credentials + .write( + CredentialKey::new( + CredentialNamespace::Asr, + Some(name.to_string()), + account, + ) + .unwrap(), + crate::SecretValue::new(value), + ) + .await + .unwrap(); + } + let mut context = DictationContext::default(); + context.asr.provider_id = name.to_string(); + context.asr.provider_type = "bailian".into(); + context.asr.model = Some("unknown-model".into()); + let prepared = prepare_cloud_transcription(&credentials, Arc::new(context)) + .await + .unwrap(); + assert_eq!(prepared.effective_provider, expected); + assert_eq!(prepared.model, "unknown-model"); + // A later settings change must not mutate an already prepared recording. + credentials + .write( + CredentialKey::new( + CredentialNamespace::Asr, + Some(name.to_string()), + ASR_ADVANCED_CONFIG_ACCOUNT, + ) + .unwrap(), + crate::SecretValue::new("{}"), + ) + .await + .unwrap(); + assert_eq!( + prepared.bailian_protocol, + BailianProtocol::from_config("bailian", Some(&raw)).unwrap() + ); + } + } use super::*; use crate::{InMemoryCredentialStore, ProviderInvocation, SecretValue}; diff --git a/openless-all/app/crates/openless-core/src/provider_rules.rs b/openless-all/app/crates/openless-core/src/provider_rules.rs index 7ed16b35f..f13127020 100644 --- a/openless-all/app/crates/openless-core/src/provider_rules.rs +++ b/openless-all/app/crates/openless-core/src/provider_rules.rs @@ -765,7 +765,10 @@ pub fn is_stepfun_realtime_provider(id: &str) -> bool { } pub fn is_mimo_provider(id: &str) -> bool { - matches!(id, MIMO_PROVIDER_ID | crate::asr::mimo::ORCAROUTER_PROVIDER_ID) + matches!( + id, + MIMO_PROVIDER_ID | crate::asr::mimo::ORCAROUTER_PROVIDER_ID + ) } pub fn is_dashscope_multimodal_provider(id: &str) -> bool { @@ -791,6 +794,67 @@ pub fn is_whisper_compatible_provider(id: &str) -> bool { ) || id == OPENAI_COMPATIBLE_ASR_PROVIDER_ID } +/// Explicit channel selection takes precedence over model-name inference. +/// Stored inside asr.advanced_config so all hosts use the existing credential lifecycle. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum BailianProtocol { + #[default] + Auto, + DashscopeRealtime, + QwenRealtime, + Multimodal, + QwenMultimodal, + AsyncTranscription, +} + +impl BailianProtocol { + pub fn from_config(provider: &str, raw: Option<&str>) -> Result { + if !is_bailian_provider(provider) { + return Ok(Self::Auto); + } + let Some(raw) = raw.filter(|s| !s.trim().is_empty()) else { + return Ok(Self::Auto); + }; + let value: serde_json::Value = + serde_json::from_str(raw).map_err(|_| "百炼接口配置不是有效 JSON".to_string())?; + match value.get("bailianProtocol") { + None => Ok(Self::Auto), + Some(value) => serde_json::from_value(value.clone()) + .map_err(|_| "不支持的百炼接口类型，请重新选择接口".to_string()), + } + } + + pub fn resolve_provider(self, provider: &str, model: &str) -> Result { + if !is_bailian_provider(provider) || self == Self::Auto { + return resolve_effective_asr_provider(provider, model); + } + if model.trim().is_empty() { + return Err("手动选择百炼接口时必须填写模型 ID".to_string()); + } + Ok(match self { + Self::DashscopeRealtime => BAILIAN_PROVIDER_ID, + Self::QwenRealtime => QWEN3_REALTIME_PROVIDER_ID, + _ => DASHSCOPE_MULTIMODAL_PROVIDER_ID, + } + .to_string()) + } + + pub fn batch_protocol(self, model: &str) -> Option { + match self { + Self::Auto => dashscope_batch_protocol_for_model(model), + Self::Multimodal | Self::QwenMultimodal => Some(DashScopeBatchProtocol::Multimodal), + Self::AsyncTranscription => Some(DashScopeBatchProtocol::AsyncTranscription), + _ => None, + } + } + + pub fn uses_qwen_envelope(self, model: &str) -> bool { + self == Self::QwenMultimodal + || (self == Self::Auto && dashscope_uses_qwen_sync_envelope(model)) + } +} + pub fn resolve_effective_asr_provider(active_asr: &str, model: &str) -> Result { if !is_bailian_provider(active_asr) { if is_dashscope_multimodal_provider(active_asr) { @@ -817,7 +881,8 @@ pub fn resolve_effective_asr_provider(active_asr: &str, model: &str) -> Result bool { - model.starts_with("fun-asr-realtime") + model == "qwen-audio-3.0-asr-flash-streaming" + || model.starts_with("fun-asr-realtime") || model.starts_with("fun-asr-flash-8k-realtime") || model.starts_with("paraformer-realtime") || model.starts_with("paraformer-8k-realtime") @@ -1066,8 +1131,64 @@ mod tests { } } + #[test] + fn manual_bailian_protocol_overrides_names_and_preserves_auto() { + for (name, expected) in [ + ("dashscope-realtime", BAILIAN_PROVIDER_ID), + ("qwen-realtime", QWEN3_REALTIME_PROVIDER_ID), + ("multimodal", DASHSCOPE_MULTIMODAL_PROVIDER_ID), + ("qwen-multimodal", DASHSCOPE_MULTIMODAL_PROVIDER_ID), + ("async-transcription", DASHSCOPE_MULTIMODAL_PROVIDER_ID), + ] { + let raw = format!(r#"{{"bailianProtocol":"{name}"}}"#); + let protocol = BailianProtocol::from_config("bailian", Some(&raw)).unwrap(); + for model in ["unknown-model", "fun-asr", "qwen3-asr-flash-realtime"] { + assert_eq!( + protocol.resolve_provider("bailian", model).unwrap(), + expected + ); + } + assert!(protocol.resolve_provider("bailian", " ").is_err()); + assert_eq!( + BailianProtocol::from_config("whisper", Some(&raw)).unwrap(), + BailianProtocol::Auto + ); + } + assert_eq!( + BailianProtocol::from_config("bailian", None).unwrap(), + BailianProtocol::Auto + ); + assert_eq!( + BailianProtocol::from_config("bailian", Some("{}")).unwrap(), + BailianProtocol::Auto + ); + assert!( + BailianProtocol::from_config("bailian", Some(r#"{"bailianProtocol":"typo"}"#)).is_err() + ); + assert!(BailianProtocol::Auto + .resolve_provider("bailian", "unknown-model") + .is_err()); + assert_eq!( + BailianProtocol::Auto + .resolve_provider("bailian", "fun-asr") + .unwrap(), + DASHSCOPE_MULTIMODAL_PROVIDER_ID + ); + } + #[test] fn routes_bailian_and_stepfun_models() { + let streaming = "qwen-audio-3.0-asr-flash-streaming"; + assert_eq!( + resolve_effective_asr_provider(BAILIAN_PROVIDER_ID, streaming).unwrap(), + BAILIAN_PROVIDER_ID + ); + assert_eq!(dashscope_batch_protocol_for_model(streaming), None); + assert_eq!( + resolve_effective_asr_provider(BAILIAN_PROVIDER_ID, "qwen-audio-3.0-asr-flash") + .unwrap(), + DASHSCOPE_MULTIMODAL_PROVIDER_ID + ); assert_eq!( resolve_effective_asr_provider(BAILIAN_PROVIDER_ID, "fun-asr-realtime").unwrap(), BAILIAN_PROVIDER_ID diff --git a/openless-all/app/crates/openless-core/src/provider_service.rs b/openless-all/app/crates/openless-core/src/provider_service.rs index 47ab7db4b..b6e03a2c4 100644 --- a/openless-all/app/crates/openless-core/src/provider_service.rs +++ b/openless-all/app/crates/openless-core/src/provider_service.rs @@ -174,7 +174,21 @@ impl ProviderService { None => None, }; + let bailian_protocol = if request.kind == ProviderKind::Asr { + let raw = self + .read( + namespace, + &provider_id, + crate::credentials::ASR_ADVANCED_CONFIG_ACCOUNT, + ) + .await?; + crate::provider_rules::BailianProtocol::from_config(&provider_type, raw.as_deref()) + .map_err(invalid_request)? + } else { + crate::provider_rules::BailianProtocol::Auto + }; Ok(ResolvedProvider { + bailian_protocol, thinking_enabled: request.thinking_enabled, protocol: if request.kind == ProviderKind::Llm { LlmProtocolConfig::load(self.credentials.as_ref(), &provider_id, &provider_type) @@ -228,11 +242,7 @@ impl ProviderService { } ensure_supported_kind(&resolved)?; validate_configuration(&resolved, true)?; - let probe = validation_probe_for( - resolved.kind, - &resolved.provider_type, - resolved.model.as_deref(), - ); + let probe = resolved.validation_probe()?; if probe == ValidationProbe::AsrNonSilent { return tokio::select! { _ = wait_for_cancellation(cancellation) => Err(cancelled_request()), @@ -389,6 +399,7 @@ impl ProviderApi for ProviderService { #[derive(Debug, Clone)] struct ResolvedProvider { thinking_enabled: bool, + bailian_protocol: crate::provider_rules::BailianProtocol, protocol: LlmProtocolConfig, kind: ProviderKind, provider_id: String, @@ -400,6 +411,30 @@ struct ResolvedProvider { } impl ResolvedProvider { + fn validation_probe(&self) -> Result { + let effective_provider = if self.kind == ProviderKind::Asr { + self.bailian_protocol + .resolve_provider( + &self.provider_type, + self.model.as_deref().unwrap_or_default(), + ) + .map_err(invalid_request)? + } else { + self.provider_type.clone() + }; + Ok( + if self.bailian_protocol != crate::provider_rules::BailianProtocol::Auto { + if self.bailian_protocol.batch_protocol("").is_some() { + ValidationProbe::AsrNonSilent + } else { + validation_probe_for(self.kind, &effective_provider, None) + } + } else { + validation_probe_for(self.kind, &effective_provider, self.model.as_deref()) + }, + ) + } + fn context(&self) -> DictationContext { let mut context = DictationContext::default(); context.polish.llm_thinking_enabled = self.thinking_enabled; @@ -540,8 +575,13 @@ async fn validate_dashscope_probe(resolved: &ResolvedProvider) -> Result<(), Bac .filter(|value| !value.trim().is_empty()) .or_else(|| default_asr_model(&resolved.provider_type)) .ok_or_else(|| invalid_request("ASR model is not configured"))?; - crate::provider_rules::validate_dashscope_multimodal_model(model).map_err(invalid_request)?; - let protocol = crate::provider_rules::dashscope_batch_protocol_for_model(model) + if resolved.bailian_protocol == crate::provider_rules::BailianProtocol::Auto { + crate::provider_rules::validate_dashscope_multimodal_model(model) + .map_err(invalid_request)?; + } + let protocol = resolved + .bailian_protocol + .batch_protocol(model) .unwrap_or(crate::provider_rules::DashScopeBatchProtocol::Multimodal); let stored_endpoint = resolved .endpoint @@ -587,9 +627,10 @@ async fn validate_dashscope_probe(resolved: &ResolvedProvider) -> Result<(), Bac let url = crate::asr::dashscope_multimodal::generation_url(&endpoint) .map_err(|_| invalid_request("ASR endpoint is invalid"))?; - let body = crate::asr::dashscope_multimodal::dashscope_multimodal_body_from_uri( + let body = crate::asr::dashscope_multimodal::dashscope_multimodal_body_with_protocol( model, DASHSCOPE_ASR_VALIDATE_SAMPLE_URL, + resolved.bailian_protocol, ); let response = crate::net::credential_http() .post(url) @@ -1412,6 +1453,58 @@ mod tests { assert!(!format!("{error:?}").contains("response-secret")); } + #[tokio::test] + async fn manual_bailian_protocol_is_loaded_per_channel_for_validation() { + let credentials = Arc::new(InMemoryCredentialStore::default()); + for (selected, model, expected) in [ + ("dashscope-realtime", "fun-asr", ValidationProbe::AsrSilence), + ( + "qwen-realtime", + "unknown-model", + ValidationProbe::AsrSilence, + ), + ( + "multimodal", + "qwen3-asr-flash-realtime", + ValidationProbe::AsrNonSilent, + ), + ( + "qwen-multimodal", + "unknown-model", + ValidationProbe::AsrNonSilent, + ), + ( + "async-transcription", + "unknown-model", + ValidationProbe::AsrNonSilent, + ), + ] { + let raw = format!(r#"{{"bailianProtocol":"{selected}"}}"#); + let channel = create_channel_with_values( + &credentials, + ChannelKind::Asr, + "bailian", + &[ + (ASR_API_KEY_ACCOUNT, "fixture-key"), + (ASR_MODEL_ACCOUNT, model), + (crate::credentials::ASR_ADVANCED_CONFIG_ACCOUNT, &raw), + ], + ) + .await; + let service = + ProviderService::new(credentials.clone(), Arc::new(crate::TokioTaskSpawner)); + let resolved = service + .resolve(ProviderRequest { + thinking_enabled: false, + kind: ProviderKind::Asr, + channel_id: Some(channel), + }) + .await + .unwrap(); + assert_eq!(resolved.validation_probe().unwrap(), expected, "{selected}"); + } + } + #[tokio::test] async fn static_model_list_runs_the_real_provider_probe_first() { let (endpoint, request) = @@ -2184,6 +2277,7 @@ mod tests { for (provider_type, expected_models) in expected { let resolved = ResolvedProvider { thinking_enabled: false, + bailian_protocol: crate::provider_rules::BailianProtocol::Auto, protocol: LlmProtocolConfig::default(), kind: ProviderKind::Asr, provider_id: provider_type.to_string(), diff --git a/openless-all/app/scripts/bailian-protocol-h5.mjs b/openless-all/app/scripts/bailian-protocol-h5.mjs new file mode 100644 index 000000000..0d007ae1f --- /dev/null +++ b/openless-all/app/scripts/bailian-protocol-h5.mjs @@ -0,0 +1,144 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { mkdirSync, writeFileSync, unlinkSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join, resolve } from 'node:path'; +import { createServer } from 'vite'; + +const out = + process.env.OPENLESS_MOTION_ARTIFACT_DIR || join(tmpdir(), 'openless-bailian-protocol-h5'); +mkdirSync(out, { recursive: true }); +const server = await createServer({ + server: { host: '127.0.0.1', port: 1441, strictPort: true, watch: null }, +}); +const fixture = resolve(`.bailian-protocol-h5-${process.pid}-${Date.now()}.html`); +writeFileSync( + fixture, + `
`, +); +await server.listen(); +const pageUrl = 'http://127.0.0.1:1441/' + basename(fixture); +const chrome = spawn( + process.env.CHROME_PATH || 'C:/Program Files/Google/Chrome/Application/chrome.exe', + [ + '--headless=new', + '--no-first-run', + '--no-default-browser-check', + '--remote-debugging-port=9441', + '--window-size=900,500', + '--user-data-dir=' + join(tmpdir(), 'openless-bailian-protocol-' + Date.now()), + pageUrl, + ], + { stdio: 'ignore', windowsHide: true }, +); +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +let socket; +try { + let target; + for (let i = 0; i < 100; i++) { + try { + target = (await (await fetch('http://127.0.0.1:9441/json/list')).json()).find( + (t) => t.type === 'page' && t.url === pageUrl, + ); + } catch {} + if (target) break; + await sleep(100); + } + assert(target, 'Chrome page target'); + socket = new WebSocket(target.webSocketDebuggerUrl); + await new Promise((r) => socket.addEventListener('open', r, { once: true })); + let id = 0; + const pending = new Map(); + socket.addEventListener('message', (e) => { + const m = JSON.parse(String(e.data)); + if (m.id) { + const p = pending.get(m.id); + pending.delete(m.id); + m.error ? p.reject(m.error) : p.resolve(m.result); + } + }); + const send = (method, params = {}) => + new Promise((resolve, reject) => { + pending.set(++id, { resolve, reject }); + socket.send(JSON.stringify({ id, method, params })); + }); + const evaluate = async (expression) => { + const r = await send('Runtime.evaluate', { + expression, + awaitPromise: true, + returnByValue: true, + }); + if (r.exceptionDetails) + throw Error(r.exceptionDetails.exception?.description || r.exceptionDetails.text); + return r.result.value; + }; + for (let i = 0; i < 150; i++) { + if (await evaluate('Boolean(window.ready)')) break; + await sleep(100); + } + assert(await evaluate('Boolean(window.ready)'), 'React fixture ready'); + const until = async (expression) => { + for (let i = 0; i < 100; i++) { + if (await evaluate(expression)) return; + await sleep(50); + } + throw new Error('Timed out: ' + expression); + }; + await until('!document.querySelector("select").disabled'); + assert.equal(await evaluate('document.querySelector("select").value'), 'auto'); + assert.equal(await evaluate('document.querySelectorAll("option").length'), 6); + for (const value of [ + 'dashscope-realtime', + 'qwen-realtime', + 'multimodal', + 'qwen-multimodal', + 'async-transcription', + ]) { + await evaluate(`window.choose(${JSON.stringify(value)});window.finish()`); + await until('!document.querySelector("select").disabled && !window.blocked'); + assert.equal(JSON.parse(await evaluate('window.read("a")')).bailianProtocol, value); + assert.equal(JSON.parse(await evaluate('window.read("a")')).enableItn, false); + } + await evaluate('window.show("b")'); + await until('!document.querySelector("select").disabled'); + assert.equal(await evaluate('document.querySelector("select").value'), 'auto'); + await evaluate('window.show("a")'); + await until('!document.querySelector("select").disabled'); + assert.equal(await evaluate('document.querySelector("select").value'), 'async-transcription'); + await evaluate('window.choose("auto")'); + await until('!document.querySelector("select").disabled && !window.blocked'); + assert.equal(JSON.parse(await evaluate('window.read("a")')).bailianProtocol, undefined); + // A malformed configuration must not be overwritten or silently accepted on close. + await evaluate('window.write("bad","invalid").then(()=>window.show("bad"))'); + await until('Boolean(document.querySelector("[role=alert]"))'); + assert.equal(await evaluate('window.finish()'), false); + assert.equal(await evaluate('window.read("bad")'), 'invalid'); + console.log( + 'PASS: six options, persistence, channel isolation, auto reset, close/save coordination, invalid configuration', + ); +} finally { + socket?.close(); + chrome.kill(); + await server.close(); + unlinkSync(fixture); +} diff --git a/openless-all/app/src/i18n/de.ts b/openless-all/app/src/i18n/de.ts index ea3628ad5..9d2a6d7dd 100644 --- a/openless-all/app/src/i18n/de.ts +++ b/openless-all/app/src/i18n/de.ts @@ -1456,6 +1456,16 @@ export const de: typeof zhCN = { bailianVocabularyIdLabel: 'Wörterbuch-ID für Begriffe (optional)', bailianVocabularyIdNote: 'Wenn du bei DashScope ein Begriffswörterbuch erstellt hast, gib seine vocab-... ID ein. Leer lassen, um keine Begriffe zu übergeben.', + bailianProtocolLabel: "API-Typ", + bailianProtocolNote: "Die manuelle Auswahl ersetzt die Erkennung anhand des Modellnamens und gilt pro Kanal für Prüfung und Aufnahme. Beachten Sie die Modelldokumentation.", + bailianProtocolOptions: { + "auto": "Automatisch", + "dashscope-realtime": "Echtzeit (DashScope)", + "qwen-realtime": "Echtzeit (Qwen Realtime)", + "multimodal": "Synchron, nicht in Echtzeit (Fun-ASR / Qwen-Audio)", + "qwen-multimodal": "Synchron, nicht in Echtzeit (Qwen3-ASR)", + "async-transcription": "Asynchron (Dateitranskription)", + }, bailianModelRealtimeHint: 'Echtzeitmodell · transkribiert während des Sprechens.', bailianModelSyncFileHint: 'Synchrones Aufnahmemodell · transkribiert nach dem Beenden (einzelne Aufnahme ≤ 5 Min.).', diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index bf2d03b75..a8344926f 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -1426,6 +1426,16 @@ export const en: typeof zhCN = { bailianVocabularyIdLabel: 'Hotword Vocabulary ID (optional)', bailianVocabularyIdNote: 'If you have created a DashScope hotword vocabulary, enter its vocab-... ID. Leave blank to skip hotwords.', + bailianProtocolLabel: "API type", + bailianProtocolNote: "Manual selection overrides model-name detection and is saved per channel for both validation and recording. Choose the API specified by the model documentation.", + bailianProtocolOptions: { + "auto": "Automatic", + "dashscope-realtime": "Realtime (DashScope)", + "qwen-realtime": "Realtime (Qwen Realtime)", + "multimodal": "Non-realtime synchronous (Fun-ASR / Qwen-Audio)", + "qwen-multimodal": "Non-realtime synchronous (Qwen3-ASR)", + "async-transcription": "Non-realtime asynchronous (file transcription)", + }, bailianModelRealtimeHint: 'Realtime model · transcribes as you speak.', bailianModelSyncFileHint: 'Synchronous recording model · transcribes after you finish (single clip ≤ 5 min).', diff --git a/openless-all/app/src/i18n/es.ts b/openless-all/app/src/i18n/es.ts index 0c71ec6f2..71760e83f 100644 --- a/openless-all/app/src/i18n/es.ts +++ b/openless-all/app/src/i18n/es.ts @@ -1447,6 +1447,16 @@ export const es: typeof zhCN = { bailianVocabularyIdLabel: 'ID del vocabulario de palabras clave (opcional)', bailianVocabularyIdNote: 'Si creaste un vocabulario en DashScope, introduce su ID vocab-... Déjalo vacío para no usar palabras clave.', + bailianProtocolLabel: "Tipo de API", + bailianProtocolNote: "La selección manual prevalece sobre el nombre del modelo y se guarda por canal para validar y grabar. Consulte la documentación del modelo.", + bailianProtocolOptions: { + "auto": "Automático", + "dashscope-realtime": "Tiempo real (DashScope)", + "qwen-realtime": "Tiempo real (Qwen Realtime)", + "multimodal": "Síncrono no en tiempo real (Fun-ASR / Qwen-Audio)", + "qwen-multimodal": "Síncrono no en tiempo real (Qwen3-ASR)", + "async-transcription": "Asíncrono (transcripción de archivos)", + }, bailianModelRealtimeHint: 'Modelo en tiempo real: transcribe mientras hablas.', bailianModelSyncFileHint: 'Modelo síncrono de grabación: transcribe al terminar (máx. 5 min por grabación).', diff --git a/openless-all/app/src/i18n/fr.ts b/openless-all/app/src/i18n/fr.ts index 692b1e9d2..e5d0812c9 100644 --- a/openless-all/app/src/i18n/fr.ts +++ b/openless-all/app/src/i18n/fr.ts @@ -1467,6 +1467,16 @@ export const fr: typeof zhCN = { bailianVocabularyIdLabel: 'ID du vocabulaire de mots-clés (facultatif)', bailianVocabularyIdNote: 'Si vous avez créé un vocabulaire dans DashScope, saisissez son ID vocab-... Laissez vide pour ne pas utiliser de mots-clés.', + bailianProtocolLabel: "Type d’API", + bailianProtocolNote: "Le choix manuel remplace la détection par nom et est enregistré par canal pour la validation et l’enregistrement. Consultez la documentation du modèle.", + bailianProtocolOptions: { + "auto": "Automatique", + "dashscope-realtime": "Temps réel (DashScope)", + "qwen-realtime": "Temps réel (Qwen Realtime)", + "multimodal": "Synchrone hors temps réel (Fun-ASR / Qwen-Audio)", + "qwen-multimodal": "Synchrone hors temps réel (Qwen3-ASR)", + "async-transcription": "Asynchrone (transcription de fichier)", + }, bailianModelRealtimeHint: 'Modèle en temps réel : transcrit pendant que vous parlez.', bailianModelSyncFileHint: 'Modèle d’enregistrement synchrone : transcrit une fois l’enregistrement terminé (5 min maximum).', diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index 7e5440262..ccc201a03 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -1412,6 +1412,16 @@ export const ja: typeof zhCN = { bailianVocabularyIdLabel: 'ホットワード Vocabulary ID（任意）', bailianVocabularyIdNote: 'DashScope でホットワード辞書を作成済みの場合は vocab-... ID を入力します。空欄なら送信しません。', + bailianProtocolLabel: "API の種類", + bailianProtocolNote: "手動選択はモデル名による判定より優先され、検証と録音用にチャネルごとに保存されます。モデルのドキュメントに従って選択してください。", + bailianProtocolOptions: { + "auto": "自動判定", + "dashscope-realtime": "リアルタイム（DashScope）", + "qwen-realtime": "リアルタイム（Qwen Realtime）", + "multimodal": "非リアルタイム同期（Fun-ASR / Qwen-Audio）", + "qwen-multimodal": "非リアルタイム同期（Qwen3-ASR）", + "async-transcription": "非リアルタイム非同期（ファイル文字起こし）", + }, bailianModelRealtimeHint: 'リアルタイムモデル · 話しながら文字起こし。', bailianModelSyncFileHint: '同期録音モデル · 話し終えてから一括で文字起こし（1 本 ≤ 5 分）。', bailianModelAsyncFileHint: diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index 6dc745a55..f599643e1 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -1404,6 +1404,16 @@ export const ko: typeof zhCN = { bailianVocabularyIdLabel: '핫워드 Vocabulary ID(선택)', bailianVocabularyIdNote: 'DashScope에서 핫워드 사전을 만들었다면 vocab-... ID를 입력하세요. 비워 두면 핫워드를 전송하지 않습니다.', + bailianProtocolLabel: "API 유형", + bailianProtocolNote: "수동 선택은 모델 이름 감지보다 우선하며 검증과 녹음에 사용할 채널별 설정으로 저장됩니다. 모델 문서에 따라 선택하세요.", + bailianProtocolOptions: { + "auto": "자동 감지", + "dashscope-realtime": "실시간 (DashScope)", + "qwen-realtime": "실시간 (Qwen Realtime)", + "multimodal": "비실시간 동기 (Fun-ASR / Qwen-Audio)", + "qwen-multimodal": "비실시간 동기 (Qwen3-ASR)", + "async-transcription": "비실시간 비동기 (파일 전사)", + }, bailianModelRealtimeHint: '실시간 모델 · 말하는 동안 바로 전사.', bailianModelSyncFileHint: '동기 녹음 모델 · 말을 마친 뒤 전체 전사(한 클립 ≤ 5분).', bailianModelAsyncFileHint: diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index f503982eb..b69b0b879 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -1355,6 +1355,16 @@ export const zhCN = { '按所选请求格式和模型支持的参数启用、关闭或降低思考，不向提示词注入控制指令。', bailianVocabularyIdLabel: '热词 Vocabulary ID（可选）', bailianVocabularyIdNote: '如已在百炼创建热词表，可填写 vocab-...；留空则不下发热词。', + bailianProtocolLabel: "接口类型", + bailianProtocolNote: "手动选择优先于模型名称，按渠道保存，同时用于验证和录音。请根据模型文档选择接口。", + bailianProtocolOptions: { + "auto": "自动识别", + "dashscope-realtime": "实时（DashScope）", + "qwen-realtime": "实时（Qwen Realtime）", + "multimodal": "非实时同步（Fun-ASR / Qwen-Audio）", + "qwen-multimodal": "非实时同步（Qwen3-ASR）", + "async-transcription": "非实时异步（文件转写）", + }, bailianModelRealtimeHint: '实时模型 · 边说边出字。', bailianModelSyncFileHint: '同步录音模型 · 说完后整段转写（单条 ≤ 5 分钟）。', bailianModelAsyncFileHint: '异步文件模型 · 录音上传后等待转写任务完成。', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index e2b3cff71..e62df0fed 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -1357,6 +1357,16 @@ export const zhTW: typeof zhCN = { '依所選請求格式與模型支援的參數啟用、關閉或降低思考，不在提示詞注入控制指令。', bailianVocabularyIdLabel: '熱詞 Vocabulary ID（可選）', bailianVocabularyIdNote: '如已在百煉建立熱詞表，可填寫 vocab-...；留空則不下發熱詞。', + bailianProtocolLabel: "介面類型", + bailianProtocolNote: "手動選擇優先於模型名稱，按渠道儲存，同時用於驗證和錄音。請依模型文件選擇介面。", + bailianProtocolOptions: { + "auto": "自動辨識", + "dashscope-realtime": "即時（DashScope）", + "qwen-realtime": "即時（Qwen Realtime）", + "multimodal": "非即時同步（Fun-ASR / Qwen-Audio）", + "qwen-multimodal": "非即時同步（Qwen3-ASR）", + "async-transcription": "非即時非同步（檔案轉寫）", + }, bailianModelRealtimeHint: '即時模型 · 邊說邊出字。', bailianModelSyncFileHint: '同步錄音模型 · 說完後整段轉寫（單條 ≤ 5 分鐘）。', bailianModelAsyncFileHint: '非同步檔案模型 · 錄音上傳後等待轉寫工作完成。', diff --git a/openless-all/app/src/lib/bailianProtocol.test.ts b/openless-all/app/src/lib/bailianProtocol.test.ts new file mode 100644 index 000000000..220ef2f92 --- /dev/null +++ b/openless-all/app/src/lib/bailianProtocol.test.ts @@ -0,0 +1,33 @@ +import { bailianProtocols, readBailianProtocol, writeBailianProtocol } from './bailianProtocol'; + +const assert = { + equal(actual: unknown, expected: unknown) { + if (actual !== expected) + throw new Error(`Expected ${String(expected)}, received ${String(actual)}`); + }, + throws(action: () => unknown) { + try { + action(); + } catch { + return; + } + throw new Error('Expected invalid configuration to be rejected'); + }, +}; + +assert.equal(readBailianProtocol(null), 'auto'); +for (const protocol of bailianProtocols) { + const raw = writeBailianProtocol('{"enableItn":false,"chunkDurationMs":1234}', protocol); + assert.equal(readBailianProtocol(raw), protocol); + assert.equal(JSON.parse(raw).enableItn, false); + assert.equal(JSON.parse(raw).chunkDurationMs, 1234); +} +const reset = writeBailianProtocol( + '{"bailianProtocol":"qwen-realtime","verboseJson":true}', + 'auto', +); +assert.equal('bailianProtocol' in JSON.parse(reset), false); +assert.equal(JSON.parse(reset).verboseJson, true); +assert.throws(() => readBailianProtocol('{"bailianProtocol":"unknown"}')); +assert.throws(() => writeBailianProtocol('broken JSON', 'multimodal')); +console.log('bailianProtocol tests passed'); diff --git a/openless-all/app/src/lib/bailianProtocol.ts b/openless-all/app/src/lib/bailianProtocol.ts new file mode 100644 index 000000000..54a899e12 --- /dev/null +++ b/openless-all/app/src/lib/bailianProtocol.ts @@ -0,0 +1,30 @@ +export const bailianProtocols = [ + 'auto', + 'dashscope-realtime', + 'qwen-realtime', + 'multimodal', + 'qwen-multimodal', + 'async-transcription', +] as const; +export type BailianProtocol = (typeof bailianProtocols)[number]; + +export function readBailianProtocol(raw: string | null | undefined): BailianProtocol { + const value: unknown = raw?.trim() ? JSON.parse(raw) : {}; + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Invalid ASR configuration'); + const protocol = (value as Record).bailianProtocol ?? 'auto'; + if (!bailianProtocols.includes(protocol as BailianProtocol)) + throw new Error('Invalid Bailian protocol'); + return protocol as BailianProtocol; +} + +export function writeBailianProtocol( + raw: string | null | undefined, + protocol: BailianProtocol, +): string { + readBailianProtocol(raw); + const value = raw?.trim() ? JSON.parse(raw) : {}; + if (protocol === 'auto') delete value.bailianProtocol; + else value.bailianProtocol = protocol; + return JSON.stringify(value); +} diff --git a/openless-all/app/src/pages/settings/BailianProtocolField.tsx b/openless-all/app/src/pages/settings/BailianProtocolField.tsx new file mode 100644 index 000000000..47322858c --- /dev/null +++ b/openless-all/app/src/pages/settings/BailianProtocolField.tsx @@ -0,0 +1,124 @@ +import { useContext, useEffect, useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { readCredential, setCredential } from '../../lib/ipc'; +import { + bailianProtocols, + readBailianProtocol, + writeBailianProtocol, + type BailianProtocol, +} from '../../lib/bailianProtocol'; +import { ProviderFormContext } from './ProviderForm'; +import { inputStyle } from './shared'; +import { emitSaved } from '../../lib/savedEvent'; + +const account = 'asr.advanced_config'; +export function BailianProtocolField({ + channelId, + onChange, + onUserMutation, + onBlockedChange, +}: { + channelId: string; + onChange: (protocol: BailianProtocol) => void; + onUserMutation?: () => void; + onBlockedChange: (account: string, blocked: boolean) => void; +}) { + const { t } = useTranslation(); + const form = useContext(ProviderFormContext); + const track = form?.track; + const register = form?.register; + const [selected, setSelected] = useState('auto'); + const [loaded, setLoaded] = useState(false); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(''); + const pending = useRef>(Promise.resolve(true)); + const writing = useRef(false); + const mounted = useRef(false); + const canLeave = useRef(false); + canLeave.current = loaded && !error; + + useEffect(() => { + mounted.current = true; + let cancelled = false; + readCredential(account, channelId) + .then((raw) => { + if (cancelled) return; + const protocol = readBailianProtocol(raw); + setSelected(protocol); + onChange(protocol); + setLoaded(true); + }) + .catch((err) => { + if (!cancelled) setError(String(err)); + }); + return () => { + cancelled = true; + mounted.current = false; + }; + }, [channelId, onChange]); + + useEffect( + () => register?.(account, async () => (await pending.current) && canLeave.current), + [register], + ); + useEffect(() => { + const blocked = !loaded || saving || Boolean(error); + track?.(account, blocked); + onBlockedChange(account, blocked); + return () => { + track?.(account, false); + onBlockedChange(account, false); + }; + }, [loaded, saving, error, track, onBlockedChange]); + + const save = (protocol: BailianProtocol) => { + if (writing.current || !loaded) return; + writing.current = true; + setSaving(true); + setError(''); + form?.invalidate(account); + track?.(account, true); + onBlockedChange(account, true); + onUserMutation?.(); + pending.current = (async () => { + try { + const raw = await readCredential(account, channelId); + await setCredential(account, writeBailianProtocol(raw, protocol), channelId); + if (mounted.current) { + setSelected(protocol); + onChange(protocol); + emitSaved('saved', t('common.saved')); + } + return true; + } catch (err) { + if (mounted.current) setError(String(err)); + return false; + } finally { + writing.current = false; + if (mounted.current) setSaving(false); + } + })(); + }; + + return ( + + ); +} diff --git a/openless-all/app/src/pages/settings/ProvidersSection.tsx b/openless-all/app/src/pages/settings/ProvidersSection.tsx index a9caf8ca3..77319b712 100644 --- a/openless-all/app/src/pages/settings/ProvidersSection.tsx +++ b/openless-all/app/src/pages/settings/ProvidersSection.tsx @@ -25,6 +25,8 @@ import { validateProviderCredentials, type ProviderDescriptor, } from '../../lib/ipc'; +import { BailianProtocolField } from './BailianProtocolField'; +import type { BailianProtocol } from '../../lib/bailianProtocol'; import { LlmProtocolFields } from './LlmProtocolFields'; import { ProviderFormContext } from './ProviderForm'; import { LocalModelPicker } from './models/LocalModelPicker'; @@ -233,6 +235,7 @@ export function ChannelCredentialFields({ const [asrModelRevision, setAsrModelRevision] = useState(0); const unifiedBailian = providerType === 'bailian'; const [bailianModel, setBailianModel] = useState(''); + const [bailianProtocol, setBailianProtocol] = useState('auto'); const [volcengineAuthMode, setVolcengineAuthMode] = useState<'app_id_token' | 'api_key'>( 'app_id_token', ); @@ -816,13 +819,16 @@ export function ChannelCredentialFields({ : undefined } /> + {unifiedBailian && } {unifiedBailian && ( )} - {unifiedBailian && bailianModelSupportsVocabulary(bailianModel) && ( + {unifiedBailian && (bailianProtocol === 'dashscope-realtime' || (bailianProtocol === 'auto' && bailianModelSupportsVocabulary(bailianModel))) && ( <> setAsrModelRevision((v) => v + 1)} @@ -1009,7 +1016,8 @@ function AsrAdvancedOptions({ // 与 qwen-audio-3.0-asr-flash 是「录音文件·说完转写」（同步）。 function bailianModelProtocol(model: string): 'realtime' | 'sync' | 'async' { const m = model.trim(); - if (!m || m.includes('realtime')) return 'realtime'; + if (!m || m.includes('realtime') || m === 'qwen-audio-3.0-asr-flash-streaming') + return 'realtime'; // qwen3-asr-flash-filetrans 仅接受公网 URL，暂不支持（后端 protocol_for_model // 显式拒绝），前端不再归为 async 提示。 if ( @@ -1028,6 +1036,7 @@ function bailianModelSupportsVocabulary(model: string): boolean { const m = model.trim(); return ( !m || + m === 'qwen-audio-3.0-asr-flash-streaming' || m.startsWith('fun-asr-realtime') || m.startsWith('paraformer-realtime') || m.startsWith('sensevoice-realtime') @@ -1036,29 +1045,12 @@ function bailianModelSupportsVocabulary(model: string): boolean { // 模型框下的一行协议提示,解决「三种模型看不出区别」——告诉用户当前模型是实时还是 // 录音文件、行为差异如何。随 asrModelRevision(拉取/选择模型时)与挂载时重读 asr.model。 -function BailianProtocolHint({ currentModel }: { currentModel: string }) { +function BailianProtocolHint({ currentModel, selectedProtocol }: { currentModel: string; selectedProtocol: BailianProtocol }) { const { t } = useTranslation(); - const [model, setModel] = useState(''); - - useEffect(() => { - let cancelled = false; - readCredential('asr.model') - .then((v) => { - if (!cancelled) setModel(v || 'fun-asr-realtime'); - }) - .catch(() => { - /* 读失败按默认实时提示 */ - }); - return () => { - cancelled = true; - }; - }, []); - - useEffect(() => { - setModel(currentModel || 'fun-asr-realtime'); - }, [currentModel]); - - const protocol = bailianModelProtocol(model); + const protocol = selectedProtocol === 'auto' + ? bailianModelProtocol(currentModel) + : selectedProtocol === 'async-transcription' ? 'async' + : selectedProtocol.endsWith('realtime') ? 'realtime' : 'sync'; const hint = protocol === 'realtime' ? t('settings.providers.bailianModelRealtimeHint') From 4db5e7bb164e6cb6595d6705006353b210daa702 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Sun, 20 Sep 2026 17:52:13 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix(core):=20=E4=BF=AE=E5=A4=8D=E4=B8=8A?= =?UTF-8?q?=E6=B8=B8=E5=9F=BA=E7=BA=BF=E7=9A=84=E4=B8=A5=E6=A0=BC=E6=A3=80?= =?UTF-8?q?=E6=9F=A5=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../crates/openless-core/src/asr/bailian.rs | 36 +++++++++++-------- .../crates/openless-core/src/cloud_sync.rs | 2 +- .../app/crates/openless-core/src/domains.rs | 2 +- .../openless-core/src/external_audio.rs | 18 ++++++---- .../src/external_audio/archive.rs | 2 +- .../openless-core/src/provider_service.rs | 4 +-- 6 files changed, 39 insertions(+), 25 deletions(-) diff --git a/openless-all/app/crates/openless-core/src/asr/bailian.rs b/openless-all/app/crates/openless-core/src/asr/bailian.rs index 9e2850e32..00a9d35be 100644 --- a/openless-all/app/crates/openless-core/src/asr/bailian.rs +++ b/openless-all/app/crates/openless-core/src/asr/bailian.rs @@ -46,14 +46,15 @@ const PER_ADDR_TCP_TIMEOUT: Duration = Duration::from_millis(1500); fn default_port_for_request( request: &tokio_tungstenite::tungstenite::handshake::client::Request, -) -> Result { +) -> Result> { let default_port = match request.uri().scheme_str() { Some("ws") => 80, Some("wss") => 443, _ => { return Err(WsError::Url( tokio_tungstenite::tungstenite::error::UrlError::UnsupportedUrlScheme, - )) + ) + .into()) } }; Ok(request.uri().port_u16().unwrap_or(default_port)) @@ -72,13 +73,14 @@ async fn connect_ws_to_addrs( WsStream, tokio_tungstenite::tungstenite::handshake::client::Response, ), - WsError, + Box, > { if addrs.is_empty() { return Err(WsError::Io(std::io::Error::new( std::io::ErrorKind::NotFound, "no addresses for websocket endpoint", - ))); + )) + .into()); } let mut last_err = None; @@ -101,12 +103,14 @@ async fn connect_ws_to_addrs( } } - Err(last_err.unwrap_or_else(|| { - WsError::Io(std::io::Error::new( - std::io::ErrorKind::NotConnected, - "no tcp candidate", - )) - })) + Err(last_err + .unwrap_or_else(|| { + WsError::Io(std::io::Error::new( + std::io::ErrorKind::NotConnected, + "no tcp candidate", + )) + }) + .into()) } async fn connect_ws_prefer_ipv4( @@ -116,7 +120,7 @@ async fn connect_ws_prefer_ipv4( WsStream, tokio_tungstenite::tungstenite::handshake::client::Response, ), - WsError, + Box, > { let port = default_port_for_request(&request)?; let host = request.uri().host().unwrap_or("").to_string(); @@ -869,13 +873,17 @@ mod tests { let request = "https://localhost/path".into_client_request().unwrap(); let explicit_port = "https://localhost:443/path".into_client_request().unwrap(); assert!(matches!( - default_port_for_request(&request), + default_port_for_request(&request) + .as_ref() + .map_err(|error| error.as_ref()), Err(WsError::Url( tokio_tungstenite::tungstenite::error::UrlError::UnsupportedUrlScheme )) )); assert!(matches!( - default_port_for_request(&explicit_port), + default_port_for_request(&explicit_port) + .as_ref() + .map_err(|error| error.as_ref()), Err(WsError::Url( tokio_tungstenite::tungstenite::error::UrlError::UnsupportedUrlScheme )) @@ -943,7 +951,7 @@ mod tests { let request = "ws://localhost/path".into_client_request().unwrap(); let result = connect_ws_to_addrs(request, Vec::new()).await; assert!( - matches!(result, Err(WsError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound) + matches!(result.as_ref().map_err(|error| error.as_ref()), Err(WsError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound) ); } diff --git a/openless-all/app/crates/openless-core/src/cloud_sync.rs b/openless-all/app/crates/openless-core/src/cloud_sync.rs index b6d9eb264..7b1865671 100644 --- a/openless-all/app/crates/openless-core/src/cloud_sync.rs +++ b/openless-all/app/crates/openless-core/src/cloud_sync.rs @@ -162,7 +162,7 @@ impl CloudSyncService { .parse::() .is_ok_and(|ip| ip.is_loopback()) }); - if (!matches!(url.scheme(), "https") && !(url.scheme() == "http" && loopback)) + if !(matches!(url.scheme(), "https") || url.scheme() == "http" && loopback) || !url.username().is_empty() || url.password().is_some() { diff --git a/openless-all/app/crates/openless-core/src/domains.rs b/openless-all/app/crates/openless-core/src/domains.rs index e8d9ee790..45e1060ee 100644 --- a/openless-all/app/crates/openless-core/src/domains.rs +++ b/openless-all/app/crates/openless-core/src/domains.rs @@ -282,7 +282,7 @@ pub trait LocalAsrApi: Send + Sync { fn delete_model(&self, target: LocalAsrTarget) -> BoxFuture<'static, Result<(), BackendError>>; fn cleanup_incomplete( &self, - target: LocalAsrTarget, + _target: LocalAsrTarget, ) -> BoxFuture<'static, Result<(), BackendError>> { unsupported("local ASR incomplete download cleanup") } diff --git a/openless-all/app/crates/openless-core/src/external_audio.rs b/openless-all/app/crates/openless-core/src/external_audio.rs index 24d0cbf8d..977ba1d44 100644 --- a/openless-all/app/crates/openless-core/src/external_audio.rs +++ b/openless-all/app/crates/openless-core/src/external_audio.rs @@ -299,8 +299,10 @@ mod tests { std::env::temp_dir().join(format!("openless-remote-archive-{}", uuid::Uuid::new_v4())); let recorder = ExternalAudioRecorder::with_recordings_directory(directory.clone()); let id = SessionId::new(); - let mut context = DictationContext::default(); - context.audio_source = DictationAudioSource::External; + let mut context = DictationContext { + audio_source: DictationAudioSource::External, + ..DictationContext::default() + }; context.recording.archive_enabled = true; let consumer = Arc::new(RecordingConsumer::default()); let recording = recorder @@ -356,8 +358,10 @@ mod tests { } let recorder = ExternalAudioRecorder::with_recordings_directory(directory.clone()); let id = SessionId::new(); - let mut context = DictationContext::default(); - context.audio_source = DictationAudioSource::External; + let mut context = DictationContext { + audio_source: DictationAudioSource::External, + ..DictationContext::default() + }; context.recording.archive_enabled = enabled; let consumer = Arc::new(RecordingConsumer::default()); let recording = recorder @@ -389,8 +393,10 @@ mod tests { std::fs::create_dir_all(&directory).unwrap(); std::fs::write(directory.join("user.wav"), b"keep").unwrap(); let recorder = ExternalAudioRecorder::with_recordings_directory(directory.clone()); - let mut context = DictationContext::default(); - context.audio_source = DictationAudioSource::External; + let mut context = DictationContext { + audio_source: DictationAudioSource::External, + ..DictationContext::default() + }; context.recording.archive_enabled = true; context.recording.max_entries = Some(2); for _ in 0..4 { diff --git a/openless-all/app/crates/openless-core/src/external_audio/archive.rs b/openless-all/app/crates/openless-core/src/external_audio/archive.rs index 14a786195..0f6681204 100644 --- a/openless-all/app/crates/openless-core/src/external_audio/archive.rs +++ b/openless-all/app/crates/openless-core/src/external_audio/archive.rs @@ -50,7 +50,7 @@ impl ExternalRecordingArchive { entries.push((path, modified)); } } - entries.sort_by(|left, right| right.1.cmp(&left.1)); + entries.sort_by_key(|entry| std::cmp::Reverse(entry.1)); let cap = plan .max_entries .map(|count| (count as usize).clamp(1, crate::history::HISTORY_CAP)) diff --git a/openless-all/app/crates/openless-core/src/provider_service.rs b/openless-all/app/crates/openless-core/src/provider_service.rs index b6e03a2c4..87c246c6a 100644 --- a/openless-all/app/crates/openless-core/src/provider_service.rs +++ b/openless-all/app/crates/openless-core/src/provider_service.rs @@ -552,8 +552,8 @@ fn validate_provider_endpoint(endpoint: &str, allow_websocket: bool) -> Result<( let url = url::Url::parse(endpoint).map_err(|_| invalid_request("provider endpoint is invalid"))?; if url.host_str().is_none() - || !matches!(url.scheme(), "http" | "https") - && !(allow_websocket && matches!(url.scheme(), "ws" | "wss")) + || !(matches!(url.scheme(), "http" | "https") + || allow_websocket && matches!(url.scheme(), "ws" | "wss")) { return Err(invalid_request("provider endpoint is invalid")); } From 00e3b868da3af89a679f00039643658684a65c1a Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Sun, 20 Sep 2026 18:05:02 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix(core):=20=E9=80=9A=E8=BF=87=E5=AE=BF?= =?UTF-8?q?=E4=B8=BB=E8=B0=83=E5=BA=A6=E5=99=A8=E5=90=AF=E5=8A=A8=E7=BC=93?= =?UTF-8?q?=E5=86=B2=E8=BD=AC=E5=86=99=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/crates/openless-core/src/api.rs | 1 + .../openless-core/src/dictation_engine.rs | 56 +++++++++++++++++-- .../app/crates/openless-core/src/ports.rs | 7 ++- 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index d75302e3c..aa9b457b1 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -2542,6 +2542,7 @@ impl OpenLessBackend { &self.deps.task_spawner, Arc::clone(&resources), Arc::clone(&self.deps.dictation_engine).start_transcription_with_progress( + Arc::clone(&self.deps.task_spawner), session_id, Arc::clone(&context), Arc::clone(&partials) as Arc, diff --git a/openless-all/app/crates/openless-core/src/dictation_engine.rs b/openless-all/app/crates/openless-core/src/dictation_engine.rs index 4e081370e..d72836062 100644 --- a/openless-all/app/crates/openless-core/src/dictation_engine.rs +++ b/openless-all/app/crates/openless-core/src/dictation_engine.rs @@ -1011,6 +1011,7 @@ pub(crate) fn buffered_transcription_session( context: Arc, partials: Arc, progress: Arc, + task_spawner: Arc, ) -> Arc { let partials = if context.recording.transcribe_after_stop { Arc::new(DiscardTextStream) as Arc @@ -1021,7 +1022,7 @@ pub(crate) fn buffered_transcription_session( prepared, partials, progress, )); if !context.recording.transcribe_after_stop { - buffered.attach_in_background(); + buffered.attach_in_background(task_spawner); } buffered } @@ -1149,13 +1150,13 @@ impl BufferedTranscriptionSession { }) } - fn attach_in_background(&self) { + fn attach_in_background(&self, task_spawner: Arc) { let attaching = self.attach(); - tokio::spawn(async move { + task_spawner.spawn(Box::pin(async move { if let Err(error) = attaching.await { log::warn!("provider-only transcription startup failed: {error}"); } - }); + })); } fn prepared(&self) -> Arc { @@ -1909,6 +1910,53 @@ mod tests { } } + #[tokio::test] + async fn provider_only_capture_uses_host_spawner_and_stable_mode_defers_start() { + #[derive(Default)] + struct QueuedSpawner(Mutex>>); + impl crate::TaskSpawner for QueuedSpawner { + fn spawn(&self, task: BoxFuture<'static, ()>) { + self.0.lock().unwrap().push(task); + } + } + for stable in [false, true] { + let fixture = fixture_engine( + false, + Ok(crate::ports::PolishOutput::text("unused")), + None, + None, + ); + let spawner = Arc::new(QueuedSpawner::default()); + let mut context = DictationContext::default(); + context.recording.transcribe_after_stop = stable; + let session = Arc::new(fixture.engine) + .start_transcription_with_progress( + spawner.clone(), + SessionId::new(), + Arc::new(context), + Arc::new(DiscardTextStream), + Arc::new(NoopRecordingProgress), + ) + .await + .unwrap(); + session.consume_pcm_chunk(&[1, 0, 2, 0]); + assert_eq!(fixture.transcription_starts.load(Ordering::SeqCst), 0); + assert!(fixture.pcm.lock().unwrap().is_empty()); + let tasks = std::mem::take(&mut *spawner.0.lock().unwrap()); + assert_eq!(tasks.len(), usize::from(!stable)); + for task in tasks { + task.await; + } + if !stable { + assert_eq!(fixture.transcription_starts.load(Ordering::SeqCst), 1); + assert_eq!(*fixture.pcm.lock().unwrap(), vec![1, 0, 2, 0]); + } + assert_eq!(session.finish().await.unwrap().text, "raw text"); + assert_eq!(fixture.transcription_starts.load(Ordering::SeqCst), 1); + assert_eq!(*fixture.pcm.lock().unwrap(), vec![1, 0, 2, 0]); + } + } + #[tokio::test] async fn pipeline_streams_pcm_progress_and_terminal_deltas() { let fixture = fixture_engine( diff --git a/openless-all/app/crates/openless-core/src/ports.rs b/openless-all/app/crates/openless-core/src/ports.rs index 0ba837b9d..7a78e7ed8 100644 --- a/openless-all/app/crates/openless-core/src/ports.rs +++ b/openless-all/app/crates/openless-core/src/ports.rs @@ -277,6 +277,7 @@ pub trait DictationEngine: Send + Sync + 'static { /// It uses the same buffered lifecycle as normal voice capture. fn start_transcription_with_progress( self: Arc, + task_spawner: Arc, session_id: SessionId, context: Arc, partials: Arc, @@ -286,7 +287,11 @@ pub trait DictationEngine: Send + Sync + 'static { Box::pin(async move { let prepared = preparation.await?; Ok(crate::dictation_engine::buffered_transcription_session( - prepared, context, partials, progress, + prepared, + context, + partials, + progress, + task_spawner, )) }) }