From 6153cd2826aaa9ed724bb45591b3c44b6b71c508 Mon Sep 17 00:00:00 2001 From: mkbk Date: Thu, 18 Jun 2026 16:19:30 +0800 Subject: [PATCH] =?UTF-8?q?[=E5=BC=82=E6=AD=A5=E6=92=AD=E6=8A=A5=E6=89=93?= =?UTF-8?q?=E6=96=AD]=EF=BC=9A=E5=AE=8C=E6=88=90=E6=92=AD=E6=94=BE?= =?UTF-8?q?=E4=B8=AD=E9=BA=A6=E5=85=8B=E9=A3=8E=E7=9B=91=E5=90=AC=E5=92=8C?= =?UTF-8?q?=E9=9F=B3=E8=89=B2=E9=9A=94=E7=A6=BB=EF=BC=8C=E5=8C=85=E5=90=AB?= =?UTF-8?q?=E5=90=8E=E5=8F=B0=E7=9B=91=E5=90=AC=E3=80=81=E5=8A=A9=E6=89=8B?= =?UTF-8?q?=E5=9B=9E=E6=94=BE=E6=8A=91=E5=88=B6=E5=92=8C=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E9=9F=B3=E8=89=B2=E6=89=93=E6=96=AD=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 5 + README.md | 7 +- .../design.md | 12 + .../proposal.md | 10 + .../specs/voice-pet-pipeline/spec.md | 20 ++ .../tasks.md | 8 + src/owner_voice_pet/assistant_pipeline.py | 102 +++----- src/owner_voice_pet/barge_in.py | 232 ++++++++++++++++++ src/owner_voice_pet/cli.py | 5 + src/owner_voice_pet/config.py | 43 ++++ src/owner_voice_pet/protocols.py | 1 + src/owner_voice_pet/transport.py | 10 + tests/test_barge_in.py | 132 ++++++++++ tests/test_live_runtime.py | 40 ++- tests/test_models_config.py | 13 + tests/test_transport.py | 15 ++ 16 files changed, 584 insertions(+), 71 deletions(-) create mode 100644 src/owner_voice_pet/barge_in.py create mode 100644 tests/test_barge_in.py diff --git a/.env.example b/.env.example index e2d7393..1bdc9cb 100644 --- a/.env.example +++ b/.env.example @@ -45,6 +45,11 @@ OWNER_FOLLOWUP_LISTEN_TIMEOUT_MS=3000 OWNER_BARGE_IN_ENABLED=1 OWNER_BARGE_IN_MIN_SPEECH_MS=250 OWNER_BARGE_IN_ECHO_GUARD_MS=500 +OWNER_BARGE_IN_SPEAKER_GATE_ENABLED=1 +OWNER_BARGE_IN_USER_SIMILARITY_THRESHOLD=0.62 +OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD=0.72 +OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20 +OWNER_BARGE_IN_CHUNK_MS=30 OWNER_END_CHIME_ENABLED=1 OWNER_END_CHIME_FILE=assets/sounds/codex-notification.wav OWNER_END_CHIME_FREQUENCY_HZ=880 diff --git a/README.md b/README.md index 47c772c..063337a 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,11 @@ OWNER_FOLLOWUP_LISTEN_TIMEOUT_MS=3000 OWNER_BARGE_IN_ENABLED=1 OWNER_BARGE_IN_MIN_SPEECH_MS=250 OWNER_BARGE_IN_ECHO_GUARD_MS=500 +OWNER_BARGE_IN_SPEAKER_GATE_ENABLED=1 +OWNER_BARGE_IN_USER_SIMILARITY_THRESHOLD=0.62 +OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD=0.72 +OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20 +OWNER_BARGE_IN_CHUNK_MS=30 OWNER_END_CHIME_ENABLED=1 OWNER_END_CHIME_FILE=assets/sounds/codex-notification.wav OWNER_END_CHIME_FREQUENCY_HZ=880 @@ -86,7 +91,7 @@ OWNER_END_CHIME_DURATION_MS=140 `OWNER_CONTINUOUS_DIALOG_ENABLED=1` 表示每轮回复播放后会自动判断是否继续对话。若助手回复里明显在问用户、要求补充信息或让用户选择，终端会输出 `继续对话：3秒内可直接回答`，这 3 秒内可以不用再说“小杰小杰”。若助手只是完成回答、报错、拒绝或判断不确定，就直接恢复待机。`OWNER_CONTINUATION_DECISION_PROVIDER=hybrid` 表示先用本地规则判断，规则不确定时才调用云端 LLM 做小分类；低于 `OWNER_CONTINUATION_CONFIDENCE_THRESHOLD=0.65` 的结果按待机处理。 -`OWNER_BARGE_IN_ENABLED=1` 表示播报中允许打断。播放开始后的 `OWNER_BARGE_IN_ECHO_GUARD_MS=500` 毫秒内忽略麦克风输入，之后如果检测到至少 `OWNER_BARGE_IN_MIN_SPEECH_MS=250` 毫秒有效用户语音，并且 realtime STT 给出有效 partial，就停止剩余播报。上下文只记录已经完整播出的 assistant 句子，未播出的内容不会写入临时历史。 +`OWNER_BARGE_IN_ENABLED=1` 表示播报中允许打断。播放回复时会启动后台麦克风监听，不再等每个播放 chunk 结束后才检查输入；`OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20` 控制监听间隔，`OWNER_BARGE_IN_CHUNK_MS=30` 控制播放停止粒度。播放开始后的 `OWNER_BARGE_IN_ECHO_GUARD_MS=500` 毫秒内忽略麦克风输入，之后如果检测到至少 `OWNER_BARGE_IN_MIN_SPEECH_MS=250` 毫秒有效用户语音，并且 realtime STT 给出有效 partial，就停止剩余播报。`OWNER_BARGE_IN_SPEAKER_GATE_ENABLED=1` 会同时建立本次会话用户临时音色画像和当前助手回放音色画像，默认要求用户相似度达到 `OWNER_BARGE_IN_USER_SIMILARITY_THRESHOLD=0.62`，并拒绝相似度高于 `OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD=0.72` 的助手回放音色，避免 AI 自己的声音触发打断或实时字幕。音色画像只在进程内使用，不写文件、不发送给 LLM。上下文只记录已经完整播出的 assistant 句子，未播出的内容不会写入临时历史。 `OWNER_END_CHIME_ENABLED=1` 表示对话自然结束或追问超时恢复待机前会播放一声项目内置提示音，默认文件是 `assets/sounds/codex-notification.wav`。提示音不走 TTS，也不会写入上下文；如果 `OWNER_END_CHIME_FILE` 指向的文件缺失，会回退到本地合成短音，`OWNER_END_CHIME_FREQUENCY_HZ` 和 `OWNER_END_CHIME_DURATION_MS` 只影响这个回退音。设置 `OWNER_END_CHIME_ENABLED=0` 可以关闭。 diff --git a/openspec/changes/separate-wake-and-realtime-transcript/design.md b/openspec/changes/separate-wake-and-realtime-transcript/design.md index e625ea5..ac45dd5 100644 --- a/openspec/changes/separate-wake-and-realtime-transcript/design.md +++ b/openspec/changes/separate-wake-and-realtime-transcript/design.md @@ -326,6 +326,18 @@ Follow-up 捕获复用现有 capture、实时字幕、final STT、LLM、TTS 流 4. 已完整播放的句子文本才写入 process-local assistant context；未播出的句子和后续未播放 token 不写入上下文。 5. 打断后的用户输入进入免唤醒 follow-up 捕获路径，保留实时字幕和 final STT，最终仍只有 final transcript 进入 LLM。 +## Async Barge-in Speaker Gate Revision + +旧打断路径由播放 chunk 回调驱动，每写完一段音频后才读取麦克风，不满足“播放同时监听”的实时性要求，也无法稳定排除助手自己的回放。修正版边界如下： + +1. `AsyncBargeInMonitor` 在可打断 TTS 回复开始时启动后台线程，独立调用 `AudioTransport.read_frames()`，默认每 20 ms 检查一次，不由播放循环触发麦克风处理。 +2. `AudioTransport.play_pcm_chunks()` 只接收 `should_stop` 停止信号；播放线程在 20-40 ms chunk 边界检查该信号并尽快停止剩余播报。 +3. `BargeInSpeakerGate` 从最近一次用户正式输入建立本次进程内临时用户音色画像，从当前 TTS PCM 建立助手回放音色画像。 +4. 播放期间候选麦克风帧必须通过 VAD，且不能匹配助手回放画像；若用户画像已存在，还必须匹配用户画像；若用户画像不足，降级为“非助手音色 + 有效 realtime partial”。 +5. 播放期间 realtime STT 只作为后台打断确认信号，不发 `transcript_partial` 事件，避免把助手回放显示成用户实时字幕。 +6. 监听线程确认打断后把已确认用户帧写入 `_pending_capture_frames`，后续 follow-up capture 直接续接，减少用户抢话首字丢失。 +7. 用户画像、助手画像、候选帧均只保存在进程内，不写入磁盘、不发送给 LLM、不跨进程复用。 + ## Migration No database migration. Users should run: diff --git a/openspec/changes/separate-wake-and-realtime-transcript/proposal.md b/openspec/changes/separate-wake-and-realtime-transcript/proposal.md index 1982455..a642614 100644 --- a/openspec/changes/separate-wake-and-realtime-transcript/proposal.md +++ b/openspec/changes/separate-wake-and-realtime-transcript/proposal.md @@ -461,6 +461,14 @@ standby - [ ] 15.3 补充 recorder finish 和回归测试；前置条件：15.2 完成；验收标准：持续有语音但 partial 不再推进时，segment end_reason 为 `partial_transcript_idle`，final STT 仍执行且 partial 不进上下文；测试要点：live runtime 单测；优先级：P0；预计：45 分钟。 - [ ] 15.4 执行门禁并提交“实时字幕端点”模块；前置条件：15.1 至 15.3 完成；验收标准：compileall、unittest、simulate-live、security-check、OpenSpec strict、git diff check 通过后中文提交；优先级：P0；预计：45 分钟。 +### 17. 异步播报打断与音色隔离 + +- [ ] 17.1 更新 OpenSpec 和 README 描述异步监听与音色隔离；前置条件：用户确认当前打断仍未真正异步且会被助手回放干扰；验收标准：proposal/design/spec/tasks/README 明确后台监听线程、停止信号、用户临时音色画像、助手回放音色抑制和非持久化边界；测试要点：OpenSpec strict；优先级：P0；预计：45 分钟。 +- [ ] 17.2 扩展配置和 Transport 停止接口；前置条件：17.1 完成；验收标准：新增 `OWNER_BARGE_IN_SPEAKER_GATE_ENABLED`、`OWNER_BARGE_IN_USER_SIMILARITY_THRESHOLD`、`OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD`、`OWNER_BARGE_IN_LISTEN_INTERVAL_MS`、`OWNER_BARGE_IN_CHUNK_MS`，`play_pcm_chunks()` 支持播放侧只检查 stop signal；测试要点：配置默认值、校验和 chunk 停止测试；优先级：P0；预计：45 分钟。 +- [ ] 17.3 实现 `AsyncBargeInMonitor` 和 `BargeInSpeakerGate`；前置条件：17.2 完成；验收标准：播放时后台线程独立读取麦克风，排除助手回放音色，确认用户音色后设置 stop event，并把已确认用户帧接入后续 capture；测试要点：异步读取、助手音色不触发、用户音色触发、画像不足降级；优先级：P0；预计：60 分钟。 +- [ ] 17.4 接入 `VoiceAssistantPipeline` 播放路径并补充文档；前置条件：17.3 完成；验收标准：TTS 回复使用异步打断，ACK 和结束提示音不打断，播放期间不输出实时字幕，README 说明新参数和调参边界；测试要点：打断事件顺序、上下文只写已播文本、pending capture 不丢首字；优先级：P0；预计：60 分钟。 +- [ ] 17.5 执行门禁并提交“异步播报打断”模块；前置条件：17.1 至 17.4 完成；验收标准：compileall、unittest、simulate-live、real-live-check --no-playback、security-check、model-check、device-check、OpenSpec strict 通过后提交；优先级：P0；预计：60 分钟。 + ## Spec Deltas ### 新增能力 @@ -487,6 +495,7 @@ standby 16. `Realtime transcript idle endpoint`：新增已有实时字幕后 1.5 秒无新文字即结束录音的端点要求，避免用户已停说但 capture 继续等待 VAD/音色端点。 17. `Automatic continuous dialog decision`：新增回复后自动判断继续/待机能力，使用规则优先、LLM 小分类兜底，不确定时默认回到待机。 18. `Barge-in playback interruption`：新增播报中用户有效说话打断能力，使用 echo guard、最短语音时长和 realtime partial 共同确认，避免播放回声误触发。 +19. `Async barge-in speaker gate`：强化播报打断为后台异步监听，新增用户临时音色画像、助手回放音色抑制、播放停止信号和非持久化隐私要求。 ### 删除项 @@ -513,6 +522,7 @@ standby 13. M13：`real-live-check` 计时输出完成后提交，使后续排查可以直接看到每段耗时。 14. M14：实时字幕停滞端点完成后提交，用户真实 `run-live` 若已有 partial 后 1.5 秒无新文字，应快速进入 final STT。 15. M15：自动持续对话判断与播报打断完成后提交，用户真实 `run-live` 应能在助手反问时免唤醒继续回答，在助手已完成回答时自动恢复待机。 +16. M16：异步播报打断与音色隔离完成后提交，用户真实 `run-live` 应能在助手播报时直接说话打断，且助手自己的回放不触发打断或实时字幕。 估时： diff --git a/openspec/changes/separate-wake-and-realtime-transcript/specs/voice-pet-pipeline/spec.md b/openspec/changes/separate-wake-and-realtime-transcript/specs/voice-pet-pipeline/spec.md index dfe4989..f0d0304 100644 --- a/openspec/changes/separate-wake-and-realtime-transcript/specs/voice-pet-pipeline/spec.md +++ b/openspec/changes/separate-wake-and-realtime-transcript/specs/voice-pet-pipeline/spec.md @@ -170,6 +170,21 @@ The live runtime SHALL support a first-version barge-in path that can stop assis - **WHEN** TTS playback starts and microphone input appears before `OWNER_BARGE_IN_ECHO_GUARD_MS` - **THEN** the runtime SHALL NOT treat that input as a user barge-in +#### Scenario: Playback is monitored asynchronously +- **WHEN** an interruptible assistant reply is playing +- **THEN** the runtime SHALL listen to microphone frames from a background monitor rather than waiting for playback chunks to invoke microphone processing +- **AND** the playback transport SHALL stop remaining chunks when the monitor raises the shared stop signal + +#### Scenario: Assistant playback voice is rejected +- **WHEN** microphone input during playback matches the current assistant playback timbre profile +- **THEN** the runtime SHALL NOT emit `barge_in_detected` +- **AND** it SHALL NOT display realtime transcript text for that assistant playback echo + +#### Scenario: User voice interrupts playback +- **WHEN** microphone input after the echo guard matches the temporary process-local user timbre profile, does not match the assistant playback profile, passes VAD, and produces a valid realtime partial +- **THEN** the runtime SHALL emit `barge_in_detected` +- **AND** it SHALL stop playback promptly and process the buffered user frames as the next no-wake input + #### Scenario: Valid user speech interrupts playback - **WHEN** TTS playback has passed the echo guard, VAD observes at least `OWNER_BARGE_IN_MIN_SPEECH_MS` of user speech, and realtime STT produces a valid partial transcript - **THEN** the runtime SHALL emit `barge_in_detected` @@ -181,6 +196,11 @@ The live runtime SHALL support a first-version barge-in path that can stop assis - **THEN** the process-local conversation context SHALL append only the assistant text that had been fully spoken before interruption - **AND** unspoken assistant text SHALL NOT be appended to the session history +#### Scenario: Barge-in speaker profiles are temporary +- **WHEN** the runtime builds user or assistant timbre profiles for interruption gating +- **THEN** those profiles SHALL remain in process memory only +- **AND** they SHALL NOT be written to disk, sent to the cloud LLM, or reused after process exit + ### Requirement: Real provider live fixture acceptance The system SHALL support scripted full-chain acceptance that drives the live pipeline with generated microphone PCM while using real local speech providers, the configured cloud LLM, and local TTS/playback capability. diff --git a/openspec/changes/separate-wake-and-realtime-transcript/tasks.md b/openspec/changes/separate-wake-and-realtime-transcript/tasks.md index 4065e86..33f6a18 100644 --- a/openspec/changes/separate-wake-and-realtime-transcript/tasks.md +++ b/openspec/changes/separate-wake-and-realtime-transcript/tasks.md @@ -126,3 +126,11 @@ - [x] 16.3 重构 `VoiceAssistantPipeline/TurnController` 支持自动 follow-up；前置条件：16.2 完成；验收标准：回复后先判定 continue/standby，continue 时播放完进入 `followup_listening`，3 秒内用户直接回答无需 wake/ACK，超时恢复 standby；测试要点：连续对话、follow-up timeout、上下文只写 final STT；优先级：P0；预计：60 分钟。 - [x] 16.4 实现可打断播放路径；前置条件：16.3 完成；验收标准：TTS 播放前 500 ms 忽略回声，之后检测到至少 250 ms 有效用户语音且 realtime partial 有效时触发 `barge_in_detected -> playback_interrupted`，上下文只写已播出的 assistant 文本；测试要点：guard window 不打断、有效输入打断、未播文本不进上下文；优先级：P0；预计：60 分钟。 - [x] 16.5 执行门禁并提交“连续对话判断”模块；前置条件：16.1 至 16.4 完成；验收标准：compileall、unittest、simulate-live --turns 3、real-live-check --turns 2 --no-playback、security-check、model-check、device-check、OpenSpec strict、git diff check 通过后提交；优先级：P0；预计：60 分钟。 + +## 17. 异步播报打断与音色隔离 + +- [x] 17.1 更新 OpenSpec 和 README 描述异步监听与音色隔离；前置条件：用户确认当前打断仍未真正异步且会被助手回放干扰；验收标准：proposal/design/spec/tasks/README 明确后台监听线程、停止信号、用户临时音色画像、助手回放音色抑制和非持久化边界；测试要点：OpenSpec strict；优先级：P0；预计：45 分钟。 +- [x] 17.2 扩展配置和 Transport 停止接口；前置条件：17.1 完成；验收标准：新增 `OWNER_BARGE_IN_SPEAKER_GATE_ENABLED`、`OWNER_BARGE_IN_USER_SIMILARITY_THRESHOLD`、`OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD`、`OWNER_BARGE_IN_LISTEN_INTERVAL_MS`、`OWNER_BARGE_IN_CHUNK_MS`，`play_pcm_chunks()` 支持播放侧只检查 stop signal；测试要点：配置默认值、校验和 chunk 停止测试；优先级：P0；预计：45 分钟。 +- [x] 17.3 实现 `AsyncBargeInMonitor` 和 `BargeInSpeakerGate`；前置条件：17.2 完成；验收标准：播放时后台线程独立读取麦克风，排除助手回放音色，确认用户音色后设置 stop event，并把已确认用户帧接入后续 capture；测试要点：异步读取、助手音色不触发、用户音色触发、画像不足降级；优先级：P0；预计：60 分钟。 +- [x] 17.4 接入 `VoiceAssistantPipeline` 播放路径并补充文档；前置条件：17.3 完成；验收标准：TTS 回复使用异步打断，ACK 和结束提示音不打断，播放期间不输出实时字幕，README 说明新参数和调参边界；测试要点：打断事件顺序、上下文只写已播文本、pending capture 不丢首字；优先级：P0；预计：60 分钟。 +- [x] 17.5 执行门禁并提交“异步播报打断”模块；前置条件：17.1 至 17.4 完成；验收标准：compileall、unittest、simulate-live、real-live-check --no-playback、security-check、model-check、device-check、OpenSpec strict 通过后提交；优先级：P0；预计：60 分钟。 diff --git a/src/owner_voice_pet/assistant_pipeline.py b/src/owner_voice_pet/assistant_pipeline.py index 5c0accd..74945d6 100644 --- a/src/owner_voice_pet/assistant_pipeline.py +++ b/src/owner_voice_pet/assistant_pipeline.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, field from typing import Protocol from .audio_preprocess import NoopAudioPreprocessor +from .barge_in import AsyncBargeInMonitor, BargeInSpeakerGate, ensure_interruptible_pcm from .config import AppConfig from .continuation import ContinuationDecision, ContinuationDecisionProvider, build_continuation_decider from .conversation import ConversationContext @@ -41,7 +42,6 @@ from .protocols import ( AudioTransport, LlmProvider, RealtimeSttProvider, - RealtimeTranscriptSession, SttProvider, TtsProvider, WakeWordProvider, @@ -124,6 +124,12 @@ class TurnController: self._pending_capture_frames: list[AudioFrame] = [] self._cached_ack_text: str | None = None self._cached_ack_segment: AudioSegment | None = None + self._barge_in_gate = BargeInSpeakerGate( + enabled=config.barge_in_speaker_gate_enabled, + user_similarity_threshold=config.barge_in_user_similarity_threshold, + assistant_reject_threshold=config.barge_in_assistant_reject_threshold, + min_rms=config.speaker_min_rms, + ) def run_turn(self, turn_id: int) -> TurnResult: self._states = [] @@ -175,6 +181,7 @@ class TurnController: ) if isinstance(user_segment, ProviderError): return user_segment + self._barge_in_gate.remember_user_segment(user_segment) self._event(STT_STARTED, PipelineState.TRANSCRIBING, "转写中：正在识别问题", turn_id=turn_id) transcript = self.stt.transcribe(user_segment) user_text = transcript.normalized_text @@ -509,89 +516,54 @@ class TurnController: return SpeakResult("") self._event(TTS_STARTED, PipelineState.SPEAKING, "播放中：正在播报回复", turn_id=turn_id) segment = self.tts.synthesize(spoken_sentence) - if not self._can_interrupt_playback(segment): + interruptible_segment = ensure_interruptible_pcm(segment) + if not self._can_interrupt_playback(interruptible_segment): playback = self.transport.play_pcm(segment) if playback.error: raise playback.error self._event(PLAYBACK_FINISHED, PipelineState.SPEAKING, "", turn_id=turn_id) self._drain_input_after_playback() return SpeakResult(spoken_sentence) - if self._play_interruptible(segment, turn_id=turn_id): + if self._play_interruptible(interruptible_segment, turn_id=turn_id): return SpeakResult("", interrupted=True) self._event(PLAYBACK_FINISHED, PipelineState.SPEAKING, "", turn_id=turn_id) self._drain_input_after_playback() return SpeakResult(spoken_sentence) - def _can_interrupt_playback(self, segment: AudioSegment) -> bool: + def _can_interrupt_playback(self, segment: AudioSegment | None) -> bool: return ( - self.config.barge_in_enabled + segment is not None + and self.config.barge_in_enabled and self.realtime_stt is not None and segment.duration_ms > self.config.barge_in_echo_guard_ms - and not segment.metadata.get("format") ) def _play_interruptible(self, segment: AudioSegment, *, turn_id: int) -> bool: - guard_cleared = False - self.vad_recorder.provider.reset() - realtime_session = self._start_realtime_transcript() - speech_ms = 0 - partial_seen = False - pending_frames: list[AudioFrame] = [] - interrupted = False - - def after_chunk(_chunk: AudioSegment, elapsed_ms: int) -> bool: - nonlocal guard_cleared, speech_ms, partial_seen, interrupted - if elapsed_ms < self.config.barge_in_echo_guard_ms: - return False - if not guard_cleared: - self.transport.flush_input() - guard_cleared = True - return False - detected, speech_ms, partial_seen, new_frames = self._detect_barge_in( - realtime_session, - turn_id=turn_id, - speech_ms=speech_ms, - partial_seen=partial_seen, - ) - pending_frames.extend(new_frames) - if detected: - self._pending_capture_frames.extend(pending_frames) - self._event(BARGE_IN_DETECTED, PipelineState.INTERRUPTED, "检测到用户打断", turn_id=turn_id) - self._event(PLAYBACK_INTERRUPTED, PipelineState.INTERRUPTED, "播报已打断", turn_id=turn_id) - interrupted = True - return True - return False - - playback = self.transport.play_pcm_chunks(segment, chunk_ms=100, after_chunk=after_chunk) + monitor = AsyncBargeInMonitor( + transport=self.transport, + vad_provider=self.vad_recorder.provider, + realtime_stt=self.realtime_stt, + speaker_gate=self._barge_in_gate, + assistant_profile=self._barge_in_gate.assistant_profile(segment), + echo_guard_ms=self.config.barge_in_echo_guard_ms, + min_speech_ms=self.config.barge_in_min_speech_ms, + listen_interval_ms=self.config.barge_in_listen_interval_ms, + ) + monitor.start() + playback = self.transport.play_pcm_chunks( + segment, + chunk_ms=self.config.barge_in_chunk_ms, + should_stop=monitor.should_stop_playback, + ) + monitor.stop() if playback.error: raise playback.error - if realtime_session is not None: - realtime_session.finish() - return interrupted - - def _detect_barge_in( - self, - realtime_session: RealtimeTranscriptSession | None, - *, - turn_id: int, - speech_ms: int, - partial_seen: bool, - ) -> tuple[bool, int, bool, list[AudioFrame]]: - frames = self.transport.read_frames(timeout_ms=0) - if not frames: - return False, speech_ms, partial_seen, [] - for frame in frames: - result = self.vad_recorder.provider.analyze(frame) - if result.is_speech: - speech_ms += int(frame.metadata.get("duration_ms", 20)) - if realtime_session is not None: - transcript = realtime_session.accept_frame(frame) - if transcript is not None and is_valid_transcript_text(transcript.normalized_text): - partial_seen = True - else: - speech_ms = 0 - detected = speech_ms >= self.config.barge_in_min_speech_ms and partial_seen - return detected, speech_ms, partial_seen, frames + if monitor.interrupted: + self._pending_capture_frames.extend(monitor.pending_frames()) + self._event(BARGE_IN_DETECTED, PipelineState.INTERRUPTED, "检测到用户打断", turn_id=turn_id) + self._event(PLAYBACK_INTERRUPTED, PipelineState.INTERRUPTED, "播报已打断", turn_id=turn_id) + return True + return False def _drain_input_after_playback(self) -> None: self.transport.flush_input() diff --git a/src/owner_voice_pet/barge_in.py b/src/owner_voice_pet/barge_in.py new file mode 100644 index 0000000..8ab3113 --- /dev/null +++ b/src/owner_voice_pet/barge_in.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import io +import subprocess +import tempfile +import threading +import time +import wave +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .models import AudioFrame, AudioSegment, ProviderError +from .protocols import AudioTransport, RealtimeSttProvider, RealtimeTranscriptSession +from .stt import is_valid_transcript_text +from .vad import cosine_similarity, extract_timbre_vector + + +_FILE_AUDIO_FORMATS = {"aiff", "wav", "mp3", "m4a", "aac"} + + +@dataclass(slots=True) +class TimbreProfile: + vector: tuple[float, ...] | None = None + speaker_id: str | None = None + + @property + def ready(self) -> bool: + return self.vector is not None or self.speaker_id is not None + + +class BargeInSpeakerGate: + def __init__( + self, + *, + enabled: bool, + user_similarity_threshold: float, + assistant_reject_threshold: float, + min_rms: float, + ) -> None: + self.enabled = enabled + self.user_similarity_threshold = user_similarity_threshold + self.assistant_reject_threshold = assistant_reject_threshold + self.min_rms = min_rms + self._user_profile = TimbreProfile() + + def remember_user_segment(self, segment: AudioSegment) -> None: + profile = self._profile_from_segment(segment) + if profile.ready: + self._user_profile = profile + + def assistant_profile(self, segment: AudioSegment) -> TimbreProfile: + return self._profile_from_segment(segment) + + def accepts_candidate(self, frame: AudioFrame, assistant_profile: TimbreProfile) -> bool: + if not self.enabled: + return True + candidate = self._profile_from_frame(frame) + if not candidate.ready: + return False + if self._matches(candidate, assistant_profile, self.assistant_reject_threshold): + return False + if self._user_profile.ready: + return self._matches(candidate, self._user_profile, self.user_similarity_threshold) + return True + + def _profile_from_segment(self, segment: AudioSegment) -> TimbreProfile: + frame = AudioFrame( + segment.pcm, + segment.sample_rate, + segment.channels, + segment.start_time_ms, + 0, + segment.metadata, + ) + return self._profile_from_frame(frame) + + def _profile_from_frame(self, frame: AudioFrame) -> TimbreProfile: + speaker_id = frame.metadata.get("speaker_id") + vector = extract_timbre_vector(frame, min_rms=self.min_rms) + return TimbreProfile(vector=vector, speaker_id=str(speaker_id) if speaker_id is not None else None) + + def _matches(self, candidate: TimbreProfile, reference: TimbreProfile, threshold: float) -> bool: + if not reference.ready: + return False + if reference.speaker_id is not None: + return candidate.speaker_id is not None and candidate.speaker_id == reference.speaker_id + if candidate.vector is None or reference.vector is None: + return False + return cosine_similarity(candidate.vector, reference.vector) >= threshold + + +class AsyncBargeInMonitor: + def __init__( + self, + *, + transport: AudioTransport, + vad_provider: Any, + realtime_stt: RealtimeSttProvider | None, + speaker_gate: BargeInSpeakerGate, + assistant_profile: TimbreProfile, + echo_guard_ms: int, + min_speech_ms: int, + listen_interval_ms: int, + ) -> None: + self.transport = transport + self.vad_provider = vad_provider + self.realtime_stt = realtime_stt + self.speaker_gate = speaker_gate + self.assistant_profile = assistant_profile + self.echo_guard_ms = max(0, echo_guard_ms) + self.min_speech_ms = max(0, min_speech_ms) + self.listen_interval_ms = max(1, listen_interval_ms) + self.stop_event = threading.Event() + self._shutdown_event = threading.Event() + self._thread: threading.Thread | None = None + self._pending_frames: list[AudioFrame] = [] + self._lock = threading.Lock() + self._interrupted = False + self.error: ProviderError | None = None + + def start(self) -> None: + self.vad_provider.reset() + self._thread = threading.Thread(target=self._run, name="owner-voice-barge-in", daemon=True) + self._thread.start() + + def stop(self) -> None: + self._shutdown_event.set() + if self._thread is not None: + self._thread.join(timeout=1.0) + + def should_stop_playback(self) -> bool: + return self.stop_event.is_set() + + @property + def interrupted(self) -> bool: + return self._interrupted + + def pending_frames(self) -> list[AudioFrame]: + with self._lock: + return list(self._pending_frames) + + def _run(self) -> None: + realtime_session = self.realtime_stt.start_stream() if self.realtime_stt is not None else None + started_at = time.monotonic() + speech_ms = 0 + partial_seen = False + candidate_frames: list[AudioFrame] = [] + try: + while not self._shutdown_event.is_set() and not self.stop_event.is_set(): + frames = self.transport.read_frames(timeout_ms=self.listen_interval_ms) + elapsed_ms = int((time.monotonic() - started_at) * 1000) + if elapsed_ms < self.echo_guard_ms: + continue + if not frames: + time.sleep(self.listen_interval_ms / 1000) + continue + for frame in frames: + result = self.vad_provider.analyze(frame) + if not result.is_speech: + speech_ms = 0 + candidate_frames = [] + continue + if not self.speaker_gate.accepts_candidate(frame, self.assistant_profile): + speech_ms = 0 + candidate_frames = [] + continue + frame_ms = int(frame.metadata.get("duration_ms", 20)) + speech_ms += frame_ms + candidate_frames.append(frame) + if realtime_session is not None: + transcript = realtime_session.accept_frame(frame) + if transcript is not None and is_valid_transcript_text(transcript.normalized_text): + partial_seen = True + if speech_ms >= self.min_speech_ms and partial_seen: + with self._lock: + self._pending_frames = list(candidate_frames) + self._interrupted = True + self.stop_event.set() + return + except ProviderError as exc: + self.error = exc + finally: + if realtime_session is not None: + realtime_session.finish() + + +def ensure_interruptible_pcm(segment: AudioSegment) -> AudioSegment | None: + fmt = str(segment.metadata.get("format", "")).lower() + if not fmt: + return segment + if fmt not in _FILE_AUDIO_FORMATS: + return None + if fmt == "wav": + return _decode_wav_bytes(segment) + return _decode_with_afconvert(segment, fmt) + + +def _decode_wav_bytes(segment: AudioSegment) -> AudioSegment | None: + try: + with wave.open(io.BytesIO(segment.pcm), "rb") as handle: + sample_rate = handle.getframerate() + channels = handle.getnchannels() + sample_width = handle.getsampwidth() + if sample_width != 2: + return None + data = handle.readframes(handle.getnframes()) + duration_ms = int(handle.getnframes() / max(1, sample_rate) * 1000) + except wave.Error: + return None + metadata = dict(segment.metadata) + metadata.pop("format", None) + metadata["source_format"] = "wav" + return AudioSegment(data, sample_rate, channels, 0, max(20, duration_ms), metadata) + + +def _decode_with_afconvert(segment: AudioSegment, fmt: str) -> AudioSegment | None: + try: + with tempfile.TemporaryDirectory() as tmp: + input_path = Path(tmp) / f"input.{fmt}" + output_path = Path(tmp) / "output.wav" + input_path.write_bytes(segment.pcm) + subprocess.run( + ["afconvert", "-f", "WAVE", "-d", "LEI16", "-c", "1", str(input_path), str(output_path)], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + decoded = AudioSegment(output_path.read_bytes(), 16000, 1, 0, segment.duration_ms, segment.metadata) + return _decode_wav_bytes(decoded) + except (OSError, subprocess.CalledProcessError): + return None diff --git a/src/owner_voice_pet/cli.py b/src/owner_voice_pet/cli.py index 2b86bca..6952123 100644 --- a/src/owner_voice_pet/cli.py +++ b/src/owner_voice_pet/cli.py @@ -104,6 +104,11 @@ def main(argv: list[str] | None = None) -> int: "barge_in_enabled": config.barge_in_enabled, "barge_in_min_speech_ms": config.barge_in_min_speech_ms, "barge_in_echo_guard_ms": config.barge_in_echo_guard_ms, + "barge_in_speaker_gate_enabled": config.barge_in_speaker_gate_enabled, + "barge_in_user_similarity_threshold": config.barge_in_user_similarity_threshold, + "barge_in_assistant_reject_threshold": config.barge_in_assistant_reject_threshold, + "barge_in_listen_interval_ms": config.barge_in_listen_interval_ms, + "barge_in_chunk_ms": config.barge_in_chunk_ms, "end_chime_enabled": config.end_chime_enabled, "end_chime_file": str(config.end_chime_file), "end_chime_frequency_hz": config.end_chime_frequency_hz, diff --git a/src/owner_voice_pet/config.py b/src/owner_voice_pet/config.py index 1d48b3c..312e696 100644 --- a/src/owner_voice_pet/config.py +++ b/src/owner_voice_pet/config.py @@ -59,6 +59,11 @@ class AppConfig: barge_in_enabled: bool = True barge_in_min_speech_ms: int = 250 barge_in_echo_guard_ms: int = 500 + barge_in_speaker_gate_enabled: bool = True + barge_in_user_similarity_threshold: float = 0.62 + barge_in_assistant_reject_threshold: float = 0.72 + barge_in_listen_interval_ms: int = 20 + barge_in_chunk_ms: int = 30 end_chime_enabled: bool = True end_chime_file: Path = Path("assets/sounds/codex-notification.wav") end_chime_frequency_hz: int = 880 @@ -134,6 +139,16 @@ class AppConfig: barge_in_enabled=(get("BARGE_IN_ENABLED", "1") or "1").lower() not in {"0", "false", "no"}, barge_in_min_speech_ms=int(get("BARGE_IN_MIN_SPEECH_MS", "250") or "250"), barge_in_echo_guard_ms=int(get("BARGE_IN_ECHO_GUARD_MS", "500") or "500"), + barge_in_speaker_gate_enabled=(get("BARGE_IN_SPEAKER_GATE_ENABLED", "1") or "1").lower() + not in {"0", "false", "no"}, + barge_in_user_similarity_threshold=float( + get("BARGE_IN_USER_SIMILARITY_THRESHOLD", "0.62") or "0.62" + ), + barge_in_assistant_reject_threshold=float( + get("BARGE_IN_ASSISTANT_REJECT_THRESHOLD", "0.72") or "0.72" + ), + barge_in_listen_interval_ms=int(get("BARGE_IN_LISTEN_INTERVAL_MS", "20") or "20"), + barge_in_chunk_ms=int(get("BARGE_IN_CHUNK_MS", "30") or "30"), end_chime_enabled=(get("END_CHIME_ENABLED", "1") or "1").lower() not in {"0", "false", "no"}, end_chime_file=Path( get("END_CHIME_FILE", "assets/sounds/codex-notification.wav") @@ -405,6 +420,34 @@ class AppConfig: "startup", ) ) + for name, value in { + "OWNER_BARGE_IN_LISTEN_INTERVAL_MS": self.barge_in_listen_interval_ms, + "OWNER_BARGE_IN_CHUNK_MS": self.barge_in_chunk_ms, + }.items(): + if value <= 0: + errors.append( + ProviderError( + ErrorCode.CONFIG_MISSING_VALUE, + f"{name} must be positive", + False, + "config", + "startup", + ) + ) + for name, value in { + "OWNER_BARGE_IN_USER_SIMILARITY_THRESHOLD": self.barge_in_user_similarity_threshold, + "OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD": self.barge_in_assistant_reject_threshold, + }.items(): + if not 0 < value <= 1: + errors.append( + ProviderError( + ErrorCode.CONFIG_MISSING_VALUE, + f"{name} must be in (0, 1]", + False, + "config", + "startup", + ) + ) for name, value in { "OWNER_END_CHIME_FREQUENCY_HZ": self.end_chime_frequency_hz, "OWNER_END_CHIME_DURATION_MS": self.end_chime_duration_ms, diff --git a/src/owner_voice_pet/protocols.py b/src/owner_voice_pet/protocols.py index eba1888..c32114a 100644 --- a/src/owner_voice_pet/protocols.py +++ b/src/owner_voice_pet/protocols.py @@ -34,6 +34,7 @@ class AudioTransport(Protocol): *, chunk_ms: int, after_chunk: Callable[[AudioSegment, int], bool] | None = None, + should_stop: Callable[[], bool] | None = None, ) -> PlaybackResult: ... diff --git a/src/owner_voice_pet/transport.py b/src/owner_voice_pet/transport.py index 862b8f7..8de04ce 100644 --- a/src/owner_voice_pet/transport.py +++ b/src/owner_voice_pet/transport.py @@ -113,13 +113,18 @@ class MemoryAudioTransport: *, chunk_ms: int, after_chunk: Callable[[AudioSegment, int], bool] | None = None, + should_stop: Callable[[], bool] | None = None, ) -> PlaybackResult: elapsed_ms = 0 for chunk in _raw_pcm_chunks(segment, chunk_ms=chunk_ms): + if should_stop is not None and should_stop(): + return PlaybackResult(True, elapsed_ms) playback = self.play_pcm(chunk) if playback.error: return playback elapsed_ms += chunk.duration_ms + if should_stop is not None and should_stop(): + return PlaybackResult(True, elapsed_ms) if after_chunk is not None and after_chunk(chunk, elapsed_ms): return PlaybackResult(True, elapsed_ms) return PlaybackResult(True, elapsed_ms) @@ -312,6 +317,7 @@ class SoundDeviceAudioTransport: *, chunk_ms: int, after_chunk: Callable[[AudioSegment, int], bool] | None = None, + should_stop: Callable[[], bool] | None = None, ) -> PlaybackResult: if self._sd is None or segment.metadata.get("format") in {"aiff", "wav", "mp3", "m4a", "aac"}: playback = self.play_pcm(segment) @@ -341,8 +347,12 @@ class SoundDeviceAudioTransport: device=_coerce_device_id(self._output_device), ) as stream: for chunk in _raw_pcm_chunks(segment, chunk_ms=chunk_ms): + if should_stop is not None and should_stop(): + return PlaybackResult(True, elapsed_ms) stream.write(chunk.pcm) elapsed_ms += chunk.duration_ms + if should_stop is not None and should_stop(): + return PlaybackResult(True, elapsed_ms) if after_chunk is not None and after_chunk(chunk, elapsed_ms): return PlaybackResult(True, elapsed_ms) except Exception as exc: diff --git a/tests/test_barge_in.py b/tests/test_barge_in.py new file mode 100644 index 0000000..857ddd9 --- /dev/null +++ b/tests/test_barge_in.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import time +import unittest + +from owner_voice_pet.barge_in import AsyncBargeInMonitor, BargeInSpeakerGate +from owner_voice_pet.models import AudioFrame, AudioSegment +from owner_voice_pet.stt import MetadataSttProvider +from owner_voice_pet.transport import MemoryAudioTransport +from owner_voice_pet.vad import EnergyVadProvider + + +def speech_frame(idx: int, speaker_id: str, partial: str = "等一下") -> AudioFrame: + return AudioFrame( + b"\xff\x7f" * 320, + 16000, + 1, + idx * 20, + idx, + { + "duration_ms": 20, + "speech": True, + "speaker_id": speaker_id, + "partial_transcript": partial, + }, + ) + + +def segment_for_speaker(speaker_id: str) -> AudioSegment: + return AudioSegment( + b"\xff\x7f" * 640, + 16000, + 1, + 0, + 40, + {"speaker_id": speaker_id}, + ) + + +class BargeInTests(unittest.TestCase): + def test_async_monitor_ignores_assistant_playback_voice(self) -> None: + gate = BargeInSpeakerGate( + enabled=True, + user_similarity_threshold=0.62, + assistant_reject_threshold=0.72, + min_rms=0.001, + ) + gate.remember_user_segment(segment_for_speaker("owner")) + transport = MemoryAudioTransport([speech_frame(1, "assistant"), speech_frame(2, "assistant")]) + transport.start_input() + monitor = AsyncBargeInMonitor( + transport=transport, + vad_provider=EnergyVadProvider(threshold=1), + realtime_stt=MetadataSttProvider(), + speaker_gate=gate, + assistant_profile=gate.assistant_profile(segment_for_speaker("assistant")), + echo_guard_ms=0, + min_speech_ms=40, + listen_interval_ms=1, + ) + monitor.vad_provider.load() + + monitor.start() + time.sleep(0.05) + monitor.stop() + + self.assertFalse(monitor.interrupted) + self.assertEqual(monitor.pending_frames(), []) + + def test_async_monitor_accepts_user_voice_and_buffers_frames(self) -> None: + gate = BargeInSpeakerGate( + enabled=True, + user_similarity_threshold=0.62, + assistant_reject_threshold=0.72, + min_rms=0.001, + ) + gate.remember_user_segment(segment_for_speaker("owner")) + transport = MemoryAudioTransport([speech_frame(1, "owner"), speech_frame(2, "owner")]) + transport.start_input() + monitor = AsyncBargeInMonitor( + transport=transport, + vad_provider=EnergyVadProvider(threshold=1), + realtime_stt=MetadataSttProvider(), + speaker_gate=gate, + assistant_profile=gate.assistant_profile(segment_for_speaker("assistant")), + echo_guard_ms=0, + min_speech_ms=40, + listen_interval_ms=1, + ) + monitor.vad_provider.load() + + monitor.start() + deadline = time.monotonic() + 1 + while not monitor.interrupted and time.monotonic() < deadline: + time.sleep(0.005) + monitor.stop() + + self.assertTrue(monitor.interrupted) + self.assertEqual([frame.metadata["speaker_id"] for frame in monitor.pending_frames()], ["owner", "owner"]) + + def test_user_profile_missing_falls_back_to_non_assistant_with_partial(self) -> None: + gate = BargeInSpeakerGate( + enabled=True, + user_similarity_threshold=0.62, + assistant_reject_threshold=0.72, + min_rms=0.001, + ) + transport = MemoryAudioTransport([speech_frame(1, "guest"), speech_frame(2, "guest")]) + transport.start_input() + monitor = AsyncBargeInMonitor( + transport=transport, + vad_provider=EnergyVadProvider(threshold=1), + realtime_stt=MetadataSttProvider(), + speaker_gate=gate, + assistant_profile=gate.assistant_profile(segment_for_speaker("assistant")), + echo_guard_ms=0, + min_speech_ms=40, + listen_interval_ms=1, + ) + monitor.vad_provider.load() + + monitor.start() + deadline = time.monotonic() + 1 + while not monitor.interrupted and time.monotonic() < deadline: + time.sleep(0.005) + monitor.stop() + + self.assertTrue(monitor.interrupted) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_live_runtime.py b/tests/test_live_runtime.py index 93c3bac..3ad608e 100644 --- a/tests/test_live_runtime.py +++ b/tests/test_live_runtime.py @@ -1,5 +1,6 @@ from __future__ import annotations +import time import unittest from owner_voice_pet.config import AppConfig @@ -238,6 +239,7 @@ class PlaybackInjectedTransport(MemoryAudioTransport): if self.play_count == self.inject_after_play_count: for frame in self.injected_frames: self.inject(frame) + time.sleep(0.002) return result def health(self) -> TransportHealth: @@ -649,15 +651,41 @@ class LiveRuntimeTests(unittest.TestCase): def test_barge_in_interrupts_playback_and_keeps_only_spoken_context(self) -> None: first_question_frames = [wake_frame(0, 0)] - first_question_frames.extend(segment_frames(1, 20, partials=["第一问", "第一问"])) + first_question_frames.extend( + [ + AudioFrame( + frame.pcm, + frame.sample_rate, + frame.channels, + frame.timestamp_ms, + frame.frame_id, + {**dict(frame.metadata), "speaker_id": "owner"}, + ) + for frame in segment_frames(1, 20, partials=["第一问", "第一问"]) + ] + ) barge_frames = [ - AudioFrame(b"\xff\x7f", 16000, 1, 600, 20, {"duration_ms": 20, "speech": True, "partial_transcript": "等一下"}), - AudioFrame(b"\xff\x7f", 16000, 1, 620, 21, {"duration_ms": 20, "speech": True, "partial_transcript": "等一下"}), + AudioFrame( + b"\xff\x7f", + 16000, + 1, + 600, + 20, + {"duration_ms": 20, "speech": True, "partial_transcript": "等一下", "speaker_id": "owner"}, + ), + AudioFrame( + b"\xff\x7f", + 16000, + 1, + 620, + 21, + {"duration_ms": 20, "speech": True, "partial_transcript": "等一下", "speaker_id": "owner"}, + ), *silence_frames(22, 640, 3), ] transport = PlaybackInjectedTransport( first_question_frames, - inject_after_play_count=8, + inject_after_play_count=18, injected_frames=barge_frames, ) stt = QueueSttProvider(["第一问", "打断问题"]) @@ -677,8 +705,10 @@ class LiveRuntimeTests(unittest.TestCase): speech_provider="cloud", wake_ack_text="", barge_in_enabled=True, - barge_in_echo_guard_ms=500, + barge_in_echo_guard_ms=0, barge_in_min_speech_ms=40, + barge_in_listen_interval_ms=1, + barge_in_chunk_ms=20, followup_listen_timeout_ms=3000, ), transport=transport, diff --git a/tests/test_models_config.py b/tests/test_models_config.py index e021e3c..4769c90 100644 --- a/tests/test_models_config.py +++ b/tests/test_models_config.py @@ -113,6 +113,11 @@ class ModelsConfigTests(unittest.TestCase): self.assertTrue(config.barge_in_enabled) self.assertEqual(config.barge_in_min_speech_ms, 250) self.assertEqual(config.barge_in_echo_guard_ms, 500) + self.assertTrue(config.barge_in_speaker_gate_enabled) + self.assertEqual(config.barge_in_user_similarity_threshold, 0.62) + self.assertEqual(config.barge_in_assistant_reject_threshold, 0.72) + self.assertEqual(config.barge_in_listen_interval_ms, 20) + self.assertEqual(config.barge_in_chunk_ms, 30) self.assertTrue(config.end_chime_enabled) self.assertEqual(str(config.end_chime_file), "assets/sounds/codex-notification.wav") self.assertEqual(config.end_chime_frequency_hz, 880) @@ -157,6 +162,10 @@ class ModelsConfigTests(unittest.TestCase): followup_listen_timeout_ms=-1, barge_in_min_speech_ms=-1, barge_in_echo_guard_ms=-1, + barge_in_user_similarity_threshold=1.5, + barge_in_assistant_reject_threshold=0.0, + barge_in_listen_interval_ms=0, + barge_in_chunk_ms=0, end_chime_frequency_hz=0, end_chime_duration_ms=0, ) @@ -167,6 +176,10 @@ class ModelsConfigTests(unittest.TestCase): self.assertTrue(any("OWNER_FOLLOWUP_LISTEN_TIMEOUT_MS" in error.message for error in errors)) self.assertTrue(any("OWNER_BARGE_IN_MIN_SPEECH_MS" in error.message for error in errors)) self.assertTrue(any("OWNER_BARGE_IN_ECHO_GUARD_MS" in error.message for error in errors)) + self.assertTrue(any("OWNER_BARGE_IN_USER_SIMILARITY_THRESHOLD" in error.message for error in errors)) + self.assertTrue(any("OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD" in error.message for error in errors)) + self.assertTrue(any("OWNER_BARGE_IN_LISTEN_INTERVAL_MS" in error.message for error in errors)) + self.assertTrue(any("OWNER_BARGE_IN_CHUNK_MS" in error.message for error in errors)) self.assertTrue(any("OWNER_END_CHIME_FREQUENCY_HZ" in error.message for error in errors)) self.assertTrue(any("OWNER_END_CHIME_DURATION_MS" in error.message for error in errors)) diff --git a/tests/test_transport.py b/tests/test_transport.py index d033b8c..c2cf3bb 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -133,6 +133,21 @@ class TransportTests(unittest.TestCase): self.assertGreater(len(fake.output_writes), 1) self.assertEqual(callbacks[-1], result.duration_ms) + def test_sounddevice_chunk_playback_stops_on_signal(self) -> None: + fake = FakeSoundDevice() + transport = SoundDeviceAudioTransport(sounddevice_module=fake) + segment = AudioSegment(b"\x00\x00" * 1600, 16000, 1, 0, 100) + + result = transport.play_pcm_chunks( + segment, + chunk_ms=20, + should_stop=lambda: len(fake.output_writes) >= 2, + ) + + self.assertTrue(result.played) + self.assertEqual(len(fake.output_writes), 2) + self.assertEqual(result.duration_ms, 40) + def test_sounddevice_device_report_uses_query_devices(self) -> None: fake = FakeSoundDevice() report = SoundDeviceAudioTransport(sounddevice_module=fake).device_report()