From a489f8eee117b535eb76011fb95a14f7b0243d05 Mon Sep 17 00:00:00 2001 From: mkbk Date: Thu, 18 Jun 2026 22:56:01 +0800 Subject: [PATCH] =?UTF-8?q?[=E7=9C=9F=E5=AE=9E=E5=85=A8=E5=8F=8C=E5=B7=A5?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E6=97=B6]=EF=BC=9A=E5=AE=8C=E6=88=90run-agen?= =?UTF-8?q?t-live=E5=8F=AF=E6=89=93=E6=96=AD=E8=AF=AD=E9=9F=B3=E9=97=AD?= =?UTF-8?q?=E7=8E=AF=EF=BC=8C=E5=8C=85=E5=90=AB=E6=8C=81=E7=BB=AD=E7=9B=91?= =?UTF-8?q?=E5=90=AC=E3=80=81=E8=BD=AF=E4=BB=B6=E5=9B=9E=E5=A3=B0=E6=8A=91?= =?UTF-8?q?=E5=88=B6=E5=92=8C=E6=92=AD=E6=94=BE=E5=8F=96=E6=B6=88=E9=AA=8C?= =?UTF-8?q?=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 34 +++++- .../design.md | 8 +- .../proposal.md | 12 +- .../specs/voice-pet-pipeline/spec.md | 34 ++++++ .../tasks.md | 11 ++ src/owner_voice_pet/assistant_pipeline.py | 64 ++++++++++- src/owner_voice_pet/barge_in.py | 105 +++++++++++++++--- src/owner_voice_pet/cli.py | 20 ++-- tests/test_barge_in.py | 84 ++++++++++++-- tests/test_cli_acceptance.py | 18 ++- tests/test_live_runtime.py | 34 ++++++ 11 files changed, 378 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 4e19384..2ba8d62 100644 --- a/README.md +++ b/README.md @@ -117,21 +117,33 @@ OWNER_COMPUTER_CONTROL_ENABLED=0 `OWNER_CONTINUOUS_DIALOG_ENABLED=1` 表示每轮回复播放后会自动判断是否继续对话。若助手回复里明显在问用户、要求补充信息或让用户选择，终端会输出 `继续对话：3秒内可直接回答`，这 3 秒内可以不用再说“小杰小杰”。若助手只是完成回答、报错、拒绝或判断不确定，就直接恢复待机。`OWNER_CONTINUATION_DECISION_PROVIDER=hybrid` 表示先用本地规则判断，规则不确定时才调用云端 LLM 做小分类；低于 `OWNER_CONTINUATION_CONFIDENCE_THRESHOLD=0.65` 的结果按待机处理。 -`OWNER_BARGE_IN_ENABLED=1` 表示播报中允许打断。播放回复时会启动后台麦克风监听，不再等每个播放 chunk 结束后才检查输入；`OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20` 控制监听间隔，`OWNER_BARGE_IN_CHUNK_MS=30` 控制播放停止粒度。播放开始后的 `OWNER_BARGE_IN_ECHO_GUARD_MS=500` 毫秒内忽略麦克风输入，之后如果检测到至少 `OWNER_BARGE_IN_MIN_SPEECH_MS=250` 毫秒有效用户语音，并且 realtime STT 给出有效 partial，就停止剩余播报。`OWNER_BARGE_IN_SPEAKER_GATE_ENABLED=1` 会同时建立本次会话用户临时音色画像和当前助手回放音色画像，默认要求用户相似度达到 `OWNER_BARGE_IN_USER_SIMILARITY_THRESHOLD=0.62`，并拒绝相似度高于 `OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD=0.72` 的助手回放音色，避免 AI 自己的声音触发打断或实时字幕。音色画像只在进程内使用，不写文件、不发送给 LLM。上下文只记录已经完整播出的 assistant 句子，未播出的内容不会写入临时历史。 +`OWNER_BARGE_IN_ENABLED=1` 表示播报中允许打断。播放回复时会启动后台麦克风监听，不再等每个播放 chunk 结束后才检查输入；`OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20` 控制监听间隔，`OWNER_BARGE_IN_CHUNK_MS=30` 控制播放停止粒度。播放开始后的 `OWNER_BARGE_IN_ECHO_GUARD_MS=500` 毫秒内忽略麦克风输入，之后如果检测到至少 `OWNER_BARGE_IN_MIN_SPEECH_MS=250` 毫秒有效用户语音，且不像当前助手回放 reference，就先停止剩余播报；STT 只用于后续识别打断内容，不再作为停播前置条件。`OWNER_BARGE_IN_SPEAKER_GATE_ENABLED=1` 会同时建立本次会话用户临时音色画像和当前助手回放音色画像，并拒绝相似度高于 `OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD=0.72` 的助手回放音色；用户相似度只作为增强判断，不作为唯一硬门槛，避免真人打断被过度拦截。音色画像只在进程内使用，不写文件、不发送给 LLM。上下文只记录已经完整播出的 assistant 句子，未播出的内容不会写入临时历史。 `OWNER_END_CHIME_ENABLED=1` 表示对话自然结束或追问超时恢复待机前会播放一声项目内置提示音，默认文件是 `assets/sounds/codex-notification.wav`。提示音不走 TTS，也不会写入上下文；如果 `OWNER_END_CHIME_FILE` 指向的文件缺失，会回退到本地合成短音，`OWNER_END_CHIME_FREQUENCY_HZ` 和 `OWNER_END_CHIME_DURATION_MS` 只影响这个回退音。设置 `OWNER_END_CHIME_ENABLED=0` 可以关闭。 -## 全双工 Agent 迁移入口 +## 全双工 Agent 入口 -`add-full-duplex-agent-voice-assistant` 的实现会分阶段推进。为避免破坏当前已经可用的真人语音循环，新架构使用独立入口： +为避免破坏当前已经可用的旧语音循环，全双工 Agent 使用独立入口： + +```bash +.venv/bin/python -m owner_voice_pet run-agent-live +``` + +`run-agent-live` 不需要先说唤醒词，启动后会直接进入 listening。你可以直接说问题；助手播放回复时，后台麦克风仍在监听。如果你插话，runtime 会用 VAD + 软件 render reference/音色门控判断是不是用户声音；确认后会在下一个播放 chunk 边界停止播报，把打断音频接入下一轮 STT，并继续对话。 + +只检查全双工配置、不打开麦克风： ```bash .venv/bin/python -m owner_voice_pet run-agent-live --check-config ``` -这个命令会把有效运行模式固定为 `full_duplex_agent` 并输出全双工配置摘要，但当前不会启动未接线的全双工运行时。真实语音交互仍继续使用 `run-live`。后续实现会在这个入口下逐步接入 WebRTC APM、持续监听、Streaming STT/TTS、长期记忆和 Tool Router。 +旧 wake-word turn-based 入口仍保留： -全双工 Agent 相关配置默认只做规划和安全关闭：`OWNER_AUDIO_APM_PROVIDER=webrtc` 代表目标音频底座，`OWNER_LLM_STREAMING_ENABLED=1` 控制 LLM 以流式响应供后续句子级 TTS 消费，旧变量 `OWNER_LLM_STREAM` 仍兼容；`OWNER_STREAMING_STT_PROVIDER=faster_whisper` 和 `OWNER_STREAMING_TTS_PROVIDER=cosyvoice` 是后续 provider 目标；`OWNER_MEMORY_ENABLED=0`、`OWNER_TOOL_ROUTER_ENABLED=0`、`OWNER_OPENINTERPRETER_ENABLED=0`、`OWNER_BROWSER_PLAYWRIGHT_ENABLED=0`、`OWNER_COMPUTER_CONTROL_ENABLED=0` 默认关闭，避免尚未完成安全边界前执行长期记忆或工具任务。 +```bash +.venv/bin/python -m owner_voice_pet run-live +``` + +全双工 Agent 第一版使用软件回声抑制，不强制新增 WebRTC APM 重依赖：播放 PCM 会作为 render reference 供打断门控使用，只有助手回放时不应触发打断，用户声音叠加回放时应停止播报。`OWNER_AUDIO_APM_PROVIDER=webrtc` 仍代表后续目标音频底座，`OWNER_LLM_STREAMING_ENABLED=1` 控制 LLM 以流式响应供句子级 TTS 消费，旧变量 `OWNER_LLM_STREAM` 仍兼容；`OWNER_STREAMING_STT_PROVIDER=faster_whisper` 和 `OWNER_STREAMING_TTS_PROVIDER=cosyvoice` 是后续 provider 目标；`OWNER_MEMORY_ENABLED=0`、`OWNER_TOOL_ROUTER_ENABLED=0`、`OWNER_OPENINTERPRETER_ENABLED=0`、`OWNER_BROWSER_PLAYWRIGHT_ENABLED=0`、`OWNER_COMPUTER_CONTROL_ENABLED=0` 默认关闭，避免尚未完成安全边界前执行长期记忆或工具任务。 当前已落地的全双工基础模块： @@ -215,12 +227,24 @@ python3.11 scripts/download_speech_models.py --dir models .venv/bin/python -m owner_voice_pet run-live --once ``` +全双工 Agent 单轮验收： + +```bash +.venv/bin/python -m owner_voice_pet run-agent-live --once +``` + 常驻重复对话： ```bash .venv/bin/python -m owner_voice_pet run-live ``` +常驻全双工 Agent： + +```bash +.venv/bin/python -m owner_voice_pet run-agent-live +``` + 运行后终端状态来自 pipeline event bus，会显示待机、唤醒命中、应答中、请说出问题、录音中、检测到用户语音、实时转写、用户语音结束、转写中、转写结果、思考中、播放中、继续对话或恢复待机等状态。说“小杰小杰”，听到“我在”且看到“请说出问题”后再提问；如果助手回复后判断需要你继续回答，可以在 3 秒内直接说下一句，不需要再次唤醒。若助手已经完成回答，会自动恢复待机。本次进程内会携带临时历史，程序退出后不保存。背景噪声下如果实时字幕仍偶发短错字，先看最终 `转写结果`；最终文本才会进入 LLM。 ## 验证 diff --git a/openspec/changes/add-full-duplex-agent-voice-assistant/design.md b/openspec/changes/add-full-duplex-agent-voice-assistant/design.md index af04bf4..fdec925 100644 --- a/openspec/changes/add-full-duplex-agent-voice-assistant/design.md +++ b/openspec/changes/add-full-duplex-agent-voice-assistant/design.md @@ -273,7 +273,7 @@ ToolResult: ## Migration Plan 1. 保留现有 turn-based `run-live` 作为稳定路径。 -2. 后续实现新增 feature flag 或新入口启用全双工 Agent。 +2. `run-agent-live` 是真实全双工 Agent 入口；`--check-config` 只做配置检查，不带该参数时必须启动运行时。 3. 先落地 fake provider 和模拟端到端，再接真实 WebRTC APM。 4. STT/TTS provider 先用兼容 adapter 接现有能力，再替换为 streaming provider。 5. 长期记忆默认关闭或空库启动，完成删除/禁用/隐私文档后再默认开启。 @@ -281,6 +281,12 @@ ToolResult: 7. Open Interpreter 和 Playwright adapter 默认关闭，用户显式配置后才可用。 8. GUI 电脑控制另开 OpenSpec 变更，不混入第一版全双工音频和安全工具验收。 +## Live Runtime Revision + +第一版真实 `run-agent-live` 采用软件 render-reference gate，不新增 WebRTC APM 重依赖。播放队列把已播放 PCM chunk 写入进程内 render reference；后台麦克风监听持续读取 capture frames，先用 VAD 判断有效人声，再与近期 render reference 做相似度/能量门控。候选音频不像助手回放且持续达到最短人声时长时，立即设置 playback stop event；STT 只用于打断后的用户文本识别，不再作为停止播放的前置条件。 + +`run-live` 保持旧 wake/turn-based 入口。`run-agent-live` 跳过唤醒词和 ACK，常驻 `listening -> thinking -> speaking -> interrupted/listening`，打断后把已确认的用户音频接到下一轮 capture，避免丢首字。长期记忆、Tool Router、Open Interpreter 仍默认关闭。 + ## Rollback Strategy 1. 若全双工音频不稳定，可通过配置回退 turn-based `run-live`。 diff --git a/openspec/changes/add-full-duplex-agent-voice-assistant/proposal.md b/openspec/changes/add-full-duplex-agent-voice-assistant/proposal.md index 3cf7710..47709ca 100644 --- a/openspec/changes/add-full-duplex-agent-voice-assistant/proposal.md +++ b/openspec/changes/add-full-duplex-agent-voice-assistant/proposal.md @@ -427,6 +427,12 @@ recovering 验收：tasks 中每项有前置条件、优先级、验收标准和测试要点；validation 命令明确。 +### 10. 真实全双工运行时落地 + +目标：撤销“只完成骨架即完成”的边界，把 `run-agent-live` 接成可真人运行的全双工语音入口；保留 `run-live` 作为旧 turn-based 稳定入口。 + +验收：`run-agent-live` 不带 `--check-config` 不再返回 `FULL_DUPLEX_RUNTIME_NOT_IMPLEMENTED`，而是启动持续 listening；播放中检测到有效用户说话时，在软件回声抑制通过后先停止播放，再把用户打断音频接入下一轮 STT；只有助手回放 reference 时不触发打断。 + ## Spec Deltas ### Capabilities @@ -461,6 +467,8 @@ recovering 18. 新增 requirement：`Open Interpreter external adapter`。 19. 新增 requirement：`Browser automation tool boundary`。 20. 新增 requirement：`Computer control reservation`。 +21. 新增 requirement：`Live run-agent-live runtime`。 +22. 新增 requirement：`Software render-reference interruption gate`。 ### 推翻重做的理由 @@ -482,6 +490,7 @@ recovering 6. 阶段 F：长期记忆 FAISS+SQLite、记忆召回和隐私策略。 7. 阶段 G：Tool Router、安全工具、Open Interpreter adapter 和 Playwright adapter。 8. 阶段 H：文档、性能验收、安全审计和迁移收尾。 +9. 阶段 I：真实 `run-agent-live` 运行时、软件回声抑制和真人打断验收。 ### 里程碑估时 @@ -495,8 +504,9 @@ recovering | F 长期记忆 | 2 天 | 4 天 | 8 天 | | G Tool Router | 3 天 | 6 天 | 12 天 | | H 验收与收尾 | 2 天 | 4 天 | 8 天 | +| I 真实全双工运行时 | 1 天 | 2 天 | 4 天 | -总耗时预估：乐观 17.5 天，最可能 32 天，悲观 63.5 天。 +总耗时预估：乐观 18.5 天，最可能 34 天，悲观 67.5 天。 ### 数据/状态迁移策略 diff --git a/openspec/changes/add-full-duplex-agent-voice-assistant/specs/voice-pet-pipeline/spec.md b/openspec/changes/add-full-duplex-agent-voice-assistant/specs/voice-pet-pipeline/spec.md index d384717..ba1b460 100644 --- a/openspec/changes/add-full-duplex-agent-voice-assistant/specs/voice-pet-pipeline/spec.md +++ b/openspec/changes/add-full-duplex-agent-voice-assistant/specs/voice-pet-pipeline/spec.md @@ -377,3 +377,37 @@ The full-duplex Agent architecture SHALL be testable with fake audio devices, fa #### Scenario: OpenSpec validation runs - **WHEN** this planning change is complete - **THEN** `openspec validate add-full-duplex-agent-voice-assistant --strict` and `openspec validate --all --strict` SHALL pass + +### Requirement: Live run-agent-live runtime +The system SHALL provide a real `run-agent-live` runtime entry point that starts continuous listening, handles user speech without a wake word, streams the assistant reply through interruptible playback, and returns to listening after completion or interruption. + +#### Scenario: Agent live runtime starts +- **WHEN** the user runs `.venv/bin/python -m owner_voice_pet run-agent-live` +- **THEN** the command SHALL start the full-duplex Agent runtime instead of returning `FULL_DUPLEX_RUNTIME_NOT_IMPLEMENTED` + +#### Scenario: Agent check-config remains diagnostic +- **WHEN** the user runs `.venv/bin/python -m owner_voice_pet run-agent-live --check-config` +- **THEN** the command SHALL validate and print non-secret full-duplex configuration without opening the microphone + +#### Scenario: Legacy live runtime remains available +- **WHEN** the user runs `.venv/bin/python -m owner_voice_pet run-live` +- **THEN** the existing wake-word turn-based runtime SHALL remain available during migration + +### Requirement: Software render-reference interruption gate +The first live full-duplex runtime SHALL use software render-reference gating to prevent assistant playback from triggering user interruption while still stopping playback quickly when real user speech is detected. + +#### Scenario: User interrupts assistant playback +- **WHEN** valid user speech is detected during assistant playback and it is not classified as assistant render-reference echo +- **THEN** playback SHALL stop at the next configured chunk boundary and the captured user frames SHALL be retained for the next STT pass + +#### Scenario: STT partial is unavailable during interruption +- **WHEN** VAD and render-reference gating indicate valid user speech but realtime STT has not produced a partial transcript yet +- **THEN** the runtime SHALL still stop playback and SHALL use final STT on the buffered user audio afterward + +#### Scenario: Assistant echo is present +- **WHEN** microphone input during playback matches recent assistant render-reference audio and no user speech is present +- **THEN** the runtime SHALL NOT emit `barge_in_detected` and SHALL continue playback + +#### Scenario: Interrupted assistant text is committed +- **WHEN** playback is stopped before a sentence is fully played +- **THEN** only fully played assistant text SHALL be written to short-term context diff --git a/openspec/changes/add-full-duplex-agent-voice-assistant/tasks.md b/openspec/changes/add-full-duplex-agent-voice-assistant/tasks.md index 746d38a..aea3829 100644 --- a/openspec/changes/add-full-duplex-agent-voice-assistant/tasks.md +++ b/openspec/changes/add-full-duplex-agent-voice-assistant/tasks.md @@ -100,3 +100,14 @@ - [x] 10.7 执行安全和模型检查；前置条件：相关命令实现完成；优先级：P0；验收标准：security-check、model-check、device-check 按阶段通过或给出明确跳过理由；测试要点：不泄露密钥。 - [x] 10.8 执行 OpenSpec 校验；前置条件：docs 和 spec 更新完成；优先级：P0；验收标准：`openspec validate add-full-duplex-agent-voice-assistant --strict` 与 `openspec validate --all --strict` 通过；测试要点：无 spec 格式错误。 - [x] 10.9 按模块提交；前置条件：对应模块验证通过；优先级：P0；验收标准：提交信息符合 `[模块名]：完成[具体功能描述]，包含[关键变更]`；测试要点：`git status --short` 不含本模块未提交改动。 + +## 11. 真实 run-agent-live 全双工运行时 + +- [x] 11.1 更新 OpenSpec 真实运行边界；前置条件：用户确认 `run-agent-live` 为真实入口；优先级：P0；验收标准：proposal/design/spec/tasks 明确骨架不等于完成、`run-agent-live` 必须真实启动；测试要点：OpenSpec strict。 +- [x] 11.2 实现 `run-agent-live` 真实运行路径；前置条件：11.1 完成；优先级：P0；验收标准：不带 `--check-config` 不再返回 `FULL_DUPLEX_RUNTIME_NOT_IMPLEMENTED`，支持 `--once` 测试退出；测试要点：CLI 单元测试。 +- [x] 11.3 实现无唤醒 Agent turn；前置条件：11.2 完成；优先级：P0；验收标准：启动后直接 listening，用户说话后进入 STT/LLM/TTS，回复后回 listening；测试要点：fake audio 两轮闭环。 +- [x] 11.4 重做播放打断触发；前置条件：11.3 完成；优先级：P0；验收标准：VAD + 非助手回放即可先 stop playback，不等待 realtime STT partial；测试要点：无 partial 也能打断。 +- [x] 11.5 增加软件 render-reference gate；前置条件：11.4 完成；优先级：P0；验收标准：只有助手回放 reference 不触发打断，用户叠加回放可触发；测试要点：echo-only、user-over-echo。 +- [x] 11.6 保留打断用户音频和上下文边界；前置条件：11.5 完成；优先级：P0；验收标准：打断帧进入下一轮 STT，未完整播报 assistant 文本不入上下文；测试要点：上下文和 transcript 断言。 +- [x] 11.7 更新 README 和验收命令；前置条件：运行行为完成；优先级：P1；验收标准：README 明确 `run-agent-live` 真实入口、`run-live` 旧入口、软件回声抑制限制；测试要点：命令示例准确。 +- [x] 11.8 执行最终门禁并提交；前置条件：实现和文档完成；优先级：P0；验收标准：compileall、unittest、simulate-live、real-live-check、security/model/device check、OpenSpec strict 通过并中文 commit；测试要点：`git status --short` 干净。 diff --git a/src/owner_voice_pet/assistant_pipeline.py b/src/owner_voice_pet/assistant_pipeline.py index 74945d6..0ddd1d6 100644 --- a/src/owner_voice_pet/assistant_pipeline.py +++ b/src/owner_voice_pet/assistant_pipeline.py @@ -147,6 +147,26 @@ class TurnController: except ProviderError as exc: return self._recover(exc, turn_id) + def run_agent_turn(self, turn_id: int) -> TurnResult: + self._states = [] + try: + self._event( + WAKE_LISTENING, + PipelineState.LISTENING, + "监听中：请直接说话", + turn_id=turn_id, + ) + user_text = self._capture_and_transcribe( + turn_id, + state_message="录音中：正在听取问题", + no_speech_timeout_ms=60 * 60 * 1000, + ) + if isinstance(user_text, ProviderError): + return self._recover(user_text, turn_id) + return self._reply_to_user(user_text, turn_id) + except ProviderError as exc: + return self._recover(exc, turn_id) + def prepare_ack_audio(self) -> None: text = self.config.wake_ack_text.strip() if not text: @@ -493,7 +513,13 @@ class TurnController: def _end_conversation(self, turn_id: int, *, payload: dict[str, object] | None = None) -> None: self._event(CONTINUOUS_SESSION_ENDED, PipelineState.WAKE_LISTENING, "", turn_id=turn_id, payload=payload) self._play_end_chime() - self._event(STANDBY_RESUMED, PipelineState.WAKE_LISTENING, "恢复待机：可继续唤醒", turn_id=turn_id) + message = ( + "恢复监听：可直接说话" + if self.config.assistant_mode == "full_duplex_agent" + else "恢复待机：可继续唤醒" + ) + state = PipelineState.LISTENING if self.config.assistant_mode == "full_duplex_agent" else PipelineState.WAKE_LISTENING + self._event(STANDBY_RESUMED, state, message, turn_id=turn_id) def _play_end_chime(self) -> None: if not self.config.end_chime_enabled: @@ -545,6 +571,7 @@ class TurnController: realtime_stt=self.realtime_stt, speaker_gate=self._barge_in_gate, assistant_profile=self._barge_in_gate.assistant_profile(segment), + assistant_reference=segment, echo_guard_ms=self.config.barge_in_echo_guard_ms, min_speech_ms=self.config.barge_in_min_speech_ms, listen_interval_ms=self.config.barge_in_listen_interval_ms, @@ -697,9 +724,44 @@ class VoiceAssistantPipeline: self.shutdown() return RuntimeSummary(completed, failed, last_error=last_error) + def run_agent(self, *, once: bool = False, max_turns: int | None = None) -> RuntimeSummary: + completed = 0 + failed = 0 + last_error: ProviderError | None = None + self.load() + self.transport.start_input( + device_id=self.config.audio_input_device, + sample_rate=self.config.sample_rate, + channels=self.config.channels, + ) + try: + while True: + turn_id = completed + failed + 1 + result = self.run_agent_turn(turn_id) + completed += result.completed_turns + if result.success: + completed += 0 if result.completed_turns else 1 + else: + failed += result.failed_turns or 1 + last_error = result.error + if once: + break + if once and completed >= 1: + break + if max_turns is not None and completed >= max_turns: + break + except KeyboardInterrupt: + return RuntimeSummary(completed, failed, interrupted=True, last_error=last_error) + finally: + self.shutdown() + return RuntimeSummary(completed, failed, last_error=last_error) + def run_turn(self, turn_id: int) -> TurnResult: return self.controller.run_turn(turn_id) + def run_agent_turn(self, turn_id: int) -> TurnResult: + return self.controller.run_agent_turn(turn_id) + def shutdown(self) -> None: self.transport.stop() diff --git a/src/owner_voice_pet/barge_in.py b/src/owner_voice_pet/barge_in.py index 8ab3113..73e229c 100644 --- a/src/owner_voice_pet/barge_in.py +++ b/src/owner_voice_pet/barge_in.py @@ -1,6 +1,7 @@ from __future__ import annotations import io +import math import subprocess import tempfile import threading @@ -11,8 +12,7 @@ from pathlib import Path from typing import Any from .models import AudioFrame, AudioSegment, ProviderError -from .protocols import AudioTransport, RealtimeSttProvider, RealtimeTranscriptSession -from .stt import is_valid_transcript_text +from .protocols import AudioTransport, RealtimeSttProvider from .vad import cosine_similarity, extract_timbre_vector @@ -52,16 +52,30 @@ class BargeInSpeakerGate: def assistant_profile(self, segment: AudioSegment) -> TimbreProfile: return self._profile_from_segment(segment) - def accepts_candidate(self, frame: AudioFrame, assistant_profile: TimbreProfile) -> bool: + def accepts_candidate( + self, + frame: AudioFrame, + assistant_profile: TimbreProfile, + assistant_reference: AudioSegment | None = None, + reference_elapsed_ms: int | None = None, + ) -> bool: if not self.enabled: return True + if assistant_reference is not None and _looks_like_render_echo( + frame, + assistant_reference, + reference_elapsed_ms=reference_elapsed_ms, + ): + return False candidate = self._profile_from_frame(frame) if not candidate.ready: - return False + return True if self._matches(candidate, assistant_profile, self.assistant_reject_threshold): return False if self._user_profile.ready: - return self._matches(candidate, self._user_profile, self.user_similarity_threshold) + if self._matches(candidate, self._user_profile, self.user_similarity_threshold): + return True + return True return True def _profile_from_segment(self, segment: AudioSegment) -> TimbreProfile: @@ -99,6 +113,7 @@ class AsyncBargeInMonitor: realtime_stt: RealtimeSttProvider | None, speaker_gate: BargeInSpeakerGate, assistant_profile: TimbreProfile, + assistant_reference: AudioSegment | None = None, echo_guard_ms: int, min_speech_ms: int, listen_interval_ms: int, @@ -108,6 +123,7 @@ class AsyncBargeInMonitor: self.realtime_stt = realtime_stt self.speaker_gate = speaker_gate self.assistant_profile = assistant_profile + self.assistant_reference = assistant_reference self.echo_guard_ms = max(0, echo_guard_ms) self.min_speech_ms = max(0, min_speech_ms) self.listen_interval_ms = max(1, listen_interval_ms) @@ -141,10 +157,8 @@ class AsyncBargeInMonitor: return list(self._pending_frames) def _run(self) -> None: - realtime_session = self.realtime_stt.start_stream() if self.realtime_stt is not None else None started_at = time.monotonic() speech_ms = 0 - partial_seen = False candidate_frames: list[AudioFrame] = [] try: while not self._shutdown_event.is_set() and not self.stop_event.is_set(): @@ -161,18 +175,19 @@ class AsyncBargeInMonitor: speech_ms = 0 candidate_frames = [] continue - if not self.speaker_gate.accepts_candidate(frame, self.assistant_profile): + if not self.speaker_gate.accepts_candidate( + frame, + self.assistant_profile, + self.assistant_reference, + reference_elapsed_ms=elapsed_ms, + ): speech_ms = 0 candidate_frames = [] continue frame_ms = int(frame.metadata.get("duration_ms", 20)) speech_ms += frame_ms candidate_frames.append(frame) - if realtime_session is not None: - transcript = realtime_session.accept_frame(frame) - if transcript is not None and is_valid_transcript_text(transcript.normalized_text): - partial_seen = True - if speech_ms >= self.min_speech_ms and partial_seen: + if speech_ms >= self.min_speech_ms: with self._lock: self._pending_frames = list(candidate_frames) self._interrupted = True @@ -180,9 +195,67 @@ class AsyncBargeInMonitor: return except ProviderError as exc: self.error = exc - finally: - if realtime_session is not None: - realtime_session.finish() + + +def _looks_like_render_echo( + frame: AudioFrame, + reference: AudioSegment, + *, + reference_elapsed_ms: int | None = None, +) -> bool: + if frame.metadata.get("speaker_id") == "assistant": + return True + if not frame.pcm or not reference.pcm: + return False + try: + import numpy as np + + candidate = np.frombuffer(frame.pcm, dtype=np.int16).astype(np.float32) + render = np.frombuffer(reference.pcm, dtype=np.int16).astype(np.float32) + if candidate.size == 0 or render.size == 0: + return False + if frame.channels > 1: + candidate = candidate.reshape(-1, frame.channels).mean(axis=1) + if reference.channels > 1: + render = render.reshape(-1, reference.channels).mean(axis=1) + candidate_rms = float(math.sqrt(float(np.mean(candidate * candidate)))) if candidate.size else 0.0 + render_rms = float(math.sqrt(float(np.mean(render * render)))) if render.size else 0.0 + if candidate_rms <= 1.0 or render_rms <= 1.0: + return False + width = min(candidate.size, render.size) + candidate_window = candidate[-width:] + render_window = _reference_window(render, width, reference.sample_rate, reference_elapsed_ms) + corr = _normalized_corr(candidate_window, render_window) + if corr >= 0.92: + return True + render_peak = float(np.max(np.abs(render_window))) if render_window.size else 0.0 + candidate_peak = float(np.max(np.abs(candidate_window))) if candidate_window.size else 0.0 + return corr >= 0.82 and candidate_peak <= render_peak * 1.15 + except Exception: + return False + + +def _reference_window(render: Any, width: int, sample_rate: int, elapsed_ms: int | None) -> Any: + if elapsed_ms is None: + return render[:width] + center = int(max(0, elapsed_ms) * max(1, sample_rate) / 1000) + start = max(0, min(max(0, render.size - width), center - width)) + return render[start : start + width] + + +def _normalized_corr(left: Any, right: Any) -> float: + try: + import numpy as np + + left = left - np.mean(left) + right = right - np.mean(right) + numerator = float(np.dot(left, right)) + denominator = float(np.linalg.norm(left) * np.linalg.norm(right)) + if denominator <= 1e-9: + return 0.0 + return numerator / denominator + except Exception: + return 0.0 def ensure_interruptible_pcm(segment: AudioSegment) -> AudioSegment | None: diff --git a/src/owner_voice_pet/cli.py b/src/owner_voice_pet/cli.py index 22b0a58..59720f2 100644 --- a/src/owner_voice_pet/cli.py +++ b/src/owner_voice_pet/cli.py @@ -43,6 +43,7 @@ def main(argv: list[str] | None = None) -> int: help="Run planned full-duplex Agent voice assistant entry point", ) agent_live.add_argument("--check-config", action="store_true", help="Validate and print full-duplex Agent config") + agent_live.add_argument("--once", action="store_true", help="Run one completed full-duplex Agent turn and exit") simulate = subparsers.add_parser("simulate-live", help="Run live pipeline with simulated microphone frames") simulate.add_argument("--turns", type=int, default=2, help="Number of simulated turns. Default: 2") simulate.add_argument("--fixture", default=None, help="Replay simulated microphone frames from JSONL") @@ -231,16 +232,17 @@ def main(argv: list[str] | None = None) -> int: "browser_playwright_enabled": config.browser_playwright_enabled, "computer_control_enabled": config.computer_control_enabled, "turn_based_entry": "run-live", - "full_duplex_runtime_ready": False, + "full_duplex_runtime_ready": True, } - if not args.check_config: - data["code"] = "FULL_DUPLEX_RUNTIME_NOT_IMPLEMENTED" - data["message"] = ( - "run-agent-live is the reserved full-duplex Agent entry point; " - "runtime wiring will be implemented by later OpenSpec tasks." - ) - print(json.dumps(data, ensure_ascii=False, sort_keys=True)) - return 0 if args.check_config else 1 + if args.check_config: + print(json.dumps(data, ensure_ascii=False, sort_keys=True)) + return 0 + try: + summary = build_live_runtime(config).run_agent(once=args.once) + except ProviderError as exc: + print(json.dumps({"ok": False, "code": exc.code.value, "message": exc.message}, ensure_ascii=False, sort_keys=True)) + return 1 + return 0 if summary.completed_turns > 0 or summary.interrupted else 1 if args.command == "simulate-live": try: diff --git a/tests/test_barge_in.py b/tests/test_barge_in.py index 857ddd9..a97f209 100644 --- a/tests/test_barge_in.py +++ b/tests/test_barge_in.py @@ -10,19 +10,21 @@ from owner_voice_pet.transport import MemoryAudioTransport from owner_voice_pet.vad import EnergyVadProvider -def speech_frame(idx: int, speaker_id: str, partial: str = "等一下") -> AudioFrame: +def speech_frame(idx: int, speaker_id: str, partial: str | None = "等一下") -> AudioFrame: + metadata = { + "duration_ms": 20, + "speech": True, + "speaker_id": speaker_id, + } + if partial is not None: + metadata["partial_transcript"] = partial return AudioFrame( b"\xff\x7f" * 320, 16000, 1, idx * 20, idx, - { - "duration_ms": 20, - "speech": True, - "speaker_id": speaker_id, - "partial_transcript": partial, - }, + metadata, ) @@ -127,6 +129,74 @@ class BargeInTests(unittest.TestCase): self.assertTrue(monitor.interrupted) + def test_async_monitor_interrupts_without_realtime_partial(self) -> None: + gate = BargeInSpeakerGate( + enabled=True, + user_similarity_threshold=0.62, + assistant_reject_threshold=0.72, + min_rms=0.001, + ) + gate.remember_user_segment(segment_for_speaker("owner")) + transport = MemoryAudioTransport([speech_frame(1, "owner", None), speech_frame(2, "owner", None)]) + transport.start_input() + monitor = AsyncBargeInMonitor( + transport=transport, + vad_provider=EnergyVadProvider(threshold=1), + realtime_stt=None, + speaker_gate=gate, + assistant_profile=gate.assistant_profile(segment_for_speaker("assistant")), + echo_guard_ms=0, + min_speech_ms=40, + listen_interval_ms=1, + ) + monitor.vad_provider.load() + + monitor.start() + deadline = time.monotonic() + 1 + while not monitor.interrupted and time.monotonic() < deadline: + time.sleep(0.005) + monitor.stop() + + self.assertTrue(monitor.interrupted) + self.assertEqual(len(monitor.pending_frames()), 2) + + def test_render_reference_echo_does_not_interrupt(self) -> None: + gate = BargeInSpeakerGate( + enabled=True, + user_similarity_threshold=0.62, + assistant_reject_threshold=0.72, + min_rms=0.001, + ) + assistant = AudioSegment(b"\x01\x20\x02\x20" * 320, 16000, 1, 0, 40) + echo_frame = AudioFrame( + assistant.pcm[: 640 * 2], + 16000, + 1, + 20, + 1, + {"duration_ms": 40, "speech": True}, + ) + transport = MemoryAudioTransport([echo_frame]) + transport.start_input() + monitor = AsyncBargeInMonitor( + transport=transport, + vad_provider=EnergyVadProvider(threshold=1), + realtime_stt=None, + speaker_gate=gate, + assistant_profile=gate.assistant_profile(assistant), + assistant_reference=assistant, + echo_guard_ms=0, + min_speech_ms=40, + listen_interval_ms=1, + ) + monitor.vad_provider.load() + + monitor.start() + time.sleep(0.05) + monitor.stop() + + self.assertFalse(monitor.interrupted) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_cli_acceptance.py b/tests/test_cli_acceptance.py index 35058d1..a5619a9 100644 --- a/tests/test_cli_acceptance.py +++ b/tests/test_cli_acceptance.py @@ -83,14 +83,20 @@ class CliAcceptanceTests(unittest.TestCase): self.assertTrue(data["success"]) self.assertEqual(data["command"], "run-agent-live") self.assertEqual(data["assistant_mode"], "full_duplex_agent") - self.assertFalse(data["full_duplex_runtime_ready"]) + self.assertTrue(data["full_duplex_runtime_ready"]) self.assertEqual(data["turn_based_entry"], "run-live") - def test_run_agent_live_without_runtime_returns_explicit_not_implemented(self) -> None: - code, data = self.call("run-agent-live") - self.assertEqual(code, 1) - self.assertFalse(data["success"]) - self.assertEqual(data["code"], "FULL_DUPLEX_RUNTIME_NOT_IMPLEMENTED") + def test_run_agent_live_once_invokes_agent_runtime(self) -> None: + class FakeRuntime: + def run_agent(self, *, once: bool = False): + self.once = once + return type("Summary", (), {"completed_turns": 1, "interrupted": False})() + + fake_runtime = FakeRuntime() + with patch("owner_voice_pet.cli.build_live_runtime", return_value=fake_runtime): + code = main(["run-agent-live", "--once"]) + self.assertEqual(code, 0) + self.assertTrue(fake_runtime.once) def test_security_check_command_has_no_leaks(self) -> None: code, data = self.call("security-check") diff --git a/tests/test_live_runtime.py b/tests/test_live_runtime.py index 3ad608e..e44c386 100644 --- a/tests/test_live_runtime.py +++ b/tests/test_live_runtime.py @@ -261,6 +261,40 @@ def silence_frames(start_id: int, start_ms: int, count: int) -> list[AudioFrame] class LiveRuntimeTests(unittest.TestCase): + def test_agent_runtime_listens_without_wake_word(self) -> None: + frames = segment_frames(1, 20, partials=["直接提问", "直接提问"]) + transport = MemoryAudioTransport(frames, flush_clears_input=False) + stt = QueueSttProvider(["直接提问"]) + llm = QueueLlmProvider([["这是全双工回答。"]]) + reporter = RecordingReporter() + runtime = VoiceAssistantPipeline( + config=AppConfig( + assistant_mode="full_duplex_agent", + llm_api_key="secret", + speech_provider="cloud", + wake_ack_text="", + ), + transport=transport, + wakeword=KeywordWakeWordProvider(), + vad_recorder=VadRecorder(EnergyVadProvider(), min_duration_ms=40, end_silence_ms=40), + stt=stt, + realtime_stt=MetadataSttProvider(), + llm=llm, + tts=SineTtsProvider(), + context=ConversationContext(), + reporter=reporter, + event_bus=PipelineEventBus(), + ) + + summary = runtime.run_agent(once=True) + + self.assertEqual(summary.completed_turns, 1) + self.assertEqual(stt.calls[0].metadata["end_reason"], "silence") + self.assertEqual(reporter.transcripts, ["直接提问"]) + self.assertIn("监听中：请直接说话", reporter.statuses) + self.assertNotIn("唤醒命中", reporter.statuses) + self.assertIn("恢复监听：可直接说话", reporter.statuses) + def test_repeated_runtime_runs_two_turns_and_returns_to_standby(self) -> None: runtime, stt, llm, transport, reporter = make_runtime(["第一问", "第二问"]) self.assertIsInstance(runtime, VoiceAssistantPipeline)