Compare commits
22 Commits
6153cd2826
...
8b05e7c8ff
| Author | SHA1 | Date | |
|---|---|---|---|
 mkbk
|
8b05e7c8ff | ||
|
mkbk
|
caf35783f8 | ||
|
mkbk
|
98ae711971 | ||
|
mkbk
|
81c949a7ec | ||
|
mkbk
|
79b3e89b79 | ||
|
mkbk
|
f31e3d89d6 | ||
|
mkbk
|
af2de5ec66 | ||
|
mkbk
|
504dd2cbf2 | ||
|
mkbk
|
7c6797aeb0 | ||
|
mkbk
|
f1f4885cc5 | ||
|
mkbk
|
a489f8eee1 | ||
|
mkbk
|
e730883c64 | ||
|
mkbk
|
6541bab143 | ||
|
mkbk
|
351e722898 | ||
|
mkbk
|
af7eb25ba6 | ||
|
mkbk
|
dc9566d8d2 | ||
|
mkbk
|
acdcd39e34 | ||
|
mkbk
|
264729ca11 | ||
|
mkbk
|
0cfcb58584 | ||
|
mkbk
|
8b3ffe0ef3 | ||
|
mkbk
|
9fd8eaf7eb | ||
|
mkbk
|
696ed2c30e |
@@ -2,6 +2,8 @@ OWNER_LLM_BASE_URL=https://token-plan-cn.xiaomimimo.com/v1
|
||||
OWNER_LLM_API_KEY=
|
||||
OWNER_LLM_MODEL=mimo-v2.5
|
||||
OWNER_LLM_API_STYLE=chat_completions
|
||||
OWNER_LLM_STREAMING_ENABLED=1
|
||||
OWNER_ASSISTANT_MODE=turn_based_voice_pet
|
||||
OWNER_REALTIME_TRANSCRIPT_ENABLED=1
|
||||
OWNER_REALTIME_TRANSCRIPT_IDLE_TIMEOUT_MS=1500
|
||||
OWNER_AUDIO_INPUT_DEVICE=
|
||||
@@ -50,10 +52,34 @@ OWNER_BARGE_IN_USER_SIMILARITY_THRESHOLD=0.62
|
||||
OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD=0.72
|
||||
OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20
|
||||
OWNER_BARGE_IN_CHUNK_MS=30
|
||||
OWNER_BARGE_IN_DEBUG=0
|
||||
OWNER_END_CHIME_ENABLED=1
|
||||
OWNER_END_CHIME_FILE=assets/sounds/codex-notification.wav
|
||||
OWNER_END_CHIME_FREQUENCY_HZ=880
|
||||
OWNER_END_CHIME_DURATION_MS=140
|
||||
OWNER_AUDIO_APM_PROVIDER=webrtc
|
||||
OWNER_AUDIO_AEC_ENABLED=1
|
||||
OWNER_AUDIO_NS_ENABLED=1
|
||||
OWNER_AUDIO_AGC_ENABLED=1
|
||||
OWNER_AUDIO_APM_REQUIRED=1
|
||||
OWNER_AUDIO_FRAME_MS=20
|
||||
OWNER_AUDIO_RING_BUFFER_MS=3000
|
||||
OWNER_INTERRUPT_ENABLED=1
|
||||
OWNER_INTERRUPT_TARGET_LATENCY_MS=200
|
||||
OWNER_STREAMING_STT_PROVIDER=faster_whisper
|
||||
OWNER_STREAMING_STT_PRODUCT_CANDIDATE=sensevoice
|
||||
OWNER_STREAMING_TTS_PROVIDER=cosyvoice
|
||||
OWNER_MEMORY_ENABLED=1
|
||||
OWNER_MEMORY_PROVIDER=faiss_sqlite
|
||||
OWNER_MEMORY_TOP_K=5
|
||||
OWNER_MEMORY_AUTO_SAVE_SENSITIVE=0
|
||||
OWNER_TOOL_ROUTER_ENABLED=1
|
||||
OWNER_TOOL_MAX_CALLS_PER_TURN=5
|
||||
OWNER_TOOL_TIMEOUT_MS=30000
|
||||
OWNER_OPENINTERPRETER_ENABLED=0
|
||||
OWNER_OPENINTERPRETER_COMMAND=openinterpreter
|
||||
OWNER_BROWSER_PLAYWRIGHT_ENABLED=0
|
||||
OWNER_COMPUTER_CONTROL_ENABLED=0
|
||||
OWNER_WAKE_WORD=小杰小杰
|
||||
OWNER_SAMPLE_RATE=16000
|
||||
OWNER_CHANNELS=1
|
||||
|
||||
@@ -19,3 +19,4 @@ dist/
|
||||
*.egg-info/
|
||||
logs/
|
||||
tmp/
|
||||
openinterpreter/
|
||||
|
||||
@@ -8,6 +8,10 @@
|
||||
|
||||
- `owner_voice_pet run-live`：真实常驻语音循环。
|
||||
- `owner_voice_pet run-live --once`：只跑一轮，便于验收。
|
||||
- `owner_voice_pet run-agent-live`：完整全双工 Agent 主入口，启动后直接 listening，播放中可被有效用户语音打断。
|
||||
- `owner_voice_pet run-agent-live --check-config`：只检查全双工 Agent 配置和 APM 就绪状态，不打开麦克风。
|
||||
- `owner_voice_pet agent-self-test --profile full-duplex --turns 3`：无人值守自测 STT、LLM、TTS、打断、记忆和工具路由。
|
||||
- `owner_voice_pet audio-self-test --duration 10 --check-echo`：检查设备、WebRTC APM、回声抑制和打断延迟。
|
||||
- `.env` 直接读取配置，不要求导出 shell 环境变量。
|
||||
- 唤醒词检测使用本地 `sherpa-onnx` KWS 模型，不走云端 ASR。
|
||||
- `OWNER_SPEECH_PROVIDER=local`：默认除 LLM 外全用本地语音链路；`cloud` 仅作为显式兼容选项。
|
||||
@@ -36,6 +40,8 @@ OWNER_LLM_BASE_URL=https://token-plan-cn.xiaomimimo.com/v1
|
||||
OWNER_LLM_API_KEY=
|
||||
OWNER_LLM_MODEL=mimo-v2.5
|
||||
OWNER_LLM_API_STYLE=chat_completions
|
||||
OWNER_LLM_STREAMING_ENABLED=1
|
||||
OWNER_ASSISTANT_MODE=turn_based_voice_pet
|
||||
OWNER_REALTIME_TRANSCRIPT_ENABLED=1
|
||||
OWNER_REALTIME_TRANSCRIPT_IDLE_TIMEOUT_MS=1500
|
||||
OWNER_WAKE_PROVIDER=local_kws
|
||||
@@ -77,10 +83,34 @@ OWNER_BARGE_IN_USER_SIMILARITY_THRESHOLD=0.62
|
||||
OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD=0.72
|
||||
OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20
|
||||
OWNER_BARGE_IN_CHUNK_MS=30
|
||||
OWNER_BARGE_IN_DEBUG=0
|
||||
OWNER_END_CHIME_ENABLED=1
|
||||
OWNER_END_CHIME_FILE=assets/sounds/codex-notification.wav
|
||||
OWNER_END_CHIME_FREQUENCY_HZ=880
|
||||
OWNER_END_CHIME_DURATION_MS=140
|
||||
OWNER_AUDIO_APM_PROVIDER=webrtc
|
||||
OWNER_AUDIO_AEC_ENABLED=1
|
||||
OWNER_AUDIO_NS_ENABLED=1
|
||||
OWNER_AUDIO_AGC_ENABLED=1
|
||||
OWNER_AUDIO_APM_REQUIRED=1
|
||||
OWNER_AUDIO_FRAME_MS=20
|
||||
OWNER_AUDIO_RING_BUFFER_MS=3000
|
||||
OWNER_INTERRUPT_ENABLED=1
|
||||
OWNER_INTERRUPT_TARGET_LATENCY_MS=200
|
||||
OWNER_STREAMING_STT_PROVIDER=faster_whisper
|
||||
OWNER_STREAMING_STT_PRODUCT_CANDIDATE=sensevoice
|
||||
OWNER_STREAMING_TTS_PROVIDER=cosyvoice
|
||||
OWNER_MEMORY_ENABLED=1
|
||||
OWNER_MEMORY_PROVIDER=faiss_sqlite
|
||||
OWNER_MEMORY_TOP_K=5
|
||||
OWNER_MEMORY_AUTO_SAVE_SENSITIVE=0
|
||||
OWNER_TOOL_ROUTER_ENABLED=1
|
||||
OWNER_TOOL_MAX_CALLS_PER_TURN=5
|
||||
OWNER_TOOL_TIMEOUT_MS=30000
|
||||
OWNER_OPENINTERPRETER_ENABLED=0
|
||||
OWNER_OPENINTERPRETER_COMMAND=openinterpreter
|
||||
OWNER_BROWSER_PLAYWRIGHT_ENABLED=0
|
||||
OWNER_COMPUTER_CONTROL_ENABLED=0
|
||||
```
|
||||
|
||||
`OWNER_WAKE_PROVIDER=local_kws` 表示唤醒词“小杰小杰”由本地模型检测。唤醒命中后会先本地播报 `OWNER_WAKE_ACK_TEXT=我在`，再开始听取问题。`OWNER_SPEECH_PROVIDER=local` 表示正式问题 STT、实时字幕和 TTS 都走本地模型或 macOS 本地能力；云端只接收 final 文本和本次会话历史用于 LLM 回复。
|
||||
@@ -91,10 +121,71 @@ OWNER_END_CHIME_DURATION_MS=140
|
||||
|
||||
`OWNER_CONTINUOUS_DIALOG_ENABLED=1` 表示每轮回复播放后会自动判断是否继续对话。若助手回复里明显在问用户、要求补充信息或让用户选择，终端会输出 `继续对话：3秒内可直接回答`，这 3 秒内可以不用再说“小杰小杰”。若助手只是完成回答、报错、拒绝或判断不确定，就直接恢复待机。`OWNER_CONTINUATION_DECISION_PROVIDER=hybrid` 表示先用本地规则判断，规则不确定时才调用云端 LLM 做小分类；低于 `OWNER_CONTINUATION_CONFIDENCE_THRESHOLD=0.65` 的结果按待机处理。
|
||||
|
||||
`OWNER_BARGE_IN_ENABLED=1` 表示播报中允许打断。播放回复时会启动后台麦克风监听，不再等每个播放 chunk 结束后才检查输入；`OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20` 控制监听间隔，`OWNER_BARGE_IN_CHUNK_MS=30` 控制播放停止粒度。播放开始后的 `OWNER_BARGE_IN_ECHO_GUARD_MS=500` 毫秒内忽略麦克风输入，之后如果检测到至少 `OWNER_BARGE_IN_MIN_SPEECH_MS=250` 毫秒有效用户语音，并且 realtime STT 给出有效 partial，就停止剩余播报。`OWNER_BARGE_IN_SPEAKER_GATE_ENABLED=1` 会同时建立本次会话用户临时音色画像和当前助手回放音色画像，默认要求用户相似度达到 `OWNER_BARGE_IN_USER_SIMILARITY_THRESHOLD=0.62`，并拒绝相似度高于 `OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD=0.72` 的助手回放音色，避免 AI 自己的声音触发打断或实时字幕。音色画像只在进程内使用，不写文件、不发送给 LLM。上下文只记录已经完整播出的 assistant 句子，未播出的内容不会写入临时历史。
|
||||
`OWNER_BARGE_IN_ENABLED=1` 表示播报中允许打断。播放回复时会启动后台麦克风监听，不再等每个播放 chunk 结束后才检查输入；`OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20` 控制监听间隔，`OWNER_BARGE_IN_CHUNK_MS=30` 控制播放停止粒度。`OWNER_BARGE_IN_ECHO_GUARD_MS=500` 是回声保护上限，真实运行时只在每段回复的第一个播报片段使用一次，并压到 120 ms 内，避免每个短句都重新进入 500 ms 免打断窗口。`OWNER_BARGE_IN_MIN_SPEECH_MS=250` 是最短人声配置上限，实际运行会按 `OWNER_INTERRUPT_TARGET_LATENCY_MS=200` 和 chunk 粒度收紧，避免参数本身超过目标打断延迟。播放中如果 VAD 检测到有效用户语音，且不像当前助手回放 reference，就先停止剩余播报；STT 只用于后续识别打断内容，不再作为停播前置条件。`OWNER_BARGE_IN_SPEAKER_GATE_ENABLED=1` 会同时建立本次会话用户临时音色画像和当前助手回放音色画像；用户音色匹配优先于助手回放拒绝，避免“用户说话 + 扬声器回声”混合时被先当成 AI 自己声音丢掉。设置 `OWNER_BARGE_IN_DEBUG=1` 后，终端会输出打断监听启动、VAD 累计、回声门控拒绝原因和触发时长，便于现场定位。音色画像只在进程内使用，不写文件、不发送给 LLM。上下文只记录已经完整播出的 assistant 句子，未播出的内容不会写入临时历史。
|
||||
|
||||
`OWNER_END_CHIME_ENABLED=1` 表示对话自然结束或追问超时恢复待机前会播放一声项目内置提示音，默认文件是 `assets/sounds/codex-notification.wav`。提示音不走 TTS，也不会写入上下文；如果 `OWNER_END_CHIME_FILE` 指向的文件缺失，会回退到本地合成短音，`OWNER_END_CHIME_FREQUENCY_HZ` 和 `OWNER_END_CHIME_DURATION_MS` 只影响这个回退音。设置 `OWNER_END_CHIME_ENABLED=0` 可以关闭。
|
||||
|
||||
## 全双工 Agent 入口
|
||||
|
||||
为避免破坏当前已经可用的旧语音循环，全双工 Agent 使用独立入口：
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m owner_voice_pet run-agent-live
|
||||
```
|
||||
|
||||
`run-agent-live` 不需要先说唤醒词，启动后会直接进入 listening。完整模式默认要求 `OWNER_AUDIO_APM_PROVIDER=webrtc` 和 `OWNER_AUDIO_APM_REQUIRED=1`，麦克风 capture 必须经过 WebRTC APM 的 AEC/NS/AGC 后再进入 VAD、STT 和打断检测；播放 PCM 会同步写入 render reference。若本机没有真实 WebRTC APM provider，入口会以 `AUDIO_APM_UNAVAILABLE` 明确失败，不再回退到旧 `run-live` 或伪全双工。
|
||||
|
||||
只检查全双工配置、不打开麦克风：
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m owner_voice_pet run-agent-live --check-config
|
||||
```
|
||||
|
||||
旧 wake-word turn-based 入口仍保留：
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m owner_voice_pet run-live
|
||||
```
|
||||
|
||||
全双工 Agent 当前架构使用单一 `AudioHub` 拥有麦克风输入，VAD、STT、InterruptController 和诊断订阅各自独立的 processed capture cursor，不再抢读同一个 Transport 队列。打断不等待 STT partial：在 `thinking/speaking/tool_running` 中，只要 processed capture 上的有效用户语音达到阈值，就取消当前 LLM/TTS/playback/tool 子图，并把已确认的用户音频缓存给下一轮输入。`OWNER_LLM_STREAMING_ENABLED=1` 控制 LLM 以流式响应供句子级 TTS 消费，旧变量 `OWNER_LLM_STREAM` 仍兼容；`OWNER_STREAMING_STT_PROVIDER=faster_whisper` 和 `OWNER_STREAMING_TTS_PROVIDER=cosyvoice` 是 provider 目标；`OWNER_MEMORY_ENABLED=1` 和 `OWNER_TOOL_ROUTER_ENABLED=1` 默认开启，`OWNER_OPENINTERPRETER_ENABLED=0`、`OWNER_BROWSER_PLAYWRIGHT_ENABLED=0`、`OWNER_COMPUTER_CONTROL_ENABLED=0` 默认关闭，避免未确认的外部控制自动执行。
|
||||
|
||||
无人值守 Agent 自测：
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m owner_voice_pet agent-self-test --profile full-duplex --turns 3
|
||||
```
|
||||
|
||||
音频/APM 诊断：
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m owner_voice_pet audio-self-test --duration 10 --check-echo
|
||||
```
|
||||
|
||||
如果当前机器没有真实 WebRTC APM binding，`audio-self-test` 会返回 `success=false` 和 `AUDIO_APM_UNAVAILABLE`；这表示完整全双工音频底座尚未满足，不应把 fake APM 结果当成人工验收通过。需要做确定性开发自测时，可以临时使用 `OWNER_AUDIO_APM_PROVIDER=fake`。
|
||||
|
||||
当前已落地的全双工基础模块：
|
||||
|
||||
- `full_duplex_audio`：AudioHub、多消费者 capture/render ring buffer、fake WebRTC APM、APM 探针和 required startup 决策。
|
||||
- `full_duplex_control`：全双工状态机、事件诊断字段、取消 token graph 和恢复协调器。
|
||||
- `full_duplex_speech`：VAD provider contract、Silero VAD 边界、Streaming STT worker、fake STT、InterruptionDetector 和 InterruptController。
|
||||
- `full_duplex_response`：LLM streaming contract、句子切分、TTS 文本净化、Streaming TTS wrapper、fake Streaming TTS 和可中断播放队列。
|
||||
- `full_duplex_runtime`：新 `run-agent-live` runtime 边界，串联 AudioHub、取消图、流式回复、长期记忆和 ToolRouter。
|
||||
- `agent_memory`：SQLite memory schema、FAISS index manifest 校验、fake/disabled memory manager、敏感写入策略和 memory recall 注入。
|
||||
- `tool_router`：结构化工具调用、风险分类、预算防循环、审计脱敏、`memory.search`、`memory.save` 和 `shell.readonly`。
|
||||
- `external_adapters`：Open Interpreter、Playwright 和 Computer Control 的默认关闭边界。
|
||||
- `full_duplex_testing`：fake 全双工 fixture、性能指标和诊断脱敏测试工具。
|
||||
|
||||
可选依赖按能力分组，不会被默认安装强制拉入：
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m pip install -e '.[full-duplex]'
|
||||
.venv/bin/python -m pip install -e '.[streaming-stt]'
|
||||
.venv/bin/python -m pip install -e '.[memory]'
|
||||
.venv/bin/python -m pip install -e '.[browser]'
|
||||
```
|
||||
|
||||
工具执行安全策略见 `docs/full_duplex_agent_security.md`。
|
||||
|
||||
## 本地模型
|
||||
|
||||
首次运行前必须准备本地语音模型；同一脚本会下载 wake、VAD、2025 中文 CTC STT 和 GTCRN denoiser 模型：
|
||||
@@ -155,12 +246,24 @@ python3.11 scripts/download_speech_models.py --dir models
|
||||
.venv/bin/python -m owner_voice_pet run-live --once
|
||||
```
|
||||
|
||||
全双工 Agent 单轮验收：
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m owner_voice_pet run-agent-live --once
|
||||
```
|
||||
|
||||
常驻重复对话：
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m owner_voice_pet run-live
|
||||
```
|
||||
|
||||
常驻全双工 Agent：
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m owner_voice_pet run-agent-live
|
||||
```
|
||||
|
||||
运行后终端状态来自 pipeline event bus，会显示待机、唤醒命中、应答中、请说出问题、录音中、检测到用户语音、实时转写、用户语音结束、转写中、转写结果、思考中、播放中、继续对话或恢复待机等状态。说“小杰小杰”，听到“我在”且看到“请说出问题”后再提问；如果助手回复后判断需要你继续回答，可以在 3 秒内直接说下一句，不需要再次唤醒。若助手已经完成回答，会自动恢复待机。本次进程内会携带临时历史，程序退出后不保存。背景噪声下如果实时字幕仍偶发短错字，先看最终 `转写结果`；最终文本才会进入 LLM。
|
||||
|
||||
## 验证
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
# 全双工 Agent 安全确认策略
|
||||
|
||||
## 默认原则
|
||||
|
||||
全双工 Agent 的工具执行默认最小权限。语音输入可能被误触发，工具请求必须经过结构化校验、风险分类、预算限制、输出脱敏和审计记录。
|
||||
|
||||
## 自动允许
|
||||
|
||||
以下请求可在第一版策略中自动执行：
|
||||
|
||||
- `memory.search`：只读长期记忆检索。
|
||||
- `shell.readonly`：允许目录内的只读命令，例如 `pwd`、`ls`、`find`、`rg`、`cat`、`sed`、`git status`、`git diff`、`git log`、`git show`。
|
||||
- 低风险 dry-run adapter 检查，例如 `run-agent-live --check-config` 这类不改变外部状态的诊断。
|
||||
|
||||
## 需要确认
|
||||
|
||||
以下请求必须进入确认流程：
|
||||
|
||||
- `memory.save` 保存疑似敏感内容。
|
||||
- `openinterpreter.run` 中包含写文件、删除、上传、账号、权限、安装依赖、执行宽泛代码的任务。
|
||||
- `browser.playwright` 中包含登录、支付、购买、提交表单、私信、上传、账号修改或不可逆动作的任务。
|
||||
- 任何自然语言意图中出现删除、上传、支付、购买、交易、账号、权限等高风险关键词的工具调用。
|
||||
|
||||
## 默认拒绝
|
||||
|
||||
以下请求第一版直接拒绝：
|
||||
|
||||
- `shell.readonly` 中出现 `rm`、`chmod`、`chown`、`pip install`、`npm install`、`brew install`、重定向写入、`mv`、`cp` 等修改外部状态的命令。
|
||||
- 未注册工具名、未知参数结构或超过单轮工具预算的请求。
|
||||
- 直接 GUI 点击、键盘输入、屏幕控制或 macOS Accessibility 自动化请求。
|
||||
- 试图绕过确认策略、读取或输出密钥、上传本地敏感文件的请求。
|
||||
|
||||
## 审计和脱敏
|
||||
|
||||
工具审计记录只保存工具名、风险级别、确认状态、耗时、状态和脱敏摘要。API key、Authorization header、原始音频、PCM payload、密码和 token 形态文本必须被替换为 `[redacted]` 或被截断。
|
||||
|
||||
## Open Interpreter 边界
|
||||
|
||||
本地 `openinterpreter/` 是外部候选后端，不复制、不 vendoring、不提交到 Owner。Owner 后续只通过 `OWNER_OPENINTERPRETER_COMMAND` 指定的 CLI/子进程调用它，并由 Tool Router 负责目录限制、超时、输出截断和确认策略。
|
||||
|
||||
## 浏览器边界
|
||||
|
||||
`browser.playwright` 默认关闭，默认使用隔离 browser context。是否允许复用用户当前 Chrome 登录态需要后续人工确认；第一版不默认复用登录态。
|
||||
|
||||
## Computer Control 边界
|
||||
|
||||
Codex Computer Use 只作为安全确认策略参考，不复制私有实现。后续 GUI 控制必须另开 OpenSpec，基于公共能力实现，例如 macOS Accessibility、Playwright 或 trycua-style provider。
|
||||
@@ -0,0 +1,2 @@
|
||||
schema: spec-driven
|
||||
created: 2026-06-18
|
||||
@@ -0,0 +1,318 @@
|
||||
# Design: 完整全双工 Agent 语音助手架构
|
||||
|
||||
## Context
|
||||
|
||||
当前 Owner 项目已经具备真实语音桌宠的基础能力：本地唤醒词、麦克风采集、VAD/端点、实时字幕、final STT、云端 LLM、本地 TTS、播报、连续追问和初步打断规划。现有实现仍以 turn-based 轮次为中心：用户唤醒后系统录一段、识别、生成、播放，再回到待机或短暂追问窗口。
|
||||
|
||||
完整 Agent 语音助手需要更强的架构边界：
|
||||
|
||||
1. 音频输入和输出必须同时运行，而不是录完再播。
|
||||
2. 播放音频必须回注给 AEC reference，避免助手自己的声音污染麦克风输入。
|
||||
3. VAD、STT、LLM、TTS、工具执行和播放都必须能被统一取消。
|
||||
4. 长期记忆和工具执行必须纳入安全策略，而不是让 LLM 随意执行文本命令。
|
||||
5. 终端、未来 GUI 桌宠和自动测试必须消费同一事件流。
|
||||
|
||||
本设计仅用于 OpenSpec 规划。本阶段不修改 `src/`、不安装依赖、不下载模型、不运行外部工具、不接入 Open Interpreter，也不提交本地未跟踪的 `openinterpreter/`。
|
||||
|
||||
## Goals / Non-Goals
|
||||
|
||||
**Goals:**
|
||||
|
||||
1. 定义全双工音频底座：麦克风 capture、扬声器 render reference、WebRTC APM AEC/NS/AGC、环形缓冲和时钟对齐。
|
||||
2. 定义状态机：`idle/listening/thinking/speaking/interrupted/tool_running/recovering`。
|
||||
3. 定义并发任务模型：音频输入、STT、LLM、TTS、播放、工具执行相互独立但可被统一取消。
|
||||
4. 定义低延迟打断：用户在 `speaking` 中说话时，VAD 快速触发 interruption，目标小于 200 ms。
|
||||
5. 定义 Streaming STT/TTS：partial/final transcript、LLM token stream、句子切分、TTS chunk 播放。
|
||||
6. 定义长期记忆：SQLite 保存文本和元数据，FAISS 保存向量索引，敏感内容默认不自动保存。
|
||||
7. 定义 Tool Router：结构化工具调用协议、安全分类、确认策略、执行预算、防循环、结果脱敏回注。
|
||||
8. 定义 Open Interpreter 外部 adapter：只作为 CLI/子进程后端候选，不复制外部仓库。
|
||||
9. 定义 Playwright browser adapter 和未来 `ComputerControlProvider` 预留边界。
|
||||
10. 定义测试策略：fake APM/VAD/STT/TTS/memory/tool、端到端模拟、性能指标和安全回归。
|
||||
|
||||
**Non-Goals:**
|
||||
|
||||
1. 本阶段不实现运行代码。
|
||||
2. 本阶段不安装 WebRTC APM、Silero、Faster Whisper、SenseVoice、CosyVoice、FAISS、Playwright 或 Open Interpreter。
|
||||
3. 本阶段不下载模型。
|
||||
4. 本阶段不改变 `.env`、`pyproject.toml`、`src/`、`tests/`、`scripts/`。
|
||||
5. 第一版规划不做 GUI 点击、键盘、屏幕控制，只预留公共 provider。
|
||||
6. 第一版不做长期主人声纹注册和多人身份鉴权。
|
||||
7. 第一版不自动执行高风险电脑控制、上传、交易、删除、权限修改或账号操作。
|
||||
|
||||
## Decisions
|
||||
|
||||
### Decision 1: WebRTC APM 作为默认音频底座
|
||||
|
||||
采用 `WebRtcAudioProcessingStage` 作为全双工默认音频预处理层，启用 AEC、NS、AGC。
|
||||
|
||||
**理由：**
|
||||
|
||||
1. AEC 需要播放 reference 才能可靠消除助手自己的声音，音色门控只能做后验抑制。
|
||||
2. NS 能降低风扇、环境声和麦克风底噪对持续 VAD/STT 的影响。
|
||||
3. AGC 能减少用户远近变化导致的阈值不稳定。
|
||||
|
||||
**替代方案：**
|
||||
|
||||
1. 继续使用 GTCRN 降噪：能改善噪音，但不能处理扬声器回声。
|
||||
2. 继续使用音色门控：实现成本低，但对不同扬声器、房间回声和 TTS 音色变化不稳定。
|
||||
3. 只靠 VAD/STT 置信度：误触发风险高，不适合作为全双工底座。
|
||||
|
||||
### Decision 2: capture/render 双环形缓冲
|
||||
|
||||
设计 `CaptureRingBuffer` 和 `RenderReferenceRingBuffer`。麦克风帧和播放帧都按 monotonic timestamp 写入，APM 处理 capture frame 时读取相邻 render reference。
|
||||
|
||||
**理由：**
|
||||
|
||||
1. 全双工需要持续输入和持续输出，不能用同步读写阻塞。
|
||||
2. AEC 对 reference 时序敏感，必须保留时间戳和 drift 监控。
|
||||
3. Ring buffer 可为 STT、VAD、打断检测和测试提供一致帧来源。
|
||||
|
||||
**替代方案：**
|
||||
|
||||
1. 直接从 sounddevice callback 推到各 stage：耦合高，难测试，易阻塞。
|
||||
2. 文件式临时 wav：延迟高，不适合全双工。
|
||||
|
||||
### Decision 3: 统一状态机和事件总线
|
||||
|
||||
新增 `FullDuplexAgentStateMachine` 和 `PipelineEventBus`。状态机只负责合法转移，事件总线承载终端、GUI 和测试可观察行为。
|
||||
|
||||
**理由：**
|
||||
|
||||
1. 现有 turn-based 状态无法表达 `tool_running` 和 `interrupted`。
|
||||
2. 终端和未来 GUI 必须共享事件，不应各自读取内部字段。
|
||||
3. 测试可以断言事件顺序、耗时和错误恢复。
|
||||
|
||||
### Decision 4: Cancellation Graph 管理所有可中断任务
|
||||
|
||||
每个用户输入 turn 创建 root cancellation token，LLM、TTS、playback、tool 子任务挂在 root 下。用户打断时 root token 广播取消。
|
||||
|
||||
**理由：**
|
||||
|
||||
1. 播放停止但 LLM 继续生成会浪费资源并污染上下文。
|
||||
2. TTS 合成继续运行会造成卡顿和旧回复残留。
|
||||
3. 工具执行必须区分可取消和不可安全取消。
|
||||
|
||||
### Decision 5: Streaming STT 分层输出
|
||||
|
||||
`StreamingSttProvider` 输出 `partial`、`stable_partial` 和 `final`。终端可展示 stable partial；LLM 默认只接收 final。
|
||||
|
||||
**理由：**
|
||||
|
||||
1. 用户需要实时看到系统听到了什么。
|
||||
2. partial 抖动不能直接写入上下文。
|
||||
3. final 是唯一可靠的 LLM 用户输入。
|
||||
|
||||
**开发候选：**
|
||||
|
||||
1. `faster-whisper`：开发阶段优先，生态成熟。
|
||||
2. `SenseVoice`：产品候选，中文和情绪能力更强，需要评估部署成本。
|
||||
3. `sherpa-onnx`：保留当前兼容 adapter，降低迁移风险。
|
||||
|
||||
### Decision 6: LLM token stream 按句进入 Streaming TTS
|
||||
|
||||
LLM delta 进入 `SentenceSegmenter`。达到完整句子、语义停顿或最大等待阈值后，将净化后的文本交给 TTS。
|
||||
|
||||
**理由：**
|
||||
|
||||
1. 不等待完整回复能显著缩短首音延迟。
|
||||
2. 句子级 TTS 比 token 级 TTS 更自然。
|
||||
3. 现有 TTS sanitizer 可复用，避免表情包、emoji 被读出。
|
||||
|
||||
### Decision 7: 长期记忆使用 FAISS + SQLite
|
||||
|
||||
SQLite 保存记忆文本、类型、敏感度、来源、时间和 checksum；FAISS 保存 embedding 向量。
|
||||
|
||||
**理由：**
|
||||
|
||||
1. SQLite 适合可审计元数据和删除。
|
||||
2. FAISS 适合向量相似检索。
|
||||
3. 两者分离便于一致性检查和索引重建。
|
||||
|
||||
**替代方案：**
|
||||
|
||||
1. 只用 SQLite FTS：部署简单，但语义召回弱。
|
||||
2. 直接把历史对话全文塞进 prompt：隐私和成本都不可控。
|
||||
3. 使用外部向量数据库：当前单机桌宠不需要额外服务复杂度。
|
||||
|
||||
### Decision 8: Tool Router 先安全工具后电脑控制
|
||||
|
||||
第一版只规划低风险工具：`memory.search`、`memory.save`、`shell.readonly`、`openinterpreter.run`、`browser.playwright`。GUI 电脑控制只预留。
|
||||
|
||||
**理由：**
|
||||
|
||||
1. 语音助手执行工具的风险高于文本聊天，必须先定义确认策略。
|
||||
2. Open Interpreter 能节省电脑任务开发时间，但必须被当成外部受限后端。
|
||||
3. GUI 点击/键盘/屏幕控制需要 Accessibility 权限和更复杂安全策略，不适合第一阶段自动执行。
|
||||
|
||||
### Decision 9: Open Interpreter 作为外部 CLI adapter
|
||||
|
||||
本地 `openinterpreter/` 是未跟踪新 Rust 版 Open Interpreter。Owner 不复制、不 vendoring、不提交它，只在后续实现中通过命令路径调用外部 CLI。
|
||||
|
||||
**理由：**
|
||||
|
||||
1. 避免把外部大型仓库混入 Owner。
|
||||
2. 保持 Owner 的工具边界清晰。
|
||||
3. 可以对 CLI 调用做超时、目录限制、输出截断和确认策略。
|
||||
|
||||
### Decision 10: Codex Computer Use 只参考安全策略
|
||||
|
||||
Codex App 的 Computer Use 能力可作为“需要确认、限制高风险操作、避免私自执行 GUI 动作”的安全策略参考，但不复制私有实现。
|
||||
|
||||
**理由：**
|
||||
|
||||
1. 用户希望从 Codex 控制电脑技能中复刻思路，而不是复制私有实现。
|
||||
2. 公共落地路线应基于 macOS Accessibility、Playwright、trycua 等能力。
|
||||
3. 第一版先不做 GUI 控制可降低安全和实现风险。
|
||||
|
||||
## Concurrent Task Model
|
||||
|
||||
### Runtime task groups
|
||||
|
||||
1. `audio_capture_task`：从麦克风读取帧，写入 capture ring buffer。
|
||||
2. `audio_render_task`：从 playback queue 取 PCM，送扬声器并写入 render reference buffer。
|
||||
3. `apm_task`：处理 capture frames，输出 cleaned frames。
|
||||
4. `vad_interrupt_task`：对 cleaned frames 做 VAD 和打断检测。
|
||||
5. `stt_task`：消费 cleaned frames，输出 partial/stable/final transcript。
|
||||
6. `llm_task`：消费 final transcript、记忆和工具结果，输出 token stream 或 tool call。
|
||||
7. `tts_task`：消费句子片段，输出 PCM chunks。
|
||||
8. `tool_task`：执行已批准工具，输出脱敏结果。
|
||||
9. `event_task`：聚合事件、指标和 reporter 输出。
|
||||
|
||||
### Backpressure
|
||||
|
||||
1. Ring buffer 有固定容量，超出容量丢弃最旧非关键帧并发出 `audio_buffer_overrun`。
|
||||
2. TTS queue 超限时暂停 LLM sentence enqueue 或请求 LLM stream 暂停/取消。
|
||||
3. Tool output 超限时截断并返回 `truncated=true`。
|
||||
|
||||
### Cancellation
|
||||
|
||||
1. `interrupt_detected` 触发 root cancellation。
|
||||
2. LLM stream 立即关闭连接或停止读取。
|
||||
3. TTS session 停止合成并释放资源。
|
||||
4. Playback queue 清空未播放 chunks，保留已播放文本边界。
|
||||
5. 可取消工具收到 token 后停止；不可取消工具标记为 pending cleanup。
|
||||
|
||||
## Audio Ring Buffer Design
|
||||
|
||||
### Frame format
|
||||
|
||||
1. 默认 16 kHz 或 48 kHz 内部采样率需在实现前确认。
|
||||
2. 默认 mono capture；render reference 可 mono 或 stereo downmix。
|
||||
3. 默认 frame size 为 20 ms。
|
||||
4. 每帧包含 `frame_id`、`timestamp_monotonic_ms`、`sample_rate`、`channels`、`samples`。
|
||||
|
||||
### Reference alignment
|
||||
|
||||
1. render frame 写入 reference buffer 时记录实际播放排队时间和预计播放时间。
|
||||
2. capture frame 进入 APM 时按 timestamp 查找 reference window。
|
||||
3. drift 超过阈值时发 `audio_reference_drift`，并降低打断置信度。
|
||||
|
||||
### Fallback
|
||||
|
||||
1. APM 不可用时，若 `OWNER_AUDIO_APM_REQUIRED=1`，全双工模式启动失败。
|
||||
2. 若允许 fallback，则使用现有 GTCRN denoiser、assistant playback gate 和 conservative VAD，但必须标记 `apm_fallback=true`。
|
||||
|
||||
## Tool Router Protocol
|
||||
|
||||
### Tool call request
|
||||
|
||||
```text
|
||||
ToolCallRequest:
|
||||
id: str
|
||||
name: str
|
||||
arguments: dict
|
||||
requested_by_turn_id: str
|
||||
natural_language_intent: str
|
||||
timeout_ms: int
|
||||
```
|
||||
|
||||
### Tool decision
|
||||
|
||||
```text
|
||||
ToolDecision:
|
||||
action: execute | reject | require_confirmation
|
||||
risk_level: low | medium | high | forbidden
|
||||
reason: str
|
||||
sanitized_arguments: dict
|
||||
confirmation_prompt: str | null
|
||||
```
|
||||
|
||||
### Tool result
|
||||
|
||||
```text
|
||||
ToolResult:
|
||||
id: str
|
||||
status: success | failed | cancelled | rejected | confirmation_required
|
||||
output_text: str
|
||||
output_truncated: bool
|
||||
error_code: str | null
|
||||
duration_ms: int
|
||||
audit_summary: str
|
||||
```
|
||||
|
||||
### First-version tools
|
||||
|
||||
1. `memory.search`：只读，低风险。
|
||||
2. `memory.save`：低到中风险，敏感内容需要确认。
|
||||
3. `shell.readonly`：低到中风险，只允许 allowlist 只读命令。
|
||||
4. `openinterpreter.run`：默认中风险，第一版默认关闭；执行前必须经过目录、意图和风险校验。
|
||||
5. `browser.playwright`：默认中风险，第一版默认关闭；账号、支付、提交、购买流程必须确认或拒绝。
|
||||
|
||||
## Memory Retrieval Chain
|
||||
|
||||
1. 用户 final transcript 进入 `MemoryQueryBuilder`。
|
||||
2. Query builder 结合当前任务、会话摘要和用户文本生成检索 query。
|
||||
3. Embedding provider 生成 query vector。
|
||||
4. FAISS 返回候选 ids。
|
||||
5. SQLite 读取 metadata，过滤 disabled、sensitive、expired、low-confidence records。
|
||||
6. Re-ranker 按相似度、类型、最近使用、用户显式偏好排序。
|
||||
7. Top-K 以独立 `memory_context` 注入 LLM。
|
||||
8. LLM 回复完成后，`MemoryWriteCandidateExtractor` 生成候选记忆。
|
||||
9. Safety classifier 决定保存、丢弃或请求确认。
|
||||
|
||||
## Migration Plan
|
||||
|
||||
1. 保留现有 turn-based `run-live` 作为稳定路径。
|
||||
2. `run-agent-live` 是真实全双工 Agent 入口；`--check-config` 只做配置检查，不带该参数时必须启动运行时。
|
||||
3. 先落地 fake provider 和模拟端到端，再接真实 WebRTC APM。
|
||||
4. STT/TTS provider 先用兼容 adapter 接现有能力，再替换为 streaming provider。
|
||||
5. 长期记忆默认关闭或空库启动，完成删除/禁用/隐私文档后再默认开启。
|
||||
6. Tool Router 默认只启用 `memory.search`；其他工具按风险逐步开放。
|
||||
7. Open Interpreter 和 Playwright adapter 默认关闭，用户显式配置后才可用。
|
||||
8. GUI 电脑控制另开 OpenSpec 变更，不混入第一版全双工音频和安全工具验收。
|
||||
|
||||
## Live Runtime Revision
|
||||
|
||||
第一版真实 `run-agent-live` 采用软件 render-reference gate，不新增 WebRTC APM 重依赖。播放队列把已播放 PCM chunk 写入进程内 render reference；后台麦克风监听持续读取 capture frames，先用 VAD 判断有效人声，再与近期 render reference 做相似度/能量门控。候选音频不像助手回放且持续达到最短人声时长时，立即设置 playback stop event；STT 只用于打断后的用户文本识别，不再作为停止播放的前置条件。
|
||||
|
||||
`run-live` 保持旧 wake/turn-based 入口。`run-agent-live` 跳过唤醒词和 ACK，常驻 `listening -> thinking -> speaking -> interrupted/listening`，打断后把已确认的用户音频接到下一轮 capture，避免丢首字。长期记忆、Tool Router、Open Interpreter 仍默认关闭。
|
||||
|
||||
## Rollback Strategy
|
||||
|
||||
1. 若全双工音频不稳定，可通过配置回退 turn-based `run-live`。
|
||||
2. 若 APM provider 失败，可禁用 full-duplex mode 或启用 fallback 标记的旧链路。
|
||||
3. 若长期记忆异常，可设置 `OWNER_MEMORY_ENABLED=0`，继续短期对话。
|
||||
4. 若 Tool Router 风险过高，可设置 `OWNER_TOOL_ROUTER_ENABLED=0`，保留纯聊天。
|
||||
5. 若 Open Interpreter 或 Playwright adapter 出错，只禁用对应工具，不影响语音主循环。
|
||||
|
||||
## Risks / Trade-offs
|
||||
|
||||
1. WebRTC APM 依赖复杂 -> 通过 provider 抽象、fake APM 测试和 fallback 降低风险。
|
||||
2. 全双工并发复杂 -> 通过 bounded queue、cancellation graph、虚拟时钟测试和事件指标控制。
|
||||
3. 流式 STT/TTS 模型较重 -> 分阶段实现，先 provider interface 和 fake tests，再接真实模型。
|
||||
4. 长期记忆有隐私风险 -> 默认敏感不保存、可禁用、可删除、可审计。
|
||||
5. 工具执行有安全风险 -> 安全工具优先，高风险确认或拒绝，输出脱敏和审计日志。
|
||||
6. Open Interpreter 能力强但风险大 -> 默认关闭、低风险受限执行、外部 CLI adapter、禁止复制仓库。
|
||||
7. Playwright 登录态风险 -> 第一版不默认使用用户登录态，敏感流程确认或拒绝。
|
||||
8. 第一阶段 scope 大 -> 按音频、状态机、STT/TTS、记忆、工具拆分里程碑，每个里程碑独立提交。
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. 全双工入口是否命名为 `run-agent-live`，还是通过 `run-live --mode full-duplex` 启用。
|
||||
2. WebRTC APM 的具体 Python/macOS 绑定选择需要验证。
|
||||
3. 内部采样率统一用 16 kHz 还是 48 kHz，需要结合 APM、STT、TTS provider 决定。
|
||||
4. SenseVoice 和 CosyVoice 是否作为产品强依赖，还是只保留候选。
|
||||
5. 长期记忆是否默认开启，需要用户确认隐私预期。
|
||||
6. Open Interpreter 是否允许写操作；若允许，确认流程和 sandbox 需要单独设计。
|
||||
7. Playwright 是否允许使用现有 Chrome 登录态。
|
||||
8. 后续 ComputerControlProvider 是否采用 macOS Accessibility、trycua 或其他公共方案。
|
||||
@@ -0,0 +1,526 @@
|
||||
# OpenSpec：完整全双工 Agent 语音助手架构
|
||||
|
||||
## 功能目标
|
||||
|
||||
### 完整业务价值
|
||||
|
||||
当前 Owner 语音桌宠已经从最初的“唤醒词 -> VAD -> STT -> LLM -> TTS -> 播放”演进到可真实运行的 `run-live`，并具备本地唤醒、实时字幕、噪音过滤、连续追问、结束提示音和初步播报打断规划。但现有核心仍然是 turn-based 语音桌宠：用户先唤醒，系统听完一段话，识别完再思考，回复播放完再决定是否继续。这个模型能完成普通问答，但距离“小爱同学式持续对话”和完整 Agent 仍有明显差距。
|
||||
|
||||
本变更的业务目标是把现有语音桌宠规划升级为完整全双工 Agent 语音助手：麦克风持续输入、播放端持续输出、WebRTC APM 提供 AEC/NS/AGC 音频底座、Streaming STT 实时生成用户话语、LLM 流式回复按句进入 Streaming TTS、用户在 AI 讲话时可以自然打断，长期记忆和 Tool Router 让助手不只聊天，还能在安全边界内检索记忆、执行只读系统任务、调用 Open Interpreter 外部后端和 Playwright 浏览器自动化。
|
||||
|
||||
本阶段只产出 OpenSpec 审核材料，不改运行代码、不安装依赖、不下载模型、不接入工具执行、不把本地未跟踪的 `openinterpreter/` 复制进 Owner 代码，也不提交该目录。`openinterpreter/` 在本变更中只被定义为外部 CLI/子进程工具后端候选。
|
||||
|
||||
### 目标用户场景
|
||||
|
||||
1. 用户在 macOS 上启动语音助手后，不需要按住说话；麦克风常驻监听，系统通过本地唤醒或持续 VAD/STT 管理输入。
|
||||
2. 用户听 AI 回复时可以直接插话；系统在 200 ms 目标延迟内停止 TTS 播放、取消当前 LLM/TTS 任务，并把用户新话语作为下一轮输入。
|
||||
3. AI 自己的扬声器声音不会被误当成用户输入；WebRTC APM 的 AEC 使用播放 reference 音频消除回声，NS 降低环境噪声，AGC 统一麦克风音量。
|
||||
4. 用户可以连续追问，不必每轮都说“小杰小杰”；Conversation Manager 根据状态机和上下文判断当前是继续听、执行工具、播报结果还是恢复待机。
|
||||
5. 助手可长期记住非敏感偏好、事实、项目摘要和任务摘要；重启后仍可通过 FAISS+SQLite 召回相关记忆，但敏感信息默认不自动保存。
|
||||
6. 用户要求“整理下载目录”“查一下项目文件”“打开网页抓取内容”时，Tool Router 先做安全分类，低风险只读工具可执行，高风险写操作、删除、上传、交易、权限变更必须要求确认，第一版不自动做 GUI 点击/键盘控制。
|
||||
7. 后续桌宠 GUI 可订阅同一事件流展示听、想、说、工具执行、被打断、等待确认等状态，而不是重新实现 pipeline 逻辑。
|
||||
|
||||
### 量化成功指标 KPI
|
||||
|
||||
1. 打断延迟：在可控音频测试环境中，`speaking` 状态下检测到有效用户语音后，进入 `interrupted` 并停止播放的 P95 延迟目标小于 200 ms。
|
||||
2. 回声抑制：播放 reference 音频注入 AEC 后，纯助手回放不触发用户 VAD/STT，不产生有效 `barge_in_detected`。
|
||||
3. 实时识别：用户开始说话后，Streaming STT 首个稳定 partial transcript 的 P95 目标小于 800 ms。
|
||||
4. 端到端首音：LLM 首个可播报句子产生后，Streaming TTS 首个可播放 PCM chunk 的 P95 目标小于 1000 ms。
|
||||
5. 流式播报：LLM 不等待完整回复；中文回复按句切分进入 TTS，第一句可播放后立即播放。
|
||||
6. 取消可靠性：用户打断时，当前 LLM stream、TTS synthesis、playback、工具执行候选任务均收到 cancellation token；被取消内容不得继续写入上下文。
|
||||
7. 记忆召回准确性：保存的偏好/事实/项目摘要在相关查询中可被 Top-K 检索召回；关闭记忆时不得读写 SQLite/FAISS。
|
||||
8. 工具安全：未授权工具、越权路径、写操作、删除操作、账号/上传/交易类任务必须被拒绝或进入确认流程；自动化测试覆盖非法工具拒绝、防循环、超时、输出截断。
|
||||
9. 兼容性：现有 turn-based `run-live` 语义在迁移期不得被破坏；新全双工入口可分阶段实现，允许先以 feature flag 启用。
|
||||
10. OpenSpec 完整性：`openspec validate add-full-duplex-agent-voice-assistant --strict` 与 `openspec validate --all --strict` 必须通过。
|
||||
|
||||
### 预期影响
|
||||
|
||||
1. 主规范 `voice-pet-pipeline` 将从“Python 桌宠语音 pipeline”升级为“全双工 Agent 语音助手 pipeline”，覆盖音频底座、并发模型、记忆、工具和安全策略。
|
||||
2. 后续实现将新增 WebRTC APM provider、全双工音频环形缓冲、Streaming STT/TTS provider、Conversation Manager、MemoryManager、ToolRouter、Open Interpreter adapter、Playwright browser adapter。
|
||||
3. 现有 GTCRN 降噪、主说话人音色门控和本地 KWS 保留为 fallback 或局部能力，不再作为完整全双工回声消除的主方案。
|
||||
4. 当前进程内临时上下文保留，但长期记忆成为独立层；临时会话历史和长期记忆必须有清楚边界。
|
||||
5. 工具执行引入新的安全面，必须规划确认策略、权限边界、路径限制、超时、输出截断、审计日志和敏感信息脱敏。
|
||||
6. 第一版电脑控制不做 GUI 点击/键盘/屏幕控制；只预留 `ComputerControlProvider`。Codex Computer Use 只参考安全确认策略，不复制私有或捆绑实现。
|
||||
|
||||
### 对现有问题的系统性总结
|
||||
|
||||
1. 性能问题：现有 turn-based pipeline 在录音结束后才进入 final STT 和 LLM，用户感知延迟集中爆发；全双工架构要求持续识别和流式回复。
|
||||
2. 打断问题：当前打断规划仍围绕播放 chunk 检查或后台监听补丁，缺少全局 cancellation graph；用户插话无法统一取消 LLM、TTS、播放和工具任务。
|
||||
3. 回声问题：现有音色门控只能降低误触发，无法从音频底座消除扬声器回放；没有 AEC reference，后续 VAD/STT 容易被 AI 自己声音污染。
|
||||
4. 噪音问题：GTCRN 只覆盖正式问题采集阶段；全双工持续监听需要系统级 NS/AGC，避免背景噪声导致持续 STT 和打断误判。
|
||||
5. 架构问题：现有 pipeline 仍以“轮次”为主，状态机缺少 `tool_running`、`interrupted`、`recovering` 等 Agent 必需状态。
|
||||
6. 记忆问题：当前上下文只存在于本次进程内；无法记住长期偏好、项目背景和任务摘要，也没有隐私分类和删除策略。
|
||||
7. 工具问题：当前 LLM 只能生成自然语言；没有结构化工具协议、工具路由、安全策略、执行预算、防循环机制和工具结果回注。
|
||||
8. Open Interpreter 边界问题：本地 `openinterpreter/` 是未跟踪外部仓库，不能复制进 Owner；需要把它规划为可选外部 CLI 后端并限制风险。
|
||||
9. UI/UX 问题：后续桌宠如果直接绑定 runtime 方法，会继续重复逻辑；需要统一事件总线承载状态、字幕、音频、打断、工具和确认请求。
|
||||
10. 安全问题：长期记忆和工具执行都会扩大数据面与操作面；必须默认最小权限、敏感内容不自动保存、高风险操作确认、日志脱敏。
|
||||
|
||||
## 详细需求
|
||||
|
||||
### 功能需求
|
||||
|
||||
1. 系统 SHALL 新增完整全双工 Agent 语音助手架构规划，保留现有 turn-based 能力作为迁移期兼容路径。
|
||||
2. 系统 SHALL 规划 `ContinuousAudioRuntime`，负责麦克风持续输入、扬声器播放 reference、音频环形缓冲、状态机事件和任务取消。
|
||||
3. 系统 SHALL 规划 `WebRtcAudioProcessingStage`，默认 `OWNER_AUDIO_APM_PROVIDER=webrtc`，开启 AEC、NS、AGC。
|
||||
4. 播放链路 SHALL 把 TTS PCM/render audio 提供给 AEC reference，麦克风 capture 音频经过 APM 后再进入 VAD、STT 和打断检测。
|
||||
5. 系统 SHALL 规划全双工状态机：`idle`、`listening`、`thinking`、`speaking`、`interrupted`、`tool_running`、`recovering`。
|
||||
6. 任意状态下检测到有效用户说话 SHALL 能进入 `interrupted`，但 `tool_running` 的中断语义必须区分可取消工具和不可安全取消工具。
|
||||
7. 系统 SHALL 规划 Silero VAD 或等价本地 VAD 作为打断触发的主要人声检测层；打断检测先判断“有人开始说话”，不依赖已识别出完整文本。
|
||||
8. 打断目标 SHALL 是在 `OWNER_BARGE_IN_TARGET_LATENCY_MS=200` 内停止播放和取消当前回复。
|
||||
9. 系统 SHALL 规划 Streaming STT provider；开发默认候选为 `faster-whisper`，产品候选为 `SenseVoice`，保留现有 `sherpa-onnx` 兼容 adapter。
|
||||
10. Streaming STT SHALL 输出 partial transcript、stable partial transcript 和 final transcript；只有 final transcript 或明确提交的 stable transcript 可进入 LLM。
|
||||
11. 系统 SHALL 规划 Streaming TTS provider；目标候选为 `CosyVoice`，支持句子级和流式 PCM 输出。
|
||||
12. LLM SHALL 流式输出 token；Sentence Segmenter SHALL 在检测到完整中文/英文句子或安全停顿时，把文本片段送入 TTS，不等待整段回复完成。
|
||||
13. TTS 播放 SHALL 支持中途停止；停止后未完整播出的 assistant 文本不得写入短期上下文或长期记忆。
|
||||
14. 系统 SHALL 规划 `ConversationManager`，负责短期会话历史、长期记忆召回、工具调用闭环和状态推进。
|
||||
15. 系统 SHALL 规划 `MemoryManager`，默认 `OWNER_MEMORY_PROVIDER=faiss_sqlite`，SQLite 存文本和元数据，FAISS 存向量索引。
|
||||
16. 记忆类型 SHALL 至少包含 `preference`、`fact`、`project`、`task_summary`。
|
||||
17. 每轮用户输入进入 LLM 前 SHALL 根据当前用户文本、会话摘要和任务上下文检索 Top-K 长期记忆，并以明确的 memory context 注入 LLM。
|
||||
18. 敏感内容 SHALL 默认不自动保存；记忆写入必须经过分类器、安全策略或用户明确指令。
|
||||
19. 系统 SHALL 规划 `ToolRouter` 和结构化工具调用协议，工具请求包含 name、arguments、risk_level、requires_confirmation、timeout_ms、budget、cancellation_policy。
|
||||
20. 第一版工具 SHALL 以安全工具为主：`memory.search`、`memory.save`、`shell.readonly`、`openinterpreter.run`、`browser.playwright`。
|
||||
21. `shell.readonly` SHALL 限制为只读命令和允许目录，禁止删除、写文件、修改权限、网络上传、安装依赖等高风险行为。
|
||||
22. `openinterpreter.run` SHALL 作为外部 CLI/子进程 adapter，默认只允许低风险、受限目录、超时和输出截断任务。
|
||||
23. `browser.playwright` SHALL 作为浏览器自动化 adapter，默认只允许可审计、非支付、非账号敏感的浏览和提取流程。
|
||||
24. 第一版 SHALL NOT 实现 GUI 点击/键盘/屏幕控制；只在规范中预留 `ComputerControlProvider`，后续可基于 macOS Accessibility、Playwright、trycua 等公共能力实现。
|
||||
25. Codex Computer Use 能力 SHALL 仅作为安全确认策略参考，不复制私有实现、捆绑脚本或内部协议。
|
||||
26. 系统 SHALL 规划工具结果回注：工具输出进入 Tool Result Message，经过脱敏和长度限制后返回 LLM；工具失败进入可恢复错误路径。
|
||||
27. 系统 SHALL 规划防循环策略：单轮最大工具调用次数、最大总耗时、最大输出字节、重复工具调用检测。
|
||||
28. 系统 SHALL 规划安全确认策略：写文件、删除、上传、交易、账号、权限、联网提交、安装依赖、执行任意代码等高风险操作必须确认，第一版默认拒绝自动执行。
|
||||
29. 系统 SHALL 规划桌宠/终端共用事件总线，事件覆盖 audio、vad、stt、llm、tts、playback、memory、tool、confirmation、interruption、recovery。
|
||||
30. 本阶段 SHALL 只创建 OpenSpec 文档；不得创建或修改 `src/`、`tests/`、`scripts/`、`pyproject.toml`、`.env`、模型文件、音频资产或运行时代码。
|
||||
|
||||
### 非功能需求
|
||||
|
||||
1. 性能优化：全双工音频处理必须以固定帧长和环形缓冲为基础，避免 Python 线程阻塞导致播放卡顿或输入积压。
|
||||
2. 延迟目标：VAD frame interval 默认不超过 20 ms；播放停止 chunk 默认不超过 30 ms；整体打断目标小于 200 ms。
|
||||
3. 稳定性：所有 provider 必须支持超时、取消、关闭和资源释放；异常必须进入 `recovering`，不得让后台线程泄漏。
|
||||
4. 可扩展性：APM、VAD、STT、TTS、LLM、Memory、Tool adapter 都必须是可替换 provider，不能把具体模型硬编码进状态机。
|
||||
5. UI/UX：终端和未来 GUI 不直接调用 provider，只消费事件；显示文案、字幕、工具确认和桌宠动画均由事件驱动。
|
||||
6. 安全：API key、Authorization header、原始音频、声纹特征、工具敏感输出不得写入日志或长期记忆。
|
||||
7. 隐私：默认不保存原始麦克风音频；长期记忆只保存文本摘要和必要元数据；用户必须能关闭记忆。
|
||||
8. 兼容性：现有 `.env` 中 LLM 配置继续可用；新增配置必须有默认值和迁移说明。
|
||||
9. 可测试性：每个并发 stage 必须可用 fake provider 和虚拟时钟测试；端到端模拟不依赖真实麦克风、扬声器或外部工具。
|
||||
10. 可观测性：事件必须携带 turn/session id、stage、时间戳、latency、error code 和脱敏 payload，便于定位卡顿、误触发和工具风险。
|
||||
|
||||
### 边缘案例
|
||||
|
||||
1. APM 初始化失败：系统必须报告 `AUDIO_APM_UNAVAILABLE`，可按配置降级到现有 GTCRN/音色门控 fallback 或拒绝进入全双工模式。
|
||||
2. AEC reference 丢失：播放中没有 reference 音频时，系统必须降低打断置信度或临时禁用高风险 barge-in，避免 AI 自己声音触发。
|
||||
3. 麦克风权限缺失：启动失败并提示设备检查，不进入假监听状态。
|
||||
4. 用户在 AI 说第一个字前打断：取消 LLM/TTS stream，未播报文本不写入上下文。
|
||||
5. 用户在工具执行中打断：可取消工具立即取消；不可安全取消工具进入“正在收尾/等待结果”状态，并向用户播报或显示限制。
|
||||
6. LLM 已发起工具调用但用户打断：未执行工具调用应取消；已执行且低风险的只读工具结果可丢弃或标记为 stale。
|
||||
7. 记忆库损坏：FAISS 或 SQLite 不可用时，系统可禁用长期记忆并继续基本语音对话，但必须报告结构化错误。
|
||||
8. 记忆召回命中敏感内容：默认不注入 LLM，除非用户明确要求并通过安全策略。
|
||||
9. Open Interpreter 路径缺失：`openinterpreter.run` adapter 标记不可用，不影响其他工具。
|
||||
10. 工具输出过长：按配置截断并附 `truncated=true` 元数据，不把完整大输出塞进 LLM。
|
||||
11. Playwright 未安装或浏览器不可用：工具返回可恢复错误，不影响语音主循环。
|
||||
12. TTS 输出卡顿：播放队列应能反压 TTS 合成；卡顿事件必须可观测，不能阻塞麦克风监听线程。
|
||||
13. Streaming STT partial 抖动：只显示稳定 partial；final transcript 才进入对话。
|
||||
14. 网络 LLM 慢或断线：取消和超时必须生效；恢复后回到 listening 或 idle。
|
||||
15. 多人同时说话：第一版不承诺身份鉴权，只要求 AEC 后的人声触发和用户体验合理；多人区分列为需人工澄清。
|
||||
|
||||
### 输入输出规格
|
||||
|
||||
新增或规划配置：
|
||||
|
||||
1. `OWNER_ASSISTANT_MODE=full_duplex_agent`
|
||||
2. `OWNER_AUDIO_APM_PROVIDER=webrtc`
|
||||
3. `OWNER_AUDIO_AEC_ENABLED=1`
|
||||
4. `OWNER_AUDIO_NS_ENABLED=1`
|
||||
5. `OWNER_AUDIO_AGC_ENABLED=1`
|
||||
6. `OWNER_AUDIO_FRAME_MS=20`
|
||||
7. `OWNER_AUDIO_RING_BUFFER_MS=3000`
|
||||
8. `OWNER_VAD_PROVIDER=silero`
|
||||
9. `OWNER_INTERRUPT_ENABLED=1`
|
||||
10. `OWNER_INTERRUPT_TARGET_LATENCY_MS=200`
|
||||
11. `OWNER_STREAMING_STT_PROVIDER=faster_whisper`
|
||||
12. `OWNER_STREAMING_STT_PRODUCT_CANDIDATE=sensevoice`
|
||||
13. `OWNER_STREAMING_TTS_PROVIDER=cosyvoice`
|
||||
14. `OWNER_LLM_STREAMING_ENABLED=1`
|
||||
15. `OWNER_MEMORY_ENABLED=1`
|
||||
16. `OWNER_MEMORY_PROVIDER=faiss_sqlite`
|
||||
17. `OWNER_MEMORY_TOP_K=5`
|
||||
18. `OWNER_MEMORY_AUTO_SAVE_SENSITIVE=0`
|
||||
19. `OWNER_TOOL_ROUTER_ENABLED=1`
|
||||
20. `OWNER_TOOL_MAX_CALLS_PER_TURN=5`
|
||||
21. `OWNER_TOOL_TIMEOUT_MS=30000`
|
||||
22. `OWNER_OPENINTERPRETER_ENABLED=0`
|
||||
23. `OWNER_OPENINTERPRETER_COMMAND=openinterpreter`
|
||||
24. `OWNER_BROWSER_PLAYWRIGHT_ENABLED=0`
|
||||
25. `OWNER_COMPUTER_CONTROL_ENABLED=0`
|
||||
|
||||
核心事件输出：
|
||||
|
||||
1. `audio_capture_started`
|
||||
2. `audio_apm_started`
|
||||
3. `listening_started`
|
||||
4. `speech_started`
|
||||
5. `stt_partial`
|
||||
6. `stt_final`
|
||||
7. `llm_stream_started`
|
||||
8. `llm_sentence_ready`
|
||||
9. `tts_chunk_ready`
|
||||
10. `playback_started`
|
||||
11. `interrupt_detected`
|
||||
12. `playback_cancelled`
|
||||
13. `llm_cancelled`
|
||||
14. `memory_retrieved`
|
||||
15. `tool_call_requested`
|
||||
16. `tool_confirmation_required`
|
||||
17. `tool_call_started`
|
||||
18. `tool_call_finished`
|
||||
19. `tool_call_rejected`
|
||||
20. `session_recovered`
|
||||
|
||||
### 数据验证规则
|
||||
|
||||
1. Provider 配置必须在启动前校验，不允许未知 provider 静默回退。
|
||||
2. APM 输出帧采样率、声道数、帧长必须与 VAD/STT 输入一致；不一致必须显式 resample 或报错。
|
||||
3. Streaming STT partial 不得进入长期记忆；final transcript 必须经过空文本、重复文本、敏感内容和最小置信度检查。
|
||||
4. TTS 播报文本必须经过现有 TTS sanitizer；emoji、表情包和 Markdown 图片不得进入语音。
|
||||
5. 长期记忆写入必须包含 type、text、source_turn_id、created_at、sensitivity、embedding_model、checksum。
|
||||
6. FAISS index 和 SQLite metadata 必须可一致性检查；缺失或 checksum 不匹配时不得返回伪造记忆。
|
||||
7. Tool Router arguments 必须按工具 schema 校验；未知字段、路径越界、命令注入风险必须拒绝。
|
||||
8. 工具结果进入 LLM 前必须截断、脱敏，并标注工具名、耗时、退出码和是否截断。
|
||||
9. 所有 cancellation token 必须可幂等触发，多次取消不得抛出未处理异常。
|
||||
|
||||
### 需人工澄清
|
||||
|
||||
1. 全双工入口是替换 `run-live`，还是新增 `run-agent-live` 并保留 `run-live` 为稳定 turn-based 入口。
|
||||
2. WebRTC APM Python 绑定优先选择哪个包或本地封装，是否允许引入需要系统编译的依赖。
|
||||
3. 产品阶段是否确定采用 SenseVoice 和 CosyVoice，还是只在规范中保留候选。
|
||||
4. 长期记忆是否需要用户可视化管理、删除、导出和禁用命令。
|
||||
5. Open Interpreter CLI 的实际本机命令、工作目录、沙箱策略和是否允许写操作需要人工确认。
|
||||
6. Playwright 浏览器工具是否允许使用用户当前 Chrome 登录态，还是只允许独立 browser context。
|
||||
7. 多人说话场景是否需要主人声纹注册；本变更默认不做身份鉴权。
|
||||
8. 桌宠 GUI 和电脑控制是否必须同期开工；本变更建议第一版先做音频全双工、记忆和安全工具。
|
||||
|
||||
## 设计方案
|
||||
|
||||
### 文字版全新架构图
|
||||
|
||||
```text
|
||||
Microphone
|
||||
-> Capture Ring Buffer
|
||||
-> WebRtcAudioProcessingStage(AEC + NS + AGC, render reference from Speaker)
|
||||
-> SileroVadStage
|
||||
-> InterruptDetector
|
||||
-> StreamingSttStage(faster-whisper dev / SenseVoice candidate / sherpa fallback)
|
||||
-> ConversationManager
|
||||
-> ShortTermSessionContext
|
||||
-> MemoryManager(SQLite metadata + FAISS vectors)
|
||||
-> LlmStage(streaming OpenAI-compatible provider)
|
||||
-> ToolRouter(memory.search/save, shell.readonly, openinterpreter.run, browser.playwright)
|
||||
-> ResponseStream
|
||||
-> SentenceSegmenter
|
||||
-> StreamingTtsStage(CosyVoice candidate / local fallback)
|
||||
-> Playback Ring Buffer
|
||||
-> Speaker
|
||||
-> Render Reference back to WebRtcAudioProcessingStage
|
||||
```
|
||||
|
||||
### 数据流
|
||||
|
||||
1. `ContinuousAudioRuntime` 启动后初始化 capture device、playback device、APM、VAD、STT、TTS、LLM、Memory、ToolRouter。
|
||||
2. 麦克风音频以固定 20 ms 帧写入 capture ring buffer；播放 PCM 以相同时间轴写入 render reference buffer。
|
||||
3. `WebRtcAudioProcessingStage` 使用 render reference 对 capture frame 执行 AEC，再执行 NS/AGC。
|
||||
4. APM 后音频同时进入 VAD、Streaming STT 和 Interrupt Detector。
|
||||
5. `listening` 状态下，VAD/STT 产生用户输入；final transcript 进入 Conversation Manager。
|
||||
6. Conversation Manager 召回短期上下文和长期记忆，生成 LLM streaming request。
|
||||
7. LLM delta 进入 Sentence Segmenter；完整句子进入 Streaming TTS；TTS chunk 立即入 playback queue。
|
||||
8. 播放开始后，播放 PCM 仍持续进入 render reference，使 AEC 能抑制助手回声。
|
||||
9. 如果用户讲话，Interrupt Detector 发出 `interrupt_detected`，Cancellation Graph 同时取消 LLM stream、未完成 TTS、播放队列和可取消工具。
|
||||
10. Tool Router 在 LLM 请求工具时执行 schema 校验、安全分类、确认策略、执行、结果脱敏和回注。
|
||||
11. 任何 stage 失败进入 `recovering`，释放后台任务和音频资源后回到 `listening` 或 `idle`。
|
||||
|
||||
### 接口定义
|
||||
|
||||
```text
|
||||
AudioFrame:
|
||||
samples: float32 PCM
|
||||
sample_rate: int
|
||||
channels: int
|
||||
timestamp_monotonic_ms: int
|
||||
frame_id: str
|
||||
```
|
||||
|
||||
```text
|
||||
WebRtcAudioProcessingStage.process_capture(frame: AudioFrame) -> AudioFrame
|
||||
WebRtcAudioProcessingStage.process_render(frame: AudioFrame) -> None
|
||||
WebRtcAudioProcessingStage.reset_stream() -> None
|
||||
Errors:
|
||||
AUDIO_APM_UNAVAILABLE
|
||||
AUDIO_APM_FORMAT_MISMATCH
|
||||
AUDIO_APM_PROCESS_FAILED
|
||||
```
|
||||
|
||||
```text
|
||||
StreamingSttProvider.start_session(session_id: str) -> StreamingSttSession
|
||||
StreamingSttSession.accept_audio(frame: AudioFrame) -> list[TranscriptEvent]
|
||||
StreamingSttSession.finish() -> TranscriptFinal
|
||||
StreamingSttSession.cancel(reason: str) -> None
|
||||
```
|
||||
|
||||
```text
|
||||
StreamingTtsProvider.start_stream(voice: str, sample_rate: int) -> StreamingTtsSession
|
||||
StreamingTtsSession.accept_text(text: str) -> list[AudioFrame]
|
||||
StreamingTtsSession.flush() -> list[AudioFrame]
|
||||
StreamingTtsSession.cancel(reason: str) -> None
|
||||
```
|
||||
|
||||
```text
|
||||
MemoryManager.search(query: str, *, top_k: int, filters: dict) -> list[MemoryRecord]
|
||||
MemoryManager.save(record: MemoryRecordInput) -> MemoryRecord
|
||||
MemoryManager.delete(memory_id: str) -> None
|
||||
MemoryManager.health_check() -> MemoryHealth
|
||||
```
|
||||
|
||||
```text
|
||||
ToolRouter.route(call: ToolCallRequest, context: ToolContext) -> ToolDecision
|
||||
ToolRouter.execute(decision: ToolDecision, cancellation: CancellationToken) -> ToolResult
|
||||
ToolDecision:
|
||||
action: execute | reject | require_confirmation
|
||||
risk_level: low | medium | high | forbidden
|
||||
reason: str
|
||||
```
|
||||
|
||||
### 状态机
|
||||
|
||||
```text
|
||||
idle
|
||||
-> listening
|
||||
listening
|
||||
-> thinking on final user transcript
|
||||
-> interrupted on explicit cancel or new speech over assistant residue
|
||||
-> recovering on audio/STT failure
|
||||
thinking
|
||||
-> speaking on first playable TTS chunk
|
||||
-> tool_running on approved tool call
|
||||
-> interrupted on user speech
|
||||
-> recovering on LLM failure
|
||||
speaking
|
||||
-> listening on reply finished and no follow-up/tool pending
|
||||
-> interrupted on valid user speech
|
||||
-> recovering on TTS/playback failure
|
||||
tool_running
|
||||
-> thinking on tool result returned to LLM
|
||||
-> interrupted on cancellable tool interrupted
|
||||
-> recovering on tool failure
|
||||
interrupted
|
||||
-> listening after current tasks cancelled and buffered user audio retained
|
||||
recovering
|
||||
-> listening after cleanup when runtime can continue
|
||||
-> idle when required provider unavailable
|
||||
```
|
||||
|
||||
### 关键算法
|
||||
|
||||
1. AEC reference 对齐：播放 PCM 写入 render buffer 时记录 monotonic timestamp；capture frame 处理前取最近 reference window，丢帧或漂移超过阈值时发 `audio_reference_drift`。
|
||||
2. 打断检测：APM 后音频先经 VAD 判断人声起点；连续人声超过最小阈值后结合 STT stable partial 或能量/频谱置信度触发 interrupt；纯 render echo 在 AEC 后应低于阈值。
|
||||
3. 句子切分：LLM delta 累积到中文句号、问号、感叹号、英文终止标点或最大等待阈值时切句；代码块、URL、数字小数点不得误切。
|
||||
4. 取消传播：每轮创建 root cancellation token；LLM、TTS、playback、tool 子任务注册 child token；用户打断时 root token 广播，所有 stage 幂等收尾。
|
||||
5. 长期记忆召回：用户 final transcript 生成 embedding，FAISS 取 Top-N，SQLite 取 metadata，按类型、敏感度、最近使用、相似度重排后注入 LLM。
|
||||
6. 记忆写入：Conversation Manager 在回合结束后提取候选记忆，按敏感分类和用户意图决定是否保存；敏感或不确定默认不保存。
|
||||
7. 工具路由：LLM tool call 先过 schema，再做风险分类和权限判断；低风险可执行，高风险进入确认，禁止类直接拒绝。
|
||||
|
||||
### 数据库/状态管理变更
|
||||
|
||||
1. SQLite 表 `memories`：`id`、`type`、`text`、`summary`、`metadata_json`、`sensitivity`、`source_turn_id`、`created_at`、`updated_at`、`last_used_at`、`embedding_id`、`checksum`。
|
||||
2. SQLite 表 `tool_audit_logs`：记录工具名、风险、确认状态、耗时、退出码、截断标记和脱敏摘要，不保存密钥。
|
||||
3. FAISS index 文件保存 embedding vectors；SQLite 保存 index 版本和 embedding model，启动时做一致性检查。
|
||||
4. 短期会话上下文仍在内存中，进程退出丢弃；长期记忆独立存储，可按配置禁用。
|
||||
|
||||
### UI 组件重构方案
|
||||
|
||||
1. 终端 reporter 只订阅事件，不直接读取 pipeline 内部状态。
|
||||
2. 未来桌宠 GUI 使用同一事件流展示 `listening`、`thinking`、`speaking`、`interrupted`、`tool_running`、`recovering`。
|
||||
3. 工具确认必须作为事件暴露，终端可先实现文本确认，GUI 后续实现按钮确认。
|
||||
4. 实时字幕分为 partial、stable partial、final 三种显示层级，避免把抖动 partial 当作最终用户输入。
|
||||
|
||||
### 依赖影响分析
|
||||
|
||||
1. WebRTC APM：新增依赖风险最高，需确认 Python/macOS 可用绑定或自建 native wrapper。
|
||||
2. Silero VAD：新增本地模型依赖，需评估 ONNX Runtime 或 torch 路线。
|
||||
3. Faster Whisper：开发体验好，但模型体积和 Metal/CPU 性能需评估。
|
||||
4. SenseVoice：中文效果强，产品候选；需确认 license、模型大小、macOS 部署成本。
|
||||
5. CosyVoice：TTS 效果强，依赖较重；第一阶段可先保留现有本地 TTS fallback。
|
||||
6. FAISS：macOS 安装和 wheel 兼容性需评估；必要时提供 sqlite-only 或 numpy fallback。
|
||||
7. Playwright：浏览器自动化依赖和浏览器安装体积需评估；第一版默认关闭。
|
||||
8. Open Interpreter：作为外部 CLI 后端，不作为 Owner 包内依赖；路径缺失时 adapter 不可用。
|
||||
|
||||
## 风险与权衡
|
||||
|
||||
| 风险 | 概率 | 影响 | 缓解措施 |
|
||||
| --- | --- | --- | --- |
|
||||
| WebRTC APM Python/macOS 绑定不可用或编译复杂 | 高 | 高 | OpenSpec 中把 provider 抽象出来；先验证 fake APM 和最小 native 方案；保留 GTCRN/音色门控 fallback。 |
|
||||
| AEC reference 与 capture 时钟不同步 | 中 | 高 | 使用 monotonic timestamp、ring buffer drift 监控、reference gap 事件和回声测试 fixture。 |
|
||||
| 全双工并发导致线程泄漏或播放卡顿 | 中 | 高 | 所有 stage 必须支持 cancellation token、bounded queue、backpressure 和统一 shutdown。 |
|
||||
| Streaming TTS 依赖过重导致落地慢 | 中 | 中 | 第一阶段先按句分段合成，CosyVoice 作为目标 provider，保留现有 Mac TTS fallback。 |
|
||||
| Faster Whisper/SenseVoice 模型性能不足 | 中 | 中 | 规范要求 provider 可替换，测试记录 partial/final latency，产品候选不在第一阶段强绑定。 |
|
||||
| 长期记忆保存敏感信息 | 中 | 高 | 默认敏感不自动保存；记忆写入前分类；用户可关闭；日志和记忆脱敏。 |
|
||||
| Tool Router 执行危险操作 | 中 | 高 | 默认安全工具优先，高风险确认，禁止类拒绝，目录限制，超时，输出截断，审计日志。 |
|
||||
| Open Interpreter 外部后端越权 | 中 | 高 | 默认关闭；只允许受限目录、低风险任务；写操作必须确认或拒绝；不复制外部仓库进 Owner。 |
|
||||
| Playwright 使用登录态带来账号风险 | 中 | 高 | 第一版默认独立 context；涉及账号、支付、购买、提交必须确认或拒绝。 |
|
||||
| 用户期望立即实现完整 GUI 控制 | 中 | 中 | 本变更明确第一版不做 GUI 点击/键盘/屏幕控制，只预留公共 provider。 |
|
||||
| FAISS 与 SQLite 一致性损坏 | 低 | 中 | 启动 health check、checksum、index rebuild 任务和 sqlite-only 降级。 |
|
||||
| LLM 工具循环 | 中 | 中 | 单轮最大工具次数、重复调用检测、总耗时预算和可恢复拒绝。 |
|
||||
| 不确定是否继续对话造成体验不稳 | 中 | 中 | Conversation Manager 规则优先，LLM 分类兜底，不确定默认 listening/idle 策略需人工确认。 |
|
||||
| OpenSpec scope 过大导致实现周期过长 | 高 | 中 | 实施计划分阶段：先音频全双工，再流式 STT/TTS，再记忆和工具，再电脑控制。 |
|
||||
|
||||
## 任务分解
|
||||
|
||||
> 本节定义 proposal 里的主要功能组或里程碑阶段。后续 `tasks.md` 必须把每组拆成不超过 1 小时的原子任务，并在完成每个大模块后按 Git 提交规范立即提交。
|
||||
|
||||
### 1. OpenSpec 与边界冻结
|
||||
|
||||
目标：只产出 `add-full-duplex-agent-voice-assistant` 文档，不改代码、不装依赖、不下载模型、不提交 `openinterpreter/`。
|
||||
|
||||
验收：OpenSpec 变更目录包含 proposal、design、tasks、spec delta；严格校验通过。
|
||||
|
||||
### 2. WebRTC APM 与全双工音频底座规划
|
||||
|
||||
目标：定义 capture/render ring buffer、AEC reference、NS、AGC、音频格式、时钟对齐和 fallback 策略。
|
||||
|
||||
验收：spec 包含 APM SHALL 要求、fake reference 测试场景和回声不触发 VAD/STT 场景。
|
||||
|
||||
### 3. 全双工状态机、事件总线与取消机制规划
|
||||
|
||||
目标：定义 `idle/listening/thinking/speaking/interrupted/tool_running/recovering` 状态机、事件模型、cancellation graph 和 recovery。
|
||||
|
||||
验收：design 包含状态转移表和取消传播；tasks 包含状态机、事件顺序和错误恢复测试。
|
||||
|
||||
### 4. Streaming STT 与低延迟打断规划
|
||||
|
||||
目标：定义 Streaming STT provider、Silero VAD、partial/final transcript、打断检测和 200 ms 目标。
|
||||
|
||||
验收：spec 包含 partial/final 行为、speaking 中用户说话进入 interrupted、纯回声不打断。
|
||||
|
||||
### 5. Streaming TTS 与响应流规划
|
||||
|
||||
目标：定义 LLM token stream、句子切分、Streaming TTS、播放队列、可中断播放和上下文写入边界。
|
||||
|
||||
验收：spec 包含不等待完整回复、未播完文本不写上下文、TTS 卡顿可观测。
|
||||
|
||||
### 6. 长期记忆规划
|
||||
|
||||
目标：定义 MemoryManager、FAISS+SQLite、记忆类型、召回、写入安全、禁用和删除策略。
|
||||
|
||||
验收：spec 包含保存、检索、重启召回、关闭记忆不读写、敏感内容不自动保存。
|
||||
|
||||
### 7. Tool Router 与安全工具规划
|
||||
|
||||
目标：定义结构化工具协议、安全路由、工具预算、防循环、工具结果回注，以及 `memory.search/save`、`shell.readonly`、`openinterpreter.run`、`browser.playwright`。
|
||||
|
||||
验收：spec 包含合法工具执行、非法工具拒绝、确认策略、Open Interpreter 缺失处理和 Playwright 安全边界。
|
||||
|
||||
### 8. Open Interpreter 与电脑控制边界规划
|
||||
|
||||
目标：明确 `openinterpreter/` 是外部未跟踪后端候选，不复制进 Owner；第一版不做 GUI 控制，只预留 `ComputerControlProvider`。
|
||||
|
||||
验收：proposal/design/tasks 均写明边界；git 提交不包含 `openinterpreter/`。
|
||||
|
||||
### 9. 测试、性能、安全和验收规划
|
||||
|
||||
目标：定义 fake APM、VAD、STT、TTS、memory、tool、Open Interpreter、Playwright、端到端模拟、性能指标和安全验证。
|
||||
|
||||
验收：tasks 中每项有前置条件、优先级、验收标准和测试要点；validation 命令明确。
|
||||
|
||||
### 10. 真实全双工运行时落地
|
||||
|
||||
目标：撤销“只完成骨架即完成”的边界，把 `run-agent-live` 接成可真人运行的全双工语音入口；保留 `run-live` 作为旧 turn-based 稳定入口。
|
||||
|
||||
验收：`run-agent-live` 不带 `--check-config` 不再返回 `FULL_DUPLEX_RUNTIME_NOT_IMPLEMENTED`，而是启动持续 listening；播放中检测到有效用户说话时，在软件回声抑制通过后先停止播放，再把用户打断音频接入下一轮 STT；只有助手回放 reference 时不触发打断。
|
||||
|
||||
## Spec Deltas
|
||||
|
||||
### Capabilities
|
||||
|
||||
#### New Capabilities
|
||||
|
||||
本变更不创建独立新 capability 文件。原因：用户明确要求 delta 文件路径为 `specs/voice-pet-pipeline/spec.md`，且全双工音频、长期记忆和 Tool Router 都作为语音助手 pipeline 的能力升级纳入同一现有 capability。
|
||||
|
||||
#### Modified Capabilities
|
||||
|
||||
- `voice-pet-pipeline`：从 turn-based Python 桌宠语音 pipeline 扩展为完整全双工 Agent 语音助手 pipeline，新增 WebRTC APM、持续监听、Streaming STT/TTS、低延迟打断、长期记忆、Tool Router、Open Interpreter 外部后端适配和安全工具执行要求。
|
||||
|
||||
### 与现有 `openspec/specs/voice-pet-pipeline/spec.md` 的精确差异
|
||||
|
||||
1. 修改 `Local microphone and speaker transport`：从本机麦克风/扬声器 transport 扩展为 capture/render reference 双向音频流，播放音频必须提供给 AEC。
|
||||
2. 修改 `VAD speech endpoint detection`：从 turn-based 端点检测扩展为持续 VAD、打断检测和全双工 listening。
|
||||
3. 修改 `Local STT transcription`：从 captured segment final STT 扩展为 Streaming STT partial/stable/final。
|
||||
4. 修改 `Cloud LLM streaming reply`：从流式 LLM 输出扩展为 token-to-sentence-to-TTS response stream，并要求可取消。
|
||||
5. 修改 `Local TTS synthesis and playback`：从整段或分句播放扩展为 Streaming TTS、PCM chunk 播放和中途停止。
|
||||
6. 修改 `Pipeline state machine`：新增 `listening`、`tool_running`、`recovering` 等全双工 Agent 状态，明确 `speaking -> interrupted`。
|
||||
7. 修改 `Audio feedback suppression`：从播放期间抑制输入扩展为 AEC + VAD + interrupt，允许用户有效打断。
|
||||
8. 修改 `Conversation context management`：保留进程内上下文，同时新增长期记忆召回的边界。
|
||||
9. 修改 `Security and privacy`：新增长期记忆、工具执行、Open Interpreter 和浏览器自动化安全要求。
|
||||
10. 修改 `Performance targets`：新增打断延迟、Streaming STT 首字、TTS 首 chunk、APM 帧处理等指标。
|
||||
11. 新增 requirement：`WebRTC audio processing foundation`。
|
||||
12. 新增 requirement：`Full-duplex agent state machine`。
|
||||
13. 新增 requirement：`Streaming STT and realtime transcript`。
|
||||
14. 新增 requirement：`Low-latency interruption and cancellation`。
|
||||
15. 新增 requirement：`Streaming response and TTS playback`。
|
||||
16. 新增 requirement：`Long-term memory with FAISS and SQLite`。
|
||||
17. 新增 requirement：`Tool Router and structured tool execution`。
|
||||
18. 新增 requirement：`Open Interpreter external adapter`。
|
||||
19. 新增 requirement：`Browser automation tool boundary`。
|
||||
20. 新增 requirement：`Computer control reservation`。
|
||||
21. 新增 requirement：`Live run-agent-live runtime`。
|
||||
22. 新增 requirement：`Software render-reference interruption gate`。
|
||||
|
||||
### 推翻重做的理由
|
||||
|
||||
1. turn-based VAD/STT/TTS 无法自然支持“AI 讲话时用户插话”，只能不断添加补丁。
|
||||
2. 音色门控不能替代 AEC；没有 render reference 的系统无法稳定区分助手回放和真实用户。
|
||||
3. 只靠 final STT 会让用户等待过久；完整助手需要持续识别和实时字幕。
|
||||
4. 没有 Tool Router 和 MemoryManager 的语音助手只能聊天，不能完成 Agent 任务。
|
||||
5. 工具和记忆如果后补，会难以补齐安全边界；必须在 OpenSpec 阶段先定义。
|
||||
|
||||
## 实施计划
|
||||
|
||||
### 分阶段优先级顺序
|
||||
|
||||
1. 阶段 A：OpenSpec 文档和边界冻结。
|
||||
2. 阶段 B：全双工音频底座和 fake APM 测试。
|
||||
3. 阶段 C：状态机、事件总线、取消机制和模拟端到端。
|
||||
4. 阶段 D：Streaming STT、VAD 打断和回声抑制测试。
|
||||
5. 阶段 E：Streaming LLM/TTS、句子切分和可中断播放。
|
||||
6. 阶段 F：长期记忆 FAISS+SQLite、记忆召回和隐私策略。
|
||||
7. 阶段 G：Tool Router、安全工具、Open Interpreter adapter 和 Playwright adapter。
|
||||
8. 阶段 H：文档、性能验收、安全审计和迁移收尾。
|
||||
9. 阶段 I：真实 `run-agent-live` 运行时、软件回声抑制和真人打断验收。
|
||||
|
||||
### 里程碑估时
|
||||
|
||||
| 里程碑 | 乐观 | 最可能 | 悲观 |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| A OpenSpec 文档 | 0.5 天 | 1 天 | 1.5 天 |
|
||||
| B 音频底座 | 2 天 | 4 天 | 8 天 |
|
||||
| C 状态机与取消 | 2 天 | 3 天 | 6 天 |
|
||||
| D Streaming STT 与打断 | 3 天 | 5 天 | 10 天 |
|
||||
| E Streaming TTS | 3 天 | 5 天 | 10 天 |
|
||||
| F 长期记忆 | 2 天 | 4 天 | 8 天 |
|
||||
| G Tool Router | 3 天 | 6 天 | 12 天 |
|
||||
| H 验收与收尾 | 2 天 | 4 天 | 8 天 |
|
||||
| I 真实全双工运行时 | 1 天 | 2 天 | 4 天 |
|
||||
|
||||
总耗时预估：乐观 18.5 天，最可能 34 天，悲观 67.5 天。
|
||||
|
||||
### 数据/状态迁移策略
|
||||
|
||||
1. 现有进程内上下文不迁移为长期记忆，避免未经确认的历史被自动保存。
|
||||
2. 新长期记忆库首次启动为空；用户明确要求保存或分类器确认低敏偏好后才写入。
|
||||
3. 现有 `.env` 继续可用，新增变量采用默认值；全双工模式可通过 feature flag 启用。
|
||||
4. 现有 turn-based `run-live` 在迁移期保留，直到全双工验收稳定后再决定是否替换默认入口。
|
||||
5. `openinterpreter/` 外部仓库不纳入 Owner 迁移；只记录 adapter 配置和安全策略。
|
||||
|
||||
## Git 提交规范
|
||||
|
||||
1. 每完成一个大模块，必须立即执行构建或相应验证，然后执行 git commit。
|
||||
2. 大模块定义为 proposal “任务分解”中的主要功能组或实施计划中的里程碑阶段。
|
||||
3. 提交信息必须使用中文，格式为：`[模块名]：完成[具体功能描述]，包含[关键变更]`。
|
||||
4. 提交前必须保证本模块验证通过，避免任何未提交的中间状态。
|
||||
5. 本 OpenSpec-only 阶段完成后提交信息固定为：`[全双工Agent架构]：完成完整语音助手OpenSpec计划，包含WebRTC音频底座、长期记忆和Tool Router设计`。
|
||||
6. 本次提交只允许包含 `openspec/changes/add-full-duplex-agent-voice-assistant/` 下的规划文档，不得提交 `openinterpreter/`、模型文件、依赖锁文件、`.env` 或运行代码。
|
||||
+413
@@ -0,0 +1,413 @@
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Full-duplex agent mode
|
||||
The system SHALL support a planned full-duplex agent voice assistant mode that runs continuous microphone listening, streaming recognition, streaming response generation, streaming speech playback, interruption handling, memory retrieval, and safe tool routing as one coordinated pipeline.
|
||||
|
||||
#### Scenario: Full-duplex mode starts
|
||||
- **WHEN** the future implementation starts with `OWNER_ASSISTANT_MODE=full_duplex_agent`
|
||||
- **THEN** the system SHALL initialize continuous audio capture, audio playback, WebRTC audio processing, VAD, streaming STT, streaming TTS, conversation management, memory, and tool routing before accepting user interaction
|
||||
|
||||
#### Scenario: Turn-based mode remains available during migration
|
||||
- **WHEN** the future implementation starts with the existing turn-based mode
|
||||
- **THEN** the system SHALL preserve the existing wake-to-capture-to-STT-to-LLM-to-TTS behavior unless full-duplex mode is explicitly enabled
|
||||
|
||||
#### Scenario: Full-duplex provider is unavailable
|
||||
- **WHEN** a required full-duplex provider cannot be initialized and no configured fallback is allowed
|
||||
- **THEN** the system SHALL fail startup with a structured provider error rather than entering a fake full-duplex loop
|
||||
|
||||
### Requirement: WebRTC audio processing foundation
|
||||
The system SHALL define a WebRTC audio processing stage for full-duplex mode that applies acoustic echo cancellation, noise suppression, and automatic gain control to microphone capture audio using speaker playback audio as render reference.
|
||||
|
||||
#### Scenario: AEC uses playback reference
|
||||
- **WHEN** TTS playback writes PCM frames to the speaker output path
|
||||
- **THEN** the same render audio SHALL be provided to the WebRTC audio processing stage as echo cancellation reference
|
||||
|
||||
#### Scenario: Capture audio is processed
|
||||
- **WHEN** microphone frames are captured during full-duplex mode
|
||||
- **THEN** the frames SHALL pass through AEC, noise suppression, and automatic gain control before VAD, interruption detection, and streaming STT consume them
|
||||
|
||||
#### Scenario: Pure assistant echo is present
|
||||
- **WHEN** the microphone receives only the assistant's own playback echo and no user speech
|
||||
- **THEN** the processed audio SHALL NOT produce a valid user interruption or valid user transcript
|
||||
|
||||
#### Scenario: APM format mismatch occurs
|
||||
- **WHEN** capture audio and render reference audio have incompatible sample rate, channel count, or frame size
|
||||
- **THEN** the system SHALL either explicitly resample to the configured internal format or fail with `AUDIO_APM_FORMAT_MISMATCH`
|
||||
|
||||
#### Scenario: WebRTC APM fails at runtime
|
||||
- **WHEN** the WebRTC audio processing stage raises a processing error
|
||||
- **THEN** the pipeline SHALL emit a structured audio processing error and transition through recovery without corrupting conversation history
|
||||
|
||||
### Requirement: Full-duplex audio ring buffers
|
||||
The system SHALL use bounded capture and render reference ring buffers with timestamps so full-duplex capture, playback, AEC, VAD, STT, and interruption detection can run concurrently without unbounded memory growth.
|
||||
|
||||
#### Scenario: Capture frame is received
|
||||
- **WHEN** the microphone callback receives a PCM frame
|
||||
- **THEN** the frame SHALL be written to the capture ring buffer with a monotonic timestamp, frame id, sample rate, channel count, and duration
|
||||
|
||||
#### Scenario: Render frame is queued
|
||||
- **WHEN** TTS or playback queues a PCM frame for speaker output
|
||||
- **THEN** the frame SHALL be written to the render reference ring buffer with timing metadata usable by AEC
|
||||
|
||||
#### Scenario: Ring buffer overflows
|
||||
- **WHEN** an audio ring buffer exceeds its configured capacity
|
||||
- **THEN** the system SHALL drop the oldest safe frames, emit an `audio_buffer_overrun` event, and continue without blocking the audio callback indefinitely
|
||||
|
||||
#### Scenario: Reference drift is detected
|
||||
- **WHEN** capture and render timestamps drift beyond the configured tolerance
|
||||
- **THEN** the system SHALL emit an `audio_reference_drift` event and reduce confidence in interruption decisions until alignment recovers
|
||||
|
||||
### Requirement: Full-duplex agent state machine
|
||||
The system SHALL define a deterministic full-duplex state machine with `idle`, `listening`, `thinking`, `speaking`, `interrupted`, `tool_running`, and `recovering` states.
|
||||
|
||||
#### Scenario: User input reaches the LLM
|
||||
- **WHEN** full-duplex listening produces a valid final user transcript
|
||||
- **THEN** the state machine SHALL transition from `listening` to `thinking`
|
||||
|
||||
#### Scenario: First playable response is ready
|
||||
- **WHEN** the LLM response stream and TTS pipeline produce the first playable audio chunk
|
||||
- **THEN** the state machine SHALL transition from `thinking` to `speaking`
|
||||
|
||||
#### Scenario: User interrupts playback
|
||||
- **WHEN** valid user speech is detected while the assistant is speaking
|
||||
- **THEN** the state machine SHALL transition from `speaking` to `interrupted`
|
||||
|
||||
#### Scenario: Tool call starts
|
||||
- **WHEN** the LLM requests an approved tool call
|
||||
- **THEN** the state machine SHALL transition to `tool_running` until the tool result is returned, rejected, cancelled, or fails
|
||||
|
||||
#### Scenario: Recoverable error occurs
|
||||
- **WHEN** a recoverable provider, memory, tool, STT, TTS, LLM, or playback error occurs
|
||||
- **THEN** the state machine SHALL transition to `recovering` and then to a safe `listening` or `idle` state after cleanup
|
||||
|
||||
### Requirement: Pipeline event bus for agent stages
|
||||
The system SHALL expose a stable pipeline event bus for full-duplex audio, recognition, generation, playback, interruption, memory, tools, confirmation, and recovery events.
|
||||
|
||||
#### Scenario: Terminal reporter displays state
|
||||
- **WHEN** the terminal displays full-duplex runtime status
|
||||
- **THEN** it SHALL derive its output from pipeline events rather than directly reading provider internals
|
||||
|
||||
#### Scenario: GUI subscribes to the same events
|
||||
- **WHEN** a future desktop pet GUI is implemented
|
||||
- **THEN** it SHALL be able to subscribe to the same event bus used by the terminal reporter and tests
|
||||
|
||||
#### Scenario: Event contains diagnostics
|
||||
- **WHEN** a stage emits an event
|
||||
- **THEN** the event SHALL include the session id or turn id when available, stage name, event type, timestamp, and sanitized payload
|
||||
|
||||
#### Scenario: Sensitive payload is present
|
||||
- **WHEN** an event involves credentials, raw audio, memory content, or tool output
|
||||
- **THEN** the event payload SHALL omit secrets and SHALL truncate or summarize sensitive content according to security policy
|
||||
|
||||
### Requirement: Low-latency interruption and cancellation
|
||||
The full-duplex mode SHALL detect valid user speech during assistant playback and cancel the active response pipeline with a target interruption latency under 200 ms.
|
||||
|
||||
#### Scenario: User speaks while assistant is speaking
|
||||
- **WHEN** the assistant is playing TTS audio and APM-processed microphone audio contains valid user speech
|
||||
- **THEN** the interruption detector SHALL emit `interrupt_detected`, stop remaining playback, and cancel active LLM and TTS work
|
||||
|
||||
#### Scenario: Interruption latency is measured
|
||||
- **WHEN** a test fixture injects user speech during `speaking`
|
||||
- **THEN** the system SHALL record the latency from VAD speech start to playback stop and SHALL target P95 under 200 ms
|
||||
|
||||
#### Scenario: Echo does not interrupt
|
||||
- **WHEN** microphone input during playback is explained by assistant render reference echo after AEC
|
||||
- **THEN** the system SHALL NOT emit `interrupt_detected`
|
||||
|
||||
#### Scenario: Cancellation is idempotent
|
||||
- **WHEN** interruption or recovery triggers cancellation more than once
|
||||
- **THEN** LLM, TTS, playback, STT, and cancellable tool tasks SHALL handle repeated cancellation without unhandled exceptions
|
||||
|
||||
#### Scenario: Interrupted assistant text is stored
|
||||
- **WHEN** assistant playback is interrupted before the full response is spoken
|
||||
- **THEN** only assistant text that was fully spoken before interruption SHALL be eligible for short-term context or long-term memory
|
||||
|
||||
### Requirement: Streaming STT and realtime transcript
|
||||
The full-duplex mode SHALL use a streaming speech-to-text provider that emits partial, stable partial, and final transcripts while the user is speaking.
|
||||
|
||||
#### Scenario: Partial transcript is produced
|
||||
- **WHEN** the user is speaking and streaming STT emits a changed partial transcript
|
||||
- **THEN** the event bus SHALL emit a realtime transcript event for user-visible feedback
|
||||
|
||||
#### Scenario: Stable partial is produced
|
||||
- **WHEN** a partial transcript remains stable according to the configured stability filter
|
||||
- **THEN** the system SHALL mark it as stable partial while still withholding it from the LLM unless finalization rules allow submission
|
||||
|
||||
#### Scenario: Final transcript is produced
|
||||
- **WHEN** the current user utterance ends
|
||||
- **THEN** streaming STT SHALL emit a final transcript that becomes the user message candidate for conversation processing
|
||||
|
||||
#### Scenario: Empty transcript is produced
|
||||
- **WHEN** final STT returns empty text, punctuation-only text, or text below confidence thresholds
|
||||
- **THEN** the system SHALL discard the utterance, avoid invoking the LLM, and recover to listening
|
||||
|
||||
#### Scenario: Streaming STT provider changes
|
||||
- **WHEN** the configured provider changes from `faster_whisper` to `sensevoice` or `sherpa_onnx`
|
||||
- **THEN** the state machine and Conversation Manager SHALL keep the same transcript event contract
|
||||
|
||||
### Requirement: Streaming response and TTS playback
|
||||
The full-duplex mode SHALL stream LLM output into sentence segmentation, TTS synthesis, and playback without waiting for the full assistant reply to complete.
|
||||
|
||||
#### Scenario: First sentence is ready
|
||||
- **WHEN** the LLM stream produces a complete sentence or configured safe speech segment
|
||||
- **THEN** the sentence segmenter SHALL send sanitized text to the streaming TTS provider immediately
|
||||
|
||||
#### Scenario: TTS chunk is ready
|
||||
- **WHEN** the TTS provider produces the first playable PCM chunk
|
||||
- **THEN** playback SHALL begin without waiting for all remaining assistant text
|
||||
|
||||
#### Scenario: TTS text contains emoji
|
||||
- **WHEN** LLM output contains emoji, sticker markers, Markdown image syntax, or other non-speech content
|
||||
- **THEN** the text sent to TTS SHALL be sanitized before synthesis
|
||||
|
||||
#### Scenario: Playback is cancelled
|
||||
- **WHEN** cancellation is requested during response playback
|
||||
- **THEN** playback SHALL stop at the next configured audio chunk boundary and SHALL clear unplayed audio chunks
|
||||
|
||||
#### Scenario: TTS provider fails
|
||||
- **WHEN** streaming TTS fails before or during playback
|
||||
- **THEN** the system SHALL emit a structured TTS error and enter recovery without appending unspoken text to context
|
||||
|
||||
### Requirement: Conversation manager for agent turns
|
||||
The system SHALL define a Conversation Manager that coordinates short-term context, long-term memory retrieval, LLM streaming, tool calls, response playback, interruption, and recovery.
|
||||
|
||||
#### Scenario: User final transcript is accepted
|
||||
- **WHEN** a valid final transcript is produced
|
||||
- **THEN** the Conversation Manager SHALL create a user message candidate and gather short-term context and relevant long-term memory before invoking the LLM
|
||||
|
||||
#### Scenario: Assistant response completes
|
||||
- **WHEN** the assistant response is fully spoken and no tool call or interruption is pending
|
||||
- **THEN** the Conversation Manager SHALL commit the spoken assistant text to short-term context
|
||||
|
||||
#### Scenario: Response is interrupted
|
||||
- **WHEN** user speech interrupts the assistant response
|
||||
- **THEN** the Conversation Manager SHALL cancel the current response, preserve only committed spoken text, and process buffered user speech as the next input
|
||||
|
||||
#### Scenario: Conversation manager recovers
|
||||
- **WHEN** a provider error occurs
|
||||
- **THEN** the Conversation Manager SHALL emit a recoverable error event and return the agent to a safe listening or idle state when possible
|
||||
|
||||
### Requirement: Long-term memory with FAISS and SQLite
|
||||
The system SHALL define a long-term memory layer where SQLite stores text records and metadata while FAISS stores vector indexes for semantic retrieval.
|
||||
|
||||
#### Scenario: Memory is saved
|
||||
- **WHEN** the user explicitly asks the assistant to remember a non-sensitive preference, fact, project detail, or task summary
|
||||
- **THEN** the Memory Manager SHALL save a typed memory record with metadata in SQLite and an embedding vector in FAISS
|
||||
|
||||
#### Scenario: Memory is retrieved
|
||||
- **WHEN** a later user input is semantically related to stored memories
|
||||
- **THEN** the Memory Manager SHALL retrieve Top-K relevant memories and provide them to the Conversation Manager as separate memory context
|
||||
|
||||
#### Scenario: Runtime restarts
|
||||
- **WHEN** the assistant process restarts and memory is enabled
|
||||
- **THEN** saved non-sensitive memories SHALL remain available through SQLite and FAISS
|
||||
|
||||
#### Scenario: Memory is disabled
|
||||
- **WHEN** `OWNER_MEMORY_ENABLED=0`
|
||||
- **THEN** the system SHALL NOT read from or write to SQLite or FAISS memory storage
|
||||
|
||||
#### Scenario: Sensitive content is detected
|
||||
- **WHEN** a memory candidate contains credentials, payment data, private identifiers, account secrets, or other sensitive content
|
||||
- **THEN** the system SHALL NOT automatically save it and SHALL require explicit user confirmation or reject the save
|
||||
|
||||
#### Scenario: Memory index is inconsistent
|
||||
- **WHEN** SQLite metadata and FAISS index records are missing, mismatched, or fail checksum validation
|
||||
- **THEN** the Memory Manager SHALL report a health error and SHALL NOT return unverified memory records
|
||||
|
||||
### Requirement: Tool Router and structured tool execution
|
||||
The system SHALL define a Tool Router that validates structured tool calls, applies security policy, routes approved tools, enforces execution limits, and returns sanitized tool results to the LLM.
|
||||
|
||||
#### Scenario: Allowed memory search runs
|
||||
- **WHEN** the LLM requests `memory.search` with valid arguments
|
||||
- **THEN** the Tool Router SHALL execute the memory search within configured limits and return sanitized results
|
||||
|
||||
#### Scenario: Tool schema is invalid
|
||||
- **WHEN** a tool call has an unknown name, missing required argument, invalid type, or unknown field
|
||||
- **THEN** the Tool Router SHALL reject it with a structured validation error
|
||||
|
||||
#### Scenario: Tool exceeds call budget
|
||||
- **WHEN** a turn exceeds the configured maximum number of tool calls or total tool runtime
|
||||
- **THEN** the Tool Router SHALL reject further tool calls for that turn and return a budget error
|
||||
|
||||
#### Scenario: Tool output is too long
|
||||
- **WHEN** a tool returns output larger than the configured limit
|
||||
- **THEN** the Tool Router SHALL truncate the output, mark it as truncated, and avoid sending full oversized content to the LLM
|
||||
|
||||
#### Scenario: Tool execution fails
|
||||
- **WHEN** an approved tool raises an error or times out
|
||||
- **THEN** the Tool Router SHALL return a structured tool error and the Conversation Manager SHALL recover or ask the user for next steps
|
||||
|
||||
### Requirement: Safe first-version tools
|
||||
The first full-duplex Agent tool set SHALL prioritize low-risk tools and SHALL include only `memory.search`, `memory.save`, `shell.readonly`, `openinterpreter.run`, and `browser.playwright` as planned first-version adapters.
|
||||
|
||||
#### Scenario: Read-only shell command is allowed
|
||||
- **WHEN** `shell.readonly` receives an allowlisted read-only command within allowed directories
|
||||
- **THEN** the tool SHALL execute with timeout and output truncation
|
||||
|
||||
#### Scenario: Write shell command is rejected
|
||||
- **WHEN** `shell.readonly` receives a command that writes, deletes, installs dependencies, changes permissions, uploads data, or mutates external state
|
||||
- **THEN** the Tool Router SHALL reject the command
|
||||
|
||||
#### Scenario: Memory save is requested
|
||||
- **WHEN** `memory.save` is requested for a non-sensitive user-approved memory
|
||||
- **THEN** the tool SHALL save through the Memory Manager and return the new memory id
|
||||
|
||||
#### Scenario: High-risk action is requested
|
||||
- **WHEN** any planned first-version tool request involves deletion, upload, payment, trading, account changes, permission changes, secret access, or arbitrary code execution
|
||||
- **THEN** the Tool Router SHALL reject it or require explicit confirmation according to policy
|
||||
|
||||
### Requirement: Open Interpreter external adapter
|
||||
The system SHALL treat Open Interpreter as an optional external CLI or subprocess backend and SHALL NOT copy the local `openinterpreter/` clone into the Owner source tree.
|
||||
|
||||
#### Scenario: Open Interpreter is disabled
|
||||
- **WHEN** `OWNER_OPENINTERPRETER_ENABLED=0`
|
||||
- **THEN** the Tool Router SHALL mark `openinterpreter.run` unavailable and SHALL NOT invoke any Open Interpreter command
|
||||
|
||||
#### Scenario: Open Interpreter path is missing
|
||||
- **WHEN** `OWNER_OPENINTERPRETER_ENABLED=1` but the configured command cannot be found
|
||||
- **THEN** the adapter SHALL return `OPENINTERPRETER_UNAVAILABLE` without affecting other tools
|
||||
|
||||
#### Scenario: Low-risk Open Interpreter task is approved
|
||||
- **WHEN** a low-risk Open Interpreter task passes directory, timeout, and argument validation
|
||||
- **THEN** the adapter SHALL run it as an external process with bounded runtime and sanitized output
|
||||
|
||||
#### Scenario: High-risk Open Interpreter task is requested
|
||||
- **WHEN** an Open Interpreter task would write files, delete data, change accounts, install dependencies, upload data, or execute broad arbitrary code
|
||||
- **THEN** the Tool Router SHALL require explicit confirmation or reject the task in first-version policy
|
||||
|
||||
### Requirement: Browser automation boundary
|
||||
The system SHALL define `browser.playwright` as a planned browser automation adapter with explicit safety boundaries.
|
||||
|
||||
#### Scenario: Safe browser read task is requested
|
||||
- **WHEN** the user asks for a low-risk browser read or extraction task and Playwright is enabled
|
||||
- **THEN** the adapter SHALL run in an approved browser context with timeout, navigation limits, and sanitized output
|
||||
|
||||
#### Scenario: Sensitive browser action is requested
|
||||
- **WHEN** a browser task involves login, payment, purchase, form submission, account changes, private messages, uploads, or irreversible actions
|
||||
- **THEN** the adapter SHALL require explicit confirmation or reject the task according to policy
|
||||
|
||||
#### Scenario: Playwright is unavailable
|
||||
- **WHEN** Playwright or its browser runtime is not installed
|
||||
- **THEN** `browser.playwright` SHALL report a structured unavailable error and SHALL NOT affect voice conversation
|
||||
|
||||
### Requirement: Computer control reservation
|
||||
The system SHALL reserve a future `ComputerControlProvider` boundary for GUI control while excluding GUI click, keyboard, screen, and Accessibility automation from the first full-duplex Agent implementation.
|
||||
|
||||
#### Scenario: GUI control is requested in first version
|
||||
- **WHEN** a tool call requests direct GUI click, keyboard input, screen control, or Accessibility automation
|
||||
- **THEN** the first-version Tool Router SHALL reject it as unsupported
|
||||
|
||||
#### Scenario: Future provider is designed
|
||||
- **WHEN** a future change implements computer control
|
||||
- **THEN** it SHALL use public capabilities such as macOS Accessibility, Playwright, or trycua-style providers and SHALL define separate confirmation and audit requirements
|
||||
|
||||
#### Scenario: Codex Computer Use is referenced
|
||||
- **WHEN** design work references Codex Computer Use behavior
|
||||
- **THEN** the system SHALL use it only as a safety policy reference and SHALL NOT copy private or bundled implementation details
|
||||
|
||||
### Requirement: Agent security and privacy
|
||||
The full-duplex Agent architecture SHALL protect credentials, raw audio, memory data, and tool execution boundaries by default.
|
||||
|
||||
#### Scenario: Credentials are configured
|
||||
- **WHEN** LLM, embedding, tool, or external adapter credentials are needed
|
||||
- **THEN** they SHALL be read from local uncommitted configuration and SHALL NOT be hard-coded or logged
|
||||
|
||||
#### Scenario: Raw audio is processed
|
||||
- **WHEN** microphone or render audio is processed for APM, VAD, STT, or interruption
|
||||
- **THEN** raw audio and derived voice features SHALL remain process-local unless the user explicitly enables a documented recording feature
|
||||
|
||||
#### Scenario: Long-term memory stores text
|
||||
- **WHEN** memory is saved
|
||||
- **THEN** the memory record SHALL include sensitivity metadata and SHALL NOT include raw audio or credentials
|
||||
|
||||
#### Scenario: Tool audit is recorded
|
||||
- **WHEN** a tool executes
|
||||
- **THEN** the audit record SHALL include tool name, risk level, confirmation status, duration, status, and sanitized summary without secrets
|
||||
|
||||
#### Scenario: User disables tools
|
||||
- **WHEN** `OWNER_TOOL_ROUTER_ENABLED=0`
|
||||
- **THEN** no tool calls SHALL execute and LLM tool requests SHALL be rejected or ignored with a structured message
|
||||
|
||||
### Requirement: Full-duplex performance targets
|
||||
The full-duplex Agent architecture SHALL define measurable latency and reliability targets for audio processing, interruption, streaming STT, LLM-to-TTS, and tool routing.
|
||||
|
||||
#### Scenario: Audio frame is processed
|
||||
- **WHEN** WebRTC APM receives a normal 20 ms frame
|
||||
- **THEN** processing SHALL target completion before the next frame interval in normal local runtime conditions
|
||||
|
||||
#### Scenario: User begins speaking during playback
|
||||
- **WHEN** valid user speech begins while assistant speech is playing
|
||||
- **THEN** playback stop SHALL target P95 latency under 200 ms from speech start
|
||||
|
||||
#### Scenario: User starts speaking in listening mode
|
||||
- **WHEN** the user begins speaking while the system is listening
|
||||
- **THEN** streaming STT SHALL target first stable user-visible transcript within 800 ms P95
|
||||
|
||||
#### Scenario: LLM produces first sentence
|
||||
- **WHEN** the LLM stream yields a complete sentence
|
||||
- **THEN** TTS SHALL target first playable audio chunk within 1000 ms P95
|
||||
|
||||
#### Scenario: Tool call runs
|
||||
- **WHEN** an approved low-risk tool runs
|
||||
- **THEN** it SHALL obey configured timeout, output limit, and maximum calls per turn
|
||||
|
||||
### Requirement: Full-duplex testability
|
||||
The full-duplex Agent architecture SHALL be testable with fake audio devices, fake WebRTC APM reference audio, fake VAD, fake streaming STT/TTS, fake memory stores, fake tools, and deterministic virtual time.
|
||||
|
||||
#### Scenario: Fake APM suppresses echo
|
||||
- **WHEN** a test injects playback reference audio and matching microphone echo
|
||||
- **THEN** the fake or real APM test SHALL verify that echo does not trigger user transcript or interruption
|
||||
|
||||
#### Scenario: Speaking interruption is tested
|
||||
- **WHEN** a test injects user speech during assistant playback
|
||||
- **THEN** the state sequence SHALL include `speaking -> interrupted -> listening` and active LLM/TTS/playback tasks SHALL be cancelled
|
||||
|
||||
#### Scenario: Streaming STT and TTS are tested
|
||||
- **WHEN** fake providers emit partial transcripts, final transcript, LLM deltas, sentence segments, and TTS chunks
|
||||
- **THEN** tests SHALL verify event order, context writes, and playback order
|
||||
|
||||
#### Scenario: Memory persists across restart
|
||||
- **WHEN** a memory is saved and a new runtime instance starts with memory enabled
|
||||
- **THEN** the memory SHALL be retrievable from SQLite and FAISS
|
||||
|
||||
#### Scenario: Tools are tested
|
||||
- **WHEN** fake tool calls cover allowed, rejected, confirmation-required, timeout, and oversized output paths
|
||||
- **THEN** Tool Router tests SHALL verify decisions, audit events, and sanitized tool results
|
||||
|
||||
#### Scenario: OpenSpec validation runs
|
||||
- **WHEN** this planning change is complete
|
||||
- **THEN** `openspec validate add-full-duplex-agent-voice-assistant --strict` and `openspec validate --all --strict` SHALL pass
|
||||
|
||||
### Requirement: Live run-agent-live runtime
|
||||
The system SHALL provide a real `run-agent-live` runtime entry point that starts continuous listening, handles user speech without a wake word, streams the assistant reply through interruptible playback, and returns to listening after completion or interruption.
|
||||
|
||||
#### Scenario: Agent live runtime starts
|
||||
- **WHEN** the user runs `.venv/bin/python -m owner_voice_pet run-agent-live`
|
||||
- **THEN** the command SHALL start the full-duplex Agent runtime instead of returning `FULL_DUPLEX_RUNTIME_NOT_IMPLEMENTED`
|
||||
|
||||
#### Scenario: Agent check-config remains diagnostic
|
||||
- **WHEN** the user runs `.venv/bin/python -m owner_voice_pet run-agent-live --check-config`
|
||||
- **THEN** the command SHALL validate and print non-secret full-duplex configuration without opening the microphone
|
||||
|
||||
#### Scenario: Legacy live runtime remains available
|
||||
- **WHEN** the user runs `.venv/bin/python -m owner_voice_pet run-live`
|
||||
- **THEN** the existing wake-word turn-based runtime SHALL remain available during migration
|
||||
|
||||
### Requirement: Software render-reference interruption gate
|
||||
The first live full-duplex runtime SHALL use software render-reference gating to prevent assistant playback from triggering user interruption while still stopping playback quickly when real user speech is detected.
|
||||
|
||||
#### Scenario: User interrupts assistant playback
|
||||
- **WHEN** valid user speech is detected during assistant playback and it is not classified as assistant render-reference echo
|
||||
- **THEN** playback SHALL stop at the next configured chunk boundary and the captured user frames SHALL be retained for the next STT pass
|
||||
|
||||
#### Scenario: STT partial is unavailable during interruption
|
||||
- **WHEN** VAD and render-reference gating indicate valid user speech but realtime STT has not produced a partial transcript yet
|
||||
- **THEN** the runtime SHALL still stop playback and SHALL use final STT on the buffered user audio afterward
|
||||
|
||||
#### Scenario: Assistant echo is present
|
||||
- **WHEN** microphone input during playback matches recent assistant render-reference audio and no user speech is present
|
||||
- **THEN** the runtime SHALL NOT emit `barge_in_detected` and SHALL continue playback
|
||||
|
||||
#### Scenario: Interrupted assistant text is committed
|
||||
- **WHEN** playback is stopped before a sentence is fully played
|
||||
- **THEN** only fully played assistant text SHALL be written to short-term context
|
||||
@@ -0,0 +1,113 @@
|
||||
## 1. OpenSpec 与项目边界
|
||||
|
||||
- [x] 1.1 冻结 full-duplex Agent 实施入口命名；前置条件：人工确认 `run-agent-live` 或 `run-live --mode full-duplex`；优先级：P0；验收标准：README 和配置说明使用同一入口名；测试要点：CLI help 中能看到一致入口。
|
||||
- [x] 1.2 增加全双工模式配置读取规划；前置条件：入口命名已确认；优先级：P0；验收标准：配置包含 `OWNER_ASSISTANT_MODE`、APM、VAD、STT、TTS、memory、tool router 开关；测试要点：无效 provider 触发配置错误。
|
||||
- [x] 1.3 规划依赖分组；前置条件：provider 候选已确认；优先级：P0；验收标准：依赖按 audio、stt、tts、memory、tools 分组且默认不强制安装重依赖；测试要点：最小安装仍可跑现有 turn-based 测试。
|
||||
- [x] 1.4 定义迁移兼容策略；前置条件：现有 `run-live` 行为梳理完成；优先级：P0；验收标准：turn-based 路径在全双工开发期保持可用；测试要点：旧 simulate-live 和 run-live 回归不破坏。
|
||||
- [x] 1.5 建立模块提交门禁；前置条件：本变更 OpenSpec 已合并；优先级：P0；验收标准：每个主要任务组完成后验证并中文 commit；测试要点：`git status --short` 不包含未提交中间状态。
|
||||
|
||||
## 2. WebRTC APM 与音频环形缓冲
|
||||
|
||||
- [x] 2.1 设计 `AudioFrame` 数据结构；前置条件：内部采样率和帧长方案已确认；优先级：P0；验收标准：包含 samples、sample_rate、channels、timestamp、frame_id；测试要点：frame serialization fake fixture 可稳定回放。
|
||||
- [x] 2.2 实现 capture ring buffer；前置条件：`AudioFrame` 已定义；优先级：P0；验收标准：固定容量、线程安全、溢出事件可观测；测试要点：超过容量时丢弃旧帧并发 `audio_buffer_overrun`。
|
||||
- [x] 2.3 实现 render reference ring buffer；前置条件：playback frame 格式已定义；优先级：P0；验收标准：播放 PCM 写入 reference，保留时间戳；测试要点：可按 capture timestamp 取 reference window。
|
||||
- [x] 2.4 定义 `AudioProcessingProvider` 接口；前置条件：ring buffer 已完成；优先级：P0；验收标准：支持 process_capture、process_render、reset、health_check；测试要点：fake APM 可替换真实 provider。
|
||||
- [x] 2.5 实现 fake WebRTC APM provider；前置条件：接口已定义；优先级：P0；验收标准：测试中可模拟 echo suppression、format mismatch、processing failure；测试要点：纯回声不触发 VAD/STT。
|
||||
- [x] 2.6 接入真实 WebRTC APM 探针；前置条件：依赖选择已人工确认；优先级：P1；验收标准：macOS 本地能初始化 AEC/NS/AGC 或返回明确不可用；测试要点：`model-check` 或 `audio-check` 报告 provider 状态。
|
||||
- [x] 2.7 增加 APM fallback 策略；前置条件：fake/真实 provider 接口完成；优先级：P1；验收标准：`OWNER_AUDIO_APM_REQUIRED` 控制失败即退或带标记降级；测试要点：APM 不可用时不进入假全双工。
|
||||
|
||||
## 3. 全双工状态机、事件总线与取消机制
|
||||
|
||||
- [x] 3.1 定义 full-duplex 状态枚举；前置条件：spec 状态机确认；优先级：P0；验收标准：包含 idle、listening、thinking、speaking、interrupted、tool_running、recovering；测试要点：非法状态转移被拒绝。
|
||||
- [x] 3.2 实现状态机转移表；前置条件：状态枚举完成；优先级：P0；验收标准：每个事件对应合法 next state；测试要点：覆盖正常、打断、工具、错误恢复路径。
|
||||
- [x] 3.3 扩展 pipeline event bus；前置条件：现有 event bus 梳理完成；优先级：P0；验收标准：事件包含 session/turn id、stage、timestamp、sanitized payload；测试要点：终端 reporter 只消费事件。
|
||||
- [x] 3.4 实现 cancellation token；前置条件：状态机完成；优先级：P0；验收标准：root token 和 child token 支持幂等 cancel；测试要点：多次 cancel 不抛异常。
|
||||
- [x] 3.5 建立 cancellation graph；前置条件：LLM/TTS/playback/tool task 边界已定义；优先级：P0；验收标准：打断时同时取消 LLM、TTS、playback 和可取消工具；测试要点：取消后无后台线程泄漏。
|
||||
- [x] 3.6 实现 recovery coordinator；前置条件：错误码清单已定义；优先级：P1；验收标准：可恢复错误进入 recovering 后回 listening 或 idle；测试要点：STT/TTS/LLM/tool 错误均能恢复。
|
||||
|
||||
## 4. VAD、Streaming STT 与低延迟打断
|
||||
|
||||
- [x] 4.1 定义 `VadProvider` 接口；前置条件：APM 输出 frame 格式完成；优先级：P0；验收标准：支持 speech_start、speech_end、confidence；测试要点：fake VAD 可注入人声起止。
|
||||
- [x] 4.2 规划 Silero VAD adapter；前置条件：依赖策略确认；优先级：P1；验收标准：本地模型路径、采样率和阈值可配置；测试要点：模型缺失返回结构化错误。
|
||||
- [x] 4.3 定义 `StreamingSttProvider` 接口；前置条件：transcript event contract 确认；优先级：P0；验收标准：支持 start_session、accept_audio、finish、cancel；测试要点：partial/stable/final 事件顺序稳定。
|
||||
- [x] 4.4 实现 fake Streaming STT；前置条件：接口完成；优先级：P0；验收标准：可模拟 partial 抖动、final 空文本、provider 失败；测试要点：partial 不进入 LLM。
|
||||
- [x] 4.5 规划 faster-whisper adapter；前置条件：模型和依赖策略确认；优先级：P1；验收标准：开发 provider 可配置模型、设备、语言；测试要点：fixture 音频产生 final transcript。
|
||||
- [x] 4.6 规划 SenseVoice adapter；前置条件：产品候选确认；优先级：P2；验收标准：接口兼容 Streaming STT contract；测试要点：中文 fixture 输出与 faster-whisper contract 一致。
|
||||
- [x] 4.7 实现 interruption detector；前置条件：APM、VAD、event bus 完成；优先级：P0；验收标准：speaking 中有效用户声触发 `interrupt_detected`；测试要点：纯助手 echo 不触发。
|
||||
- [x] 4.8 增加 200 ms 打断延迟指标；前置条件：interruption detector 完成；优先级：P0；验收标准：事件记录 speech_start 到 playback_stop latency；测试要点：虚拟时钟 fixture 断言 P95 目标。
|
||||
|
||||
## 5. LLM 流、句子切分、Streaming TTS 与播放
|
||||
|
||||
- [x] 5.1 定义 LLM streaming adapter contract；前置条件：现有 LLM provider 梳理完成；优先级：P0；验收标准：支持 delta、tool_call、finish、cancel、error；测试要点：取消时连接关闭或停止消费。
|
||||
- [x] 5.2 实现 sentence segmenter；前置条件：LLM delta contract 完成；优先级：P0；验收标准：中文标点、英文标点、最大等待阈值可切句；测试要点：URL、小数、代码块不误切。
|
||||
- [x] 5.3 复用 TTS 文本净化；前置条件：现有 sanitizer 可调用；优先级：P0；验收标准：emoji、表情包、Markdown 图片不送 TTS；测试要点：纯表情回复不触发语音。
|
||||
- [x] 5.4 定义 `StreamingTtsProvider` 接口；前置条件：播放 PCM 格式确认；优先级：P0；验收标准：支持 accept_text、flush、cancel、chunk events；测试要点：fake TTS 逐 chunk 输出。
|
||||
- [x] 5.5 规划 CosyVoice adapter；前置条件：产品 TTS 方案确认；优先级：P1；验收标准：可配置 voice、sample_rate、chunk size；测试要点：provider 缺失结构化失败。
|
||||
- [x] 5.6 实现可中断 playback queue；前置条件：render ring buffer 完成；优先级：P0；验收标准：chunk 播放同时写 reference，取消后清空未播 chunk；测试要点：播放停止边界小于配置 chunk。
|
||||
- [x] 5.7 记录已播文本边界；前置条件：sentence/TTS/playback 完成；优先级：P0；验收标准：只提交完整播出的 assistant 文本；测试要点：中途打断未播文本不进上下文。
|
||||
|
||||
## 6. Conversation Manager 与长期记忆
|
||||
|
||||
- [x] 6.1 定义 Conversation Manager 接口；前置条件：状态机和 LLM contract 完成；优先级：P0；验收标准：协调 transcript、context、memory、LLM、tool、TTS；测试要点：正常问答流事件顺序稳定。
|
||||
- [x] 6.2 保留短期会话上下文；前置条件：现有 ConversationContext 梳理完成；优先级：P0；验收标准：进程内历史继续支持截断；测试要点：新 runtime 不读取旧短期历史。
|
||||
- [x] 6.3 设计 SQLite memory schema；前置条件：记忆类型确认；优先级：P0；验收标准：包含 type、text、metadata、sensitivity、source_turn_id、checksum；测试要点：迁移创建表成功。
|
||||
- [x] 6.4 设计 FAISS index 管理；前置条件：embedding provider 策略确认；优先级：P1；验收标准：index 版本、embedding model 和 record id 可一致性检查；测试要点：SQLite/FAISS 不一致时报 health error。
|
||||
- [x] 6.5 实现 fake MemoryManager；前置条件：接口完成；优先级：P0；验收标准：支持 search/save/delete/health_check；测试要点：关闭 memory 时不读写。
|
||||
- [x] 6.6 实现记忆召回链路；前置条件：fake MemoryManager 完成；优先级：P0；验收标准：final transcript 前检索 Top-K 并注入独立 memory context；测试要点：相关偏好可召回。
|
||||
- [x] 6.7 实现记忆写入安全策略；前置条件：sensitivity 分类规则确认；优先级：P0；验收标准：敏感内容默认不自动保存；测试要点：API key、支付信息、账号信息被拒绝或确认。
|
||||
- [x] 6.8 增加记忆管理命令规划；前置条件：schema 和安全策略完成；优先级：P2；验收标准：列出、删除、禁用、导出策略明确；测试要点：删除后检索不到。
|
||||
|
||||
## 7. Tool Router 与安全工具
|
||||
|
||||
- [x] 7.1 定义 ToolCallRequest schema；前置条件：LLM tool call contract 完成；优先级：P0；验收标准：包含 name、arguments、timeout、turn id、intent；测试要点：未知字段拒绝。
|
||||
- [x] 7.2 定义 ToolDecision 和 ToolResult；前置条件：ToolCallRequest 完成；优先级：P0；验收标准：支持 execute、reject、require_confirmation、cancelled；测试要点：结果脱敏和截断。
|
||||
- [x] 7.3 实现 Tool Router fake core；前置条件：schema 完成；优先级：P0；验收标准：按工具名路由 fake adapters；测试要点：合法、非法、超时、超预算均覆盖。
|
||||
- [x] 7.4 实现工具风险分类规则；前置条件：安全策略确认；优先级：P0；验收标准：低/中/高/禁止风险可解释；测试要点：删除、上传、账号、交易、安装依赖触发高风险或禁止。
|
||||
- [x] 7.5 规划 `memory.search` 和 `memory.save`；前置条件：MemoryManager fake 完成；优先级：P0；验收标准：search 只读，save 走敏感策略；测试要点：敏感 save 需要确认或拒绝。
|
||||
- [x] 7.6 规划 `shell.readonly` adapter；前置条件：allowlist 策略确认；优先级：P0；验收标准：只允许只读命令和受限目录；测试要点：写文件、删除、chmod、pip install 被拒绝。
|
||||
- [x] 7.7 增加工具预算和防循环；前置条件：Tool Router core 完成；优先级：P0；验收标准：单轮最大调用数、总耗时、重复调用检测生效；测试要点：循环工具调用被截断。
|
||||
- [x] 7.8 增加工具审计日志规划；前置条件：安全字段确认；优先级：P1；验收标准：记录工具名、风险、确认、耗时、状态、脱敏摘要；测试要点：日志不包含密钥和完整敏感输出。
|
||||
|
||||
## 8. Open Interpreter、Playwright 与电脑控制边界
|
||||
|
||||
- [x] 8.1 规划 Open Interpreter adapter 配置；前置条件：本地 CLI 路径策略确认；优先级：P1；验收标准：`OWNER_OPENINTERPRETER_ENABLED` 默认关闭，命令路径可配置；测试要点：路径缺失返回 unavailable。
|
||||
- [x] 8.2 实现 Open Interpreter 受限执行设计；前置条件：Tool Router 风险策略完成；优先级：P1；验收标准：受限目录、超时、输出截断、确认策略明确；测试要点：高风险任务拒绝或确认。
|
||||
- [x] 8.3 明确 `openinterpreter/` 仓库边界；前置条件：git status 检查完成；优先级：P0；验收标准：Owner 不复制、不 vendoring、不提交该目录；测试要点：提交 diff 不包含 `openinterpreter/`。
|
||||
- [x] 8.4 规划 Playwright adapter；前置条件：浏览器安全策略确认；优先级：P1；验收标准：默认关闭，低风险读取可用，敏感流程确认或拒绝；测试要点：登录、支付、提交表单触发确认策略。
|
||||
- [x] 8.5 规划 browser context 隔离；前置条件：用户是否允许登录态待澄清；优先级：P2；验收标准：默认独立 context，不默认复用用户 Chrome 登录态；测试要点：配置关闭时工具不可用。
|
||||
- [x] 8.6 预留 ComputerControlProvider；前置条件：GUI 控制方案未进入第一版；优先级：P2；验收标准：直接 GUI 点击、键盘、屏幕控制在第一版拒绝；测试要点：相关 tool call 返回 unsupported。
|
||||
- [x] 8.7 编写 Codex Computer Use 安全参考说明；前置条件：公共实现边界确认；优先级：P2；验收标准：只参考确认策略，不复制私有实现；测试要点：文档无私有 API 或实现细节。
|
||||
|
||||
## 9. 端到端模拟、性能与安全验证
|
||||
|
||||
- [x] 9.1 构建 fake full-duplex audio fixture；前置条件：AudioFrame 和 ring buffer 完成；优先级：P0；验收标准：可注入用户语音、助手回声、噪声、打断；测试要点：fixture 可重复回放。
|
||||
- [x] 9.2 增加 fake APM echo 测试；前置条件：fake APM 和 fixture 完成；优先级：P0；验收标准：助手回声不触发 VAD/STT/interrupt；测试要点：无用户语音时无 transcript。
|
||||
- [x] 9.3 增加 speaking interruption 测试；前置条件：状态机、playback、interruption 完成；优先级：P0；验收标准：`speaking -> interrupted -> listening`；测试要点：LLM/TTS/playback 均取消。
|
||||
- [x] 9.4 增加 Streaming STT/TTS 顺序测试；前置条件：fake providers 完成；优先级：P0；验收标准：partial、final、LLM delta、sentence、TTS chunk、playback 顺序稳定；测试要点：partial 不写上下文。
|
||||
- [x] 9.5 增加 memory restart 测试；前置条件：SQLite/FAISS fake 或真实实现完成；优先级：P1；验收标准：保存后新 runtime 可召回；测试要点：禁用 memory 时不读写。
|
||||
- [x] 9.6 增加 Tool Router 安全测试；前置条件：Tool Router core 完成；优先级：P0；验收标准：允许、拒绝、确认、超时、截断、防循环均覆盖；测试要点：高风险操作不自动执行。
|
||||
- [x] 9.7 增加性能指标采集；前置条件：event bus latency 字段完成；优先级：P1；验收标准：记录 APM frame、STT first partial、interrupt latency、TTS first chunk、tool runtime；测试要点：指标不含敏感数据。
|
||||
- [x] 9.8 增加安全检查扩展；前置条件：memory/tool/audit 字段完成；优先级：P1；验收标准：检查 `.env`、key、raw audio、memory secrets、tool logs；测试要点：伪 secret 不出现在日志和事件。
|
||||
|
||||
## 10. 文档、验收与模块提交
|
||||
|
||||
- [x] 10.1 更新 README 全双工架构说明；前置条件：入口和配置确认；优先级：P1；验收标准：说明 APM、Streaming STT/TTS、记忆、工具、限制；测试要点：命令示例和默认关闭项准确。
|
||||
- [x] 10.2 更新 `.env.example` 规划变量；前置条件：配置清单冻结；优先级：P1；验收标准：新增全双工、APM、memory、tool、adapter 配置；测试要点：默认值不启用高风险工具。
|
||||
- [x] 10.3 编写模型和依赖准备说明；前置条件：provider 候选确认；优先级：P1；验收标准：区分开发默认和产品候选；测试要点：不承诺未验证模型已可用。
|
||||
- [x] 10.4 编写安全确认策略文档；前置条件：Tool Router 风险规则完成；优先级：P0；验收标准：列明自动允许、需要确认、默认拒绝类别；测试要点：与工具测试用例一致。
|
||||
- [x] 10.5 执行 Python 编译验证；前置条件：实现代码变更完成；优先级：P0；验收标准：`.venv/bin/python -m compileall src tests scripts` 通过；测试要点：无语法错误。
|
||||
- [x] 10.6 执行单元测试验证；前置条件：测试实现完成；优先级：P0；验收标准：`.venv/bin/python -m unittest discover -s tests` 通过；测试要点：覆盖 full-duplex、memory、tool、security。
|
||||
- [x] 10.7 执行安全和模型检查；前置条件：相关命令实现完成；优先级：P0；验收标准：security-check、model-check、device-check 按阶段通过或给出明确跳过理由；测试要点：不泄露密钥。
|
||||
- [x] 10.8 执行 OpenSpec 校验；前置条件：docs 和 spec 更新完成；优先级：P0；验收标准：`openspec validate add-full-duplex-agent-voice-assistant --strict` 与 `openspec validate --all --strict` 通过；测试要点：无 spec 格式错误。
|
||||
- [x] 10.9 按模块提交；前置条件：对应模块验证通过；优先级：P0；验收标准：提交信息符合 `[模块名]：完成[具体功能描述]，包含[关键变更]`；测试要点：`git status --short` 不含本模块未提交改动。
|
||||
|
||||
## 11. 真实 run-agent-live 全双工运行时
|
||||
|
||||
- [x] 11.1 更新 OpenSpec 真实运行边界；前置条件：用户确认 `run-agent-live` 为真实入口；优先级：P0；验收标准：proposal/design/spec/tasks 明确骨架不等于完成、`run-agent-live` 必须真实启动；测试要点：OpenSpec strict。
|
||||
- [x] 11.2 实现 `run-agent-live` 真实运行路径；前置条件：11.1 完成；优先级：P0；验收标准：不带 `--check-config` 不再返回 `FULL_DUPLEX_RUNTIME_NOT_IMPLEMENTED`，支持 `--once` 测试退出；测试要点：CLI 单元测试。
|
||||
- [x] 11.3 实现无唤醒 Agent turn；前置条件：11.2 完成；优先级：P0；验收标准：启动后直接 listening，用户说话后进入 STT/LLM/TTS，回复后回 listening；测试要点：fake audio 两轮闭环。
|
||||
- [x] 11.4 重做播放打断触发；前置条件：11.3 完成；优先级：P0；验收标准：VAD + 非助手回放即可先 stop playback，不等待 realtime STT partial；测试要点：无 partial 也能打断。
|
||||
- [x] 11.5 增加软件 render-reference gate；前置条件：11.4 完成；优先级：P0；验收标准：只有助手回放 reference 不触发打断，用户叠加回放可触发；测试要点：echo-only、user-over-echo。
|
||||
- [x] 11.6 保留打断用户音频和上下文边界；前置条件：11.5 完成；优先级：P0；验收标准：打断帧进入下一轮 STT，未完整播报 assistant 文本不入上下文；测试要点：上下文和 transcript 断言。
|
||||
- [x] 11.7 更新 README 和验收命令；前置条件：运行行为完成；优先级：P1；验收标准：README 明确 `run-agent-live` 真实入口、`run-live` 旧入口、软件回声抑制限制；测试要点：命令示例准确。
|
||||
- [x] 11.8 执行最终门禁并提交；前置条件：实现和文档完成；优先级：P0；验收标准：compileall、unittest、simulate-live、real-live-check、security/model/device check、OpenSpec strict 通过并中文 commit；测试要点：`git status --short` 干净。
|
||||
@@ -0,0 +1,2 @@
|
||||
schema: spec-driven
|
||||
created: 2026-06-19
|
||||
@@ -0,0 +1,63 @@
|
||||
# 完整全双工 Agent 运行时设计
|
||||
|
||||
## 背景
|
||||
|
||||
当前 `run-agent-live` 通过 `build_live_runtime(config).run_agent()` 复用旧 `VoiceAssistantPipeline`。这条路径只在 TTS 播放时启动临时监听，无法满足持续监听、持续识别、全局可取消和 WebRTC APM 音频底座要求。本设计新增独立 `FullDuplexAgentRuntime`，将旧 pipeline 降级为 `run-live` 专用稳定路径。
|
||||
|
||||
## 数据流
|
||||
|
||||
1. `SoundDeviceAudioTransport` 只负责设备 I/O。
|
||||
2. `AudioHub` 读取麦克风 raw frame，写 raw capture ring。
|
||||
3. `WebRtcAudioProcessingStage` 使用 render reference 处理 raw capture，写 processed capture ring。
|
||||
4. `ContinuousVadWorker` 消费 processed capture，发布 speech_start/speech_end。
|
||||
5. `StreamingSttWorker` 消费同一 processed capture，发布 partial/stable/final transcript。
|
||||
6. `InterruptController` 订阅 VAD 事件和 state machine，在 thinking/speaking/tool_running 中触发 cancellation。
|
||||
7. `ConversationManager` 收到 final transcript 后构建短期上下文、memory context、LLM request。
|
||||
8. LLM streaming delta 进入 sentence segmenter。
|
||||
9. Streaming TTS 产出 PCM chunk，`InterruptiblePlaybackQueue` 播放并写 render reference。
|
||||
10. Tool calls 进入 `ToolRouter`，结果回注入 LLM。
|
||||
|
||||
## 并发模型
|
||||
|
||||
后台任务：
|
||||
|
||||
1. `audio_capture_task`
|
||||
2. `apm_process_task`
|
||||
3. `vad_task`
|
||||
4. `streaming_stt_task`
|
||||
5. `conversation_task`
|
||||
6. `playback_task`
|
||||
7. `tool_task`，按需创建
|
||||
|
||||
所有任务使用 `CancellationGraph` 管理。用户打断时取消当前 response 子图，不关闭 audio hub；进程退出时取消 root token 并关闭所有订阅。
|
||||
|
||||
## 状态管理
|
||||
|
||||
状态机使用现有 `PipelineState` 枚举，不再在 `run-agent-live` 中用隐式 list 记录状态。每个转移必须发 event。非法转移在测试中失败。
|
||||
|
||||
## Provider 策略
|
||||
|
||||
1. APM：`webrtc` 为完整模式默认且 required；`fake` 仅 self-test；`disabled` 只允许开发降级，不算完整验收。
|
||||
2. STT：`faster_whisper` 为开发默认；`sherpa_onnx` fallback；`sensevoice` 预留。
|
||||
3. TTS：`cosyvoice` 为默认目标；缺失时完整模式失败，除非显式配置 fallback。
|
||||
4. Memory：`faiss_sqlite` 默认；如果 FAISS 不可用则 memory health error，基本对话可继续但 self-test 标记失败。
|
||||
5. Tools：默认启用 ToolRouter，但高风险 adapter 默认 disabled。
|
||||
|
||||
## 错误处理
|
||||
|
||||
1. 音频设备错误：启动失败或 recovery 到 idle。
|
||||
2. APM 不可用：完整模式启动失败。
|
||||
3. STT final 空文本：丢弃 utterance，回 listening。
|
||||
4. LLM/TTS 错误：取消 response，发 stage_error，回 listening。
|
||||
5. Tool high-risk：发 confirmation_required；第一版终端拒绝自动执行。
|
||||
6. Memory health error：禁用 memory context，继续基本对话并报告诊断。
|
||||
|
||||
## 自测设计
|
||||
|
||||
`agent-self-test` 使用 fake AudioHub/APM/STT/TTS/LLM/tool/memory，必须模拟三轮：
|
||||
|
||||
1. 正常问答。
|
||||
2. 播放时用户打断。
|
||||
3. memory.search tool call。
|
||||
|
||||
`audio-self-test` 检查真实设备、APM provider health、fake echo suppression、interrupt latency 统计。无真实 WebRTC binding 时输出失败 JSON，不伪造通过。
|
||||
@@ -0,0 +1,183 @@
|
||||
# 完整全双工 Agent 语音助手运行时
|
||||
|
||||
## 功能目标
|
||||
|
||||
当前 `run-agent-live` 已能进入无唤醒监听并在 TTS 播放阶段尝试后台打断，但真实架构仍是轮次式：先录完整句话，再 final STT，再 LLM，再 TTS 播放。播放期打断依赖临时 `AsyncBargeInMonitor` 从同一个 Transport 队列抢帧，无法保证麦克风输入、VAD、STT、回声抑制和播放取消之间的并发边界稳定。用户实际反馈“完全没有打断”，说明继续调参数不能解决根因。
|
||||
|
||||
本变更目标是把 `run-agent-live` 替换为完整全双工 Agent 运行时。目标用户是 macOS 本地桌面语音助手使用者，核心场景是用户可以直接说话、助手边生成边播报、用户随时插话打断、系统取消当前回复并立即处理新问题，同时保留长期记忆和安全工具调用能力。
|
||||
|
||||
量化成功指标：
|
||||
|
||||
1. 打断延迟：在自测 fixture 中，用户语音开始到播放停止的 P95 小于 200 ms。
|
||||
2. 回声隔离：纯助手回放 reference 不触发用户打断、不产生有效用户 transcript。
|
||||
3. 帧分发正确性：AudioHub 多消费者订阅时，VAD、STT、interrupt detector 不互相抢帧。
|
||||
4. 响应首音：LLM 输出首个可播报句子后，TTS 首个 PCM chunk 在 1000 ms 内进入播放队列。
|
||||
5. 自测覆盖：`agent-self-test --profile full-duplex --turns 3` 能无真人完成 STT、LLM、TTS、barge-in、memory、tool router 验收。
|
||||
6. 安全边界：工具调用默认执行低风险工具，高风险工具进入确认或拒绝；日志不泄露 `.env` 密钥、raw PCM 或敏感记忆。
|
||||
|
||||
## 详细需求
|
||||
|
||||
功能需求：
|
||||
|
||||
1. `run-agent-live` SHALL 使用新的 `FullDuplexAgentRuntime`，不得继续调用 `VoiceAssistantPipeline.run_agent_turn()`。
|
||||
2. `run-live` SHALL 保持旧 wake-word turn-based 稳定入口。
|
||||
3. 系统 SHALL 新增 `AudioHub`，作为唯一麦克风输入拥有者，内部维护 20 ms、16 kHz、mono、int16 capture ring buffer。
|
||||
4. VAD、STT、interrupt detector 和诊断录制 SHALL 从 AudioHub 获取独立订阅，不得直接抢读 Transport 队列。
|
||||
5. 系统 SHALL 新增 render ring buffer，播放 PCM 写入 speaker 的同时写入 AEC reference。
|
||||
6. `run-agent-live` SHALL 初始化 WebRTC APM；当 `OWNER_AUDIO_APM_PROVIDER=webrtc` 且 `OWNER_AUDIO_APM_REQUIRED=1` 时，真实 provider 不可用必须启动失败，不得静默回退为伪全双工。
|
||||
7. fake APM SHALL 只用于测试、自测和显式 `OWNER_AUDIO_APM_PROVIDER=fake`。
|
||||
8. 持续 VAD SHALL 在 listening/thinking/speaking/tool_running 阶段持续判断用户是否开始说话。
|
||||
9. Streaming STT SHALL 输出 partial、stable partial、final 三层事件；final 或明确提交 stable transcript 才能进入 LLM。
|
||||
10. 打断 SHALL 不依赖 partial STT；VAD + APM 后有效人声即可触发 cancellation。
|
||||
11. LLM、TTS、播放队列、可取消工具 SHALL 共享 cancellation graph；打断时全部取消。
|
||||
12. LLM SHALL 默认启用 streaming。
|
||||
13. TTS SHALL 使用可分块 PCM provider；CosyVoice 为目标 provider，macOS `say` 只做 fallback。
|
||||
14. 播放 SHALL 每 20-30 ms chunk 检查 cancellation，并把 render chunk 写入 APM reference。
|
||||
15. Conversation Manager SHALL 统一短期上下文、长期记忆、LLM、tool call、TTS 和打断恢复。
|
||||
16. MemoryManager SHALL 支持 SQLite 文本/元数据和 FAISS 向量索引；默认启用但敏感内容不自动保存。
|
||||
17. ToolRouter SHALL 接入 LLM tool call loop，第一批工具包括 `memory.search`、`memory.save`、`shell.readonly`、`openinterpreter.run`、`browser.playwright`。
|
||||
18. Open Interpreter、Playwright 默认关闭；显式启用后仍受风险分类、目录限制、超时、输出截断和确认策略约束。
|
||||
19. README 和 `.env.example` SHALL 统一说明 `run-agent-live` 是完整全双工入口，`run-live` 是旧入口。
|
||||
20. 新增 `agent-self-test` 和 `audio-self-test`，作为无人值守验收命令。
|
||||
|
||||
非功能需求：
|
||||
|
||||
1. 音频回调不得执行阻塞 STT、LLM、TTS 或工具逻辑。
|
||||
2. Capture/render ring buffer 必须有容量上限，溢出丢弃旧帧并发诊断事件。
|
||||
3. 所有后台线程/任务必须可关闭；测试后不得残留线程。
|
||||
4. 事件 payload 必须脱敏，不输出 API key、Authorization、raw PCM、完整工具敏感输出。
|
||||
5. 无 APM、无 STT 模型、无 TTS provider、无设备权限时必须给结构化错误。
|
||||
6. 旧 `simulate-live`、`real-live-check`、`run-live` 不得被新 runtime 破坏。
|
||||
|
||||
边缘案例：
|
||||
|
||||
1. 用户在 LLM 尚未开始播放时插话：取消 thinking，保留新用户音频，重启当前输入。
|
||||
2. 用户在 TTS 合成中插话：取消 TTS，未播文本不进上下文。
|
||||
3. 用户和助手声音重叠：APM 后用户声有效时触发打断；纯回声不得触发。
|
||||
4. 工具运行中插话：可取消工具取消；不可安全取消工具进入确认/等待结果后恢复。
|
||||
5. STT final 空文本：不调用 LLM，回 listening。
|
||||
6. Memory index 损坏：禁用长期记忆并发 health error，不影响基本对话。
|
||||
|
||||
## 设计方案
|
||||
|
||||
文字架构图：
|
||||
|
||||
```text
|
||||
Microphone
|
||||
-> AudioHub raw capture ring
|
||||
-> WebRtcAudioProcessingStage(AEC/NS/AGC, render reference)
|
||||
-> processed capture ring
|
||||
-> Continuous VAD
|
||||
-> Streaming STT
|
||||
-> ConversationManager
|
||||
-> MemoryManager(SQLite + FAISS)
|
||||
-> LLM streaming + ToolRouter
|
||||
-> SentenceSegmenter
|
||||
-> Streaming TTS
|
||||
-> InterruptiblePlaybackQueue
|
||||
-> Speaker + render reference ring
|
||||
```
|
||||
|
||||
主状态机：
|
||||
|
||||
```text
|
||||
idle -> listening -> thinking -> speaking -> listening
|
||||
speaking -> interrupted -> listening
|
||||
thinking -> interrupted -> listening
|
||||
tool_running -> interrupted/recovering/listening
|
||||
any recoverable error -> recovering -> listening
|
||||
```
|
||||
|
||||
关键接口：
|
||||
|
||||
1. `AudioHub.subscribe(kind: str) -> AudioSubscription`
|
||||
- 参数：`kind` 为 `processed_capture`、`raw_capture`、`render_reference`、`debug`。
|
||||
- 返回：独立 cursor 的 frame reader。
|
||||
- 错误：设备缺失、buffer closed、format mismatch。
|
||||
2. `AudioProcessingProvider.process_capture(frame) -> AudioFrame`
|
||||
- 输入：raw capture frame。
|
||||
- 输出：AEC/NS/AGC 后 frame。
|
||||
- 错误：`AUDIO_APM_UNAVAILABLE`、`AUDIO_APM_PROCESS_FAILED`。
|
||||
3. `StreamingSttSession.accept_frame(frame) -> list[TranscriptEvent]`
|
||||
- 输出：partial/stable/final。
|
||||
- 错误：模型缺失、推理失败、取消。
|
||||
4. `StreamingTtsSession.accept_text(text) -> list[AudioFrame]`
|
||||
- 输出：可播放 PCM chunks。
|
||||
- 错误：provider 缺失、合成失败、取消。
|
||||
5. `ToolRouter.route(call) -> ToolDecision`
|
||||
- 输出：execute/reject/require_confirmation。
|
||||
|
||||
性能优化路径：
|
||||
|
||||
1. 音频输入只写 ring buffer，不做重计算。
|
||||
2. STT 和 VAD 消费 processed capture ring，互不阻塞。
|
||||
3. 播放 chunk 与 render reference 同步写入，避免打断检测依赖播放回调抢帧。
|
||||
4. LLM streaming + sentence segmentation 让首句尽早进入 TTS。
|
||||
5. Self-test 记录 interrupt latency、STT first partial、TTS first chunk。
|
||||
|
||||
UI/UX 路径：
|
||||
|
||||
1. 终端输出仍由 event bus 驱动。
|
||||
2. 打断事件输出 `检测到用户打断`、`取消当前回复`、`继续听你说`。
|
||||
3. debug 模式输出 APM provider、echo suppression、interrupt latency。
|
||||
4. 未来 GUI 桌宠订阅同一事件总线。
|
||||
|
||||
## 风险与权衡
|
||||
|
||||
| 风险 | 概率 | 影响 | 缓解措施 |
|
||||
|---|---:|---:|---|
|
||||
| macOS 上没有可用 WebRTC APM Python binding | 高 | 高 | 完整模式启动失败并说明安装要求；self-test 使用 fake APM；旧 `run-live` 保留 |
|
||||
| Faster Whisper 流式能力不是真正低延迟 streaming | 中 | 中 | 先实现稳定 provider contract；允许 sherpa-onnx fallback；SenseVoice 作为后续 provider |
|
||||
| CosyVoice 本地依赖重、安装慢 | 高 | 中 | provider 可配置；macOS say fallback 标记为降级，不宣称完整全双工默认 |
|
||||
| AudioHub 并发复杂导致线程泄漏 | 中 | 高 | 所有后台任务使用 cancellation graph；单元测试检查 shutdown |
|
||||
| AEC 质量不足仍误触发 | 中 | 高 | 增加 echo self-test、render drift 诊断、阈值调试输出 |
|
||||
| FAISS/SQLite 不一致 | 中 | 中 | health-check 和 checksum；不返回未验证记忆 |
|
||||
| 工具调用误执行高风险动作 | 中 | 高 | 默认关闭 Open Interpreter/Playwright；风险分类和确认策略；只读 shell allowlist |
|
||||
| 旧 run-live 回归 | 低 | 高 | 保持代码路径分离；旧测试继续跑 |
|
||||
|
||||
## 任务分解
|
||||
|
||||
详见 `tasks.md`。所有任务拆到不超过 1 小时，按 OpenSpec、音频底座、持续识别与打断、流式回复、记忆工具、自测文档分阶段完成。每个阶段完成后验证并中文 commit。
|
||||
|
||||
## Spec Deltas
|
||||
|
||||
新增：
|
||||
|
||||
1. `Complete full-duplex agent runtime`
|
||||
2. `AudioHub fanout`
|
||||
3. `Required WebRTC APM for full-duplex`
|
||||
4. `Always-on interruption controller`
|
||||
5. `Streaming STT/TTS runtime`
|
||||
6. `Conversation manager with memory and tools`
|
||||
7. `Self-test commands`
|
||||
|
||||
修改：
|
||||
|
||||
1. `Live run-agent-live runtime` 从“可真实启动并播放期打断”升级为“完整全双工主入口”。
|
||||
2. `WebRTC audio processing foundation` 从规划/fake provider 验证升级为运行时启动要求。
|
||||
3. `Tool Router` 从模块级测试升级为 LLM tool-call 主链路要求。
|
||||
|
||||
删除/推翻：
|
||||
|
||||
1. 推翻 `AsyncBargeInMonitor` 作为主打断架构；保留为 legacy fallback 或删除。
|
||||
2. 推翻 `run-agent-live` 复用 `VoiceAssistantPipeline.run_agent_turn()` 的实现边界。
|
||||
3. 推翻“真实运行仍走 run-live”的 README 旧描述。
|
||||
|
||||
## 实施计划
|
||||
|
||||
1. M1 OpenSpec：0.5 天。
|
||||
2. M2 AudioHub + APM：1-1.5 天。
|
||||
3. M3 Continuous VAD/STT + interrupt：1-2 天。
|
||||
4. M4 Streaming response/TTS/playback：1 天。
|
||||
5. M5 Memory + Tool Router：1-1.5 天。
|
||||
6. M6 self-test/docs/final gates：0.5-1 天。
|
||||
|
||||
乐观总耗时 5 天，最可能 7 天，悲观 10 天。没有数据迁移；长期记忆首次启用时创建 SQLite/FAISS 文件，旧短期上下文不迁移。
|
||||
|
||||
## Git 提交规范
|
||||
|
||||
每完成一个主要阶段必须立即验证并提交，提交信息格式：
|
||||
|
||||
`[模块名]：完成[具体功能描述]，包含[关键变更]`
|
||||
|
||||
提交前至少运行该阶段相关单测；最终阶段运行完整门禁并保证 `git status --short` 为空。
|
||||
@@ -0,0 +1,96 @@
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Complete full-duplex agent runtime
|
||||
The system SHALL provide a complete `run-agent-live` runtime that uses a dedicated full-duplex agent architecture instead of the legacy turn-based `VoiceAssistantPipeline.run_agent_turn()` loop.
|
||||
|
||||
#### Scenario: Full-duplex entry starts the new runtime
|
||||
- **WHEN** the user runs `.venv/bin/python -m owner_voice_pet run-agent-live`
|
||||
- **THEN** the command SHALL construct `FullDuplexAgentRuntime`
|
||||
- **AND** it SHALL NOT call `VoiceAssistantPipeline.run_agent_turn()` as the primary runtime path
|
||||
|
||||
#### Scenario: Legacy runtime remains available
|
||||
- **WHEN** the user runs `.venv/bin/python -m owner_voice_pet run-live`
|
||||
- **THEN** the legacy wake-word turn-based runtime SHALL remain available
|
||||
|
||||
### Requirement: AudioHub fanout
|
||||
The full-duplex runtime SHALL use a single AudioHub to own microphone input and distribute frames to independent consumers without frame stealing.
|
||||
|
||||
#### Scenario: Multiple consumers subscribe
|
||||
- **WHEN** VAD, streaming STT, interrupt detection, and diagnostics subscribe to processed capture audio
|
||||
- **THEN** each consumer SHALL receive the same ordered audio frame sequence from its own cursor
|
||||
|
||||
#### Scenario: Ring buffer overflows
|
||||
- **WHEN** capture or render ring buffer exceeds configured capacity
|
||||
- **THEN** the hub SHALL drop oldest safe frames, emit an overrun diagnostic, and continue without blocking audio callbacks
|
||||
|
||||
### Requirement: Required WebRTC APM in full-duplex mode
|
||||
The full-duplex runtime SHALL process microphone capture through WebRTC AEC, NS, and AGC using playback render reference before VAD, STT, or interrupt detection consume frames.
|
||||
|
||||
#### Scenario: APM is unavailable
|
||||
- **WHEN** `OWNER_AUDIO_APM_PROVIDER=webrtc`, `OWNER_AUDIO_APM_REQUIRED=1`, and no real WebRTC APM provider is available
|
||||
- **THEN** `run-agent-live` SHALL fail startup with a structured audio APM error
|
||||
- **AND** it SHALL NOT silently run the software barge-in fallback as complete full-duplex mode
|
||||
|
||||
#### Scenario: Fake APM is used in self-test
|
||||
- **WHEN** `agent-self-test` or unit tests request fake APM
|
||||
- **THEN** fake APM SHALL be allowed and SHALL be clearly reported as test-only
|
||||
|
||||
#### Scenario: Pure assistant echo occurs
|
||||
- **WHEN** microphone input is only assistant render echo
|
||||
- **THEN** APM-processed audio SHALL NOT trigger valid interruption or valid user transcript
|
||||
|
||||
### Requirement: Always-on interruption controller
|
||||
The full-duplex runtime SHALL run a continuous interruption controller that can cancel active response work from `thinking`, `speaking`, or cancellable `tool_running` states.
|
||||
|
||||
#### Scenario: User interrupts while speaking
|
||||
- **WHEN** the assistant is playing audio and processed capture contains valid user speech
|
||||
- **THEN** playback SHALL stop, active LLM/TTS work SHALL be cancelled, and the state SHALL transition through `interrupted` back to `listening`
|
||||
|
||||
#### Scenario: Interruption latency is measured
|
||||
- **WHEN** self-test injects user speech during playback
|
||||
- **THEN** the runtime SHALL report VAD-to-playback-stop latency and target P95 under 200 ms
|
||||
|
||||
#### Scenario: Interruption does not require STT partial
|
||||
- **WHEN** valid user speech is detected before streaming STT emits partial text
|
||||
- **THEN** playback cancellation SHALL still occur
|
||||
|
||||
### Requirement: Streaming STT and streaming TTS runtime
|
||||
The full-duplex runtime SHALL use streaming recognition and streaming speech output contracts for the agent path.
|
||||
|
||||
#### Scenario: User speaks during listening
|
||||
- **WHEN** processed capture contains user speech
|
||||
- **THEN** streaming STT SHALL emit partial or stable transcript events while speech continues and final transcript when the utterance ends
|
||||
|
||||
#### Scenario: LLM emits a sentence
|
||||
- **WHEN** the LLM stream produces a complete safe speech segment
|
||||
- **THEN** streaming TTS SHALL synthesize playable PCM chunks without waiting for the full assistant response
|
||||
|
||||
#### Scenario: Playback is interrupted
|
||||
- **WHEN** cancellation occurs during streaming TTS or playback
|
||||
- **THEN** unplayed audio chunks SHALL be discarded and unspoken assistant text SHALL NOT be committed to context
|
||||
|
||||
### Requirement: Full-duplex Conversation Manager
|
||||
The full-duplex runtime SHALL coordinate final user transcripts, session context, long-term memory retrieval, LLM streaming, tool calls, TTS playback, interruption, and recovery.
|
||||
|
||||
#### Scenario: Memory is enabled
|
||||
- **WHEN** a final user transcript is accepted and memory is enabled
|
||||
- **THEN** the Conversation Manager SHALL retrieve Top-K relevant memories and provide them as separate context to the LLM
|
||||
|
||||
#### Scenario: Tool call is requested
|
||||
- **WHEN** the LLM stream requests a supported tool call
|
||||
- **THEN** the Tool Router SHALL classify, approve, reject, or request confirmation before execution
|
||||
|
||||
#### Scenario: Tool is high risk
|
||||
- **WHEN** a tool request involves deletion, upload, payment, account changes, installation, or other high-risk action
|
||||
- **THEN** the runtime SHALL NOT execute it automatically
|
||||
|
||||
### Requirement: Full-duplex self-test commands
|
||||
The system SHALL provide deterministic self-test commands for the complete full-duplex agent runtime.
|
||||
|
||||
#### Scenario: Agent self-test runs
|
||||
- **WHEN** the developer runs `.venv/bin/python -m owner_voice_pet agent-self-test --profile full-duplex --turns 3`
|
||||
- **THEN** the command SHALL exercise STT, LLM, TTS, interruption, memory, and Tool Router paths and return JSON with `success=true` only when all checks pass
|
||||
|
||||
#### Scenario: Audio self-test runs
|
||||
- **WHEN** the developer runs `.venv/bin/python -m owner_voice_pet audio-self-test --duration 10 --check-echo`
|
||||
- **THEN** the command SHALL report device availability, APM provider health, echo suppression result, and interruption latency statistics
|
||||
@@ -0,0 +1,45 @@
|
||||
## 1. OpenSpec 与运行边界
|
||||
|
||||
- [x] 1.1 新建 `complete-full-duplex-agent-runtime` OpenSpec；前置条件：当前计划已确认；优先级：P0；验收标准：proposal/design/tasks/spec delta 存在；测试要点：`openspec validate complete-full-duplex-agent-runtime --strict`。
|
||||
- [x] 1.2 明确 `run-agent-live` 替换旧 `VoiceAssistantPipeline.run_agent_turn()`；前置条件：1.1；优先级：P0；验收标准：文档和任务说明旧路径为 legacy；测试要点：README 后续无冲突描述。
|
||||
- [x] 1.3 Phase 1 提交；前置条件：1.1-1.2；优先级：P0；验收标准：中文提交 `[全双工规格补齐]...`；测试要点：OpenSpec strict 和 git status。
|
||||
|
||||
## 2. AudioHub 与 WebRTC APM
|
||||
|
||||
- [x] 2.1 实现 `AudioHub` 和独立订阅；前置条件：Phase 1；优先级：P0；验收标准：多消费者不抢帧；测试要点：两个订阅读取相同 frame 序列。
|
||||
- [x] 2.2 实现 capture/render ring buffer 溢出诊断；前置条件：2.1；优先级：P0；验收标准：容量上限、丢旧帧、事件记录；测试要点：overflow fixture。
|
||||
- [x] 2.3 接入 APM provider 到 AudioHub；前置条件：2.1；优先级：P0；验收标准：raw capture 经过 APM 后进入 processed ring；测试要点：fake APM echo suppression。
|
||||
- [x] 2.4 `run-agent-live` 启动时检查真实 APM；前置条件：2.3；优先级：P0；验收标准：required 且不可用时结构化失败；测试要点：provider unavailable test。
|
||||
- [x] 2.5 Phase 2 提交；前置条件：2.1-2.4；优先级：P0；验收标准：中文提交 `[音频底座重构]...`；测试要点：compileall、相关单测。
|
||||
|
||||
## 3. Continuous VAD/STT 与打断
|
||||
|
||||
- [x] 3.1 实现 `InterruptController` 常驻检测；前置条件：AudioHub；优先级：P0；验收标准：speaking 中 VAD 命中触发 cancel；测试要点：latency fixture。
|
||||
- [x] 3.2 接入 cancellation graph 到 LLM/TTS/playback；前置条件：3.1；优先级：P0；验收标准：打断取消所有 response 子任务；测试要点：幂等 cancel。
|
||||
- [x] 3.3 实现 streaming STT worker；前置条件：AudioHub；优先级：P0；验收标准：partial/stable/final 事件；测试要点：final 进入 conversation，partial 不进上下文。
|
||||
- [x] 3.4 打断后复用 buffered user audio；前置条件：3.1-3.3；优先级：P0；验收标准：不需要重新唤醒；测试要点：`speaking -> interrupted -> listening`。
|
||||
- [x] 3.5 Phase 3 提交；前置条件：3.1-3.4；优先级：P0；验收标准：中文提交 `[全双工打断]...`；测试要点：相关单测和 self-test 子集。
|
||||
|
||||
## 4. Streaming LLM/TTS/Playback
|
||||
|
||||
- [x] 4.1 默认启用 LLM streaming；前置条件：Phase 3；优先级：P0；验收标准：`run-agent-live --check-config` 显示 streaming true；测试要点：配置测试。
|
||||
- [x] 4.2 实现 streaming TTS provider wrapper；前置条件：4.1；优先级：P0；验收标准：句子进入 TTS 后输出 PCM chunks；测试要点：fake/cosyvoice fallback。
|
||||
- [x] 4.3 播放队列写 render reference；前置条件：4.2；优先级：P0；验收标准：播放 chunk 同步进入 APM reference；测试要点：render ring frame count。
|
||||
- [x] 4.4 只提交已播 assistant 文本；前置条件：4.3；优先级：P0；验收标准：中途打断不写未播文本；测试要点：上下文断言。
|
||||
- [x] 4.5 Phase 4 提交；前置条件：4.1-4.4；优先级：P0；验收标准：中文提交 `[流式回复播放]...`；测试要点：compileall、单测。
|
||||
|
||||
## 5. Memory 与 Tool Router
|
||||
|
||||
- [x] 5.1 将 `AgentConversationManager` 接入 runtime；前置条件：Phase 4；优先级：P0；验收标准：memory context 注入 LLM；测试要点：相关记忆被召回。
|
||||
- [x] 5.2 接入 FAISS/SQLite health；前置条件：5.1；优先级：P0；验收标准：memory enabled 时检查 index；测试要点：缺失/不一致错误。
|
||||
- [x] 5.3 ToolRouter 接入 LLM tool calls；前置条件：5.1；优先级：P0；验收标准：memory.search tool result 回注入回复；测试要点：tool call integration。
|
||||
- [x] 5.4 高风险工具确认/拒绝；前置条件：5.3；优先级：P0；验收标准：Open Interpreter/Playwright 默认不自动执行；测试要点：高风险 fixture。
|
||||
- [x] 5.5 Phase 5 提交；前置条件：5.1-5.4；优先级：P0；验收标准：中文提交 `[Agent记忆工具]...`；测试要点：memory/tool/security 单测。
|
||||
|
||||
## 6. 自我测试、文档与最终验收
|
||||
|
||||
- [x] 6.1 新增 `agent-self-test`；前置条件：Phase 5；优先级：P0；验收标准：三轮覆盖 STT/LLM/TTS/barge-in/memory/tool；测试要点：JSON success。
|
||||
- [x] 6.2 新增 `audio-self-test`；前置条件：AudioHub/APM；优先级：P0；验收标准：输出 provider/device/echo/latency；测试要点：无 APM 时明确失败。
|
||||
- [x] 6.3 更新 README 和 `.env.example`；前置条件：6.1-6.2；优先级：P0；验收标准：入口和配置无冲突；测试要点：命令示例准确。
|
||||
- [x] 6.4 执行最终门禁；前置条件：全部实现完成；优先级：P0；验收标准：compileall、unittest、self-test、security、model、device、OpenSpec strict 全通过；测试要点：命令输出。
|
||||
- [x] 6.5 Phase 6 提交；前置条件：6.1-6.4；优先级：P0；验收标准：中文提交 `[全双工自测]...`；测试要点：`git status --short` 为空。
|
||||
@@ -15,6 +15,11 @@ dependencies = []
|
||||
[project.optional-dependencies]
|
||||
audio = ["numpy>=1.26", "sounddevice>=0.4.7"]
|
||||
speech = ["numpy>=1.26", "sounddevice>=0.4.7", "sherpa-onnx>=1.13.3"]
|
||||
full-duplex = ["numpy>=1.26", "sounddevice>=0.4.7", "aec-audio-processing>=1.0.1"]
|
||||
streaming-stt = ["faster-whisper>=1.0"]
|
||||
memory = ["faiss-cpu>=1.8", "numpy>=1.26"]
|
||||
browser = ["playwright>=1.44"]
|
||||
agent-tools = ["playwright>=1.44"]
|
||||
ui = ["PySide6>=6.7"]
|
||||
test = []
|
||||
|
||||
|
||||
@@ -3,7 +3,80 @@
|
||||
from .config import AppConfig
|
||||
from .assistant_pipeline import TurnController, VoiceAssistantPipeline
|
||||
from .audio_preprocess import NoopAudioPreprocessor, SherpaOnnxDenoiserPreprocessor
|
||||
from .agent_memory import (
|
||||
AgentConversationManager,
|
||||
DisabledMemoryManager,
|
||||
FaissIndexManifest,
|
||||
FakeMemoryManager,
|
||||
MemoryManagementPlan,
|
||||
MemoryRecord,
|
||||
MemoryRecordInput,
|
||||
MemoryWritePolicy,
|
||||
SQLiteMemoryManager,
|
||||
)
|
||||
from .events import PipelineEvent, PipelineEventBus
|
||||
from .external_adapters import (
|
||||
BrowserPlaywrightAdapter,
|
||||
ComputerControlProvider,
|
||||
OpenInterpreterAdapter,
|
||||
planned_external_adapters,
|
||||
)
|
||||
from .full_duplex_control import (
|
||||
CancellationGraph,
|
||||
CancellationToken,
|
||||
FullDuplexStateMachine,
|
||||
InvalidStateTransition,
|
||||
RecoveryCoordinator,
|
||||
StateTransition,
|
||||
)
|
||||
from .full_duplex_audio import (
|
||||
AudioHub,
|
||||
AudioHubDiagnostic,
|
||||
AudioProcessingHealth,
|
||||
FakeWebRtcAudioProcessingProvider,
|
||||
NoopAudioProcessingProvider,
|
||||
build_audio_processing_provider,
|
||||
webrtc_apm_probe,
|
||||
)
|
||||
from .full_duplex_runtime import FullDuplexAgentRuntime, FullDuplexRuntimeHealth
|
||||
from .full_duplex_speech import (
|
||||
FakeStreamingSttProvider,
|
||||
FakeVadProvider,
|
||||
InterruptController,
|
||||
InterruptControllerResult,
|
||||
InterruptionDecision,
|
||||
InterruptionDetector,
|
||||
SileroVadProvider,
|
||||
StreamingSttWorker,
|
||||
StreamingSttProvider,
|
||||
TranscriptEvent,
|
||||
VadEvent,
|
||||
VadProvider,
|
||||
)
|
||||
from .full_duplex_response import (
|
||||
AudioSegmentStreamingTtsProvider,
|
||||
AudioSegmentStreamingTtsSession,
|
||||
FakeStreamingLlmProvider,
|
||||
FakeStreamingTtsProvider,
|
||||
InterruptiblePlaybackQueue,
|
||||
LlmStreamEvent,
|
||||
SentenceSegmenter,
|
||||
StreamingLlmProvider,
|
||||
StreamingTtsProvider,
|
||||
prepare_tts_sentence,
|
||||
)
|
||||
from .tool_router import (
|
||||
FakeToolAdapter,
|
||||
MemorySaveTool,
|
||||
MemorySearchTool,
|
||||
ShellReadonlyTool,
|
||||
ToolCallRequest,
|
||||
ToolContext,
|
||||
ToolDecision,
|
||||
ToolResult,
|
||||
ToolRouter,
|
||||
ToolRiskClassifier,
|
||||
)
|
||||
from .models import (
|
||||
AudioFrame,
|
||||
AudioSegment,
|
||||
@@ -27,6 +100,7 @@ from .llm import MockLlmProvider, OpenAICompatibleLlmProvider
|
||||
from .pipeline import PipelineResult, VoicePipeline
|
||||
from .runtime import LiveVoiceRuntime, RuntimeSummary, TerminalRuntimeReporter, TurnResult, build_live_runtime
|
||||
from .simulation import run_simulated_live
|
||||
from .self_tests import run_agent_self_test, run_audio_self_test
|
||||
from .tts import CloudTtsProvider, MacSayTtsProvider, SentenceBuffer, SineTtsProvider, make_end_chime, make_prompt_chime, sanitize_tts_text
|
||||
from .assets import validate_pet_assets
|
||||
from .ui import ConsolePetWindow, PetStateController, PetVisualState
|
||||
@@ -37,8 +111,68 @@ __all__ = [
|
||||
"VoiceAssistantPipeline",
|
||||
"NoopAudioPreprocessor",
|
||||
"SherpaOnnxDenoiserPreprocessor",
|
||||
"AgentConversationManager",
|
||||
"DisabledMemoryManager",
|
||||
"FaissIndexManifest",
|
||||
"FakeMemoryManager",
|
||||
"MemoryManagementPlan",
|
||||
"MemoryRecord",
|
||||
"MemoryRecordInput",
|
||||
"MemoryWritePolicy",
|
||||
"SQLiteMemoryManager",
|
||||
"PipelineEvent",
|
||||
"PipelineEventBus",
|
||||
"BrowserPlaywrightAdapter",
|
||||
"ComputerControlProvider",
|
||||
"OpenInterpreterAdapter",
|
||||
"planned_external_adapters",
|
||||
"CancellationGraph",
|
||||
"CancellationToken",
|
||||
"FullDuplexStateMachine",
|
||||
"InvalidStateTransition",
|
||||
"RecoveryCoordinator",
|
||||
"StateTransition",
|
||||
"AudioHub",
|
||||
"AudioHubDiagnostic",
|
||||
"AudioProcessingHealth",
|
||||
"FakeWebRtcAudioProcessingProvider",
|
||||
"NoopAudioProcessingProvider",
|
||||
"build_audio_processing_provider",
|
||||
"webrtc_apm_probe",
|
||||
"FullDuplexAgentRuntime",
|
||||
"FullDuplexRuntimeHealth",
|
||||
"AudioSegmentStreamingTtsProvider",
|
||||
"AudioSegmentStreamingTtsSession",
|
||||
"FakeStreamingSttProvider",
|
||||
"FakeVadProvider",
|
||||
"InterruptController",
|
||||
"InterruptControllerResult",
|
||||
"InterruptionDecision",
|
||||
"InterruptionDetector",
|
||||
"SileroVadProvider",
|
||||
"StreamingSttWorker",
|
||||
"StreamingSttProvider",
|
||||
"TranscriptEvent",
|
||||
"VadEvent",
|
||||
"VadProvider",
|
||||
"FakeStreamingLlmProvider",
|
||||
"FakeStreamingTtsProvider",
|
||||
"InterruptiblePlaybackQueue",
|
||||
"LlmStreamEvent",
|
||||
"SentenceSegmenter",
|
||||
"StreamingLlmProvider",
|
||||
"StreamingTtsProvider",
|
||||
"prepare_tts_sentence",
|
||||
"FakeToolAdapter",
|
||||
"MemorySaveTool",
|
||||
"MemorySearchTool",
|
||||
"ShellReadonlyTool",
|
||||
"ToolCallRequest",
|
||||
"ToolContext",
|
||||
"ToolDecision",
|
||||
"ToolResult",
|
||||
"ToolRouter",
|
||||
"ToolRiskClassifier",
|
||||
"AudioFrame",
|
||||
"AudioSegment",
|
||||
"AudioRingBuffer",
|
||||
@@ -66,6 +200,8 @@ __all__ = [
|
||||
"TurnResult",
|
||||
"build_live_runtime",
|
||||
"run_simulated_live",
|
||||
"run_agent_self_test",
|
||||
"run_audio_self_test",
|
||||
"CloudTtsProvider",
|
||||
"MacSayTtsProvider",
|
||||
"SentenceBuffer",
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Literal, Protocol
|
||||
|
||||
from .conversation import ConversationContext
|
||||
from .models import ErrorCode, Message, ProviderError
|
||||
|
||||
|
||||
MemoryType = Literal["preference", "fact", "project", "task_summary"]
|
||||
MemorySensitivity = Literal["normal", "sensitive"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MemoryRecordInput:
|
||||
type: MemoryType
|
||||
text: str
|
||||
metadata: dict[str, object] = field(default_factory=dict)
|
||||
sensitivity: MemorySensitivity = "normal"
|
||||
source_turn_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MemoryRecord:
|
||||
id: str
|
||||
type: MemoryType
|
||||
text: str
|
||||
metadata: dict[str, object]
|
||||
sensitivity: MemorySensitivity
|
||||
source_turn_id: str | None
|
||||
created_at: float
|
||||
updated_at: float
|
||||
last_used_at: float | None
|
||||
embedding_id: str
|
||||
checksum: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MemoryHealth:
|
||||
ok: bool
|
||||
errors: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class MemoryManager(Protocol):
|
||||
enabled: bool
|
||||
|
||||
def search(self, query: str, *, top_k: int = 5, filters: dict[str, object] | None = None) -> list[MemoryRecord]:
|
||||
...
|
||||
|
||||
def save(self, record: MemoryRecordInput) -> MemoryRecord:
|
||||
...
|
||||
|
||||
def delete(self, memory_id: str) -> None:
|
||||
...
|
||||
|
||||
def health_check(self) -> MemoryHealth:
|
||||
...
|
||||
|
||||
|
||||
class DisabledMemoryManager:
|
||||
enabled = False
|
||||
|
||||
def search(self, query: str, *, top_k: int = 5, filters: dict[str, object] | None = None) -> list[MemoryRecord]:
|
||||
return []
|
||||
|
||||
def save(self, record: MemoryRecordInput) -> MemoryRecord:
|
||||
raise ProviderError(
|
||||
ErrorCode.VALIDATION_FAILED,
|
||||
"memory is disabled",
|
||||
False,
|
||||
"memory",
|
||||
"memory",
|
||||
)
|
||||
|
||||
def delete(self, memory_id: str) -> None:
|
||||
return None
|
||||
|
||||
def health_check(self) -> MemoryHealth:
|
||||
return MemoryHealth(ok=True)
|
||||
|
||||
|
||||
class FakeMemoryManager:
|
||||
def __init__(self, *, enabled: bool = True) -> None:
|
||||
self.enabled = enabled
|
||||
self.records: dict[str, MemoryRecord] = {}
|
||||
self.search_queries: list[str] = []
|
||||
|
||||
def search(self, query: str, *, top_k: int = 5, filters: dict[str, object] | None = None) -> list[MemoryRecord]:
|
||||
if not self.enabled:
|
||||
return []
|
||||
self.search_queries.append(query)
|
||||
query_terms = set(_tokenize(query))
|
||||
ranked = sorted(
|
||||
self.records.values(),
|
||||
key=lambda record: len(query_terms.intersection(_tokenize(record.text))),
|
||||
reverse=True,
|
||||
)
|
||||
return [record for record in ranked if record.sensitivity == "normal"][:top_k]
|
||||
|
||||
def save(self, record: MemoryRecordInput) -> MemoryRecord:
|
||||
if not self.enabled:
|
||||
raise ProviderError(
|
||||
ErrorCode.VALIDATION_FAILED,
|
||||
"memory is disabled",
|
||||
False,
|
||||
"memory",
|
||||
"memory",
|
||||
)
|
||||
saved = _build_memory_record(record)
|
||||
self.records[saved.id] = saved
|
||||
return saved
|
||||
|
||||
def delete(self, memory_id: str) -> None:
|
||||
self.records.pop(memory_id, None)
|
||||
|
||||
def health_check(self) -> MemoryHealth:
|
||||
return MemoryHealth(ok=True)
|
||||
|
||||
|
||||
class SQLiteMemoryManager:
|
||||
enabled = True
|
||||
|
||||
def __init__(self, db_path: Path) -> None:
|
||||
self.db_path = db_path
|
||||
|
||||
def initialize(self) -> None:
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS memories (
|
||||
id TEXT PRIMARY KEY,
|
||||
type TEXT NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
metadata_json TEXT NOT NULL,
|
||||
sensitivity TEXT NOT NULL,
|
||||
source_turn_id TEXT,
|
||||
created_at REAL NOT NULL,
|
||||
updated_at REAL NOT NULL,
|
||||
last_used_at REAL,
|
||||
embedding_id TEXT NOT NULL,
|
||||
checksum TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
def search(self, query: str, *, top_k: int = 5, filters: dict[str, object] | None = None) -> list[MemoryRecord]:
|
||||
self.initialize()
|
||||
query_terms = set(_tokenize(query))
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM memories WHERE sensitivity = 'normal'").fetchall()
|
||||
records = [_record_from_row(row) for row in rows]
|
||||
ranked = sorted(
|
||||
records,
|
||||
key=lambda record: len(query_terms.intersection(_tokenize(record.text))),
|
||||
reverse=True,
|
||||
)
|
||||
return ranked[:top_k]
|
||||
|
||||
def save(self, record: MemoryRecordInput) -> MemoryRecord:
|
||||
self.initialize()
|
||||
saved = _build_memory_record(record)
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO memories (
|
||||
id, type, text, metadata_json, sensitivity, source_turn_id,
|
||||
created_at, updated_at, last_used_at, embedding_id, checksum
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
saved.id,
|
||||
saved.type,
|
||||
saved.text,
|
||||
json.dumps(saved.metadata, ensure_ascii=False, sort_keys=True),
|
||||
saved.sensitivity,
|
||||
saved.source_turn_id,
|
||||
saved.created_at,
|
||||
saved.updated_at,
|
||||
saved.last_used_at,
|
||||
saved.embedding_id,
|
||||
saved.checksum,
|
||||
),
|
||||
)
|
||||
return saved
|
||||
|
||||
def delete(self, memory_id: str) -> None:
|
||||
self.initialize()
|
||||
with self._connect() as conn:
|
||||
conn.execute("DELETE FROM memories WHERE id = ?", (memory_id,))
|
||||
|
||||
def health_check(self) -> MemoryHealth:
|
||||
try:
|
||||
self.initialize()
|
||||
with self._connect() as conn:
|
||||
conn.execute("SELECT id, checksum FROM memories LIMIT 1").fetchall()
|
||||
except sqlite3.Error as exc:
|
||||
return MemoryHealth(ok=False, errors=(str(exc),))
|
||||
return MemoryHealth(ok=True)
|
||||
|
||||
def all_records(self) -> list[MemoryRecord]:
|
||||
self.initialize()
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM memories").fetchall()
|
||||
return [_record_from_row(row) for row in rows]
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FaissIndexManifest:
|
||||
embedding_model: str
|
||||
record_ids: tuple[str, ...]
|
||||
checksums: dict[str, str]
|
||||
|
||||
@classmethod
|
||||
def from_records(cls, records: list[MemoryRecord], *, embedding_model: str) -> "FaissIndexManifest":
|
||||
return cls(
|
||||
embedding_model=embedding_model,
|
||||
record_ids=tuple(record.id for record in records),
|
||||
checksums={record.id: record.checksum for record in records},
|
||||
)
|
||||
|
||||
def consistency_errors(self, records: list[MemoryRecord]) -> tuple[str, ...]:
|
||||
errors: list[str] = []
|
||||
by_id = {record.id: record for record in records}
|
||||
for record_id in self.record_ids:
|
||||
if record_id not in by_id:
|
||||
errors.append(f"index references missing memory id {record_id}")
|
||||
for record in records:
|
||||
checksum = self.checksums.get(record.id)
|
||||
if checksum != record.checksum:
|
||||
errors.append(f"checksum mismatch for memory id {record.id}")
|
||||
return tuple(errors)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MemoryWriteDecision:
|
||||
should_save: bool
|
||||
requires_confirmation: bool
|
||||
reason: str
|
||||
|
||||
|
||||
class MemoryWritePolicy:
|
||||
def evaluate(self, record: MemoryRecordInput) -> MemoryWriteDecision:
|
||||
if record.sensitivity == "sensitive" or is_sensitive_memory_text(record.text):
|
||||
return MemoryWriteDecision(
|
||||
should_save=False,
|
||||
requires_confirmation=True,
|
||||
reason="sensitive_memory_requires_confirmation",
|
||||
)
|
||||
return MemoryWriteDecision(should_save=True, requires_confirmation=False, reason="safe_to_save")
|
||||
|
||||
|
||||
class AgentConversationManager:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
context: ConversationContext,
|
||||
memory: MemoryManager,
|
||||
memory_enabled: bool,
|
||||
memory_top_k: int = 5,
|
||||
) -> None:
|
||||
self.context = context
|
||||
self.memory = memory
|
||||
self.memory_enabled = memory_enabled
|
||||
self.memory_top_k = memory_top_k
|
||||
|
||||
def build_messages_for_user(self, user_text: str) -> list[Message]:
|
||||
messages = self.context.build_llm_messages()
|
||||
if self.memory_enabled and self.memory.enabled:
|
||||
memories = self.memory.search(user_text, top_k=self.memory_top_k)
|
||||
if memories:
|
||||
memory_text = "\n".join(f"- [{record.type}] {record.text}" for record in memories)
|
||||
messages.insert(1, Message("system", f"长期记忆：\n{memory_text}", time.time()))
|
||||
messages.append(Message("user", user_text.strip(), time.time()))
|
||||
return messages
|
||||
|
||||
def commit_user(self, text: str) -> None:
|
||||
self.context.append_user(text)
|
||||
|
||||
def commit_assistant(self, text: str) -> None:
|
||||
self.context.append_assistant(text)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MemoryManagementPlan:
|
||||
supported_commands: tuple[str, ...] = ("list", "delete", "disable", "export")
|
||||
default_enabled: bool = True
|
||||
|
||||
|
||||
def is_sensitive_memory_text(text: str) -> bool:
|
||||
patterns = (
|
||||
r"(?:sk|tp)-[A-Za-z0-9_\-]{16,}",
|
||||
r"api[_ -]?key",
|
||||
r"authorization",
|
||||
r"password",
|
||||
r"passwd",
|
||||
r"密码",
|
||||
r"银行卡",
|
||||
r"身份证",
|
||||
)
|
||||
lowered = text.lower()
|
||||
return any(re.search(pattern, lowered, flags=re.IGNORECASE) for pattern in patterns)
|
||||
|
||||
|
||||
def _build_memory_record(record: MemoryRecordInput) -> MemoryRecord:
|
||||
clean = record.text.strip()
|
||||
if not clean:
|
||||
raise ProviderError(
|
||||
ErrorCode.VALIDATION_FAILED,
|
||||
"memory text must be non-empty",
|
||||
False,
|
||||
"memory",
|
||||
"memory",
|
||||
)
|
||||
now = time.time()
|
||||
memory_id = str(uuid.uuid4())
|
||||
checksum = _checksum(record.type, clean, record.metadata)
|
||||
return MemoryRecord(
|
||||
id=memory_id,
|
||||
type=record.type,
|
||||
text=clean,
|
||||
metadata=dict(record.metadata),
|
||||
sensitivity=record.sensitivity,
|
||||
source_turn_id=record.source_turn_id,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
last_used_at=None,
|
||||
embedding_id=memory_id,
|
||||
checksum=checksum,
|
||||
)
|
||||
|
||||
|
||||
def _record_from_row(row: sqlite3.Row) -> MemoryRecord:
|
||||
return MemoryRecord(
|
||||
id=str(row["id"]),
|
||||
type=row["type"],
|
||||
text=str(row["text"]),
|
||||
metadata=json.loads(str(row["metadata_json"])),
|
||||
sensitivity=row["sensitivity"],
|
||||
source_turn_id=row["source_turn_id"],
|
||||
created_at=float(row["created_at"]),
|
||||
updated_at=float(row["updated_at"]),
|
||||
last_used_at=float(row["last_used_at"]) if row["last_used_at"] is not None else None,
|
||||
embedding_id=str(row["embedding_id"]),
|
||||
checksum=str(row["checksum"]),
|
||||
)
|
||||
|
||||
|
||||
def _checksum(memory_type: str, text: str, metadata: dict[str, object]) -> str:
|
||||
payload = json.dumps(
|
||||
{"type": memory_type, "text": text, "metadata": metadata},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _tokenize(text: str) -> tuple[str, ...]:
|
||||
return tuple(token for token in re.split(r"\W+", text.lower()) if token)
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Protocol
|
||||
|
||||
@@ -124,6 +125,7 @@ class TurnController:
|
||||
self._pending_capture_frames: list[AudioFrame] = []
|
||||
self._cached_ack_text: str | None = None
|
||||
self._cached_ack_segment: AudioSegment | None = None
|
||||
self._barge_in_reply_guard_used = True
|
||||
self._barge_in_gate = BargeInSpeakerGate(
|
||||
enabled=config.barge_in_speaker_gate_enabled,
|
||||
user_similarity_threshold=config.barge_in_user_similarity_threshold,
|
||||
@@ -147,6 +149,26 @@ class TurnController:
|
||||
except ProviderError as exc:
|
||||
return self._recover(exc, turn_id)
|
||||
|
||||
def run_agent_turn(self, turn_id: int) -> TurnResult:
|
||||
self._states = []
|
||||
try:
|
||||
self._event(
|
||||
WAKE_LISTENING,
|
||||
PipelineState.LISTENING,
|
||||
"监听中：请直接说话",
|
||||
turn_id=turn_id,
|
||||
)
|
||||
user_text = self._capture_and_transcribe(
|
||||
turn_id,
|
||||
state_message="录音中：正在听取问题",
|
||||
no_speech_timeout_ms=60 * 60 * 1000,
|
||||
)
|
||||
if isinstance(user_text, ProviderError):
|
||||
return self._recover(user_text, turn_id)
|
||||
return self._reply_to_user(user_text, turn_id)
|
||||
except ProviderError as exc:
|
||||
return self._recover(exc, turn_id)
|
||||
|
||||
def prepare_ack_audio(self) -> None:
|
||||
text = self.config.wake_ack_text.strip()
|
||||
if not text:
|
||||
@@ -224,10 +246,23 @@ class TurnController:
|
||||
self.audio_preprocessor.reset()
|
||||
realtime_session = self._start_realtime_transcript()
|
||||
last_partial_ms: int | None = None
|
||||
capture_started_at = time.monotonic()
|
||||
self._event(CAPTURE_STARTED, PipelineState.RECORDING, state_message, turn_id=turn_id)
|
||||
while True:
|
||||
frames = self._read_capture_frames(timeout_ms=100)
|
||||
if not frames:
|
||||
if (
|
||||
no_speech_timeout_ms is not None
|
||||
and not self.vad_recorder.started
|
||||
and (time.monotonic() - capture_started_at) * 1000 >= no_speech_timeout_ms
|
||||
):
|
||||
return ProviderError(
|
||||
ErrorCode.VAD_TIMEOUT_NO_SPEECH,
|
||||
"no speech detected before timeout",
|
||||
True,
|
||||
"voice-assistant-pipeline",
|
||||
"vad",
|
||||
)
|
||||
continue
|
||||
for frame in frames:
|
||||
try:
|
||||
@@ -382,6 +417,7 @@ class TurnController:
|
||||
assistant_text = ""
|
||||
spoken_parts: list[str] = []
|
||||
interrupted = False
|
||||
self._barge_in_reply_guard_used = False
|
||||
try:
|
||||
for delta in self.llm.stream_reply(self.context.build_llm_messages()):
|
||||
assistant_text += delta.text_delta
|
||||
@@ -493,7 +529,13 @@ class TurnController:
|
||||
def _end_conversation(self, turn_id: int, *, payload: dict[str, object] | None = None) -> None:
|
||||
self._event(CONTINUOUS_SESSION_ENDED, PipelineState.WAKE_LISTENING, "", turn_id=turn_id, payload=payload)
|
||||
self._play_end_chime()
|
||||
self._event(STANDBY_RESUMED, PipelineState.WAKE_LISTENING, "恢复待机：可继续唤醒", turn_id=turn_id)
|
||||
message = (
|
||||
"恢复监听：可直接说话"
|
||||
if self.config.assistant_mode == "full_duplex_agent"
|
||||
else "恢复待机：可继续唤醒"
|
||||
)
|
||||
state = PipelineState.LISTENING if self.config.assistant_mode == "full_duplex_agent" else PipelineState.WAKE_LISTENING
|
||||
self._event(STANDBY_RESUMED, state, message, turn_id=turn_id)
|
||||
|
||||
def _play_end_chime(self) -> None:
|
||||
if not self.config.end_chime_enabled:
|
||||
@@ -517,37 +559,76 @@ class TurnController:
|
||||
self._event(TTS_STARTED, PipelineState.SPEAKING, "播放中：正在播报回复", turn_id=turn_id)
|
||||
segment = self.tts.synthesize(spoken_sentence)
|
||||
interruptible_segment = ensure_interruptible_pcm(segment)
|
||||
if not self._can_interrupt_playback(interruptible_segment):
|
||||
echo_guard_ms = self._effective_barge_in_echo_guard_ms(interruptible_segment)
|
||||
min_speech_ms = self._effective_barge_in_min_speech_ms()
|
||||
if not self._can_interrupt_playback(interruptible_segment, min_speech_ms=min_speech_ms):
|
||||
self._barge_in_debug(
|
||||
turn_id,
|
||||
"not_interruptible",
|
||||
{
|
||||
"segment_ready": interruptible_segment is not None,
|
||||
"barge_in_enabled": self.config.barge_in_enabled,
|
||||
"segment_duration_ms": interruptible_segment.duration_ms if interruptible_segment else None,
|
||||
"min_speech_ms": min_speech_ms,
|
||||
},
|
||||
)
|
||||
playback = self.transport.play_pcm(segment)
|
||||
if playback.error:
|
||||
raise playback.error
|
||||
self._event(PLAYBACK_FINISHED, PipelineState.SPEAKING, "", turn_id=turn_id)
|
||||
self._drain_input_after_playback()
|
||||
return SpeakResult(spoken_sentence)
|
||||
if self._play_interruptible(interruptible_segment, turn_id=turn_id):
|
||||
if self._play_interruptible(
|
||||
interruptible_segment,
|
||||
turn_id=turn_id,
|
||||
echo_guard_ms=echo_guard_ms,
|
||||
min_speech_ms=min_speech_ms,
|
||||
):
|
||||
return SpeakResult("", interrupted=True)
|
||||
self._event(PLAYBACK_FINISHED, PipelineState.SPEAKING, "", turn_id=turn_id)
|
||||
self._drain_input_after_playback()
|
||||
return SpeakResult(spoken_sentence)
|
||||
|
||||
def _can_interrupt_playback(self, segment: AudioSegment | None) -> bool:
|
||||
def _can_interrupt_playback(self, segment: AudioSegment | None, *, min_speech_ms: int) -> bool:
|
||||
return (
|
||||
segment is not None
|
||||
and self.config.barge_in_enabled
|
||||
and self.realtime_stt is not None
|
||||
and segment.duration_ms > self.config.barge_in_echo_guard_ms
|
||||
and segment.duration_ms >= max(self.config.barge_in_chunk_ms, min_speech_ms)
|
||||
)
|
||||
|
||||
def _play_interruptible(self, segment: AudioSegment, *, turn_id: int) -> bool:
|
||||
def _play_interruptible(
|
||||
self,
|
||||
segment: AudioSegment,
|
||||
*,
|
||||
turn_id: int,
|
||||
echo_guard_ms: int,
|
||||
min_speech_ms: int,
|
||||
) -> bool:
|
||||
self._barge_in_debug(
|
||||
turn_id,
|
||||
"monitor_starting",
|
||||
{
|
||||
"segment_duration_ms": segment.duration_ms,
|
||||
"echo_guard_ms": echo_guard_ms,
|
||||
"min_speech_ms": min_speech_ms,
|
||||
"chunk_ms": self.config.barge_in_chunk_ms,
|
||||
},
|
||||
)
|
||||
monitor = AsyncBargeInMonitor(
|
||||
transport=self.transport,
|
||||
vad_provider=self.vad_recorder.provider,
|
||||
realtime_stt=self.realtime_stt,
|
||||
speaker_gate=self._barge_in_gate,
|
||||
assistant_profile=self._barge_in_gate.assistant_profile(segment),
|
||||
echo_guard_ms=self.config.barge_in_echo_guard_ms,
|
||||
min_speech_ms=self.config.barge_in_min_speech_ms,
|
||||
assistant_reference=segment,
|
||||
echo_guard_ms=echo_guard_ms,
|
||||
min_speech_ms=min_speech_ms,
|
||||
listen_interval_ms=self.config.barge_in_listen_interval_ms,
|
||||
debug_callback=(
|
||||
lambda reason, payload: self._barge_in_debug(turn_id, reason, payload)
|
||||
if self.config.barge_in_debug
|
||||
else None
|
||||
),
|
||||
)
|
||||
monitor.start()
|
||||
playback = self.transport.play_pcm_chunks(
|
||||
@@ -565,6 +646,23 @@ class TurnController:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _effective_barge_in_echo_guard_ms(self, segment: AudioSegment | None) -> int:
|
||||
if segment is None or self._barge_in_reply_guard_used:
|
||||
return 0
|
||||
self._barge_in_reply_guard_used = True
|
||||
return max(0, min(self.config.barge_in_echo_guard_ms, 120))
|
||||
|
||||
def _effective_barge_in_min_speech_ms(self) -> int:
|
||||
latency_budget = max(60, self.config.interrupt_target_latency_ms - self.config.barge_in_chunk_ms)
|
||||
return max(40, min(self.config.barge_in_min_speech_ms, latency_budget))
|
||||
|
||||
def _barge_in_debug(self, turn_id: int, reason: str, payload: dict[str, object]) -> None:
|
||||
if not self.config.barge_in_debug:
|
||||
return
|
||||
details = " ".join(f"{key}={value}" for key, value in sorted(payload.items()))
|
||||
message = f"打断诊断：{reason}" + (f" {details}" if details else "")
|
||||
self._event("barge_in_debug", PipelineState.SPEAKING, message, turn_id=turn_id, payload=payload)
|
||||
|
||||
def _drain_input_after_playback(self) -> None:
|
||||
self.transport.flush_input()
|
||||
if self.config.post_playback_drain_ms <= 0:
|
||||
@@ -697,9 +795,44 @@ class VoiceAssistantPipeline:
|
||||
self.shutdown()
|
||||
return RuntimeSummary(completed, failed, last_error=last_error)
|
||||
|
||||
def run_agent(self, *, once: bool = False, max_turns: int | None = None) -> RuntimeSummary:
|
||||
completed = 0
|
||||
failed = 0
|
||||
last_error: ProviderError | None = None
|
||||
self.load()
|
||||
self.transport.start_input(
|
||||
device_id=self.config.audio_input_device,
|
||||
sample_rate=self.config.sample_rate,
|
||||
channels=self.config.channels,
|
||||
)
|
||||
try:
|
||||
while True:
|
||||
turn_id = completed + failed + 1
|
||||
result = self.run_agent_turn(turn_id)
|
||||
completed += result.completed_turns
|
||||
if result.success:
|
||||
completed += 0 if result.completed_turns else 1
|
||||
else:
|
||||
failed += result.failed_turns or 1
|
||||
last_error = result.error
|
||||
if once:
|
||||
break
|
||||
if once and completed >= 1:
|
||||
break
|
||||
if max_turns is not None and completed >= max_turns:
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
return RuntimeSummary(completed, failed, interrupted=True, last_error=last_error)
|
||||
finally:
|
||||
self.shutdown()
|
||||
return RuntimeSummary(completed, failed, last_error=last_error)
|
||||
|
||||
def run_turn(self, turn_id: int) -> TurnResult:
|
||||
return self.controller.run_turn(turn_id)
|
||||
|
||||
def run_agent_turn(self, turn_id: int) -> TurnResult:
|
||||
return self.controller.run_agent_turn(turn_id)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.transport.stop()
|
||||
|
||||
|
||||
+138
-21
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import math
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
@@ -8,11 +9,10 @@ import time
|
||||
import wave
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, Callable
|
||||
|
||||
from .models import AudioFrame, AudioSegment, ProviderError
|
||||
from .protocols import AudioTransport, RealtimeSttProvider, RealtimeTranscriptSession
|
||||
from .stt import is_valid_transcript_text
|
||||
from .protocols import AudioTransport, RealtimeSttProvider
|
||||
from .vad import cosine_similarity, extract_timbre_vector
|
||||
|
||||
|
||||
@@ -52,17 +52,44 @@ class BargeInSpeakerGate:
|
||||
def assistant_profile(self, segment: AudioSegment) -> TimbreProfile:
|
||||
return self._profile_from_segment(segment)
|
||||
|
||||
def accepts_candidate(self, frame: AudioFrame, assistant_profile: TimbreProfile) -> bool:
|
||||
def accepts_candidate(
|
||||
self,
|
||||
frame: AudioFrame,
|
||||
assistant_profile: TimbreProfile,
|
||||
assistant_reference: AudioSegment | None = None,
|
||||
reference_elapsed_ms: int | None = None,
|
||||
) -> bool:
|
||||
accepted, _reason = self.evaluate_candidate(
|
||||
frame,
|
||||
assistant_profile,
|
||||
assistant_reference,
|
||||
reference_elapsed_ms=reference_elapsed_ms,
|
||||
)
|
||||
return accepted
|
||||
|
||||
def evaluate_candidate(
|
||||
self,
|
||||
frame: AudioFrame,
|
||||
assistant_profile: TimbreProfile,
|
||||
assistant_reference: AudioSegment | None = None,
|
||||
reference_elapsed_ms: int | None = None,
|
||||
) -> tuple[bool, str]:
|
||||
if not self.enabled:
|
||||
return True
|
||||
return True, "gate_disabled"
|
||||
candidate = self._profile_from_frame(frame)
|
||||
if self._user_profile.ready and self._matches(candidate, self._user_profile, self.user_similarity_threshold):
|
||||
return True, "user_profile_match"
|
||||
if assistant_reference is not None and _looks_like_render_echo(
|
||||
frame,
|
||||
assistant_reference,
|
||||
reference_elapsed_ms=reference_elapsed_ms,
|
||||
):
|
||||
return False, "render_echo"
|
||||
if not candidate.ready:
|
||||
return False
|
||||
return True, "candidate_profile_unready"
|
||||
if self._matches(candidate, assistant_profile, self.assistant_reject_threshold):
|
||||
return False
|
||||
if self._user_profile.ready:
|
||||
return self._matches(candidate, self._user_profile, self.user_similarity_threshold)
|
||||
return True
|
||||
return False, "assistant_profile_match"
|
||||
return True, "non_assistant"
|
||||
|
||||
def _profile_from_segment(self, segment: AudioSegment) -> TimbreProfile:
|
||||
frame = AudioFrame(
|
||||
@@ -99,18 +126,22 @@ class AsyncBargeInMonitor:
|
||||
realtime_stt: RealtimeSttProvider | None,
|
||||
speaker_gate: BargeInSpeakerGate,
|
||||
assistant_profile: TimbreProfile,
|
||||
assistant_reference: AudioSegment | None = None,
|
||||
echo_guard_ms: int,
|
||||
min_speech_ms: int,
|
||||
listen_interval_ms: int,
|
||||
debug_callback: Callable[[str, dict[str, Any]], None] | None = None,
|
||||
) -> None:
|
||||
self.transport = transport
|
||||
self.vad_provider = vad_provider
|
||||
self.realtime_stt = realtime_stt
|
||||
self.speaker_gate = speaker_gate
|
||||
self.assistant_profile = assistant_profile
|
||||
self.assistant_reference = assistant_reference
|
||||
self.echo_guard_ms = max(0, echo_guard_ms)
|
||||
self.min_speech_ms = max(0, min_speech_ms)
|
||||
self.listen_interval_ms = max(1, listen_interval_ms)
|
||||
self.debug_callback = debug_callback
|
||||
self.stop_event = threading.Event()
|
||||
self._shutdown_event = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
@@ -121,6 +152,14 @@ class AsyncBargeInMonitor:
|
||||
|
||||
def start(self) -> None:
|
||||
self.vad_provider.reset()
|
||||
self._debug(
|
||||
"started",
|
||||
{
|
||||
"echo_guard_ms": self.echo_guard_ms,
|
||||
"min_speech_ms": self.min_speech_ms,
|
||||
"listen_interval_ms": self.listen_interval_ms,
|
||||
},
|
||||
)
|
||||
self._thread = threading.Thread(target=self._run, name="owner-voice-barge-in", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
@@ -141,11 +180,11 @@ class AsyncBargeInMonitor:
|
||||
return list(self._pending_frames)
|
||||
|
||||
def _run(self) -> None:
|
||||
realtime_session = self.realtime_stt.start_stream() if self.realtime_stt is not None else None
|
||||
started_at = time.monotonic()
|
||||
speech_ms = 0
|
||||
partial_seen = False
|
||||
candidate_frames: list[AudioFrame] = []
|
||||
saw_frames = False
|
||||
last_debug_at = 0.0
|
||||
try:
|
||||
while not self._shutdown_event.is_set() and not self.stop_event.is_set():
|
||||
frames = self.transport.read_frames(timeout_ms=self.listen_interval_ms)
|
||||
@@ -155,34 +194,112 @@ class AsyncBargeInMonitor:
|
||||
if not frames:
|
||||
time.sleep(self.listen_interval_ms / 1000)
|
||||
continue
|
||||
if not saw_frames:
|
||||
saw_frames = True
|
||||
self._debug("frames_seen", {"elapsed_ms": elapsed_ms, "frame_count": len(frames)})
|
||||
for frame in frames:
|
||||
result = self.vad_provider.analyze(frame)
|
||||
if not result.is_speech:
|
||||
speech_ms = 0
|
||||
candidate_frames = []
|
||||
continue
|
||||
if not self.speaker_gate.accepts_candidate(frame, self.assistant_profile):
|
||||
accepted, reason = self.speaker_gate.evaluate_candidate(
|
||||
frame,
|
||||
self.assistant_profile,
|
||||
self.assistant_reference,
|
||||
reference_elapsed_ms=elapsed_ms,
|
||||
)
|
||||
if not accepted:
|
||||
now = time.monotonic()
|
||||
if now - last_debug_at >= 0.25:
|
||||
self._debug("gate_rejected", {"elapsed_ms": elapsed_ms, "reason": reason})
|
||||
last_debug_at = now
|
||||
speech_ms = 0
|
||||
candidate_frames = []
|
||||
continue
|
||||
frame_ms = int(frame.metadata.get("duration_ms", 20))
|
||||
speech_ms += frame_ms
|
||||
candidate_frames.append(frame)
|
||||
if realtime_session is not None:
|
||||
transcript = realtime_session.accept_frame(frame)
|
||||
if transcript is not None and is_valid_transcript_text(transcript.normalized_text):
|
||||
partial_seen = True
|
||||
if speech_ms >= self.min_speech_ms and partial_seen:
|
||||
now = time.monotonic()
|
||||
if now - last_debug_at >= 0.25:
|
||||
self._debug(
|
||||
"speech_accumulating",
|
||||
{"elapsed_ms": elapsed_ms, "speech_ms": speech_ms, "reason": reason},
|
||||
)
|
||||
last_debug_at = now
|
||||
if speech_ms >= self.min_speech_ms:
|
||||
with self._lock:
|
||||
self._pending_frames = list(candidate_frames)
|
||||
self._interrupted = True
|
||||
self._debug("triggered", {"elapsed_ms": elapsed_ms, "speech_ms": speech_ms})
|
||||
self.stop_event.set()
|
||||
return
|
||||
except ProviderError as exc:
|
||||
self.error = exc
|
||||
finally:
|
||||
if realtime_session is not None:
|
||||
realtime_session.finish()
|
||||
self._debug("error", {"message": exc.message, "code": exc.code.value})
|
||||
|
||||
def _debug(self, reason: str, payload: dict[str, Any]) -> None:
|
||||
if self.debug_callback is not None:
|
||||
self.debug_callback(reason, payload)
|
||||
|
||||
|
||||
def _looks_like_render_echo(
|
||||
frame: AudioFrame,
|
||||
reference: AudioSegment,
|
||||
*,
|
||||
reference_elapsed_ms: int | None = None,
|
||||
) -> bool:
|
||||
if frame.metadata.get("speaker_id") == "assistant":
|
||||
return True
|
||||
if not frame.pcm or not reference.pcm:
|
||||
return False
|
||||
try:
|
||||
import numpy as np
|
||||
|
||||
candidate = np.frombuffer(frame.pcm, dtype=np.int16).astype(np.float32)
|
||||
render = np.frombuffer(reference.pcm, dtype=np.int16).astype(np.float32)
|
||||
if candidate.size == 0 or render.size == 0:
|
||||
return False
|
||||
if frame.channels > 1:
|
||||
candidate = candidate.reshape(-1, frame.channels).mean(axis=1)
|
||||
if reference.channels > 1:
|
||||
render = render.reshape(-1, reference.channels).mean(axis=1)
|
||||
candidate_rms = float(math.sqrt(float(np.mean(candidate * candidate)))) if candidate.size else 0.0
|
||||
render_rms = float(math.sqrt(float(np.mean(render * render)))) if render.size else 0.0
|
||||
if candidate_rms <= 1.0 or render_rms <= 1.0:
|
||||
return False
|
||||
width = min(candidate.size, render.size)
|
||||
candidate_window = candidate[-width:]
|
||||
render_window = _reference_window(render, width, reference.sample_rate, reference_elapsed_ms)
|
||||
corr = _normalized_corr(candidate_window, render_window)
|
||||
if corr >= 0.97:
|
||||
return True
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _reference_window(render: Any, width: int, sample_rate: int, elapsed_ms: int | None) -> Any:
|
||||
if elapsed_ms is None:
|
||||
return render[:width]
|
||||
center = int(max(0, elapsed_ms) * max(1, sample_rate) / 1000)
|
||||
start = max(0, min(max(0, render.size - width), center - width))
|
||||
return render[start : start + width]
|
||||
|
||||
|
||||
def _normalized_corr(left: Any, right: Any) -> float:
|
||||
try:
|
||||
import numpy as np
|
||||
|
||||
left = left - np.mean(left)
|
||||
right = right - np.mean(right)
|
||||
numerator = float(np.dot(left, right))
|
||||
denominator = float(np.linalg.norm(left) * np.linalg.norm(right))
|
||||
if denominator <= 1e-9:
|
||||
return 0.0
|
||||
return numerator / denominator
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
def ensure_interruptible_pcm(segment: AudioSegment) -> AudioSegment | None:
|
||||
|
||||
@@ -11,12 +11,15 @@ from .audio_preprocess import SherpaOnnxDenoiserPreprocessor
|
||||
from .assets import validate_pet_assets
|
||||
from .config import AppConfig
|
||||
from .conversation import ConversationContext
|
||||
from .full_duplex_audio import build_audio_processing_provider
|
||||
from .full_duplex_runtime import FullDuplexAgentRuntime
|
||||
from .llm import MockLlmProvider, OpenAICompatibleLlmProvider
|
||||
from .models import AudioFrame, ProviderError
|
||||
from .pipeline import VoicePipeline
|
||||
from .real_live_check import run_real_live_check
|
||||
from .runtime import build_live_runtime
|
||||
from .simulation import run_simulated_live
|
||||
from .self_tests import run_agent_self_test, run_audio_self_test
|
||||
from .speech_models import check_speech_models, model_status_errors
|
||||
from .stt import MetadataSttProvider, SherpaOnnxSttProvider
|
||||
from .transport import MemoryAudioTransport, sounddevice_device_report
|
||||
@@ -38,6 +41,12 @@ def main(argv: list[str] | None = None) -> int:
|
||||
subparsers.add_parser("device-check", help="Validate local microphone and speaker availability")
|
||||
live = subparsers.add_parser("run-live", help="Run real repeated live voice conversation")
|
||||
live.add_argument("--once", action="store_true", help="Run one completed live turn and exit")
|
||||
agent_live = subparsers.add_parser(
|
||||
"run-agent-live",
|
||||
help="Run planned full-duplex Agent voice assistant entry point",
|
||||
)
|
||||
agent_live.add_argument("--check-config", action="store_true", help="Validate and print full-duplex Agent config")
|
||||
agent_live.add_argument("--once", action="store_true", help="Run one completed full-duplex Agent turn and exit")
|
||||
simulate = subparsers.add_parser("simulate-live", help="Run live pipeline with simulated microphone frames")
|
||||
simulate.add_argument("--turns", type=int, default=2, help="Number of simulated turns. Default: 2")
|
||||
simulate.add_argument("--fixture", default=None, help="Replay simulated microphone frames from JSONL")
|
||||
@@ -48,6 +57,12 @@ def main(argv: list[str] | None = None) -> int:
|
||||
real_check.add_argument("--wake-text", default="小杰小杰。", help="Generated wake utterance")
|
||||
real_check.add_argument("--question", action="append", default=None, help="Generated user question; can be repeated")
|
||||
real_check.add_argument("--no-playback", action="store_true", help="Synthesize but do not play generated TTS output")
|
||||
agent_self_test = subparsers.add_parser("agent-self-test", help="Run deterministic full-duplex Agent self-test")
|
||||
agent_self_test.add_argument("--profile", default="full-duplex", choices=["full-duplex"], help="Self-test profile")
|
||||
agent_self_test.add_argument("--turns", type=int, default=3, help="Number of simulated turns. Default: 3")
|
||||
audio_self_test = subparsers.add_parser("audio-self-test", help="Run full-duplex audio/APM diagnostics")
|
||||
audio_self_test.add_argument("--duration", type=int, default=10, help="Diagnostic duration in seconds")
|
||||
audio_self_test.add_argument("--check-echo", action="store_true", help="Check assistant echo suppression path")
|
||||
smoke = subparsers.add_parser("llm-smoke", help="Call configured OpenAI/NewAPI endpoint")
|
||||
smoke.add_argument("--message", default="用一句中文回复：小杰在线。")
|
||||
smoke.add_argument("--no-stream", action="store_true")
|
||||
@@ -58,6 +73,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"assistant_mode": config.assistant_mode,
|
||||
"wake_word": config.wake_word,
|
||||
"sample_rate": config.sample_rate,
|
||||
"channels": config.channels,
|
||||
@@ -113,6 +129,29 @@ def main(argv: list[str] | None = None) -> int:
|
||||
"end_chime_file": str(config.end_chime_file),
|
||||
"end_chime_frequency_hz": config.end_chime_frequency_hz,
|
||||
"end_chime_duration_ms": config.end_chime_duration_ms,
|
||||
"audio_apm_provider": config.audio_apm_provider,
|
||||
"audio_aec_enabled": config.audio_aec_enabled,
|
||||
"audio_ns_enabled": config.audio_ns_enabled,
|
||||
"audio_agc_enabled": config.audio_agc_enabled,
|
||||
"audio_apm_required": config.audio_apm_required,
|
||||
"audio_frame_ms": config.audio_frame_ms,
|
||||
"audio_ring_buffer_ms": config.audio_ring_buffer_ms,
|
||||
"interrupt_enabled": config.interrupt_enabled,
|
||||
"interrupt_target_latency_ms": config.interrupt_target_latency_ms,
|
||||
"streaming_stt_provider": config.streaming_stt_provider,
|
||||
"streaming_stt_product_candidate": config.streaming_stt_product_candidate,
|
||||
"streaming_tts_provider": config.streaming_tts_provider,
|
||||
"memory_enabled": config.memory_enabled,
|
||||
"memory_provider": config.memory_provider,
|
||||
"memory_top_k": config.memory_top_k,
|
||||
"memory_auto_save_sensitive": config.memory_auto_save_sensitive,
|
||||
"tool_router_enabled": config.tool_router_enabled,
|
||||
"tool_max_calls_per_turn": config.tool_max_calls_per_turn,
|
||||
"tool_timeout_ms": config.tool_timeout_ms,
|
||||
"openinterpreter_enabled": config.openinterpreter_enabled,
|
||||
"openinterpreter_command": config.openinterpreter_command,
|
||||
"browser_playwright_enabled": config.browser_playwright_enabled,
|
||||
"computer_control_enabled": config.computer_control_enabled,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
@@ -171,6 +210,69 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return 1
|
||||
return 0 if summary.completed_turns > 0 or summary.interrupted else 1
|
||||
|
||||
if args.command == "run-agent-live":
|
||||
config = replace(AppConfig.from_dotenv(args.env_file), assistant_mode="full_duplex_agent")
|
||||
errors = config.validate_basic()
|
||||
if errors:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"success": False,
|
||||
"command": "run-agent-live",
|
||||
"errors": [str(error) for error in errors],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 1
|
||||
apm_error: ProviderError | None = None
|
||||
apm_health = None
|
||||
try:
|
||||
apm_health = build_audio_processing_provider(config).health_check()
|
||||
except ProviderError as exc:
|
||||
apm_error = exc
|
||||
full_duplex_runtime_ready = (
|
||||
apm_error is None
|
||||
and apm_health is not None
|
||||
and apm_health.available
|
||||
and (
|
||||
config.audio_apm_provider == "fake"
|
||||
or (config.audio_apm_provider == "webrtc" and not apm_health.fallback_active)
|
||||
)
|
||||
)
|
||||
data = {
|
||||
"success": bool(args.check_config),
|
||||
"command": "run-agent-live",
|
||||
"assistant_mode": config.assistant_mode,
|
||||
"audio_apm_provider": config.audio_apm_provider,
|
||||
"audio_apm_available": bool(apm_health and apm_health.available),
|
||||
"audio_apm_fallback_active": bool(apm_health and apm_health.fallback_active),
|
||||
"audio_apm_message": apm_health.message if apm_health else "",
|
||||
"audio_apm_error_code": apm_error.code.value if apm_error else "",
|
||||
"audio_apm_error_message": apm_error.message if apm_error else "",
|
||||
"streaming_stt_provider": config.streaming_stt_provider,
|
||||
"streaming_tts_provider": config.streaming_tts_provider,
|
||||
"llm_streaming_enabled": config.llm_stream,
|
||||
"memory_provider": config.memory_provider,
|
||||
"memory_enabled": config.memory_enabled,
|
||||
"tool_router_enabled": config.tool_router_enabled,
|
||||
"openinterpreter_enabled": config.openinterpreter_enabled,
|
||||
"browser_playwright_enabled": config.browser_playwright_enabled,
|
||||
"computer_control_enabled": config.computer_control_enabled,
|
||||
"turn_based_entry": "run-live",
|
||||
"full_duplex_runtime_ready": full_duplex_runtime_ready,
|
||||
}
|
||||
if args.check_config:
|
||||
print(json.dumps(data, ensure_ascii=False, sort_keys=True))
|
||||
return 0
|
||||
try:
|
||||
summary = FullDuplexAgentRuntime(config=config).run(once=args.once)
|
||||
except ProviderError as exc:
|
||||
print(json.dumps({"ok": False, "code": exc.code.value, "message": exc.message}, ensure_ascii=False, sort_keys=True))
|
||||
return 1
|
||||
return 0 if summary.completed_turns > 0 or summary.interrupted else 1
|
||||
|
||||
if args.command == "simulate-live":
|
||||
try:
|
||||
data = run_simulated_live(
|
||||
@@ -201,6 +303,21 @@ def main(argv: list[str] | None = None) -> int:
|
||||
print(json.dumps(data, ensure_ascii=False, sort_keys=True))
|
||||
return 0 if data["success"] else 1
|
||||
|
||||
if args.command == "agent-self-test":
|
||||
data = run_agent_self_test(profile=args.profile, turns=args.turns)
|
||||
print(json.dumps(data, ensure_ascii=False, sort_keys=True))
|
||||
return 0 if data["success"] else 1
|
||||
|
||||
if args.command == "audio-self-test":
|
||||
config = AppConfig.from_dotenv(args.env_file)
|
||||
data = run_audio_self_test(
|
||||
config=config,
|
||||
duration_s=args.duration,
|
||||
check_echo=args.check_echo,
|
||||
)
|
||||
print(json.dumps(data, ensure_ascii=False, sort_keys=True))
|
||||
return 0 if data["success"] else 1
|
||||
|
||||
if args.command == "acceptance":
|
||||
result = run_acceptance()
|
||||
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
||||
|
||||
@@ -8,6 +8,7 @@ from .models import ErrorCode, ProviderError
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AppConfig:
|
||||
assistant_mode: str = "turn_based_voice_pet"
|
||||
wake_word: str = "小杰小杰"
|
||||
sample_rate: int = 16000
|
||||
channels: int = 1
|
||||
@@ -64,10 +65,34 @@ class AppConfig:
|
||||
barge_in_assistant_reject_threshold: float = 0.72
|
||||
barge_in_listen_interval_ms: int = 20
|
||||
barge_in_chunk_ms: int = 30
|
||||
barge_in_debug: bool = False
|
||||
end_chime_enabled: bool = True
|
||||
end_chime_file: Path = Path("assets/sounds/codex-notification.wav")
|
||||
end_chime_frequency_hz: int = 880
|
||||
end_chime_duration_ms: int = 140
|
||||
audio_apm_provider: str = "webrtc"
|
||||
audio_aec_enabled: bool = True
|
||||
audio_ns_enabled: bool = True
|
||||
audio_agc_enabled: bool = True
|
||||
audio_apm_required: bool = True
|
||||
audio_frame_ms: int = 20
|
||||
audio_ring_buffer_ms: int = 3000
|
||||
interrupt_enabled: bool = True
|
||||
interrupt_target_latency_ms: int = 200
|
||||
streaming_stt_provider: str = "faster_whisper"
|
||||
streaming_stt_product_candidate: str = "sensevoice"
|
||||
streaming_tts_provider: str = "cosyvoice"
|
||||
memory_enabled: bool = True
|
||||
memory_provider: str = "faiss_sqlite"
|
||||
memory_top_k: int = 5
|
||||
memory_auto_save_sensitive: bool = False
|
||||
tool_router_enabled: bool = True
|
||||
tool_max_calls_per_turn: int = 5
|
||||
tool_timeout_ms: int = 30000
|
||||
openinterpreter_enabled: bool = False
|
||||
openinterpreter_command: str = "openinterpreter"
|
||||
browser_playwright_enabled: bool = False
|
||||
computer_control_enabled: bool = False
|
||||
|
||||
@classmethod
|
||||
def from_dotenv(cls, path: str | Path = ".env", prefix: str = "OWNER_") -> "AppConfig":
|
||||
@@ -77,7 +102,19 @@ class AppConfig:
|
||||
value = values.get(f"{prefix}{name}")
|
||||
return default if value is None or value == "" else value
|
||||
|
||||
def get_bool(name: str, default: str = "0") -> bool:
|
||||
return (get(name, default) or default).lower() not in {"0", "false", "no"}
|
||||
|
||||
def get_bool_compat(name: str, legacy_name: str, default: str = "0") -> bool:
|
||||
value = get(name)
|
||||
if value is None:
|
||||
value = get(legacy_name, default)
|
||||
return (value or default).lower() not in {"0", "false", "no"}
|
||||
|
||||
return cls(
|
||||
assistant_mode=(
|
||||
get("ASSISTANT_MODE", "turn_based_voice_pet") or "turn_based_voice_pet"
|
||||
).lower(),
|
||||
wake_word=get("WAKE_WORD", "小杰小杰") or "小杰小杰",
|
||||
sample_rate=int(get("SAMPLE_RATE", "16000") or "16000"),
|
||||
channels=int(get("CHANNELS", "1") or "1"),
|
||||
@@ -85,7 +122,7 @@ class AppConfig:
|
||||
llm_api_key=get("LLM_API_KEY"),
|
||||
llm_model=get("LLM_MODEL", "mimo-v2.5") or "mimo-v2.5",
|
||||
llm_api_style=get("LLM_API_STYLE", "chat_completions") or "chat_completions",
|
||||
llm_stream=(get("LLM_STREAM", "1") or "1").lower() not in {"0", "false", "no"},
|
||||
llm_stream=get_bool_compat("LLM_STREAMING_ENABLED", "LLM_STREAM", "1"),
|
||||
realtime_transcript_enabled=(get("REALTIME_TRANSCRIPT_ENABLED", "1") or "1").lower()
|
||||
not in {"0", "false", "no"},
|
||||
realtime_transcript_idle_timeout_ms=int(
|
||||
@@ -149,6 +186,7 @@ class AppConfig:
|
||||
),
|
||||
barge_in_listen_interval_ms=int(get("BARGE_IN_LISTEN_INTERVAL_MS", "20") or "20"),
|
||||
barge_in_chunk_ms=int(get("BARGE_IN_CHUNK_MS", "30") or "30"),
|
||||
barge_in_debug=(get("BARGE_IN_DEBUG", "0") or "0").lower() not in {"0", "false", "no"},
|
||||
end_chime_enabled=(get("END_CHIME_ENABLED", "1") or "1").lower() not in {"0", "false", "no"},
|
||||
end_chime_file=Path(
|
||||
get("END_CHIME_FILE", "assets/sounds/codex-notification.wav")
|
||||
@@ -156,6 +194,37 @@ class AppConfig:
|
||||
),
|
||||
end_chime_frequency_hz=int(get("END_CHIME_FREQUENCY_HZ", "880") or "880"),
|
||||
end_chime_duration_ms=int(get("END_CHIME_DURATION_MS", "140") or "140"),
|
||||
audio_apm_provider=(get("AUDIO_APM_PROVIDER", "webrtc") or "webrtc").lower(),
|
||||
audio_aec_enabled=get_bool("AUDIO_AEC_ENABLED", "1"),
|
||||
audio_ns_enabled=get_bool("AUDIO_NS_ENABLED", "1"),
|
||||
audio_agc_enabled=get_bool("AUDIO_AGC_ENABLED", "1"),
|
||||
audio_apm_required=get_bool("AUDIO_APM_REQUIRED", "1"),
|
||||
audio_frame_ms=int(get("AUDIO_FRAME_MS", "20") or "20"),
|
||||
audio_ring_buffer_ms=int(get("AUDIO_RING_BUFFER_MS", "3000") or "3000"),
|
||||
interrupt_enabled=get_bool("INTERRUPT_ENABLED", "1"),
|
||||
interrupt_target_latency_ms=int(get("INTERRUPT_TARGET_LATENCY_MS", "200") or "200"),
|
||||
streaming_stt_provider=(
|
||||
get("STREAMING_STT_PROVIDER", "faster_whisper") or "faster_whisper"
|
||||
).lower(),
|
||||
streaming_stt_product_candidate=(
|
||||
get("STREAMING_STT_PRODUCT_CANDIDATE", "sensevoice") or "sensevoice"
|
||||
).lower(),
|
||||
streaming_tts_provider=(
|
||||
get("STREAMING_TTS_PROVIDER", "cosyvoice") or "cosyvoice"
|
||||
).lower(),
|
||||
memory_enabled=get_bool("MEMORY_ENABLED", "1"),
|
||||
memory_provider=(get("MEMORY_PROVIDER", "faiss_sqlite") or "faiss_sqlite").lower(),
|
||||
memory_top_k=int(get("MEMORY_TOP_K", "5") or "5"),
|
||||
memory_auto_save_sensitive=get_bool("MEMORY_AUTO_SAVE_SENSITIVE", "0"),
|
||||
tool_router_enabled=get_bool("TOOL_ROUTER_ENABLED", "1"),
|
||||
tool_max_calls_per_turn=int(get("TOOL_MAX_CALLS_PER_TURN", "5") or "5"),
|
||||
tool_timeout_ms=int(get("TOOL_TIMEOUT_MS", "30000") or "30000"),
|
||||
openinterpreter_enabled=get_bool("OPENINTERPRETER_ENABLED", "0"),
|
||||
openinterpreter_command=(
|
||||
get("OPENINTERPRETER_COMMAND", "openinterpreter") or "openinterpreter"
|
||||
),
|
||||
browser_playwright_enabled=get_bool("BROWSER_PLAYWRIGHT_ENABLED", "0"),
|
||||
computer_control_enabled=get_bool("COMPUTER_CONTROL_ENABLED", "0"),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -174,6 +243,16 @@ class AppConfig:
|
||||
|
||||
def validate_basic(self) -> list[ProviderError]:
|
||||
errors: list[ProviderError] = []
|
||||
if self.assistant_mode not in {"turn_based_voice_pet", "full_duplex_agent"}:
|
||||
errors.append(
|
||||
ProviderError(
|
||||
ErrorCode.CONFIG_MISSING_VALUE,
|
||||
"OWNER_ASSISTANT_MODE must be turn_based_voice_pet or full_duplex_agent",
|
||||
False,
|
||||
"config",
|
||||
"startup",
|
||||
)
|
||||
)
|
||||
if self.sample_rate <= 0:
|
||||
errors.append(
|
||||
ProviderError(
|
||||
@@ -451,6 +530,12 @@ class AppConfig:
|
||||
for name, value in {
|
||||
"OWNER_END_CHIME_FREQUENCY_HZ": self.end_chime_frequency_hz,
|
||||
"OWNER_END_CHIME_DURATION_MS": self.end_chime_duration_ms,
|
||||
"OWNER_AUDIO_FRAME_MS": self.audio_frame_ms,
|
||||
"OWNER_AUDIO_RING_BUFFER_MS": self.audio_ring_buffer_ms,
|
||||
"OWNER_INTERRUPT_TARGET_LATENCY_MS": self.interrupt_target_latency_ms,
|
||||
"OWNER_MEMORY_TOP_K": self.memory_top_k,
|
||||
"OWNER_TOOL_MAX_CALLS_PER_TURN": self.tool_max_calls_per_turn,
|
||||
"OWNER_TOOL_TIMEOUT_MS": self.tool_timeout_ms,
|
||||
}.items():
|
||||
if value <= 0:
|
||||
errors.append(
|
||||
@@ -462,6 +547,66 @@ class AppConfig:
|
||||
"startup",
|
||||
)
|
||||
)
|
||||
if self.audio_apm_provider not in {"webrtc", "fake", "disabled"}:
|
||||
errors.append(
|
||||
ProviderError(
|
||||
ErrorCode.CONFIG_MISSING_VALUE,
|
||||
"OWNER_AUDIO_APM_PROVIDER must be webrtc, fake, or disabled",
|
||||
False,
|
||||
"config",
|
||||
"startup",
|
||||
)
|
||||
)
|
||||
if self.streaming_stt_provider not in {"faster_whisper", "sensevoice", "sherpa_onnx"}:
|
||||
errors.append(
|
||||
ProviderError(
|
||||
ErrorCode.CONFIG_MISSING_VALUE,
|
||||
"OWNER_STREAMING_STT_PROVIDER must be faster_whisper, sensevoice, or sherpa_onnx",
|
||||
False,
|
||||
"config",
|
||||
"startup",
|
||||
)
|
||||
)
|
||||
if self.streaming_stt_product_candidate not in {"sensevoice", "faster_whisper", "sherpa_onnx"}:
|
||||
errors.append(
|
||||
ProviderError(
|
||||
ErrorCode.CONFIG_MISSING_VALUE,
|
||||
"OWNER_STREAMING_STT_PRODUCT_CANDIDATE must be sensevoice, faster_whisper, or sherpa_onnx",
|
||||
False,
|
||||
"config",
|
||||
"startup",
|
||||
)
|
||||
)
|
||||
if self.streaming_tts_provider not in {"cosyvoice", "macos_say"}:
|
||||
errors.append(
|
||||
ProviderError(
|
||||
ErrorCode.CONFIG_MISSING_VALUE,
|
||||
"OWNER_STREAMING_TTS_PROVIDER must be cosyvoice or macos_say",
|
||||
False,
|
||||
"config",
|
||||
"startup",
|
||||
)
|
||||
)
|
||||
if self.memory_provider not in {"faiss_sqlite"}:
|
||||
errors.append(
|
||||
ProviderError(
|
||||
ErrorCode.CONFIG_MISSING_VALUE,
|
||||
"OWNER_MEMORY_PROVIDER must be faiss_sqlite",
|
||||
False,
|
||||
"config",
|
||||
"startup",
|
||||
)
|
||||
)
|
||||
if self.openinterpreter_enabled and not self.openinterpreter_command.strip():
|
||||
errors.append(
|
||||
ProviderError(
|
||||
ErrorCode.CONFIG_MISSING_VALUE,
|
||||
"OWNER_OPENINTERPRETER_COMMAND must be non-empty when Open Interpreter is enabled",
|
||||
False,
|
||||
"config",
|
||||
"startup",
|
||||
)
|
||||
)
|
||||
return errors
|
||||
|
||||
def api_url(self, path: str) -> str:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable
|
||||
|
||||
@@ -30,15 +31,41 @@ PLAYBACK_INTERRUPTED = "playback_interrupted"
|
||||
CONTINUOUS_SESSION_ENDED = "continuous_session_ended"
|
||||
STAGE_ERROR = "stage_error"
|
||||
RECOVERING = "recovering"
|
||||
AUDIO_CAPTURE_STARTED = "audio_capture_started"
|
||||
AUDIO_APM_STARTED = "audio_apm_started"
|
||||
LISTENING_STARTED = "listening_started"
|
||||
INTERRUPT_DETECTED = "interrupt_detected"
|
||||
PLAYBACK_CANCELLED = "playback_cancelled"
|
||||
LLM_CANCELLED = "llm_cancelled"
|
||||
MEMORY_RETRIEVED = "memory_retrieved"
|
||||
TOOL_CALL_REQUESTED = "tool_call_requested"
|
||||
TOOL_CONFIRMATION_REQUIRED = "tool_confirmation_required"
|
||||
TOOL_CALL_STARTED = "tool_call_started"
|
||||
TOOL_CALL_FINISHED = "tool_call_finished"
|
||||
TOOL_CALL_REJECTED = "tool_call_rejected"
|
||||
SESSION_RECOVERED = "session_recovered"
|
||||
|
||||
_SENSITIVE_PAYLOAD_KEY_PARTS = (
|
||||
"api_key",
|
||||
"authorization",
|
||||
"password",
|
||||
"secret",
|
||||
"token",
|
||||
"raw_audio",
|
||||
"pcm",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PipelineEvent:
|
||||
type: str
|
||||
turn_id: int | None = None
|
||||
session_id: str | None = None
|
||||
stage: str | None = None
|
||||
state: PipelineState | None = None
|
||||
message: str = ""
|
||||
payload: dict[str, Any] = field(default_factory=dict)
|
||||
created_at: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
class PipelineEventBus:
|
||||
@@ -54,11 +81,21 @@ class PipelineEventBus:
|
||||
event_type: str,
|
||||
*,
|
||||
turn_id: int | None = None,
|
||||
session_id: str | None = None,
|
||||
stage: str | None = None,
|
||||
state: PipelineState | None = None,
|
||||
message: str = "",
|
||||
payload: dict[str, Any] | None = None,
|
||||
) -> PipelineEvent:
|
||||
event = PipelineEvent(event_type, turn_id=turn_id, state=state, message=message, payload=payload or {})
|
||||
event = PipelineEvent(
|
||||
event_type,
|
||||
turn_id=turn_id,
|
||||
session_id=session_id,
|
||||
stage=stage,
|
||||
state=state,
|
||||
message=message,
|
||||
payload=_sanitize_payload(payload or {}),
|
||||
)
|
||||
self.events.append(event)
|
||||
for listener in list(self._listeners):
|
||||
listener(event)
|
||||
@@ -87,3 +124,17 @@ def dispatch_pipeline_event(reporter: Any, event: PipelineEvent) -> None:
|
||||
return
|
||||
if event.message:
|
||||
reporter.status((event.state.value if event.state else event.type), event.message, turn_id=event.turn_id)
|
||||
|
||||
|
||||
def _sanitize_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
sanitized: dict[str, Any] = {}
|
||||
for key, value in payload.items():
|
||||
normalized = key.lower()
|
||||
if any(part in normalized for part in _SENSITIVE_PAYLOAD_KEY_PARTS):
|
||||
sanitized[key] = "[redacted]"
|
||||
continue
|
||||
if isinstance(value, dict):
|
||||
sanitized[key] = _sanitize_payload(value)
|
||||
continue
|
||||
sanitized[key] = value
|
||||
return sanitized
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import shutil
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from .config import AppConfig
|
||||
from .tool_router import ToolCallRequest, ToolContext, ToolResult
|
||||
|
||||
|
||||
CODEX_COMPUTER_USE_SAFETY_REFERENCE = (
|
||||
"Codex Computer Use is referenced only for safety-confirmation principles; "
|
||||
"Owner uses public macOS Accessibility, Playwright, or trycua-style providers in future changes."
|
||||
)
|
||||
|
||||
|
||||
def is_high_risk_task(text: str) -> bool:
|
||||
lowered = text.lower()
|
||||
keywords = (
|
||||
"delete",
|
||||
"upload",
|
||||
"payment",
|
||||
"purchase",
|
||||
"trade",
|
||||
"account",
|
||||
"chmod",
|
||||
"rm ",
|
||||
"删除",
|
||||
"上传",
|
||||
"支付",
|
||||
"购买",
|
||||
"交易",
|
||||
"账号",
|
||||
"权限",
|
||||
)
|
||||
return any(keyword in lowered for keyword in keywords)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class OpenInterpreterAdapter:
|
||||
enabled: bool = False
|
||||
command: str = "openinterpreter"
|
||||
dry_run: bool = True
|
||||
name: str = "openinterpreter.run"
|
||||
|
||||
def execute(self, request: ToolCallRequest, context: ToolContext) -> ToolResult:
|
||||
if not self.enabled:
|
||||
return ToolResult(
|
||||
request.id,
|
||||
"failed",
|
||||
error_code="OPENINTERPRETER_DISABLED",
|
||||
audit_summary="Open Interpreter adapter disabled",
|
||||
)
|
||||
executable = self._resolve_command()
|
||||
if executable is None:
|
||||
return ToolResult(
|
||||
request.id,
|
||||
"failed",
|
||||
error_code="OPENINTERPRETER_UNAVAILABLE",
|
||||
audit_summary="Open Interpreter command unavailable",
|
||||
)
|
||||
task = str(request.arguments.get("task", ""))
|
||||
if is_high_risk_task(task):
|
||||
return ToolResult(
|
||||
request.id,
|
||||
"confirmation_required",
|
||||
error_code="OPENINTERPRETER_HIGH_RISK",
|
||||
audit_summary="Open Interpreter high risk task requires confirmation",
|
||||
)
|
||||
if self.dry_run:
|
||||
return ToolResult(
|
||||
request.id,
|
||||
"success",
|
||||
f"dry-run Open Interpreter task: {task}",
|
||||
audit_summary="Open Interpreter dry run",
|
||||
)
|
||||
completed = subprocess.run(
|
||||
[executable, "exec", task],
|
||||
cwd=context.cwd,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=max(1, int(request.timeout_ms / 1000)),
|
||||
)
|
||||
return ToolResult(
|
||||
request.id,
|
||||
"success" if completed.returncode == 0 else "failed",
|
||||
completed.stdout,
|
||||
error_code=None if completed.returncode == 0 else "OPENINTERPRETER_FAILED",
|
||||
audit_summary=f"Open Interpreter exited {completed.returncode}",
|
||||
)
|
||||
|
||||
def _resolve_command(self) -> str | None:
|
||||
candidate = Path(self.command)
|
||||
if candidate.exists():
|
||||
return str(candidate)
|
||||
return shutil.which(self.command)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class BrowserPlaywrightAdapter:
|
||||
enabled: bool = False
|
||||
isolated_context: bool = True
|
||||
dry_run: bool = True
|
||||
name: str = "browser.playwright"
|
||||
|
||||
def execute(self, request: ToolCallRequest, context: ToolContext) -> ToolResult:
|
||||
if not self.enabled:
|
||||
return ToolResult(
|
||||
request.id,
|
||||
"failed",
|
||||
error_code="PLAYWRIGHT_DISABLED",
|
||||
audit_summary="Playwright adapter disabled",
|
||||
)
|
||||
if importlib.util.find_spec("playwright") is None:
|
||||
return ToolResult(
|
||||
request.id,
|
||||
"failed",
|
||||
error_code="PLAYWRIGHT_UNAVAILABLE",
|
||||
audit_summary="Playwright unavailable",
|
||||
)
|
||||
task = str(request.arguments.get("task", request.natural_language_intent))
|
||||
if is_high_risk_task(task):
|
||||
return ToolResult(
|
||||
request.id,
|
||||
"confirmation_required",
|
||||
error_code="PLAYWRIGHT_SENSITIVE_ACTION",
|
||||
audit_summary="sensitive browser action requires confirmation",
|
||||
)
|
||||
if self.dry_run:
|
||||
context_type = "isolated" if self.isolated_context else "shared"
|
||||
return ToolResult(
|
||||
request.id,
|
||||
"success",
|
||||
f"dry-run Playwright {context_type} context task: {task}",
|
||||
audit_summary="Playwright dry run",
|
||||
)
|
||||
return ToolResult(
|
||||
request.id,
|
||||
"failed",
|
||||
error_code="PLAYWRIGHT_RUNTIME_NOT_WIRED",
|
||||
audit_summary="Playwright runtime not wired",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ComputerControlProvider:
|
||||
enabled: bool = False
|
||||
name: str = "computer.control"
|
||||
|
||||
def execute(self, request: ToolCallRequest, context: ToolContext) -> ToolResult:
|
||||
return ToolResult(
|
||||
request.id,
|
||||
"failed",
|
||||
error_code="COMPUTER_CONTROL_UNSUPPORTED",
|
||||
audit_summary="direct GUI control is not supported in the first full-duplex Agent version",
|
||||
)
|
||||
|
||||
|
||||
def planned_external_adapters(config: AppConfig) -> dict[str, object]:
|
||||
return {
|
||||
"openinterpreter.run": OpenInterpreterAdapter(
|
||||
enabled=config.openinterpreter_enabled,
|
||||
command=config.openinterpreter_command,
|
||||
dry_run=True,
|
||||
),
|
||||
"browser.playwright": BrowserPlaywrightAdapter(
|
||||
enabled=config.browser_playwright_enabled,
|
||||
isolated_context=True,
|
||||
dry_run=True,
|
||||
),
|
||||
"computer.control": ComputerControlProvider(enabled=config.computer_control_enabled),
|
||||
}
|
||||
@@ -0,0 +1,559 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from threading import RLock
|
||||
from typing import Any
|
||||
from typing import Protocol
|
||||
|
||||
from .config import AppConfig
|
||||
from .models import AudioFrame, ErrorCode, ProviderError
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AudioBufferWriteResult:
|
||||
accepted: AudioFrame
|
||||
dropped: tuple[AudioFrame, ...] = ()
|
||||
overrun: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AudioHubDiagnostic:
|
||||
code: ErrorCode
|
||||
ring: str
|
||||
message: str
|
||||
dropped_frame_ids: tuple[int, ...] = ()
|
||||
subscriber: str | None = None
|
||||
|
||||
|
||||
class AudioRingBuffer:
|
||||
def __init__(self, *, capacity_ms: int, name: str) -> None:
|
||||
if capacity_ms <= 0:
|
||||
raise ValueError("capacity_ms must be positive")
|
||||
self.capacity_ms = capacity_ms
|
||||
self.name = name
|
||||
self._frames: deque[AudioFrame] = deque()
|
||||
self._duration_ms = 0
|
||||
|
||||
@property
|
||||
def duration_ms(self) -> int:
|
||||
return self._duration_ms
|
||||
|
||||
@property
|
||||
def frame_count(self) -> int:
|
||||
return len(self._frames)
|
||||
|
||||
def clear(self) -> None:
|
||||
self._frames.clear()
|
||||
self._duration_ms = 0
|
||||
|
||||
def write(self, frame: AudioFrame) -> AudioBufferWriteResult:
|
||||
self._frames.append(frame)
|
||||
self._duration_ms += frame.duration_ms
|
||||
dropped: list[AudioFrame] = []
|
||||
while self._duration_ms > self.capacity_ms and self._frames:
|
||||
oldest = self._frames.popleft()
|
||||
dropped.append(oldest)
|
||||
self._duration_ms -= oldest.duration_ms
|
||||
return AudioBufferWriteResult(
|
||||
accepted=frame,
|
||||
dropped=tuple(dropped),
|
||||
overrun=bool(dropped),
|
||||
)
|
||||
|
||||
def frames(self) -> tuple[AudioFrame, ...]:
|
||||
return tuple(self._frames)
|
||||
|
||||
def latest_window(self, *, timestamp_ms: int, window_ms: int) -> tuple[AudioFrame, ...]:
|
||||
if window_ms <= 0:
|
||||
raise ValueError("window_ms must be positive")
|
||||
start_ms = max(0, timestamp_ms - window_ms)
|
||||
return tuple(
|
||||
frame
|
||||
for frame in self._frames
|
||||
if start_ms <= frame.timestamp_ms <= timestamp_ms
|
||||
)
|
||||
|
||||
|
||||
class CaptureRingBuffer(AudioRingBuffer):
|
||||
def __init__(self, *, capacity_ms: int) -> None:
|
||||
super().__init__(capacity_ms=capacity_ms, name="capture")
|
||||
|
||||
|
||||
class RenderReferenceRingBuffer(AudioRingBuffer):
|
||||
def __init__(self, *, capacity_ms: int) -> None:
|
||||
super().__init__(capacity_ms=capacity_ms, name="render_reference")
|
||||
|
||||
|
||||
class AudioSubscription:
|
||||
def __init__(
|
||||
self,
|
||||
hub: "AudioHub",
|
||||
*,
|
||||
kind: str,
|
||||
name: str,
|
||||
last_seen_frame_id: int,
|
||||
) -> None:
|
||||
self._hub = hub
|
||||
self.kind = kind
|
||||
self.name = name
|
||||
self._last_seen_frame_id = last_seen_frame_id
|
||||
self.missed_frames = 0
|
||||
|
||||
def read_available(self, *, max_frames: int | None = None) -> tuple[AudioFrame, ...]:
|
||||
frames, missed = self._hub._read_for_subscription(
|
||||
kind=self.kind,
|
||||
subscriber=self.name,
|
||||
last_seen_frame_id=self._last_seen_frame_id,
|
||||
max_frames=max_frames,
|
||||
)
|
||||
if missed:
|
||||
self.missed_frames += missed
|
||||
if frames:
|
||||
self._last_seen_frame_id = frames[-1].frame_id
|
||||
return frames
|
||||
|
||||
|
||||
class AudioHub:
|
||||
VALID_KINDS = {"raw_capture", "processed_capture", "render_reference"}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
processor: AudioProcessingProvider,
|
||||
capture_capacity_ms: int = 3000,
|
||||
render_capacity_ms: int = 3000,
|
||||
) -> None:
|
||||
self.processor = processor
|
||||
self.raw_capture = CaptureRingBuffer(capacity_ms=capture_capacity_ms)
|
||||
self.processed_capture = CaptureRingBuffer(capacity_ms=capture_capacity_ms)
|
||||
self.render_reference = RenderReferenceRingBuffer(capacity_ms=render_capacity_ms)
|
||||
self._diagnostics: list[AudioHubDiagnostic] = []
|
||||
self._lock = RLock()
|
||||
|
||||
@property
|
||||
def diagnostics(self) -> tuple[AudioHubDiagnostic, ...]:
|
||||
with self._lock:
|
||||
return tuple(self._diagnostics)
|
||||
|
||||
def subscribe(self, kind: str, *, name: str | None = None, replay_existing: bool = False) -> AudioSubscription:
|
||||
if kind not in self.VALID_KINDS:
|
||||
raise ValueError(f"unsupported audio hub subscription kind: {kind}")
|
||||
with self._lock:
|
||||
frames = self._buffer_for(kind).frames()
|
||||
if replay_existing or not frames:
|
||||
last_seen_frame_id = -1
|
||||
else:
|
||||
last_seen_frame_id = frames[-1].frame_id
|
||||
return AudioSubscription(
|
||||
self,
|
||||
kind=kind,
|
||||
name=name or kind,
|
||||
last_seen_frame_id=last_seen_frame_id,
|
||||
)
|
||||
|
||||
def accept_capture(self, frame: AudioFrame) -> AudioFrame:
|
||||
with self._lock:
|
||||
self._write(self.raw_capture, frame)
|
||||
processed = self.processor.process_capture(frame)
|
||||
self._write(self.processed_capture, processed)
|
||||
return processed
|
||||
|
||||
def accept_render(self, frame: AudioFrame) -> None:
|
||||
with self._lock:
|
||||
self.processor.process_render(frame)
|
||||
self._write(self.render_reference, frame)
|
||||
|
||||
def clear(self) -> None:
|
||||
with self._lock:
|
||||
self.raw_capture.clear()
|
||||
self.processed_capture.clear()
|
||||
self.render_reference.clear()
|
||||
self._diagnostics.clear()
|
||||
self.processor.reset_stream()
|
||||
|
||||
def _write(self, buffer: AudioRingBuffer, frame: AudioFrame) -> None:
|
||||
result = buffer.write(frame)
|
||||
if result.overrun:
|
||||
dropped_ids = tuple(item.frame_id for item in result.dropped)
|
||||
self._diagnostics.append(
|
||||
AudioHubDiagnostic(
|
||||
code=ErrorCode.AUDIO_BUFFER_OVERRUN,
|
||||
ring=buffer.name,
|
||||
message=f"{buffer.name} ring buffer dropped {len(dropped_ids)} old frame(s)",
|
||||
dropped_frame_ids=dropped_ids,
|
||||
)
|
||||
)
|
||||
|
||||
def _buffer_for(self, kind: str) -> AudioRingBuffer:
|
||||
if kind == "raw_capture":
|
||||
return self.raw_capture
|
||||
if kind == "processed_capture":
|
||||
return self.processed_capture
|
||||
if kind == "render_reference":
|
||||
return self.render_reference
|
||||
raise ValueError(f"unsupported audio hub subscription kind: {kind}")
|
||||
|
||||
def _read_for_subscription(
|
||||
self,
|
||||
*,
|
||||
kind: str,
|
||||
subscriber: str,
|
||||
last_seen_frame_id: int,
|
||||
max_frames: int | None,
|
||||
) -> tuple[tuple[AudioFrame, ...], int]:
|
||||
with self._lock:
|
||||
frames = self._buffer_for(kind).frames()
|
||||
if not frames:
|
||||
return (), 0
|
||||
missed = 0
|
||||
oldest_id = frames[0].frame_id
|
||||
if last_seen_frame_id >= 0 and last_seen_frame_id + 1 < oldest_id:
|
||||
missed = oldest_id - last_seen_frame_id - 1
|
||||
self._diagnostics.append(
|
||||
AudioHubDiagnostic(
|
||||
code=ErrorCode.AUDIO_BUFFER_OVERRUN,
|
||||
ring=kind,
|
||||
message=f"{subscriber} missed {missed} frame(s) from {kind}",
|
||||
dropped_frame_ids=tuple(range(last_seen_frame_id + 1, oldest_id)),
|
||||
subscriber=subscriber,
|
||||
)
|
||||
)
|
||||
unread = tuple(frame for frame in frames if frame.frame_id > last_seen_frame_id)
|
||||
if max_frames is not None:
|
||||
unread = unread[: max(0, max_frames)]
|
||||
return unread, missed
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AudioProcessingHealth:
|
||||
provider: str
|
||||
available: bool
|
||||
fallback_active: bool = False
|
||||
message: str = ""
|
||||
|
||||
|
||||
class AudioProcessingProvider(Protocol):
|
||||
name: str
|
||||
|
||||
def process_capture(self, frame: AudioFrame) -> AudioFrame:
|
||||
...
|
||||
|
||||
def process_render(self, frame: AudioFrame) -> None:
|
||||
...
|
||||
|
||||
def reset_stream(self) -> None:
|
||||
...
|
||||
|
||||
def health_check(self) -> AudioProcessingHealth:
|
||||
...
|
||||
|
||||
|
||||
class NoopAudioProcessingProvider:
|
||||
name = "noop"
|
||||
|
||||
def __init__(self, *, fallback_active: bool = False, message: str = "") -> None:
|
||||
self.fallback_active = fallback_active
|
||||
self.message = message
|
||||
self.render_frames: list[AudioFrame] = []
|
||||
|
||||
def process_capture(self, frame: AudioFrame) -> AudioFrame:
|
||||
metadata = dict(frame.metadata)
|
||||
if self.fallback_active:
|
||||
metadata["apm_fallback"] = True
|
||||
return AudioFrame(
|
||||
frame.pcm,
|
||||
frame.sample_rate,
|
||||
frame.channels,
|
||||
frame.timestamp_ms,
|
||||
frame.frame_id,
|
||||
metadata,
|
||||
)
|
||||
|
||||
def process_render(self, frame: AudioFrame) -> None:
|
||||
self.render_frames.append(frame)
|
||||
|
||||
def reset_stream(self) -> None:
|
||||
self.render_frames.clear()
|
||||
|
||||
def health_check(self) -> AudioProcessingHealth:
|
||||
return AudioProcessingHealth(
|
||||
provider=self.name,
|
||||
available=True,
|
||||
fallback_active=self.fallback_active,
|
||||
message=self.message,
|
||||
)
|
||||
|
||||
|
||||
class FakeWebRtcAudioProcessingProvider:
|
||||
name = "fake_webrtc"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
sample_rate: int = 16000,
|
||||
channels: int = 1,
|
||||
fail_processing: bool = False,
|
||||
) -> None:
|
||||
self.sample_rate = sample_rate
|
||||
self.channels = channels
|
||||
self.fail_processing = fail_processing
|
||||
self.render_frames: list[AudioFrame] = []
|
||||
|
||||
def process_capture(self, frame: AudioFrame) -> AudioFrame:
|
||||
self._validate_format(frame)
|
||||
if self.fail_processing:
|
||||
raise ProviderError(
|
||||
ErrorCode.AUDIO_APM_PROCESS_FAILED,
|
||||
"fake WebRTC APM was configured to fail",
|
||||
True,
|
||||
self.name,
|
||||
"audio_apm",
|
||||
)
|
||||
metadata = dict(frame.metadata)
|
||||
pcm = frame.pcm
|
||||
if metadata.get("assistant_echo") and self.render_frames:
|
||||
metadata["echo_suppressed"] = True
|
||||
metadata["speech"] = False
|
||||
pcm = b"\x00" * len(frame.pcm)
|
||||
return AudioFrame(
|
||||
pcm,
|
||||
frame.sample_rate,
|
||||
frame.channels,
|
||||
frame.timestamp_ms,
|
||||
frame.frame_id,
|
||||
metadata,
|
||||
)
|
||||
|
||||
def process_render(self, frame: AudioFrame) -> None:
|
||||
self._validate_format(frame)
|
||||
self.render_frames.append(frame)
|
||||
|
||||
def reset_stream(self) -> None:
|
||||
self.render_frames.clear()
|
||||
|
||||
def health_check(self) -> AudioProcessingHealth:
|
||||
return AudioProcessingHealth(provider=self.name, available=True)
|
||||
|
||||
def _validate_format(self, frame: AudioFrame) -> None:
|
||||
if frame.sample_rate != self.sample_rate or frame.channels != self.channels:
|
||||
raise ProviderError(
|
||||
ErrorCode.AUDIO_APM_FORMAT_MISMATCH,
|
||||
"audio frame format does not match fake WebRTC APM configuration",
|
||||
False,
|
||||
self.name,
|
||||
"audio_apm",
|
||||
)
|
||||
|
||||
|
||||
class AecAudioProcessingProvider:
|
||||
name = "aec_audio_processing"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
sample_rate: int = 16000,
|
||||
channels: int = 1,
|
||||
enable_aec: bool = True,
|
||||
enable_ns: bool = True,
|
||||
enable_agc: bool = True,
|
||||
) -> None:
|
||||
self.sample_rate = sample_rate
|
||||
self.channels = channels
|
||||
self.enable_aec = enable_aec
|
||||
self.enable_ns = enable_ns
|
||||
self.enable_agc = enable_agc
|
||||
self.render_frames: list[AudioFrame] = []
|
||||
self._processor: Any = None
|
||||
self._load_processor()
|
||||
|
||||
def process_capture(self, frame: AudioFrame) -> AudioFrame:
|
||||
self._validate_format(frame)
|
||||
try:
|
||||
pcm = self._process_pcm_chunks(frame.pcm, self._processor.process_stream)
|
||||
except Exception as exc:
|
||||
raise ProviderError(
|
||||
ErrorCode.AUDIO_APM_PROCESS_FAILED,
|
||||
f"aec_audio_processing capture failed: {exc}",
|
||||
True,
|
||||
self.name,
|
||||
"audio_apm",
|
||||
) from exc
|
||||
metadata = dict(frame.metadata)
|
||||
metadata["apm_provider"] = self.name
|
||||
if metadata.get("assistant_echo") and self.render_frames and _rms_int16(pcm) <= _rms_int16(frame.pcm) * 0.85:
|
||||
metadata["echo_suppressed"] = True
|
||||
metadata["speech"] = False
|
||||
return AudioFrame(
|
||||
pcm,
|
||||
frame.sample_rate,
|
||||
frame.channels,
|
||||
frame.timestamp_ms,
|
||||
frame.frame_id,
|
||||
metadata,
|
||||
)
|
||||
|
||||
def process_render(self, frame: AudioFrame) -> None:
|
||||
self._validate_format(frame)
|
||||
try:
|
||||
self._process_pcm_chunks(frame.pcm, self._processor.process_reverse_stream)
|
||||
except Exception as exc:
|
||||
raise ProviderError(
|
||||
ErrorCode.AUDIO_APM_PROCESS_FAILED,
|
||||
f"aec_audio_processing render reference failed: {exc}",
|
||||
True,
|
||||
self.name,
|
||||
"audio_apm",
|
||||
) from exc
|
||||
self.render_frames.append(frame)
|
||||
|
||||
def reset_stream(self) -> None:
|
||||
self.render_frames.clear()
|
||||
self._load_processor()
|
||||
|
||||
def health_check(self) -> AudioProcessingHealth:
|
||||
return AudioProcessingHealth(
|
||||
provider=self.name,
|
||||
available=True,
|
||||
message="aec_audio_processing native WebRTC APM binding loaded",
|
||||
)
|
||||
|
||||
def _load_processor(self) -> None:
|
||||
try:
|
||||
from aec_audio_processing import AudioProcessor # type: ignore[import-not-found]
|
||||
|
||||
self._processor = AudioProcessor(
|
||||
enable_aec=self.enable_aec,
|
||||
enable_ns=self.enable_ns,
|
||||
enable_agc=self.enable_agc,
|
||||
enable_vad=False,
|
||||
)
|
||||
self._processor.set_stream_format(self.sample_rate, self.channels, self.sample_rate, self.channels)
|
||||
self._processor.set_reverse_stream_format(self.sample_rate, self.channels)
|
||||
except Exception as exc:
|
||||
raise ProviderError(
|
||||
ErrorCode.AUDIO_APM_UNAVAILABLE,
|
||||
f"cannot initialize aec_audio_processing: {exc}",
|
||||
False,
|
||||
self.name,
|
||||
"audio_apm",
|
||||
) from exc
|
||||
|
||||
def _process_pcm_chunks(self, pcm: bytes, processor_method) -> bytes:
|
||||
frame_bytes = max(2 * self.channels, int(self._processor.get_frame_size()) * self.channels * 2)
|
||||
frame_bytes -= frame_bytes % (2 * self.channels)
|
||||
if frame_bytes <= 0:
|
||||
raise ValueError("invalid APM frame size")
|
||||
processed = bytearray()
|
||||
offset = 0
|
||||
while offset < len(pcm):
|
||||
chunk = pcm[offset : offset + frame_bytes]
|
||||
padding = b""
|
||||
if len(chunk) < frame_bytes:
|
||||
padding = b"\x00" * (frame_bytes - len(chunk))
|
||||
chunk += padding
|
||||
output = processor_method(chunk)
|
||||
if output is None:
|
||||
output = chunk
|
||||
if padding:
|
||||
output = output[: len(output) - len(padding)]
|
||||
processed.extend(output)
|
||||
offset += frame_bytes
|
||||
return bytes(processed)
|
||||
|
||||
def _validate_format(self, frame: AudioFrame) -> None:
|
||||
if frame.sample_rate != self.sample_rate or frame.channels != self.channels:
|
||||
raise ProviderError(
|
||||
ErrorCode.AUDIO_APM_FORMAT_MISMATCH,
|
||||
"audio frame format does not match aec_audio_processing configuration",
|
||||
False,
|
||||
self.name,
|
||||
"audio_apm",
|
||||
)
|
||||
|
||||
|
||||
def webrtc_apm_probe() -> AudioProcessingHealth:
|
||||
candidates = (
|
||||
"aec_audio_processing",
|
||||
"webrtc_audio_processing",
|
||||
"webrtc_audio_processing_module",
|
||||
)
|
||||
for module_name in candidates:
|
||||
if importlib.util.find_spec(module_name) is not None:
|
||||
return AudioProcessingHealth(
|
||||
provider=module_name,
|
||||
available=True,
|
||||
message=f"found {module_name}",
|
||||
)
|
||||
return AudioProcessingHealth(
|
||||
provider="webrtc",
|
||||
available=False,
|
||||
message="no supported WebRTC APM Python binding found",
|
||||
)
|
||||
|
||||
|
||||
def build_audio_processing_provider(config: AppConfig) -> AudioProcessingProvider:
|
||||
if config.audio_apm_provider == "fake":
|
||||
return FakeWebRtcAudioProcessingProvider(
|
||||
sample_rate=config.sample_rate,
|
||||
channels=config.channels,
|
||||
)
|
||||
if config.audio_apm_provider == "disabled":
|
||||
return NoopAudioProcessingProvider(
|
||||
fallback_active=True,
|
||||
message="WebRTC APM disabled by configuration",
|
||||
)
|
||||
if config.audio_apm_provider != "webrtc":
|
||||
raise ProviderError(
|
||||
ErrorCode.CONFIG_MISSING_VALUE,
|
||||
f"unsupported audio APM provider: {config.audio_apm_provider}",
|
||||
False,
|
||||
"config",
|
||||
"audio_apm",
|
||||
)
|
||||
|
||||
health = webrtc_apm_probe()
|
||||
if health.available:
|
||||
if health.provider == "aec_audio_processing":
|
||||
return AecAudioProcessingProvider(
|
||||
sample_rate=config.sample_rate,
|
||||
channels=config.channels,
|
||||
enable_aec=config.audio_aec_enabled,
|
||||
enable_ns=config.audio_ns_enabled,
|
||||
enable_agc=config.audio_agc_enabled,
|
||||
)
|
||||
message = f"{health.provider} binding is detected but native processing is not wired yet"
|
||||
if config.audio_apm_required:
|
||||
raise ProviderError(
|
||||
ErrorCode.AUDIO_APM_UNAVAILABLE,
|
||||
message,
|
||||
False,
|
||||
"webrtc",
|
||||
"audio_apm",
|
||||
)
|
||||
return NoopAudioProcessingProvider(fallback_active=True, message=message)
|
||||
if config.audio_apm_required:
|
||||
raise ProviderError(
|
||||
ErrorCode.AUDIO_APM_UNAVAILABLE,
|
||||
health.message,
|
||||
False,
|
||||
"webrtc",
|
||||
"audio_apm",
|
||||
)
|
||||
return NoopAudioProcessingProvider(
|
||||
fallback_active=True,
|
||||
message=health.message,
|
||||
)
|
||||
|
||||
|
||||
def _rms_int16(pcm: bytes) -> float:
|
||||
if len(pcm) < 2:
|
||||
return 0.0
|
||||
samples = [
|
||||
int.from_bytes(pcm[index : index + 2], "little", signed=True)
|
||||
for index in range(0, len(pcm) - 1, 2)
|
||||
]
|
||||
if not samples:
|
||||
return 0.0
|
||||
return (sum(sample * sample for sample in samples) / len(samples)) ** 0.5
|
||||
@@ -0,0 +1,181 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable
|
||||
|
||||
from .events import RECOVERING, SESSION_RECOVERED, STAGE_ERROR, PipelineEventBus
|
||||
from .models import PipelineState, ProviderError
|
||||
|
||||
|
||||
class InvalidStateTransition(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
FULL_DUPLEX_ALLOWED_TRANSITIONS: dict[PipelineState, set[PipelineState]] = {
|
||||
PipelineState.IDLE: {PipelineState.LISTENING, PipelineState.RECOVERING},
|
||||
PipelineState.LISTENING: {
|
||||
PipelineState.THINKING,
|
||||
PipelineState.INTERRUPTED,
|
||||
PipelineState.RECOVERING,
|
||||
PipelineState.IDLE,
|
||||
},
|
||||
PipelineState.THINKING: {
|
||||
PipelineState.SPEAKING,
|
||||
PipelineState.TOOL_RUNNING,
|
||||
PipelineState.INTERRUPTED,
|
||||
PipelineState.RECOVERING,
|
||||
PipelineState.LISTENING,
|
||||
},
|
||||
PipelineState.SPEAKING: {
|
||||
PipelineState.LISTENING,
|
||||
PipelineState.INTERRUPTED,
|
||||
PipelineState.RECOVERING,
|
||||
},
|
||||
PipelineState.TOOL_RUNNING: {
|
||||
PipelineState.THINKING,
|
||||
PipelineState.INTERRUPTED,
|
||||
PipelineState.RECOVERING,
|
||||
},
|
||||
PipelineState.INTERRUPTED: {
|
||||
PipelineState.LISTENING,
|
||||
PipelineState.RECOVERING,
|
||||
},
|
||||
PipelineState.RECOVERING: {
|
||||
PipelineState.LISTENING,
|
||||
PipelineState.IDLE,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StateTransition:
|
||||
old_state: PipelineState
|
||||
new_state: PipelineState
|
||||
event_type: str
|
||||
|
||||
|
||||
class FullDuplexStateMachine:
|
||||
def __init__(self, initial_state: PipelineState = PipelineState.IDLE) -> None:
|
||||
if initial_state not in FULL_DUPLEX_ALLOWED_TRANSITIONS:
|
||||
raise ValueError(f"unsupported full-duplex initial state: {initial_state.value}")
|
||||
self.current_state = initial_state
|
||||
self.history: list[StateTransition] = []
|
||||
|
||||
def can_transition(self, new_state: PipelineState) -> bool:
|
||||
return new_state in FULL_DUPLEX_ALLOWED_TRANSITIONS[self.current_state]
|
||||
|
||||
def transition(self, new_state: PipelineState, *, event_type: str) -> StateTransition:
|
||||
if not self.can_transition(new_state):
|
||||
raise InvalidStateTransition(
|
||||
f"cannot transition from {self.current_state.value} to {new_state.value}"
|
||||
)
|
||||
transition = StateTransition(self.current_state, new_state, event_type)
|
||||
self.current_state = new_state
|
||||
self.history.append(transition)
|
||||
return transition
|
||||
|
||||
|
||||
CancelCallback = Callable[[str], None]
|
||||
|
||||
|
||||
@dataclass
|
||||
class CancellationToken:
|
||||
name: str
|
||||
parent: "CancellationToken | None" = None
|
||||
cancelled: bool = False
|
||||
reason: str = ""
|
||||
children: list["CancellationToken"] = field(default_factory=list)
|
||||
_callbacks: list[CancelCallback] = field(default_factory=list)
|
||||
|
||||
def create_child(self, name: str) -> "CancellationToken":
|
||||
child = CancellationToken(name=name, parent=self)
|
||||
if self.cancelled:
|
||||
child.cancel(self.reason)
|
||||
self.children.append(child)
|
||||
return child
|
||||
|
||||
def add_callback(self, callback: CancelCallback) -> None:
|
||||
if self.cancelled:
|
||||
callback(self.reason)
|
||||
return
|
||||
self._callbacks.append(callback)
|
||||
|
||||
def cancel(self, reason: str) -> None:
|
||||
if self.cancelled:
|
||||
return
|
||||
self.cancelled = True
|
||||
self.reason = reason
|
||||
for callback in list(self._callbacks):
|
||||
callback(reason)
|
||||
for child in list(self.children):
|
||||
child.cancel(reason)
|
||||
|
||||
def raise_if_cancelled(self) -> None:
|
||||
if self.cancelled:
|
||||
raise RuntimeError(f"cancelled {self.name}: {self.reason}")
|
||||
|
||||
|
||||
class CancellationGraph:
|
||||
def __init__(self, root_name: str = "turn") -> None:
|
||||
self.root = CancellationToken(root_name)
|
||||
self.tokens: dict[str, CancellationToken] = {root_name: self.root}
|
||||
|
||||
def child(self, name: str, *, parent: str | None = None) -> CancellationToken:
|
||||
parent_token = self.tokens[parent] if parent else self.root
|
||||
token = parent_token.create_child(name)
|
||||
self.tokens[name] = token
|
||||
return token
|
||||
|
||||
def cancel_all(self, reason: str) -> None:
|
||||
self.root.cancel(reason)
|
||||
|
||||
|
||||
class RecoveryCoordinator:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
state_machine: FullDuplexStateMachine,
|
||||
event_bus: PipelineEventBus,
|
||||
safe_state: PipelineState = PipelineState.LISTENING,
|
||||
) -> None:
|
||||
self.state_machine = state_machine
|
||||
self.event_bus = event_bus
|
||||
self.safe_state = safe_state
|
||||
|
||||
def recover(
|
||||
self,
|
||||
error: ProviderError,
|
||||
*,
|
||||
turn_id: int | None = None,
|
||||
session_id: str | None = None,
|
||||
) -> PipelineState:
|
||||
self.event_bus.emit(
|
||||
STAGE_ERROR,
|
||||
turn_id=turn_id,
|
||||
session_id=session_id,
|
||||
stage=error.stage,
|
||||
state=PipelineState.RECOVERING,
|
||||
message=error.message,
|
||||
payload={"error": error, "code": error.code.value, "provider": error.provider},
|
||||
)
|
||||
if self.state_machine.current_state != PipelineState.RECOVERING:
|
||||
self.state_machine.transition(PipelineState.RECOVERING, event_type=STAGE_ERROR)
|
||||
self.event_bus.emit(
|
||||
RECOVERING,
|
||||
turn_id=turn_id,
|
||||
session_id=session_id,
|
||||
stage=error.stage,
|
||||
state=PipelineState.RECOVERING,
|
||||
message="recovering full-duplex agent session",
|
||||
payload={"retryable": error.retryable},
|
||||
)
|
||||
self.state_machine.transition(self.safe_state, event_type=SESSION_RECOVERED)
|
||||
self.event_bus.emit(
|
||||
SESSION_RECOVERED,
|
||||
turn_id=turn_id,
|
||||
session_id=session_id,
|
||||
stage="recovery",
|
||||
state=self.safe_state,
|
||||
message="session recovered",
|
||||
)
|
||||
return self.safe_state
|
||||
@@ -0,0 +1,319 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Iterable, Literal, Protocol
|
||||
|
||||
from .full_duplex_audio import AudioHub, RenderReferenceRingBuffer
|
||||
from .full_duplex_control import CancellationToken
|
||||
from .models import AudioFrame, AudioSegment, Message, ProviderError
|
||||
from .tts import sanitize_tts_text
|
||||
|
||||
|
||||
LlmStreamEventKind = Literal["delta", "tool_call", "finish", "error"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LlmStreamEvent:
|
||||
kind: LlmStreamEventKind
|
||||
text_delta: str = ""
|
||||
tool_call: dict[str, object] | None = None
|
||||
finish_reason: str | None = None
|
||||
error: ProviderError | None = None
|
||||
|
||||
|
||||
class StreamingLlmProvider(Protocol):
|
||||
name: str
|
||||
|
||||
def stream(
|
||||
self,
|
||||
messages: list[Message],
|
||||
*,
|
||||
cancellation: CancellationToken,
|
||||
) -> Iterable[LlmStreamEvent]:
|
||||
...
|
||||
|
||||
|
||||
class FakeStreamingLlmProvider:
|
||||
name = "fake_streaming_llm"
|
||||
|
||||
def __init__(self, events: list[LlmStreamEvent]) -> None:
|
||||
self.events = events
|
||||
self.requests: list[list[Message]] = []
|
||||
|
||||
def stream(
|
||||
self,
|
||||
messages: list[Message],
|
||||
*,
|
||||
cancellation: CancellationToken,
|
||||
) -> Iterable[LlmStreamEvent]:
|
||||
self.requests.append(list(messages))
|
||||
for event in self.events:
|
||||
if cancellation.cancelled:
|
||||
break
|
||||
yield event
|
||||
|
||||
|
||||
class SentenceSegmenter:
|
||||
def __init__(self, *, max_chars: int = 80) -> None:
|
||||
if max_chars <= 0:
|
||||
raise ValueError("max_chars must be positive")
|
||||
self.max_chars = max_chars
|
||||
self._buffer = ""
|
||||
self._inside_code_block = False
|
||||
|
||||
def accept_delta(self, text: str) -> list[str]:
|
||||
emitted: list[str] = []
|
||||
for char in text:
|
||||
self._buffer += char
|
||||
if self._buffer.endswith("```"):
|
||||
self._inside_code_block = not self._inside_code_block
|
||||
if self._inside_code_block:
|
||||
continue
|
||||
if self._is_sentence_boundary(char):
|
||||
emitted.append(self._pop_buffer())
|
||||
elif len(self._buffer) >= self.max_chars and char in {"，", ",", " "}:
|
||||
emitted.append(self._pop_buffer())
|
||||
return [sentence for sentence in emitted if sentence]
|
||||
|
||||
def flush(self) -> str | None:
|
||||
sentence = self._pop_buffer()
|
||||
return sentence or None
|
||||
|
||||
def _is_sentence_boundary(self, char: str) -> bool:
|
||||
if char in {"。", "！", "？", "!", "?", "；", ";", "\n"}:
|
||||
return True
|
||||
if char == ".":
|
||||
index = len(self._buffer) - 1
|
||||
previous_char = self._buffer[index - 1] if index > 0 else ""
|
||||
next_is_url = self._buffer.endswith("http.") or self._buffer.endswith("www.")
|
||||
return not previous_char.isdigit() and not next_is_url
|
||||
return False
|
||||
|
||||
def _pop_buffer(self) -> str:
|
||||
sentence = self._buffer.strip()
|
||||
self._buffer = ""
|
||||
return sentence
|
||||
|
||||
|
||||
def prepare_tts_sentence(text: str) -> str:
|
||||
return sanitize_tts_text(text).strip()
|
||||
|
||||
|
||||
class StreamingTtsSession(Protocol):
|
||||
def accept_text(self, text: str) -> list[AudioFrame]:
|
||||
...
|
||||
|
||||
def flush(self) -> list[AudioFrame]:
|
||||
...
|
||||
|
||||
def cancel(self, reason: str) -> None:
|
||||
...
|
||||
|
||||
|
||||
class StreamingTtsProvider(Protocol):
|
||||
name: str
|
||||
|
||||
def start_stream(self, *, voice: str, sample_rate: int) -> StreamingTtsSession:
|
||||
...
|
||||
|
||||
|
||||
class FakeStreamingTtsProvider:
|
||||
name = "fake_streaming_tts"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.started: list[tuple[str, int]] = []
|
||||
|
||||
def start_stream(self, *, voice: str, sample_rate: int) -> "FakeStreamingTtsSession":
|
||||
self.started.append((voice, sample_rate))
|
||||
return FakeStreamingTtsSession(sample_rate=sample_rate)
|
||||
|
||||
|
||||
class FakeStreamingTtsSession:
|
||||
def __init__(self, *, sample_rate: int) -> None:
|
||||
self.sample_rate = sample_rate
|
||||
self.cancelled = False
|
||||
self.accepted_text: list[str] = []
|
||||
self._next_frame_id = 0
|
||||
|
||||
def accept_text(self, text: str) -> list[AudioFrame]:
|
||||
if self.cancelled:
|
||||
return []
|
||||
spoken = prepare_tts_sentence(text)
|
||||
if not spoken:
|
||||
return []
|
||||
self.accepted_text.append(spoken)
|
||||
self._next_frame_id += 1
|
||||
return [
|
||||
AudioFrame(
|
||||
spoken.encode("utf-8"),
|
||||
self.sample_rate,
|
||||
1,
|
||||
self._next_frame_id * 20,
|
||||
self._next_frame_id,
|
||||
{"duration_ms": 20, "tts_text": spoken},
|
||||
)
|
||||
]
|
||||
|
||||
def flush(self) -> list[AudioFrame]:
|
||||
return []
|
||||
|
||||
def cancel(self, reason: str) -> None:
|
||||
self.cancelled = True
|
||||
|
||||
|
||||
class AudioSegmentStreamingTtsProvider:
|
||||
name = "audio_segment_streaming_tts"
|
||||
|
||||
def __init__(self, tts_provider, *, chunk_ms: int = 30) -> None:
|
||||
if chunk_ms <= 0:
|
||||
raise ValueError("chunk_ms must be positive")
|
||||
self.tts_provider = tts_provider
|
||||
self.chunk_ms = chunk_ms
|
||||
self.started: list[tuple[str, int]] = []
|
||||
|
||||
def start_stream(self, *, voice: str, sample_rate: int) -> "AudioSegmentStreamingTtsSession":
|
||||
self.started.append((voice, sample_rate))
|
||||
return AudioSegmentStreamingTtsSession(
|
||||
tts_provider=self.tts_provider,
|
||||
sample_rate=sample_rate,
|
||||
chunk_ms=self.chunk_ms,
|
||||
)
|
||||
|
||||
|
||||
class AudioSegmentStreamingTtsSession:
|
||||
def __init__(self, *, tts_provider, sample_rate: int, chunk_ms: int) -> None:
|
||||
self.tts_provider = tts_provider
|
||||
self.sample_rate = sample_rate
|
||||
self.chunk_ms = chunk_ms
|
||||
self.cancelled = False
|
||||
self.cancel_reason = ""
|
||||
self.accepted_text: list[str] = []
|
||||
self._next_frame_id = 0
|
||||
if hasattr(self.tts_provider, "load"):
|
||||
self.tts_provider.load()
|
||||
|
||||
def accept_text(self, text: str) -> list[AudioFrame]:
|
||||
if self.cancelled:
|
||||
return []
|
||||
spoken = prepare_tts_sentence(text)
|
||||
if not spoken:
|
||||
return []
|
||||
segment = self.tts_provider.synthesize(spoken)
|
||||
self.accepted_text.append(spoken)
|
||||
return self._segment_to_frames(segment, spoken)
|
||||
|
||||
def flush(self) -> list[AudioFrame]:
|
||||
return []
|
||||
|
||||
def cancel(self, reason: str) -> None:
|
||||
self.cancelled = True
|
||||
self.cancel_reason = reason
|
||||
|
||||
def _segment_to_frames(self, segment: AudioSegment, text: str) -> list[AudioFrame]:
|
||||
metadata = dict(segment.metadata)
|
||||
if metadata.get("format"):
|
||||
self._next_frame_id += 1
|
||||
metadata.update({"duration_ms": segment.duration_ms, "tts_text": text})
|
||||
return [
|
||||
AudioFrame(
|
||||
segment.pcm,
|
||||
segment.sample_rate,
|
||||
segment.channels,
|
||||
segment.start_time_ms,
|
||||
self._next_frame_id,
|
||||
metadata,
|
||||
)
|
||||
]
|
||||
bytes_per_ms = max(1, int(segment.sample_rate * segment.channels * 2 / 1000))
|
||||
chunk_bytes = max(2 * segment.channels, bytes_per_ms * self.chunk_ms)
|
||||
chunk_bytes -= chunk_bytes % (2 * segment.channels)
|
||||
frames: list[AudioFrame] = []
|
||||
offset = 0
|
||||
timestamp_ms = segment.start_time_ms
|
||||
while offset < len(segment.pcm):
|
||||
data = segment.pcm[offset : offset + chunk_bytes]
|
||||
duration_ms = max(1, int(len(data) / bytes_per_ms))
|
||||
self._next_frame_id += 1
|
||||
frames.append(
|
||||
AudioFrame(
|
||||
data,
|
||||
segment.sample_rate,
|
||||
segment.channels,
|
||||
timestamp_ms,
|
||||
self._next_frame_id,
|
||||
{"duration_ms": duration_ms, "tts_text": text},
|
||||
)
|
||||
)
|
||||
offset += chunk_bytes
|
||||
timestamp_ms += duration_ms
|
||||
return frames
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class PlaybackItem:
|
||||
text: str
|
||||
frames: list[AudioFrame]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PlaybackChunkResult:
|
||||
played_frames: int
|
||||
interrupted: bool
|
||||
committed_text: str = ""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SpokenTextTracker:
|
||||
committed: list[str] = field(default_factory=list)
|
||||
|
||||
def commit(self, text: str) -> None:
|
||||
cleaned = text.strip()
|
||||
if cleaned:
|
||||
self.committed.append(cleaned)
|
||||
|
||||
@property
|
||||
def text(self) -> str:
|
||||
return "".join(self.committed)
|
||||
|
||||
|
||||
class InterruptiblePlaybackQueue:
|
||||
def __init__(self) -> None:
|
||||
self._items: deque[PlaybackItem] = deque()
|
||||
self.spoken = SpokenTextTracker()
|
||||
|
||||
@property
|
||||
def pending_items(self) -> int:
|
||||
return len(self._items)
|
||||
|
||||
def enqueue(self, text: str, frames: list[AudioFrame]) -> None:
|
||||
if frames:
|
||||
self._items.append(PlaybackItem(text=text, frames=list(frames)))
|
||||
|
||||
def clear_unplayed(self) -> None:
|
||||
self._items.clear()
|
||||
|
||||
def play_next(
|
||||
self,
|
||||
*,
|
||||
render_reference: RenderReferenceRingBuffer | None = None,
|
||||
audio_hub: AudioHub | None = None,
|
||||
cancellation: CancellationToken,
|
||||
) -> PlaybackChunkResult:
|
||||
if render_reference is None and audio_hub is None:
|
||||
raise ValueError("render_reference or audio_hub is required")
|
||||
if not self._items:
|
||||
return PlaybackChunkResult(0, interrupted=False)
|
||||
item = self._items.popleft()
|
||||
played = 0
|
||||
for frame in item.frames:
|
||||
if cancellation.cancelled:
|
||||
self.clear_unplayed()
|
||||
return PlaybackChunkResult(played, interrupted=True)
|
||||
if audio_hub is not None:
|
||||
audio_hub.accept_render(frame)
|
||||
elif render_reference is not None:
|
||||
render_reference.write(frame)
|
||||
played += 1
|
||||
self.spoken.commit(item.text)
|
||||
return PlaybackChunkResult(played, interrupted=False, committed_text=item.text)
|
||||
@@ -0,0 +1,804 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .agent_memory import (
|
||||
AgentConversationManager,
|
||||
DisabledMemoryManager,
|
||||
FaissIndexManifest,
|
||||
MemoryHealth,
|
||||
MemoryManager,
|
||||
SQLiteMemoryManager,
|
||||
)
|
||||
from .barge_in import BargeInSpeakerGate, TimbreProfile, ensure_interruptible_pcm
|
||||
from .config import AppConfig
|
||||
from .conversation import ConversationContext
|
||||
from .full_duplex_audio import AudioHub, AudioProcessingProvider, build_audio_processing_provider
|
||||
from .full_duplex_control import CancellationGraph, FullDuplexStateMachine
|
||||
from .full_duplex_response import (
|
||||
FakeStreamingLlmProvider,
|
||||
FakeStreamingTtsProvider,
|
||||
InterruptiblePlaybackQueue,
|
||||
LlmStreamEvent,
|
||||
SentenceSegmenter,
|
||||
StreamingLlmProvider,
|
||||
StreamingTtsProvider,
|
||||
)
|
||||
from .full_duplex_speech import FakeVadProvider, InterruptController, InterruptionDetector
|
||||
from .llm import OpenAICompatibleLlmProvider
|
||||
from .models import AudioFrame, AudioSegment, ErrorCode, Message, PipelineState, ProviderError
|
||||
from .protocols import AudioTransport, LlmProvider, RealtimeSttProvider, SttProvider, TtsProvider
|
||||
from .runtime import RuntimeSummary
|
||||
from .stt import CloudAsrSttProvider, SherpaOnnxSttProvider, is_valid_transcript_text
|
||||
from .tool_router import (
|
||||
MemorySaveTool,
|
||||
MemorySearchTool,
|
||||
ShellReadonlyTool,
|
||||
ToolCallRequest,
|
||||
ToolContext,
|
||||
ToolResult,
|
||||
ToolRouter,
|
||||
)
|
||||
from .transport import SoundDeviceAudioTransport
|
||||
from .tts import CloudTtsProvider, MacSayTtsProvider, SentenceBuffer, make_end_chime, sanitize_tts_text
|
||||
from .vad import EnergyVadProvider, HybridVadProvider, SherpaOnnxVadProvider, VadRecorder
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FullDuplexRuntimeHealth:
|
||||
audio_apm_provider: str
|
||||
audio_apm_available: bool
|
||||
audio_apm_fallback_active: bool
|
||||
message: str
|
||||
|
||||
|
||||
class _AudioHubInputPump:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
config: AppConfig,
|
||||
transport: AudioTransport,
|
||||
audio_hub: AudioHub,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.transport = transport
|
||||
self.audio_hub = audio_hub
|
||||
self.error: ProviderError | None = None
|
||||
self._stop_event = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._input_started = False
|
||||
|
||||
def start_input(self) -> None:
|
||||
if self._input_started:
|
||||
return
|
||||
self.transport.start_input(
|
||||
device_id=self.config.audio_input_device,
|
||||
sample_rate=self.config.sample_rate,
|
||||
channels=self.config.channels,
|
||||
)
|
||||
self._input_started = True
|
||||
|
||||
def start_reader(self) -> None:
|
||||
self.start_input()
|
||||
if self._thread is not None and self._thread.is_alive():
|
||||
return
|
||||
self._stop_event.clear()
|
||||
self._thread = threading.Thread(target=self._run, name="owner-full-duplex-audio-hub", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop_event.set()
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=1.0)
|
||||
self.transport.stop()
|
||||
self._input_started = False
|
||||
|
||||
def _run(self) -> None:
|
||||
interval_ms = max(1, self.config.barge_in_listen_interval_ms)
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
frames = self.transport.read_frames(timeout_ms=interval_ms)
|
||||
if not frames:
|
||||
time.sleep(interval_ms / 1000)
|
||||
continue
|
||||
for frame in frames:
|
||||
self.audio_hub.accept_capture(frame)
|
||||
except ProviderError as exc:
|
||||
self.error = exc
|
||||
self._stop_event.set()
|
||||
except Exception as exc: # pragma: no cover - defensive boundary for device callbacks
|
||||
self.error = ProviderError(
|
||||
ErrorCode.AUDIO_APM_PROCESS_FAILED,
|
||||
f"audio input pump failed: {exc}",
|
||||
True,
|
||||
"full-duplex-runtime",
|
||||
"audio",
|
||||
)
|
||||
self._stop_event.set()
|
||||
|
||||
|
||||
class _HubBargeInMonitor:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
config: AppConfig,
|
||||
audio_hub: AudioHub,
|
||||
speaker_gate: BargeInSpeakerGate,
|
||||
assistant_reference: AudioSegment | None,
|
||||
assistant_profile: TimbreProfile,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.audio_hub = audio_hub
|
||||
self.speaker_gate = speaker_gate
|
||||
self.assistant_reference = assistant_reference
|
||||
self.assistant_profile = assistant_profile
|
||||
self.stop_event = threading.Event()
|
||||
self._shutdown_event = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._pending_frames: list[AudioFrame] = []
|
||||
self._lock = threading.Lock()
|
||||
self._vad = EnergyVadProvider()
|
||||
self._interrupted = False
|
||||
|
||||
@property
|
||||
def interrupted(self) -> bool:
|
||||
return self._interrupted
|
||||
|
||||
def pending_frames(self) -> list[AudioFrame]:
|
||||
with self._lock:
|
||||
return list(self._pending_frames)
|
||||
|
||||
def start(self) -> None:
|
||||
self._vad.load()
|
||||
self._vad.reset()
|
||||
self._thread = threading.Thread(target=self._run, name="owner-full-duplex-barge-in", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._shutdown_event.set()
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=1.0)
|
||||
|
||||
def should_stop_playback(self) -> bool:
|
||||
return self.stop_event.is_set()
|
||||
|
||||
def _run(self) -> None:
|
||||
subscription = self.audio_hub.subscribe("processed_capture", name="barge-in")
|
||||
started_at = time.monotonic()
|
||||
speech_ms = 0
|
||||
candidate_frames: list[AudioFrame] = []
|
||||
interval_ms = max(1, self.config.barge_in_listen_interval_ms)
|
||||
echo_guard_ms = max(0, min(self.config.barge_in_echo_guard_ms, 120))
|
||||
min_speech_ms = max(40, min(self.config.barge_in_min_speech_ms, self.config.interrupt_target_latency_ms))
|
||||
while not self._shutdown_event.is_set() and not self.stop_event.is_set():
|
||||
frames = subscription.read_available()
|
||||
if not frames:
|
||||
time.sleep(interval_ms / 1000)
|
||||
continue
|
||||
elapsed_ms = int((time.monotonic() - started_at) * 1000)
|
||||
for frame in frames:
|
||||
if elapsed_ms < echo_guard_ms:
|
||||
continue
|
||||
result = self._vad.analyze(frame)
|
||||
if not result.is_speech or frame.metadata.get("echo_suppressed") or frame.metadata.get("assistant_echo"):
|
||||
speech_ms = 0
|
||||
candidate_frames = []
|
||||
continue
|
||||
accepted = self.speaker_gate.accepts_candidate(
|
||||
frame,
|
||||
self.assistant_profile,
|
||||
self.assistant_reference,
|
||||
reference_elapsed_ms=elapsed_ms,
|
||||
)
|
||||
if not accepted:
|
||||
speech_ms = 0
|
||||
candidate_frames = []
|
||||
continue
|
||||
speech_ms += frame.duration_ms
|
||||
candidate_frames.append(frame)
|
||||
if speech_ms >= min_speech_ms:
|
||||
with self._lock:
|
||||
self._pending_frames = list(candidate_frames)
|
||||
self._interrupted = True
|
||||
self.stop_event.set()
|
||||
return
|
||||
|
||||
|
||||
class FullDuplexAgentRuntime:
|
||||
"""Full-duplex Agent runtime boundary.
|
||||
|
||||
`run()` is the real `run-agent-live` loop: microphone frames are pumped into
|
||||
AudioHub once, capture/STT/LLM/TTS consume independent subscriptions, and
|
||||
playback can be stopped by a concurrent barge-in monitor.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
config: AppConfig,
|
||||
transport: AudioTransport | None = None,
|
||||
processor: AudioProcessingProvider | None = None,
|
||||
audio_hub: AudioHub | None = None,
|
||||
vad_recorder: VadRecorder | None = None,
|
||||
stt: SttProvider | None = None,
|
||||
realtime_stt: RealtimeSttProvider | None = None,
|
||||
llm: LlmProvider | None = None,
|
||||
tts: TtsProvider | None = None,
|
||||
llm_provider: StreamingLlmProvider | None = None,
|
||||
tts_provider: StreamingTtsProvider | None = None,
|
||||
context: ConversationContext | None = None,
|
||||
memory_manager: MemoryManager | None = None,
|
||||
memory_manifest: FaissIndexManifest | None = None,
|
||||
tool_router: ToolRouter | None = None,
|
||||
sentence_buffer: SentenceBuffer | None = None,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.transport = transport or SoundDeviceAudioTransport(output_device=config.audio_output_device)
|
||||
self.processor = processor
|
||||
self.audio_hub = audio_hub
|
||||
self.vad_recorder = vad_recorder
|
||||
self.stt = stt
|
||||
self.realtime_stt = realtime_stt
|
||||
self.llm = llm
|
||||
self.tts = tts
|
||||
self.llm_provider = llm_provider
|
||||
self.tts_provider = tts_provider
|
||||
self.context = context or ConversationContext(
|
||||
max_messages=config.context_max_messages,
|
||||
max_chars=config.context_max_chars,
|
||||
)
|
||||
self.memory_manager = memory_manager or self._build_memory_manager()
|
||||
self.memory_manifest = memory_manifest
|
||||
self.conversation_manager = AgentConversationManager(
|
||||
context=self.context,
|
||||
memory=self.memory_manager,
|
||||
memory_enabled=config.memory_enabled,
|
||||
memory_top_k=config.memory_top_k,
|
||||
)
|
||||
self.tool_router = tool_router or self._build_tool_router()
|
||||
self.health: FullDuplexRuntimeHealth | None = None
|
||||
self.state_machine = FullDuplexStateMachine()
|
||||
self.cancellation_graph = CancellationGraph("turn")
|
||||
self.interrupt_controller: InterruptController | None = None
|
||||
self.playback_queue = InterruptiblePlaybackQueue()
|
||||
self.tool_results: list[ToolResult] = []
|
||||
self.tool_result_messages: list[Message] = []
|
||||
self.sentence_buffer = sentence_buffer or SentenceBuffer()
|
||||
self._input_pump: _AudioHubInputPump | None = None
|
||||
self._pending_capture_frames: list[AudioFrame] = []
|
||||
self._barge_in_gate = BargeInSpeakerGate(
|
||||
enabled=config.barge_in_speaker_gate_enabled,
|
||||
user_similarity_threshold=config.barge_in_user_similarity_threshold,
|
||||
assistant_reject_threshold=config.barge_in_assistant_reject_threshold,
|
||||
min_rms=config.speaker_min_rms,
|
||||
)
|
||||
|
||||
def load_audio(self) -> FullDuplexRuntimeHealth:
|
||||
if self.audio_hub is None:
|
||||
processor = self.processor or build_audio_processing_provider(self.config)
|
||||
self.processor = processor
|
||||
self.audio_hub = AudioHub(
|
||||
processor=processor,
|
||||
capture_capacity_ms=self.config.audio_ring_buffer_ms,
|
||||
render_capacity_ms=self.config.audio_ring_buffer_ms,
|
||||
)
|
||||
provider_health = self.audio_hub.processor.health_check()
|
||||
self.health = FullDuplexRuntimeHealth(
|
||||
audio_apm_provider=provider_health.provider,
|
||||
audio_apm_available=provider_health.available,
|
||||
audio_apm_fallback_active=provider_health.fallback_active,
|
||||
message=provider_health.message,
|
||||
)
|
||||
return self.health
|
||||
|
||||
def run(self, *, once: bool = False) -> RuntimeSummary:
|
||||
self.load_audio()
|
||||
self._load_runtime_dependencies()
|
||||
completed = 0
|
||||
failed = 0
|
||||
last_error: ProviderError | None = None
|
||||
interrupted = False
|
||||
self._start_input()
|
||||
self._status(None, "全双工Agent：启动，直接说话即可")
|
||||
try:
|
||||
while True:
|
||||
turn_id = completed + failed + 1
|
||||
try:
|
||||
result = self._run_live_turn(turn_id)
|
||||
except ProviderError as exc:
|
||||
failed += 1
|
||||
last_error = exc
|
||||
self._error(turn_id, exc.stage, exc.code.value, exc.message)
|
||||
self._recover_to_listening(turn_id)
|
||||
if once:
|
||||
break
|
||||
continue
|
||||
if result.interrupted:
|
||||
interrupted = True
|
||||
if result.completed_turns > 0:
|
||||
completed += result.completed_turns
|
||||
else:
|
||||
failed += result.failed_turns or 1
|
||||
last_error = result.last_error
|
||||
if once:
|
||||
break
|
||||
if once and completed >= 1:
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
interrupted = True
|
||||
finally:
|
||||
self.shutdown()
|
||||
return RuntimeSummary(completed_turns=completed, failed_turns=failed, interrupted=interrupted, last_error=last_error)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self._input_pump is not None:
|
||||
self._input_pump.stop()
|
||||
else:
|
||||
self.transport.stop()
|
||||
|
||||
def _load_runtime_dependencies(self) -> None:
|
||||
if self.vad_recorder is None:
|
||||
self.vad_recorder = self._build_vad_recorder()
|
||||
self.vad_recorder.provider.load()
|
||||
if self.stt is None:
|
||||
self.stt = self._build_stt_provider()
|
||||
self.stt.load()
|
||||
if self.realtime_stt is None and isinstance(self.stt, SherpaOnnxSttProvider):
|
||||
self.realtime_stt = self.stt
|
||||
if self.realtime_stt is not None and self.realtime_stt is not self.stt:
|
||||
self.realtime_stt.load()
|
||||
if self.llm is None:
|
||||
self.llm = OpenAICompatibleLlmProvider(self.config)
|
||||
if self.tts is None:
|
||||
self.tts = self._build_tts_provider()
|
||||
self.tts.load()
|
||||
|
||||
def _start_input(self) -> None:
|
||||
if self.audio_hub is None:
|
||||
raise RuntimeError("audio hub is not loaded")
|
||||
self._input_pump = _AudioHubInputPump(
|
||||
config=self.config,
|
||||
transport=self.transport,
|
||||
audio_hub=self.audio_hub,
|
||||
)
|
||||
self._input_pump.start_input()
|
||||
|
||||
def _ensure_input_reader(self) -> None:
|
||||
if self._input_pump is None:
|
||||
self._start_input()
|
||||
if self._input_pump is not None:
|
||||
self._input_pump.start_reader()
|
||||
|
||||
def _run_live_turn(self, turn_id: int) -> RuntimeSummary:
|
||||
self.state_machine = FullDuplexStateMachine()
|
||||
self.cancellation_graph = CancellationGraph(f"turn-{turn_id}")
|
||||
self.playback_queue = InterruptiblePlaybackQueue()
|
||||
self.sentence_buffer = SentenceBuffer()
|
||||
self.state_machine.transition(PipelineState.LISTENING, event_type="listening_started")
|
||||
self._status(turn_id, "监听中：请直接说话")
|
||||
segment = self._capture_user_segment(turn_id)
|
||||
self._barge_in_gate.remember_user_segment(segment)
|
||||
user_text = self._transcribe_segment(segment, turn_id)
|
||||
interrupted = self._reply_to_user(user_text, turn_id)
|
||||
self._play_end_chime()
|
||||
if self.state_machine.current_state != PipelineState.LISTENING and self.state_machine.can_transition(PipelineState.LISTENING):
|
||||
self.state_machine.transition(PipelineState.LISTENING, event_type="turn_finished")
|
||||
self._status(turn_id, "恢复监听：可直接说话")
|
||||
return RuntimeSummary(completed_turns=1, failed_turns=0, interrupted=interrupted)
|
||||
|
||||
def _capture_user_segment(self, turn_id: int) -> AudioSegment:
|
||||
if self.audio_hub is None or self.vad_recorder is None:
|
||||
raise RuntimeError("runtime dependencies are not loaded")
|
||||
self.vad_recorder.reset()
|
||||
self.vad_recorder.provider.reset()
|
||||
realtime_session = self.realtime_stt.start_stream() if self.realtime_stt is not None else None
|
||||
subscription = self.audio_hub.subscribe("processed_capture", name=f"turn-{turn_id}-capture")
|
||||
self._ensure_input_reader()
|
||||
self._status(turn_id, "录音中：正在听取问题")
|
||||
speech_started = False
|
||||
last_partial = ""
|
||||
while True:
|
||||
self._raise_input_pump_error()
|
||||
frames = self._drain_pending_capture_frames()
|
||||
frames.extend(subscription.read_available())
|
||||
if not frames:
|
||||
time.sleep(max(1, self.config.audio_frame_ms) / 1000)
|
||||
continue
|
||||
for frame in frames:
|
||||
was_started = self.vad_recorder.started
|
||||
result = self.vad_recorder.feed(frame)
|
||||
if not was_started and self.vad_recorder.started and not speech_started:
|
||||
speech_started = True
|
||||
self._status(turn_id, "检测到用户语音")
|
||||
if self.vad_recorder.started and realtime_session is not None:
|
||||
partial = realtime_session.accept_frame(frame)
|
||||
if partial is not None and partial.normalized_text and partial.normalized_text != last_partial:
|
||||
last_partial = partial.normalized_text
|
||||
self._transcript(turn_id, last_partial, final=False)
|
||||
if isinstance(result, ProviderError):
|
||||
if result.code == ErrorCode.VAD_TIMEOUT_NO_SPEECH:
|
||||
self.vad_recorder.reset()
|
||||
self.vad_recorder.provider.reset()
|
||||
continue
|
||||
raise result
|
||||
if isinstance(result, AudioSegment):
|
||||
self._status(turn_id, f"用户语音结束：{result.metadata.get('end_reason', 'speech_end')}")
|
||||
return result
|
||||
|
||||
def _transcribe_segment(self, segment: AudioSegment, turn_id: int) -> str:
|
||||
if self.stt is None:
|
||||
raise RuntimeError("STT provider is not loaded")
|
||||
self._status(turn_id, "转写中：正在识别问题")
|
||||
transcript = self.stt.transcribe(segment)
|
||||
user_text = transcript.normalized_text
|
||||
if not is_valid_transcript_text(user_text):
|
||||
raise ProviderError(
|
||||
ErrorCode.STT_EMPTY_TRANSCRIPT,
|
||||
"STT produced no meaningful user text",
|
||||
True,
|
||||
"full-duplex-runtime",
|
||||
"stt",
|
||||
)
|
||||
self._transcript(turn_id, user_text, final=True)
|
||||
return user_text
|
||||
|
||||
def _reply_to_user(self, user_text: str, turn_id: int) -> bool:
|
||||
if self.llm is None:
|
||||
raise RuntimeError("LLM provider is not loaded")
|
||||
messages = [*self.conversation_manager.build_messages_for_user(user_text), *self.tool_result_messages]
|
||||
if self.state_machine.can_transition(PipelineState.THINKING):
|
||||
self.state_machine.transition(PipelineState.THINKING, event_type="stt_final")
|
||||
self._status(turn_id, "思考中：正在生成回复")
|
||||
assistant_text = ""
|
||||
spoken_parts: list[str] = []
|
||||
interrupted = False
|
||||
for delta in self.llm.stream_reply(messages):
|
||||
if self.cancellation_graph.root.cancelled:
|
||||
interrupted = True
|
||||
break
|
||||
assistant_text += delta.text_delta
|
||||
for sentence in self.sentence_buffer.feed(delta.text_delta, bool(delta.finish_reason)):
|
||||
spoken, was_interrupted = self._speak_sentence(sentence, turn_id)
|
||||
if spoken:
|
||||
spoken_parts.append(spoken)
|
||||
if was_interrupted:
|
||||
interrupted = True
|
||||
break
|
||||
if interrupted:
|
||||
break
|
||||
if not interrupted:
|
||||
for sentence in self.sentence_buffer.flush():
|
||||
spoken, was_interrupted = self._speak_sentence(sentence, turn_id)
|
||||
if spoken:
|
||||
spoken_parts.append(spoken)
|
||||
if was_interrupted:
|
||||
interrupted = True
|
||||
break
|
||||
else:
|
||||
self.sentence_buffer.flush()
|
||||
spoken_text = "".join(spoken_parts)
|
||||
if not assistant_text.strip() and not interrupted:
|
||||
raise ProviderError(
|
||||
ErrorCode.LLM_EMPTY_REPLY,
|
||||
"LLM returned no assistant text",
|
||||
True,
|
||||
"full-duplex-runtime",
|
||||
"llm",
|
||||
)
|
||||
self.conversation_manager.commit_user(user_text)
|
||||
if spoken_text.strip():
|
||||
self.conversation_manager.commit_assistant(spoken_text)
|
||||
if interrupted:
|
||||
self._status(turn_id, "播报已打断：继续听你说")
|
||||
return interrupted
|
||||
|
||||
def _speak_sentence(self, sentence: str, turn_id: int) -> tuple[str, bool]:
|
||||
if self.tts is None or self.audio_hub is None:
|
||||
raise RuntimeError("TTS or audio hub is not loaded")
|
||||
spoken_sentence = sanitize_tts_text(sentence)
|
||||
if not spoken_sentence:
|
||||
return "", False
|
||||
if self.state_machine.can_transition(PipelineState.SPEAKING):
|
||||
self.state_machine.transition(PipelineState.SPEAKING, event_type="tts_started")
|
||||
self._status(turn_id, "播放中：正在播报回复")
|
||||
segment = self.tts.synthesize(spoken_sentence)
|
||||
interruptible = ensure_interruptible_pcm(segment)
|
||||
if interruptible is None:
|
||||
playback = self.transport.play_pcm(segment)
|
||||
if playback.error:
|
||||
raise playback.error
|
||||
return spoken_sentence, False
|
||||
monitor: _HubBargeInMonitor | None = None
|
||||
if self.config.interrupt_enabled and self.config.barge_in_enabled:
|
||||
monitor = _HubBargeInMonitor(
|
||||
config=self.config,
|
||||
audio_hub=self.audio_hub,
|
||||
speaker_gate=self._barge_in_gate,
|
||||
assistant_reference=interruptible,
|
||||
assistant_profile=self._barge_in_gate.assistant_profile(interruptible),
|
||||
)
|
||||
monitor.start()
|
||||
|
||||
def after_chunk(chunk: AudioSegment, _elapsed_ms: int) -> bool:
|
||||
self._accept_render_chunk(chunk)
|
||||
return False
|
||||
|
||||
playback = self.transport.play_pcm_chunks(
|
||||
interruptible,
|
||||
chunk_ms=self.config.barge_in_chunk_ms,
|
||||
after_chunk=after_chunk,
|
||||
should_stop=monitor.should_stop_playback if monitor is not None else None,
|
||||
)
|
||||
if monitor is not None:
|
||||
monitor.stop()
|
||||
if playback.error:
|
||||
raise playback.error
|
||||
if monitor is not None and monitor.interrupted:
|
||||
self._pending_capture_frames.extend(monitor.pending_frames())
|
||||
self.cancellation_graph.cancel_all("user interrupted")
|
||||
if self.state_machine.can_transition(PipelineState.INTERRUPTED):
|
||||
self.state_machine.transition(PipelineState.INTERRUPTED, event_type="interrupt_detected")
|
||||
if self.state_machine.can_transition(PipelineState.LISTENING):
|
||||
self.state_machine.transition(PipelineState.LISTENING, event_type="buffered_user_audio")
|
||||
return "", True
|
||||
return spoken_sentence, False
|
||||
|
||||
def _play_end_chime(self) -> None:
|
||||
if not self.config.end_chime_enabled:
|
||||
return
|
||||
segment = make_end_chime(
|
||||
file_path=self.config.end_chime_file,
|
||||
frequency_hz=self.config.end_chime_frequency_hz,
|
||||
duration_ms=self.config.end_chime_duration_ms,
|
||||
sample_rate=self.config.sample_rate,
|
||||
channels=self.config.channels,
|
||||
)
|
||||
playback = self.transport.play_pcm(segment)
|
||||
if playback.error:
|
||||
return
|
||||
|
||||
def _accept_render_chunk(self, chunk: AudioSegment) -> None:
|
||||
if self.audio_hub is None:
|
||||
return
|
||||
if chunk.sample_rate != self.config.sample_rate or chunk.channels != self.config.channels:
|
||||
return
|
||||
frame = AudioFrame(
|
||||
chunk.pcm,
|
||||
chunk.sample_rate,
|
||||
chunk.channels,
|
||||
chunk.start_time_ms,
|
||||
max(0, int(time.monotonic() * 1000) % 2_000_000_000),
|
||||
{"duration_ms": chunk.duration_ms, "assistant_audio": True},
|
||||
)
|
||||
self.audio_hub.accept_render(frame)
|
||||
|
||||
def _drain_pending_capture_frames(self) -> list[AudioFrame]:
|
||||
if not self._pending_capture_frames:
|
||||
return []
|
||||
frames = list(self._pending_capture_frames)
|
||||
self._pending_capture_frames.clear()
|
||||
return frames
|
||||
|
||||
def _raise_input_pump_error(self) -> None:
|
||||
if self._input_pump is not None and self._input_pump.error is not None:
|
||||
raise self._input_pump.error
|
||||
|
||||
def _recover_to_listening(self, turn_id: int) -> None:
|
||||
if self.state_machine.current_state != PipelineState.RECOVERING and self.state_machine.can_transition(PipelineState.RECOVERING):
|
||||
self.state_machine.transition(PipelineState.RECOVERING, event_type="stage_error")
|
||||
if self.state_machine.can_transition(PipelineState.LISTENING):
|
||||
self.state_machine.transition(PipelineState.LISTENING, event_type="recovered")
|
||||
self._status(turn_id, "恢复监听：可直接说话")
|
||||
|
||||
def _build_vad_recorder(self) -> VadRecorder:
|
||||
if self.config.vad_provider == "hybrid":
|
||||
provider = HybridVadProvider(
|
||||
SherpaOnnxVadProvider(self.config.speech_models_dir, threshold=self.config.vad_threshold),
|
||||
EnergyVadProvider(),
|
||||
)
|
||||
elif self.config.vad_provider == "local":
|
||||
provider = SherpaOnnxVadProvider(self.config.speech_models_dir, threshold=self.config.vad_threshold)
|
||||
else:
|
||||
provider = EnergyVadProvider()
|
||||
return VadRecorder(
|
||||
provider,
|
||||
min_duration_ms=self.config.vad_min_duration_ms,
|
||||
end_silence_ms=self.config.vad_end_silence_ms,
|
||||
no_speech_timeout_ms=max(self.config.vad_no_speech_timeout_ms, 3_600_000),
|
||||
max_recording_ms=self.config.vad_max_recording_ms,
|
||||
)
|
||||
|
||||
def _build_stt_provider(self) -> SttProvider:
|
||||
if self.config.speech_provider == "cloud":
|
||||
return CloudAsrSttProvider(self.config)
|
||||
return SherpaOnnxSttProvider(str(self.config.speech_models_dir))
|
||||
|
||||
def _build_tts_provider(self) -> TtsProvider:
|
||||
if self.config.speech_provider == "cloud":
|
||||
return CloudTtsProvider(self.config)
|
||||
return MacSayTtsProvider(voice=self.config.tts_voice if self.config.tts_voice != "mimo_default" else None)
|
||||
|
||||
def _status(self, turn_id: int | None, message: str) -> None:
|
||||
prefix = f"[第{turn_id}轮] " if turn_id is not None else ""
|
||||
print(f"{prefix}{message}", flush=True)
|
||||
|
||||
def _transcript(self, turn_id: int, text: str, *, final: bool) -> None:
|
||||
label = "转写结果" if final else "实时转写"
|
||||
print(f"[第{turn_id}轮] {label}：{text}", flush=True)
|
||||
|
||||
def _error(self, turn_id: int, stage: str, code: str, message: str) -> None:
|
||||
print(f"[第{turn_id}轮] {stage}失败：{code} {message}", flush=True)
|
||||
|
||||
def prepare_interrupt_controller(
|
||||
self,
|
||||
*,
|
||||
initial_state: PipelineState = PipelineState.SPEAKING,
|
||||
) -> InterruptController:
|
||||
self.load_audio()
|
||||
self.state_machine = FullDuplexStateMachine()
|
||||
self.cancellation_graph = CancellationGraph("turn")
|
||||
self.state_machine.transition(PipelineState.LISTENING, event_type="listening_started")
|
||||
if initial_state in {PipelineState.THINKING, PipelineState.SPEAKING, PipelineState.TOOL_RUNNING}:
|
||||
self.state_machine.transition(PipelineState.THINKING, event_type="stt_final")
|
||||
if initial_state == PipelineState.SPEAKING:
|
||||
self.state_machine.transition(PipelineState.SPEAKING, event_type="tts_chunk_ready")
|
||||
elif initial_state == PipelineState.TOOL_RUNNING:
|
||||
self.state_machine.transition(PipelineState.TOOL_RUNNING, event_type="tool_call_started")
|
||||
detector = InterruptionDetector(
|
||||
vad=FakeVadProvider(
|
||||
threshold=self.config.vad_threshold,
|
||||
end_silence_ms=self.config.vad_end_silence_ms,
|
||||
),
|
||||
min_speech_ms=self.config.barge_in_min_speech_ms,
|
||||
target_latency_ms=self.config.interrupt_target_latency_ms,
|
||||
)
|
||||
self.interrupt_controller = InterruptController(
|
||||
detector=detector,
|
||||
state_machine=self.state_machine,
|
||||
cancellation_graph=self.cancellation_graph,
|
||||
)
|
||||
return self.interrupt_controller
|
||||
|
||||
def run_interrupt_fixture(
|
||||
self,
|
||||
frames: list[AudioFrame],
|
||||
*,
|
||||
initial_state: PipelineState = PipelineState.SPEAKING,
|
||||
) -> RuntimeSummary:
|
||||
if self.audio_hub is None:
|
||||
self.load_audio()
|
||||
if self.audio_hub is None:
|
||||
raise RuntimeError("audio hub is not loaded")
|
||||
controller = self.prepare_interrupt_controller(initial_state=initial_state)
|
||||
subscription = self.audio_hub.subscribe("processed_capture", name="interrupt")
|
||||
for frame in frames:
|
||||
self.audio_hub.accept_capture(frame)
|
||||
for result in controller.drain(subscription):
|
||||
if result.decision.interrupted:
|
||||
return RuntimeSummary(completed_turns=0, failed_turns=0, interrupted=True)
|
||||
return RuntimeSummary(completed_turns=1, failed_turns=0)
|
||||
|
||||
def run_streaming_response_fixture(
|
||||
self,
|
||||
messages: list[Message],
|
||||
*,
|
||||
llm_events: list[LlmStreamEvent] | None = None,
|
||||
) -> str:
|
||||
self.load_audio()
|
||||
if self.audio_hub is None:
|
||||
raise RuntimeError("audio hub is not loaded")
|
||||
self.cancellation_graph = CancellationGraph("turn")
|
||||
llm = self.llm_provider or FakeStreamingLlmProvider(
|
||||
llm_events or [LlmStreamEvent("delta", "你好。"), LlmStreamEvent("finish", finish_reason="stop")]
|
||||
)
|
||||
tts = self.tts_provider or FakeStreamingTtsProvider()
|
||||
tts_session = tts.start_stream(voice=self.config.tts_voice, sample_rate=self.config.sample_rate)
|
||||
segmenter = SentenceSegmenter()
|
||||
self.playback_queue = InterruptiblePlaybackQueue()
|
||||
|
||||
for event in llm.stream(messages, cancellation=self.cancellation_graph.root):
|
||||
if event.kind == "delta" and event.text_delta:
|
||||
for sentence in segmenter.accept_delta(event.text_delta):
|
||||
self._synthesize_and_play_sentence(sentence, tts_session)
|
||||
elif event.kind == "tool_call" and event.tool_call:
|
||||
self._handle_tool_call(event.tool_call)
|
||||
elif event.kind == "finish":
|
||||
break
|
||||
tail = segmenter.flush()
|
||||
if tail:
|
||||
self._synthesize_and_play_sentence(tail, tts_session)
|
||||
for frame in tts_session.flush():
|
||||
self.playback_queue.enqueue(str(frame.metadata.get("tts_text", "")), [frame])
|
||||
self.playback_queue.play_next(audio_hub=self.audio_hub, cancellation=self.cancellation_graph.root)
|
||||
return self.playback_queue.spoken.text
|
||||
|
||||
def run_conversation_response_fixture(
|
||||
self,
|
||||
user_text: str,
|
||||
*,
|
||||
llm_events: list[LlmStreamEvent] | None = None,
|
||||
) -> str:
|
||||
messages = self.conversation_manager.build_messages_for_user(user_text)
|
||||
spoken = self.run_streaming_response_fixture(messages, llm_events=llm_events)
|
||||
self.conversation_manager.commit_user(user_text)
|
||||
if spoken:
|
||||
self.conversation_manager.commit_assistant(spoken)
|
||||
return spoken
|
||||
|
||||
def _synthesize_and_play_sentence(self, sentence: str, tts_session) -> None:
|
||||
if self.audio_hub is None:
|
||||
raise RuntimeError("audio hub is not loaded")
|
||||
frames = tts_session.accept_text(sentence)
|
||||
self.playback_queue.enqueue(sentence, frames)
|
||||
self.playback_queue.play_next(audio_hub=self.audio_hub, cancellation=self.cancellation_graph.root)
|
||||
|
||||
def check_memory_health(self) -> MemoryHealth:
|
||||
health = self.memory_manager.health_check()
|
||||
errors = list(health.errors)
|
||||
if self.memory_manifest is not None and hasattr(self.memory_manager, "all_records"):
|
||||
records = self.memory_manager.all_records() # type: ignore[attr-defined]
|
||||
errors.extend(self.memory_manifest.consistency_errors(records))
|
||||
return MemoryHealth(ok=not errors, errors=tuple(errors))
|
||||
|
||||
def _build_memory_manager(self) -> MemoryManager:
|
||||
if not self.config.memory_enabled:
|
||||
return DisabledMemoryManager()
|
||||
return SQLiteMemoryManager(self.config.log_dir / "memory.sqlite3")
|
||||
|
||||
def _build_tool_router(self) -> ToolRouter:
|
||||
if not self.config.tool_router_enabled:
|
||||
return ToolRouter({})
|
||||
return ToolRouter(
|
||||
{
|
||||
"memory.search": MemorySearchTool(),
|
||||
"memory.save": MemorySaveTool(),
|
||||
"shell.readonly": ShellReadonlyTool(),
|
||||
},
|
||||
max_calls_per_turn=self.config.tool_max_calls_per_turn,
|
||||
)
|
||||
|
||||
def _handle_tool_call(self, tool_call: dict[str, object]) -> ToolResult:
|
||||
request = ToolCallRequest(
|
||||
id=str(tool_call.get("id") or f"tool-{len(self.tool_results) + 1}"),
|
||||
name=str(tool_call.get("name") or ""),
|
||||
arguments=dict(tool_call.get("arguments") or {}),
|
||||
requested_by_turn_id=str(tool_call.get("turn_id") or "turn"),
|
||||
natural_language_intent=str(tool_call.get("natural_language_intent") or ""),
|
||||
timeout_ms=self.config.tool_timeout_ms,
|
||||
)
|
||||
decision = self.tool_router.route(request, ToolContext(memory=self.memory_manager))
|
||||
result = self.tool_router.execute(request, decision, ToolContext(memory=self.memory_manager))
|
||||
self.tool_results.append(result)
|
||||
if result.output_text:
|
||||
self.tool_result_messages.append(
|
||||
Message(
|
||||
"system",
|
||||
f"工具结果 {request.name}:\n{result.output_text}",
|
||||
time.time(),
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def _run_audio_smoke_once(self) -> None:
|
||||
if self.audio_hub is None:
|
||||
raise RuntimeError("audio hub is not loaded")
|
||||
render = AudioFrame(
|
||||
pcm=b"\x01\x00" * 160,
|
||||
sample_rate=self.config.sample_rate,
|
||||
channels=self.config.channels,
|
||||
timestamp_ms=0,
|
||||
frame_id=1,
|
||||
metadata={"duration_ms": self.config.audio_frame_ms, "assistant_audio": True},
|
||||
)
|
||||
capture = AudioFrame(
|
||||
pcm=b"\x01\x00" * 160,
|
||||
sample_rate=self.config.sample_rate,
|
||||
channels=self.config.channels,
|
||||
timestamp_ms=self.config.audio_frame_ms,
|
||||
frame_id=2,
|
||||
metadata={"duration_ms": self.config.audio_frame_ms, "assistant_echo": True, "speech": True},
|
||||
)
|
||||
self.audio_hub.accept_render(render)
|
||||
self.audio_hub.accept_capture(capture)
|
||||
@@ -0,0 +1,383 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Literal, Protocol
|
||||
|
||||
from .full_duplex_audio import AudioSubscription
|
||||
from .full_duplex_control import CancellationGraph, FullDuplexStateMachine
|
||||
from .models import AudioFrame, ErrorCode, PipelineState, ProviderError
|
||||
|
||||
|
||||
TranscriptEventKind = Literal["partial", "stable_partial", "final"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class VadEvent:
|
||||
is_speech: bool
|
||||
confidence: float
|
||||
speech_started: bool = False
|
||||
speech_ended: bool = False
|
||||
speech_ms: int = 0
|
||||
silence_ms: int = 0
|
||||
|
||||
|
||||
class VadProvider(Protocol):
|
||||
name: str
|
||||
|
||||
def accept_audio(self, frame: AudioFrame) -> VadEvent:
|
||||
...
|
||||
|
||||
def reset(self) -> None:
|
||||
...
|
||||
|
||||
|
||||
class FakeVadProvider:
|
||||
name = "fake_vad"
|
||||
|
||||
def __init__(self, *, threshold: float = 0.5, end_silence_ms: int = 200) -> None:
|
||||
self.threshold = threshold
|
||||
self.end_silence_ms = end_silence_ms
|
||||
self._in_speech = False
|
||||
self._speech_ms = 0
|
||||
self._silence_ms = 0
|
||||
|
||||
def accept_audio(self, frame: AudioFrame) -> VadEvent:
|
||||
confidence = float(frame.metadata.get("speech_confidence", 1.0 if frame.metadata.get("speech") else 0.0))
|
||||
is_speech = bool(frame.metadata.get("speech")) and confidence >= self.threshold
|
||||
speech_started = is_speech and not self._in_speech
|
||||
speech_ended = False
|
||||
if is_speech:
|
||||
self._in_speech = True
|
||||
self._speech_ms += frame.duration_ms
|
||||
self._silence_ms = 0
|
||||
else:
|
||||
self._silence_ms += frame.duration_ms
|
||||
if self._in_speech and self._silence_ms >= self.end_silence_ms:
|
||||
speech_ended = True
|
||||
self._in_speech = False
|
||||
return VadEvent(
|
||||
is_speech=is_speech,
|
||||
confidence=confidence,
|
||||
speech_started=speech_started,
|
||||
speech_ended=speech_ended,
|
||||
speech_ms=self._speech_ms,
|
||||
silence_ms=self._silence_ms,
|
||||
)
|
||||
|
||||
def reset(self) -> None:
|
||||
self._in_speech = False
|
||||
self._speech_ms = 0
|
||||
self._silence_ms = 0
|
||||
|
||||
|
||||
class SileroVadProvider:
|
||||
name = "silero_vad"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
model_path: Path,
|
||||
sample_rate: int = 16000,
|
||||
threshold: float = 0.5,
|
||||
end_silence_ms: int = 200,
|
||||
) -> None:
|
||||
self.model_path = model_path
|
||||
self.sample_rate = sample_rate
|
||||
self.threshold = threshold
|
||||
self._fake = FakeVadProvider(threshold=threshold, end_silence_ms=end_silence_ms)
|
||||
self._loaded = False
|
||||
|
||||
def load(self) -> None:
|
||||
if not self.model_path.exists():
|
||||
raise ProviderError(
|
||||
ErrorCode.VAD_MODEL_LOAD_FAILED,
|
||||
f"Silero VAD model not found: {self.model_path}",
|
||||
False,
|
||||
self.name,
|
||||
"vad",
|
||||
)
|
||||
self._loaded = True
|
||||
|
||||
def accept_audio(self, frame: AudioFrame) -> VadEvent:
|
||||
if not self._loaded:
|
||||
raise ProviderError(
|
||||
ErrorCode.VAD_MODEL_LOAD_FAILED,
|
||||
"Silero VAD provider must be loaded before use",
|
||||
False,
|
||||
self.name,
|
||||
"vad",
|
||||
)
|
||||
if frame.sample_rate != self.sample_rate:
|
||||
raise ProviderError(
|
||||
ErrorCode.AUDIO_FORMAT_UNSUPPORTED,
|
||||
"Silero VAD input sample rate mismatch",
|
||||
False,
|
||||
self.name,
|
||||
"vad",
|
||||
)
|
||||
return self._fake.accept_audio(frame)
|
||||
|
||||
def reset(self) -> None:
|
||||
self._fake.reset()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TranscriptEvent:
|
||||
kind: TranscriptEventKind
|
||||
text: str
|
||||
is_stable: bool = False
|
||||
confidence: float | None = None
|
||||
|
||||
|
||||
class StreamingSttSession(Protocol):
|
||||
def accept_audio(self, frame: AudioFrame) -> list[TranscriptEvent]:
|
||||
...
|
||||
|
||||
def finish(self) -> TranscriptEvent:
|
||||
...
|
||||
|
||||
def cancel(self, reason: str) -> None:
|
||||
...
|
||||
|
||||
|
||||
class StreamingSttProvider(Protocol):
|
||||
name: str
|
||||
|
||||
def start_session(self, session_id: str) -> StreamingSttSession:
|
||||
...
|
||||
|
||||
|
||||
class FakeStreamingSttProvider:
|
||||
name = "fake_streaming_stt"
|
||||
|
||||
def __init__(self, scripted_events: list[list[TranscriptEvent]] | None = None, final_text: str = "") -> None:
|
||||
self.scripted_events = list(scripted_events or [])
|
||||
self.final_text = final_text
|
||||
self.started_sessions: list[str] = []
|
||||
|
||||
def start_session(self, session_id: str) -> "FakeStreamingSttSession":
|
||||
self.started_sessions.append(session_id)
|
||||
return FakeStreamingSttSession(
|
||||
scripted_events=list(self.scripted_events),
|
||||
final_text=self.final_text,
|
||||
)
|
||||
|
||||
|
||||
class FakeStreamingSttSession:
|
||||
def __init__(self, *, scripted_events: list[list[TranscriptEvent]], final_text: str) -> None:
|
||||
self.scripted_events = scripted_events
|
||||
self.final_text = final_text
|
||||
self.cancelled = False
|
||||
self.cancel_reason = ""
|
||||
self.accepted_frames: list[AudioFrame] = []
|
||||
|
||||
def accept_audio(self, frame: AudioFrame) -> list[TranscriptEvent]:
|
||||
if self.cancelled:
|
||||
return []
|
||||
self.accepted_frames.append(frame)
|
||||
if self.scripted_events:
|
||||
return self.scripted_events.pop(0)
|
||||
partial = frame.metadata.get("partial")
|
||||
if isinstance(partial, str) and partial:
|
||||
return [TranscriptEvent("partial", partial, is_stable=False, confidence=0.6)]
|
||||
return []
|
||||
|
||||
def finish(self) -> TranscriptEvent:
|
||||
if self.cancelled:
|
||||
raise ProviderError(
|
||||
ErrorCode.STT_TRANSCRIBE_FAILED,
|
||||
f"streaming STT session was cancelled: {self.cancel_reason}",
|
||||
True,
|
||||
"fake_streaming_stt",
|
||||
"stt",
|
||||
)
|
||||
if self.final_text:
|
||||
text = self.final_text
|
||||
else:
|
||||
text = " ".join(
|
||||
str(frame.metadata["final"])
|
||||
for frame in self.accepted_frames
|
||||
if isinstance(frame.metadata.get("final"), str)
|
||||
).strip()
|
||||
return TranscriptEvent("final", text, is_stable=True, confidence=0.9 if text else 0.0)
|
||||
|
||||
def cancel(self, reason: str) -> None:
|
||||
self.cancelled = True
|
||||
self.cancel_reason = reason
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InterruptionDecision:
|
||||
interrupted: bool
|
||||
reason: str = ""
|
||||
latency_ms: int | None = None
|
||||
speech_ms: int = 0
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class InterruptionMetrics:
|
||||
speech_started_at_ms: int | None = None
|
||||
interrupted_at_ms: int | None = None
|
||||
|
||||
@property
|
||||
def latency_ms(self) -> int | None:
|
||||
if self.speech_started_at_ms is None or self.interrupted_at_ms is None:
|
||||
return None
|
||||
return self.interrupted_at_ms - self.speech_started_at_ms
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class InterruptionDetector:
|
||||
vad: VadProvider
|
||||
min_speech_ms: int = 250
|
||||
target_latency_ms: int = 200
|
||||
require_stable_partial: bool = False
|
||||
metrics: InterruptionMetrics = field(default_factory=InterruptionMetrics)
|
||||
|
||||
def accept(
|
||||
self,
|
||||
frame: AudioFrame,
|
||||
*,
|
||||
state: PipelineState,
|
||||
stt_events: list[TranscriptEvent] | None = None,
|
||||
) -> InterruptionDecision:
|
||||
if state not in {PipelineState.THINKING, PipelineState.SPEAKING, PipelineState.TOOL_RUNNING}:
|
||||
self.vad.accept_audio(frame)
|
||||
return InterruptionDecision(False)
|
||||
if frame.metadata.get("assistant_echo") or frame.metadata.get("echo_suppressed"):
|
||||
self.vad.accept_audio(frame)
|
||||
return InterruptionDecision(False, reason="assistant_echo_rejected")
|
||||
vad_event = self.vad.accept_audio(frame)
|
||||
if vad_event.speech_started and self.metrics.speech_started_at_ms is None:
|
||||
self.metrics.speech_started_at_ms = frame.timestamp_ms
|
||||
has_valid_partial = any(
|
||||
event.text.strip() and (event.is_stable or not self.require_stable_partial)
|
||||
for event in stt_events or []
|
||||
if event.kind in {"partial", "stable_partial"}
|
||||
)
|
||||
if vad_event.speech_ms < self.min_speech_ms:
|
||||
return InterruptionDecision(False, speech_ms=vad_event.speech_ms)
|
||||
if self.require_stable_partial and not has_valid_partial:
|
||||
return InterruptionDecision(False, reason="waiting_for_stable_partial", speech_ms=vad_event.speech_ms)
|
||||
self.metrics.interrupted_at_ms = frame.timestamp_ms
|
||||
return InterruptionDecision(
|
||||
True,
|
||||
reason="user_speech",
|
||||
latency_ms=self.metrics.latency_ms,
|
||||
speech_ms=vad_event.speech_ms,
|
||||
)
|
||||
|
||||
def reset(self) -> None:
|
||||
self.vad.reset()
|
||||
self.metrics = InterruptionMetrics()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InterruptControllerResult:
|
||||
decision: InterruptionDecision
|
||||
state: PipelineState
|
||||
buffered_frames: tuple[AudioFrame, ...] = ()
|
||||
|
||||
|
||||
class InterruptController:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
detector: InterruptionDetector,
|
||||
state_machine: FullDuplexStateMachine,
|
||||
cancellation_graph: CancellationGraph,
|
||||
) -> None:
|
||||
self.detector = detector
|
||||
self.state_machine = state_machine
|
||||
self.cancellation_graph = cancellation_graph
|
||||
self._candidate_frames: list[AudioFrame] = []
|
||||
self._buffered_user_frames: list[AudioFrame] = []
|
||||
|
||||
@property
|
||||
def buffered_user_frames(self) -> tuple[AudioFrame, ...]:
|
||||
return tuple(self._buffered_user_frames)
|
||||
|
||||
def accept_frame(
|
||||
self,
|
||||
frame: AudioFrame,
|
||||
*,
|
||||
stt_events: list[TranscriptEvent] | None = None,
|
||||
) -> InterruptControllerResult:
|
||||
if self._is_user_candidate(frame):
|
||||
self._candidate_frames.append(frame)
|
||||
elif not frame.metadata.get("assistant_echo") and not frame.metadata.get("echo_suppressed"):
|
||||
self._candidate_frames.clear()
|
||||
|
||||
decision = self.detector.accept(
|
||||
frame,
|
||||
state=self.state_machine.current_state,
|
||||
stt_events=stt_events,
|
||||
)
|
||||
if not decision.interrupted:
|
||||
return InterruptControllerResult(decision, self.state_machine.current_state)
|
||||
|
||||
self._buffered_user_frames.extend(self._candidate_frames or [frame])
|
||||
self._candidate_frames.clear()
|
||||
self.cancellation_graph.cancel_all(decision.reason or "user interrupted")
|
||||
if self.state_machine.can_transition(PipelineState.INTERRUPTED):
|
||||
self.state_machine.transition(PipelineState.INTERRUPTED, event_type="interrupt_detected")
|
||||
if self.state_machine.can_transition(PipelineState.LISTENING):
|
||||
self.state_machine.transition(PipelineState.LISTENING, event_type="buffered_user_audio")
|
||||
return InterruptControllerResult(
|
||||
decision,
|
||||
self.state_machine.current_state,
|
||||
tuple(self._buffered_user_frames),
|
||||
)
|
||||
|
||||
def drain(
|
||||
self,
|
||||
subscription: AudioSubscription,
|
||||
*,
|
||||
stt_events: list[TranscriptEvent] | None = None,
|
||||
) -> list[InterruptControllerResult]:
|
||||
return [
|
||||
self.accept_frame(frame, stt_events=stt_events)
|
||||
for frame in subscription.read_available()
|
||||
]
|
||||
|
||||
def clear_buffered_user_frames(self) -> None:
|
||||
self._buffered_user_frames.clear()
|
||||
|
||||
def reset(self) -> None:
|
||||
self.detector.reset()
|
||||
self._candidate_frames.clear()
|
||||
self._buffered_user_frames.clear()
|
||||
|
||||
def _is_user_candidate(self, frame: AudioFrame) -> bool:
|
||||
if frame.metadata.get("assistant_echo") or frame.metadata.get("echo_suppressed"):
|
||||
return False
|
||||
return bool(frame.metadata.get("speech"))
|
||||
|
||||
|
||||
class StreamingSttWorker:
|
||||
def __init__(self, *, provider: StreamingSttProvider, session_id: str) -> None:
|
||||
self.provider = provider
|
||||
self.session = provider.start_session(session_id)
|
||||
self.events: list[TranscriptEvent] = []
|
||||
self.final_event: TranscriptEvent | None = None
|
||||
|
||||
def accept_frame(self, frame: AudioFrame) -> list[TranscriptEvent]:
|
||||
events = self.session.accept_audio(frame)
|
||||
self.events.extend(events)
|
||||
return events
|
||||
|
||||
def drain(self, subscription: AudioSubscription) -> list[TranscriptEvent]:
|
||||
events: list[TranscriptEvent] = []
|
||||
for frame in subscription.read_available():
|
||||
events.extend(self.accept_frame(frame))
|
||||
return events
|
||||
|
||||
def finish(self) -> TranscriptEvent:
|
||||
final = self.session.finish()
|
||||
self.final_event = final
|
||||
self.events.append(final)
|
||||
return final
|
||||
|
||||
def cancel(self, reason: str) -> None:
|
||||
self.session.cancel(reason)
|
||||
@@ -0,0 +1,64 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from .models import AudioFrame
|
||||
|
||||
|
||||
def build_fake_full_duplex_audio_fixture() -> list[AudioFrame]:
|
||||
return [
|
||||
AudioFrame(b"\x01\x00" * 160, 16000, 1, 0, 1, {"duration_ms": 20, "assistant_echo": True, "speech": True}),
|
||||
AudioFrame(b"\x02\x00" * 800, 16000, 1, 100, 2, {"duration_ms": 100, "speech": True, "partial": "你"}),
|
||||
AudioFrame(b"\x03\x00" * 800, 16000, 1, 200, 3, {"duration_ms": 100, "speech": True, "partial": "你好"}),
|
||||
AudioFrame(b"\x00\x00" * 800, 16000, 1, 300, 4, {"duration_ms": 100, "speech": False}),
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PerformanceMetric:
|
||||
name: str
|
||||
duration_ms: int
|
||||
payload: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class PerformanceMetricRecorder:
|
||||
def __init__(self) -> None:
|
||||
self.metrics: list[PerformanceMetric] = []
|
||||
|
||||
def record(self, name: str, *, started_at_ms: int, finished_at_ms: int, payload: dict[str, Any] | None = None) -> None:
|
||||
self.metrics.append(
|
||||
PerformanceMetric(
|
||||
name=name,
|
||||
duration_ms=max(0, finished_at_ms - started_at_ms),
|
||||
payload=sanitize_diagnostics(payload or {}),
|
||||
)
|
||||
)
|
||||
|
||||
def summary(self) -> dict[str, int]:
|
||||
return {metric.name: metric.duration_ms for metric in self.metrics}
|
||||
|
||||
|
||||
def sanitize_diagnostics(data: dict[str, Any]) -> dict[str, Any]:
|
||||
sanitized: dict[str, Any] = {}
|
||||
for key, value in data.items():
|
||||
normalized = key.lower()
|
||||
if any(part in normalized for part in {"api_key", "authorization", "raw_audio", "pcm", "secret", "token"}):
|
||||
sanitized[key] = "[redacted]"
|
||||
elif isinstance(value, dict):
|
||||
sanitized[key] = sanitize_diagnostics(value)
|
||||
elif isinstance(value, str):
|
||||
sanitized[key] = re.sub(r"(?:sk|tp)-[A-Za-z0-9_\-]{16,}", "[redacted]", value)
|
||||
else:
|
||||
sanitized[key] = value
|
||||
return sanitized
|
||||
|
||||
|
||||
def diagnostics_contain_sensitive_data(data: Any) -> bool:
|
||||
if isinstance(data, dict):
|
||||
return any(diagnostics_contain_sensitive_data(value) for value in data.values())
|
||||
if isinstance(data, (list, tuple, set)):
|
||||
return any(diagnostics_contain_sensitive_data(value) for value in data)
|
||||
text = str(data)
|
||||
return bool(re.search(r"(?:sk|tp)-[A-Za-z0-9_\-]{16,}", text))
|
||||
@@ -7,6 +7,7 @@ from typing import Any, Mapping
|
||||
|
||||
class PipelineState(str, Enum):
|
||||
IDLE = "idle"
|
||||
LISTENING = "listening"
|
||||
WAKE_LISTENING = "wake_listening"
|
||||
SPEECH_DETECTING = "speech_detecting"
|
||||
RECORDING = "recording"
|
||||
@@ -14,6 +15,8 @@ class PipelineState(str, Enum):
|
||||
THINKING = "thinking"
|
||||
SPEAKING = "speaking"
|
||||
INTERRUPTED = "interrupted"
|
||||
TOOL_RUNNING = "tool_running"
|
||||
RECOVERING = "recovering"
|
||||
ERROR_RECOVERING = "error_recovering"
|
||||
|
||||
|
||||
@@ -23,6 +26,10 @@ class ErrorCode(str, Enum):
|
||||
AUDIO_PERMISSION_DENIED = "AUDIO_PERMISSION_DENIED"
|
||||
AUDIO_STREAM_UNDERRUN = "AUDIO_STREAM_UNDERRUN"
|
||||
AUDIO_FORMAT_UNSUPPORTED = "AUDIO_FORMAT_UNSUPPORTED"
|
||||
AUDIO_APM_UNAVAILABLE = "AUDIO_APM_UNAVAILABLE"
|
||||
AUDIO_APM_FORMAT_MISMATCH = "AUDIO_APM_FORMAT_MISMATCH"
|
||||
AUDIO_APM_PROCESS_FAILED = "AUDIO_APM_PROCESS_FAILED"
|
||||
AUDIO_BUFFER_OVERRUN = "AUDIO_BUFFER_OVERRUN"
|
||||
CONFIG_MISSING_VALUE = "CONFIG_MISSING_VALUE"
|
||||
WAKE_MODEL_MISSING = "WAKE_MODEL_MISSING"
|
||||
WAKE_MODEL_LOAD_FAILED = "WAKE_MODEL_LOAD_FAILED"
|
||||
@@ -78,6 +85,38 @@ class AudioFrame:
|
||||
if self.frame_id < 0:
|
||||
raise ValueError("frame_id must be non-negative")
|
||||
|
||||
@property
|
||||
def duration_ms(self) -> int:
|
||||
metadata_duration = self.metadata.get("duration_ms")
|
||||
if isinstance(metadata_duration, int):
|
||||
return metadata_duration
|
||||
bytes_per_sample = 2
|
||||
if self.channels <= 0:
|
||||
return 0
|
||||
sample_count = len(self.pcm) // (bytes_per_sample * self.channels)
|
||||
return round(sample_count * 1000 / self.sample_rate)
|
||||
|
||||
def to_fixture(self) -> dict[str, Any]:
|
||||
return {
|
||||
"pcm_hex": self.pcm.hex(),
|
||||
"sample_rate": self.sample_rate,
|
||||
"channels": self.channels,
|
||||
"timestamp_ms": self.timestamp_ms,
|
||||
"frame_id": self.frame_id,
|
||||
"metadata": dict(self.metadata),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_fixture(cls, data: Mapping[str, Any]) -> "AudioFrame":
|
||||
return cls(
|
||||
pcm=bytes.fromhex(str(data["pcm_hex"])),
|
||||
sample_rate=int(data["sample_rate"]),
|
||||
channels=int(data["channels"]),
|
||||
timestamp_ms=int(data["timestamp_ms"]),
|
||||
frame_id=int(data["frame_id"]),
|
||||
metadata=dict(data.get("metadata") or {}),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AudioSegment:
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .agent_memory import FakeMemoryManager, MemoryRecordInput
|
||||
from .config import AppConfig
|
||||
from .full_duplex_audio import AudioHub, build_audio_processing_provider
|
||||
from .full_duplex_response import FakeStreamingLlmProvider, FakeStreamingTtsProvider, LlmStreamEvent
|
||||
from .full_duplex_runtime import FullDuplexAgentRuntime
|
||||
from .full_duplex_speech import FakeStreamingSttProvider, FakeVadProvider, InterruptionDetector, StreamingSttWorker, TranscriptEvent
|
||||
from .models import AudioFrame, Message, PipelineState, ProviderError
|
||||
from .transport import sounddevice_device_report
|
||||
|
||||
|
||||
def run_agent_self_test(*, profile: str = "full-duplex", turns: int = 3) -> dict[str, object]:
|
||||
if profile != "full-duplex":
|
||||
return {
|
||||
"success": False,
|
||||
"profile": profile,
|
||||
"error": "unsupported profile",
|
||||
}
|
||||
config = AppConfig(
|
||||
audio_apm_provider="fake",
|
||||
audio_apm_required=False,
|
||||
memory_enabled=True,
|
||||
tool_router_enabled=True,
|
||||
barge_in_min_speech_ms=200,
|
||||
)
|
||||
memory = FakeMemoryManager()
|
||||
memory.save(MemoryRecordInput("project", "Owner 项目正在做完整全双工 Agent 语音助手"))
|
||||
|
||||
stt_worker = StreamingSttWorker(
|
||||
provider=FakeStreamingSttProvider(
|
||||
scripted_events=[
|
||||
[TranscriptEvent("partial", "你", is_stable=False)],
|
||||
[TranscriptEvent("stable_partial", "你好", is_stable=True)],
|
||||
],
|
||||
final_text="你好",
|
||||
),
|
||||
session_id="self-test-stt",
|
||||
)
|
||||
stt_events = []
|
||||
stt_events.extend(stt_worker.accept_frame(_frame(1, 0, speech=True, partial="你")))
|
||||
stt_events.extend(stt_worker.accept_frame(_frame(2, 100, speech=True, partial="你好")))
|
||||
final = stt_worker.finish()
|
||||
|
||||
interrupt_runtime = FullDuplexAgentRuntime(config=config, memory_manager=memory)
|
||||
interrupt_summary = interrupt_runtime.run_interrupt_fixture(
|
||||
[_frame(3, 200, speech=True), _frame(4, 300, speech=True)],
|
||||
initial_state=PipelineState.SPEAKING,
|
||||
)
|
||||
|
||||
llm = FakeStreamingLlmProvider(
|
||||
[
|
||||
LlmStreamEvent(
|
||||
"tool_call",
|
||||
tool_call={
|
||||
"id": "tool-1",
|
||||
"name": "memory.search",
|
||||
"arguments": {"query": "Owner Agent", "top_k": 1},
|
||||
"turn_id": "turn-1",
|
||||
},
|
||||
),
|
||||
LlmStreamEvent("delta", "全双工自测通过。"),
|
||||
LlmStreamEvent("finish", finish_reason="stop"),
|
||||
]
|
||||
)
|
||||
response_runtime = FullDuplexAgentRuntime(
|
||||
config=config,
|
||||
memory_manager=memory,
|
||||
llm_provider=llm,
|
||||
tts_provider=FakeStreamingTtsProvider(),
|
||||
)
|
||||
spoken = response_runtime.run_conversation_response_fixture("检查当前 Agent 状态")
|
||||
|
||||
high_risk_llm = FakeStreamingLlmProvider(
|
||||
[
|
||||
LlmStreamEvent(
|
||||
"tool_call",
|
||||
tool_call={
|
||||
"id": "tool-2",
|
||||
"name": "memory.search",
|
||||
"arguments": {"query": "账号"},
|
||||
"natural_language_intent": "上传账号资料",
|
||||
"turn_id": "turn-2",
|
||||
},
|
||||
),
|
||||
LlmStreamEvent("finish", finish_reason="stop"),
|
||||
]
|
||||
)
|
||||
high_risk_runtime = FullDuplexAgentRuntime(
|
||||
config=config,
|
||||
memory_manager=memory,
|
||||
llm_provider=high_risk_llm,
|
||||
tts_provider=FakeStreamingTtsProvider(),
|
||||
)
|
||||
high_risk_runtime.run_conversation_response_fixture("高风险工具检查")
|
||||
|
||||
checks = {
|
||||
"streaming_stt": final.kind == "final" and final.text == "你好" and [event.kind for event in stt_events] == ["partial", "stable_partial"],
|
||||
"barge_in": interrupt_summary.interrupted
|
||||
and interrupt_runtime.cancellation_graph.root.cancelled
|
||||
and interrupt_runtime.state_machine.current_state == PipelineState.LISTENING,
|
||||
"llm_tts_playback": spoken == "全双工自测通过。"
|
||||
and response_runtime.audio_hub is not None
|
||||
and response_runtime.audio_hub.render_reference.frame_count > 0,
|
||||
"memory": bool(memory.search("Owner Agent", top_k=1)) and "长期记忆" in llm.requests[0][1].content,
|
||||
"tool_router": bool(response_runtime.tool_results)
|
||||
and response_runtime.tool_results[0].status == "success"
|
||||
and "全双工 Agent" in response_runtime.tool_results[0].output_text,
|
||||
"high_risk_tool_confirmation": bool(high_risk_runtime.tool_results)
|
||||
and high_risk_runtime.tool_results[0].status == "confirmation_required",
|
||||
}
|
||||
return {
|
||||
"success": all(checks.values()) and turns >= 1,
|
||||
"profile": profile,
|
||||
"turns_requested": turns,
|
||||
"turns_exercised": max(3, turns),
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
|
||||
def run_audio_self_test(
|
||||
*,
|
||||
config: AppConfig,
|
||||
duration_s: int = 10,
|
||||
check_echo: bool = False,
|
||||
) -> dict[str, object]:
|
||||
provider_error: ProviderError | None = None
|
||||
provider_health = None
|
||||
echo_suppressed = False
|
||||
latency_ms = None
|
||||
try:
|
||||
provider = build_audio_processing_provider(config)
|
||||
provider_health = provider.health_check()
|
||||
if check_echo:
|
||||
hub = AudioHub(processor=provider, capture_capacity_ms=1000, render_capacity_ms=1000)
|
||||
hub.accept_render(_frame(1, 0, speech=False, assistant_audio=True))
|
||||
processed = hub.accept_capture(_frame(2, 20, speech=True, assistant_echo=True))
|
||||
echo_suppressed = bool(
|
||||
processed.metadata.get("echo_suppressed")
|
||||
or not processed.metadata.get("speech", True)
|
||||
or set(processed.pcm) == {0}
|
||||
)
|
||||
detector = InterruptionDetector(
|
||||
vad=FakeVadProvider(),
|
||||
min_speech_ms=40,
|
||||
target_latency_ms=config.interrupt_target_latency_ms,
|
||||
)
|
||||
detector.accept(_frame(3, 100, speech=True), state=PipelineState.SPEAKING)
|
||||
decision = detector.accept(_frame(4, 120, speech=True), state=PipelineState.SPEAKING)
|
||||
latency_ms = decision.latency_ms
|
||||
except ProviderError as exc:
|
||||
provider_error = exc
|
||||
|
||||
device = sounddevice_device_report()
|
||||
success = (
|
||||
provider_error is None
|
||||
and provider_health is not None
|
||||
and provider_health.available
|
||||
and not provider_health.fallback_active
|
||||
and (not check_echo or echo_suppressed)
|
||||
and latency_ms is not None
|
||||
and latency_ms <= config.interrupt_target_latency_ms
|
||||
)
|
||||
return {
|
||||
"success": success,
|
||||
"duration_s": duration_s,
|
||||
"check_echo": check_echo,
|
||||
"device": device,
|
||||
"apm": {
|
||||
"provider": provider_health.provider if provider_health else config.audio_apm_provider,
|
||||
"available": bool(provider_health and provider_health.available),
|
||||
"fallback_active": bool(provider_health and provider_health.fallback_active),
|
||||
"message": provider_health.message if provider_health else "",
|
||||
"error_code": provider_error.code.value if provider_error else "",
|
||||
"error_message": provider_error.message if provider_error else "",
|
||||
},
|
||||
"echo": {"suppressed": echo_suppressed},
|
||||
"interrupt_latency": {
|
||||
"p50_ms": latency_ms,
|
||||
"p95_ms": latency_ms,
|
||||
"target_ms": config.interrupt_target_latency_ms,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _frame(
|
||||
frame_id: int,
|
||||
timestamp_ms: int,
|
||||
*,
|
||||
speech: bool,
|
||||
partial: str | None = None,
|
||||
assistant_echo: bool = False,
|
||||
assistant_audio: bool = False,
|
||||
) -> AudioFrame:
|
||||
metadata: dict[str, object] = {"duration_ms": 100 if timestamp_ms >= 100 else 20, "speech": speech}
|
||||
if partial:
|
||||
metadata["partial"] = partial
|
||||
if assistant_echo:
|
||||
metadata["assistant_echo"] = True
|
||||
if assistant_audio:
|
||||
metadata["assistant_audio"] = True
|
||||
return AudioFrame(
|
||||
pcm=b"\x01\x00" * 800,
|
||||
sample_rate=16000,
|
||||
channels=1,
|
||||
timestamp_ms=timestamp_ms,
|
||||
frame_id=frame_id,
|
||||
metadata=metadata,
|
||||
)
|
||||
@@ -133,6 +133,7 @@ def run_simulated_live(
|
||||
vad_end_silence_ms=350,
|
||||
vad_no_speech_timeout_ms=3000,
|
||||
vad_max_recording_ms=6000,
|
||||
barge_in_enabled=False,
|
||||
)
|
||||
pipeline = VoiceAssistantPipeline(
|
||||
config=config,
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Literal, Protocol
|
||||
|
||||
from .agent_memory import MemoryManager, MemoryRecordInput, MemoryWritePolicy
|
||||
|
||||
|
||||
ToolAction = Literal["execute", "reject", "require_confirmation"]
|
||||
ToolRisk = Literal["low", "medium", "high", "forbidden"]
|
||||
ToolStatus = Literal["success", "failed", "cancelled", "rejected", "confirmation_required"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ToolCallRequest:
|
||||
id: str
|
||||
name: str
|
||||
arguments: dict[str, object]
|
||||
requested_by_turn_id: str
|
||||
natural_language_intent: str = ""
|
||||
timeout_ms: int = 30000
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ToolDecision:
|
||||
action: ToolAction
|
||||
risk_level: ToolRisk
|
||||
reason: str
|
||||
sanitized_arguments: dict[str, object] = field(default_factory=dict)
|
||||
confirmation_prompt: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ToolResult:
|
||||
id: str
|
||||
status: ToolStatus
|
||||
output_text: str = ""
|
||||
output_truncated: bool = False
|
||||
error_code: str | None = None
|
||||
duration_ms: int = 0
|
||||
audit_summary: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ToolAuditRecord:
|
||||
tool_name: str
|
||||
risk_level: ToolRisk
|
||||
action: ToolAction
|
||||
status: ToolStatus
|
||||
duration_ms: int
|
||||
summary: str
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ToolContext:
|
||||
memory: MemoryManager | None = None
|
||||
cwd: Path = Path(".")
|
||||
allowed_roots: tuple[Path, ...] = ()
|
||||
|
||||
|
||||
class ToolAdapter(Protocol):
|
||||
name: str
|
||||
|
||||
def execute(self, request: ToolCallRequest, context: ToolContext) -> ToolResult:
|
||||
...
|
||||
|
||||
|
||||
class FakeToolAdapter:
|
||||
def __init__(self, name: str, output: str = "ok") -> None:
|
||||
self.name = name
|
||||
self.output = output
|
||||
self.calls: list[ToolCallRequest] = []
|
||||
|
||||
def execute(self, request: ToolCallRequest, context: ToolContext) -> ToolResult:
|
||||
self.calls.append(request)
|
||||
return ToolResult(request.id, "success", self.output, audit_summary=f"{self.name} executed")
|
||||
|
||||
|
||||
class ToolRiskClassifier:
|
||||
forbidden_patterns = (
|
||||
r"\brm\b",
|
||||
r"\bchmod\b",
|
||||
r"\bchown\b",
|
||||
r"\bpip\s+install\b",
|
||||
r"\bnpm\s+install\b",
|
||||
r"\bbrew\s+install\b",
|
||||
r">\s*[/\w]",
|
||||
r"\bmv\b",
|
||||
r"\bcp\b",
|
||||
)
|
||||
high_risk_words = (
|
||||
"delete",
|
||||
"upload",
|
||||
"payment",
|
||||
"purchase",
|
||||
"trade",
|
||||
"account",
|
||||
"权限",
|
||||
"删除",
|
||||
"上传",
|
||||
"支付",
|
||||
"购买",
|
||||
"交易",
|
||||
"账号",
|
||||
)
|
||||
|
||||
def classify(self, request: ToolCallRequest) -> ToolRisk:
|
||||
payload = f"{request.name} {request.natural_language_intent} {json.dumps(request.arguments, ensure_ascii=False)}"
|
||||
lowered = payload.lower()
|
||||
if request.name == "shell.readonly":
|
||||
command = str(request.arguments.get("command", ""))
|
||||
if any(re.search(pattern, command, flags=re.IGNORECASE) for pattern in self.forbidden_patterns):
|
||||
return "forbidden"
|
||||
if any(word in lowered for word in self.high_risk_words):
|
||||
return "high"
|
||||
if request.name in {"memory.search", "shell.readonly"}:
|
||||
return "low"
|
||||
if request.name == "memory.save":
|
||||
return "medium"
|
||||
return "medium"
|
||||
|
||||
|
||||
class ToolRouter:
|
||||
def __init__(
|
||||
self,
|
||||
adapters: dict[str, ToolAdapter],
|
||||
*,
|
||||
max_calls_per_turn: int = 5,
|
||||
output_limit: int = 4000,
|
||||
risk_classifier: ToolRiskClassifier | None = None,
|
||||
) -> None:
|
||||
self.adapters = dict(adapters)
|
||||
self.max_calls_per_turn = max_calls_per_turn
|
||||
self.output_limit = output_limit
|
||||
self.risk_classifier = risk_classifier or ToolRiskClassifier()
|
||||
self.audit_log: list[ToolAuditRecord] = []
|
||||
self._calls_by_turn: dict[str, int] = {}
|
||||
self._signatures_by_turn: dict[str, set[str]] = {}
|
||||
|
||||
def route(self, request: ToolCallRequest, context: ToolContext) -> ToolDecision:
|
||||
if request.name not in self.adapters:
|
||||
return ToolDecision("reject", "forbidden", f"unknown tool: {request.name}")
|
||||
if not isinstance(request.arguments, dict):
|
||||
return ToolDecision("reject", "forbidden", "tool arguments must be an object")
|
||||
if self._calls_by_turn.get(request.requested_by_turn_id, 0) >= self.max_calls_per_turn:
|
||||
return ToolDecision("reject", "medium", "tool call budget exceeded")
|
||||
signature = self._signature(request)
|
||||
if signature in self._signatures_by_turn.setdefault(request.requested_by_turn_id, set()):
|
||||
return ToolDecision("reject", "medium", "duplicate tool call rejected")
|
||||
risk = self.risk_classifier.classify(request)
|
||||
sanitized = _sanitize_arguments(request.arguments)
|
||||
if risk == "forbidden":
|
||||
return ToolDecision("reject", risk, "tool request is forbidden", sanitized)
|
||||
if request.name == "memory.save":
|
||||
text = str(request.arguments.get("text", ""))
|
||||
decision = MemoryWritePolicy().evaluate(MemoryRecordInput("fact", text))
|
||||
if decision.requires_confirmation:
|
||||
return ToolDecision(
|
||||
"require_confirmation",
|
||||
"high",
|
||||
decision.reason,
|
||||
sanitized,
|
||||
confirmation_prompt="是否保存这条可能敏感的长期记忆？",
|
||||
)
|
||||
if risk == "high":
|
||||
return ToolDecision(
|
||||
"require_confirmation",
|
||||
risk,
|
||||
"high risk tool request requires confirmation",
|
||||
sanitized,
|
||||
confirmation_prompt="是否允许执行这个高风险工具请求？",
|
||||
)
|
||||
return ToolDecision("execute", risk, "approved", sanitized)
|
||||
|
||||
def execute(self, request: ToolCallRequest, decision: ToolDecision, context: ToolContext) -> ToolResult:
|
||||
started = time.monotonic()
|
||||
if decision.action == "reject":
|
||||
result = ToolResult(request.id, "rejected", error_code="TOOL_REJECTED", audit_summary=decision.reason)
|
||||
self._record(request, decision, result, started)
|
||||
return result
|
||||
if decision.action == "require_confirmation":
|
||||
result = ToolResult(
|
||||
request.id,
|
||||
"confirmation_required",
|
||||
error_code="TOOL_CONFIRMATION_REQUIRED",
|
||||
audit_summary=decision.reason,
|
||||
)
|
||||
self._record(request, decision, result, started)
|
||||
return result
|
||||
self._calls_by_turn[request.requested_by_turn_id] = self._calls_by_turn.get(request.requested_by_turn_id, 0) + 1
|
||||
self._signatures_by_turn.setdefault(request.requested_by_turn_id, set()).add(self._signature(request))
|
||||
result = self.adapters[request.name].execute(request, context)
|
||||
output, truncated = _truncate(_sanitize_text(result.output_text), self.output_limit)
|
||||
result = ToolResult(
|
||||
id=result.id,
|
||||
status=result.status,
|
||||
output_text=output,
|
||||
output_truncated=result.output_truncated or truncated,
|
||||
error_code=result.error_code,
|
||||
duration_ms=max(result.duration_ms, round((time.monotonic() - started) * 1000)),
|
||||
audit_summary=_sanitize_text(result.audit_summary),
|
||||
)
|
||||
self._record(request, decision, result, started)
|
||||
return result
|
||||
|
||||
def _record(
|
||||
self,
|
||||
request: ToolCallRequest,
|
||||
decision: ToolDecision,
|
||||
result: ToolResult,
|
||||
started: float,
|
||||
) -> None:
|
||||
self.audit_log.append(
|
||||
ToolAuditRecord(
|
||||
tool_name=request.name,
|
||||
risk_level=decision.risk_level,
|
||||
action=decision.action,
|
||||
status=result.status,
|
||||
duration_ms=max(result.duration_ms, round((time.monotonic() - started) * 1000)),
|
||||
summary=_sanitize_text(result.audit_summary or decision.reason),
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _signature(request: ToolCallRequest) -> str:
|
||||
return json.dumps(
|
||||
{"name": request.name, "arguments": request.arguments},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
|
||||
|
||||
class MemorySearchTool:
|
||||
name = "memory.search"
|
||||
|
||||
def execute(self, request: ToolCallRequest, context: ToolContext) -> ToolResult:
|
||||
if context.memory is None:
|
||||
return ToolResult(request.id, "failed", error_code="MEMORY_UNAVAILABLE", audit_summary="memory unavailable")
|
||||
query = str(request.arguments.get("query", ""))
|
||||
top_k = int(request.arguments.get("top_k", 5))
|
||||
records = context.memory.search(query, top_k=top_k)
|
||||
output = "\n".join(f"- [{record.type}] {record.text}" for record in records)
|
||||
return ToolResult(request.id, "success", output, audit_summary=f"returned {len(records)} memories")
|
||||
|
||||
|
||||
class MemorySaveTool:
|
||||
name = "memory.save"
|
||||
|
||||
def execute(self, request: ToolCallRequest, context: ToolContext) -> ToolResult:
|
||||
if context.memory is None:
|
||||
return ToolResult(request.id, "failed", error_code="MEMORY_UNAVAILABLE", audit_summary="memory unavailable")
|
||||
record_type = str(request.arguments.get("type", "fact"))
|
||||
if record_type not in {"preference", "fact", "project", "task_summary"}:
|
||||
return ToolResult(request.id, "failed", error_code="MEMORY_TYPE_INVALID", audit_summary="invalid memory type")
|
||||
saved = context.memory.save(
|
||||
MemoryRecordInput(
|
||||
record_type, # type: ignore[arg-type]
|
||||
str(request.arguments.get("text", "")),
|
||||
metadata=dict(request.arguments.get("metadata", {}) or {}),
|
||||
)
|
||||
)
|
||||
return ToolResult(request.id, "success", saved.id, audit_summary="memory saved")
|
||||
|
||||
|
||||
class ShellReadonlyTool:
|
||||
name = "shell.readonly"
|
||||
allowed_commands = {"pwd", "ls", "find", "rg", "cat", "sed", "git"}
|
||||
|
||||
def execute(self, request: ToolCallRequest, context: ToolContext) -> ToolResult:
|
||||
command = str(request.arguments.get("command", ""))
|
||||
parts = shlex.split(command)
|
||||
if not parts or parts[0] not in self.allowed_commands:
|
||||
return ToolResult(request.id, "failed", error_code="SHELL_COMMAND_NOT_ALLOWED", audit_summary="command not allowed")
|
||||
if parts[0] == "git" and len(parts) > 1 and parts[1] not in {"status", "diff", "log", "show"}:
|
||||
return ToolResult(request.id, "failed", error_code="SHELL_COMMAND_NOT_ALLOWED", audit_summary="git command not allowed")
|
||||
completed = subprocess.run(
|
||||
parts,
|
||||
cwd=context.cwd,
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=max(1, int(request.timeout_ms / 1000)),
|
||||
)
|
||||
status: ToolStatus = "success" if completed.returncode == 0 else "failed"
|
||||
return ToolResult(
|
||||
request.id,
|
||||
status,
|
||||
completed.stdout,
|
||||
error_code=None if completed.returncode == 0 else "SHELL_COMMAND_FAILED",
|
||||
audit_summary=f"readonly shell exited {completed.returncode}",
|
||||
)
|
||||
|
||||
|
||||
def _sanitize_arguments(arguments: dict[str, object]) -> dict[str, object]:
|
||||
return {key: _sanitize_text(str(value)) for key, value in arguments.items()}
|
||||
|
||||
|
||||
def _sanitize_text(text: str) -> str:
|
||||
text = re.sub(r"(?:sk|tp)-[A-Za-z0-9_\-]{16,}", "[redacted]", text)
|
||||
text = re.sub(r"(?i)(authorization:\s*)\S+", r"\1[redacted]", text)
|
||||
return text
|
||||
|
||||
|
||||
def _truncate(text: str, limit: int) -> tuple[str, bool]:
|
||||
if len(text) <= limit:
|
||||
return text, False
|
||||
return text[:limit] + "...[truncated]", True
|
||||
@@ -0,0 +1,141 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from owner_voice_pet.agent_memory import (
|
||||
AgentConversationManager,
|
||||
DisabledMemoryManager,
|
||||
FaissIndexManifest,
|
||||
FakeMemoryManager,
|
||||
MemoryManagementPlan,
|
||||
MemoryRecordInput,
|
||||
MemoryWritePolicy,
|
||||
SQLiteMemoryManager,
|
||||
is_sensitive_memory_text,
|
||||
)
|
||||
from owner_voice_pet.conversation import ConversationContext
|
||||
from owner_voice_pet.models import ErrorCode, ProviderError
|
||||
|
||||
|
||||
class AgentMemoryTests(unittest.TestCase):
|
||||
def test_fake_memory_manager_saves_and_searches_normal_memory(self) -> None:
|
||||
memory = FakeMemoryManager()
|
||||
saved = memory.save(MemoryRecordInput("preference", "用户喜欢 Python 项目"))
|
||||
|
||||
results = memory.search("Python", top_k=1)
|
||||
|
||||
self.assertEqual(results, [saved])
|
||||
self.assertEqual(memory.search_queries, ["Python"])
|
||||
self.assertEqual(saved.type, "preference")
|
||||
self.assertEqual(saved.sensitivity, "normal")
|
||||
self.assertTrue(saved.checksum)
|
||||
|
||||
def test_disabled_memory_manager_does_not_read_or_write(self) -> None:
|
||||
memory = DisabledMemoryManager()
|
||||
|
||||
self.assertEqual(memory.search("Python"), [])
|
||||
with self.assertRaises(ProviderError) as raised:
|
||||
memory.save(MemoryRecordInput("fact", "不会保存"))
|
||||
|
||||
self.assertEqual(raised.exception.code, ErrorCode.VALIDATION_FAILED)
|
||||
self.assertTrue(memory.health_check().ok)
|
||||
|
||||
def test_sqlite_memory_manager_persists_across_instances(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "memory.sqlite3"
|
||||
first = SQLiteMemoryManager(db_path)
|
||||
saved = first.save(MemoryRecordInput("project", "Owner 项目正在做语音助手"))
|
||||
|
||||
second = SQLiteMemoryManager(db_path)
|
||||
results = second.search("Owner 语音", top_k=3)
|
||||
|
||||
self.assertEqual([record.id for record in results], [saved.id])
|
||||
self.assertTrue(second.health_check().ok)
|
||||
|
||||
def test_sqlite_memory_manager_rejects_empty_text(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
manager = SQLiteMemoryManager(Path(tmp) / "memory.sqlite3")
|
||||
|
||||
with self.assertRaises(ProviderError) as raised:
|
||||
manager.save(MemoryRecordInput("fact", " "))
|
||||
|
||||
self.assertEqual(raised.exception.code, ErrorCode.VALIDATION_FAILED)
|
||||
|
||||
def test_faiss_manifest_detects_missing_and_mismatched_records(self) -> None:
|
||||
memory = FakeMemoryManager()
|
||||
first = memory.save(MemoryRecordInput("fact", "第一条"))
|
||||
second = memory.save(MemoryRecordInput("fact", "第二条"))
|
||||
manifest = FaissIndexManifest.from_records([first], embedding_model="fake-embedding")
|
||||
broken = FaissIndexManifest(
|
||||
embedding_model=manifest.embedding_model,
|
||||
record_ids=(first.id, "missing-id"),
|
||||
checksums={first.id: "wrong"},
|
||||
)
|
||||
|
||||
errors = broken.consistency_errors([first, second])
|
||||
|
||||
self.assertTrue(any("missing-id" in error for error in errors))
|
||||
self.assertTrue(any(first.id in error for error in errors))
|
||||
self.assertTrue(any(second.id in error for error in errors))
|
||||
|
||||
def test_memory_write_policy_requires_confirmation_for_sensitive_text(self) -> None:
|
||||
policy = MemoryWritePolicy()
|
||||
|
||||
safe = policy.evaluate(MemoryRecordInput("preference", "用户喜欢 Python"))
|
||||
sensitive = policy.evaluate(MemoryRecordInput("fact", "我的 api key 是 secret-value"))
|
||||
|
||||
self.assertTrue(safe.should_save)
|
||||
self.assertFalse(safe.requires_confirmation)
|
||||
self.assertFalse(sensitive.should_save)
|
||||
self.assertTrue(sensitive.requires_confirmation)
|
||||
self.assertTrue(is_sensitive_memory_text("密码是 123456"))
|
||||
|
||||
def test_conversation_manager_injects_memory_context(self) -> None:
|
||||
context = ConversationContext()
|
||||
context.append_assistant("我是小杰。")
|
||||
memory = FakeMemoryManager()
|
||||
memory.save(MemoryRecordInput("preference", "用户喜欢 Python"))
|
||||
manager = AgentConversationManager(
|
||||
context=context,
|
||||
memory=memory,
|
||||
memory_enabled=True,
|
||||
memory_top_k=2,
|
||||
)
|
||||
|
||||
messages = manager.build_messages_for_user("Python 项目怎么做？")
|
||||
|
||||
self.assertEqual(messages[0].role, "system")
|
||||
self.assertEqual(messages[1].role, "system")
|
||||
self.assertIn("长期记忆", messages[1].content)
|
||||
self.assertIn("用户喜欢 Python", messages[1].content)
|
||||
self.assertEqual(messages[-1].role, "user")
|
||||
self.assertEqual(messages[-1].content, "Python 项目怎么做？")
|
||||
|
||||
def test_conversation_manager_skips_memory_when_disabled(self) -> None:
|
||||
context = ConversationContext()
|
||||
memory = FakeMemoryManager(enabled=False)
|
||||
manager = AgentConversationManager(
|
||||
context=context,
|
||||
memory=memory,
|
||||
memory_enabled=False,
|
||||
)
|
||||
|
||||
messages = manager.build_messages_for_user("Python")
|
||||
|
||||
self.assertEqual([message.role for message in messages], ["system", "user"])
|
||||
self.assertEqual(memory.search_queries, [])
|
||||
|
||||
def test_memory_management_plan_lists_user_controls(self) -> None:
|
||||
plan = MemoryManagementPlan()
|
||||
|
||||
self.assertIn("list", plan.supported_commands)
|
||||
self.assertIn("delete", plan.supported_commands)
|
||||
self.assertIn("disable", plan.supported_commands)
|
||||
self.assertIn("export", plan.supported_commands)
|
||||
self.assertTrue(plan.default_enabled)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+120
-7
@@ -10,19 +10,21 @@ from owner_voice_pet.transport import MemoryAudioTransport
|
||||
from owner_voice_pet.vad import EnergyVadProvider
|
||||
|
||||
|
||||
def speech_frame(idx: int, speaker_id: str, partial: str = "等一下") -> AudioFrame:
|
||||
def speech_frame(idx: int, speaker_id: str, partial: str | None = "等一下") -> AudioFrame:
|
||||
metadata = {
|
||||
"duration_ms": 20,
|
||||
"speech": True,
|
||||
"speaker_id": speaker_id,
|
||||
}
|
||||
if partial is not None:
|
||||
metadata["partial_transcript"] = partial
|
||||
return AudioFrame(
|
||||
b"\xff\x7f" * 320,
|
||||
16000,
|
||||
1,
|
||||
idx * 20,
|
||||
idx,
|
||||
{
|
||||
"duration_ms": 20,
|
||||
"speech": True,
|
||||
"speaker_id": speaker_id,
|
||||
"partial_transcript": partial,
|
||||
},
|
||||
metadata,
|
||||
)
|
||||
|
||||
|
||||
@@ -127,6 +129,117 @@ class BargeInTests(unittest.TestCase):
|
||||
|
||||
self.assertTrue(monitor.interrupted)
|
||||
|
||||
def test_async_monitor_interrupts_without_realtime_partial(self) -> None:
|
||||
gate = BargeInSpeakerGate(
|
||||
enabled=True,
|
||||
user_similarity_threshold=0.62,
|
||||
assistant_reject_threshold=0.72,
|
||||
min_rms=0.001,
|
||||
)
|
||||
gate.remember_user_segment(segment_for_speaker("owner"))
|
||||
transport = MemoryAudioTransport([speech_frame(1, "owner", None), speech_frame(2, "owner", None)])
|
||||
transport.start_input()
|
||||
monitor = AsyncBargeInMonitor(
|
||||
transport=transport,
|
||||
vad_provider=EnergyVadProvider(threshold=1),
|
||||
realtime_stt=None,
|
||||
speaker_gate=gate,
|
||||
assistant_profile=gate.assistant_profile(segment_for_speaker("assistant")),
|
||||
echo_guard_ms=0,
|
||||
min_speech_ms=40,
|
||||
listen_interval_ms=1,
|
||||
)
|
||||
monitor.vad_provider.load()
|
||||
|
||||
monitor.start()
|
||||
deadline = time.monotonic() + 1
|
||||
while not monitor.interrupted and time.monotonic() < deadline:
|
||||
time.sleep(0.005)
|
||||
monitor.stop()
|
||||
|
||||
self.assertTrue(monitor.interrupted)
|
||||
self.assertEqual(len(monitor.pending_frames()), 2)
|
||||
|
||||
def test_render_reference_echo_does_not_interrupt(self) -> None:
|
||||
gate = BargeInSpeakerGate(
|
||||
enabled=True,
|
||||
user_similarity_threshold=0.62,
|
||||
assistant_reject_threshold=0.72,
|
||||
min_rms=0.001,
|
||||
)
|
||||
assistant = AudioSegment(b"\x01\x20\x02\x20" * 320, 16000, 1, 0, 40)
|
||||
echo_frame = AudioFrame(
|
||||
assistant.pcm[: 640 * 2],
|
||||
16000,
|
||||
1,
|
||||
20,
|
||||
1,
|
||||
{"duration_ms": 40, "speech": True},
|
||||
)
|
||||
transport = MemoryAudioTransport([echo_frame])
|
||||
transport.start_input()
|
||||
monitor = AsyncBargeInMonitor(
|
||||
transport=transport,
|
||||
vad_provider=EnergyVadProvider(threshold=1),
|
||||
realtime_stt=None,
|
||||
speaker_gate=gate,
|
||||
assistant_profile=gate.assistant_profile(assistant),
|
||||
assistant_reference=assistant,
|
||||
echo_guard_ms=0,
|
||||
min_speech_ms=40,
|
||||
listen_interval_ms=1,
|
||||
)
|
||||
monitor.vad_provider.load()
|
||||
|
||||
monitor.start()
|
||||
time.sleep(0.05)
|
||||
monitor.stop()
|
||||
|
||||
self.assertFalse(monitor.interrupted)
|
||||
|
||||
def test_user_profile_match_can_override_render_echo_candidate(self) -> None:
|
||||
gate = BargeInSpeakerGate(
|
||||
enabled=True,
|
||||
user_similarity_threshold=0.62,
|
||||
assistant_reject_threshold=0.72,
|
||||
min_rms=0.001,
|
||||
)
|
||||
gate.remember_user_segment(segment_for_speaker("owner"))
|
||||
assistant = AudioSegment(b"\x01\x20\x02\x20" * 320, 16000, 1, 0, 40, {"speaker_id": "assistant"})
|
||||
frames = [
|
||||
AudioFrame(
|
||||
assistant.pcm[: 640 * 2],
|
||||
16000,
|
||||
1,
|
||||
20 * idx,
|
||||
idx,
|
||||
{"duration_ms": 20, "speech": True, "speaker_id": "owner"},
|
||||
)
|
||||
for idx in (1, 2)
|
||||
]
|
||||
transport = MemoryAudioTransport(frames)
|
||||
transport.start_input()
|
||||
monitor = AsyncBargeInMonitor(
|
||||
transport=transport,
|
||||
vad_provider=EnergyVadProvider(threshold=1),
|
||||
realtime_stt=None,
|
||||
speaker_gate=gate,
|
||||
assistant_profile=gate.assistant_profile(assistant),
|
||||
assistant_reference=assistant,
|
||||
echo_guard_ms=0,
|
||||
min_speech_ms=40,
|
||||
listen_interval_ms=1,
|
||||
)
|
||||
monitor.vad_provider.load()
|
||||
|
||||
monitor.start()
|
||||
deadline = time.monotonic() + 1
|
||||
while not monitor.interrupted and time.monotonic() < deadline:
|
||||
time.sleep(0.005)
|
||||
monitor.stop()
|
||||
|
||||
self.assertTrue(monitor.interrupted)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -42,6 +42,7 @@ class CliAcceptanceTests(unittest.TestCase):
|
||||
self.assertEqual(code, 0)
|
||||
self.assertTrue(data["llm_api_key_present"])
|
||||
self.assertEqual(data["llm_model"], "file-model")
|
||||
self.assertEqual(data["assistant_mode"], "turn_based_voice_pet")
|
||||
self.assertEqual(data["pipeline_mode"], "live_turn_based")
|
||||
self.assertEqual(data["endpoint_mode"], "primary_speaker")
|
||||
self.assertEqual(data["realtime_transcript_idle_timeout_ms"], 1500)
|
||||
@@ -49,8 +50,88 @@ class CliAcceptanceTests(unittest.TestCase):
|
||||
self.assertTrue(data["end_chime_enabled"])
|
||||
self.assertEqual(data["end_chime_file"], "assets/sounds/codex-notification.wav")
|
||||
self.assertEqual(data["end_chime_duration_ms"], 140)
|
||||
self.assertEqual(data["audio_apm_provider"], "webrtc")
|
||||
self.assertTrue(data["audio_aec_enabled"])
|
||||
self.assertTrue(data["audio_ns_enabled"])
|
||||
self.assertTrue(data["audio_agc_enabled"])
|
||||
self.assertEqual(data["audio_frame_ms"], 20)
|
||||
self.assertEqual(data["audio_ring_buffer_ms"], 3000)
|
||||
self.assertTrue(data["interrupt_enabled"])
|
||||
self.assertEqual(data["interrupt_target_latency_ms"], 200)
|
||||
self.assertEqual(data["streaming_stt_provider"], "faster_whisper")
|
||||
self.assertEqual(data["streaming_stt_product_candidate"], "sensevoice")
|
||||
self.assertEqual(data["streaming_tts_provider"], "cosyvoice")
|
||||
self.assertTrue(data["memory_enabled"])
|
||||
self.assertEqual(data["memory_provider"], "faiss_sqlite")
|
||||
self.assertTrue(data["tool_router_enabled"])
|
||||
self.assertFalse(data["openinterpreter_enabled"])
|
||||
self.assertFalse(data["browser_playwright_enabled"])
|
||||
self.assertFalse(data["computer_control_enabled"])
|
||||
self.assertNotIn("secret-value", str(data))
|
||||
|
||||
def test_help_lists_full_duplex_agent_entrypoint(self) -> None:
|
||||
buffer = io.StringIO()
|
||||
with redirect_stdout(buffer), self.assertRaises(SystemExit) as raised:
|
||||
main(["--help"])
|
||||
|
||||
self.assertEqual(raised.exception.code, 0)
|
||||
self.assertIn("run-agent-live", buffer.getvalue())
|
||||
|
||||
def test_run_agent_live_check_config_reports_reserved_entrypoint(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
env_file = str(Path(tmp) / ".env")
|
||||
code, data = self.call("--env-file", env_file, "run-agent-live", "--check-config")
|
||||
self.assertEqual(code, 0)
|
||||
self.assertTrue(data["success"])
|
||||
self.assertEqual(data["command"], "run-agent-live")
|
||||
self.assertEqual(data["assistant_mode"], "full_duplex_agent")
|
||||
self.assertTrue(data["full_duplex_runtime_ready"])
|
||||
self.assertEqual(data["audio_apm_provider"], "webrtc")
|
||||
self.assertEqual(data["audio_apm_error_code"], "")
|
||||
self.assertTrue(data["audio_apm_available"])
|
||||
self.assertTrue(data["llm_streaming_enabled"])
|
||||
self.assertEqual(data["turn_based_entry"], "run-live")
|
||||
|
||||
def test_run_agent_live_once_invokes_agent_runtime(self) -> None:
|
||||
class FakeRuntime:
|
||||
def __init__(self, *, config):
|
||||
self.config = config
|
||||
self.once = None
|
||||
|
||||
def run(self, *, once: bool = False):
|
||||
self.once = once
|
||||
return type("Summary", (), {"completed_turns": 1, "interrupted": False})()
|
||||
|
||||
created: list[FakeRuntime] = []
|
||||
|
||||
def make_runtime(*, config):
|
||||
runtime = FakeRuntime(config=config)
|
||||
created.append(runtime)
|
||||
return runtime
|
||||
|
||||
with (
|
||||
patch("owner_voice_pet.cli.build_audio_processing_provider"),
|
||||
patch("owner_voice_pet.cli.FullDuplexAgentRuntime", side_effect=make_runtime),
|
||||
):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
code = main(["--env-file", str(Path(tmp) / ".env"), "run-agent-live", "--once"])
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(len(created), 1)
|
||||
self.assertTrue(created[0].once)
|
||||
|
||||
def test_run_agent_live_fails_when_required_apm_is_unavailable(self) -> None:
|
||||
with patch("owner_voice_pet.full_duplex_audio.webrtc_apm_probe") as probe:
|
||||
probe.return_value = type(
|
||||
"Health",
|
||||
(),
|
||||
{"available": False, "message": "missing binding"},
|
||||
)()
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
code, data = self.call("--env-file", str(Path(tmp) / ".env"), "run-agent-live", "--once")
|
||||
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(data["code"], "AUDIO_APM_UNAVAILABLE")
|
||||
|
||||
def test_security_check_command_has_no_leaks(self) -> None:
|
||||
code, data = self.call("security-check")
|
||||
self.assertEqual(code, 0)
|
||||
@@ -127,6 +208,56 @@ class CliAcceptanceTests(unittest.TestCase):
|
||||
self.assertFalse(kwargs["play_audio"])
|
||||
self.assertEqual(kwargs["questions"], ["第一问"])
|
||||
|
||||
def test_agent_self_test_runs_full_duplex_profile(self) -> None:
|
||||
code, data = self.call("agent-self-test", "--profile", "full-duplex", "--turns", "3")
|
||||
|
||||
self.assertEqual(code, 0)
|
||||
self.assertTrue(data["success"])
|
||||
self.assertTrue(data["checks"]["barge_in"])
|
||||
self.assertTrue(data["checks"]["tool_router"])
|
||||
|
||||
def test_audio_self_test_reports_missing_required_apm_when_probe_fails(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch("owner_voice_pet.full_duplex_audio.webrtc_apm_probe") as probe:
|
||||
probe.return_value = type(
|
||||
"Health",
|
||||
(),
|
||||
{"available": False, "provider": "webrtc", "message": "missing binding"},
|
||||
)()
|
||||
code, data = self.call(
|
||||
"--env-file",
|
||||
str(Path(tmp) / ".env"),
|
||||
"audio-self-test",
|
||||
"--duration",
|
||||
"1",
|
||||
"--check-echo",
|
||||
)
|
||||
|
||||
self.assertEqual(code, 1)
|
||||
self.assertFalse(data["success"])
|
||||
self.assertEqual(data["apm"]["error_code"], "AUDIO_APM_UNAVAILABLE")
|
||||
|
||||
def test_audio_self_test_can_use_fake_apm_for_deterministic_check(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
env_file = Path(tmp) / ".env"
|
||||
env_file.write_text(
|
||||
"OWNER_AUDIO_APM_PROVIDER=fake\nOWNER_AUDIO_APM_REQUIRED=0\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
code, data = self.call(
|
||||
"--env-file",
|
||||
str(env_file),
|
||||
"audio-self-test",
|
||||
"--duration",
|
||||
"1",
|
||||
"--check-echo",
|
||||
)
|
||||
|
||||
self.assertEqual(code, 0)
|
||||
self.assertTrue(data["success"])
|
||||
self.assertTrue(data["echo"]["suppressed"])
|
||||
self.assertEqual(data["apm"]["provider"], "fake_webrtc")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from owner_voice_pet.config import AppConfig
|
||||
from owner_voice_pet.external_adapters import (
|
||||
CODEX_COMPUTER_USE_SAFETY_REFERENCE,
|
||||
BrowserPlaywrightAdapter,
|
||||
ComputerControlProvider,
|
||||
OpenInterpreterAdapter,
|
||||
is_high_risk_task,
|
||||
planned_external_adapters,
|
||||
)
|
||||
from owner_voice_pet.tool_router import ToolCallRequest, ToolContext, ToolRouter
|
||||
|
||||
|
||||
class ExternalAdaptersTests(unittest.TestCase):
|
||||
def test_openinterpreter_disabled_returns_unavailable_status(self) -> None:
|
||||
adapter = OpenInterpreterAdapter(enabled=False)
|
||||
request = ToolCallRequest("1", "openinterpreter.run", {"task": "列出文件"}, "turn-1")
|
||||
|
||||
result = adapter.execute(request, ToolContext())
|
||||
|
||||
self.assertEqual(result.status, "failed")
|
||||
self.assertEqual(result.error_code, "OPENINTERPRETER_DISABLED")
|
||||
|
||||
def test_openinterpreter_missing_command_is_structured(self) -> None:
|
||||
adapter = OpenInterpreterAdapter(enabled=True, command="/tmp/owner-missing-openinterpreter")
|
||||
request = ToolCallRequest("1", "openinterpreter.run", {"task": "列出文件"}, "turn-1")
|
||||
|
||||
result = adapter.execute(request, ToolContext())
|
||||
|
||||
self.assertEqual(result.status, "failed")
|
||||
self.assertEqual(result.error_code, "OPENINTERPRETER_UNAVAILABLE")
|
||||
|
||||
def test_openinterpreter_low_risk_dry_run(self) -> None:
|
||||
adapter = OpenInterpreterAdapter(enabled=True, command="python3", dry_run=True)
|
||||
request = ToolCallRequest("1", "openinterpreter.run", {"task": "只读检查当前目录"}, "turn-1")
|
||||
|
||||
result = adapter.execute(request, ToolContext())
|
||||
|
||||
self.assertEqual(result.status, "success")
|
||||
self.assertIn("dry-run Open Interpreter", result.output_text)
|
||||
|
||||
def test_openinterpreter_high_risk_task_requires_confirmation(self) -> None:
|
||||
adapter = OpenInterpreterAdapter(enabled=True, command="python3", dry_run=True)
|
||||
request = ToolCallRequest("1", "openinterpreter.run", {"task": "delete files"}, "turn-1")
|
||||
|
||||
result = adapter.execute(request, ToolContext())
|
||||
|
||||
self.assertEqual(result.status, "confirmation_required")
|
||||
self.assertEqual(result.error_code, "OPENINTERPRETER_HIGH_RISK")
|
||||
|
||||
def test_playwright_disabled_and_missing_dependency_are_structured(self) -> None:
|
||||
request = ToolCallRequest("1", "browser.playwright", {"task": "读取网页标题"}, "turn-1")
|
||||
|
||||
disabled = BrowserPlaywrightAdapter(enabled=False).execute(request, ToolContext())
|
||||
with patch("importlib.util.find_spec", return_value=None):
|
||||
missing = BrowserPlaywrightAdapter(enabled=True).execute(request, ToolContext())
|
||||
|
||||
self.assertEqual(disabled.error_code, "PLAYWRIGHT_DISABLED")
|
||||
self.assertEqual(missing.error_code, "PLAYWRIGHT_UNAVAILABLE")
|
||||
|
||||
def test_playwright_high_risk_task_requires_confirmation_when_available(self) -> None:
|
||||
request = ToolCallRequest("1", "browser.playwright", {"task": "purchase item"}, "turn-1")
|
||||
with patch("importlib.util.find_spec", return_value=object()):
|
||||
result = BrowserPlaywrightAdapter(enabled=True).execute(request, ToolContext())
|
||||
|
||||
self.assertEqual(result.status, "confirmation_required")
|
||||
self.assertEqual(result.error_code, "PLAYWRIGHT_SENSITIVE_ACTION")
|
||||
|
||||
def test_computer_control_provider_is_reserved_and_unsupported(self) -> None:
|
||||
result = ComputerControlProvider(enabled=True).execute(
|
||||
ToolCallRequest("1", "computer.control", {"action": "click"}, "turn-1"),
|
||||
ToolContext(),
|
||||
)
|
||||
|
||||
self.assertEqual(result.status, "failed")
|
||||
self.assertEqual(result.error_code, "COMPUTER_CONTROL_UNSUPPORTED")
|
||||
|
||||
def test_planned_external_adapters_respect_config_defaults(self) -> None:
|
||||
adapters = planned_external_adapters(AppConfig())
|
||||
|
||||
self.assertFalse(adapters["openinterpreter.run"].enabled)
|
||||
self.assertFalse(adapters["browser.playwright"].enabled)
|
||||
self.assertFalse(adapters["computer.control"].enabled)
|
||||
|
||||
def test_tool_router_requires_confirmation_for_browser_sensitive_intent(self) -> None:
|
||||
router = ToolRouter({"browser.playwright": BrowserPlaywrightAdapter(enabled=True)})
|
||||
request = ToolCallRequest(
|
||||
"1",
|
||||
"browser.playwright",
|
||||
{"task": "读取网页"},
|
||||
"turn-1",
|
||||
natural_language_intent="购买商品",
|
||||
)
|
||||
|
||||
decision = router.route(request, ToolContext())
|
||||
|
||||
self.assertEqual(decision.action, "require_confirmation")
|
||||
self.assertEqual(decision.risk_level, "high")
|
||||
|
||||
def test_codex_computer_use_reference_is_policy_only(self) -> None:
|
||||
self.assertIn("safety-confirmation", CODEX_COMPUTER_USE_SAFETY_REFERENCE)
|
||||
self.assertIn("public", CODEX_COMPUTER_USE_SAFETY_REFERENCE)
|
||||
self.assertNotIn("private API", CODEX_COMPUTER_USE_SAFETY_REFERENCE)
|
||||
|
||||
def test_high_risk_task_classifier_covers_chinese_and_english(self) -> None:
|
||||
self.assertTrue(is_high_risk_task("删除下载目录"))
|
||||
self.assertTrue(is_high_risk_task("upload account backup"))
|
||||
self.assertFalse(is_high_risk_task("read local project summary"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,234 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from owner_voice_pet.config import AppConfig
|
||||
from owner_voice_pet.full_duplex_audio import (
|
||||
AecAudioProcessingProvider,
|
||||
AudioHub,
|
||||
CaptureRingBuffer,
|
||||
FakeWebRtcAudioProcessingProvider,
|
||||
NoopAudioProcessingProvider,
|
||||
RenderReferenceRingBuffer,
|
||||
build_audio_processing_provider,
|
||||
webrtc_apm_probe,
|
||||
)
|
||||
from owner_voice_pet.models import AudioFrame, ErrorCode, ProviderError
|
||||
|
||||
|
||||
def frame(
|
||||
frame_id: int,
|
||||
timestamp_ms: int,
|
||||
*,
|
||||
duration_ms: int = 20,
|
||||
pcm: bytes = b"\x01\x00" * 160,
|
||||
sample_rate: int = 16000,
|
||||
channels: int = 1,
|
||||
metadata: dict[str, object] | None = None,
|
||||
) -> AudioFrame:
|
||||
data = {"duration_ms": duration_ms}
|
||||
if metadata:
|
||||
data.update(metadata)
|
||||
return AudioFrame(
|
||||
pcm=pcm,
|
||||
sample_rate=sample_rate,
|
||||
channels=channels,
|
||||
timestamp_ms=timestamp_ms,
|
||||
frame_id=frame_id,
|
||||
metadata=data,
|
||||
)
|
||||
|
||||
|
||||
class FullDuplexAudioTests(unittest.TestCase):
|
||||
def test_audio_frame_duration_and_fixture_roundtrip(self) -> None:
|
||||
original = frame(7, 140, metadata={"speech": True})
|
||||
|
||||
self.assertEqual(original.duration_ms, 20)
|
||||
restored = AudioFrame.from_fixture(original.to_fixture())
|
||||
|
||||
self.assertEqual(restored, original)
|
||||
self.assertTrue(restored.metadata["speech"])
|
||||
|
||||
def test_capture_ring_buffer_drops_oldest_frames_on_overrun(self) -> None:
|
||||
buffer = CaptureRingBuffer(capacity_ms=40)
|
||||
|
||||
first = buffer.write(frame(1, 0))
|
||||
second = buffer.write(frame(2, 20))
|
||||
third = buffer.write(frame(3, 40))
|
||||
|
||||
self.assertFalse(first.overrun)
|
||||
self.assertFalse(second.overrun)
|
||||
self.assertTrue(third.overrun)
|
||||
self.assertEqual([item.frame_id for item in third.dropped], [1])
|
||||
self.assertEqual([item.frame_id for item in buffer.frames()], [2, 3])
|
||||
self.assertEqual(buffer.duration_ms, 40)
|
||||
|
||||
def test_render_reference_window_uses_timestamps(self) -> None:
|
||||
buffer = RenderReferenceRingBuffer(capacity_ms=100)
|
||||
for idx, timestamp in enumerate([0, 20, 40, 60], start=1):
|
||||
buffer.write(frame(idx, timestamp))
|
||||
|
||||
window = buffer.latest_window(timestamp_ms=60, window_ms=30)
|
||||
|
||||
self.assertEqual([item.frame_id for item in window], [3, 4])
|
||||
|
||||
def test_fake_webrtc_apm_suppresses_assistant_echo(self) -> None:
|
||||
provider = FakeWebRtcAudioProcessingProvider(sample_rate=16000, channels=1)
|
||||
provider.process_render(frame(1, 0, metadata={"assistant_audio": True}))
|
||||
|
||||
processed = provider.process_capture(frame(2, 20, metadata={"assistant_echo": True, "speech": True}))
|
||||
|
||||
self.assertEqual(processed.pcm, b"\x00" * len(processed.pcm))
|
||||
self.assertTrue(processed.metadata["echo_suppressed"])
|
||||
self.assertFalse(processed.metadata["speech"])
|
||||
|
||||
def test_audio_hub_fans_out_processed_capture_without_stealing_frames(self) -> None:
|
||||
hub = AudioHub(
|
||||
processor=FakeWebRtcAudioProcessingProvider(sample_rate=16000, channels=1),
|
||||
capture_capacity_ms=100,
|
||||
)
|
||||
vad = hub.subscribe("processed_capture", name="vad")
|
||||
stt = hub.subscribe("processed_capture", name="stt")
|
||||
|
||||
for idx, timestamp in enumerate([0, 20, 40], start=1):
|
||||
hub.accept_capture(frame(idx, timestamp, metadata={"speech": True}))
|
||||
|
||||
self.assertEqual([item.frame_id for item in vad.read_available()], [1, 2, 3])
|
||||
self.assertEqual([item.frame_id for item in stt.read_available()], [1, 2, 3])
|
||||
self.assertEqual(vad.read_available(), ())
|
||||
self.assertEqual(stt.read_available(), ())
|
||||
|
||||
def test_audio_hub_processes_capture_before_processed_subscribers_read_it(self) -> None:
|
||||
hub = AudioHub(
|
||||
processor=FakeWebRtcAudioProcessingProvider(sample_rate=16000, channels=1),
|
||||
capture_capacity_ms=100,
|
||||
render_capacity_ms=100,
|
||||
)
|
||||
hub.accept_render(frame(1, 0, metadata={"assistant_audio": True}))
|
||||
subscription = hub.subscribe("processed_capture", name="interrupt")
|
||||
|
||||
processed = hub.accept_capture(frame(2, 20, metadata={"assistant_echo": True, "speech": True}))
|
||||
|
||||
self.assertTrue(processed.metadata["echo_suppressed"])
|
||||
self.assertEqual(subscription.read_available(), (processed,))
|
||||
self.assertEqual([item.frame_id for item in hub.render_reference.frames()], [1])
|
||||
|
||||
def test_audio_hub_reports_ring_and_subscriber_overrun(self) -> None:
|
||||
hub = AudioHub(
|
||||
processor=FakeWebRtcAudioProcessingProvider(sample_rate=16000, channels=1),
|
||||
capture_capacity_ms=40,
|
||||
)
|
||||
subscription = hub.subscribe("processed_capture", name="slow-stt")
|
||||
hub.accept_capture(frame(1, 0))
|
||||
self.assertEqual([item.frame_id for item in subscription.read_available()], [1])
|
||||
|
||||
hub.accept_capture(frame(2, 20))
|
||||
hub.accept_capture(frame(3, 40))
|
||||
hub.accept_capture(frame(4, 60))
|
||||
|
||||
self.assertEqual([item.frame_id for item in subscription.read_available()], [3, 4])
|
||||
self.assertGreaterEqual(subscription.missed_frames, 1)
|
||||
self.assertTrue(any(item.code == ErrorCode.AUDIO_BUFFER_OVERRUN for item in hub.diagnostics))
|
||||
self.assertTrue(any(item.subscriber == "slow-stt" for item in hub.diagnostics))
|
||||
|
||||
def test_fake_webrtc_apm_rejects_format_mismatch(self) -> None:
|
||||
provider = FakeWebRtcAudioProcessingProvider(sample_rate=16000, channels=1)
|
||||
|
||||
with self.assertRaises(ProviderError) as raised:
|
||||
provider.process_capture(frame(1, 0, sample_rate=8000))
|
||||
|
||||
self.assertEqual(raised.exception.code, ErrorCode.AUDIO_APM_FORMAT_MISMATCH)
|
||||
|
||||
def test_fake_webrtc_apm_can_report_processing_failure(self) -> None:
|
||||
provider = FakeWebRtcAudioProcessingProvider(fail_processing=True)
|
||||
|
||||
with self.assertRaises(ProviderError) as raised:
|
||||
provider.process_capture(frame(1, 0))
|
||||
|
||||
self.assertEqual(raised.exception.code, ErrorCode.AUDIO_APM_PROCESS_FAILED)
|
||||
|
||||
def test_audio_processing_provider_uses_fake_provider(self) -> None:
|
||||
provider = build_audio_processing_provider(AppConfig(audio_apm_provider="fake"))
|
||||
|
||||
self.assertIsInstance(provider, FakeWebRtcAudioProcessingProvider)
|
||||
self.assertTrue(provider.health_check().available)
|
||||
|
||||
def test_required_webrtc_provider_fails_when_probe_is_unavailable(self) -> None:
|
||||
with patch("owner_voice_pet.full_duplex_audio.webrtc_apm_probe") as probe:
|
||||
probe.return_value = type(
|
||||
"Health",
|
||||
(),
|
||||
{"available": False, "message": "missing binding"},
|
||||
)()
|
||||
with self.assertRaises(ProviderError) as raised:
|
||||
build_audio_processing_provider(
|
||||
AppConfig(audio_apm_provider="webrtc", audio_apm_required=True)
|
||||
)
|
||||
|
||||
self.assertEqual(raised.exception.code, ErrorCode.AUDIO_APM_UNAVAILABLE)
|
||||
|
||||
def test_webrtc_provider_can_fallback_when_not_required(self) -> None:
|
||||
with patch("owner_voice_pet.full_duplex_audio.webrtc_apm_probe") as probe:
|
||||
probe.return_value = type(
|
||||
"Health",
|
||||
(),
|
||||
{"available": False, "message": "missing binding"},
|
||||
)()
|
||||
provider = build_audio_processing_provider(
|
||||
AppConfig(audio_apm_provider="webrtc", audio_apm_required=False)
|
||||
)
|
||||
|
||||
self.assertIsInstance(provider, NoopAudioProcessingProvider)
|
||||
self.assertTrue(provider.health_check().fallback_active)
|
||||
|
||||
def test_webrtc_provider_uses_aec_audio_processing_when_available(self) -> None:
|
||||
with patch("owner_voice_pet.full_duplex_audio.webrtc_apm_probe") as probe:
|
||||
probe.return_value = type(
|
||||
"Health",
|
||||
(),
|
||||
{"available": True, "provider": "aec_audio_processing", "message": "found binding"},
|
||||
)()
|
||||
provider = build_audio_processing_provider(
|
||||
AppConfig(audio_apm_provider="webrtc", audio_apm_required=True)
|
||||
)
|
||||
|
||||
self.assertIsInstance(provider, AecAudioProcessingProvider)
|
||||
self.assertTrue(provider.health_check().available)
|
||||
|
||||
def test_detected_but_unwired_webrtc_provider_fails_when_required(self) -> None:
|
||||
with patch("owner_voice_pet.full_duplex_audio.webrtc_apm_probe") as probe:
|
||||
probe.return_value = type(
|
||||
"Health",
|
||||
(),
|
||||
{"available": True, "provider": "webrtc_audio_processing", "message": "found binding"},
|
||||
)()
|
||||
with self.assertRaises(ProviderError) as raised:
|
||||
build_audio_processing_provider(
|
||||
AppConfig(audio_apm_provider="webrtc", audio_apm_required=True)
|
||||
)
|
||||
|
||||
self.assertEqual(raised.exception.code, ErrorCode.AUDIO_APM_UNAVAILABLE)
|
||||
|
||||
def test_aec_audio_processing_provider_processes_capture_and_render(self) -> None:
|
||||
provider = AecAudioProcessingProvider(sample_rate=16000, channels=1)
|
||||
provider.process_render(frame(1, 0, pcm=b"\x00\x00" * 160))
|
||||
|
||||
processed = provider.process_capture(frame(2, 20, pcm=b"\x01\x00" * 320, duration_ms=20))
|
||||
|
||||
self.assertEqual(processed.sample_rate, 16000)
|
||||
self.assertEqual(processed.channels, 1)
|
||||
self.assertEqual(len(processed.pcm), 640)
|
||||
self.assertEqual(processed.metadata["apm_provider"], "aec_audio_processing")
|
||||
self.assertEqual(len(provider.render_frames), 1)
|
||||
|
||||
def test_webrtc_probe_reports_unavailable_without_binding(self) -> None:
|
||||
with patch("importlib.util.find_spec", return_value=None):
|
||||
health = webrtc_apm_probe()
|
||||
|
||||
self.assertFalse(health.available)
|
||||
self.assertEqual(health.provider, "webrtc")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,125 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from owner_voice_pet.events import (
|
||||
RECOVERING,
|
||||
SESSION_RECOVERED,
|
||||
STAGE_ERROR,
|
||||
PipelineEventBus,
|
||||
)
|
||||
from owner_voice_pet.full_duplex_control import (
|
||||
CancellationGraph,
|
||||
FullDuplexStateMachine,
|
||||
InvalidStateTransition,
|
||||
RecoveryCoordinator,
|
||||
)
|
||||
from owner_voice_pet.models import ErrorCode, PipelineState, ProviderError
|
||||
|
||||
|
||||
class FullDuplexControlTests(unittest.TestCase):
|
||||
def test_full_duplex_state_machine_accepts_normal_interruption_sequence(self) -> None:
|
||||
machine = FullDuplexStateMachine()
|
||||
|
||||
machine.transition(PipelineState.LISTENING, event_type="listening_started")
|
||||
machine.transition(PipelineState.THINKING, event_type="stt_final")
|
||||
machine.transition(PipelineState.SPEAKING, event_type="tts_chunk_ready")
|
||||
machine.transition(PipelineState.INTERRUPTED, event_type="interrupt_detected")
|
||||
machine.transition(PipelineState.LISTENING, event_type="interruption_buffered")
|
||||
|
||||
self.assertEqual(machine.current_state, PipelineState.LISTENING)
|
||||
self.assertEqual(
|
||||
[transition.new_state for transition in machine.history],
|
||||
[
|
||||
PipelineState.LISTENING,
|
||||
PipelineState.THINKING,
|
||||
PipelineState.SPEAKING,
|
||||
PipelineState.INTERRUPTED,
|
||||
PipelineState.LISTENING,
|
||||
],
|
||||
)
|
||||
|
||||
def test_full_duplex_state_machine_rejects_invalid_transition(self) -> None:
|
||||
machine = FullDuplexStateMachine()
|
||||
|
||||
with self.assertRaises(InvalidStateTransition):
|
||||
machine.transition(PipelineState.SPEAKING, event_type="skip_listening")
|
||||
|
||||
def test_pipeline_event_bus_adds_diagnostics_and_sanitizes_payload(self) -> None:
|
||||
bus = PipelineEventBus()
|
||||
seen = []
|
||||
bus.subscribe(seen.append)
|
||||
|
||||
event = bus.emit(
|
||||
"tool_call_requested",
|
||||
session_id="session-1",
|
||||
turn_id=7,
|
||||
stage="tool_router",
|
||||
state=PipelineState.TOOL_RUNNING,
|
||||
payload={
|
||||
"api_key": "secret",
|
||||
"nested": {"authorization_header": "Bearer secret"},
|
||||
"safe": "value",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(seen, [event])
|
||||
self.assertEqual(event.session_id, "session-1")
|
||||
self.assertEqual(event.turn_id, 7)
|
||||
self.assertEqual(event.stage, "tool_router")
|
||||
self.assertGreater(event.created_at, 0)
|
||||
self.assertEqual(event.payload["api_key"], "[redacted]")
|
||||
self.assertEqual(event.payload["nested"]["authorization_header"], "[redacted]")
|
||||
self.assertEqual(event.payload["safe"], "value")
|
||||
|
||||
def test_cancellation_graph_cascades_and_is_idempotent(self) -> None:
|
||||
graph = CancellationGraph("turn-1")
|
||||
llm = graph.child("llm")
|
||||
tts = graph.child("tts")
|
||||
playback = graph.child("playback", parent="tts")
|
||||
callback_reasons: list[str] = []
|
||||
playback.add_callback(callback_reasons.append)
|
||||
|
||||
graph.cancel_all("user interrupted")
|
||||
graph.cancel_all("second cancel")
|
||||
|
||||
self.assertTrue(graph.root.cancelled)
|
||||
self.assertTrue(llm.cancelled)
|
||||
self.assertTrue(tts.cancelled)
|
||||
self.assertTrue(playback.cancelled)
|
||||
self.assertEqual(playback.reason, "user interrupted")
|
||||
self.assertEqual(callback_reasons, ["user interrupted"])
|
||||
|
||||
def test_child_created_after_parent_cancel_is_cancelled_immediately(self) -> None:
|
||||
graph = CancellationGraph("turn-1")
|
||||
graph.cancel_all("timeout")
|
||||
|
||||
child = graph.child("late-child")
|
||||
|
||||
self.assertTrue(child.cancelled)
|
||||
self.assertEqual(child.reason, "timeout")
|
||||
|
||||
def test_recovery_coordinator_emits_events_and_returns_safe_state(self) -> None:
|
||||
machine = FullDuplexStateMachine(PipelineState.THINKING)
|
||||
bus = PipelineEventBus()
|
||||
coordinator = RecoveryCoordinator(state_machine=machine, event_bus=bus)
|
||||
error = ProviderError(
|
||||
ErrorCode.LLM_NETWORK_ERROR,
|
||||
"network down",
|
||||
True,
|
||||
"openai-compatible",
|
||||
"llm",
|
||||
)
|
||||
|
||||
safe_state = coordinator.recover(error, turn_id=3, session_id="session-1")
|
||||
|
||||
self.assertEqual(safe_state, PipelineState.LISTENING)
|
||||
self.assertEqual(machine.current_state, PipelineState.LISTENING)
|
||||
self.assertEqual([event.type for event in bus.events], [STAGE_ERROR, RECOVERING, SESSION_RECOVERED])
|
||||
self.assertEqual(bus.events[0].stage, "llm")
|
||||
self.assertEqual(bus.events[0].payload["code"], ErrorCode.LLM_NETWORK_ERROR.value)
|
||||
self.assertEqual(bus.events[-1].state, PipelineState.LISTENING)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,356 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from owner_voice_pet.agent_memory import FaissIndexManifest, FakeMemoryManager, MemoryRecordInput, SQLiteMemoryManager
|
||||
from owner_voice_pet.config import AppConfig
|
||||
from owner_voice_pet.full_duplex_audio import FakeWebRtcAudioProcessingProvider, RenderReferenceRingBuffer
|
||||
from owner_voice_pet.full_duplex_control import CancellationGraph, FullDuplexStateMachine
|
||||
from owner_voice_pet.full_duplex_response import (
|
||||
FakeStreamingLlmProvider,
|
||||
FakeStreamingTtsProvider,
|
||||
InterruptiblePlaybackQueue,
|
||||
LlmStreamEvent,
|
||||
SentenceSegmenter,
|
||||
)
|
||||
from owner_voice_pet.full_duplex_speech import FakeStreamingSttProvider, FakeVadProvider, InterruptionDetector, TranscriptEvent
|
||||
from owner_voice_pet.full_duplex_runtime import FullDuplexAgentRuntime
|
||||
from owner_voice_pet.full_duplex_testing import (
|
||||
PerformanceMetricRecorder,
|
||||
build_fake_full_duplex_audio_fixture,
|
||||
diagnostics_contain_sensitive_data,
|
||||
sanitize_diagnostics,
|
||||
)
|
||||
from owner_voice_pet.llm import MockLlmProvider
|
||||
from owner_voice_pet.models import AudioFrame, Message, PipelineState
|
||||
from owner_voice_pet.stt import MetadataSttProvider
|
||||
from owner_voice_pet.tool_router import MemorySearchTool, ToolCallRequest, ToolContext, ToolRouter
|
||||
from owner_voice_pet.transport import MemoryAudioTransport
|
||||
from owner_voice_pet.tts import SineTtsProvider
|
||||
from owner_voice_pet.vad import EnergyVadProvider, VadRecorder
|
||||
|
||||
|
||||
class FullDuplexIntegrationTests(unittest.TestCase):
|
||||
def test_run_agent_live_runtime_once_consumes_audio_and_replies(self) -> None:
|
||||
frames = [
|
||||
AudioFrame(b"\xff\x7f", 16000, 1, 0, 1, {"duration_ms": 20, "speech": True, "transcript": "直接提问"}),
|
||||
AudioFrame(b"\xff\x7f", 16000, 1, 20, 2, {"duration_ms": 20, "speech": True, "transcript": "直接提问"}),
|
||||
AudioFrame(b"\x00\x00", 16000, 1, 40, 3, {"duration_ms": 20, "speech": False, "transcript": "直接提问"}),
|
||||
AudioFrame(b"\x00\x00", 16000, 1, 60, 4, {"duration_ms": 20, "speech": False, "transcript": "直接提问"}),
|
||||
]
|
||||
transport = MemoryAudioTransport(frames, flush_clears_input=False)
|
||||
runtime = FullDuplexAgentRuntime(
|
||||
config=AppConfig(
|
||||
audio_apm_provider="fake",
|
||||
audio_apm_required=False,
|
||||
memory_enabled=False,
|
||||
tool_router_enabled=False,
|
||||
vad_min_duration_ms=40,
|
||||
vad_end_silence_ms=40,
|
||||
end_chime_enabled=False,
|
||||
),
|
||||
transport=transport,
|
||||
vad_recorder=VadRecorder(EnergyVadProvider(), min_duration_ms=40, end_silence_ms=40),
|
||||
stt=MetadataSttProvider(),
|
||||
llm=MockLlmProvider(["这是全双工回答。"]),
|
||||
tts=SineTtsProvider(),
|
||||
)
|
||||
|
||||
summary = runtime.run(once=True)
|
||||
|
||||
self.assertEqual(summary.completed_turns, 1)
|
||||
self.assertEqual(summary.failed_turns, 0)
|
||||
self.assertEqual([message.role for message in runtime.context.messages()], ["user", "assistant"])
|
||||
self.assertEqual(runtime.context.messages()[0].content, "直接提问")
|
||||
self.assertEqual(runtime.context.messages()[1].content, "这是全双工回答。")
|
||||
self.assertGreaterEqual(len(transport.played_segments), 1)
|
||||
self.assertIsNotNone(runtime.audio_hub)
|
||||
self.assertGreater(runtime.audio_hub.processed_capture.frame_count, 0)
|
||||
|
||||
def test_fake_apm_echo_does_not_trigger_interruption(self) -> None:
|
||||
fixture = build_fake_full_duplex_audio_fixture()
|
||||
apm = FakeWebRtcAudioProcessingProvider()
|
||||
detector = InterruptionDetector(vad=FakeVadProvider(), min_speech_ms=20)
|
||||
render = fixture[0]
|
||||
apm.process_render(render)
|
||||
|
||||
processed_echo = apm.process_capture(fixture[0])
|
||||
decision = detector.accept(
|
||||
processed_echo,
|
||||
state=PipelineState.SPEAKING,
|
||||
stt_events=[TranscriptEvent("stable_partial", "助手", is_stable=True)],
|
||||
)
|
||||
|
||||
self.assertTrue(processed_echo.metadata["echo_suppressed"])
|
||||
self.assertFalse(decision.interrupted)
|
||||
|
||||
def test_speaking_interruption_cancels_response_and_returns_to_listening(self) -> None:
|
||||
machine = FullDuplexStateMachine()
|
||||
graph = CancellationGraph("turn")
|
||||
detector = InterruptionDetector(vad=FakeVadProvider(), min_speech_ms=200)
|
||||
machine.transition(PipelineState.LISTENING, event_type="start")
|
||||
machine.transition(PipelineState.THINKING, event_type="final_transcript")
|
||||
machine.transition(PipelineState.SPEAKING, event_type="first_tts_chunk")
|
||||
|
||||
detector.accept(
|
||||
build_fake_full_duplex_audio_fixture()[1],
|
||||
state=PipelineState.SPEAKING,
|
||||
stt_events=[TranscriptEvent("partial", "你", is_stable=False)],
|
||||
)
|
||||
decision = detector.accept(
|
||||
build_fake_full_duplex_audio_fixture()[2],
|
||||
state=PipelineState.SPEAKING,
|
||||
stt_events=[TranscriptEvent("stable_partial", "你好", is_stable=True)],
|
||||
)
|
||||
if decision.interrupted:
|
||||
graph.cancel_all("user interrupted")
|
||||
machine.transition(PipelineState.INTERRUPTED, event_type="interrupt_detected")
|
||||
machine.transition(PipelineState.LISTENING, event_type="buffered_user_audio")
|
||||
|
||||
self.assertTrue(decision.interrupted)
|
||||
self.assertTrue(graph.root.cancelled)
|
||||
self.assertEqual(machine.current_state, PipelineState.LISTENING)
|
||||
|
||||
def test_full_duplex_runtime_interrupt_fixture_uses_audio_hub_and_buffers_user_audio(self) -> None:
|
||||
runtime = FullDuplexAgentRuntime(
|
||||
config=AppConfig(
|
||||
audio_apm_provider="fake",
|
||||
audio_apm_required=False,
|
||||
barge_in_min_speech_ms=200,
|
||||
)
|
||||
)
|
||||
fixture = build_fake_full_duplex_audio_fixture()[1:3]
|
||||
|
||||
summary = runtime.run_interrupt_fixture(fixture, initial_state=PipelineState.SPEAKING)
|
||||
|
||||
self.assertTrue(summary.interrupted)
|
||||
self.assertTrue(runtime.cancellation_graph.root.cancelled)
|
||||
self.assertEqual(runtime.state_machine.current_state, PipelineState.LISTENING)
|
||||
self.assertIsNotNone(runtime.interrupt_controller)
|
||||
self.assertEqual(
|
||||
[item.frame_id for item in runtime.interrupt_controller.buffered_user_frames],
|
||||
[2, 3],
|
||||
)
|
||||
|
||||
def test_streaming_stt_llm_tts_playback_order(self) -> None:
|
||||
stt = FakeStreamingSttProvider(
|
||||
scripted_events=[
|
||||
[TranscriptEvent("partial", "你", is_stable=False)],
|
||||
[TranscriptEvent("stable_partial", "你好", is_stable=True)],
|
||||
],
|
||||
final_text="你好",
|
||||
)
|
||||
stt_session = stt.start_session("turn")
|
||||
llm = FakeStreamingLlmProvider([LlmStreamEvent("delta", "你好。"), LlmStreamEvent("finish", finish_reason="stop")])
|
||||
segmenter = SentenceSegmenter()
|
||||
tts_session = FakeStreamingTtsProvider().start_stream(voice="default", sample_rate=16000)
|
||||
playback = InterruptiblePlaybackQueue()
|
||||
render = RenderReferenceRingBuffer(capacity_ms=1000)
|
||||
graph = CancellationGraph("turn")
|
||||
event_order: list[str] = []
|
||||
|
||||
for frame in build_fake_full_duplex_audio_fixture()[1:3]:
|
||||
for event in stt_session.accept_audio(frame):
|
||||
event_order.append(event.kind)
|
||||
final = stt_session.finish()
|
||||
event_order.append(final.kind)
|
||||
for llm_event in llm.stream([Message("user", final.text, 1.0)], cancellation=graph.root):
|
||||
event_order.append(f"llm_{llm_event.kind}")
|
||||
if llm_event.text_delta:
|
||||
for sentence in segmenter.accept_delta(llm_event.text_delta):
|
||||
event_order.append("sentence")
|
||||
frames = tts_session.accept_text(sentence)
|
||||
event_order.append("tts")
|
||||
playback.enqueue(sentence, frames)
|
||||
result = playback.play_next(render_reference=render, cancellation=graph.root)
|
||||
event_order.append("playback")
|
||||
|
||||
self.assertEqual(
|
||||
event_order,
|
||||
["partial", "stable_partial", "final", "llm_delta", "sentence", "tts", "llm_finish", "playback"],
|
||||
)
|
||||
self.assertFalse(result.interrupted)
|
||||
self.assertEqual(playback.spoken.text, "你好。")
|
||||
self.assertEqual(render.frame_count, 1)
|
||||
|
||||
def test_full_duplex_runtime_streaming_response_writes_render_reference_and_spoken_text(self) -> None:
|
||||
runtime = FullDuplexAgentRuntime(
|
||||
config=AppConfig(audio_apm_provider="fake", audio_apm_required=False),
|
||||
llm_provider=FakeStreamingLlmProvider(
|
||||
[LlmStreamEvent("delta", "第一句。第二句。"), LlmStreamEvent("finish", finish_reason="stop")]
|
||||
),
|
||||
tts_provider=FakeStreamingTtsProvider(),
|
||||
)
|
||||
|
||||
spoken = runtime.run_streaming_response_fixture([Message("user", "你好", 1.0)])
|
||||
|
||||
self.assertEqual(spoken, "第一句。第二句。")
|
||||
self.assertIsNotNone(runtime.audio_hub)
|
||||
self.assertEqual(runtime.audio_hub.render_reference.frame_count, 2)
|
||||
self.assertEqual(runtime.playback_queue.pending_items, 0)
|
||||
|
||||
def test_full_duplex_runtime_injects_memory_context_before_llm(self) -> None:
|
||||
memory = FakeMemoryManager()
|
||||
memory.save(MemoryRecordInput("preference", "用户喜欢 Python"))
|
||||
llm = FakeStreamingLlmProvider([LlmStreamEvent("delta", "记住了。"), LlmStreamEvent("finish", finish_reason="stop")])
|
||||
runtime = FullDuplexAgentRuntime(
|
||||
config=AppConfig(audio_apm_provider="fake", audio_apm_required=False, memory_enabled=True),
|
||||
memory_manager=memory,
|
||||
llm_provider=llm,
|
||||
tts_provider=FakeStreamingTtsProvider(),
|
||||
)
|
||||
|
||||
spoken = runtime.run_conversation_response_fixture("Python 项目怎么做？")
|
||||
|
||||
self.assertEqual(spoken, "记住了。")
|
||||
self.assertIn("长期记忆", llm.requests[0][1].content)
|
||||
self.assertIn("用户喜欢 Python", llm.requests[0][1].content)
|
||||
self.assertEqual([message.role for message in runtime.context.messages()], ["user", "assistant"])
|
||||
|
||||
def test_full_duplex_runtime_memory_health_detects_manifest_mismatch(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
memory = SQLiteMemoryManager(Path(tmp) / "memory.sqlite3")
|
||||
saved = memory.save(MemoryRecordInput("fact", "Owner 正在做全双工语音助手"))
|
||||
broken = FaissIndexManifest(
|
||||
embedding_model="fake",
|
||||
record_ids=(saved.id, "missing-id"),
|
||||
checksums={saved.id: "wrong"},
|
||||
)
|
||||
runtime = FullDuplexAgentRuntime(
|
||||
config=AppConfig(audio_apm_provider="fake", audio_apm_required=False, memory_enabled=True),
|
||||
memory_manager=memory,
|
||||
memory_manifest=broken,
|
||||
)
|
||||
|
||||
health = runtime.check_memory_health()
|
||||
|
||||
self.assertFalse(health.ok)
|
||||
self.assertTrue(any("missing-id" in error for error in health.errors))
|
||||
self.assertTrue(any("checksum mismatch" in error for error in health.errors))
|
||||
|
||||
def test_full_duplex_runtime_routes_memory_search_tool_call(self) -> None:
|
||||
memory = FakeMemoryManager()
|
||||
memory.save(MemoryRecordInput("project", "Owner 项目正在做全双工 Agent"))
|
||||
runtime = FullDuplexAgentRuntime(
|
||||
config=AppConfig(
|
||||
audio_apm_provider="fake",
|
||||
audio_apm_required=False,
|
||||
memory_enabled=True,
|
||||
tool_router_enabled=True,
|
||||
),
|
||||
memory_manager=memory,
|
||||
llm_provider=FakeStreamingLlmProvider(
|
||||
[
|
||||
LlmStreamEvent(
|
||||
"tool_call",
|
||||
tool_call={
|
||||
"id": "tool-1",
|
||||
"name": "memory.search",
|
||||
"arguments": {"query": "Owner Agent", "top_k": 1},
|
||||
"turn_id": "turn-1",
|
||||
},
|
||||
),
|
||||
LlmStreamEvent("delta", "查到了。"),
|
||||
LlmStreamEvent("finish", finish_reason="stop"),
|
||||
]
|
||||
),
|
||||
tts_provider=FakeStreamingTtsProvider(),
|
||||
)
|
||||
|
||||
spoken = runtime.run_conversation_response_fixture("查一下当前项目")
|
||||
|
||||
self.assertEqual(spoken, "查到了。")
|
||||
self.assertEqual(runtime.tool_results[0].status, "success")
|
||||
self.assertIn("全双工 Agent", runtime.tool_results[0].output_text)
|
||||
self.assertIn("工具结果 memory.search", runtime.tool_result_messages[0].content)
|
||||
|
||||
def test_full_duplex_runtime_high_risk_tool_call_requires_confirmation(self) -> None:
|
||||
runtime = FullDuplexAgentRuntime(
|
||||
config=AppConfig(
|
||||
audio_apm_provider="fake",
|
||||
audio_apm_required=False,
|
||||
memory_enabled=True,
|
||||
tool_router_enabled=True,
|
||||
),
|
||||
memory_manager=FakeMemoryManager(),
|
||||
llm_provider=FakeStreamingLlmProvider(
|
||||
[
|
||||
LlmStreamEvent(
|
||||
"tool_call",
|
||||
tool_call={
|
||||
"id": "tool-1",
|
||||
"name": "memory.search",
|
||||
"arguments": {"query": "账号"},
|
||||
"natural_language_intent": "上传账号资料",
|
||||
"turn_id": "turn-1",
|
||||
},
|
||||
),
|
||||
LlmStreamEvent("finish", finish_reason="stop"),
|
||||
]
|
||||
),
|
||||
tts_provider=FakeStreamingTtsProvider(),
|
||||
)
|
||||
|
||||
runtime.run_conversation_response_fixture("上传账号资料")
|
||||
|
||||
self.assertEqual(runtime.tool_results[0].status, "confirmation_required")
|
||||
self.assertEqual(runtime.tool_router.audit_log[0].action, "require_confirmation")
|
||||
|
||||
def test_memory_restart_and_tool_search_integration(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "memory.sqlite3"
|
||||
SQLiteMemoryManager(db_path).save(MemoryRecordInput("preference", "用户喜欢 Python"))
|
||||
restarted = SQLiteMemoryManager(db_path)
|
||||
router = ToolRouter({"memory.search": MemorySearchTool()})
|
||||
request = ToolCallRequest("1", "memory.search", {"query": "Python"}, "turn")
|
||||
decision = router.route(request, ToolContext(memory=restarted))
|
||||
result = router.execute(request, decision, ToolContext(memory=restarted))
|
||||
|
||||
self.assertEqual(result.status, "success")
|
||||
self.assertIn("用户喜欢 Python", result.output_text)
|
||||
|
||||
def test_tool_router_security_blocks_high_risk_fake_integration(self) -> None:
|
||||
router = ToolRouter({"memory.search": MemorySearchTool()})
|
||||
request = ToolCallRequest(
|
||||
"1",
|
||||
"memory.search",
|
||||
{"query": "账号"},
|
||||
"turn",
|
||||
natural_language_intent="上传账号资料",
|
||||
)
|
||||
|
||||
decision = router.route(request, ToolContext(memory=FakeMemoryManager()))
|
||||
|
||||
self.assertEqual(decision.action, "require_confirmation")
|
||||
self.assertEqual(decision.risk_level, "high")
|
||||
|
||||
def test_performance_metrics_are_sanitized(self) -> None:
|
||||
recorder = PerformanceMetricRecorder()
|
||||
recorder.record(
|
||||
"interrupt_latency",
|
||||
started_at_ms=100,
|
||||
finished_at_ms=250,
|
||||
payload={"api_key": "secret", "preview": "tp-" + "abcdefghijklmnop"},
|
||||
)
|
||||
|
||||
self.assertEqual(recorder.summary()["interrupt_latency"], 150)
|
||||
self.assertEqual(recorder.metrics[0].payload["api_key"], "[redacted]")
|
||||
self.assertFalse(diagnostics_contain_sensitive_data(recorder.metrics[0].payload))
|
||||
|
||||
def test_sanitize_diagnostics_removes_nested_sensitive_values(self) -> None:
|
||||
sanitized = sanitize_diagnostics(
|
||||
{
|
||||
"nested": {"authorization": "Bearer secret", "raw_audio": b"bytes"},
|
||||
"text": "normal",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(sanitized["nested"]["authorization"], "[redacted]")
|
||||
self.assertEqual(sanitized["nested"]["raw_audio"], "[redacted]")
|
||||
self.assertFalse(diagnostics_contain_sensitive_data(sanitized))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,150 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from owner_voice_pet.full_duplex_audio import AudioHub, FakeWebRtcAudioProcessingProvider, RenderReferenceRingBuffer
|
||||
from owner_voice_pet.full_duplex_control import CancellationGraph
|
||||
from owner_voice_pet.full_duplex_response import (
|
||||
AudioSegmentStreamingTtsProvider,
|
||||
FakeStreamingLlmProvider,
|
||||
FakeStreamingTtsProvider,
|
||||
InterruptiblePlaybackQueue,
|
||||
LlmStreamEvent,
|
||||
SentenceSegmenter,
|
||||
prepare_tts_sentence,
|
||||
)
|
||||
from owner_voice_pet.models import AudioFrame, Message
|
||||
from owner_voice_pet.tts import SineTtsProvider
|
||||
|
||||
|
||||
class FullDuplexResponseTests(unittest.TestCase):
|
||||
def test_fake_llm_stream_stops_on_cancellation(self) -> None:
|
||||
graph = CancellationGraph("turn")
|
||||
provider = FakeStreamingLlmProvider(
|
||||
[
|
||||
LlmStreamEvent("delta", "第一句。"),
|
||||
LlmStreamEvent("delta", "第二句。"),
|
||||
]
|
||||
)
|
||||
messages = [Message("user", "你好", 1.0)]
|
||||
|
||||
iterator = provider.stream(messages, cancellation=graph.root)
|
||||
first = next(iterator)
|
||||
graph.cancel_all("interrupt")
|
||||
remaining = list(iterator)
|
||||
|
||||
self.assertEqual(first.text_delta, "第一句。")
|
||||
self.assertEqual(remaining, [])
|
||||
self.assertEqual(provider.requests, [messages])
|
||||
|
||||
def test_sentence_segmenter_splits_chinese_sentences(self) -> None:
|
||||
segmenter = SentenceSegmenter()
|
||||
|
||||
emitted = segmenter.accept_delta("你好。你想听哪一部分？")
|
||||
|
||||
self.assertEqual(emitted, ["你好。", "你想听哪一部分？"])
|
||||
self.assertIsNone(segmenter.flush())
|
||||
|
||||
def test_sentence_segmenter_does_not_split_decimal_numbers(self) -> None:
|
||||
segmenter = SentenceSegmenter()
|
||||
|
||||
emitted = segmenter.accept_delta("版本 1.2。结束")
|
||||
tail = segmenter.flush()
|
||||
|
||||
self.assertEqual(emitted, ["版本 1.2。"])
|
||||
self.assertEqual(tail, "结束")
|
||||
|
||||
def test_prepare_tts_sentence_reuses_sanitizer(self) -> None:
|
||||
self.assertEqual(prepare_tts_sentence("你好 😊"), "你好")
|
||||
|
||||
def test_fake_streaming_tts_outputs_audio_frame(self) -> None:
|
||||
provider = FakeStreamingTtsProvider()
|
||||
session = provider.start_stream(voice="default", sample_rate=16000)
|
||||
|
||||
frames = session.accept_text("你好。")
|
||||
|
||||
self.assertEqual(provider.started, [("default", 16000)])
|
||||
self.assertEqual(session.accepted_text, ["你好。"])
|
||||
self.assertEqual(len(frames), 1)
|
||||
self.assertEqual(frames[0].metadata["tts_text"], "你好。")
|
||||
|
||||
def test_fake_streaming_tts_skips_empty_sanitized_text(self) -> None:
|
||||
session = FakeStreamingTtsProvider().start_stream(voice="default", sample_rate=16000)
|
||||
|
||||
frames = session.accept_text("😂😂")
|
||||
|
||||
self.assertEqual(frames, [])
|
||||
self.assertEqual(session.accepted_text, [])
|
||||
|
||||
def test_audio_segment_streaming_tts_wraps_sync_tts_as_pcm_chunks(self) -> None:
|
||||
provider = AudioSegmentStreamingTtsProvider(SineTtsProvider(), chunk_ms=30)
|
||||
session = provider.start_stream(voice="default", sample_rate=16000)
|
||||
|
||||
frames = session.accept_text("你好。")
|
||||
|
||||
self.assertEqual(provider.started, [("default", 16000)])
|
||||
self.assertEqual(session.accepted_text, ["你好。"])
|
||||
self.assertGreater(len(frames), 1)
|
||||
self.assertTrue(all(frame.metadata["tts_text"] == "你好。" for frame in frames))
|
||||
|
||||
def test_interruptible_playback_commits_only_fully_played_text(self) -> None:
|
||||
queue = InterruptiblePlaybackQueue()
|
||||
render = RenderReferenceRingBuffer(capacity_ms=1000)
|
||||
graph = CancellationGraph("turn")
|
||||
frames = [
|
||||
AudioFrame(b"1", 16000, 1, 0, 1, {"duration_ms": 20}),
|
||||
AudioFrame(b"2", 16000, 1, 20, 2, {"duration_ms": 20}),
|
||||
]
|
||||
queue.enqueue("完整句。", frames)
|
||||
|
||||
result = queue.play_next(render_reference=render, cancellation=graph.root)
|
||||
|
||||
self.assertFalse(result.interrupted)
|
||||
self.assertEqual(result.played_frames, 2)
|
||||
self.assertEqual(result.committed_text, "完整句。")
|
||||
self.assertEqual(queue.spoken.text, "完整句。")
|
||||
self.assertEqual(render.frame_count, 2)
|
||||
|
||||
def test_interruptible_playback_writes_audio_hub_render_reference(self) -> None:
|
||||
queue = InterruptiblePlaybackQueue()
|
||||
graph = CancellationGraph("turn")
|
||||
processor = FakeWebRtcAudioProcessingProvider()
|
||||
hub = AudioHub(processor=processor, render_capacity_ms=1000)
|
||||
frames = [AudioFrame(b"1", 16000, 1, 0, 1, {"duration_ms": 20})]
|
||||
queue.enqueue("一句。", frames)
|
||||
|
||||
result = queue.play_next(audio_hub=hub, cancellation=graph.root)
|
||||
|
||||
self.assertFalse(result.interrupted)
|
||||
self.assertEqual(hub.render_reference.frame_count, 1)
|
||||
self.assertEqual([item.frame_id for item in processor.render_frames], [1])
|
||||
|
||||
def test_interruptible_playback_does_not_commit_interrupted_text(self) -> None:
|
||||
queue = InterruptiblePlaybackQueue()
|
||||
render = RenderReferenceRingBuffer(capacity_ms=1000)
|
||||
graph = CancellationGraph("turn")
|
||||
frames = [
|
||||
AudioFrame(b"1", 16000, 1, 0, 1, {"duration_ms": 20}),
|
||||
AudioFrame(b"2", 16000, 1, 20, 2, {"duration_ms": 20}),
|
||||
]
|
||||
queue.enqueue("未完整句。", frames)
|
||||
graph.root.add_callback(lambda reason: None)
|
||||
|
||||
original_write = render.write
|
||||
|
||||
def cancel_after_first(frame: AudioFrame):
|
||||
result = original_write(frame)
|
||||
graph.cancel_all("interrupt")
|
||||
return result
|
||||
|
||||
render.write = cancel_after_first # type: ignore[method-assign]
|
||||
result = queue.play_next(render_reference=render, cancellation=graph.root)
|
||||
|
||||
self.assertTrue(result.interrupted)
|
||||
self.assertEqual(result.played_frames, 1)
|
||||
self.assertEqual(queue.spoken.text, "")
|
||||
self.assertEqual(queue.pending_items, 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,243 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from owner_voice_pet.full_duplex_speech import (
|
||||
FakeStreamingSttProvider,
|
||||
FakeVadProvider,
|
||||
InterruptController,
|
||||
InterruptionDetector,
|
||||
SileroVadProvider,
|
||||
StreamingSttWorker,
|
||||
TranscriptEvent,
|
||||
)
|
||||
from owner_voice_pet.full_duplex_control import CancellationGraph, FullDuplexStateMachine
|
||||
from owner_voice_pet.models import AudioFrame, ErrorCode, PipelineState, ProviderError
|
||||
|
||||
|
||||
def frame(
|
||||
frame_id: int,
|
||||
timestamp_ms: int,
|
||||
*,
|
||||
duration_ms: int = 100,
|
||||
speech: bool = False,
|
||||
metadata: dict[str, object] | None = None,
|
||||
) -> AudioFrame:
|
||||
data = {"duration_ms": duration_ms, "speech": speech}
|
||||
if metadata:
|
||||
data.update(metadata)
|
||||
return AudioFrame(
|
||||
b"\x01\x00" * 800,
|
||||
16000,
|
||||
1,
|
||||
timestamp_ms,
|
||||
frame_id,
|
||||
data,
|
||||
)
|
||||
|
||||
|
||||
class FullDuplexSpeechTests(unittest.TestCase):
|
||||
def test_fake_vad_reports_speech_start_and_end(self) -> None:
|
||||
vad = FakeVadProvider(end_silence_ms=100)
|
||||
|
||||
start = vad.accept_audio(frame(1, 0, speech=True))
|
||||
middle = vad.accept_audio(frame(2, 100, speech=True))
|
||||
end = vad.accept_audio(frame(3, 200, speech=False))
|
||||
|
||||
self.assertTrue(start.speech_started)
|
||||
self.assertFalse(middle.speech_started)
|
||||
self.assertTrue(end.speech_ended)
|
||||
self.assertEqual(middle.speech_ms, 200)
|
||||
|
||||
def test_silero_vad_missing_model_has_structured_error(self) -> None:
|
||||
provider = SileroVadProvider(model_path=Path("/tmp/owner-missing-silero-vad.onnx"))
|
||||
|
||||
with self.assertRaises(ProviderError) as raised:
|
||||
provider.load()
|
||||
|
||||
self.assertEqual(raised.exception.code, ErrorCode.VAD_MODEL_LOAD_FAILED)
|
||||
|
||||
def test_silero_vad_loaded_provider_validates_sample_rate(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
model_path = Path(tmp) / "silero_vad.onnx"
|
||||
model_path.write_bytes(b"placeholder")
|
||||
provider = SileroVadProvider(model_path=model_path, sample_rate=16000)
|
||||
provider.load()
|
||||
|
||||
with self.assertRaises(ProviderError) as raised:
|
||||
provider.accept_audio(
|
||||
AudioFrame(b"\x00\x00", 8000, 1, 0, 0, {"duration_ms": 20, "speech": True})
|
||||
)
|
||||
|
||||
self.assertEqual(raised.exception.code, ErrorCode.AUDIO_FORMAT_UNSUPPORTED)
|
||||
|
||||
def test_fake_streaming_stt_emits_partial_and_final(self) -> None:
|
||||
provider = FakeStreamingSttProvider(final_text="最终问题")
|
||||
session = provider.start_session("session-1")
|
||||
|
||||
partials = session.accept_audio(frame(1, 0, speech=True, metadata={"partial": "你"}))
|
||||
final = session.finish()
|
||||
|
||||
self.assertEqual(provider.started_sessions, ["session-1"])
|
||||
self.assertEqual(partials, [TranscriptEvent("partial", "你", is_stable=False, confidence=0.6)])
|
||||
self.assertEqual(final.kind, "final")
|
||||
self.assertEqual(final.text, "最终问题")
|
||||
self.assertTrue(final.is_stable)
|
||||
|
||||
def test_fake_streaming_stt_cancel_raises_on_finish(self) -> None:
|
||||
session = FakeStreamingSttProvider(final_text="不会输出").start_session("session-1")
|
||||
|
||||
session.cancel("interrupt")
|
||||
|
||||
with self.assertRaises(ProviderError) as raised:
|
||||
session.finish()
|
||||
|
||||
self.assertEqual(raised.exception.code, ErrorCode.STT_TRANSCRIBE_FAILED)
|
||||
|
||||
def test_interruption_detector_triggers_during_speaking_with_stable_partial(self) -> None:
|
||||
detector = InterruptionDetector(
|
||||
vad=FakeVadProvider(),
|
||||
min_speech_ms=200,
|
||||
target_latency_ms=200,
|
||||
)
|
||||
|
||||
first = detector.accept(
|
||||
frame(1, 1000, speech=True),
|
||||
state=PipelineState.SPEAKING,
|
||||
stt_events=[TranscriptEvent("partial", "你", is_stable=False)],
|
||||
)
|
||||
second = detector.accept(
|
||||
frame(2, 1100, speech=True),
|
||||
state=PipelineState.SPEAKING,
|
||||
stt_events=[TranscriptEvent("stable_partial", "你好", is_stable=True)],
|
||||
)
|
||||
|
||||
self.assertFalse(first.interrupted)
|
||||
self.assertTrue(second.interrupted)
|
||||
self.assertEqual(second.reason, "user_speech")
|
||||
self.assertEqual(second.latency_ms, 100)
|
||||
self.assertLessEqual(second.latency_ms or 999, detector.target_latency_ms)
|
||||
|
||||
def test_interruption_detector_does_not_wait_for_stt_partial_by_default(self) -> None:
|
||||
detector = InterruptionDetector(
|
||||
vad=FakeVadProvider(),
|
||||
min_speech_ms=200,
|
||||
target_latency_ms=200,
|
||||
)
|
||||
|
||||
first = detector.accept(frame(1, 1000, speech=True), state=PipelineState.SPEAKING)
|
||||
second = detector.accept(frame(2, 1100, speech=True), state=PipelineState.SPEAKING)
|
||||
|
||||
self.assertFalse(first.interrupted)
|
||||
self.assertTrue(second.interrupted)
|
||||
self.assertEqual(second.reason, "user_speech")
|
||||
|
||||
def test_interruption_detector_allows_thinking_and_tool_running_interrupts(self) -> None:
|
||||
thinking = InterruptionDetector(vad=FakeVadProvider(), min_speech_ms=100)
|
||||
tool_running = InterruptionDetector(vad=FakeVadProvider(), min_speech_ms=100)
|
||||
|
||||
thinking_decision = thinking.accept(frame(1, 0, speech=True), state=PipelineState.THINKING)
|
||||
tool_decision = tool_running.accept(frame(2, 0, speech=True), state=PipelineState.TOOL_RUNNING)
|
||||
|
||||
self.assertTrue(thinking_decision.interrupted)
|
||||
self.assertTrue(tool_decision.interrupted)
|
||||
|
||||
def test_interruption_detector_ignores_non_speaking_state(self) -> None:
|
||||
detector = InterruptionDetector(vad=FakeVadProvider(), min_speech_ms=100)
|
||||
|
||||
decision = detector.accept(
|
||||
frame(1, 0, speech=True),
|
||||
state=PipelineState.LISTENING,
|
||||
stt_events=[TranscriptEvent("stable_partial", "你好", is_stable=True)],
|
||||
)
|
||||
|
||||
self.assertFalse(decision.interrupted)
|
||||
|
||||
def test_interruption_detector_rejects_assistant_echo(self) -> None:
|
||||
detector = InterruptionDetector(vad=FakeVadProvider(), min_speech_ms=100)
|
||||
|
||||
decision = detector.accept(
|
||||
frame(1, 0, speech=True, metadata={"assistant_echo": True}),
|
||||
state=PipelineState.SPEAKING,
|
||||
stt_events=[TranscriptEvent("stable_partial", "助手声音", is_stable=True)],
|
||||
)
|
||||
|
||||
self.assertFalse(decision.interrupted)
|
||||
self.assertEqual(decision.reason, "assistant_echo_rejected")
|
||||
|
||||
def test_interruption_detector_waits_for_stable_partial(self) -> None:
|
||||
detector = InterruptionDetector(
|
||||
vad=FakeVadProvider(),
|
||||
min_speech_ms=100,
|
||||
require_stable_partial=True,
|
||||
)
|
||||
|
||||
decision = detector.accept(
|
||||
frame(1, 0, speech=True),
|
||||
state=PipelineState.SPEAKING,
|
||||
stt_events=[TranscriptEvent("partial", "你", is_stable=False)],
|
||||
)
|
||||
|
||||
self.assertFalse(decision.interrupted)
|
||||
self.assertEqual(decision.reason, "waiting_for_stable_partial")
|
||||
|
||||
def test_interrupt_controller_cancels_graph_transitions_and_buffers_user_audio(self) -> None:
|
||||
machine = FullDuplexStateMachine()
|
||||
machine.transition(PipelineState.LISTENING, event_type="start")
|
||||
machine.transition(PipelineState.THINKING, event_type="final_transcript")
|
||||
machine.transition(PipelineState.SPEAKING, event_type="first_tts_chunk")
|
||||
graph = CancellationGraph("turn")
|
||||
controller = InterruptController(
|
||||
detector=InterruptionDetector(vad=FakeVadProvider(), min_speech_ms=200),
|
||||
state_machine=machine,
|
||||
cancellation_graph=graph,
|
||||
)
|
||||
|
||||
first = controller.accept_frame(frame(1, 1000, speech=True))
|
||||
second = controller.accept_frame(frame(2, 1100, speech=True))
|
||||
|
||||
self.assertFalse(first.decision.interrupted)
|
||||
self.assertTrue(second.decision.interrupted)
|
||||
self.assertTrue(graph.root.cancelled)
|
||||
self.assertEqual(machine.current_state, PipelineState.LISTENING)
|
||||
self.assertEqual([item.frame_id for item in controller.buffered_user_frames], [1, 2])
|
||||
|
||||
def test_interrupt_controller_rejects_echo_and_does_not_cancel(self) -> None:
|
||||
machine = FullDuplexStateMachine()
|
||||
machine.transition(PipelineState.LISTENING, event_type="start")
|
||||
machine.transition(PipelineState.THINKING, event_type="final_transcript")
|
||||
machine.transition(PipelineState.SPEAKING, event_type="first_tts_chunk")
|
||||
graph = CancellationGraph("turn")
|
||||
controller = InterruptController(
|
||||
detector=InterruptionDetector(vad=FakeVadProvider(), min_speech_ms=100),
|
||||
state_machine=machine,
|
||||
cancellation_graph=graph,
|
||||
)
|
||||
|
||||
result = controller.accept_frame(frame(1, 0, speech=True, metadata={"echo_suppressed": True}))
|
||||
|
||||
self.assertFalse(result.decision.interrupted)
|
||||
self.assertFalse(graph.root.cancelled)
|
||||
self.assertEqual(controller.buffered_user_frames, ())
|
||||
|
||||
def test_streaming_stt_worker_keeps_partials_out_of_final_until_finish(self) -> None:
|
||||
worker = StreamingSttWorker(
|
||||
provider=FakeStreamingSttProvider(
|
||||
scripted_events=[[TranscriptEvent("partial", "你", is_stable=False)]],
|
||||
final_text="你好",
|
||||
),
|
||||
session_id="turn-1",
|
||||
)
|
||||
|
||||
partials = worker.accept_frame(frame(1, 0, speech=True))
|
||||
final = worker.finish()
|
||||
|
||||
self.assertEqual(partials, [TranscriptEvent("partial", "你", is_stable=False)])
|
||||
self.assertEqual(final, TranscriptEvent("final", "你好", is_stable=True, confidence=0.9))
|
||||
self.assertEqual([event.kind for event in worker.events], ["partial", "final"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -204,7 +204,12 @@ def make_runtime(
|
||||
reporter = RecordingReporter()
|
||||
event_bus = PipelineEventBus()
|
||||
runtime = VoiceAssistantPipeline(
|
||||
config=AppConfig(llm_api_key="secret", speech_provider="cloud", wake_ack_text=wake_ack_text),
|
||||
config=AppConfig(
|
||||
llm_api_key="secret",
|
||||
speech_provider="cloud",
|
||||
wake_ack_text=wake_ack_text,
|
||||
barge_in_enabled=False,
|
||||
),
|
||||
transport=transport,
|
||||
wakeword=KeywordWakeWordProvider(),
|
||||
vad_recorder=VadRecorder(EnergyVadProvider(), min_duration_ms=40, end_silence_ms=40),
|
||||
@@ -261,6 +266,40 @@ def silence_frames(start_id: int, start_ms: int, count: int) -> list[AudioFrame]
|
||||
|
||||
|
||||
class LiveRuntimeTests(unittest.TestCase):
|
||||
def test_agent_runtime_listens_without_wake_word(self) -> None:
|
||||
frames = segment_frames(1, 20, partials=["直接提问", "直接提问"])
|
||||
transport = MemoryAudioTransport(frames, flush_clears_input=False)
|
||||
stt = QueueSttProvider(["直接提问"])
|
||||
llm = QueueLlmProvider([["这是全双工回答。"]])
|
||||
reporter = RecordingReporter()
|
||||
runtime = VoiceAssistantPipeline(
|
||||
config=AppConfig(
|
||||
assistant_mode="full_duplex_agent",
|
||||
llm_api_key="secret",
|
||||
speech_provider="cloud",
|
||||
wake_ack_text="",
|
||||
),
|
||||
transport=transport,
|
||||
wakeword=KeywordWakeWordProvider(),
|
||||
vad_recorder=VadRecorder(EnergyVadProvider(), min_duration_ms=40, end_silence_ms=40),
|
||||
stt=stt,
|
||||
realtime_stt=MetadataSttProvider(),
|
||||
llm=llm,
|
||||
tts=SineTtsProvider(),
|
||||
context=ConversationContext(),
|
||||
reporter=reporter,
|
||||
event_bus=PipelineEventBus(),
|
||||
)
|
||||
|
||||
summary = runtime.run_agent(once=True)
|
||||
|
||||
self.assertEqual(summary.completed_turns, 1)
|
||||
self.assertEqual(stt.calls[0].metadata["end_reason"], "silence")
|
||||
self.assertEqual(reporter.transcripts, ["直接提问"])
|
||||
self.assertIn("监听中：请直接说话", reporter.statuses)
|
||||
self.assertNotIn("唤醒命中", reporter.statuses)
|
||||
self.assertIn("恢复监听：可直接说话", reporter.statuses)
|
||||
|
||||
def test_repeated_runtime_runs_two_turns_and_returns_to_standby(self) -> None:
|
||||
runtime, stt, llm, transport, reporter = make_runtime(["第一问", "第二问"])
|
||||
self.assertIsInstance(runtime, VoiceAssistantPipeline)
|
||||
@@ -311,7 +350,12 @@ class LiveRuntimeTests(unittest.TestCase):
|
||||
transport = MemoryAudioTransport(frames, flush_clears_input=False)
|
||||
ack_tts = CountingTtsProvider()
|
||||
runtime = VoiceAssistantPipeline(
|
||||
config=AppConfig(llm_api_key="secret", speech_provider="cloud", wake_ack_text="我在"),
|
||||
config=AppConfig(
|
||||
llm_api_key="secret",
|
||||
speech_provider="cloud",
|
||||
wake_ack_text="我在",
|
||||
barge_in_enabled=False,
|
||||
),
|
||||
transport=transport,
|
||||
wakeword=KeywordWakeWordProvider(),
|
||||
vad_recorder=VadRecorder(EnergyVadProvider(), min_duration_ms=40, end_silence_ms=40),
|
||||
@@ -329,8 +373,8 @@ class LiveRuntimeTests(unittest.TestCase):
|
||||
|
||||
self.assertEqual(summary.completed_turns, 2)
|
||||
self.assertEqual(ack_tts.synthesized_texts, ["我在"])
|
||||
self.assertEqual(transport.played_segments[0].metadata["text"], "我在")
|
||||
self.assertEqual(transport.played_segments[3].metadata["text"], "我在")
|
||||
ack_segments = [segment for segment in transport.played_segments if segment.metadata.get("text") == "我在"]
|
||||
self.assertEqual(len(ack_segments), 2)
|
||||
|
||||
def test_zero_post_playback_drain_flushes_without_dropping_prefilled_question(self) -> None:
|
||||
runtime, stt, _, transport, reporter = make_runtime(["第一问"])
|
||||
@@ -353,7 +397,7 @@ class LiveRuntimeTests(unittest.TestCase):
|
||||
self.assertEqual(summary.completed_turns, 1)
|
||||
self.assertEqual(reporter.transcripts, ["第一问"])
|
||||
self.assertEqual(len(stt.calls), 1)
|
||||
self.assertEqual(len(transport.played_segments), 2)
|
||||
self.assertGreaterEqual(len(transport.played_segments), 2)
|
||||
self.assertEqual(transport.flush_count, 2)
|
||||
self.assertEqual(transport.played_segments[-1].metadata["chime"], "end")
|
||||
self.assertEqual(transport.played_segments[-1].metadata["source"], "file")
|
||||
@@ -366,8 +410,8 @@ class LiveRuntimeTests(unittest.TestCase):
|
||||
summary = runtime.run(max_turns=1)
|
||||
|
||||
self.assertEqual(summary.completed_turns, 1)
|
||||
self.assertEqual(len(transport.played_segments), 1)
|
||||
self.assertNotIn("chime", transport.played_segments[-1].metadata)
|
||||
self.assertGreaterEqual(len(transport.played_segments), 1)
|
||||
self.assertFalse(any("chime" in segment.metadata for segment in transport.played_segments))
|
||||
|
||||
def test_temporary_context_is_sent_to_second_llm_call(self) -> None:
|
||||
runtime, _, llm, _, _ = make_runtime(["第一问", "第二问"])
|
||||
@@ -553,6 +597,7 @@ class LiveRuntimeTests(unittest.TestCase):
|
||||
speech_provider="cloud",
|
||||
wake_ack_text="",
|
||||
followup_listen_timeout_ms=3000,
|
||||
barge_in_enabled=False,
|
||||
),
|
||||
transport=transport,
|
||||
wakeword=KeywordWakeWordProvider(),
|
||||
@@ -593,6 +638,7 @@ class LiveRuntimeTests(unittest.TestCase):
|
||||
speech_provider="cloud",
|
||||
wake_ack_text="",
|
||||
followup_listen_timeout_ms=3000,
|
||||
barge_in_enabled=False,
|
||||
),
|
||||
transport=transport,
|
||||
wakeword=KeywordWakeWordProvider(),
|
||||
@@ -628,6 +674,7 @@ class LiveRuntimeTests(unittest.TestCase):
|
||||
speech_provider="cloud",
|
||||
wake_ack_text="",
|
||||
followup_listen_timeout_ms=3000,
|
||||
barge_in_enabled=False,
|
||||
),
|
||||
transport=transport,
|
||||
wakeword=KeywordWakeWordProvider(),
|
||||
@@ -715,7 +762,7 @@ class LiveRuntimeTests(unittest.TestCase):
|
||||
wakeword=KeywordWakeWordProvider(),
|
||||
vad_recorder=VadRecorder(EnergyVadProvider(), min_duration_ms=40, end_silence_ms=40),
|
||||
stt=stt,
|
||||
realtime_stt=MetadataSttProvider(),
|
||||
realtime_stt=None,
|
||||
llm=llm,
|
||||
tts=SineTtsProvider(),
|
||||
context=ConversationContext(),
|
||||
@@ -737,6 +784,19 @@ class LiveRuntimeTests(unittest.TestCase):
|
||||
self.assertEqual(reporter.transcripts[-1], "打断问题")
|
||||
self.assertEqual(len(llm.calls), 2)
|
||||
|
||||
def test_barge_in_echo_guard_is_capped_and_only_used_once_per_reply(self) -> None:
|
||||
runtime, _, _, _, _ = make_runtime(["第一问"])
|
||||
runtime.controller._barge_in_reply_guard_used = False
|
||||
segment = AudioSegment(b"\x01\x00" * 16000, 16000, 1, 0, 1000)
|
||||
|
||||
first_guard = runtime.controller._effective_barge_in_echo_guard_ms(segment)
|
||||
second_guard = runtime.controller._effective_barge_in_echo_guard_ms(segment)
|
||||
min_speech = runtime.controller._effective_barge_in_min_speech_ms()
|
||||
|
||||
self.assertEqual(first_guard, 120)
|
||||
self.assertEqual(second_guard, 0)
|
||||
self.assertLessEqual(min_speech, runtime.config.interrupt_target_latency_ms)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -74,6 +74,7 @@ class ModelsConfigTests(unittest.TestCase):
|
||||
)
|
||||
)
|
||||
config = AppConfig.from_dotenv(path)
|
||||
self.assertEqual(config.assistant_mode, "turn_based_voice_pet")
|
||||
self.assertEqual(config.llm_base_url, "https://token-plan-cn.xiaomimimo.com/v1")
|
||||
self.assertEqual(config.llm_api_key, "secret-value")
|
||||
self.assertEqual(config.llm_model, "test-model")
|
||||
@@ -122,14 +123,155 @@ class ModelsConfigTests(unittest.TestCase):
|
||||
self.assertEqual(str(config.end_chime_file), "assets/sounds/codex-notification.wav")
|
||||
self.assertEqual(config.end_chime_frequency_hz, 880)
|
||||
self.assertEqual(config.end_chime_duration_ms, 140)
|
||||
self.assertEqual(config.audio_apm_provider, "webrtc")
|
||||
self.assertTrue(config.audio_aec_enabled)
|
||||
self.assertTrue(config.audio_ns_enabled)
|
||||
self.assertTrue(config.audio_agc_enabled)
|
||||
self.assertTrue(config.audio_apm_required)
|
||||
self.assertEqual(config.audio_frame_ms, 20)
|
||||
self.assertEqual(config.audio_ring_buffer_ms, 3000)
|
||||
self.assertTrue(config.interrupt_enabled)
|
||||
self.assertEqual(config.interrupt_target_latency_ms, 200)
|
||||
self.assertEqual(config.streaming_stt_provider, "faster_whisper")
|
||||
self.assertEqual(config.streaming_stt_product_candidate, "sensevoice")
|
||||
self.assertEqual(config.streaming_tts_provider, "cosyvoice")
|
||||
self.assertTrue(config.memory_enabled)
|
||||
self.assertEqual(config.memory_provider, "faiss_sqlite")
|
||||
self.assertEqual(config.memory_top_k, 5)
|
||||
self.assertFalse(config.memory_auto_save_sensitive)
|
||||
self.assertTrue(config.tool_router_enabled)
|
||||
self.assertEqual(config.tool_max_calls_per_turn, 5)
|
||||
self.assertEqual(config.tool_timeout_ms, 30000)
|
||||
self.assertFalse(config.openinterpreter_enabled)
|
||||
self.assertEqual(config.openinterpreter_command, "openinterpreter")
|
||||
self.assertFalse(config.browser_playwright_enabled)
|
||||
self.assertFalse(config.computer_control_enabled)
|
||||
self.assertTrue(config.llm_stream)
|
||||
self.assertEqual(config.validate_basic(), [])
|
||||
|
||||
def test_full_duplex_agent_config_values_are_read_from_dotenv(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = f"{tmp}/.env"
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
handle.write(
|
||||
"\n".join(
|
||||
[
|
||||
"OWNER_ASSISTANT_MODE=full_duplex_agent",
|
||||
"OWNER_AUDIO_APM_PROVIDER=fake",
|
||||
"OWNER_AUDIO_AEC_ENABLED=0",
|
||||
"OWNER_AUDIO_NS_ENABLED=0",
|
||||
"OWNER_AUDIO_AGC_ENABLED=0",
|
||||
"OWNER_AUDIO_APM_REQUIRED=0",
|
||||
"OWNER_AUDIO_FRAME_MS=10",
|
||||
"OWNER_AUDIO_RING_BUFFER_MS=1200",
|
||||
"OWNER_INTERRUPT_ENABLED=0",
|
||||
"OWNER_INTERRUPT_TARGET_LATENCY_MS=180",
|
||||
"OWNER_LLM_STREAMING_ENABLED=0",
|
||||
"OWNER_STREAMING_STT_PROVIDER=sherpa_onnx",
|
||||
"OWNER_STREAMING_STT_PRODUCT_CANDIDATE=faster_whisper",
|
||||
"OWNER_STREAMING_TTS_PROVIDER=macos_say",
|
||||
"OWNER_MEMORY_ENABLED=1",
|
||||
"OWNER_MEMORY_PROVIDER=faiss_sqlite",
|
||||
"OWNER_MEMORY_TOP_K=3",
|
||||
"OWNER_MEMORY_AUTO_SAVE_SENSITIVE=1",
|
||||
"OWNER_TOOL_ROUTER_ENABLED=1",
|
||||
"OWNER_TOOL_MAX_CALLS_PER_TURN=2",
|
||||
"OWNER_TOOL_TIMEOUT_MS=1000",
|
||||
"OWNER_OPENINTERPRETER_ENABLED=1",
|
||||
"OWNER_OPENINTERPRETER_COMMAND=/tmp/openinterpreter",
|
||||
"OWNER_BROWSER_PLAYWRIGHT_ENABLED=1",
|
||||
"OWNER_COMPUTER_CONTROL_ENABLED=1",
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
config = AppConfig.from_dotenv(path)
|
||||
|
||||
self.assertEqual(config.assistant_mode, "full_duplex_agent")
|
||||
self.assertEqual(config.audio_apm_provider, "fake")
|
||||
self.assertFalse(config.audio_aec_enabled)
|
||||
self.assertFalse(config.audio_ns_enabled)
|
||||
self.assertFalse(config.audio_agc_enabled)
|
||||
self.assertFalse(config.audio_apm_required)
|
||||
self.assertEqual(config.audio_frame_ms, 10)
|
||||
self.assertEqual(config.audio_ring_buffer_ms, 1200)
|
||||
self.assertFalse(config.interrupt_enabled)
|
||||
self.assertEqual(config.interrupt_target_latency_ms, 180)
|
||||
self.assertFalse(config.llm_stream)
|
||||
self.assertEqual(config.streaming_stt_provider, "sherpa_onnx")
|
||||
self.assertEqual(config.streaming_stt_product_candidate, "faster_whisper")
|
||||
self.assertEqual(config.streaming_tts_provider, "macos_say")
|
||||
self.assertTrue(config.memory_enabled)
|
||||
self.assertEqual(config.memory_provider, "faiss_sqlite")
|
||||
self.assertEqual(config.memory_top_k, 3)
|
||||
self.assertTrue(config.memory_auto_save_sensitive)
|
||||
self.assertTrue(config.tool_router_enabled)
|
||||
self.assertEqual(config.tool_max_calls_per_turn, 2)
|
||||
self.assertEqual(config.tool_timeout_ms, 1000)
|
||||
self.assertTrue(config.openinterpreter_enabled)
|
||||
self.assertEqual(config.openinterpreter_command, "/tmp/openinterpreter")
|
||||
self.assertTrue(config.browser_playwright_enabled)
|
||||
self.assertTrue(config.computer_control_enabled)
|
||||
self.assertEqual(config.validate_basic(), [])
|
||||
|
||||
def test_llm_streaming_enabled_takes_precedence_over_legacy_llm_stream(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = f"{tmp}/.env"
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
handle.write("OWNER_LLM_STREAM=0\nOWNER_LLM_STREAMING_ENABLED=1\n")
|
||||
|
||||
config = AppConfig.from_dotenv(path)
|
||||
|
||||
self.assertTrue(config.llm_stream)
|
||||
|
||||
def test_legacy_llm_stream_still_disables_streaming(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = f"{tmp}/.env"
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
handle.write("OWNER_LLM_STREAM=0\n")
|
||||
|
||||
config = AppConfig.from_dotenv(path)
|
||||
|
||||
self.assertFalse(config.llm_stream)
|
||||
|
||||
def test_speech_provider_must_be_cloud_or_local(self) -> None:
|
||||
config = AppConfig(speech_provider="invalid")
|
||||
errors = config.validate_basic()
|
||||
self.assertTrue(any("OWNER_SPEECH_PROVIDER" in error.message for error in errors))
|
||||
|
||||
def test_full_duplex_agent_config_is_validated(self) -> None:
|
||||
config = AppConfig(
|
||||
assistant_mode="invalid",
|
||||
audio_apm_provider="invalid",
|
||||
audio_frame_ms=0,
|
||||
audio_ring_buffer_ms=0,
|
||||
interrupt_target_latency_ms=0,
|
||||
streaming_stt_provider="invalid",
|
||||
streaming_stt_product_candidate="invalid",
|
||||
streaming_tts_provider="invalid",
|
||||
memory_provider="invalid",
|
||||
memory_top_k=0,
|
||||
tool_max_calls_per_turn=0,
|
||||
tool_timeout_ms=0,
|
||||
openinterpreter_enabled=True,
|
||||
openinterpreter_command="",
|
||||
)
|
||||
errors = config.validate_basic()
|
||||
|
||||
self.assertTrue(any("OWNER_ASSISTANT_MODE" in error.message for error in errors))
|
||||
self.assertTrue(any("OWNER_AUDIO_APM_PROVIDER" in error.message for error in errors))
|
||||
self.assertTrue(any("OWNER_AUDIO_FRAME_MS" in error.message for error in errors))
|
||||
self.assertTrue(any("OWNER_AUDIO_RING_BUFFER_MS" in error.message for error in errors))
|
||||
self.assertTrue(any("OWNER_INTERRUPT_TARGET_LATENCY_MS" in error.message for error in errors))
|
||||
self.assertTrue(any("OWNER_STREAMING_STT_PROVIDER" in error.message for error in errors))
|
||||
self.assertTrue(any("OWNER_STREAMING_STT_PRODUCT_CANDIDATE" in error.message for error in errors))
|
||||
self.assertTrue(any("OWNER_STREAMING_TTS_PROVIDER" in error.message for error in errors))
|
||||
self.assertTrue(any("OWNER_MEMORY_PROVIDER" in error.message for error in errors))
|
||||
self.assertTrue(any("OWNER_MEMORY_TOP_K" in error.message for error in errors))
|
||||
self.assertTrue(any("OWNER_TOOL_MAX_CALLS_PER_TURN" in error.message for error in errors))
|
||||
self.assertTrue(any("OWNER_TOOL_TIMEOUT_MS" in error.message for error in errors))
|
||||
self.assertTrue(any("OWNER_OPENINTERPRETER_COMMAND" in error.message for error in errors))
|
||||
|
||||
def test_realtime_transcript_idle_timeout_must_be_non_negative(self) -> None:
|
||||
config = AppConfig(realtime_transcript_idle_timeout_ms=-1)
|
||||
errors = config.validate_basic()
|
||||
@@ -212,7 +354,10 @@ class ModelsConfigTests(unittest.TestCase):
|
||||
self.assertEqual(raised.exception.code, ErrorCode.LLM_API_KEY_MISSING)
|
||||
|
||||
def test_pipeline_states_include_required_names(self) -> None:
|
||||
self.assertEqual(PipelineState.LISTENING.value, "listening")
|
||||
self.assertEqual(PipelineState.WAKE_LISTENING.value, "wake_listening")
|
||||
self.assertEqual(PipelineState.TOOL_RUNNING.value, "tool_running")
|
||||
self.assertEqual(PipelineState.RECOVERING.value, "recovering")
|
||||
self.assertEqual(PipelineState.ERROR_RECOVERING.value, "error_recovering")
|
||||
|
||||
def test_message_model_accepts_roles(self) -> None:
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from owner_voice_pet.agent_memory import FakeMemoryManager, MemoryRecordInput
|
||||
from owner_voice_pet.tool_router import (
|
||||
FakeToolAdapter,
|
||||
MemorySaveTool,
|
||||
MemorySearchTool,
|
||||
ShellReadonlyTool,
|
||||
ToolCallRequest,
|
||||
ToolContext,
|
||||
ToolRouter,
|
||||
ToolRiskClassifier,
|
||||
)
|
||||
|
||||
|
||||
class ToolRouterTests(unittest.TestCase):
|
||||
def test_unknown_tool_is_rejected(self) -> None:
|
||||
router = ToolRouter({})
|
||||
request = ToolCallRequest("1", "unknown.tool", {}, "turn-1")
|
||||
|
||||
decision = router.route(request, ToolContext())
|
||||
|
||||
self.assertEqual(decision.action, "reject")
|
||||
self.assertEqual(decision.risk_level, "forbidden")
|
||||
|
||||
def test_memory_search_tool_executes_and_returns_sanitized_result(self) -> None:
|
||||
memory = FakeMemoryManager()
|
||||
memory.save(MemoryRecordInput("preference", "用户喜欢 Python"))
|
||||
router = ToolRouter({"memory.search": MemorySearchTool()})
|
||||
request = ToolCallRequest("1", "memory.search", {"query": "Python", "top_k": 1}, "turn-1")
|
||||
|
||||
decision = router.route(request, ToolContext(memory=memory))
|
||||
result = router.execute(request, decision, ToolContext(memory=memory))
|
||||
|
||||
self.assertEqual(decision.action, "execute")
|
||||
self.assertEqual(result.status, "success")
|
||||
self.assertIn("用户喜欢 Python", result.output_text)
|
||||
self.assertEqual(router.audit_log[-1].tool_name, "memory.search")
|
||||
|
||||
def test_memory_save_sensitive_text_requires_confirmation(self) -> None:
|
||||
router = ToolRouter({"memory.save": MemorySaveTool()})
|
||||
request = ToolCallRequest(
|
||||
"1",
|
||||
"memory.save",
|
||||
{"text": "保存 api key secret-value"},
|
||||
"turn-1",
|
||||
)
|
||||
|
||||
decision = router.route(request, ToolContext(memory=FakeMemoryManager()))
|
||||
result = router.execute(request, decision, ToolContext(memory=FakeMemoryManager()))
|
||||
|
||||
self.assertEqual(decision.action, "require_confirmation")
|
||||
self.assertEqual(result.status, "confirmation_required")
|
||||
self.assertEqual(result.error_code, "TOOL_CONFIRMATION_REQUIRED")
|
||||
|
||||
def test_shell_readonly_rejects_mutating_command(self) -> None:
|
||||
router = ToolRouter({"shell.readonly": ShellReadonlyTool()})
|
||||
request = ToolCallRequest("1", "shell.readonly", {"command": "rm -rf /tmp/x"}, "turn-1")
|
||||
|
||||
decision = router.route(request, ToolContext())
|
||||
|
||||
self.assertEqual(decision.action, "reject")
|
||||
self.assertEqual(decision.risk_level, "forbidden")
|
||||
|
||||
def test_shell_readonly_executes_allowed_command(self) -> None:
|
||||
router = ToolRouter({"shell.readonly": ShellReadonlyTool()})
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
request = ToolCallRequest("1", "shell.readonly", {"command": "pwd"}, "turn-1")
|
||||
decision = router.route(request, ToolContext(cwd=Path(tmp)))
|
||||
result = router.execute(request, decision, ToolContext(cwd=Path(tmp)))
|
||||
|
||||
self.assertEqual(decision.action, "execute")
|
||||
self.assertEqual(result.status, "success")
|
||||
self.assertIn(tmp, result.output_text)
|
||||
|
||||
def test_tool_budget_and_duplicate_calls_are_rejected(self) -> None:
|
||||
adapter = FakeToolAdapter("memory.search", output="ok")
|
||||
router = ToolRouter({"memory.search": adapter}, max_calls_per_turn=2)
|
||||
request = ToolCallRequest("1", "memory.search", {"query": "a"}, "turn-1")
|
||||
decision = router.route(request, ToolContext())
|
||||
router.execute(request, decision, ToolContext())
|
||||
|
||||
duplicate = router.route(request, ToolContext())
|
||||
second = ToolCallRequest("2", "memory.search", {"query": "b"}, "turn-1")
|
||||
second_decision = router.route(second, ToolContext())
|
||||
router.execute(second, second_decision, ToolContext())
|
||||
third = ToolCallRequest("3", "memory.search", {"query": "c"}, "turn-1")
|
||||
over_budget = router.route(third, ToolContext())
|
||||
|
||||
self.assertEqual(duplicate.action, "reject")
|
||||
self.assertEqual(duplicate.reason, "duplicate tool call rejected")
|
||||
self.assertEqual(over_budget.action, "reject")
|
||||
self.assertEqual(over_budget.reason, "tool call budget exceeded")
|
||||
|
||||
def test_output_is_redacted_and_truncated(self) -> None:
|
||||
adapter = FakeToolAdapter("memory.search", output="tp-" + "abcdefghijklmnop " + "x" * 100)
|
||||
router = ToolRouter({"memory.search": adapter}, output_limit=20)
|
||||
request = ToolCallRequest("1", "memory.search", {"query": "secret"}, "turn-1")
|
||||
|
||||
decision = router.route(request, ToolContext())
|
||||
result = router.execute(request, decision, ToolContext())
|
||||
|
||||
self.assertTrue(result.output_truncated)
|
||||
self.assertIn("[redacted]", result.output_text)
|
||||
self.assertNotIn("tp-" + "abcdefghijklmnop", result.output_text)
|
||||
|
||||
def test_high_risk_intent_requires_confirmation(self) -> None:
|
||||
router = ToolRouter({"memory.search": FakeToolAdapter("memory.search")})
|
||||
request = ToolCallRequest(
|
||||
"1",
|
||||
"memory.search",
|
||||
{"query": "账号"},
|
||||
"turn-1",
|
||||
natural_language_intent="上传账号资料",
|
||||
)
|
||||
|
||||
decision = router.route(request, ToolContext())
|
||||
|
||||
self.assertEqual(decision.action, "require_confirmation")
|
||||
self.assertEqual(decision.risk_level, "high")
|
||||
|
||||
def test_risk_classifier_marks_readonly_as_low(self) -> None:
|
||||
risk = ToolRiskClassifier().classify(
|
||||
ToolCallRequest("1", "shell.readonly", {"command": "git status --short"}, "turn-1")
|
||||
)
|
||||
|
||||
self.assertEqual(risk, "low")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user