From 445eebeaec4ff73676ffde060e7b4af6e752acfb Mon Sep 17 00:00:00 2001 From: mkbk Date: Wed, 17 Jun 2026 20:17:04 +0800 Subject: [PATCH] =?UTF-8?q?[=E6=96=87=E6=A1=A3=E3=80=81=E9=AA=8C=E6=94=B6?= =?UTF-8?q?=E4=B8=8E=E5=BD=92=E6=A1=A3]=EF=BC=9A=E5=AE=8C=E6=88=90?= =?UTF-8?q?=E7=9C=9F=E5=AE=9E=E8=BF=90=E8=A1=8C=E8=AF=B4=E6=98=8E=E5=92=8C?= =?UTF-8?q?OpenSpec=E5=BD=92=E6=A1=A3=EF=BC=8C=E5=8C=85=E5=90=ABREADME?= =?UTF-8?q?=E3=80=81=E6=9C=80=E7=BB=88=E9=97=A8=E7=A6=81=E5=92=8C=E4=B8=BB?= =?UTF-8?q?=E8=A7=84=E8=8C=83=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 4 +- README.md | 115 +++++++---- .../.openspec.yaml | 0 .../README.md | 0 .../design.md | 2 +- .../proposal.md | 2 +- .../specs/voice-pet-pipeline/spec.md | 4 +- .../tasks.md | 14 +- openspec/specs/voice-pet-pipeline/spec.md | 187 ++++++++++++++---- src/owner_voice_pet/config.py | 8 +- src/owner_voice_pet/stt.py | 59 +++--- src/owner_voice_pet/tts.py | 31 ++- tests/test_models_config.py | 2 +- tests/test_pipeline_llm_tts.py | 22 ++- tests/test_wake_vad_stt.py | 13 +- 15 files changed, 320 insertions(+), 143 deletions(-) rename openspec/changes/{complete-live-repeat-voice-runtime => archive/2026-06-17-complete-live-repeat-voice-runtime}/.openspec.yaml (100%) rename openspec/changes/{complete-live-repeat-voice-runtime => archive/2026-06-17-complete-live-repeat-voice-runtime}/README.md (100%) rename openspec/changes/{complete-live-repeat-voice-runtime => archive/2026-06-17-complete-live-repeat-voice-runtime}/design.md (99%) rename openspec/changes/{complete-live-repeat-voice-runtime => archive/2026-06-17-complete-live-repeat-voice-runtime}/proposal.md (99%) rename openspec/changes/{complete-live-repeat-voice-runtime => archive/2026-06-17-complete-live-repeat-voice-runtime}/specs/voice-pet-pipeline/spec.md (99%) rename openspec/changes/{complete-live-repeat-voice-runtime => archive/2026-06-17-complete-live-repeat-voice-runtime}/tasks.md (95%) diff --git a/.env.example b/.env.example index 7e5757c..4015871 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,6 @@ OWNER_LLM_BASE_URL=https://token-plan-cn.xiaomimimo.com/v1 OWNER_LLM_API_KEY= -OWNER_LLM_MODEL=gpt-5.4-mini +OWNER_LLM_MODEL=mimo-v2.5 OWNER_LLM_API_STYLE=chat_completions OWNER_AUDIO_INPUT_DEVICE= OWNER_AUDIO_OUTPUT_DEVICE= @@ -9,7 +9,7 @@ OWNER_LOG_DIR=logs OWNER_SPEECH_PROVIDER=cloud OWNER_ASR_MODEL=mimo-v2.5-asr OWNER_TTS_MODEL=mimo-v2.5-tts -OWNER_TTS_VOICE=alloy +OWNER_TTS_VOICE=mimo_default OWNER_SPEECH_MODELS_DIR=models OWNER_CONTEXT_MAX_MESSAGES=12 OWNER_CONTEXT_MAX_CHARS=12000 diff --git a/README.md b/README.md index 29cbb5b..e8b284a 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,102 @@ # Owner 语音桌宠 -这是 `add-voice-pet-pipeline` OpenSpec 变更交付的 Python 桌宠语音 Pipeline。 +这是一个 Python 语音桌宠运行程序。当前第一版先提供无 GUI 的真实实时语音循环： -当前实现覆盖： +`小杰小杰` 唤醒 -> 麦克风录音 -> VAD 切分 -> ASR 转文字 -> 携带本次进程内临时历史调用 LLM -> TTS 生成语音 -> 本机扬声器播放 -> 回到待机继续监听。 -- “小杰小杰”唤醒词入口 -- 本机音频 Transport 抽象和文件/内存回放测试通道 -- VAD 人声端点检测 -- 本地 STT Provider 边界和测试 Provider -- OpenAI/NewAPI 兼容 LLM Provider -- 本地 TTS Provider 边界和测试 TTS -- 桌宠状态映射、无 GUI fallback、透明 PNG 状态资产 -- OpenSpec 主规范、自动化测试、CLI 验收和安全扫描 +## 当前能力 -## 配置 +- `owner_voice_pet run-live`：真实常驻语音循环。 +- `owner_voice_pet run-live --once`：只跑一轮，便于验收。 +- `.env` 直接读取配置，不要求导出 shell 环境变量。 +- `OWNER_SPEECH_PROVIDER=cloud|local`：选择云端语音模型或本地语音模型。 +- 默认云端语音模型：`mimo-v2.5-asr`、`mimo-v2.5-tts`。 +- 本地设备：`sounddevice` 读取麦克风，扬声器或 `afplay` 播放。 +- 本地模型：`models/` 存放 `sherpa-onnx` VAD/STT 模型，目录不提交 Git。 +- 临时上下文：同一次 `run-live` 进程内携带最近 user/assistant 历史，退出即清空。 -密钥不得写入仓库。运行时默认读取项目根目录的 `.env` 文件： - -- `OWNER_LLM_BASE_URL` -- `OWNER_LLM_API_KEY` -- `OWNER_LLM_MODEL` -- `OWNER_LLM_API_STYLE` - -默认配置面向当前 NewAPI： +## 首次准备 ```bash +python3.11 -m venv .venv +.venv/bin/python -m pip install --upgrade pip +.venv/bin/python -m pip install -e '.[speech]' cp .env.example .env -# 然后编辑 .env，填入 OWNER_LLM_API_KEY -OWNER_LLM_BASE_URL=https://newapi.mkbk.shop -OWNER_LLM_MODEL=gpt-5.4-mini -OWNER_LLM_API_STYLE=chat_completions ``` -## 验收命令 +然后编辑 `.env`，填写本地密钥和模型配置。当前默认示例： + +```dotenv +OWNER_LLM_BASE_URL=https://token-plan-cn.xiaomimimo.com/v1 +OWNER_LLM_API_KEY= +OWNER_LLM_MODEL=mimo-v2.5 +OWNER_LLM_API_STYLE=chat_completions +OWNER_SPEECH_PROVIDER=cloud +OWNER_ASR_MODEL=mimo-v2.5-asr +OWNER_TTS_MODEL=mimo-v2.5-tts +OWNER_TTS_VOICE=mimo_default +``` + +`OWNER_SPEECH_PROVIDER=cloud` 会把 VAD 切出来的语音片段发送到云端 ASR；不会上传连续麦克风流。改成 `local` 时使用项目 `models/` 下的本地语音模型路径。 + +## 本地模型 + +即使默认走云端语音，也可以准备本地模型，便于切换到 `OWNER_SPEECH_PROVIDER=local`： ```bash -PYTHONPATH=src python3.11 -m compileall src tests scripts -PYTHONPATH=src python3.11 -m unittest discover -s tests -PYTHONPATH=src python3.11 -m owner_voice_pet acceptance -PYTHONPATH=src python3.11 -m owner_voice_pet validate-assets -PYTHONPATH=src python3.11 -m owner_voice_pet security-check +python3.11 scripts/download_speech_models.py --dir models +.venv/bin/python -m owner_voice_pet model-check --models-dir models +``` + +`models/` 已在 `.gitignore` 中，不会提交大模型文件。 + +## 设备检查 + +```bash +.venv/bin/python -m owner_voice_pet device-check +``` + +输出里 `ok=true` 表示已检测到可用麦克风和扬声器。 + +## 运行 + +单轮验收： + +```bash +.venv/bin/python -m owner_voice_pet run-live --once +``` + +常驻重复对话： + +```bash +.venv/bin/python -m owner_voice_pet run-live +``` + +运行后终端会显示待机、唤醒命中、录音中、转写中、思考中、播放中、恢复待机等状态。说“小杰小杰”后提问；听到回复后可以再次说“小杰小杰”继续下一轮。本次进程内会携带临时历史，程序退出后不保存。 + +## 验证 + +```bash +.venv/bin/python -m compileall src tests scripts +.venv/bin/python -m unittest discover -s tests +.venv/bin/python -m owner_voice_pet acceptance +.venv/bin/python -m owner_voice_pet validate-assets +.venv/bin/python -m owner_voice_pet security-check +.venv/bin/python -m owner_voice_pet model-check --models-dir models +.venv/bin/python -m owner_voice_pet device-check openspec validate --all --strict ``` -真实 NewAPI smoke 测试示例： +LLM smoke： ```bash -PYTHONPATH=src python3.11 -m owner_voice_pet llm-smoke --no-stream --message '用一句中文回复：小杰在线。' +.venv/bin/python -m owner_voice_pet llm-smoke --no-stream --message '用一句中文回复：小杰在线。' ``` ## 安全约束 -- API key 只从环境变量读取。 -- 仓库内不保存真实密钥。 -- 默认不持久化原始麦克风音频。 -- `security-check` 会扫描已跟踪文本文件中的 `sk-...` 形式密钥。 +- API key 只写入本地 `.env`，`.env` 不提交 Git。 +- `models/`、`.venv/` 和临时音频文件不提交 Git。 +- 默认不持久化麦克风原始音频。 +- 云端 ASR 只接收 VAD 切分后的语音片段。 +- `security-check` 会扫描已跟踪文本文件中的 `sk-...` 和 `tp-...` 形式密钥。 diff --git a/openspec/changes/complete-live-repeat-voice-runtime/.openspec.yaml b/openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/.openspec.yaml similarity index 100% rename from openspec/changes/complete-live-repeat-voice-runtime/.openspec.yaml rename to openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/.openspec.yaml diff --git a/openspec/changes/complete-live-repeat-voice-runtime/README.md b/openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/README.md similarity index 100% rename from openspec/changes/complete-live-repeat-voice-runtime/README.md rename to openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/README.md diff --git a/openspec/changes/complete-live-repeat-voice-runtime/design.md b/openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/design.md similarity index 99% rename from openspec/changes/complete-live-repeat-voice-runtime/design.md rename to openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/design.md index 2edf0bd..86e8300 100644 --- a/openspec/changes/complete-live-repeat-voice-runtime/design.md +++ b/openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/design.md @@ -186,7 +186,7 @@ Test requirements: First implementation can use one of two strategies: -1. Default cloud mode: local VAD cuts speech segments, `mimo-v2.5-asr` transcribes wake and user segments through NewAPI-compatible `/v1/audio/transcriptions`. +1. Default cloud mode: local VAD cuts speech segments, `mimo-v2.5-asr` transcribes wake and user segments through NewAPI-compatible `/v1/chat/completions` audio input messages. 2. Local mode: sherpa-onnx VAD/STT uses project-local model assets under `models/`. 3. Future replacement: dedicated local KWS provider for “小杰小杰”. diff --git a/openspec/changes/complete-live-repeat-voice-runtime/proposal.md b/openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/proposal.md similarity index 99% rename from openspec/changes/complete-live-repeat-voice-runtime/proposal.md rename to openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/proposal.md index 012788d..41b2159 100644 --- a/openspec/changes/complete-live-repeat-voice-runtime/proposal.md +++ b/openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/proposal.md @@ -138,7 +138,7 @@ #### `.env` 输入 -1. `OWNER_LLM_BASE_URL`：默认 `https://newapi.mkbk.shop`，作为 OpenAI/NewAPI 兼容接口 base URL。 +1. `OWNER_LLM_BASE_URL`：默认 `https://token-plan-cn.xiaomimimo.com/v1`，作为 OpenAI/NewAPI 兼容接口 base URL。 2. `OWNER_LLM_API_KEY`：真实密钥，只允许存在于 `.env` 或用户本地未提交配置。 3. `OWNER_LLM_MODEL`：云端模型名，通过 `.env` 配置指定。 4. `OWNER_LLM_API_STYLE`：第一版保留 `chat_completions` 或现有兼容值。 diff --git a/openspec/changes/complete-live-repeat-voice-runtime/specs/voice-pet-pipeline/spec.md b/openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/specs/voice-pet-pipeline/spec.md similarity index 99% rename from openspec/changes/complete-live-repeat-voice-runtime/specs/voice-pet-pipeline/spec.md rename to openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/specs/voice-pet-pipeline/spec.md index 0a43f4b..193955e 100644 --- a/openspec/changes/complete-live-repeat-voice-runtime/specs/voice-pet-pipeline/spec.md +++ b/openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/specs/voice-pet-pipeline/spec.md @@ -138,7 +138,7 @@ The system SHALL use the local microphone as the first-version input Transport a - **WHEN** the microphone or speaker is missing, denied, unsupported, or inaccessible through `sounddevice` - **THEN** the system SHALL expose a recoverable Transport error with a stable error code and SHALL NOT silently fall back to fixture audio in live mode -### Requirement: Configured STT transcription +### Requirement: Local STT transcription The system SHALL transcribe captured user utterances through the configured STT provider; cloud mode SHALL use NewAPI-compatible ASR and local mode SHALL use project-local `sherpa-onnx` model assets under `models/`. #### Scenario: STT succeeds @@ -153,7 +153,7 @@ The system SHALL transcribe captured user utterances through the configured STT - **WHEN** the STT provider raises an error, cloud ASR fails, or local model loading fails - **THEN** the system SHALL emit an STT or model error code and SHALL recover to a state where future wake attempts are possible if startup can continue safely -### Requirement: Configured TTS synthesis and playback +### Requirement: Local TTS synthesis and playback The system SHALL synthesize assistant replies through the configured TTS provider and SHALL play synthesized speech through the local system output path; cloud mode SHALL use NewAPI-compatible TTS and local mode SHALL use a local playback-capable TTS provider. #### Scenario: Reply text is ready for speech diff --git a/openspec/changes/complete-live-repeat-voice-runtime/tasks.md b/openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/tasks.md similarity index 95% rename from openspec/changes/complete-live-repeat-voice-runtime/tasks.md rename to openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/tasks.md index cd550e3..cd4d258 100644 --- a/openspec/changes/complete-live-repeat-voice-runtime/tasks.md +++ b/openspec/changes/archive/2026-06-17-complete-live-repeat-voice-runtime/tasks.md @@ -45,10 +45,10 @@ ## 6. 文档、真实验收、归档 -- [ ] 6.1 更新 README 中文运行说明；前置条件：CLI 和脚本完成；验收标准：包含 `.venv`、`.env`、模型下载、model-check、device-check、run-live、--once；测试要点：命令可复制执行；优先级：P0；预计：45 分钟。 -- [ ] 6.2 执行本地模型验收；前置条件：模型已下载；验收标准：`python3.11 scripts/download_speech_models.py --dir models` 和 `model-check` 通过或给出明确设备/模型失败证据；测试要点：输出不泄露 key；优先级：P0；预计：60 分钟。 -- [ ] 6.3 执行设备验收；前置条件：sounddevice 安装；验收标准：`device-check` 通过或输出明确权限/设备原因；测试要点：真实设备清单；优先级：P0；预计：30 分钟。 -- [ ] 6.4 执行真实运行验收；前置条件：`.env`、模型、设备齐全；验收标准：`run-live` 可被人工唤醒并完成至少两轮；第二轮可引用第一轮临时历史；测试要点：终端状态和听到播放；优先级：P0；预计：60 分钟。 -- [ ] 6.5 执行最终自动化门禁；前置条件：全部实现完成；验收标准：compileall、unittest、security-check、OpenSpec validate 全通过；测试要点：`git status --short` 没有未提交源码/文档变更；优先级：P0；预计：45 分钟。 -- [ ] 6.6 归档 `complete-live-repeat-voice-runtime`；前置条件：任务全完成并验证通过；验收标准：主 spec 更新，change 移入 archive，`openspec validate --all --strict` 通过；测试要点：归档后 spec 包含 live runtime 要求；优先级：P0；预计：30 分钟。 -- [ ] 6.7 最终提交“文档、验收与归档”模块；前置条件：6.1 至 6.6 完成；验收标准：最终门禁通过后 commit，工作树干净；测试要点：提交信息为中文格式；优先级：P0；预计：20 分钟。 +- [x] 6.1 更新 README 中文运行说明；前置条件：CLI 和脚本完成；验收标准：包含 `.venv`、`.env`、模型下载、model-check、device-check、run-live、--once；测试要点：命令可复制执行；优先级：P0；预计：45 分钟。 +- [x] 6.2 执行本地模型验收；前置条件：模型已下载；验收标准：`python3.11 scripts/download_speech_models.py --dir models` 和 `model-check` 通过或给出明确设备/模型失败证据；测试要点：输出不泄露 key；优先级：P0；预计：60 分钟。 +- [x] 6.3 执行设备验收；前置条件：sounddevice 安装；验收标准：`device-check` 通过或输出明确权限/设备原因；测试要点：真实设备清单；优先级：P0；预计：30 分钟。 +- [x] 6.4 执行真实运行验收；前置条件：`.env`、模型、设备齐全；验收标准：`run-live` 可被人工唤醒并完成至少两轮；第二轮可引用第一轮临时历史；测试要点：终端状态和听到播放；优先级：P0；预计：60 分钟。 +- [x] 6.5 执行最终自动化门禁；前置条件：全部实现完成；验收标准：compileall、unittest、security-check、OpenSpec validate 全通过；测试要点：`git status --short` 没有未提交源码/文档变更；优先级：P0；预计：45 分钟。 +- [x] 6.6 归档 `complete-live-repeat-voice-runtime`；前置条件：任务全完成并验证通过；验收标准：主 spec 更新，change 移入 archive，`openspec validate --all --strict` 通过；测试要点：归档后 spec 包含 live runtime 要求；优先级：P0；预计：30 分钟。 +- [x] 6.7 最终提交“文档、验收与归档”模块；前置条件：6.1 至 6.6 完成；验收标准：最终门禁通过后 commit，工作树干净；测试要点：提交信息为中文格式；优先级：P0；预计：20 分钟。 diff --git a/openspec/specs/voice-pet-pipeline/spec.md b/openspec/specs/voice-pet-pipeline/spec.md index 806dbd8..8716ed6 100644 --- a/openspec/specs/voice-pet-pipeline/spec.md +++ b/openspec/specs/voice-pet-pipeline/spec.md @@ -15,19 +15,19 @@ The system SHALL be specified as a local Python desktop pet application that own - **THEN** the repository SHALL contain Python source code, automated tests, validation commands, and project-local pet assets aligned with the planning artifacts ### Requirement: Local microphone and speaker transport -The system SHALL use the local microphone as the first-version input Transport and the local system speaker as the first-version output Transport. +The system SHALL use the local microphone as the first-version input Transport and the local system speaker as the first-version output Transport, and `run-live` SHALL exercise those real local devices by default. #### Scenario: Microphone input is available - **WHEN** the configured microphone is available and permitted -- **THEN** the Transport SHALL provide streaming audio frames to the wakeword and VAD stages +- **THEN** the live Transport SHALL provide streaming audio frames to wake detection, VAD, and STT stages #### Scenario: Speaker output is available -- **WHEN** TTS returns a valid audio segment and the configured speaker is available -- **THEN** the Transport SHALL play the audio segment through the local speaker +- **WHEN** TTS returns a valid local audio segment or audio file and the configured speaker is available +- **THEN** the live Transport or macOS playback command SHALL play the audio through the local speaker #### Scenario: Audio device is unavailable -- **WHEN** the microphone or speaker is missing, denied, or unsupported -- **THEN** the system SHALL expose a recoverable Transport error with a stable error code +- **WHEN** the microphone or speaker is missing, denied, unsupported, or inaccessible through `sounddevice` +- **THEN** the system SHALL expose a recoverable Transport error with a stable error code and SHALL NOT silently fall back to fixture audio in live mode ### Requirement: Wake word detection The system SHALL listen locally for the Chinese wake word “小杰小杰” before accepting user speech for a conversation turn. @@ -64,37 +64,41 @@ After wakeword detection, the system SHALL use VAD to identify when the user sta - **THEN** the pipeline SHALL end the segment, mark the end reason, and continue to STT with the captured audio ### Requirement: Local STT transcription -The system SHALL transcribe the captured user utterance through a local STT provider, with `sherpa-onnx` as the recommended first implementation candidate. +The system SHALL transcribe captured user utterances through the configured STT provider; cloud mode SHALL use NewAPI-compatible ASR and local mode SHALL use project-local `sherpa-onnx` model assets under `models/`. #### Scenario: STT succeeds -- **WHEN** STT returns non-empty text for a captured audio segment -- **THEN** the pipeline SHALL add the trimmed text as a user message to the conversation context +- **WHEN** configured STT returns non-empty text for a captured live audio segment +- **THEN** the pipeline SHALL add the trimmed text as a user message to the current process conversation context #### Scenario: STT returns empty text - **WHEN** STT returns empty text, punctuation-only text, or an invalid transcript -- **THEN** the pipeline SHALL skip LLM invocation and return to wake listening with a recoverable status +- **THEN** the live runtime SHALL skip LLM invocation and return to standby with a recoverable status #### Scenario: STT provider fails -- **WHEN** the STT provider raises an error or cannot load its local model -- **THEN** the system SHALL emit an STT error code and SHALL recover to a state where future wake attempts are possible +- **WHEN** the STT provider raises an error, cloud ASR fails, or local model loading fails +- **THEN** the system SHALL emit an STT or model error code and SHALL recover to a state where future wake attempts are possible if startup can continue safely ### Requirement: Conversation context management -The system SHALL maintain an in-memory conversation context for the active desktop pet session. +The system SHALL maintain an in-memory conversation context for the active desktop pet session, and live repeated voice runtime SHALL define that session as the current `run-live` process only. #### Scenario: User transcript is accepted -- **WHEN** a valid STT transcript is produced -- **THEN** the system SHALL append it to the context as a user message before invoking the LLM +- **WHEN** a valid STT transcript is produced during a live turn +- **THEN** the system SHALL append it to the current process context as a user message before invoking the LLM #### Scenario: Assistant reply completes -- **WHEN** the LLM and TTS stages complete a reply -- **THEN** the system SHALL append the assistant text to the context +- **WHEN** the LLM and TTS stages complete a non-empty reply during a live turn +- **THEN** the system SHALL append the assistant text to the current process context #### Scenario: Context exceeds configured budget -- **WHEN** the context exceeds the configured message, character, or token budget +- **WHEN** the context exceeds the configured message or character budget - **THEN** the system SHALL preserve the system prompt and most recent conversation turns while removing older ordinary messages +#### Scenario: Runtime exits +- **WHEN** the `run-live` process exits +- **THEN** the system SHALL discard the context and SHALL NOT persist it across process restarts + ### Requirement: Cloud LLM streaming reply -The system SHALL use a cloud LLM provider for reply generation, with OpenAI Responses API streaming as the recommended first implementation. +The system SHALL use a cloud LLM provider for reply generation, with an OpenAI/NewAPI-compatible chat completions endpoint as the first-version runtime target. #### Scenario: LLM stream begins - **WHEN** the LLM provider receives valid context messages and configuration @@ -102,26 +106,26 @@ The system SHALL use a cloud LLM provider for reply generation, with OpenAI Resp #### Scenario: LLM model is configured - **WHEN** the system starts -- **THEN** the LLM model name SHALL be read from configuration and SHALL NOT be hard-coded in pipeline logic +- **THEN** the LLM base URL, API key, and model name SHALL be read from `.env` or local uncommitted configuration and SHALL NOT be hard-coded in pipeline logic #### Scenario: LLM request fails - **WHEN** the LLM provider times out, is rate limited, lacks credentials, or encounters a network error - **THEN** the pipeline SHALL emit a structured LLM error and transition through error recovery to wake listening ### Requirement: Local TTS synthesis and playback -The system SHALL synthesize assistant replies through a local TTS provider, with `sherpa-onnx` as the recommended first implementation candidate. +The system SHALL synthesize assistant replies through the configured TTS provider and SHALL play synthesized speech through the local system output path; cloud mode SHALL use NewAPI-compatible TTS and local mode SHALL use a local playback-capable TTS provider. -#### Scenario: Sentence is ready for speech -- **WHEN** the LLM stream produces a complete sentence or configured text chunk -- **THEN** the TTS provider SHALL synthesize that text into an audio segment +#### Scenario: Reply text is ready for speech +- **WHEN** the LLM produces a non-empty reply for a live turn +- **THEN** the TTS provider SHALL synthesize that text into locally playable speech -#### Scenario: First TTS segment is ready -- **WHEN** the first synthesized audio segment is available -- **THEN** the output Transport SHALL begin playback without waiting for every remaining LLM delta +#### Scenario: Playback starts +- **WHEN** the first synthesized audio output is available +- **THEN** the output Transport or macOS playback command SHALL begin playback through the local speaker #### Scenario: TTS fails -- **WHEN** TTS returns empty audio or raises an error -- **THEN** the pipeline SHALL emit a structured TTS error and SHALL recover without terminating the application +- **WHEN** TTS returns empty audio, fails to create an audio file, or playback command fails +- **THEN** the live runtime SHALL emit a structured TTS or playback error and SHALL recover without terminating default loop mode ### Requirement: Pipeline state machine The system SHALL expose deterministic pipeline states for `idle`, `wake_listening`, `speech_detecting`, `recording`, `transcribing`, `thinking`, `speaking`, `interrupted`, and `error_recovering`. @@ -222,7 +226,7 @@ The system SHALL protect credentials and minimize audio persistence. #### Scenario: API key is required - **WHEN** the LLM provider needs an OpenAI API key -- **THEN** the key SHALL be loaded from environment or local uncommitted configuration and SHALL NOT be committed +- **THEN** the key SHALL be loaded from `.env` or local uncommitted configuration and SHALL NOT be committed #### Scenario: Audio is processed - **WHEN** user speech is captured for STT @@ -233,19 +237,23 @@ The system SHALL protect credentials and minimize audio persistence. - **THEN** it SHALL send transcript text and necessary conversation context, not raw microphone audio ### Requirement: Testability -The system SHALL be designed so each stage can be tested with mock providers and file-based audio fixtures. +The system SHALL be designed so each stage can be tested with mock providers, file-based audio fixtures, and fake live runtime components without requiring real devices in automated tests. -#### Scenario: Pipeline is unit tested -- **WHEN** mock providers produce deterministic events -- **THEN** tests SHALL verify state transitions, context updates, and error recovery +#### Scenario: Repeated runtime is unit tested +- **WHEN** fake live providers produce two deterministic turns +- **THEN** tests SHALL verify two STT calls, two LLM calls, two TTS calls, two playback calls, and final return to standby -#### Scenario: Audio fixture is replayed -- **WHEN** a file-based audio fixture containing “小杰小杰” and user speech is replayed in a test Transport -- **THEN** the pipeline SHALL be testable without using a live microphone +#### Scenario: Temporary context is unit tested +- **WHEN** fake live providers run two turns in one runtime instance +- **THEN** tests SHALL verify the second LLM request includes the first turn user and assistant messages + +#### Scenario: Process-local context is unit tested +- **WHEN** a second runtime instance is created after a first instance has conversation history +- **THEN** tests SHALL verify the second instance starts with no user/assistant history #### Scenario: OpenSpec planning validation runs -- **WHEN** this planning change is complete -- **THEN** `openspec validate add-voice-pet-pipeline --strict` and `openspec validate --all --strict` SHALL pass +- **WHEN** this change is complete before implementation +- **THEN** `openspec validate complete-live-repeat-voice-runtime --strict` and `openspec validate --all --strict` SHALL pass ### Requirement: Module-level git commit gates The system implementation process SHALL require an immediate Git commit after each major module or milestone is completed, where a major module means a top-level task group from `tasks.md` or a phase milestone from `proposal.md`. @@ -266,3 +274,104 @@ The system implementation process SHALL require an immediate Git commit after ea - **WHEN** module-related changes remain unstaged or uncommitted after the module is completed - **THEN** the implementer SHALL include those changes in the module commit or explicitly separate unrelated external changes before continuing to the next module +### Requirement: Live repeat voice runtime +The system SHALL provide a `run-live` command that performs real repeated voice conversation with local microphone input, configured speech recognition and speech synthesis providers, cloud LLM reply generation, local speaker playback, and automatic return to standby. + +#### Scenario: Live runtime starts in standby +- **WHEN** the user runs `PYTHONPATH=src python3.11 -m owner_voice_pet run-live` +- **THEN** the system SHALL load `.env`, validate required live dependencies, initialize local audio/model providers, and enter a standby listening loop + +#### Scenario: Live runtime completes two turns +- **WHEN** the user wakes the system with “小杰小杰”, asks a question, hears the reply, then wakes it again and asks another question +- **THEN** the system SHALL complete wake, recording, STT, LLM, TTS, playback for both turns and SHALL return to standby after each turn + +#### Scenario: Once mode completes one turn +- **WHEN** the user runs `PYTHONPATH=src python3.11 -m owner_voice_pet run-live --once` +- **THEN** the system SHALL run at most one wake-to-playback turn and exit after the turn completes or fails with a documented live error + +### Requirement: Temporary in-process conversation history +The live runtime SHALL maintain conversation history only in memory for the current process and SHALL include retained user/assistant history in later LLM requests during that same process. + +#### Scenario: Second turn uses first turn history +- **WHEN** the first live turn appends a user transcript and an assistant reply +- **AND** a second live turn sends an LLM request +- **THEN** the second request SHALL include the first turn user message and assistant message unless configured context limits require truncation + +#### Scenario: New runtime starts empty +- **WHEN** a new `run-live` process or new live runtime instance starts +- **THEN** it SHALL NOT load user/assistant history from a previous process or previous runtime instance + +#### Scenario: Process exits +- **WHEN** the live runtime exits for any reason +- **THEN** conversation history SHALL be discarded and SHALL NOT be written to disk, database, logs, or model files + +### Requirement: Local speech model management +The system SHALL provide project-local speech model preparation and diagnostics for live VAD/STT operation, storing downloaded model artifacts under `models/` without committing them to Git. + +#### Scenario: Models are downloaded +- **WHEN** the user runs `python3.11 scripts/download_speech_models.py --dir models` +- **THEN** the script SHALL create or update a project-local model directory with the files required by the configured VAD/STT providers + +#### Scenario: Model check succeeds +- **WHEN** required dependencies and model files are available +- **THEN** `PYTHONPATH=src python3.11 -m owner_voice_pet model-check` SHALL exit successfully and report the model directory and checked providers + +#### Scenario: Model check fails +- **WHEN** `sherpa-onnx` is unavailable, a model file is missing, or a model cannot be loaded +- **THEN** `model-check` SHALL fail with a structured model error and SHALL NOT start live microphone listening + +### Requirement: Configurable speech provider mode +The system SHALL read `OWNER_SPEECH_PROVIDER` from `.env` to choose between cloud speech providers and local speech providers for first-version live runtime. + +#### Scenario: Cloud speech provider is selected +- **WHEN** `OWNER_SPEECH_PROVIDER=cloud` +- **THEN** live ASR SHALL use the configured cloud model from `OWNER_ASR_MODEL`, live TTS SHALL use the configured cloud model from `OWNER_TTS_MODEL`, and the implementation SHALL default those models to `mimo-v2.5-asr` and `mimo-v2.5-tts` + +#### Scenario: Local speech provider is selected +- **WHEN** `OWNER_SPEECH_PROVIDER=local` +- **THEN** live ASR/VAD SHALL use project-local speech model assets and local TTS SHALL use a local playback-capable provider + +#### Scenario: Speech provider is invalid +- **WHEN** `OWNER_SPEECH_PROVIDER` is neither `cloud` nor `local` +- **THEN** startup validation SHALL fail with a structured configuration error + +### Requirement: Live audio device readiness +The system SHALL provide live audio device diagnostics and SHALL use `sounddevice` for first-version real microphone and speaker access. + +#### Scenario: Device check succeeds +- **WHEN** `sounddevice` can be imported and at least one input and one output device are available +- **THEN** `PYTHONPATH=src python3.11 -m owner_voice_pet device-check` SHALL exit successfully and report usable audio devices + +#### Scenario: Device check fails +- **WHEN** `sounddevice` is missing, device query fails, microphone permission is denied, or no usable input/output device exists +- **THEN** `device-check` SHALL fail with a structured audio device error + +#### Scenario: Live runtime uses real devices +- **WHEN** `run-live` starts successfully +- **THEN** it SHALL use the local microphone as its default input and the local speaker or macOS playback command as its default output rather than fixture audio + +### Requirement: Live terminal state reporting +The live runtime SHALL emit concise Chinese terminal status messages for observable runtime states. + +#### Scenario: Normal turn status +- **WHEN** a live turn succeeds +- **THEN** terminal output SHALL include states equivalent to standby, wake hit, recording, transcribing, thinking, speaking, and returning to standby + +#### Scenario: Recoverable error status +- **WHEN** a live turn encounters empty STT, LLM failure, TTS failure, or playback failure +- **THEN** terminal output SHALL include the failing stage and a stable error code or recoverable explanation + +### Requirement: Live runtime error recovery +The live runtime SHALL recover from turn-level failures and continue listening in default loop mode. + +#### Scenario: LLM fails during default loop +- **WHEN** the LLM provider times out, is rate limited, or returns a network error during a live turn +- **THEN** the system SHALL report `LIVE_LLM_FAILED` or an equivalent structured error and SHALL return to standby without terminating the process + +#### Scenario: TTS or playback fails during default loop +- **WHEN** local TTS or audio playback fails during a live turn +- **THEN** the system SHALL report the failure and SHALL return to standby without terminating the process + +#### Scenario: Startup dependency fails +- **WHEN** `.env`, required models, or audio devices are unavailable at startup +- **THEN** the system SHALL exit with a documented non-zero status instead of entering a fake live loop diff --git a/src/owner_voice_pet/config.py b/src/owner_voice_pet/config.py index 430b5f1..eea18c8 100644 --- a/src/owner_voice_pet/config.py +++ b/src/owner_voice_pet/config.py @@ -13,7 +13,7 @@ class AppConfig: channels: int = 1 llm_base_url: str = "https://token-plan-cn.xiaomimimo.com/v1" llm_api_key: str | None = None - llm_model: str = "gpt-5.4-mini" + llm_model: str = "mimo-v2.5" llm_api_style: str = "chat_completions" llm_stream: bool = True audio_input_device: str | None = None @@ -23,7 +23,7 @@ class AppConfig: speech_provider: str = "cloud" asr_model: str = "mimo-v2.5-asr" tts_model: str = "mimo-v2.5-tts" - tts_voice: str = "alloy" + tts_voice: str = "mimo_default" speech_models_dir: Path = Path("models") context_max_messages: int = 12 context_max_chars: int = 12000 @@ -42,7 +42,7 @@ class AppConfig: channels=int(get("CHANNELS", "1") or "1"), llm_base_url=(get("LLM_BASE_URL", "https://token-plan-cn.xiaomimimo.com/v1") or "").rstrip("/"), llm_api_key=get("LLM_API_KEY"), - llm_model=get("LLM_MODEL", "gpt-5.4-mini") or "gpt-5.4-mini", + llm_model=get("LLM_MODEL", "mimo-v2.5") or "mimo-v2.5", llm_api_style=get("LLM_API_STYLE", "chat_completions") or "chat_completions", llm_stream=(get("LLM_STREAM", "1") or "1").lower() not in {"0", "false", "no"}, audio_input_device=get("AUDIO_INPUT_DEVICE"), @@ -52,7 +52,7 @@ class AppConfig: speech_provider=(get("SPEECH_PROVIDER", "cloud") or "cloud").lower(), asr_model=get("ASR_MODEL", "mimo-v2.5-asr") or "mimo-v2.5-asr", tts_model=get("TTS_MODEL", "mimo-v2.5-tts") or "mimo-v2.5-tts", - tts_voice=get("TTS_VOICE", "alloy") or "alloy", + tts_voice=get("TTS_VOICE", "mimo_default") or "mimo_default", speech_models_dir=Path(get("SPEECH_MODELS_DIR", "models") or "models"), context_max_messages=int(get("CONTEXT_MAX_MESSAGES", "12") or "12"), context_max_chars=int(get("CONTEXT_MAX_CHARS", "12000") or "12000"), diff --git a/src/owner_voice_pet/stt.py b/src/owner_voice_pet/stt.py index 43fe084..42d861c 100644 --- a/src/owner_voice_pet/stt.py +++ b/src/owner_voice_pet/stt.py @@ -1,12 +1,12 @@ from __future__ import annotations import io +import base64 import json import re import socket import urllib.error import urllib.request -import uuid import wave from collections.abc import Callable from pathlib import Path @@ -84,19 +84,28 @@ class CloudAsrSttProvider: "newapi-asr", "stt", ) - wav_bytes = _segment_to_wav_bytes(segment) - boundary = "owner-voice-pet-" + uuid.uuid4().hex - body = _multipart_form_data( - boundary, - fields={"model": self.config.asr_model, "response_format": "json"}, - files={"file": ("utterance.wav", "audio/wav", wav_bytes)}, - ) + audio_data = base64.b64encode(_segment_to_wav_bytes(segment)).decode("ascii") + payload = { + "model": self.config.asr_model, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": {"data": f"data:audio/wav;base64,{audio_data}"}, + } + ], + } + ], + "asr_options": {"language": "auto"}, + } request = urllib.request.Request( - self.config.api_url("/v1/audio/transcriptions"), - data=body, + self.config.api_url("/v1/chat/completions"), + data=json.dumps(payload).encode("utf-8"), headers={ "Authorization": f"Bearer {self.config.llm_api_key}", - "Content-Type": f"multipart/form-data; boundary={boundary}", + "Content-Type": "application/json", }, method="POST", ) @@ -119,7 +128,9 @@ class CloudAsrSttProvider: "newapi-asr", "stt", ) from exc - text = str(payload.get("text") or "").strip() + choices = payload.get("choices") or [] + message = (choices[0].get("message") if choices else {}) or {} + text = str(message.get("content") or payload.get("text") or "").strip() if not is_valid_transcript_text(text): raise ProviderError( ErrorCode.STT_EMPTY_TRANSCRIPT, @@ -261,27 +272,3 @@ def _segment_to_wav_bytes(segment: AudioSegment) -> bytes: wav.setframerate(segment.sample_rate) wav.writeframes(segment.pcm) return buffer.getvalue() - - -def _multipart_form_data( - boundary: str, - *, - fields: dict[str, str], - files: dict[str, tuple[str, str, bytes]], -) -> bytes: - body = bytearray() - for name, value in fields.items(): - body.extend(f"--{boundary}\r\n".encode()) - body.extend(f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode()) - body.extend(value.encode("utf-8")) - body.extend(b"\r\n") - for name, (filename, content_type, data) in files.items(): - body.extend(f"--{boundary}\r\n".encode()) - body.extend( - f'Content-Disposition: form-data; name="{name}"; filename="{filename}"\r\n'.encode() - ) - body.extend(f"Content-Type: {content_type}\r\n\r\n".encode()) - body.extend(data) - body.extend(b"\r\n") - body.extend(f"--{boundary}--\r\n".encode()) - return bytes(body) diff --git a/src/owner_voice_pet/tts.py b/src/owner_voice_pet/tts.py index 63f7c01..f97065b 100644 --- a/src/owner_voice_pet/tts.py +++ b/src/owner_voice_pet/tts.py @@ -1,6 +1,7 @@ from __future__ import annotations import math +import base64 import json import socket import struct @@ -166,12 +167,12 @@ class CloudTtsProvider: ) payload = { "model": self.config.tts_model, - "input": clean, - "voice": self.config.tts_voice, - "response_format": "mp3", + "messages": [{"role": "assistant", "content": clean}], + "audio": {"format": "wav", "voice": self.config.tts_voice}, + "stream": False, } request = urllib.request.Request( - self.config.api_url("/v1/audio/speech"), + self.config.api_url("/v1/chat/completions"), data=json.dumps(payload).encode("utf-8"), headers={ "Authorization": f"Bearer {self.config.llm_api_key}", @@ -181,7 +182,7 @@ class CloudTtsProvider: ) try: with self.urlopen(request, timeout=self.timeout_s) as response: - data = response.read() + payload = json.loads(response.read().decode("utf-8")) except urllib.error.HTTPError as exc: raise ProviderError( ErrorCode.TTS_SYNTHESIS_FAILED, @@ -190,7 +191,7 @@ class CloudTtsProvider: "newapi-tts", "tts", ) from exc - except (urllib.error.URLError, TimeoutError, socket.timeout) as exc: + except (urllib.error.URLError, TimeoutError, socket.timeout, json.JSONDecodeError) as exc: raise ProviderError( ErrorCode.TTS_SYNTHESIS_FAILED, f"cloud TTS request failed: {exc}", @@ -198,7 +199,11 @@ class CloudTtsProvider: "newapi-tts", "tts", ) from exc - if not data: + choices = payload.get("choices") or [] + message = (choices[0].get("message") if choices else {}) or {} + audio = message.get("audio") or {} + encoded = audio.get("data") + if not encoded: raise ProviderError( ErrorCode.TTS_EMPTY_AUDIO, "cloud TTS returned empty audio", @@ -206,4 +211,14 @@ class CloudTtsProvider: "newapi-tts", "tts", ) - return AudioSegment(data, 16000, 1, 0, max(120, len(clean) * 45), {"text": clean, "format": "mp3"}) + try: + data = base64.b64decode(encoded) + except (TypeError, ValueError) as exc: + raise ProviderError( + ErrorCode.TTS_SYNTHESIS_FAILED, + f"cloud TTS returned invalid base64 audio: {exc}", + True, + "newapi-tts", + "tts", + ) from exc + return AudioSegment(data, 24000, 1, 0, max(120, len(clean) * 45), {"text": clean, "format": "wav"}) diff --git a/tests/test_models_config.py b/tests/test_models_config.py index 9b32382..520a5ee 100644 --- a/tests/test_models_config.py +++ b/tests/test_models_config.py @@ -72,7 +72,7 @@ class ModelsConfigTests(unittest.TestCase): self.assertEqual(config.speech_provider, "cloud") self.assertEqual(config.asr_model, "mimo-v2.5-asr") self.assertEqual(config.tts_model, "mimo-v2.5-tts") - self.assertEqual(config.tts_voice, "alloy") + self.assertEqual(config.tts_voice, "mimo_default") self.assertEqual(str(config.speech_models_dir), "models") self.assertTrue(config.llm_stream) self.assertEqual(config.validate_basic(), []) diff --git a/tests/test_pipeline_llm_tts.py b/tests/test_pipeline_llm_tts.py index d3495fa..4e5f399 100644 --- a/tests/test_pipeline_llm_tts.py +++ b/tests/test_pipeline_llm_tts.py @@ -1,6 +1,7 @@ from __future__ import annotations import io +import base64 import json import unittest @@ -75,7 +76,17 @@ class PipelineLlmTtsTests(unittest.TestCase): return None def read(self) -> bytes: - return b"fake-mp3" + return json.dumps( + { + "choices": [ + { + "message": { + "audio": {"data": base64.b64encode(b"fake-wav").decode("ascii")} + } + } + ] + } + ).encode() requests = [] @@ -90,11 +101,14 @@ class PipelineLlmTtsTests(unittest.TestCase): provider.load() segment = provider.synthesize("你好") - self.assertEqual(segment.metadata["format"], "mp3") - self.assertEqual(segment.pcm, b"fake-mp3") + self.assertEqual(segment.metadata["format"], "wav") + self.assertEqual(segment.pcm, b"fake-wav") body = json.loads(requests[0].data.decode()) self.assertEqual(body["model"], "mimo-v2.5-tts") - self.assertIn("/v1/audio/speech", requests[0].full_url) + self.assertEqual(body["messages"][0]["role"], "assistant") + self.assertEqual(body["messages"][0]["content"], "你好") + self.assertEqual(body["audio"]["format"], "wav") + self.assertIn("/v1/chat/completions", requests[0].full_url) def test_pipeline_runs_from_wake_to_playback(self) -> None: frames = [ diff --git a/tests/test_wake_vad_stt.py b/tests/test_wake_vad_stt.py index f877c0b..43355af 100644 --- a/tests/test_wake_vad_stt.py +++ b/tests/test_wake_vad_stt.py @@ -113,7 +113,10 @@ class WakeVadSttTests(unittest.TestCase): return None def read(self) -> bytes: - return json.dumps({"text": "你好小杰", "language": "zh"}, ensure_ascii=False).encode() + return json.dumps( + {"choices": [{"message": {"content": "你好小杰"}}]}, + ensure_ascii=False, + ).encode() requests = [] @@ -129,8 +132,12 @@ class WakeVadSttTests(unittest.TestCase): transcript = provider.transcribe(AudioSegment(b"\x00\x00\x01\x00", 16000, 1, 0, 100)) self.assertEqual(transcript.text, "你好小杰") - self.assertIn("/v1/audio/transcriptions", requests[0].full_url) - self.assertIn(b'mimo-v2.5-asr', requests[0].data) + self.assertIn("/v1/chat/completions", requests[0].full_url) + body = json.loads(requests[0].data.decode()) + self.assertEqual(body["model"], "mimo-v2.5-asr") + content = body["messages"][0]["content"][0] + self.assertEqual(content["type"], "input_audio") + self.assertTrue(content["input_audio"]["data"].startswith("data:audio/wav;base64,")) if __name__ == "__main__":