From b9b501cf74c970c0cd273813d268fb05bee8c2ed Mon Sep 17 00:00:00 2001 From: mkbk Date: Wed, 17 Jun 2026 23:57:24 +0800 Subject: [PATCH] =?UTF-8?q?[=E7=9C=9F=E5=AE=9E=E6=B5=81=E7=A8=8B=E5=91=BD?= =?UTF-8?q?=E4=BB=A4]=EF=BC=9A=E5=AE=8C=E6=88=90=E5=AE=8C=E6=95=B4?= =?UTF-8?q?=E9=93=BE=E8=B7=AF=E8=87=AA=E6=B5=8B=E5=85=A5=E5=8F=A3=EF=BC=8C?= =?UTF-8?q?=E5=8C=85=E5=90=AB=E7=9C=9F=E5=AE=9EProvider=20CLI=E3=80=81?= =?UTF-8?q?=E6=92=AD=E6=94=BE=E9=AA=8C=E6=94=B6=E5=92=8C=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 9 + .../design.md | 5 +- .../proposal.md | 12 + .../specs/voice-pet-pipeline/spec.md | 10 +- .../tasks.md | 7 + src/owner_voice_pet/cli.py | 24 ++ src/owner_voice_pet/real_live_check.py | 339 ++++++++++++++++++ tests/test_cli_acceptance.py | 12 + tests/test_real_live_check.py | 47 +++ 9 files changed, 462 insertions(+), 3 deletions(-) create mode 100644 src/owner_voice_pet/real_live_check.py create mode 100644 tests/test_real_live_check.py diff --git a/README.md b/README.md index d02c513..21aa129 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,14 @@ python3.11 scripts/download_speech_models.py --dir models .venv/bin/python -m owner_voice_pet simulate-live --turns 2 --fixture /tmp/owner-simulated-mic.jsonl ``` +真实 Provider 完整链路自测： + +```bash +.venv/bin/python -m owner_voice_pet real-live-check --turns 2 +``` + +这个命令不用真人对着麦克风说话，会用 macOS `say/afconvert` 生成“小杰小杰”和两轮问题音频，再驱动真实 `VoiceAssistantPipeline`：本地 KWS、本地 VAD、本地降噪、本地实时字幕、本地 final STT、云端 LLM、本地 TTS、Transport 播放、恢复待机都会跑到。默认会真实播放 ACK 和回复；如果只想检查链路但不发声，可以加 `--no-playback`。输出 JSON 中 `success=true` 表示完整链路通过，并会检查第二轮 LLM 请求是否携带第一轮临时历史。 + 单轮验收： ```bash @@ -134,6 +142,7 @@ python3.11 scripts/download_speech_models.py --dir models .venv/bin/python -m compileall src tests scripts .venv/bin/python -m unittest discover -s tests .venv/bin/python -m owner_voice_pet simulate-live --turns 2 +.venv/bin/python -m owner_voice_pet real-live-check --turns 2 .venv/bin/python -m owner_voice_pet acceptance .venv/bin/python -m owner_voice_pet validate-assets .venv/bin/python -m owner_voice_pet security-check diff --git a/openspec/changes/separate-wake-and-realtime-transcript/design.md b/openspec/changes/separate-wake-and-realtime-transcript/design.md index d3bbaf2..9fa412e 100644 --- a/openspec/changes/separate-wake-and-realtime-transcript/design.md +++ b/openspec/changes/separate-wake-and-realtime-transcript/design.md @@ -243,7 +243,8 @@ The transport is bounded: if frames are exhausted unexpectedly, it raises a stru The simulated microphone command validates pipeline ordering with fake providers, but it does not prove that real local models and real playback can run together. The next acceptance layer uses generated 16 kHz PCM utterances as microphone input while keeping the provider chain real: ```text -generated wake/question PCM +owner_voice_pet real-live-check + -> generated wake/question PCM via macOS say + afconvert -> MemoryAudioTransport(flush preserves prefilled fixture frames) -> SherpaOnnxKeywordWakeWordProvider(real KWS) -> HybridVadProvider(real sherpa VAD + energy fallback) @@ -255,6 +256,8 @@ generated wake/question PCM -> SoundDeviceAudioTransport.play_pcm(real speaker smoke) ``` +The CLI defaults to two generated turns: “我叫阿明，请你记住我的名字。” followed by “我叫什么名字？”. A recording wrapper around the real LLM provider stores the messages sent to each LLM call, so the command can verify that the second request contains first-turn process-local user/assistant history without exposing the API key. `--question` can override the generated prompts, `--voice` controls the macOS `say` voice, and `--no-playback` keeps TTS synthesis and playback events while skipping speaker output for quiet automated runs. + Playback drain semantics are intentionally split by transport type: 1. `SoundDeviceAudioTransport.flush_input()` clears the realtime callback queue that accumulated during ACK/TTS playback. diff --git a/openspec/changes/separate-wake-and-realtime-transcript/proposal.md b/openspec/changes/separate-wake-and-realtime-transcript/proposal.md index 17745eb..7f996b1 100644 --- a/openspec/changes/separate-wake-and-realtime-transcript/proposal.md +++ b/openspec/changes/separate-wake-and-realtime-transcript/proposal.md @@ -102,6 +102,9 @@ 37. 当 ACK 文本为空且没有实际播放行为时，runtime SHALL NOT 执行 post-playback drain，避免无播放场景误清正式问题开头。 38. 模拟麦克风和 fixture replay 场景 SHALL 能保留预排的未来输入帧，并仍记录 flush 调用次数，用于验证真实队列清理动作不吞掉测试中尚未“发生”的后续用户语音。 39. 完整链路自测 SHALL 尽量使用真实本地 provider 和真实云端 LLM：真实 KWS、VAD、降噪、STT、LLM、TTS 和播放；无法自动使用真人麦克风时，允许用生成的 16 kHz PCM 音频帧替代物理麦克风输入。 +40. 系统 SHALL 提供 `owner_voice_pet real-live-check` 命令，把真实 Provider 完整链路自测固化为可重复入口，而不是依赖一次性内联脚本。 +41. `real-live-check` SHALL 默认生成两轮音频并执行真实本地 KWS/VAD/降噪/STT、本地 TTS、Transport 播放和云端 LLM；输出 JSON 必须包含完成轮数、final transcripts、事件计数、播放段数、flush 次数、临时上下文检查和错误列表。 +42. `real-live-check --no-playback` SHALL 仍执行本地 TTS 合成和 pipeline 播放事件记录，但不实际向扬声器发声，便于自动化测试和无声环境排查。 ### 非功能需求 @@ -432,6 +435,13 @@ standby - [ ] 12.4 执行真实 provider 脚本化闭环验收；前置条件：本地模型、`.env` LLM key、macOS TTS/播放可用；验收标准：生成 wake/question 音频帧驱动真实 KWS、VAD、denoiser、本地 STT、云端 LLM、本地 TTS、播放，两轮上下文可引用第一轮历史；测试要点：输出不泄露 key，模型文件不进 Git；优先级：P0；预计：60 分钟。 - [ ] 12.5 执行全量门禁并提交“真实链路自测”模块；前置条件：12.1 至 12.4 完成；验收标准：compileall、unittest、simulate-live、security-check、model-check、device-check、OpenSpec strict、git diff check 全通过后中文 commit；优先级：P0；预计：45 分钟。 +### 13. 固化真实 Provider 完整流程 CLI + +- [ ] 13.1 更新 OpenSpec 和 README 描述 `real-live-check`；前置条件：12 组发现临时脚本证据不可重复；验收标准：proposal/design/spec/tasks/README 明确命令用途、默认真实播放、`--no-playback` 和 JSON 检查项；测试要点：OpenSpec strict；优先级：P0；预计：35 分钟。 +- [ ] 13.2 实现 `owner_voice_pet real-live-check`；前置条件：13.1 完成；验收标准：命令从 `.env` 读取配置，生成 wake/question PCM，运行真实 KWS/VAD/denoise/STT/LLM/TTS/pipeline/playback，并输出结构化 JSON；测试要点：CLI wiring 测试、transport 单测；优先级：P0；预计：60 分钟。 +- [ ] 13.3 执行真实命令验收和全量门禁；前置条件：13.2 完成；验收标准：`real-live-check --turns 2`、compileall、unittest、simulate-live、security-check、model-check、device-check、OpenSpec strict、git diff check 通过；优先级：P0；预计：60 分钟。 +- [ ] 13.4 提交“真实流程命令”模块；前置条件：13.3 通过；验收标准：中文 commit 信息为 `[真实流程命令]：完成完整链路自测入口，包含真实Provider CLI、播放验收和文档测试`，提交后 `git status --short` 为空；优先级：P0；预计：10 分钟。 + ## Spec Deltas ### 新增能力 @@ -453,6 +463,7 @@ standby 11. `Local voice chain and noise filtering`：新增本地 STT/TTS 默认、GTCRN 降噪、CTC 模型、partial 稳定过滤和音频不上传要求。 12. `Simulated microphone live acceptance`：新增不依赖真人麦克风的 live pipeline 模拟验收，覆盖两轮重复对话、短噪声过滤、背景噪声端点、fixture 写入/回放和空帧防卡死。 13. `Real provider live fixture acceptance`：新增真实 provider 脚本化闭环验收和 ACK/TTS 播放后 0ms flush 语义，确保真实本地模型、云端 LLM、本地 TTS/播放在同一 pipeline 中可连续运行。 +14. `Real live check CLI`：新增 `real-live-check` 可重复命令，把完整真实 Provider 链路验收从一次性脚本升级为仓库内正式工具。 ### 删除项 @@ -475,6 +486,7 @@ standby 9. M9：本地语音链路、高质量实时字幕和正式问题降噪完成后提交，继续保留真人 `run-live` 验收任务，不在用户确认前归档。 10. M10：模拟麦克风自动验收完成后提交，作为真人验收前的可重复自测入口；仍不归档变更，直到真实 `run-live` 行为由用户确认。 11. M11：真实 provider 脚本化闭环和 ACK 缓冲修正完成后提交；仍保留用户真人 `run-live` 体验验收，不在用户确认前归档。 +12. M12：`real-live-check` 正式 CLI、文档、测试和真实命令验收完成后提交；仍保留物理麦克风 `run-live` 真人体验验收。 估时： diff --git a/openspec/changes/separate-wake-and-realtime-transcript/specs/voice-pet-pipeline/spec.md b/openspec/changes/separate-wake-and-realtime-transcript/specs/voice-pet-pipeline/spec.md index 5db23fc..002ed78 100644 --- a/openspec/changes/separate-wake-and-realtime-transcript/specs/voice-pet-pipeline/spec.md +++ b/openspec/changes/separate-wake-and-realtime-transcript/specs/voice-pet-pipeline/spec.md @@ -134,13 +134,14 @@ The system SHALL provide a deterministic simulated microphone acceptance command The system SHALL support scripted full-chain acceptance that drives the live pipeline with generated microphone PCM while using real local speech providers, the configured cloud LLM, and local TTS/playback capability. #### Scenario: Real provider fixture completes one turn -- **WHEN** generated wake and formal question PCM frames are passed through the live pipeline +- **WHEN** the developer runs `.venv/bin/python -m owner_voice_pet real-live-check --turns 1` - **THEN** the pipeline SHALL use the real local KWS, local VAD, local denoiser, local STT, configured cloud LLM, local TTS, and playback provider path to complete a wake-to-standby turn #### Scenario: Real provider fixture completes repeated turns with temporary context -- **WHEN** two generated wake/question turns run in one process and the first turn gives a user fact +- **WHEN** the developer runs `.venv/bin/python -m owner_voice_pet real-live-check --turns 2` and the first generated turn gives a user fact - **THEN** the second LLM request SHALL include the first turn's user and assistant history from process-local memory - **AND** no history SHALL be read from disk or persisted after the process exits +- **AND** JSON output SHALL include checks for completed turns, event counts, final transcripts, playback, flush count, and temporary context #### Scenario: Zero playback drain still flushes queued echo - **WHEN** ACK or TTS playback completes and `OWNER_POST_PLAYBACK_DRAIN_MS=0` @@ -155,6 +156,11 @@ The system SHALL support scripted full-chain acceptance that drives the live pip - **WHEN** a memory or fixture transport represents pre-generated future microphone frames - **THEN** tests SHALL be able to configure flush to record the cleanup call without deleting those future frames +#### Scenario: Real provider check runs quietly +- **WHEN** the developer runs `.venv/bin/python -m owner_voice_pet real-live-check --turns 2 --no-playback` +- **THEN** the command SHALL still synthesize local TTS output and emit pipeline playback events +- **AND** it SHALL skip actual speaker output for automated or quiet environments + ### Requirement: Wake acknowledgement before recording The live runtime SHALL provide an audible local acknowledgement after local wake detection and before it starts recording the user's formal question. diff --git a/openspec/changes/separate-wake-and-realtime-transcript/tasks.md b/openspec/changes/separate-wake-and-realtime-transcript/tasks.md index 3cc10ea..1d41c33 100644 --- a/openspec/changes/separate-wake-and-realtime-transcript/tasks.md +++ b/openspec/changes/separate-wake-and-realtime-transcript/tasks.md @@ -97,3 +97,10 @@ - [x] 12.3 调整 `MemoryAudioTransport` 和模拟验收；前置条件：12.2 完成；验收标准：默认 flush 仍清空输入，模拟 live 显式保留预排帧并记录 flush 调用次数；测试要点：transport flush 双语义测试、两轮 simulated live 回归；优先级：P0；预计：35 分钟。 - [x] 12.4 执行真实 provider 脚本化闭环验收；前置条件：本地模型、`.env` LLM key、macOS TTS/播放可用；验收标准：生成 wake/question 音频帧驱动真实 KWS、VAD、denoiser、本地 STT、云端 LLM、本地 TTS、播放，两轮上下文可引用第一轮历史；测试要点：输出不泄露 key，模型文件不进 Git；优先级：P0；预计：60 分钟。 - [x] 12.5 执行全量门禁并提交“真实链路自测”模块；前置条件：12.1 至 12.4 完成；验收标准：compileall、unittest、simulate-live、security-check、model-check、device-check、OpenSpec strict、git diff check 全通过后中文 commit；优先级：P0；预计：45 分钟。 + +## 13. 固化真实 Provider 完整流程 CLI + +- [x] 13.1 更新 OpenSpec 和 README 描述 `real-live-check`；前置条件：12 组发现临时脚本证据不可重复；验收标准：proposal/design/spec/tasks/README 明确命令用途、默认真实播放、`--no-playback` 和 JSON 检查项；测试要点：OpenSpec strict；优先级：P0；预计：35 分钟。 +- [x] 13.2 实现 `owner_voice_pet real-live-check`；前置条件：13.1 完成；验收标准：命令从 `.env` 读取配置，生成 wake/question PCM，运行真实 KWS/VAD/denoise/STT/LLM/TTS/pipeline/playback，并输出结构化 JSON；测试要点：CLI wiring 测试、transport 单测；优先级：P0；预计：60 分钟。 +- [x] 13.3 执行真实命令验收和全量门禁；前置条件：13.2 完成；验收标准：`real-live-check --turns 2`、compileall、unittest、simulate-live、security-check、model-check、device-check、OpenSpec strict、git diff check 通过；优先级：P0；预计：60 分钟。 +- [x] 13.4 提交“真实流程命令”模块；前置条件：13.3 通过；验收标准：中文 commit 信息为 `[真实流程命令]：完成完整链路自测入口，包含真实Provider CLI、播放验收和文档测试`，提交后 `git status --short` 为空；优先级：P0；预计：10 分钟。 diff --git a/src/owner_voice_pet/cli.py b/src/owner_voice_pet/cli.py index da4d1c4..f2c4a34 100644 --- a/src/owner_voice_pet/cli.py +++ b/src/owner_voice_pet/cli.py @@ -13,6 +13,7 @@ from .conversation import ConversationContext from .llm import MockLlmProvider, OpenAICompatibleLlmProvider from .models import AudioFrame, ProviderError from .pipeline import VoicePipeline +from .real_live_check import run_real_live_check from .runtime import build_live_runtime from .simulation import run_simulated_live from .speech_models import check_speech_models, model_status_errors @@ -40,6 +41,12 @@ def main(argv: list[str] | None = None) -> int: simulate.add_argument("--turns", type=int, default=2, help="Number of simulated turns. Default: 2") simulate.add_argument("--fixture", default=None, help="Replay simulated microphone frames from JSONL") simulate.add_argument("--write-fixture", default=None, help="Write generated simulated microphone frames to JSONL") + real_check = subparsers.add_parser("real-live-check", help="Run generated-audio live check with real providers") + real_check.add_argument("--turns", type=int, default=2, help="Number of generated live turns. Default: 2") + real_check.add_argument("--voice", default="Tingting", help="macOS say voice used for generated microphone input") + real_check.add_argument("--wake-text", default="小杰小杰。", help="Generated wake utterance") + real_check.add_argument("--question", action="append", default=None, help="Generated user question; can be repeated") + real_check.add_argument("--no-playback", action="store_true", help="Synthesize but do not play generated TTS output") smoke = subparsers.add_parser("llm-smoke", help="Call configured OpenAI/NewAPI endpoint") smoke.add_argument("--message", default="用一句中文回复：小杰在线。") smoke.add_argument("--no-stream", action="store_true") @@ -159,6 +166,23 @@ def main(argv: list[str] | None = None) -> int: print(json.dumps(data, ensure_ascii=False, sort_keys=True)) return 0 if data["success"] else 1 + if args.command == "real-live-check": + config = AppConfig.from_dotenv(args.env_file) + try: + data = run_real_live_check( + config=config, + turns=args.turns, + voice=args.voice, + wake_text=args.wake_text, + questions=args.question, + play_audio=not args.no_playback, + ) + except (ProviderError, ValueError) as exc: + print(json.dumps({"success": False, "error": str(exc)}, ensure_ascii=False, sort_keys=True)) + return 1 + print(json.dumps(data, ensure_ascii=False, sort_keys=True)) + return 0 if data["success"] else 1 + if args.command == "acceptance": result = run_acceptance() print(json.dumps(result, ensure_ascii=False, sort_keys=True)) diff --git a/src/owner_voice_pet/real_live_check.py b/src/owner_voice_pet/real_live_check.py new file mode 100644 index 0000000..9cdfcb7 --- /dev/null +++ b/src/owner_voice_pet/real_live_check.py @@ -0,0 +1,339 @@ +from __future__ import annotations + +import subprocess +import tempfile +import wave +from collections import Counter, deque +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field, replace +from pathlib import Path + +from .assistant_pipeline import VoiceAssistantPipeline +from .audio_preprocess import SherpaOnnxDenoiserPreprocessor +from .config import AppConfig +from .conversation import ConversationContext +from .events import PipelineEventBus +from .llm import OpenAICompatibleLlmProvider +from .models import AudioFrame, AudioSegment, Message, PlaybackResult, ReplyDelta, TransportHealth +from .stt import SherpaOnnxSttProvider +from .transport import SoundDeviceAudioTransport +from .tts import MacSayTtsProvider +from .vad import EnergyVadProvider, HybridVadProvider, PrimarySpeakerVadRecorder, SherpaOnnxVadProvider +from .wakeword import SherpaOnnxKeywordWakeWordProvider + +FRAME_MS = 20 +SAMPLE_RATE = 16000 +FRAME_BYTES = int(SAMPLE_RATE * FRAME_MS / 1000) * 2 +DEFAULT_WAKE_TEXT = "小杰小杰。" +DEFAULT_QUESTIONS = [ + "我叫阿明，请你记住我的名字。", + "我叫什么名字？", +] + + +@dataclass(slots=True) +class RealLiveCheckReporter: + statuses: list[dict[str, object]] = field(default_factory=list) + partials: list[dict[str, object]] = field(default_factory=list) + finals: list[dict[str, object]] = field(default_factory=list) + errors: list[dict[str, object]] = field(default_factory=list) + + def status(self, state: str, message: str, *, turn_id: int | None = None) -> None: + self.statuses.append({"turn": turn_id, "state": state, "message": message}) + + def transcript(self, text: str, *, final: bool, turn_id: int | None = None) -> None: + target = self.finals if final else self.partials + target.append({"turn": turn_id, "text": text}) + + def error(self, stage: str, code: str, message: str, *, turn_id: int | None = None) -> None: + self.errors.append({"turn": turn_id, "stage": stage, "code": code, "message": message}) + + +class RecordingLlmProvider: + def __init__(self, delegate: OpenAICompatibleLlmProvider) -> None: + self.delegate = delegate + self.calls: list[list[Message]] = [] + + def stream_reply(self, messages: Sequence[Message]) -> Iterable[ReplyDelta]: + self.calls.append(list(messages)) + yield from self.delegate.stream_reply(messages) + + +class FixtureLiveAudioTransport: + """Generated microphone input with optional real speaker output.""" + + def __init__( + self, + frames: list[AudioFrame], + *, + play_audio: bool = True, + output_transport: SoundDeviceAudioTransport | None = None, + ) -> None: + self._frames: deque[AudioFrame] = deque(frames) + self.play_audio = play_audio + self.output_transport = output_transport or SoundDeviceAudioTransport() + self.played_segments: list[AudioSegment] = [] + self.flush_count = 0 + self.started = False + + def start_input(self, device_id: str | None = None, sample_rate: int = 16000, channels: int = 1) -> None: + self.started = True + + def read_frames(self, timeout_ms: int) -> list[AudioFrame]: + if not self.started or not self._frames: + return [] + return [self._frames.popleft()] + + def play_pcm(self, segment: AudioSegment, interrupt: bool = False) -> PlaybackResult: + if interrupt: + self.played_segments.clear() + self.played_segments.append(segment) + if not self.play_audio: + return PlaybackResult(True, segment.duration_ms) + return self.output_transport.play_pcm(segment, interrupt=interrupt) + + def flush_input(self) -> int: + self.flush_count += 1 + return 0 + + def stop(self) -> None: + self.started = False + + def health(self) -> TransportHealth: + output = self.output_transport.health() + return TransportHealth(True, output.output_available if self.play_audio else True, "fixture input") + + +def run_real_live_check( + *, + config: AppConfig, + turns: int = 2, + voice: str = "Tingting", + wake_text: str = DEFAULT_WAKE_TEXT, + questions: list[str] | None = None, + play_audio: bool = True, +) -> dict[str, object]: + if turns <= 0: + raise ValueError("turns must be positive") + selected_questions = _question_list(turns, questions) + config = replace( + config, + speech_provider="local", + noise_filter_enabled=True, + realtime_transcript_enabled=True, + llm_stream=False, + post_playback_drain_ms=0, + ) + config.require_llm_credentials() + frames = _generated_live_frames(config, turns=turns, voice=voice, wake_text=wake_text, questions=selected_questions) + reporter = RealLiveCheckReporter() + event_bus = PipelineEventBus() + stt = SherpaOnnxSttProvider(str(config.speech_models_dir)) + tts = MacSayTtsProvider() + llm = RecordingLlmProvider(OpenAICompatibleLlmProvider(config, timeout_s=60)) + transport = FixtureLiveAudioTransport( + frames, + play_audio=play_audio, + output_transport=SoundDeviceAudioTransport(output_device=config.audio_output_device), + ) + pipeline = VoiceAssistantPipeline( + config=config, + transport=transport, + wakeword=SherpaOnnxKeywordWakeWordProvider( + config.speech_models_dir, + keyword=config.wake_word, + keywords_file=config.wake_keywords_file, + threshold=config.wake_kws_threshold, + score=config.wake_kws_score, + ), + vad_recorder=PrimarySpeakerVadRecorder( + HybridVadProvider( + SherpaOnnxVadProvider(config.speech_models_dir, threshold=config.vad_threshold), + EnergyVadProvider(), + ), + min_duration_ms=config.vad_min_duration_ms, + end_silence_ms=config.vad_end_silence_ms, + no_speech_timeout_ms=config.vad_no_speech_timeout_ms, + max_recording_ms=config.vad_max_recording_ms, + speaker_profile_ms=config.speaker_profile_ms, + speaker_profile_min_ms=config.speaker_profile_min_ms, + speaker_absent_ms=config.speaker_absent_ms, + similarity_threshold=config.speaker_similarity_threshold, + min_rms=config.speaker_min_rms, + ), + audio_preprocessor=SherpaOnnxDenoiserPreprocessor(config.speech_models_dir), + stt=stt, + realtime_stt=stt, + llm=llm, + tts=tts, + ack_tts=tts, + context=ConversationContext(max_messages=config.context_max_messages, max_chars=config.context_max_chars), + reporter=reporter, + event_bus=event_bus, + ) + summary = pipeline.run(max_turns=turns) + event_counts = Counter(event.type for event in event_bus.events) + context_messages = [{"role": item.role, "content": item.content[:120]} for item in pipeline.context.messages()] + checks = { + "completed_turns": summary.completed_turns == turns, + "no_failed_turns": summary.failed_turns == 0, + "wake_per_turn": event_counts["wake_detected"] >= turns, + "speech_per_turn": event_counts["speech_started"] >= turns, + "final_stt_per_turn": event_counts["transcript_final"] >= turns, + "llm_per_turn": len(llm.calls) >= turns, + "tts_per_turn": event_counts["tts_started"] >= turns, + "playback_per_turn": event_counts["playback_finished"] >= turns, + "standby_per_turn": event_counts["standby_resumed"] >= turns, + "temporary_context_in_second_llm": _second_call_contains_first_turn(llm.calls), + "real_output_playback": (not play_audio) or len(transport.played_segments) >= turns * 2, + } + return { + "success": all(checks.values()) and not reporter.errors, + "turns": turns, + "completed_turns": summary.completed_turns, + "failed_turns": summary.failed_turns, + "checks": checks, + "final_transcripts": [str(item["text"]) for item in reporter.finals], + "partials_preview": reporter.partials[:8], + "event_counts": dict(sorted(event_counts.items())), + "played_segments": len(transport.played_segments), + "flush_count": transport.flush_count, + "context_messages": context_messages, + "errors": reporter.errors, + "play_audio": play_audio, + "voice": voice, + } + + +def _question_list(turns: int, questions: list[str] | None) -> list[str]: + selected = list(questions or DEFAULT_QUESTIONS) + while len(selected) < turns: + selected.append(f"第{len(selected) + 1}轮完整流程测试。") + return selected[:turns] + + +def _generated_live_frames( + config: AppConfig, + *, + turns: int, + voice: str, + wake_text: str, + questions: list[str], +) -> list[AudioFrame]: + with tempfile.TemporaryDirectory() as tmp_dir: + tmp = Path(tmp_dir) + wake_pcm = _synthesize_text_pcm(wake_text, tmp, "wake", voice=voice) + question_pcm = [_synthesize_text_pcm(text, tmp, f"q{idx + 1}", voice=voice) for idx, text in enumerate(questions)] + wake_frames = _trim_wake_frames_to_detection(config, _frames_from_pcm(wake_pcm, 0, 0, prefix="wake")) + frames: list[AudioFrame] = [] + frame_id = 0 + timestamp_ms = 0 + for turn_index, pcm in enumerate(question_pcm, start=1): + copied_wake = [ + AudioFrame( + item.pcm, + item.sample_rate, + item.channels, + timestamp_ms + idx * FRAME_MS, + frame_id + idx, + dict(item.metadata), + ) + for idx, item in enumerate(wake_frames) + ] + frames.extend(copied_wake) + frame_id += len(copied_wake) + timestamp_ms += len(copied_wake) * FRAME_MS + gap = _silence(frame_id, timestamp_ms, 160, prefix=f"turn{turn_index}-gap") + frames.extend(gap) + frame_id += len(gap) + timestamp_ms += len(gap) * FRAME_MS + question_frames = _frames_from_pcm(pcm, frame_id, timestamp_ms, prefix=f"turn{turn_index}-question") + frames.extend(question_frames) + frame_id += len(question_frames) + timestamp_ms += len(question_frames) * FRAME_MS + tail = _silence(frame_id, timestamp_ms, 900, prefix=f"turn{turn_index}-tail") + frames.extend(tail) + frame_id += len(tail) + timestamp_ms += len(tail) * FRAME_MS + return frames + + +def _synthesize_text_pcm(text: str, tmp: Path, name: str, *, voice: str) -> bytes: + aiff_path = tmp / f"{name}.aiff" + wav_path = tmp / f"{name}.wav" + try: + subprocess.run( + ["say", "-v", voice, "-o", str(aiff_path), text], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + subprocess.run( + ["afconvert", "-f", "WAVE", "-d", "LEI16@16000", "-c", "1", str(aiff_path), str(wav_path)], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + except (FileNotFoundError, subprocess.CalledProcessError) as exc: + raise ValueError(f"failed to generate fixture speech with macOS say/afconvert: {exc}") from exc + with wave.open(str(wav_path), "rb") as handle: + if handle.getframerate() != SAMPLE_RATE or handle.getnchannels() != 1 or handle.getsampwidth() != 2: + raise ValueError("generated speech must be 16 kHz mono int16 PCM") + return handle.readframes(handle.getnframes()) + + +def _frames_from_pcm(pcm: bytes, start_id: int, start_ms: int, *, prefix: str) -> list[AudioFrame]: + frames: list[AudioFrame] = [] + frame_id = start_id + timestamp_ms = start_ms + for offset in range(0, len(pcm), FRAME_BYTES): + chunk = pcm[offset : offset + FRAME_BYTES] + if len(chunk) < FRAME_BYTES: + chunk += b"\x00" * (FRAME_BYTES - len(chunk)) + frames.append( + AudioFrame(chunk, SAMPLE_RATE, 1, timestamp_ms, frame_id, {"duration_ms": FRAME_MS, "fixture": prefix}) + ) + frame_id += 1 + timestamp_ms += FRAME_MS + return frames + + +def _silence(start_id: int, start_ms: int, duration_ms: int, *, prefix: str) -> list[AudioFrame]: + return [ + AudioFrame( + b"\x00" * FRAME_BYTES, + SAMPLE_RATE, + 1, + start_ms + idx * FRAME_MS, + start_id + idx, + {"duration_ms": FRAME_MS, "fixture": prefix}, + ) + for idx in range(duration_ms // FRAME_MS) + ] + + +def _trim_wake_frames_to_detection(config: AppConfig, frames: list[AudioFrame]) -> list[AudioFrame]: + probe_frames = list(frames) + if probe_frames: + last = probe_frames[-1] + probe_frames.extend(_silence(last.frame_id + 1, last.timestamp_ms + FRAME_MS, 1400, prefix="wake-tail")) + wake = SherpaOnnxKeywordWakeWordProvider( + config.speech_models_dir, + keyword=config.wake_word, + keywords_file=config.wake_keywords_file, + threshold=config.wake_kws_threshold, + score=config.wake_kws_score, + ) + wake.load() + wake.reset() + for idx, frame in enumerate(probe_frames): + if wake.detect(frame) is not None: + return probe_frames[: idx + 1] + raise ValueError("generated wake audio did not trigger local KWS") + + +def _second_call_contains_first_turn(calls: list[list[Message]]) -> bool: + if len(calls) < 2: + return False + second_contents = "\n".join(message.content for message in calls[1]) + return "我叫阿明" in second_contents and any(message.role == "assistant" for message in calls[1]) diff --git a/tests/test_cli_acceptance.py b/tests/test_cli_acceptance.py index 82d371f..b9b3791 100644 --- a/tests/test_cli_acceptance.py +++ b/tests/test_cli_acceptance.py @@ -111,6 +111,18 @@ class CliAcceptanceTests(unittest.TestCase): self.assertEqual(data["final_transcripts"], ["第1轮模拟问题", "第2轮模拟问题"]) self.assertTrue(data["checks"]["partial_noise_filtered"]) + def test_real_live_check_command_invokes_real_fixture_runner(self) -> None: + expected = {"success": True, "completed_turns": 2} + with patch("owner_voice_pet.cli.run_real_live_check", return_value=expected) as runner: + code, data = self.call("real-live-check", "--turns", "2", "--no-playback", "--question", "第一问") + + self.assertEqual(code, 0) + self.assertEqual(data, expected) + kwargs = runner.call_args.kwargs + self.assertEqual(kwargs["turns"], 2) + self.assertFalse(kwargs["play_audio"]) + self.assertEqual(kwargs["questions"], ["第一问"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_real_live_check.py b/tests/test_real_live_check.py new file mode 100644 index 0000000..a2dbf43 --- /dev/null +++ b/tests/test_real_live_check.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import unittest + +from owner_voice_pet.models import AudioFrame, AudioSegment, PlaybackResult, TransportHealth +from owner_voice_pet.real_live_check import FixtureLiveAudioTransport + + +def frame(idx: int) -> AudioFrame: + return AudioFrame(b"\x00\x00", 16000, 1, idx * 20, idx, {"duration_ms": 20}) + + +class FakeOutputTransport: + def __init__(self) -> None: + self.played: list[AudioSegment] = [] + + def play_pcm(self, segment: AudioSegment, interrupt: bool = False) -> PlaybackResult: + self.played.append(segment) + return PlaybackResult(True, segment.duration_ms) + + def health(self) -> TransportHealth: + return TransportHealth(True, True, "fake") + + +class RealLiveCheckTests(unittest.TestCase): + def test_fixture_transport_flush_preserves_generated_future_frames(self) -> None: + transport = FixtureLiveAudioTransport([frame(1), frame(2)], play_audio=False) + transport.start_input() + + self.assertEqual(transport.flush_input(), 0) + self.assertEqual(transport.flush_count, 1) + self.assertEqual([item.frame_id for item in transport.read_frames(10)], [1]) + + def test_fixture_transport_can_delegate_real_playback(self) -> None: + output = FakeOutputTransport() + transport = FixtureLiveAudioTransport([], play_audio=True, output_transport=output) + segment = AudioSegment(b"\x00\x00", 16000, 1, 0, 20) + + result = transport.play_pcm(segment) + + self.assertTrue(result.played) + self.assertEqual(output.played, [segment]) + self.assertEqual(transport.played_segments, [segment]) + + +if __name__ == "__main__": + unittest.main()