from __future__ import annotations import time from contextlib import contextmanager from datetime import datetime import subprocess import tempfile import wave from collections import Counter, deque from collections.abc import Iterable, Sequence from dataclasses import dataclass, field, replace from pathlib import Path from .assistant_pipeline import VoiceAssistantPipeline from .audio_preprocess import SherpaOnnxDenoiserPreprocessor from .config import AppConfig from .conversation import ConversationContext from .events import PipelineEventBus from .llm import OpenAICompatibleLlmProvider from .models import AudioFrame, AudioSegment, Message, PlaybackResult, ReplyDelta, TransportHealth from .stt import SherpaOnnxSttProvider from .transport import SoundDeviceAudioTransport from .tts import MacSayTtsProvider from .vad import EnergyVadProvider, HybridVadProvider, PrimarySpeakerVadRecorder, SherpaOnnxVadProvider from .wakeword import SherpaOnnxKeywordWakeWordProvider FRAME_MS = 20 SAMPLE_RATE = 16000 FRAME_BYTES = int(SAMPLE_RATE * FRAME_MS / 1000) * 2 DEFAULT_WAKE_TEXT = "小杰小杰。" DEFAULT_QUESTIONS = [ "我叫阿明，请你记住我的名字。", "我叫什么名字？", ] STAGE_PAIRS = [ ("wake_wait", "wake_listening", "wake_detected"), ("acknowledgement", "ack_started", "question_prompt"), ("wait_for_speech", "capture_started", "speech_started"), ("capture", "speech_started", "speech_ended"), ("final_stt", "stt_started", "transcript_final"), ("llm_to_first_tts", "llm_started", "tts_started"), ("turn_total", "wake_listening", "standby_resumed"), ] def _now_iso() -> str: return datetime.now().astimezone().isoformat(timespec="milliseconds") def _epoch_ms() -> int: return int(time.time() * 1000) def _elapsed_ms(start: float, end: float | None = None) -> int: return int(round(((end if end is not None else time.perf_counter()) - start) * 1000)) class TimingRecorder: def __init__(self) -> None: self.started_at = _now_iso() self.started_epoch_ms = _epoch_ms() self._started_mono = time.perf_counter() self.finished_at: str | None = None self.finished_epoch_ms: int | None = None self.duration_ms: int | None = None self.phases: list[dict[str, object]] = [] self.events: list[dict[str, object]] = [] @contextmanager def phase(self, name: str) -> Iterable[None]: started_mono = time.perf_counter() record: dict[str, object] = { "name": name, "started_at": _now_iso(), "start_offset_ms": _elapsed_ms(self._started_mono, started_mono), } try: yield finally: finished_mono = time.perf_counter() record["finished_at"] = _now_iso() record["duration_ms"] = _elapsed_ms(started_mono, finished_mono) self.phases.append(record) def record_event(self, event_type: str, *, turn_id: int | None, message: str) -> None: self.events.append( { "type": event_type, "turn": turn_id, "at": _now_iso(), "offset_ms": _elapsed_ms(self._started_mono), "message": message, } ) def finish(self) -> None: self.finished_at = _now_iso() self.finished_epoch_ms = _epoch_ms() self.duration_ms = _elapsed_ms(self._started_mono) def to_json(self) -> dict[str, object]: if self.duration_ms is None: self.finish() return { "started_at": self.started_at, "started_epoch_ms": self.started_epoch_ms, "finished_at": self.finished_at, "finished_epoch_ms": self.finished_epoch_ms, "duration_ms": self.duration_ms, "phases": self.phases, "events": self.events, "stage_timings": _stage_timings_from_events(self.events), } @dataclass(slots=True) class RealLiveCheckReporter: statuses: list[dict[str, object]] = field(default_factory=list) partials: list[dict[str, object]] = field(default_factory=list) finals: list[dict[str, object]] = field(default_factory=list) errors: list[dict[str, object]] = field(default_factory=list) def status(self, state: str, message: str, *, turn_id: int | None = None) -> None: self.statuses.append({"turn": turn_id, "state": state, "message": message}) def transcript(self, text: str, *, final: bool, turn_id: int | None = None) -> None: target = self.finals if final else self.partials target.append({"turn": turn_id, "text": text}) def error(self, stage: str, code: str, message: str, *, turn_id: int | None = None) -> None: self.errors.append({"turn": turn_id, "stage": stage, "code": code, "message": message}) class RecordingLlmProvider: def __init__(self, delegate: OpenAICompatibleLlmProvider, timing: TimingRecorder) -> None: self.delegate = delegate self.timing = timing self.calls: list[list[Message]] = [] self.request_timings: list[dict[str, object]] = [] def stream_reply(self, messages: Sequence[Message]) -> Iterable[ReplyDelta]: self.calls.append(list(messages)) sent_mono = time.perf_counter() request_timing: dict[str, object] = { "call_index": len(self.calls), "sent_at": _now_iso(), "sent_offset_ms": _elapsed_ms(self.timing._started_mono, sent_mono), "message_count": len(messages), "last_user_preview": _last_user_preview(messages), } first_delta_mono: float | None = None try: for delta in self.delegate.stream_reply(messages): if first_delta_mono is None and delta.text_delta: first_delta_mono = time.perf_counter() request_timing["first_delta_at"] = _now_iso() request_timing["first_delta_ms"] = _elapsed_ms(sent_mono, first_delta_mono) yield delta finally: finished_mono = time.perf_counter() request_timing["finished_at"] = _now_iso() request_timing["duration_ms"] = _elapsed_ms(sent_mono, finished_mono) if first_delta_mono is None: request_timing["first_delta_ms"] = None self.request_timings.append(request_timing) class FixtureLiveAudioTransport: """Generated microphone input with optional real speaker output.""" def __init__( self, frames: list[AudioFrame], *, play_audio: bool = True, output_transport: SoundDeviceAudioTransport | None = None, ) -> None: self._frames: deque[AudioFrame] = deque(frames) self.play_audio = play_audio self.output_transport = output_transport or SoundDeviceAudioTransport() self.played_segments: list[AudioSegment] = [] self.flush_count = 0 self.started = False def start_input(self, device_id: str | None = None, sample_rate: int = 16000, channels: int = 1) -> None: self.started = True def read_frames(self, timeout_ms: int) -> list[AudioFrame]: if not self.started or not self._frames: return [] return [self._frames.popleft()] def play_pcm(self, segment: AudioSegment, interrupt: bool = False) -> PlaybackResult: if interrupt: self.played_segments.clear() self.played_segments.append(segment) if not self.play_audio: return PlaybackResult(True, segment.duration_ms) return self.output_transport.play_pcm(segment, interrupt=interrupt) def flush_input(self) -> int: self.flush_count += 1 return 0 def stop(self) -> None: self.started = False def health(self) -> TransportHealth: output = self.output_transport.health() return TransportHealth(True, output.output_available if self.play_audio else True, "fixture input") def run_real_live_check( *, config: AppConfig, turns: int = 2, voice: str = "Tingting", wake_text: str = DEFAULT_WAKE_TEXT, questions: list[str] | None = None, play_audio: bool = True, ) -> dict[str, object]: timing = TimingRecorder() if turns <= 0: raise ValueError("turns must be positive") with timing.phase("prepare_config"): selected_questions = _question_list(turns, questions) config = replace( config, speech_provider="local", noise_filter_enabled=True, realtime_transcript_enabled=True, continuous_dialog_enabled=False, barge_in_enabled=False, llm_stream=False, post_playback_drain_ms=0, ) config.require_llm_credentials() with timing.phase("generate_fixture_audio"): frames = _generated_live_frames(config, turns=turns, voice=voice, wake_text=wake_text, questions=selected_questions) with timing.phase("build_pipeline"): reporter = RealLiveCheckReporter() event_bus = PipelineEventBus() event_bus.subscribe(lambda event: timing.record_event(event.type, turn_id=event.turn_id, message=event.message)) stt = SherpaOnnxSttProvider(str(config.speech_models_dir)) tts = MacSayTtsProvider() llm = RecordingLlmProvider(OpenAICompatibleLlmProvider(config, timeout_s=60), timing) transport = FixtureLiveAudioTransport( frames, play_audio=play_audio, output_transport=SoundDeviceAudioTransport(output_device=config.audio_output_device), ) pipeline = VoiceAssistantPipeline( config=config, transport=transport, wakeword=SherpaOnnxKeywordWakeWordProvider( config.speech_models_dir, keyword=config.wake_word, keywords_file=config.wake_keywords_file, threshold=config.wake_kws_threshold, score=config.wake_kws_score, ), vad_recorder=PrimarySpeakerVadRecorder( HybridVadProvider( SherpaOnnxVadProvider(config.speech_models_dir, threshold=config.vad_threshold), EnergyVadProvider(), ), min_duration_ms=config.vad_min_duration_ms, end_silence_ms=config.vad_end_silence_ms, no_speech_timeout_ms=config.vad_no_speech_timeout_ms, max_recording_ms=config.vad_max_recording_ms, speaker_profile_ms=config.speaker_profile_ms, speaker_profile_min_ms=config.speaker_profile_min_ms, speaker_absent_ms=config.speaker_absent_ms, similarity_threshold=config.speaker_similarity_threshold, min_rms=config.speaker_min_rms, ), audio_preprocessor=SherpaOnnxDenoiserPreprocessor(config.speech_models_dir), stt=stt, realtime_stt=stt, llm=llm, tts=tts, ack_tts=tts, context=ConversationContext(max_messages=config.context_max_messages, max_chars=config.context_max_chars), reporter=reporter, event_bus=event_bus, ) with timing.phase("pipeline_run"): summary = pipeline.run(max_turns=turns) timing.finish() event_counts = Counter(event.type for event in event_bus.events) timing_data = timing.to_json() context_messages = [{"role": item.role, "content": item.content[:120]} for item in pipeline.context.messages()] checks = { "completed_turns": summary.completed_turns == turns, "no_failed_turns": summary.failed_turns == 0, "wake_per_turn": event_counts["wake_detected"] >= turns, "speech_per_turn": event_counts["speech_started"] >= turns, "final_stt_per_turn": event_counts["transcript_final"] >= turns, "llm_per_turn": len(llm.calls) >= turns, "tts_per_turn": event_counts["tts_started"] >= turns, "playback_per_turn": event_counts["playback_finished"] >= turns, "standby_per_turn": event_counts["standby_resumed"] >= turns, "temporary_context_in_second_llm": turns < 2 or _second_call_contains_first_turn(llm.calls), "real_output_playback": (not play_audio) or len(transport.played_segments) >= turns * 2, "timing_present": bool(timing_data.get("duration_ms") and timing_data.get("stage_timings")), "llm_request_timing_present": len(llm.request_timings) >= turns and all("sent_at" in item and "duration_ms" in item for item in llm.request_timings), } return { "success": all(checks.values()) and not reporter.errors, "turns": turns, "completed_turns": summary.completed_turns, "failed_turns": summary.failed_turns, "checks": checks, "final_transcripts": [str(item["text"]) for item in reporter.finals], "partials_preview": reporter.partials[:8], "event_counts": dict(sorted(event_counts.items())), "played_segments": len(transport.played_segments), "flush_count": transport.flush_count, "context_messages": context_messages, "errors": reporter.errors, "play_audio": play_audio, "voice": voice, "timing": timing_data, "llm_request_timings": llm.request_timings, } def _question_list(turns: int, questions: list[str] | None) -> list[str]: selected = list(questions or DEFAULT_QUESTIONS) while len(selected) < turns: selected.append(f"第{len(selected) + 1}轮完整流程测试。") return selected[:turns] def _generated_live_frames( config: AppConfig, *, turns: int, voice: str, wake_text: str, questions: list[str], ) -> list[AudioFrame]: with tempfile.TemporaryDirectory() as tmp_dir: tmp = Path(tmp_dir) wake_pcm = _synthesize_text_pcm(wake_text, tmp, "wake", voice=voice) question_pcm = [_synthesize_text_pcm(text, tmp, f"q{idx + 1}", voice=voice) for idx, text in enumerate(questions)] wake_frames = _trim_wake_frames_to_detection(config, _frames_from_pcm(wake_pcm, 0, 0, prefix="wake")) frames: list[AudioFrame] = [] frame_id = 0 timestamp_ms = 0 for turn_index, pcm in enumerate(question_pcm, start=1): copied_wake = [ AudioFrame( item.pcm, item.sample_rate, item.channels, timestamp_ms + idx * FRAME_MS, frame_id + idx, dict(item.metadata), ) for idx, item in enumerate(wake_frames) ] frames.extend(copied_wake) frame_id += len(copied_wake) timestamp_ms += len(copied_wake) * FRAME_MS gap = _silence(frame_id, timestamp_ms, 160, prefix=f"turn{turn_index}-gap") frames.extend(gap) frame_id += len(gap) timestamp_ms += len(gap) * FRAME_MS question_frames = _frames_from_pcm(pcm, frame_id, timestamp_ms, prefix=f"turn{turn_index}-question") frames.extend(question_frames) frame_id += len(question_frames) timestamp_ms += len(question_frames) * FRAME_MS tail = _silence(frame_id, timestamp_ms, 900, prefix=f"turn{turn_index}-tail") frames.extend(tail) frame_id += len(tail) timestamp_ms += len(tail) * FRAME_MS return frames def _synthesize_text_pcm(text: str, tmp: Path, name: str, *, voice: str) -> bytes: aiff_path = tmp / f"{name}.aiff" wav_path = tmp / f"{name}.wav" try: subprocess.run( ["say", "-v", voice, "-o", str(aiff_path), text], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) subprocess.run( ["afconvert", "-f", "WAVE", "-d", "LEI16@16000", "-c", "1", str(aiff_path), str(wav_path)], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) except (FileNotFoundError, subprocess.CalledProcessError) as exc: raise ValueError(f"failed to generate fixture speech with macOS say/afconvert: {exc}") from exc with wave.open(str(wav_path), "rb") as handle: if handle.getframerate() != SAMPLE_RATE or handle.getnchannels() != 1 or handle.getsampwidth() != 2: raise ValueError("generated speech must be 16 kHz mono int16 PCM") return handle.readframes(handle.getnframes()) def _frames_from_pcm(pcm: bytes, start_id: int, start_ms: int, *, prefix: str) -> list[AudioFrame]: frames: list[AudioFrame] = [] frame_id = start_id timestamp_ms = start_ms for offset in range(0, len(pcm), FRAME_BYTES): chunk = pcm[offset : offset + FRAME_BYTES] if len(chunk) < FRAME_BYTES: chunk += b"\x00" * (FRAME_BYTES - len(chunk)) frames.append( AudioFrame(chunk, SAMPLE_RATE, 1, timestamp_ms, frame_id, {"duration_ms": FRAME_MS, "fixture": prefix}) ) frame_id += 1 timestamp_ms += FRAME_MS return frames def _silence(start_id: int, start_ms: int, duration_ms: int, *, prefix: str) -> list[AudioFrame]: return [ AudioFrame( b"\x00" * FRAME_BYTES, SAMPLE_RATE, 1, start_ms + idx * FRAME_MS, start_id + idx, {"duration_ms": FRAME_MS, "fixture": prefix}, ) for idx in range(duration_ms // FRAME_MS) ] def _trim_wake_frames_to_detection(config: AppConfig, frames: list[AudioFrame]) -> list[AudioFrame]: probe_frames = list(frames) if probe_frames: last = probe_frames[-1] probe_frames.extend(_silence(last.frame_id + 1, last.timestamp_ms + FRAME_MS, 1400, prefix="wake-tail")) wake = SherpaOnnxKeywordWakeWordProvider( config.speech_models_dir, keyword=config.wake_word, keywords_file=config.wake_keywords_file, threshold=config.wake_kws_threshold, score=config.wake_kws_score, ) wake.load() wake.reset() for idx, frame in enumerate(probe_frames): if wake.detect(frame) is not None: return probe_frames[: idx + 1] raise ValueError("generated wake audio did not trigger local KWS") def _second_call_contains_first_turn(calls: list[list[Message]]) -> bool: if len(calls) < 2: return False second_contents = "\n".join(message.content for message in calls[1]) return "我叫阿明" in second_contents and any(message.role == "assistant" for message in calls[1]) def _last_user_preview(messages: Sequence[Message]) -> str: for message in reversed(messages): if message.role == "user": return message.content[:80] return "" def _stage_timings_from_events(events: list[dict[str, object]]) -> list[dict[str, object]]: timings: list[dict[str, object]] = [] turns = sorted({int(event["turn"]) for event in events if event.get("turn") is not None}) for turn_id in turns: turn_events = [event for event in events if event.get("turn") == turn_id] for stage_name, start_type, end_type in STAGE_PAIRS: timing = _pair_timing(turn_events, stage_name, start_type, end_type) if timing is not None: timings.append(timing) timings.extend(_repeated_pair_timings(turn_events, "tts_playback", "tts_started", "playback_finished")) return timings def _pair_timing( events: list[dict[str, object]], name: str, start_type: str, end_type: str, ) -> dict[str, object] | None: start = next((event for event in events if event.get("type") == start_type), None) if start is None: return None end = next( (event for event in events if event.get("type") == end_type and int(event["offset_ms"]) >= int(start["offset_ms"])), None, ) if end is None: return None return _timing_record(name, start, end) def _repeated_pair_timings( events: list[dict[str, object]], name: str, start_type: str, end_type: str, ) -> list[dict[str, object]]: timings: list[dict[str, object]] = [] pending: dict[str, object] | None = None segment_index = 1 for event in events: if event.get("type") == start_type: pending = event continue if pending is not None and event.get("type") == end_type: timing = _timing_record(name, pending, event) timing["segment_index"] = segment_index timings.append(timing) segment_index += 1 pending = None return timings def _timing_record(name: str, start: dict[str, object], end: dict[str, object]) -> dict[str, object]: return { "turn": start.get("turn"), "name": name, "started_at": start.get("at"), "finished_at": end.get("at"), "start_offset_ms": start.get("offset_ms"), "end_offset_ms": end.get("offset_ms"), "duration_ms": int(end["offset_ms"]) - int(start["offset_ms"]), "start_event": start.get("type"), "end_event": end.get("type"), }