[真实流程命令]：完成完整链路自测入口，包含真实Provider CLI、播放验收和文档测试
This commit is contained in:
mkbk
@@ -0,0 +1,339 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import tempfile
|
||||
import wave
|
||||
from collections import Counter, deque
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from pathlib import Path
|
||||
|
||||
from .assistant_pipeline import VoiceAssistantPipeline
|
||||
from .audio_preprocess import SherpaOnnxDenoiserPreprocessor
|
||||
from .config import AppConfig
|
||||
from .conversation import ConversationContext
|
||||
from .events import PipelineEventBus
|
||||
from .llm import OpenAICompatibleLlmProvider
|
||||
from .models import AudioFrame, AudioSegment, Message, PlaybackResult, ReplyDelta, TransportHealth
|
||||
from .stt import SherpaOnnxSttProvider
|
||||
from .transport import SoundDeviceAudioTransport
|
||||
from .tts import MacSayTtsProvider
|
||||
from .vad import EnergyVadProvider, HybridVadProvider, PrimarySpeakerVadRecorder, SherpaOnnxVadProvider
|
||||
from .wakeword import SherpaOnnxKeywordWakeWordProvider
|
||||
|
||||
FRAME_MS = 20
|
||||
SAMPLE_RATE = 16000
|
||||
FRAME_BYTES = int(SAMPLE_RATE * FRAME_MS / 1000) * 2
|
||||
DEFAULT_WAKE_TEXT = "小杰小杰。"
|
||||
DEFAULT_QUESTIONS = [
|
||||
"我叫阿明，请你记住我的名字。",
|
||||
"我叫什么名字？",
|
||||
]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RealLiveCheckReporter:
|
||||
statuses: list[dict[str, object]] = field(default_factory=list)
|
||||
partials: list[dict[str, object]] = field(default_factory=list)
|
||||
finals: list[dict[str, object]] = field(default_factory=list)
|
||||
errors: list[dict[str, object]] = field(default_factory=list)
|
||||
|
||||
def status(self, state: str, message: str, *, turn_id: int | None = None) -> None:
|
||||
self.statuses.append({"turn": turn_id, "state": state, "message": message})
|
||||
|
||||
def transcript(self, text: str, *, final: bool, turn_id: int | None = None) -> None:
|
||||
target = self.finals if final else self.partials
|
||||
target.append({"turn": turn_id, "text": text})
|
||||
|
||||
def error(self, stage: str, code: str, message: str, *, turn_id: int | None = None) -> None:
|
||||
self.errors.append({"turn": turn_id, "stage": stage, "code": code, "message": message})
|
||||
|
||||
|
||||
class RecordingLlmProvider:
|
||||
def __init__(self, delegate: OpenAICompatibleLlmProvider) -> None:
|
||||
self.delegate = delegate
|
||||
self.calls: list[list[Message]] = []
|
||||
|
||||
def stream_reply(self, messages: Sequence[Message]) -> Iterable[ReplyDelta]:
|
||||
self.calls.append(list(messages))
|
||||
yield from self.delegate.stream_reply(messages)
|
||||
|
||||
|
||||
class FixtureLiveAudioTransport:
|
||||
"""Generated microphone input with optional real speaker output."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
frames: list[AudioFrame],
|
||||
*,
|
||||
play_audio: bool = True,
|
||||
output_transport: SoundDeviceAudioTransport | None = None,
|
||||
) -> None:
|
||||
self._frames: deque[AudioFrame] = deque(frames)
|
||||
self.play_audio = play_audio
|
||||
self.output_transport = output_transport or SoundDeviceAudioTransport()
|
||||
self.played_segments: list[AudioSegment] = []
|
||||
self.flush_count = 0
|
||||
self.started = False
|
||||
|
||||
def start_input(self, device_id: str | None = None, sample_rate: int = 16000, channels: int = 1) -> None:
|
||||
self.started = True
|
||||
|
||||
def read_frames(self, timeout_ms: int) -> list[AudioFrame]:
|
||||
if not self.started or not self._frames:
|
||||
return []
|
||||
return [self._frames.popleft()]
|
||||
|
||||
def play_pcm(self, segment: AudioSegment, interrupt: bool = False) -> PlaybackResult:
|
||||
if interrupt:
|
||||
self.played_segments.clear()
|
||||
self.played_segments.append(segment)
|
||||
if not self.play_audio:
|
||||
return PlaybackResult(True, segment.duration_ms)
|
||||
return self.output_transport.play_pcm(segment, interrupt=interrupt)
|
||||
|
||||
def flush_input(self) -> int:
|
||||
self.flush_count += 1
|
||||
return 0
|
||||
|
||||
def stop(self) -> None:
|
||||
self.started = False
|
||||
|
||||
def health(self) -> TransportHealth:
|
||||
output = self.output_transport.health()
|
||||
return TransportHealth(True, output.output_available if self.play_audio else True, "fixture input")
|
||||
|
||||
|
||||
def run_real_live_check(
|
||||
*,
|
||||
config: AppConfig,
|
||||
turns: int = 2,
|
||||
voice: str = "Tingting",
|
||||
wake_text: str = DEFAULT_WAKE_TEXT,
|
||||
questions: list[str] | None = None,
|
||||
play_audio: bool = True,
|
||||
) -> dict[str, object]:
|
||||
if turns <= 0:
|
||||
raise ValueError("turns must be positive")
|
||||
selected_questions = _question_list(turns, questions)
|
||||
config = replace(
|
||||
config,
|
||||
speech_provider="local",
|
||||
noise_filter_enabled=True,
|
||||
realtime_transcript_enabled=True,
|
||||
llm_stream=False,
|
||||
post_playback_drain_ms=0,
|
||||
)
|
||||
config.require_llm_credentials()
|
||||
frames = _generated_live_frames(config, turns=turns, voice=voice, wake_text=wake_text, questions=selected_questions)
|
||||
reporter = RealLiveCheckReporter()
|
||||
event_bus = PipelineEventBus()
|
||||
stt = SherpaOnnxSttProvider(str(config.speech_models_dir))
|
||||
tts = MacSayTtsProvider()
|
||||
llm = RecordingLlmProvider(OpenAICompatibleLlmProvider(config, timeout_s=60))
|
||||
transport = FixtureLiveAudioTransport(
|
||||
frames,
|
||||
play_audio=play_audio,
|
||||
output_transport=SoundDeviceAudioTransport(output_device=config.audio_output_device),
|
||||
)
|
||||
pipeline = VoiceAssistantPipeline(
|
||||
config=config,
|
||||
transport=transport,
|
||||
wakeword=SherpaOnnxKeywordWakeWordProvider(
|
||||
config.speech_models_dir,
|
||||
keyword=config.wake_word,
|
||||
keywords_file=config.wake_keywords_file,
|
||||
threshold=config.wake_kws_threshold,
|
||||
score=config.wake_kws_score,
|
||||
),
|
||||
vad_recorder=PrimarySpeakerVadRecorder(
|
||||
HybridVadProvider(
|
||||
SherpaOnnxVadProvider(config.speech_models_dir, threshold=config.vad_threshold),
|
||||
EnergyVadProvider(),
|
||||
),
|
||||
min_duration_ms=config.vad_min_duration_ms,
|
||||
end_silence_ms=config.vad_end_silence_ms,
|
||||
no_speech_timeout_ms=config.vad_no_speech_timeout_ms,
|
||||
max_recording_ms=config.vad_max_recording_ms,
|
||||
speaker_profile_ms=config.speaker_profile_ms,
|
||||
speaker_profile_min_ms=config.speaker_profile_min_ms,
|
||||
speaker_absent_ms=config.speaker_absent_ms,
|
||||
similarity_threshold=config.speaker_similarity_threshold,
|
||||
min_rms=config.speaker_min_rms,
|
||||
),
|
||||
audio_preprocessor=SherpaOnnxDenoiserPreprocessor(config.speech_models_dir),
|
||||
stt=stt,
|
||||
realtime_stt=stt,
|
||||
llm=llm,
|
||||
tts=tts,
|
||||
ack_tts=tts,
|
||||
context=ConversationContext(max_messages=config.context_max_messages, max_chars=config.context_max_chars),
|
||||
reporter=reporter,
|
||||
event_bus=event_bus,
|
||||
)
|
||||
summary = pipeline.run(max_turns=turns)
|
||||
event_counts = Counter(event.type for event in event_bus.events)
|
||||
context_messages = [{"role": item.role, "content": item.content[:120]} for item in pipeline.context.messages()]
|
||||
checks = {
|
||||
"completed_turns": summary.completed_turns == turns,
|
||||
"no_failed_turns": summary.failed_turns == 0,
|
||||
"wake_per_turn": event_counts["wake_detected"] >= turns,
|
||||
"speech_per_turn": event_counts["speech_started"] >= turns,
|
||||
"final_stt_per_turn": event_counts["transcript_final"] >= turns,
|
||||
"llm_per_turn": len(llm.calls) >= turns,
|
||||
"tts_per_turn": event_counts["tts_started"] >= turns,
|
||||
"playback_per_turn": event_counts["playback_finished"] >= turns,
|
||||
"standby_per_turn": event_counts["standby_resumed"] >= turns,
|
||||
"temporary_context_in_second_llm": _second_call_contains_first_turn(llm.calls),
|
||||
"real_output_playback": (not play_audio) or len(transport.played_segments) >= turns * 2,
|
||||
}
|
||||
return {
|
||||
"success": all(checks.values()) and not reporter.errors,
|
||||
"turns": turns,
|
||||
"completed_turns": summary.completed_turns,
|
||||
"failed_turns": summary.failed_turns,
|
||||
"checks": checks,
|
||||
"final_transcripts": [str(item["text"]) for item in reporter.finals],
|
||||
"partials_preview": reporter.partials[:8],
|
||||
"event_counts": dict(sorted(event_counts.items())),
|
||||
"played_segments": len(transport.played_segments),
|
||||
"flush_count": transport.flush_count,
|
||||
"context_messages": context_messages,
|
||||
"errors": reporter.errors,
|
||||
"play_audio": play_audio,
|
||||
"voice": voice,
|
||||
}
|
||||
|
||||
|
||||
def _question_list(turns: int, questions: list[str] | None) -> list[str]:
|
||||
selected = list(questions or DEFAULT_QUESTIONS)
|
||||
while len(selected) < turns:
|
||||
selected.append(f"第{len(selected) + 1}轮完整流程测试。")
|
||||
return selected[:turns]
|
||||
|
||||
|
||||
def _generated_live_frames(
|
||||
config: AppConfig,
|
||||
*,
|
||||
turns: int,
|
||||
voice: str,
|
||||
wake_text: str,
|
||||
questions: list[str],
|
||||
) -> list[AudioFrame]:
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp = Path(tmp_dir)
|
||||
wake_pcm = _synthesize_text_pcm(wake_text, tmp, "wake", voice=voice)
|
||||
question_pcm = [_synthesize_text_pcm(text, tmp, f"q{idx + 1}", voice=voice) for idx, text in enumerate(questions)]
|
||||
wake_frames = _trim_wake_frames_to_detection(config, _frames_from_pcm(wake_pcm, 0, 0, prefix="wake"))
|
||||
frames: list[AudioFrame] = []
|
||||
frame_id = 0
|
||||
timestamp_ms = 0
|
||||
for turn_index, pcm in enumerate(question_pcm, start=1):
|
||||
copied_wake = [
|
||||
AudioFrame(
|
||||
item.pcm,
|
||||
item.sample_rate,
|
||||
item.channels,
|
||||
timestamp_ms + idx * FRAME_MS,
|
||||
frame_id + idx,
|
||||
dict(item.metadata),
|
||||
)
|
||||
for idx, item in enumerate(wake_frames)
|
||||
]
|
||||
frames.extend(copied_wake)
|
||||
frame_id += len(copied_wake)
|
||||
timestamp_ms += len(copied_wake) * FRAME_MS
|
||||
gap = _silence(frame_id, timestamp_ms, 160, prefix=f"turn{turn_index}-gap")
|
||||
frames.extend(gap)
|
||||
frame_id += len(gap)
|
||||
timestamp_ms += len(gap) * FRAME_MS
|
||||
question_frames = _frames_from_pcm(pcm, frame_id, timestamp_ms, prefix=f"turn{turn_index}-question")
|
||||
frames.extend(question_frames)
|
||||
frame_id += len(question_frames)
|
||||
timestamp_ms += len(question_frames) * FRAME_MS
|
||||
tail = _silence(frame_id, timestamp_ms, 900, prefix=f"turn{turn_index}-tail")
|
||||
frames.extend(tail)
|
||||
frame_id += len(tail)
|
||||
timestamp_ms += len(tail) * FRAME_MS
|
||||
return frames
|
||||
|
||||
|
||||
def _synthesize_text_pcm(text: str, tmp: Path, name: str, *, voice: str) -> bytes:
|
||||
aiff_path = tmp / f"{name}.aiff"
|
||||
wav_path = tmp / f"{name}.wav"
|
||||
try:
|
||||
subprocess.run(
|
||||
["say", "-v", voice, "-o", str(aiff_path), text],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
subprocess.run(
|
||||
["afconvert", "-f", "WAVE", "-d", "LEI16@16000", "-c", "1", str(aiff_path), str(wav_path)],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
except (FileNotFoundError, subprocess.CalledProcessError) as exc:
|
||||
raise ValueError(f"failed to generate fixture speech with macOS say/afconvert: {exc}") from exc
|
||||
with wave.open(str(wav_path), "rb") as handle:
|
||||
if handle.getframerate() != SAMPLE_RATE or handle.getnchannels() != 1 or handle.getsampwidth() != 2:
|
||||
raise ValueError("generated speech must be 16 kHz mono int16 PCM")
|
||||
return handle.readframes(handle.getnframes())
|
||||
|
||||
|
||||
def _frames_from_pcm(pcm: bytes, start_id: int, start_ms: int, *, prefix: str) -> list[AudioFrame]:
|
||||
frames: list[AudioFrame] = []
|
||||
frame_id = start_id
|
||||
timestamp_ms = start_ms
|
||||
for offset in range(0, len(pcm), FRAME_BYTES):
|
||||
chunk = pcm[offset : offset + FRAME_BYTES]
|
||||
if len(chunk) < FRAME_BYTES:
|
||||
chunk += b"\x00" * (FRAME_BYTES - len(chunk))
|
||||
frames.append(
|
||||
AudioFrame(chunk, SAMPLE_RATE, 1, timestamp_ms, frame_id, {"duration_ms": FRAME_MS, "fixture": prefix})
|
||||
)
|
||||
frame_id += 1
|
||||
timestamp_ms += FRAME_MS
|
||||
return frames
|
||||
|
||||
|
||||
def _silence(start_id: int, start_ms: int, duration_ms: int, *, prefix: str) -> list[AudioFrame]:
|
||||
return [
|
||||
AudioFrame(
|
||||
b"\x00" * FRAME_BYTES,
|
||||
SAMPLE_RATE,
|
||||
1,
|
||||
start_ms + idx * FRAME_MS,
|
||||
start_id + idx,
|
||||
{"duration_ms": FRAME_MS, "fixture": prefix},
|
||||
)
|
||||
for idx in range(duration_ms // FRAME_MS)
|
||||
]
|
||||
|
||||
|
||||
def _trim_wake_frames_to_detection(config: AppConfig, frames: list[AudioFrame]) -> list[AudioFrame]:
|
||||
probe_frames = list(frames)
|
||||
if probe_frames:
|
||||
last = probe_frames[-1]
|
||||
probe_frames.extend(_silence(last.frame_id + 1, last.timestamp_ms + FRAME_MS, 1400, prefix="wake-tail"))
|
||||
wake = SherpaOnnxKeywordWakeWordProvider(
|
||||
config.speech_models_dir,
|
||||
keyword=config.wake_word,
|
||||
keywords_file=config.wake_keywords_file,
|
||||
threshold=config.wake_kws_threshold,
|
||||
score=config.wake_kws_score,
|
||||
)
|
||||
wake.load()
|
||||
wake.reset()
|
||||
for idx, frame in enumerate(probe_frames):
|
||||
if wake.detect(frame) is not None:
|
||||
return probe_frames[: idx + 1]
|
||||
raise ValueError("generated wake audio did not trigger local KWS")
|
||||
|
||||
|
||||
def _second_call_contains_first_turn(calls: list[list[Message]]) -> bool:
|
||||
if len(calls) < 2:
|
||||
return False
|
||||
second_contents = "\n".join(message.content for message in calls[1])
|
||||
return "我叫阿明" in second_contents and any(message.role == "assistant" for message in calls[1])
|
||||
Reference in New Issue
Block a user