mkbk
535 lines
21 KiB
Python
535 lines
21 KiB
Python
from __future__ import annotations
|
|
|
|
import time
|
|
from contextlib import contextmanager
|
|
from datetime import datetime
|
|
import subprocess
|
|
import tempfile
|
|
import wave
|
|
from collections import Counter, deque
|
|
from collections.abc import Iterable, Sequence
|
|
from dataclasses import dataclass, field, replace
|
|
from pathlib import Path
|
|
|
|
from .assistant_pipeline import VoiceAssistantPipeline
|
|
from .audio_preprocess import SherpaOnnxDenoiserPreprocessor
|
|
from .config import AppConfig
|
|
from .conversation import ConversationContext
|
|
from .events import PipelineEventBus
|
|
from .llm import OpenAICompatibleLlmProvider
|
|
from .models import AudioFrame, AudioSegment, Message, PlaybackResult, ReplyDelta, TransportHealth
|
|
from .stt import SherpaOnnxSttProvider
|
|
from .transport import SoundDeviceAudioTransport
|
|
from .tts import MacSayTtsProvider
|
|
from .vad import EnergyVadProvider, HybridVadProvider, PrimarySpeakerVadRecorder, SherpaOnnxVadProvider
|
|
from .wakeword import SherpaOnnxKeywordWakeWordProvider
|
|
|
|
FRAME_MS = 20
|
|
SAMPLE_RATE = 16000
|
|
FRAME_BYTES = int(SAMPLE_RATE * FRAME_MS / 1000) * 2
|
|
DEFAULT_WAKE_TEXT = "小杰小杰。"
|
|
DEFAULT_QUESTIONS = [
|
|
"我叫阿明，请你记住我的名字。",
|
|
"我叫什么名字？",
|
|
]
|
|
STAGE_PAIRS = [
|
|
("wake_wait", "wake_listening", "wake_detected"),
|
|
("acknowledgement", "ack_started", "question_prompt"),
|
|
("wait_for_speech", "capture_started", "speech_started"),
|
|
("capture", "speech_started", "speech_ended"),
|
|
("final_stt", "stt_started", "transcript_final"),
|
|
("llm_to_first_tts", "llm_started", "tts_started"),
|
|
("turn_total", "wake_listening", "standby_resumed"),
|
|
]
|
|
|
|
|
|
def _now_iso() -> str:
|
|
return datetime.now().astimezone().isoformat(timespec="milliseconds")
|
|
|
|
|
|
def _epoch_ms() -> int:
|
|
return int(time.time() * 1000)
|
|
|
|
|
|
def _elapsed_ms(start: float, end: float | None = None) -> int:
|
|
return int(round(((end if end is not None else time.perf_counter()) - start) * 1000))
|
|
|
|
|
|
class TimingRecorder:
|
|
def __init__(self) -> None:
|
|
self.started_at = _now_iso()
|
|
self.started_epoch_ms = _epoch_ms()
|
|
self._started_mono = time.perf_counter()
|
|
self.finished_at: str | None = None
|
|
self.finished_epoch_ms: int | None = None
|
|
self.duration_ms: int | None = None
|
|
self.phases: list[dict[str, object]] = []
|
|
self.events: list[dict[str, object]] = []
|
|
|
|
@contextmanager
|
|
def phase(self, name: str) -> Iterable[None]:
|
|
started_mono = time.perf_counter()
|
|
record: dict[str, object] = {
|
|
"name": name,
|
|
"started_at": _now_iso(),
|
|
"start_offset_ms": _elapsed_ms(self._started_mono, started_mono),
|
|
}
|
|
try:
|
|
yield
|
|
finally:
|
|
finished_mono = time.perf_counter()
|
|
record["finished_at"] = _now_iso()
|
|
record["duration_ms"] = _elapsed_ms(started_mono, finished_mono)
|
|
self.phases.append(record)
|
|
|
|
def record_event(self, event_type: str, *, turn_id: int | None, message: str) -> None:
|
|
self.events.append(
|
|
{
|
|
"type": event_type,
|
|
"turn": turn_id,
|
|
"at": _now_iso(),
|
|
"offset_ms": _elapsed_ms(self._started_mono),
|
|
"message": message,
|
|
}
|
|
)
|
|
|
|
def finish(self) -> None:
|
|
self.finished_at = _now_iso()
|
|
self.finished_epoch_ms = _epoch_ms()
|
|
self.duration_ms = _elapsed_ms(self._started_mono)
|
|
|
|
def to_json(self) -> dict[str, object]:
|
|
if self.duration_ms is None:
|
|
self.finish()
|
|
return {
|
|
"started_at": self.started_at,
|
|
"started_epoch_ms": self.started_epoch_ms,
|
|
"finished_at": self.finished_at,
|
|
"finished_epoch_ms": self.finished_epoch_ms,
|
|
"duration_ms": self.duration_ms,
|
|
"phases": self.phases,
|
|
"events": self.events,
|
|
"stage_timings": _stage_timings_from_events(self.events),
|
|
}
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class RealLiveCheckReporter:
|
|
statuses: list[dict[str, object]] = field(default_factory=list)
|
|
partials: list[dict[str, object]] = field(default_factory=list)
|
|
finals: list[dict[str, object]] = field(default_factory=list)
|
|
errors: list[dict[str, object]] = field(default_factory=list)
|
|
|
|
def status(self, state: str, message: str, *, turn_id: int | None = None) -> None:
|
|
self.statuses.append({"turn": turn_id, "state": state, "message": message})
|
|
|
|
def transcript(self, text: str, *, final: bool, turn_id: int | None = None) -> None:
|
|
target = self.finals if final else self.partials
|
|
target.append({"turn": turn_id, "text": text})
|
|
|
|
def error(self, stage: str, code: str, message: str, *, turn_id: int | None = None) -> None:
|
|
self.errors.append({"turn": turn_id, "stage": stage, "code": code, "message": message})
|
|
|
|
|
|
class RecordingLlmProvider:
|
|
def __init__(self, delegate: OpenAICompatibleLlmProvider, timing: TimingRecorder) -> None:
|
|
self.delegate = delegate
|
|
self.timing = timing
|
|
self.calls: list[list[Message]] = []
|
|
self.request_timings: list[dict[str, object]] = []
|
|
|
|
def stream_reply(self, messages: Sequence[Message]) -> Iterable[ReplyDelta]:
|
|
self.calls.append(list(messages))
|
|
sent_mono = time.perf_counter()
|
|
request_timing: dict[str, object] = {
|
|
"call_index": len(self.calls),
|
|
"sent_at": _now_iso(),
|
|
"sent_offset_ms": _elapsed_ms(self.timing._started_mono, sent_mono),
|
|
"message_count": len(messages),
|
|
"last_user_preview": _last_user_preview(messages),
|
|
}
|
|
first_delta_mono: float | None = None
|
|
try:
|
|
for delta in self.delegate.stream_reply(messages):
|
|
if first_delta_mono is None and delta.text_delta:
|
|
first_delta_mono = time.perf_counter()
|
|
request_timing["first_delta_at"] = _now_iso()
|
|
request_timing["first_delta_ms"] = _elapsed_ms(sent_mono, first_delta_mono)
|
|
yield delta
|
|
finally:
|
|
finished_mono = time.perf_counter()
|
|
request_timing["finished_at"] = _now_iso()
|
|
request_timing["duration_ms"] = _elapsed_ms(sent_mono, finished_mono)
|
|
if first_delta_mono is None:
|
|
request_timing["first_delta_ms"] = None
|
|
self.request_timings.append(request_timing)
|
|
|
|
|
|
class FixtureLiveAudioTransport:
|
|
"""Generated microphone input with optional real speaker output."""
|
|
|
|
def __init__(
|
|
self,
|
|
frames: list[AudioFrame],
|
|
*,
|
|
play_audio: bool = True,
|
|
output_transport: SoundDeviceAudioTransport | None = None,
|
|
) -> None:
|
|
self._frames: deque[AudioFrame] = deque(frames)
|
|
self.play_audio = play_audio
|
|
self.output_transport = output_transport or SoundDeviceAudioTransport()
|
|
self.played_segments: list[AudioSegment] = []
|
|
self.flush_count = 0
|
|
self.started = False
|
|
|
|
def start_input(self, device_id: str | None = None, sample_rate: int = 16000, channels: int = 1) -> None:
|
|
self.started = True
|
|
|
|
def read_frames(self, timeout_ms: int) -> list[AudioFrame]:
|
|
if not self.started or not self._frames:
|
|
return []
|
|
return [self._frames.popleft()]
|
|
|
|
def play_pcm(self, segment: AudioSegment, interrupt: bool = False) -> PlaybackResult:
|
|
if interrupt:
|
|
self.played_segments.clear()
|
|
self.played_segments.append(segment)
|
|
if not self.play_audio:
|
|
return PlaybackResult(True, segment.duration_ms)
|
|
return self.output_transport.play_pcm(segment, interrupt=interrupt)
|
|
|
|
def flush_input(self) -> int:
|
|
self.flush_count += 1
|
|
return 0
|
|
|
|
def stop(self) -> None:
|
|
self.started = False
|
|
|
|
def health(self) -> TransportHealth:
|
|
output = self.output_transport.health()
|
|
return TransportHealth(True, output.output_available if self.play_audio else True, "fixture input")
|
|
|
|
|
|
def run_real_live_check(
|
|
*,
|
|
config: AppConfig,
|
|
turns: int = 2,
|
|
voice: str = "Tingting",
|
|
wake_text: str = DEFAULT_WAKE_TEXT,
|
|
questions: list[str] | None = None,
|
|
play_audio: bool = True,
|
|
) -> dict[str, object]:
|
|
timing = TimingRecorder()
|
|
if turns <= 0:
|
|
raise ValueError("turns must be positive")
|
|
with timing.phase("prepare_config"):
|
|
selected_questions = _question_list(turns, questions)
|
|
config = replace(
|
|
config,
|
|
speech_provider="local",
|
|
noise_filter_enabled=True,
|
|
realtime_transcript_enabled=True,
|
|
continuous_dialog_enabled=False,
|
|
barge_in_enabled=False,
|
|
llm_stream=False,
|
|
post_playback_drain_ms=0,
|
|
)
|
|
config.require_llm_credentials()
|
|
with timing.phase("generate_fixture_audio"):
|
|
frames = _generated_live_frames(config, turns=turns, voice=voice, wake_text=wake_text, questions=selected_questions)
|
|
with timing.phase("build_pipeline"):
|
|
reporter = RealLiveCheckReporter()
|
|
event_bus = PipelineEventBus()
|
|
event_bus.subscribe(lambda event: timing.record_event(event.type, turn_id=event.turn_id, message=event.message))
|
|
stt = SherpaOnnxSttProvider(str(config.speech_models_dir))
|
|
tts = MacSayTtsProvider()
|
|
llm = RecordingLlmProvider(OpenAICompatibleLlmProvider(config, timeout_s=60), timing)
|
|
transport = FixtureLiveAudioTransport(
|
|
frames,
|
|
play_audio=play_audio,
|
|
output_transport=SoundDeviceAudioTransport(output_device=config.audio_output_device),
|
|
)
|
|
pipeline = VoiceAssistantPipeline(
|
|
config=config,
|
|
transport=transport,
|
|
wakeword=SherpaOnnxKeywordWakeWordProvider(
|
|
config.speech_models_dir,
|
|
keyword=config.wake_word,
|
|
keywords_file=config.wake_keywords_file,
|
|
threshold=config.wake_kws_threshold,
|
|
score=config.wake_kws_score,
|
|
),
|
|
vad_recorder=PrimarySpeakerVadRecorder(
|
|
HybridVadProvider(
|
|
SherpaOnnxVadProvider(config.speech_models_dir, threshold=config.vad_threshold),
|
|
EnergyVadProvider(),
|
|
),
|
|
min_duration_ms=config.vad_min_duration_ms,
|
|
end_silence_ms=config.vad_end_silence_ms,
|
|
no_speech_timeout_ms=config.vad_no_speech_timeout_ms,
|
|
max_recording_ms=config.vad_max_recording_ms,
|
|
speaker_profile_ms=config.speaker_profile_ms,
|
|
speaker_profile_min_ms=config.speaker_profile_min_ms,
|
|
speaker_absent_ms=config.speaker_absent_ms,
|
|
similarity_threshold=config.speaker_similarity_threshold,
|
|
min_rms=config.speaker_min_rms,
|
|
),
|
|
audio_preprocessor=SherpaOnnxDenoiserPreprocessor(config.speech_models_dir),
|
|
stt=stt,
|
|
realtime_stt=stt,
|
|
llm=llm,
|
|
tts=tts,
|
|
ack_tts=tts,
|
|
context=ConversationContext(max_messages=config.context_max_messages, max_chars=config.context_max_chars),
|
|
reporter=reporter,
|
|
event_bus=event_bus,
|
|
)
|
|
with timing.phase("pipeline_run"):
|
|
summary = pipeline.run(max_turns=turns)
|
|
timing.finish()
|
|
event_counts = Counter(event.type for event in event_bus.events)
|
|
timing_data = timing.to_json()
|
|
context_messages = [{"role": item.role, "content": item.content[:120]} for item in pipeline.context.messages()]
|
|
checks = {
|
|
"completed_turns": summary.completed_turns == turns,
|
|
"no_failed_turns": summary.failed_turns == 0,
|
|
"wake_per_turn": event_counts["wake_detected"] >= turns,
|
|
"speech_per_turn": event_counts["speech_started"] >= turns,
|
|
"final_stt_per_turn": event_counts["transcript_final"] >= turns,
|
|
"llm_per_turn": len(llm.calls) >= turns,
|
|
"tts_per_turn": event_counts["tts_started"] >= turns,
|
|
"playback_per_turn": event_counts["playback_finished"] >= turns,
|
|
"standby_per_turn": event_counts["standby_resumed"] >= turns,
|
|
"temporary_context_in_second_llm": turns < 2 or _second_call_contains_first_turn(llm.calls),
|
|
"real_output_playback": (not play_audio) or len(transport.played_segments) >= turns * 2,
|
|
"timing_present": bool(timing_data.get("duration_ms") and timing_data.get("stage_timings")),
|
|
"llm_request_timing_present": len(llm.request_timings) >= turns
|
|
and all("sent_at" in item and "duration_ms" in item for item in llm.request_timings),
|
|
}
|
|
return {
|
|
"success": all(checks.values()) and not reporter.errors,
|
|
"turns": turns,
|
|
"completed_turns": summary.completed_turns,
|
|
"failed_turns": summary.failed_turns,
|
|
"checks": checks,
|
|
"final_transcripts": [str(item["text"]) for item in reporter.finals],
|
|
"partials_preview": reporter.partials[:8],
|
|
"event_counts": dict(sorted(event_counts.items())),
|
|
"played_segments": len(transport.played_segments),
|
|
"flush_count": transport.flush_count,
|
|
"context_messages": context_messages,
|
|
"errors": reporter.errors,
|
|
"play_audio": play_audio,
|
|
"voice": voice,
|
|
"timing": timing_data,
|
|
"llm_request_timings": llm.request_timings,
|
|
}
|
|
|
|
|
|
def _question_list(turns: int, questions: list[str] | None) -> list[str]:
|
|
selected = list(questions or DEFAULT_QUESTIONS)
|
|
while len(selected) < turns:
|
|
selected.append(f"第{len(selected) + 1}轮完整流程测试。")
|
|
return selected[:turns]
|
|
|
|
|
|
def _generated_live_frames(
|
|
config: AppConfig,
|
|
*,
|
|
turns: int,
|
|
voice: str,
|
|
wake_text: str,
|
|
questions: list[str],
|
|
) -> list[AudioFrame]:
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
tmp = Path(tmp_dir)
|
|
wake_pcm = _synthesize_text_pcm(wake_text, tmp, "wake", voice=voice)
|
|
question_pcm = [_synthesize_text_pcm(text, tmp, f"q{idx + 1}", voice=voice) for idx, text in enumerate(questions)]
|
|
wake_frames = _trim_wake_frames_to_detection(config, _frames_from_pcm(wake_pcm, 0, 0, prefix="wake"))
|
|
frames: list[AudioFrame] = []
|
|
frame_id = 0
|
|
timestamp_ms = 0
|
|
for turn_index, pcm in enumerate(question_pcm, start=1):
|
|
copied_wake = [
|
|
AudioFrame(
|
|
item.pcm,
|
|
item.sample_rate,
|
|
item.channels,
|
|
timestamp_ms + idx * FRAME_MS,
|
|
frame_id + idx,
|
|
dict(item.metadata),
|
|
)
|
|
for idx, item in enumerate(wake_frames)
|
|
]
|
|
frames.extend(copied_wake)
|
|
frame_id += len(copied_wake)
|
|
timestamp_ms += len(copied_wake) * FRAME_MS
|
|
gap = _silence(frame_id, timestamp_ms, 160, prefix=f"turn{turn_index}-gap")
|
|
frames.extend(gap)
|
|
frame_id += len(gap)
|
|
timestamp_ms += len(gap) * FRAME_MS
|
|
question_frames = _frames_from_pcm(pcm, frame_id, timestamp_ms, prefix=f"turn{turn_index}-question")
|
|
frames.extend(question_frames)
|
|
frame_id += len(question_frames)
|
|
timestamp_ms += len(question_frames) * FRAME_MS
|
|
tail = _silence(frame_id, timestamp_ms, 900, prefix=f"turn{turn_index}-tail")
|
|
frames.extend(tail)
|
|
frame_id += len(tail)
|
|
timestamp_ms += len(tail) * FRAME_MS
|
|
return frames
|
|
|
|
|
|
def _synthesize_text_pcm(text: str, tmp: Path, name: str, *, voice: str) -> bytes:
|
|
aiff_path = tmp / f"{name}.aiff"
|
|
wav_path = tmp / f"{name}.wav"
|
|
try:
|
|
subprocess.run(
|
|
["say", "-v", voice, "-o", str(aiff_path), text],
|
|
check=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
subprocess.run(
|
|
["afconvert", "-f", "WAVE", "-d", "LEI16@16000", "-c", "1", str(aiff_path), str(wav_path)],
|
|
check=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
except (FileNotFoundError, subprocess.CalledProcessError) as exc:
|
|
raise ValueError(f"failed to generate fixture speech with macOS say/afconvert: {exc}") from exc
|
|
with wave.open(str(wav_path), "rb") as handle:
|
|
if handle.getframerate() != SAMPLE_RATE or handle.getnchannels() != 1 or handle.getsampwidth() != 2:
|
|
raise ValueError("generated speech must be 16 kHz mono int16 PCM")
|
|
return handle.readframes(handle.getnframes())
|
|
|
|
|
|
def _frames_from_pcm(pcm: bytes, start_id: int, start_ms: int, *, prefix: str) -> list[AudioFrame]:
|
|
frames: list[AudioFrame] = []
|
|
frame_id = start_id
|
|
timestamp_ms = start_ms
|
|
for offset in range(0, len(pcm), FRAME_BYTES):
|
|
chunk = pcm[offset : offset + FRAME_BYTES]
|
|
if len(chunk) < FRAME_BYTES:
|
|
chunk += b"\x00" * (FRAME_BYTES - len(chunk))
|
|
frames.append(
|
|
AudioFrame(chunk, SAMPLE_RATE, 1, timestamp_ms, frame_id, {"duration_ms": FRAME_MS, "fixture": prefix})
|
|
)
|
|
frame_id += 1
|
|
timestamp_ms += FRAME_MS
|
|
return frames
|
|
|
|
|
|
def _silence(start_id: int, start_ms: int, duration_ms: int, *, prefix: str) -> list[AudioFrame]:
|
|
return [
|
|
AudioFrame(
|
|
b"\x00" * FRAME_BYTES,
|
|
SAMPLE_RATE,
|
|
1,
|
|
start_ms + idx * FRAME_MS,
|
|
start_id + idx,
|
|
{"duration_ms": FRAME_MS, "fixture": prefix},
|
|
)
|
|
for idx in range(duration_ms // FRAME_MS)
|
|
]
|
|
|
|
|
|
def _trim_wake_frames_to_detection(config: AppConfig, frames: list[AudioFrame]) -> list[AudioFrame]:
|
|
probe_frames = list(frames)
|
|
if probe_frames:
|
|
last = probe_frames[-1]
|
|
probe_frames.extend(_silence(last.frame_id + 1, last.timestamp_ms + FRAME_MS, 1400, prefix="wake-tail"))
|
|
wake = SherpaOnnxKeywordWakeWordProvider(
|
|
config.speech_models_dir,
|
|
keyword=config.wake_word,
|
|
keywords_file=config.wake_keywords_file,
|
|
threshold=config.wake_kws_threshold,
|
|
score=config.wake_kws_score,
|
|
)
|
|
wake.load()
|
|
wake.reset()
|
|
for idx, frame in enumerate(probe_frames):
|
|
if wake.detect(frame) is not None:
|
|
return probe_frames[: idx + 1]
|
|
raise ValueError("generated wake audio did not trigger local KWS")
|
|
|
|
|
|
def _second_call_contains_first_turn(calls: list[list[Message]]) -> bool:
|
|
if len(calls) < 2:
|
|
return False
|
|
second_contents = "\n".join(message.content for message in calls[1])
|
|
return "我叫阿明" in second_contents and any(message.role == "assistant" for message in calls[1])
|
|
|
|
|
|
def _last_user_preview(messages: Sequence[Message]) -> str:
|
|
for message in reversed(messages):
|
|
if message.role == "user":
|
|
return message.content[:80]
|
|
return ""
|
|
|
|
|
|
def _stage_timings_from_events(events: list[dict[str, object]]) -> list[dict[str, object]]:
|
|
timings: list[dict[str, object]] = []
|
|
turns = sorted({int(event["turn"]) for event in events if event.get("turn") is not None})
|
|
for turn_id in turns:
|
|
turn_events = [event for event in events if event.get("turn") == turn_id]
|
|
for stage_name, start_type, end_type in STAGE_PAIRS:
|
|
timing = _pair_timing(turn_events, stage_name, start_type, end_type)
|
|
if timing is not None:
|
|
timings.append(timing)
|
|
timings.extend(_repeated_pair_timings(turn_events, "tts_playback", "tts_started", "playback_finished"))
|
|
return timings
|
|
|
|
|
|
def _pair_timing(
|
|
events: list[dict[str, object]],
|
|
name: str,
|
|
start_type: str,
|
|
end_type: str,
|
|
) -> dict[str, object] | None:
|
|
start = next((event for event in events if event.get("type") == start_type), None)
|
|
if start is None:
|
|
return None
|
|
end = next(
|
|
(event for event in events if event.get("type") == end_type and int(event["offset_ms"]) >= int(start["offset_ms"])),
|
|
None,
|
|
)
|
|
if end is None:
|
|
return None
|
|
return _timing_record(name, start, end)
|
|
|
|
|
|
def _repeated_pair_timings(
|
|
events: list[dict[str, object]],
|
|
name: str,
|
|
start_type: str,
|
|
end_type: str,
|
|
) -> list[dict[str, object]]:
|
|
timings: list[dict[str, object]] = []
|
|
pending: dict[str, object] | None = None
|
|
segment_index = 1
|
|
for event in events:
|
|
if event.get("type") == start_type:
|
|
pending = event
|
|
continue
|
|
if pending is not None and event.get("type") == end_type:
|
|
timing = _timing_record(name, pending, event)
|
|
timing["segment_index"] = segment_index
|
|
timings.append(timing)
|
|
segment_index += 1
|
|
pending = None
|
|
return timings
|
|
|
|
|
|
def _timing_record(name: str, start: dict[str, object], end: dict[str, object]) -> dict[str, object]:
|
|
return {
|
|
"turn": start.get("turn"),
|
|
"name": name,
|
|
"started_at": start.get("at"),
|
|
"finished_at": end.get("at"),
|
|
"start_offset_ms": start.get("offset_ms"),
|
|
"end_offset_ms": end.get("offset_ms"),
|
|
"duration_ms": int(end["offset_ms"]) - int(start["offset_ms"]),
|
|
"start_event": start.get("type"),
|
|
"end_event": end.get("type"),
|
|
}
|