mkbk/Owner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[模拟麦克风验收]：完成自动化语音闭环自测，包含模拟音频输入、两轮Pipeline验证和fixture回放

Browse Source
This commit is contained in:
mkbk
2026-06-17 23:04:20 +08:00
parent 8c75fc5baf
commit b3566c2424
10 changed files with 445 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/owner_voice_pet/__init__.py
+2
View File
@@ -26,6 +26,7 @@ from .conversation import ConversationContext
from .llm import MockLlmProvider, OpenAICompatibleLlmProvider
from .pipeline import PipelineResult, VoicePipeline
from .runtime import LiveVoiceRuntime, RuntimeSummary, TerminalRuntimeReporter, TurnResult, build_live_runtime
from .simulation import run_simulated_live
from .tts import CloudTtsProvider, MacSayTtsProvider, SentenceBuffer, SineTtsProvider
from .assets import validate_pet_assets
from .ui import ConsolePetWindow, PetStateController, PetVisualState
@@ -64,6 +65,7 @@ __all__ = [
"TerminalRuntimeReporter",
"TurnResult",
"build_live_runtime",
"run_simulated_live",
"CloudTtsProvider",
"MacSayTtsProvider",
"SentenceBuffer",
src/owner_voice_pet/cli.py
+18
View File
@@ -14,6 +14,7 @@ from .llm import MockLlmProvider, OpenAICompatibleLlmProvider
from .models import AudioFrame, ProviderError
from .pipeline import VoicePipeline
from .runtime import build_live_runtime
from .simulation import run_simulated_live
from .speech_models import check_speech_models, model_status_errors
from .stt import MetadataSttProvider, SherpaOnnxSttProvider
from .transport import MemoryAudioTransport, sounddevice_device_report
@@ -35,6 +36,10 @@ def main(argv: list[str] | None = None) -> int:
subparsers.add_parser("device-check", help="Validate local microphone and speaker availability")
live = subparsers.add_parser("run-live", help="Run real repeated live voice conversation")
live.add_argument("--once", action="store_true", help="Run one completed live turn and exit")
simulate = subparsers.add_parser("simulate-live", help="Run live pipeline with simulated microphone frames")
simulate.add_argument("--turns", type=int, default=2, help="Number of simulated turns. Default: 2")
simulate.add_argument("--fixture", default=None, help="Replay simulated microphone frames from JSONL")
simulate.add_argument("--write-fixture", default=None, help="Write generated simulated microphone frames to JSONL")
smoke = subparsers.add_parser("llm-smoke", help="Call configured OpenAI/NewAPI endpoint")
smoke.add_argument("--message", default="用一句中文回复：小杰在线。")
smoke.add_argument("--no-stream", action="store_true")
@@ -141,6 +146,19 @@ def main(argv: list[str] | None = None) -> int:
return 1
return 0 if summary.completed_turns > 0 or summary.interrupted else 1
if args.command == "simulate-live":
try:
data = run_simulated_live(
turns=args.turns,
fixture_path=args.fixture,
write_fixture=args.write_fixture,
)
except (ProviderError, ValueError) as exc:
print(json.dumps({"success": False, "error": str(exc)}, ensure_ascii=False, sort_keys=True))
return 1
print(json.dumps(data, ensure_ascii=False, sort_keys=True))
return 0 if data["success"] else 1
if args.command == "acceptance":
result = run_acceptance()
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
src/owner_voice_pet/simulation.py
+283
View File
@@ -0,0 +1,283 @@
from __future__ import annotations
import math
import struct
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from .audio_preprocess import NoopAudioPreprocessor
from .assistant_pipeline import VoiceAssistantPipeline
from .config import AppConfig
from .conversation import ConversationContext
from .events import (
LLM_STARTED,
PLAYBACK_FINISHED,
SPEECH_ENDED,
SPEECH_STARTED,
STANDBY_RESUMED,
STT_STARTED,
TRANSCRIPT_FINAL,
TRANSCRIPT_PARTIAL,
TTS_STARTED,
WAKE_DETECTED,
WAKE_LISTENING,
)
from .llm import MockLlmProvider
from .models import AudioFrame, AudioSegment, ErrorCode, ProviderError
from .stt import MetadataSttProvider
from .transport import FileReplayTransport, MemoryAudioTransport
from .tts import SineTtsProvider
from .vad import EnergyVadProvider, PrimarySpeakerVadRecorder
from .wakeword import KeywordWakeWordProvider
@dataclass(slots=True)
class SimulationReporter:
statuses: list[str] = field(default_factory=list)
partials: list[str] = field(default_factory=list)
finals: list[str] = field(default_factory=list)
errors: list[str] = field(default_factory=list)
def status(self, state: str, message: str, *, turn_id: int | None = None) -> None:
prefix = f"第{turn_id}轮:" if turn_id is not None else ""
self.statuses.append(f"{prefix}{message}")
def transcript(self, text: str, *, final: bool, turn_id: int | None = None) -> None:
if final:
self.finals.append(text)
else:
self.partials.append(text)
def error(self, stage: str, code: str, message: str, *, turn_id: int | None = None) -> None:
self.errors.append(f"{stage}:{code}:{message}")
class BoundedMemoryAudioTransport(MemoryAudioTransport):
def __init__(self, frames: list[AudioFrame], *, max_empty_reads: int = 5) -> None:
super().__init__(frames)
self.max_empty_reads = max_empty_reads
self.empty_reads = 0
def read_frames(self, timeout_ms: int) -> list[AudioFrame]:
frames = super().read_frames(timeout_ms)
if frames:
self.empty_reads = 0
return frames
self.empty_reads += 1
if self.empty_reads > self.max_empty_reads:
raise ProviderError(
ErrorCode.VALIDATION_FAILED,
"simulated microphone frames were exhausted",
False,
"simulated-microphone",
"transport",
)
return []
class SimulatedNoiseFilter(NoopAudioPreprocessor):
def __init__(self) -> None:
self.loaded = False
self.processed_frames = 0
def load(self) -> None:
self.loaded = True
def reset(self) -> None:
return None
def process_frame(self, frame: AudioFrame) -> AudioFrame:
self.processed_frames += 1
metadata = dict(frame.metadata)
metadata["denoised"] = True
metadata["noise_filter_provider"] = "simulated"
return AudioFrame(
pcm=frame.pcm,
sample_rate=frame.sample_rate,
channels=frame.channels,
timestamp_ms=frame.timestamp_ms,
frame_id=frame.frame_id,
metadata=metadata,
)
def run_simulated_live(
*,
turns: int = 2,
fixture_path: str | Path | None = None,
write_fixture: str | Path | None = None,
) -> dict[str, Any]:
if turns <= 0:
raise ValueError("turns must be positive")
frames = FileReplayTransport.from_jsonl(fixture_path)._frames if fixture_path else _simulated_turn_frames(turns)
frame_list = list(frames)
if write_fixture:
FileReplayTransport.write_jsonl(write_fixture, frame_list)
transport = BoundedMemoryAudioTransport(frame_list)
reporter = SimulationReporter()
preprocessor = SimulatedNoiseFilter()
llm = MockLlmProvider(["这是模拟回复。"])
config = AppConfig(
llm_api_key="simulated",
speech_provider="local",
realtime_transcript_enabled=True,
noise_filter_enabled=True,
post_playback_drain_ms=0,
endpoint_mode="primary_speaker",
speaker_profile_ms=120,
speaker_profile_min_ms=120,
speaker_absent_ms=300,
vad_min_duration_ms=250,
vad_end_silence_ms=350,
vad_no_speech_timeout_ms=3000,
vad_max_recording_ms=6000,
)
pipeline = VoiceAssistantPipeline(
config=config,
transport=transport,
wakeword=KeywordWakeWordProvider(threshold=0.5),
vad_recorder=PrimarySpeakerVadRecorder(
EnergyVadProvider(),
min_duration_ms=config.vad_min_duration_ms,
end_silence_ms=config.vad_end_silence_ms,
no_speech_timeout_ms=config.vad_no_speech_timeout_ms,
max_recording_ms=config.vad_max_recording_ms,
speaker_profile_ms=config.speaker_profile_ms,
speaker_profile_min_ms=config.speaker_profile_min_ms,
speaker_absent_ms=config.speaker_absent_ms,
similarity_threshold=config.speaker_similarity_threshold,
min_rms=config.speaker_min_rms,
),
audio_preprocessor=preprocessor,
stt=MetadataSttProvider(),
realtime_stt=MetadataSttProvider(),
llm=llm,
tts=SineTtsProvider(),
ack_tts=SineTtsProvider(),
context=ConversationContext(),
reporter=reporter,
)
completed_turns = 0
failed_turns = 0
pipeline.load()
transport.start_input(sample_rate=config.sample_rate, channels=config.channels)
try:
for turn_id in range(1, turns + 1):
result = pipeline.run_turn(turn_id)
if result.success:
completed_turns += 1
continue
failed_turns += 1
break
finally:
pipeline.shutdown()
event_types = [event.type for event in pipeline.event_bus.events]
expected_transcripts = [f"第{index}轮模拟问题" for index in range(1, turns + 1)]
checks = {
"completed_turns": completed_turns == turns,
"no_failed_turns": failed_turns == 0,
"wake_per_turn": event_types.count(WAKE_DETECTED) == turns,
"speech_per_turn": event_types.count(SPEECH_STARTED) == turns and event_types.count(SPEECH_ENDED) == turns,
"stt_per_turn": event_types.count(STT_STARTED) == turns and event_types.count(TRANSCRIPT_FINAL) == turns,
"llm_per_turn": event_types.count(LLM_STARTED) == turns and len(llm.calls) == turns,
"tts_per_turn": event_types.count(TTS_STARTED) == turns and event_types.count(PLAYBACK_FINISHED) >= turns,
"standby_per_turn": event_types.count(STANDBY_RESUMED) == turns,
"transcripts_match": reporter.finals == expected_transcripts,
"partial_noise_filtered": "家" not in reporter.partials and "家确" not in reporter.partials,
"denoised_capture": preprocessor.loaded and preprocessor.processed_frames > 0,
"context_in_second_turn": turns < 2 or _second_turn_has_first_history(llm.calls, expected_transcripts[0]),
}
return {
"success": all(checks.values()) and not reporter.errors,
"turns": turns,
"completed_turns": completed_turns,
"failed_turns": failed_turns,
"checks": checks,
"errors": reporter.errors,
"partials": reporter.partials,
"final_transcripts": reporter.finals,
"played_segments": len(transport.played_segments),
"llm_calls": len(llm.calls),
"event_types": event_types,
}
def _second_turn_has_first_history(calls: list[list[Any]], first_user_text: str) -> bool:
if len(calls) < 2:
return False
return any(getattr(message, "content", "") == first_user_text for message in calls[1])
def _simulated_turn_frames(turns: int) -> list[AudioFrame]:
frames: list[AudioFrame] = []
frame_id = 0
timestamp_ms = 0
for index in range(1, turns + 1):
frames.append(
_frame(
frame_id,
timestamp_ms,
frequency=880.0,
metadata={"duration_ms": 20, "wake_word": "小杰小杰", "wake_confidence": 0.99},
)
)
frame_id += 1
timestamp_ms += 20
question = f"第{index}轮模拟问题"
for speech_index in range(6):
partial = "家" if speech_index == 0 else "家确" if speech_index == 1 else question
metadata: dict[str, object] = {
"duration_ms": 20,
"speech": True,
"speaker_id": "owner",
"partial_transcript": partial,
}
if speech_index == 0:
metadata["transcript"] = question
frames.append(
_frame(
frame_id,
timestamp_ms,
frequency=440.0,
metadata=metadata,
)
)
frame_id += 1
timestamp_ms += 20
for _ in range(15):
frames.append(
_frame(
frame_id,
timestamp_ms,
frequency=180.0,
amplitude=500,
metadata={
"duration_ms": 20,
"speech": True,
"speaker_id": "background",
"partial_transcript": "家确",
},
)
)
frame_id += 1
timestamp_ms += 20
return frames
def _frame(
frame_id: int,
timestamp_ms: int,
*,
frequency: float,
amplitude: int = 8000,
metadata: dict[str, object] | None = None,
) -> AudioFrame:
sample_rate = 16000
samples = int(sample_rate * 0.02)
pcm = bytearray()
for index in range(samples):
value = int(math.sin(2 * math.pi * frequency * index / sample_rate) * amplitude)
pcm.extend(struct.pack("<h", value))
return AudioFrame(bytes(pcm), sample_rate, 1, timestamp_ms, frame_id, metadata or {"duration_ms": 20})