mkbk/Owner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[实时转写]：完成录音期间转写显示，包含partial事件、本地streaming STT和终端实时反馈

Browse Source
This commit is contained in:
mkbk
2026-06-17 22:16:56 +08:00
parent a64bb86da4
commit a77a172412
16 changed files with 331 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_live_runtime.py
+41 -8
View File
@@ -14,6 +14,7 @@ from owner_voice_pet.events import (
STANDBY_RESUMED,
STT_STARTED,
TRANSCRIPT_FINAL,
TRANSCRIPT_PARTIAL,
TTS_STARTED,
WAKE_DETECTED,
WAKE_LISTENING,
@@ -23,16 +24,24 @@ from owner_voice_pet.llm import MockLlmProvider
from owner_voice_pet.models import AudioFrame, AudioSegment, Transcript
from owner_voice_pet.assistant_pipeline import VoiceAssistantPipeline
from owner_voice_pet.runtime import build_live_runtime
from owner_voice_pet.stt import MetadataSttProvider
from owner_voice_pet.transport import MemoryAudioTransport
from owner_voice_pet.tts import SineTtsProvider
from owner_voice_pet.vad import EnergyVadProvider, PrimarySpeakerVadRecorder, VadRecorder
from owner_voice_pet.wakeword import KeywordWakeWordProvider
def segment_frames(start_id: int, start_ms: int) -> list[AudioFrame]:
def segment_frames(start_id: int, start_ms: int, partials: list[str] | None = None) -> list[AudioFrame]:
partials = partials or []
first_metadata: dict[str, object] = {"duration_ms": 20, "speech": True}
second_metadata: dict[str, object] = {"duration_ms": 20, "speech": True}
if len(partials) >= 1:
first_metadata["partial_transcript"] = partials[0]
if len(partials) >= 2:
second_metadata["partial_transcript"] = partials[1]
return [
AudioFrame(b"\xff\x7f", 16000, 1, start_ms, start_id, {"duration_ms": 20, "speech": True}),
AudioFrame(b"\xff\x7f", 16000, 1, start_ms + 20, start_id + 1, {"duration_ms": 20, "speech": True}),
AudioFrame(b"\xff\x7f", 16000, 1, start_ms, start_id, first_metadata),
AudioFrame(b"\xff\x7f", 16000, 1, start_ms + 20, start_id + 1, second_metadata),
AudioFrame(b"\x00\x00", 16000, 1, start_ms + 40, start_id + 2, {"duration_ms": 20, "speech": False}),
AudioFrame(b"\x00\x00", 16000, 1, start_ms + 60, start_id + 3, {"duration_ms": 20, "speech": False}),
]
@@ -68,6 +77,7 @@ class RecordingReporter:
def __init__(self) -> None:
self.statuses: list[str] = []
self.transcripts: list[str] = []
self.partials: list[str] = []
self.errors: list[str] = []
self.events: list[str] = []
@@ -76,20 +86,29 @@ class RecordingReporter:
self.events.append(f"status:{message}")
def transcript(self, text: str, *, final: bool, turn_id: int | None = None) -> None:
self.transcripts.append(text)
self.events.append(f"transcript:{text}")
if final:
self.transcripts.append(text)
self.events.append(f"transcript:final:{text}")
else:
self.partials.append(text)
self.events.append(f"transcript:partial:{text}")
def error(self, stage: str, code: str, message: str, *, turn_id: int | None = None) -> None:
self.errors.append(f"{stage}:{code}:{message}")
def make_runtime(texts: list[str], context: ConversationContext | None = None) -> tuple[VoiceAssistantPipeline, QueueSttProvider, MockLlmProvider, MemoryAudioTransport, RecordingReporter]:
def make_runtime(
texts: list[str],
context: ConversationContext | None = None,
partial_texts: list[list[str]] | None = None,
) -> tuple[VoiceAssistantPipeline, QueueSttProvider, MockLlmProvider, MemoryAudioTransport, RecordingReporter]:
frames = []
for idx, _text in enumerate(texts):
base_id = idx * 5
base_ms = idx * 120
frames.append(wake_frame(base_id, base_ms))
frames.extend(segment_frames(base_id + 1, base_ms + 20))
partials = partial_texts[idx] if partial_texts and idx < len(partial_texts) else None
frames.extend(segment_frames(base_id + 1, base_ms + 20, partials=partials))
transport = MemoryAudioTransport(frames)
stt = QueueSttProvider(texts)
llm = MockLlmProvider(["这是答复。"])
@@ -102,6 +121,7 @@ def make_runtime(texts: list[str], context: ConversationContext | None = None) -
wakeword=KeywordWakeWordProvider(),
vad_recorder=VadRecorder(EnergyVadProvider(), min_duration_ms=40, end_silence_ms=40),
stt=stt,
realtime_stt=MetadataSttProvider() if partial_texts is not None else None,
llm=llm,
tts=tts,
context=context or ConversationContext(),
@@ -170,12 +190,24 @@ class LiveRuntimeTests(unittest.TestCase):
runtime, _, _, _, reporter = make_runtime(["第一问"])
runtime.run(max_turns=1)
transcript_index = reporter.events.index("transcript:第一问")
transcript_index = reporter.events.index("transcript:final:第一问")
thinking_index = next(
index for index, event in enumerate(reporter.events) if event == "status:思考中：正在生成回复"
)
self.assertLess(transcript_index, thinking_index)
def test_realtime_transcript_is_reported_while_capturing(self) -> None:
runtime, _, llm, _, reporter = make_runtime(["第一问"], partial_texts=[["第一", "第一问"]])
runtime.run(max_turns=1)
self.assertEqual(reporter.partials, ["第一", "第一问"])
self.assertEqual(reporter.transcripts, ["第一问"])
self.assertEqual(llm.calls[0][-1].content, "第一问")
event_types = [event.type for event in runtime.event_bus.events]
self.assertLess(event_types.index(SPEECH_STARTED), event_types.index(TRANSCRIPT_PARTIAL))
self.assertLess(event_types.index(TRANSCRIPT_PARTIAL), event_types.index(SPEECH_ENDED))
self.assertLess(event_types.index(TRANSCRIPT_PARTIAL), event_types.index(TRANSCRIPT_FINAL))
def test_wake_keyword_does_not_pollute_llm_user_message(self) -> None:
runtime, _, llm, _, _ = make_runtime(["第一问"])
runtime.run(max_turns=1)
@@ -186,6 +218,7 @@ class LiveRuntimeTests(unittest.TestCase):
def test_live_runtime_uses_primary_speaker_endpoint_by_default(self) -> None:
runtime = build_live_runtime(AppConfig(llm_api_key="secret"))
self.assertIsInstance(runtime.vad_recorder, PrimarySpeakerVadRecorder)
self.assertIsNotNone(runtime.realtime_stt)
if __name__ == "__main__":