mkbk/Owner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[异步播报打断]：完成播放中麦克风监听和音色隔离，包含后台监听、助手回放抑制和用户音色打断测试

Browse Source
This commit is contained in:
mkbk
2026-06-18 16:19:30 +08:00
parent d83f5a19cb
commit 6153cd2826
16 changed files with 584 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_barge_in.py
+132
View File
@@ -0,0 +1,132 @@
from __future__ import annotations
import time
import unittest
from owner_voice_pet.barge_in import AsyncBargeInMonitor, BargeInSpeakerGate
from owner_voice_pet.models import AudioFrame, AudioSegment
from owner_voice_pet.stt import MetadataSttProvider
from owner_voice_pet.transport import MemoryAudioTransport
from owner_voice_pet.vad import EnergyVadProvider
def speech_frame(idx: int, speaker_id: str, partial: str = "等一下") -> AudioFrame:
return AudioFrame(
b"\xff\x7f" * 320,
16000,
1,
idx * 20,
idx,
{
"duration_ms": 20,
"speech": True,
"speaker_id": speaker_id,
"partial_transcript": partial,
},
)
def segment_for_speaker(speaker_id: str) -> AudioSegment:
return AudioSegment(
b"\xff\x7f" * 640,
16000,
1,
0,
40,
{"speaker_id": speaker_id},
)
class BargeInTests(unittest.TestCase):
def test_async_monitor_ignores_assistant_playback_voice(self) -> None:
gate = BargeInSpeakerGate(
enabled=True,
user_similarity_threshold=0.62,
assistant_reject_threshold=0.72,
min_rms=0.001,
)
gate.remember_user_segment(segment_for_speaker("owner"))
transport = MemoryAudioTransport([speech_frame(1, "assistant"), speech_frame(2, "assistant")])
transport.start_input()
monitor = AsyncBargeInMonitor(
transport=transport,
vad_provider=EnergyVadProvider(threshold=1),
realtime_stt=MetadataSttProvider(),
speaker_gate=gate,
assistant_profile=gate.assistant_profile(segment_for_speaker("assistant")),
echo_guard_ms=0,
min_speech_ms=40,
listen_interval_ms=1,
)
monitor.vad_provider.load()
monitor.start()
time.sleep(0.05)
monitor.stop()
self.assertFalse(monitor.interrupted)
self.assertEqual(monitor.pending_frames(), [])
def test_async_monitor_accepts_user_voice_and_buffers_frames(self) -> None:
gate = BargeInSpeakerGate(
enabled=True,
user_similarity_threshold=0.62,
assistant_reject_threshold=0.72,
min_rms=0.001,
)
gate.remember_user_segment(segment_for_speaker("owner"))
transport = MemoryAudioTransport([speech_frame(1, "owner"), speech_frame(2, "owner")])
transport.start_input()
monitor = AsyncBargeInMonitor(
transport=transport,
vad_provider=EnergyVadProvider(threshold=1),
realtime_stt=MetadataSttProvider(),
speaker_gate=gate,
assistant_profile=gate.assistant_profile(segment_for_speaker("assistant")),
echo_guard_ms=0,
min_speech_ms=40,
listen_interval_ms=1,
)
monitor.vad_provider.load()
monitor.start()
deadline = time.monotonic() + 1
while not monitor.interrupted and time.monotonic() < deadline:
time.sleep(0.005)
monitor.stop()
self.assertTrue(monitor.interrupted)
self.assertEqual([frame.metadata["speaker_id"] for frame in monitor.pending_frames()], ["owner", "owner"])
def test_user_profile_missing_falls_back_to_non_assistant_with_partial(self) -> None:
gate = BargeInSpeakerGate(
enabled=True,
user_similarity_threshold=0.62,
assistant_reject_threshold=0.72,
min_rms=0.001,
)
transport = MemoryAudioTransport([speech_frame(1, "guest"), speech_frame(2, "guest")])
transport.start_input()
monitor = AsyncBargeInMonitor(
transport=transport,
vad_provider=EnergyVadProvider(threshold=1),
realtime_stt=MetadataSttProvider(),
speaker_gate=gate,
assistant_profile=gate.assistant_profile(segment_for_speaker("assistant")),
echo_guard_ms=0,
min_speech_ms=40,
listen_interval_ms=1,
)
monitor.vad_provider.load()
monitor.start()
deadline = time.monotonic() + 1
while not monitor.interrupted and time.monotonic() < deadline:
time.sleep(0.005)
monitor.stop()
self.assertTrue(monitor.interrupted)
if __name__ == "__main__":
unittest.main()
tests/test_live_runtime.py
+35 -5
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import time
import unittest
from owner_voice_pet.config import AppConfig
@@ -238,6 +239,7 @@ class PlaybackInjectedTransport(MemoryAudioTransport):
if self.play_count == self.inject_after_play_count:
for frame in self.injected_frames:
self.inject(frame)
time.sleep(0.002)
return result
def health(self) -> TransportHealth:
@@ -649,15 +651,41 @@ class LiveRuntimeTests(unittest.TestCase):
def test_barge_in_interrupts_playback_and_keeps_only_spoken_context(self) -> None:
first_question_frames = [wake_frame(0, 0)]
first_question_frames.extend(segment_frames(1, 20, partials=["第一问", "第一问"]))
first_question_frames.extend(
[
AudioFrame(
frame.pcm,
frame.sample_rate,
frame.channels,
frame.timestamp_ms,
frame.frame_id,
{**dict(frame.metadata), "speaker_id": "owner"},
)
for frame in segment_frames(1, 20, partials=["第一问", "第一问"])
]
)
barge_frames = [
AudioFrame(b"\xff\x7f", 16000, 1, 600, 20, {"duration_ms": 20, "speech": True, "partial_transcript": "等一下"}),
AudioFrame(b"\xff\x7f", 16000, 1, 620, 21, {"duration_ms": 20, "speech": True, "partial_transcript": "等一下"}),
AudioFrame(
b"\xff\x7f",
16000,
1,
600,
20,
{"duration_ms": 20, "speech": True, "partial_transcript": "等一下", "speaker_id": "owner"},
),
AudioFrame(
b"\xff\x7f",
16000,
1,
620,
21,
{"duration_ms": 20, "speech": True, "partial_transcript": "等一下", "speaker_id": "owner"},
),
*silence_frames(22, 640, 3),
]
transport = PlaybackInjectedTransport(
first_question_frames,
inject_after_play_count=8,
inject_after_play_count=18,
injected_frames=barge_frames,
)
stt = QueueSttProvider(["第一问", "打断问题"])
@@ -677,8 +705,10 @@ class LiveRuntimeTests(unittest.TestCase):
speech_provider="cloud",
wake_ack_text="",
barge_in_enabled=True,
barge_in_echo_guard_ms=500,
barge_in_echo_guard_ms=0,
barge_in_min_speech_ms=40,
barge_in_listen_interval_ms=1,
barge_in_chunk_ms=20,
followup_listen_timeout_ms=3000,
),
transport=transport,
tests/test_models_config.py
+13
View File
@@ -113,6 +113,11 @@ class ModelsConfigTests(unittest.TestCase):
self.assertTrue(config.barge_in_enabled)
self.assertEqual(config.barge_in_min_speech_ms, 250)
self.assertEqual(config.barge_in_echo_guard_ms, 500)
self.assertTrue(config.barge_in_speaker_gate_enabled)
self.assertEqual(config.barge_in_user_similarity_threshold, 0.62)
self.assertEqual(config.barge_in_assistant_reject_threshold, 0.72)
self.assertEqual(config.barge_in_listen_interval_ms, 20)
self.assertEqual(config.barge_in_chunk_ms, 30)
self.assertTrue(config.end_chime_enabled)
self.assertEqual(str(config.end_chime_file), "assets/sounds/codex-notification.wav")
self.assertEqual(config.end_chime_frequency_hz, 880)
@@ -157,6 +162,10 @@ class ModelsConfigTests(unittest.TestCase):
followup_listen_timeout_ms=-1,
barge_in_min_speech_ms=-1,
barge_in_echo_guard_ms=-1,
barge_in_user_similarity_threshold=1.5,
barge_in_assistant_reject_threshold=0.0,
barge_in_listen_interval_ms=0,
barge_in_chunk_ms=0,
end_chime_frequency_hz=0,
end_chime_duration_ms=0,
)
@@ -167,6 +176,10 @@ class ModelsConfigTests(unittest.TestCase):
self.assertTrue(any("OWNER_FOLLOWUP_LISTEN_TIMEOUT_MS" in error.message for error in errors))
self.assertTrue(any("OWNER_BARGE_IN_MIN_SPEECH_MS" in error.message for error in errors))
self.assertTrue(any("OWNER_BARGE_IN_ECHO_GUARD_MS" in error.message for error in errors))
self.assertTrue(any("OWNER_BARGE_IN_USER_SIMILARITY_THRESHOLD" in error.message for error in errors))
self.assertTrue(any("OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD" in error.message for error in errors))
self.assertTrue(any("OWNER_BARGE_IN_LISTEN_INTERVAL_MS" in error.message for error in errors))
self.assertTrue(any("OWNER_BARGE_IN_CHUNK_MS" in error.message for error in errors))
self.assertTrue(any("OWNER_END_CHIME_FREQUENCY_HZ" in error.message for error in errors))
self.assertTrue(any("OWNER_END_CHIME_DURATION_MS" in error.message for error in errors))
tests/test_transport.py
+15
View File
@@ -133,6 +133,21 @@ class TransportTests(unittest.TestCase):
self.assertGreater(len(fake.output_writes), 1)
self.assertEqual(callbacks[-1], result.duration_ms)
def test_sounddevice_chunk_playback_stops_on_signal(self) -> None:
fake = FakeSoundDevice()
transport = SoundDeviceAudioTransport(sounddevice_module=fake)
segment = AudioSegment(b"\x00\x00" * 1600, 16000, 1, 0, 100)
result = transport.play_pcm_chunks(
segment,
chunk_ms=20,
should_stop=lambda: len(fake.output_writes) >= 2,
)
self.assertTrue(result.played)
self.assertEqual(len(fake.output_writes), 2)
self.assertEqual(result.duration_ms, 40)
def test_sounddevice_device_report_uses_query_devices(self) -> None:
fake = FakeSoundDevice()
report = SoundDeviceAudioTransport(sounddevice_module=fake).device_report()