mkbk/Owner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[打断低延迟修复]：完成真实播报打断门控修正，包含短保护窗口、用户音色优先和诊断输出

Browse Source
This commit is contained in:
mkbk
2026-06-19 11:27:34 +08:00
parent f1f4885cc5
commit 7c6797aeb0
8 changed files with 222 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/owner_voice_pet/assistant_pipeline.py
+79 -7
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import time
from dataclasses import dataclass, field
from typing import Protocol
@@ -124,6 +125,7 @@ class TurnController:
self._pending_capture_frames: list[AudioFrame] = []
self._cached_ack_text: str | None = None
self._cached_ack_segment: AudioSegment | None = None
self._barge_in_reply_guard_used = True
self._barge_in_gate = BargeInSpeakerGate(
enabled=config.barge_in_speaker_gate_enabled,
user_similarity_threshold=config.barge_in_user_similarity_threshold,
@@ -244,10 +246,23 @@ class TurnController:
self.audio_preprocessor.reset()
realtime_session = self._start_realtime_transcript()
last_partial_ms: int | None = None
capture_started_at = time.monotonic()
self._event(CAPTURE_STARTED, PipelineState.RECORDING, state_message, turn_id=turn_id)
while True:
frames = self._read_capture_frames(timeout_ms=100)
if not frames:
if (
no_speech_timeout_ms is not None
and not self.vad_recorder.started
and (time.monotonic() - capture_started_at) * 1000 >= no_speech_timeout_ms
):
return ProviderError(
ErrorCode.VAD_TIMEOUT_NO_SPEECH,
"no speech detected before timeout",
True,
"voice-assistant-pipeline",
"vad",
)
continue
for frame in frames:
try:
@@ -402,6 +417,7 @@ class TurnController:
assistant_text = ""
spoken_parts: list[str] = []
interrupted = False
self._barge_in_reply_guard_used = False
try:
for delta in self.llm.stream_reply(self.context.build_llm_messages()):
assistant_text += delta.text_delta
@@ -543,27 +559,61 @@ class TurnController:
self._event(TTS_STARTED, PipelineState.SPEAKING, "播放中：正在播报回复", turn_id=turn_id)
segment = self.tts.synthesize(spoken_sentence)
interruptible_segment = ensure_interruptible_pcm(segment)
if not self._can_interrupt_playback(interruptible_segment):
echo_guard_ms = self._effective_barge_in_echo_guard_ms(interruptible_segment)
min_speech_ms = self._effective_barge_in_min_speech_ms()
if not self._can_interrupt_playback(interruptible_segment, min_speech_ms=min_speech_ms):
self._barge_in_debug(
turn_id,
"not_interruptible",
{
"segment_ready": interruptible_segment is not None,
"barge_in_enabled": self.config.barge_in_enabled,
"segment_duration_ms": interruptible_segment.duration_ms if interruptible_segment else None,
"min_speech_ms": min_speech_ms,
},
)
playback = self.transport.play_pcm(segment)
if playback.error:
raise playback.error
self._event(PLAYBACK_FINISHED, PipelineState.SPEAKING, "", turn_id=turn_id)
self._drain_input_after_playback()
return SpeakResult(spoken_sentence)
if self._play_interruptible(interruptible_segment, turn_id=turn_id):
if self._play_interruptible(
interruptible_segment,
turn_id=turn_id,
echo_guard_ms=echo_guard_ms,
min_speech_ms=min_speech_ms,
):
return SpeakResult("", interrupted=True)
self._event(PLAYBACK_FINISHED, PipelineState.SPEAKING, "", turn_id=turn_id)
self._drain_input_after_playback()
return SpeakResult(spoken_sentence)
def _can_interrupt_playback(self, segment: AudioSegment | None) -> bool:
def _can_interrupt_playback(self, segment: AudioSegment | None, *, min_speech_ms: int) -> bool:
return (
segment is not None
and self.config.barge_in_enabled
and segment.duration_ms > self.config.barge_in_echo_guard_ms
and segment.duration_ms >= max(self.config.barge_in_chunk_ms, min_speech_ms)
)
def _play_interruptible(self, segment: AudioSegment, *, turn_id: int) -> bool:
def _play_interruptible(
self,
segment: AudioSegment,
*,
turn_id: int,
echo_guard_ms: int,
min_speech_ms: int,
) -> bool:
self._barge_in_debug(
turn_id,
"monitor_starting",
{
"segment_duration_ms": segment.duration_ms,
"echo_guard_ms": echo_guard_ms,
"min_speech_ms": min_speech_ms,
"chunk_ms": self.config.barge_in_chunk_ms,
},
)
monitor = AsyncBargeInMonitor(
transport=self.transport,
vad_provider=self.vad_recorder.provider,
@@ -571,9 +621,14 @@ class TurnController:
speaker_gate=self._barge_in_gate,
assistant_profile=self._barge_in_gate.assistant_profile(segment),
assistant_reference=segment,
echo_guard_ms=self.config.barge_in_echo_guard_ms,
min_speech_ms=self.config.barge_in_min_speech_ms,
echo_guard_ms=echo_guard_ms,
min_speech_ms=min_speech_ms,
listen_interval_ms=self.config.barge_in_listen_interval_ms,
debug_callback=(
lambda reason, payload: self._barge_in_debug(turn_id, reason, payload)
if self.config.barge_in_debug
else None
),
)
monitor.start()
playback = self.transport.play_pcm_chunks(
@@ -591,6 +646,23 @@ class TurnController:
return True
return False
def _effective_barge_in_echo_guard_ms(self, segment: AudioSegment | None) -> int:
if segment is None or self._barge_in_reply_guard_used:
return 0
self._barge_in_reply_guard_used = True
return max(0, min(self.config.barge_in_echo_guard_ms, 120))
def _effective_barge_in_min_speech_ms(self) -> int:
latency_budget = max(60, self.config.interrupt_target_latency_ms - self.config.barge_in_chunk_ms)
return max(40, min(self.config.barge_in_min_speech_ms, latency_budget))
def _barge_in_debug(self, turn_id: int, reason: str, payload: dict[str, object]) -> None:
if not self.config.barge_in_debug:
return
details = " ".join(f"{key}={value}" for key, value in sorted(payload.items()))
message = f"打断诊断：{reason}" + (f" {details}" if details else "")
self._event("barge_in_debug", PipelineState.SPEAKING, message, turn_id=turn_id, payload=payload)
def _drain_input_after_playback(self) -> None:
self.transport.flush_input()
if self.config.post_playback_drain_ms <= 0:
src/owner_voice_pet/barge_in.py
+61 -17
View File
@@ -9,7 +9,7 @@ import time
import wave
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from typing import Any, Callable
from .models import AudioFrame, AudioSegment, ProviderError
from .protocols import AudioTransport, RealtimeSttProvider
@@ -59,24 +59,37 @@ class BargeInSpeakerGate:
assistant_reference: AudioSegment | None = None,
reference_elapsed_ms: int | None = None,
) -> bool:
accepted, _reason = self.evaluate_candidate(
frame,
assistant_profile,
assistant_reference,
reference_elapsed_ms=reference_elapsed_ms,
)
return accepted
def evaluate_candidate(
self,
frame: AudioFrame,
assistant_profile: TimbreProfile,
assistant_reference: AudioSegment | None = None,
reference_elapsed_ms: int | None = None,
) -> tuple[bool, str]:
if not self.enabled:
return True
return True, "gate_disabled"
candidate = self._profile_from_frame(frame)
if self._user_profile.ready and self._matches(candidate, self._user_profile, self.user_similarity_threshold):
return True, "user_profile_match"
if assistant_reference is not None and _looks_like_render_echo(
frame,
assistant_reference,
reference_elapsed_ms=reference_elapsed_ms,
):
return False
candidate = self._profile_from_frame(frame)
return False, "render_echo"
if not candidate.ready:
return True
return True, "candidate_profile_unready"
if self._matches(candidate, assistant_profile, self.assistant_reject_threshold):
return False
if self._user_profile.ready:
if self._matches(candidate, self._user_profile, self.user_similarity_threshold):
return True
return True
return True
return False, "assistant_profile_match"
return True, "non_assistant"
def _profile_from_segment(self, segment: AudioSegment) -> TimbreProfile:
frame = AudioFrame(
@@ -117,6 +130,7 @@ class AsyncBargeInMonitor:
echo_guard_ms: int,
min_speech_ms: int,
listen_interval_ms: int,
debug_callback: Callable[[str, dict[str, Any]], None] | None = None,
) -> None:
self.transport = transport
self.vad_provider = vad_provider
@@ -127,6 +141,7 @@ class AsyncBargeInMonitor:
self.echo_guard_ms = max(0, echo_guard_ms)
self.min_speech_ms = max(0, min_speech_ms)
self.listen_interval_ms = max(1, listen_interval_ms)
self.debug_callback = debug_callback
self.stop_event = threading.Event()
self._shutdown_event = threading.Event()
self._thread: threading.Thread | None = None
@@ -137,6 +152,14 @@ class AsyncBargeInMonitor:
def start(self) -> None:
self.vad_provider.reset()
self._debug(
"started",
{
"echo_guard_ms": self.echo_guard_ms,
"min_speech_ms": self.min_speech_ms,
"listen_interval_ms": self.listen_interval_ms,
},
)
self._thread = threading.Thread(target=self._run, name="owner-voice-barge-in", daemon=True)
self._thread.start()
@@ -160,6 +183,8 @@ class AsyncBargeInMonitor:
started_at = time.monotonic()
speech_ms = 0
candidate_frames: list[AudioFrame] = []
saw_frames = False
last_debug_at = 0.0
try:
while not self._shutdown_event.is_set() and not self.stop_event.is_set():
frames = self.transport.read_frames(timeout_ms=self.listen_interval_ms)
@@ -169,32 +194,53 @@ class AsyncBargeInMonitor:
if not frames:
time.sleep(self.listen_interval_ms / 1000)
continue
if not saw_frames:
saw_frames = True
self._debug("frames_seen", {"elapsed_ms": elapsed_ms, "frame_count": len(frames)})
for frame in frames:
result = self.vad_provider.analyze(frame)
if not result.is_speech:
speech_ms = 0
candidate_frames = []
continue
if not self.speaker_gate.accepts_candidate(
accepted, reason = self.speaker_gate.evaluate_candidate(
frame,
self.assistant_profile,
self.assistant_reference,
reference_elapsed_ms=elapsed_ms,
):
)
if not accepted:
now = time.monotonic()
if now - last_debug_at >= 0.25:
self._debug("gate_rejected", {"elapsed_ms": elapsed_ms, "reason": reason})
last_debug_at = now
speech_ms = 0
candidate_frames = []
continue
frame_ms = int(frame.metadata.get("duration_ms", 20))
speech_ms += frame_ms
candidate_frames.append(frame)
now = time.monotonic()
if now - last_debug_at >= 0.25:
self._debug(
"speech_accumulating",
{"elapsed_ms": elapsed_ms, "speech_ms": speech_ms, "reason": reason},
)
last_debug_at = now
if speech_ms >= self.min_speech_ms:
with self._lock:
self._pending_frames = list(candidate_frames)
self._interrupted = True
self._debug("triggered", {"elapsed_ms": elapsed_ms, "speech_ms": speech_ms})
self.stop_event.set()
return
except ProviderError as exc:
self.error = exc
self._debug("error", {"message": exc.message, "code": exc.code.value})
def _debug(self, reason: str, payload: dict[str, Any]) -> None:
if self.debug_callback is not None:
self.debug_callback(reason, payload)
def _looks_like_render_echo(
@@ -226,11 +272,9 @@ def _looks_like_render_echo(
candidate_window = candidate[-width:]
render_window = _reference_window(render, width, reference.sample_rate, reference_elapsed_ms)
corr = _normalized_corr(candidate_window, render_window)
if corr >= 0.92:
if corr >= 0.97:
return True
render_peak = float(np.max(np.abs(render_window))) if render_window.size else 0.0
candidate_peak = float(np.max(np.abs(candidate_window))) if candidate_window.size else 0.0
return corr >= 0.82 and candidate_peak <= render_peak * 1.15
return False
except Exception:
return False
src/owner_voice_pet/config.py
+2
View File
@@ -65,6 +65,7 @@ class AppConfig:
barge_in_assistant_reject_threshold: float = 0.72
barge_in_listen_interval_ms: int = 20
barge_in_chunk_ms: int = 30
barge_in_debug: bool = False
end_chime_enabled: bool = True
end_chime_file: Path = Path("assets/sounds/codex-notification.wav")
end_chime_frequency_hz: int = 880
@@ -185,6 +186,7 @@ class AppConfig:
),
barge_in_listen_interval_ms=int(get("BARGE_IN_LISTEN_INTERVAL_MS", "20") or "20"),
barge_in_chunk_ms=int(get("BARGE_IN_CHUNK_MS", "30") or "30"),
barge_in_debug=(get("BARGE_IN_DEBUG", "0") or "0").lower() not in {"0", "false", "no"},
end_chime_enabled=(get("END_CHIME_ENABLED", "1") or "1").lower() not in {"0", "false", "no"},
end_chime_file=Path(
get("END_CHIME_FILE", "assets/sounds/codex-notification.wav")
src/owner_voice_pet/simulation.py
+1
View File
@@ -133,6 +133,7 @@ def run_simulated_live(
vad_end_silence_ms=350,
vad_no_speech_timeout_ms=3000,
vad_max_recording_ms=6000,
barge_in_enabled=False,
)
pipeline = VoiceAssistantPipeline(
config=config,