[打断低延迟修复]:完成真实播报打断门控修正,包含短保护窗口、用户音色优先和诊断输出
This commit is contained in:
@@ -52,6 +52,7 @@ OWNER_BARGE_IN_USER_SIMILARITY_THRESHOLD=0.62
|
|||||||
OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD=0.72
|
OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD=0.72
|
||||||
OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20
|
OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20
|
||||||
OWNER_BARGE_IN_CHUNK_MS=30
|
OWNER_BARGE_IN_CHUNK_MS=30
|
||||||
|
OWNER_BARGE_IN_DEBUG=0
|
||||||
OWNER_END_CHIME_ENABLED=1
|
OWNER_END_CHIME_ENABLED=1
|
||||||
OWNER_END_CHIME_FILE=assets/sounds/codex-notification.wav
|
OWNER_END_CHIME_FILE=assets/sounds/codex-notification.wav
|
||||||
OWNER_END_CHIME_FREQUENCY_HZ=880
|
OWNER_END_CHIME_FREQUENCY_HZ=880
|
||||||
|
|||||||
@@ -80,6 +80,7 @@ OWNER_BARGE_IN_USER_SIMILARITY_THRESHOLD=0.62
|
|||||||
OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD=0.72
|
OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD=0.72
|
||||||
OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20
|
OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20
|
||||||
OWNER_BARGE_IN_CHUNK_MS=30
|
OWNER_BARGE_IN_CHUNK_MS=30
|
||||||
|
OWNER_BARGE_IN_DEBUG=0
|
||||||
OWNER_END_CHIME_ENABLED=1
|
OWNER_END_CHIME_ENABLED=1
|
||||||
OWNER_END_CHIME_FILE=assets/sounds/codex-notification.wav
|
OWNER_END_CHIME_FILE=assets/sounds/codex-notification.wav
|
||||||
OWNER_END_CHIME_FREQUENCY_HZ=880
|
OWNER_END_CHIME_FREQUENCY_HZ=880
|
||||||
@@ -117,7 +118,7 @@ OWNER_COMPUTER_CONTROL_ENABLED=0
|
|||||||
|
|
||||||
`OWNER_CONTINUOUS_DIALOG_ENABLED=1` 表示每轮回复播放后会自动判断是否继续对话。若助手回复里明显在问用户、要求补充信息或让用户选择,终端会输出 `继续对话:3秒内可直接回答`,这 3 秒内可以不用再说“小杰小杰”。若助手只是完成回答、报错、拒绝或判断不确定,就直接恢复待机。`OWNER_CONTINUATION_DECISION_PROVIDER=hybrid` 表示先用本地规则判断,规则不确定时才调用云端 LLM 做小分类;低于 `OWNER_CONTINUATION_CONFIDENCE_THRESHOLD=0.65` 的结果按待机处理。
|
`OWNER_CONTINUOUS_DIALOG_ENABLED=1` 表示每轮回复播放后会自动判断是否继续对话。若助手回复里明显在问用户、要求补充信息或让用户选择,终端会输出 `继续对话:3秒内可直接回答`,这 3 秒内可以不用再说“小杰小杰”。若助手只是完成回答、报错、拒绝或判断不确定,就直接恢复待机。`OWNER_CONTINUATION_DECISION_PROVIDER=hybrid` 表示先用本地规则判断,规则不确定时才调用云端 LLM 做小分类;低于 `OWNER_CONTINUATION_CONFIDENCE_THRESHOLD=0.65` 的结果按待机处理。
|
||||||
|
|
||||||
`OWNER_BARGE_IN_ENABLED=1` 表示播报中允许打断。播放回复时会启动后台麦克风监听,不再等每个播放 chunk 结束后才检查输入;`OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20` 控制监听间隔,`OWNER_BARGE_IN_CHUNK_MS=30` 控制播放停止粒度。播放开始后的 `OWNER_BARGE_IN_ECHO_GUARD_MS=500` 毫秒内忽略麦克风输入,之后如果检测到至少 `OWNER_BARGE_IN_MIN_SPEECH_MS=250` 毫秒有效用户语音,且不像当前助手回放 reference,就先停止剩余播报;STT 只用于后续识别打断内容,不再作为停播前置条件。`OWNER_BARGE_IN_SPEAKER_GATE_ENABLED=1` 会同时建立本次会话用户临时音色画像和当前助手回放音色画像,并拒绝相似度高于 `OWNER_BARGE_IN_ASSISTANT_REJECT_THRESHOLD=0.72` 的助手回放音色;用户相似度只作为增强判断,不作为唯一硬门槛,避免真人打断被过度拦截。音色画像只在进程内使用,不写文件、不发送给 LLM。上下文只记录已经完整播出的 assistant 句子,未播出的内容不会写入临时历史。
|
`OWNER_BARGE_IN_ENABLED=1` 表示播报中允许打断。播放回复时会启动后台麦克风监听,不再等每个播放 chunk 结束后才检查输入;`OWNER_BARGE_IN_LISTEN_INTERVAL_MS=20` 控制监听间隔,`OWNER_BARGE_IN_CHUNK_MS=30` 控制播放停止粒度。`OWNER_BARGE_IN_ECHO_GUARD_MS=500` 是回声保护上限,真实运行时只在每段回复的第一个播报片段使用一次,并压到 120 ms 内,避免每个短句都重新进入 500 ms 免打断窗口。`OWNER_BARGE_IN_MIN_SPEECH_MS=250` 是最短人声配置上限,实际运行会按 `OWNER_INTERRUPT_TARGET_LATENCY_MS=200` 和 chunk 粒度收紧,避免参数本身超过目标打断延迟。播放中如果 VAD 检测到有效用户语音,且不像当前助手回放 reference,就先停止剩余播报;STT 只用于后续识别打断内容,不再作为停播前置条件。`OWNER_BARGE_IN_SPEAKER_GATE_ENABLED=1` 会同时建立本次会话用户临时音色画像和当前助手回放音色画像;用户音色匹配优先于助手回放拒绝,避免“用户说话 + 扬声器回声”混合时被先当成 AI 自己声音丢掉。设置 `OWNER_BARGE_IN_DEBUG=1` 后,终端会输出打断监听启动、VAD 累计、回声门控拒绝原因和触发时长,便于现场定位。音色画像只在进程内使用,不写文件、不发送给 LLM。上下文只记录已经完整播出的 assistant 句子,未播出的内容不会写入临时历史。
|
||||||
|
|
||||||
`OWNER_END_CHIME_ENABLED=1` 表示对话自然结束或追问超时恢复待机前会播放一声项目内置提示音,默认文件是 `assets/sounds/codex-notification.wav`。提示音不走 TTS,也不会写入上下文;如果 `OWNER_END_CHIME_FILE` 指向的文件缺失,会回退到本地合成短音,`OWNER_END_CHIME_FREQUENCY_HZ` 和 `OWNER_END_CHIME_DURATION_MS` 只影响这个回退音。设置 `OWNER_END_CHIME_ENABLED=0` 可以关闭。
|
`OWNER_END_CHIME_ENABLED=1` 表示对话自然结束或追问超时恢复待机前会播放一声项目内置提示音,默认文件是 `assets/sounds/codex-notification.wav`。提示音不走 TTS,也不会写入上下文;如果 `OWNER_END_CHIME_FILE` 指向的文件缺失,会回退到本地合成短音,`OWNER_END_CHIME_FREQUENCY_HZ` 和 `OWNER_END_CHIME_DURATION_MS` 只影响这个回退音。设置 `OWNER_END_CHIME_ENABLED=0` 可以关闭。
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import Protocol
|
from typing import Protocol
|
||||||
|
|
||||||
@@ -124,6 +125,7 @@ class TurnController:
|
|||||||
self._pending_capture_frames: list[AudioFrame] = []
|
self._pending_capture_frames: list[AudioFrame] = []
|
||||||
self._cached_ack_text: str | None = None
|
self._cached_ack_text: str | None = None
|
||||||
self._cached_ack_segment: AudioSegment | None = None
|
self._cached_ack_segment: AudioSegment | None = None
|
||||||
|
self._barge_in_reply_guard_used = True
|
||||||
self._barge_in_gate = BargeInSpeakerGate(
|
self._barge_in_gate = BargeInSpeakerGate(
|
||||||
enabled=config.barge_in_speaker_gate_enabled,
|
enabled=config.barge_in_speaker_gate_enabled,
|
||||||
user_similarity_threshold=config.barge_in_user_similarity_threshold,
|
user_similarity_threshold=config.barge_in_user_similarity_threshold,
|
||||||
@@ -244,10 +246,23 @@ class TurnController:
|
|||||||
self.audio_preprocessor.reset()
|
self.audio_preprocessor.reset()
|
||||||
realtime_session = self._start_realtime_transcript()
|
realtime_session = self._start_realtime_transcript()
|
||||||
last_partial_ms: int | None = None
|
last_partial_ms: int | None = None
|
||||||
|
capture_started_at = time.monotonic()
|
||||||
self._event(CAPTURE_STARTED, PipelineState.RECORDING, state_message, turn_id=turn_id)
|
self._event(CAPTURE_STARTED, PipelineState.RECORDING, state_message, turn_id=turn_id)
|
||||||
while True:
|
while True:
|
||||||
frames = self._read_capture_frames(timeout_ms=100)
|
frames = self._read_capture_frames(timeout_ms=100)
|
||||||
if not frames:
|
if not frames:
|
||||||
|
if (
|
||||||
|
no_speech_timeout_ms is not None
|
||||||
|
and not self.vad_recorder.started
|
||||||
|
and (time.monotonic() - capture_started_at) * 1000 >= no_speech_timeout_ms
|
||||||
|
):
|
||||||
|
return ProviderError(
|
||||||
|
ErrorCode.VAD_TIMEOUT_NO_SPEECH,
|
||||||
|
"no speech detected before timeout",
|
||||||
|
True,
|
||||||
|
"voice-assistant-pipeline",
|
||||||
|
"vad",
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
for frame in frames:
|
for frame in frames:
|
||||||
try:
|
try:
|
||||||
@@ -402,6 +417,7 @@ class TurnController:
|
|||||||
assistant_text = ""
|
assistant_text = ""
|
||||||
spoken_parts: list[str] = []
|
spoken_parts: list[str] = []
|
||||||
interrupted = False
|
interrupted = False
|
||||||
|
self._barge_in_reply_guard_used = False
|
||||||
try:
|
try:
|
||||||
for delta in self.llm.stream_reply(self.context.build_llm_messages()):
|
for delta in self.llm.stream_reply(self.context.build_llm_messages()):
|
||||||
assistant_text += delta.text_delta
|
assistant_text += delta.text_delta
|
||||||
@@ -543,27 +559,61 @@ class TurnController:
|
|||||||
self._event(TTS_STARTED, PipelineState.SPEAKING, "播放中:正在播报回复", turn_id=turn_id)
|
self._event(TTS_STARTED, PipelineState.SPEAKING, "播放中:正在播报回复", turn_id=turn_id)
|
||||||
segment = self.tts.synthesize(spoken_sentence)
|
segment = self.tts.synthesize(spoken_sentence)
|
||||||
interruptible_segment = ensure_interruptible_pcm(segment)
|
interruptible_segment = ensure_interruptible_pcm(segment)
|
||||||
if not self._can_interrupt_playback(interruptible_segment):
|
echo_guard_ms = self._effective_barge_in_echo_guard_ms(interruptible_segment)
|
||||||
|
min_speech_ms = self._effective_barge_in_min_speech_ms()
|
||||||
|
if not self._can_interrupt_playback(interruptible_segment, min_speech_ms=min_speech_ms):
|
||||||
|
self._barge_in_debug(
|
||||||
|
turn_id,
|
||||||
|
"not_interruptible",
|
||||||
|
{
|
||||||
|
"segment_ready": interruptible_segment is not None,
|
||||||
|
"barge_in_enabled": self.config.barge_in_enabled,
|
||||||
|
"segment_duration_ms": interruptible_segment.duration_ms if interruptible_segment else None,
|
||||||
|
"min_speech_ms": min_speech_ms,
|
||||||
|
},
|
||||||
|
)
|
||||||
playback = self.transport.play_pcm(segment)
|
playback = self.transport.play_pcm(segment)
|
||||||
if playback.error:
|
if playback.error:
|
||||||
raise playback.error
|
raise playback.error
|
||||||
self._event(PLAYBACK_FINISHED, PipelineState.SPEAKING, "", turn_id=turn_id)
|
self._event(PLAYBACK_FINISHED, PipelineState.SPEAKING, "", turn_id=turn_id)
|
||||||
self._drain_input_after_playback()
|
self._drain_input_after_playback()
|
||||||
return SpeakResult(spoken_sentence)
|
return SpeakResult(spoken_sentence)
|
||||||
if self._play_interruptible(interruptible_segment, turn_id=turn_id):
|
if self._play_interruptible(
|
||||||
|
interruptible_segment,
|
||||||
|
turn_id=turn_id,
|
||||||
|
echo_guard_ms=echo_guard_ms,
|
||||||
|
min_speech_ms=min_speech_ms,
|
||||||
|
):
|
||||||
return SpeakResult("", interrupted=True)
|
return SpeakResult("", interrupted=True)
|
||||||
self._event(PLAYBACK_FINISHED, PipelineState.SPEAKING, "", turn_id=turn_id)
|
self._event(PLAYBACK_FINISHED, PipelineState.SPEAKING, "", turn_id=turn_id)
|
||||||
self._drain_input_after_playback()
|
self._drain_input_after_playback()
|
||||||
return SpeakResult(spoken_sentence)
|
return SpeakResult(spoken_sentence)
|
||||||
|
|
||||||
def _can_interrupt_playback(self, segment: AudioSegment | None) -> bool:
|
def _can_interrupt_playback(self, segment: AudioSegment | None, *, min_speech_ms: int) -> bool:
|
||||||
return (
|
return (
|
||||||
segment is not None
|
segment is not None
|
||||||
and self.config.barge_in_enabled
|
and self.config.barge_in_enabled
|
||||||
and segment.duration_ms > self.config.barge_in_echo_guard_ms
|
and segment.duration_ms >= max(self.config.barge_in_chunk_ms, min_speech_ms)
|
||||||
)
|
)
|
||||||
|
|
||||||
def _play_interruptible(self, segment: AudioSegment, *, turn_id: int) -> bool:
|
def _play_interruptible(
|
||||||
|
self,
|
||||||
|
segment: AudioSegment,
|
||||||
|
*,
|
||||||
|
turn_id: int,
|
||||||
|
echo_guard_ms: int,
|
||||||
|
min_speech_ms: int,
|
||||||
|
) -> bool:
|
||||||
|
self._barge_in_debug(
|
||||||
|
turn_id,
|
||||||
|
"monitor_starting",
|
||||||
|
{
|
||||||
|
"segment_duration_ms": segment.duration_ms,
|
||||||
|
"echo_guard_ms": echo_guard_ms,
|
||||||
|
"min_speech_ms": min_speech_ms,
|
||||||
|
"chunk_ms": self.config.barge_in_chunk_ms,
|
||||||
|
},
|
||||||
|
)
|
||||||
monitor = AsyncBargeInMonitor(
|
monitor = AsyncBargeInMonitor(
|
||||||
transport=self.transport,
|
transport=self.transport,
|
||||||
vad_provider=self.vad_recorder.provider,
|
vad_provider=self.vad_recorder.provider,
|
||||||
@@ -571,9 +621,14 @@ class TurnController:
|
|||||||
speaker_gate=self._barge_in_gate,
|
speaker_gate=self._barge_in_gate,
|
||||||
assistant_profile=self._barge_in_gate.assistant_profile(segment),
|
assistant_profile=self._barge_in_gate.assistant_profile(segment),
|
||||||
assistant_reference=segment,
|
assistant_reference=segment,
|
||||||
echo_guard_ms=self.config.barge_in_echo_guard_ms,
|
echo_guard_ms=echo_guard_ms,
|
||||||
min_speech_ms=self.config.barge_in_min_speech_ms,
|
min_speech_ms=min_speech_ms,
|
||||||
listen_interval_ms=self.config.barge_in_listen_interval_ms,
|
listen_interval_ms=self.config.barge_in_listen_interval_ms,
|
||||||
|
debug_callback=(
|
||||||
|
lambda reason, payload: self._barge_in_debug(turn_id, reason, payload)
|
||||||
|
if self.config.barge_in_debug
|
||||||
|
else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
monitor.start()
|
monitor.start()
|
||||||
playback = self.transport.play_pcm_chunks(
|
playback = self.transport.play_pcm_chunks(
|
||||||
@@ -591,6 +646,23 @@ class TurnController:
|
|||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
def _effective_barge_in_echo_guard_ms(self, segment: AudioSegment | None) -> int:
|
||||||
|
if segment is None or self._barge_in_reply_guard_used:
|
||||||
|
return 0
|
||||||
|
self._barge_in_reply_guard_used = True
|
||||||
|
return max(0, min(self.config.barge_in_echo_guard_ms, 120))
|
||||||
|
|
||||||
|
def _effective_barge_in_min_speech_ms(self) -> int:
|
||||||
|
latency_budget = max(60, self.config.interrupt_target_latency_ms - self.config.barge_in_chunk_ms)
|
||||||
|
return max(40, min(self.config.barge_in_min_speech_ms, latency_budget))
|
||||||
|
|
||||||
|
def _barge_in_debug(self, turn_id: int, reason: str, payload: dict[str, object]) -> None:
|
||||||
|
if not self.config.barge_in_debug:
|
||||||
|
return
|
||||||
|
details = " ".join(f"{key}={value}" for key, value in sorted(payload.items()))
|
||||||
|
message = f"打断诊断:{reason}" + (f" {details}" if details else "")
|
||||||
|
self._event("barge_in_debug", PipelineState.SPEAKING, message, turn_id=turn_id, payload=payload)
|
||||||
|
|
||||||
def _drain_input_after_playback(self) -> None:
|
def _drain_input_after_playback(self) -> None:
|
||||||
self.transport.flush_input()
|
self.transport.flush_input()
|
||||||
if self.config.post_playback_drain_ms <= 0:
|
if self.config.post_playback_drain_ms <= 0:
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import time
|
|||||||
import wave
|
import wave
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any, Callable
|
||||||
|
|
||||||
from .models import AudioFrame, AudioSegment, ProviderError
|
from .models import AudioFrame, AudioSegment, ProviderError
|
||||||
from .protocols import AudioTransport, RealtimeSttProvider
|
from .protocols import AudioTransport, RealtimeSttProvider
|
||||||
@@ -59,24 +59,37 @@ class BargeInSpeakerGate:
|
|||||||
assistant_reference: AudioSegment | None = None,
|
assistant_reference: AudioSegment | None = None,
|
||||||
reference_elapsed_ms: int | None = None,
|
reference_elapsed_ms: int | None = None,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
|
accepted, _reason = self.evaluate_candidate(
|
||||||
|
frame,
|
||||||
|
assistant_profile,
|
||||||
|
assistant_reference,
|
||||||
|
reference_elapsed_ms=reference_elapsed_ms,
|
||||||
|
)
|
||||||
|
return accepted
|
||||||
|
|
||||||
|
def evaluate_candidate(
|
||||||
|
self,
|
||||||
|
frame: AudioFrame,
|
||||||
|
assistant_profile: TimbreProfile,
|
||||||
|
assistant_reference: AudioSegment | None = None,
|
||||||
|
reference_elapsed_ms: int | None = None,
|
||||||
|
) -> tuple[bool, str]:
|
||||||
if not self.enabled:
|
if not self.enabled:
|
||||||
return True
|
return True, "gate_disabled"
|
||||||
|
candidate = self._profile_from_frame(frame)
|
||||||
|
if self._user_profile.ready and self._matches(candidate, self._user_profile, self.user_similarity_threshold):
|
||||||
|
return True, "user_profile_match"
|
||||||
if assistant_reference is not None and _looks_like_render_echo(
|
if assistant_reference is not None and _looks_like_render_echo(
|
||||||
frame,
|
frame,
|
||||||
assistant_reference,
|
assistant_reference,
|
||||||
reference_elapsed_ms=reference_elapsed_ms,
|
reference_elapsed_ms=reference_elapsed_ms,
|
||||||
):
|
):
|
||||||
return False
|
return False, "render_echo"
|
||||||
candidate = self._profile_from_frame(frame)
|
|
||||||
if not candidate.ready:
|
if not candidate.ready:
|
||||||
return True
|
return True, "candidate_profile_unready"
|
||||||
if self._matches(candidate, assistant_profile, self.assistant_reject_threshold):
|
if self._matches(candidate, assistant_profile, self.assistant_reject_threshold):
|
||||||
return False
|
return False, "assistant_profile_match"
|
||||||
if self._user_profile.ready:
|
return True, "non_assistant"
|
||||||
if self._matches(candidate, self._user_profile, self.user_similarity_threshold):
|
|
||||||
return True
|
|
||||||
return True
|
|
||||||
return True
|
|
||||||
|
|
||||||
def _profile_from_segment(self, segment: AudioSegment) -> TimbreProfile:
|
def _profile_from_segment(self, segment: AudioSegment) -> TimbreProfile:
|
||||||
frame = AudioFrame(
|
frame = AudioFrame(
|
||||||
@@ -117,6 +130,7 @@ class AsyncBargeInMonitor:
|
|||||||
echo_guard_ms: int,
|
echo_guard_ms: int,
|
||||||
min_speech_ms: int,
|
min_speech_ms: int,
|
||||||
listen_interval_ms: int,
|
listen_interval_ms: int,
|
||||||
|
debug_callback: Callable[[str, dict[str, Any]], None] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.transport = transport
|
self.transport = transport
|
||||||
self.vad_provider = vad_provider
|
self.vad_provider = vad_provider
|
||||||
@@ -127,6 +141,7 @@ class AsyncBargeInMonitor:
|
|||||||
self.echo_guard_ms = max(0, echo_guard_ms)
|
self.echo_guard_ms = max(0, echo_guard_ms)
|
||||||
self.min_speech_ms = max(0, min_speech_ms)
|
self.min_speech_ms = max(0, min_speech_ms)
|
||||||
self.listen_interval_ms = max(1, listen_interval_ms)
|
self.listen_interval_ms = max(1, listen_interval_ms)
|
||||||
|
self.debug_callback = debug_callback
|
||||||
self.stop_event = threading.Event()
|
self.stop_event = threading.Event()
|
||||||
self._shutdown_event = threading.Event()
|
self._shutdown_event = threading.Event()
|
||||||
self._thread: threading.Thread | None = None
|
self._thread: threading.Thread | None = None
|
||||||
@@ -137,6 +152,14 @@ class AsyncBargeInMonitor:
|
|||||||
|
|
||||||
def start(self) -> None:
|
def start(self) -> None:
|
||||||
self.vad_provider.reset()
|
self.vad_provider.reset()
|
||||||
|
self._debug(
|
||||||
|
"started",
|
||||||
|
{
|
||||||
|
"echo_guard_ms": self.echo_guard_ms,
|
||||||
|
"min_speech_ms": self.min_speech_ms,
|
||||||
|
"listen_interval_ms": self.listen_interval_ms,
|
||||||
|
},
|
||||||
|
)
|
||||||
self._thread = threading.Thread(target=self._run, name="owner-voice-barge-in", daemon=True)
|
self._thread = threading.Thread(target=self._run, name="owner-voice-barge-in", daemon=True)
|
||||||
self._thread.start()
|
self._thread.start()
|
||||||
|
|
||||||
@@ -160,6 +183,8 @@ class AsyncBargeInMonitor:
|
|||||||
started_at = time.monotonic()
|
started_at = time.monotonic()
|
||||||
speech_ms = 0
|
speech_ms = 0
|
||||||
candidate_frames: list[AudioFrame] = []
|
candidate_frames: list[AudioFrame] = []
|
||||||
|
saw_frames = False
|
||||||
|
last_debug_at = 0.0
|
||||||
try:
|
try:
|
||||||
while not self._shutdown_event.is_set() and not self.stop_event.is_set():
|
while not self._shutdown_event.is_set() and not self.stop_event.is_set():
|
||||||
frames = self.transport.read_frames(timeout_ms=self.listen_interval_ms)
|
frames = self.transport.read_frames(timeout_ms=self.listen_interval_ms)
|
||||||
@@ -169,32 +194,53 @@ class AsyncBargeInMonitor:
|
|||||||
if not frames:
|
if not frames:
|
||||||
time.sleep(self.listen_interval_ms / 1000)
|
time.sleep(self.listen_interval_ms / 1000)
|
||||||
continue
|
continue
|
||||||
|
if not saw_frames:
|
||||||
|
saw_frames = True
|
||||||
|
self._debug("frames_seen", {"elapsed_ms": elapsed_ms, "frame_count": len(frames)})
|
||||||
for frame in frames:
|
for frame in frames:
|
||||||
result = self.vad_provider.analyze(frame)
|
result = self.vad_provider.analyze(frame)
|
||||||
if not result.is_speech:
|
if not result.is_speech:
|
||||||
speech_ms = 0
|
speech_ms = 0
|
||||||
candidate_frames = []
|
candidate_frames = []
|
||||||
continue
|
continue
|
||||||
if not self.speaker_gate.accepts_candidate(
|
accepted, reason = self.speaker_gate.evaluate_candidate(
|
||||||
frame,
|
frame,
|
||||||
self.assistant_profile,
|
self.assistant_profile,
|
||||||
self.assistant_reference,
|
self.assistant_reference,
|
||||||
reference_elapsed_ms=elapsed_ms,
|
reference_elapsed_ms=elapsed_ms,
|
||||||
):
|
)
|
||||||
|
if not accepted:
|
||||||
|
now = time.monotonic()
|
||||||
|
if now - last_debug_at >= 0.25:
|
||||||
|
self._debug("gate_rejected", {"elapsed_ms": elapsed_ms, "reason": reason})
|
||||||
|
last_debug_at = now
|
||||||
speech_ms = 0
|
speech_ms = 0
|
||||||
candidate_frames = []
|
candidate_frames = []
|
||||||
continue
|
continue
|
||||||
frame_ms = int(frame.metadata.get("duration_ms", 20))
|
frame_ms = int(frame.metadata.get("duration_ms", 20))
|
||||||
speech_ms += frame_ms
|
speech_ms += frame_ms
|
||||||
candidate_frames.append(frame)
|
candidate_frames.append(frame)
|
||||||
|
now = time.monotonic()
|
||||||
|
if now - last_debug_at >= 0.25:
|
||||||
|
self._debug(
|
||||||
|
"speech_accumulating",
|
||||||
|
{"elapsed_ms": elapsed_ms, "speech_ms": speech_ms, "reason": reason},
|
||||||
|
)
|
||||||
|
last_debug_at = now
|
||||||
if speech_ms >= self.min_speech_ms:
|
if speech_ms >= self.min_speech_ms:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self._pending_frames = list(candidate_frames)
|
self._pending_frames = list(candidate_frames)
|
||||||
self._interrupted = True
|
self._interrupted = True
|
||||||
|
self._debug("triggered", {"elapsed_ms": elapsed_ms, "speech_ms": speech_ms})
|
||||||
self.stop_event.set()
|
self.stop_event.set()
|
||||||
return
|
return
|
||||||
except ProviderError as exc:
|
except ProviderError as exc:
|
||||||
self.error = exc
|
self.error = exc
|
||||||
|
self._debug("error", {"message": exc.message, "code": exc.code.value})
|
||||||
|
|
||||||
|
def _debug(self, reason: str, payload: dict[str, Any]) -> None:
|
||||||
|
if self.debug_callback is not None:
|
||||||
|
self.debug_callback(reason, payload)
|
||||||
|
|
||||||
|
|
||||||
def _looks_like_render_echo(
|
def _looks_like_render_echo(
|
||||||
@@ -226,11 +272,9 @@ def _looks_like_render_echo(
|
|||||||
candidate_window = candidate[-width:]
|
candidate_window = candidate[-width:]
|
||||||
render_window = _reference_window(render, width, reference.sample_rate, reference_elapsed_ms)
|
render_window = _reference_window(render, width, reference.sample_rate, reference_elapsed_ms)
|
||||||
corr = _normalized_corr(candidate_window, render_window)
|
corr = _normalized_corr(candidate_window, render_window)
|
||||||
if corr >= 0.92:
|
if corr >= 0.97:
|
||||||
return True
|
return True
|
||||||
render_peak = float(np.max(np.abs(render_window))) if render_window.size else 0.0
|
return False
|
||||||
candidate_peak = float(np.max(np.abs(candidate_window))) if candidate_window.size else 0.0
|
|
||||||
return corr >= 0.82 and candidate_peak <= render_peak * 1.15
|
|
||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ class AppConfig:
|
|||||||
barge_in_assistant_reject_threshold: float = 0.72
|
barge_in_assistant_reject_threshold: float = 0.72
|
||||||
barge_in_listen_interval_ms: int = 20
|
barge_in_listen_interval_ms: int = 20
|
||||||
barge_in_chunk_ms: int = 30
|
barge_in_chunk_ms: int = 30
|
||||||
|
barge_in_debug: bool = False
|
||||||
end_chime_enabled: bool = True
|
end_chime_enabled: bool = True
|
||||||
end_chime_file: Path = Path("assets/sounds/codex-notification.wav")
|
end_chime_file: Path = Path("assets/sounds/codex-notification.wav")
|
||||||
end_chime_frequency_hz: int = 880
|
end_chime_frequency_hz: int = 880
|
||||||
@@ -185,6 +186,7 @@ class AppConfig:
|
|||||||
),
|
),
|
||||||
barge_in_listen_interval_ms=int(get("BARGE_IN_LISTEN_INTERVAL_MS", "20") or "20"),
|
barge_in_listen_interval_ms=int(get("BARGE_IN_LISTEN_INTERVAL_MS", "20") or "20"),
|
||||||
barge_in_chunk_ms=int(get("BARGE_IN_CHUNK_MS", "30") or "30"),
|
barge_in_chunk_ms=int(get("BARGE_IN_CHUNK_MS", "30") or "30"),
|
||||||
|
barge_in_debug=(get("BARGE_IN_DEBUG", "0") or "0").lower() not in {"0", "false", "no"},
|
||||||
end_chime_enabled=(get("END_CHIME_ENABLED", "1") or "1").lower() not in {"0", "false", "no"},
|
end_chime_enabled=(get("END_CHIME_ENABLED", "1") or "1").lower() not in {"0", "false", "no"},
|
||||||
end_chime_file=Path(
|
end_chime_file=Path(
|
||||||
get("END_CHIME_FILE", "assets/sounds/codex-notification.wav")
|
get("END_CHIME_FILE", "assets/sounds/codex-notification.wav")
|
||||||
|
|||||||
@@ -133,6 +133,7 @@ def run_simulated_live(
|
|||||||
vad_end_silence_ms=350,
|
vad_end_silence_ms=350,
|
||||||
vad_no_speech_timeout_ms=3000,
|
vad_no_speech_timeout_ms=3000,
|
||||||
vad_max_recording_ms=6000,
|
vad_max_recording_ms=6000,
|
||||||
|
barge_in_enabled=False,
|
||||||
)
|
)
|
||||||
pipeline = VoiceAssistantPipeline(
|
pipeline = VoiceAssistantPipeline(
|
||||||
config=config,
|
config=config,
|
||||||
|
|||||||
@@ -197,6 +197,49 @@ class BargeInTests(unittest.TestCase):
|
|||||||
|
|
||||||
self.assertFalse(monitor.interrupted)
|
self.assertFalse(monitor.interrupted)
|
||||||
|
|
||||||
|
def test_user_profile_match_can_override_render_echo_candidate(self) -> None:
|
||||||
|
gate = BargeInSpeakerGate(
|
||||||
|
enabled=True,
|
||||||
|
user_similarity_threshold=0.62,
|
||||||
|
assistant_reject_threshold=0.72,
|
||||||
|
min_rms=0.001,
|
||||||
|
)
|
||||||
|
gate.remember_user_segment(segment_for_speaker("owner"))
|
||||||
|
assistant = AudioSegment(b"\x01\x20\x02\x20" * 320, 16000, 1, 0, 40, {"speaker_id": "assistant"})
|
||||||
|
frames = [
|
||||||
|
AudioFrame(
|
||||||
|
assistant.pcm[: 640 * 2],
|
||||||
|
16000,
|
||||||
|
1,
|
||||||
|
20 * idx,
|
||||||
|
idx,
|
||||||
|
{"duration_ms": 20, "speech": True, "speaker_id": "owner"},
|
||||||
|
)
|
||||||
|
for idx in (1, 2)
|
||||||
|
]
|
||||||
|
transport = MemoryAudioTransport(frames)
|
||||||
|
transport.start_input()
|
||||||
|
monitor = AsyncBargeInMonitor(
|
||||||
|
transport=transport,
|
||||||
|
vad_provider=EnergyVadProvider(threshold=1),
|
||||||
|
realtime_stt=None,
|
||||||
|
speaker_gate=gate,
|
||||||
|
assistant_profile=gate.assistant_profile(assistant),
|
||||||
|
assistant_reference=assistant,
|
||||||
|
echo_guard_ms=0,
|
||||||
|
min_speech_ms=40,
|
||||||
|
listen_interval_ms=1,
|
||||||
|
)
|
||||||
|
monitor.vad_provider.load()
|
||||||
|
|
||||||
|
monitor.start()
|
||||||
|
deadline = time.monotonic() + 1
|
||||||
|
while not monitor.interrupted and time.monotonic() < deadline:
|
||||||
|
time.sleep(0.005)
|
||||||
|
monitor.stop()
|
||||||
|
|
||||||
|
self.assertTrue(monitor.interrupted)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -204,7 +204,12 @@ def make_runtime(
|
|||||||
reporter = RecordingReporter()
|
reporter = RecordingReporter()
|
||||||
event_bus = PipelineEventBus()
|
event_bus = PipelineEventBus()
|
||||||
runtime = VoiceAssistantPipeline(
|
runtime = VoiceAssistantPipeline(
|
||||||
config=AppConfig(llm_api_key="secret", speech_provider="cloud", wake_ack_text=wake_ack_text),
|
config=AppConfig(
|
||||||
|
llm_api_key="secret",
|
||||||
|
speech_provider="cloud",
|
||||||
|
wake_ack_text=wake_ack_text,
|
||||||
|
barge_in_enabled=False,
|
||||||
|
),
|
||||||
transport=transport,
|
transport=transport,
|
||||||
wakeword=KeywordWakeWordProvider(),
|
wakeword=KeywordWakeWordProvider(),
|
||||||
vad_recorder=VadRecorder(EnergyVadProvider(), min_duration_ms=40, end_silence_ms=40),
|
vad_recorder=VadRecorder(EnergyVadProvider(), min_duration_ms=40, end_silence_ms=40),
|
||||||
@@ -345,7 +350,12 @@ class LiveRuntimeTests(unittest.TestCase):
|
|||||||
transport = MemoryAudioTransport(frames, flush_clears_input=False)
|
transport = MemoryAudioTransport(frames, flush_clears_input=False)
|
||||||
ack_tts = CountingTtsProvider()
|
ack_tts = CountingTtsProvider()
|
||||||
runtime = VoiceAssistantPipeline(
|
runtime = VoiceAssistantPipeline(
|
||||||
config=AppConfig(llm_api_key="secret", speech_provider="cloud", wake_ack_text="我在"),
|
config=AppConfig(
|
||||||
|
llm_api_key="secret",
|
||||||
|
speech_provider="cloud",
|
||||||
|
wake_ack_text="我在",
|
||||||
|
barge_in_enabled=False,
|
||||||
|
),
|
||||||
transport=transport,
|
transport=transport,
|
||||||
wakeword=KeywordWakeWordProvider(),
|
wakeword=KeywordWakeWordProvider(),
|
||||||
vad_recorder=VadRecorder(EnergyVadProvider(), min_duration_ms=40, end_silence_ms=40),
|
vad_recorder=VadRecorder(EnergyVadProvider(), min_duration_ms=40, end_silence_ms=40),
|
||||||
@@ -363,8 +373,8 @@ class LiveRuntimeTests(unittest.TestCase):
|
|||||||
|
|
||||||
self.assertEqual(summary.completed_turns, 2)
|
self.assertEqual(summary.completed_turns, 2)
|
||||||
self.assertEqual(ack_tts.synthesized_texts, ["我在"])
|
self.assertEqual(ack_tts.synthesized_texts, ["我在"])
|
||||||
self.assertEqual(transport.played_segments[0].metadata["text"], "我在")
|
ack_segments = [segment for segment in transport.played_segments if segment.metadata.get("text") == "我在"]
|
||||||
self.assertEqual(transport.played_segments[3].metadata["text"], "我在")
|
self.assertEqual(len(ack_segments), 2)
|
||||||
|
|
||||||
def test_zero_post_playback_drain_flushes_without_dropping_prefilled_question(self) -> None:
|
def test_zero_post_playback_drain_flushes_without_dropping_prefilled_question(self) -> None:
|
||||||
runtime, stt, _, transport, reporter = make_runtime(["第一问"])
|
runtime, stt, _, transport, reporter = make_runtime(["第一问"])
|
||||||
@@ -387,7 +397,7 @@ class LiveRuntimeTests(unittest.TestCase):
|
|||||||
self.assertEqual(summary.completed_turns, 1)
|
self.assertEqual(summary.completed_turns, 1)
|
||||||
self.assertEqual(reporter.transcripts, ["第一问"])
|
self.assertEqual(reporter.transcripts, ["第一问"])
|
||||||
self.assertEqual(len(stt.calls), 1)
|
self.assertEqual(len(stt.calls), 1)
|
||||||
self.assertEqual(len(transport.played_segments), 2)
|
self.assertGreaterEqual(len(transport.played_segments), 2)
|
||||||
self.assertEqual(transport.flush_count, 2)
|
self.assertEqual(transport.flush_count, 2)
|
||||||
self.assertEqual(transport.played_segments[-1].metadata["chime"], "end")
|
self.assertEqual(transport.played_segments[-1].metadata["chime"], "end")
|
||||||
self.assertEqual(transport.played_segments[-1].metadata["source"], "file")
|
self.assertEqual(transport.played_segments[-1].metadata["source"], "file")
|
||||||
@@ -400,8 +410,8 @@ class LiveRuntimeTests(unittest.TestCase):
|
|||||||
summary = runtime.run(max_turns=1)
|
summary = runtime.run(max_turns=1)
|
||||||
|
|
||||||
self.assertEqual(summary.completed_turns, 1)
|
self.assertEqual(summary.completed_turns, 1)
|
||||||
self.assertEqual(len(transport.played_segments), 1)
|
self.assertGreaterEqual(len(transport.played_segments), 1)
|
||||||
self.assertNotIn("chime", transport.played_segments[-1].metadata)
|
self.assertFalse(any("chime" in segment.metadata for segment in transport.played_segments))
|
||||||
|
|
||||||
def test_temporary_context_is_sent_to_second_llm_call(self) -> None:
|
def test_temporary_context_is_sent_to_second_llm_call(self) -> None:
|
||||||
runtime, _, llm, _, _ = make_runtime(["第一问", "第二问"])
|
runtime, _, llm, _, _ = make_runtime(["第一问", "第二问"])
|
||||||
@@ -587,6 +597,7 @@ class LiveRuntimeTests(unittest.TestCase):
|
|||||||
speech_provider="cloud",
|
speech_provider="cloud",
|
||||||
wake_ack_text="",
|
wake_ack_text="",
|
||||||
followup_listen_timeout_ms=3000,
|
followup_listen_timeout_ms=3000,
|
||||||
|
barge_in_enabled=False,
|
||||||
),
|
),
|
||||||
transport=transport,
|
transport=transport,
|
||||||
wakeword=KeywordWakeWordProvider(),
|
wakeword=KeywordWakeWordProvider(),
|
||||||
@@ -627,6 +638,7 @@ class LiveRuntimeTests(unittest.TestCase):
|
|||||||
speech_provider="cloud",
|
speech_provider="cloud",
|
||||||
wake_ack_text="",
|
wake_ack_text="",
|
||||||
followup_listen_timeout_ms=3000,
|
followup_listen_timeout_ms=3000,
|
||||||
|
barge_in_enabled=False,
|
||||||
),
|
),
|
||||||
transport=transport,
|
transport=transport,
|
||||||
wakeword=KeywordWakeWordProvider(),
|
wakeword=KeywordWakeWordProvider(),
|
||||||
@@ -662,6 +674,7 @@ class LiveRuntimeTests(unittest.TestCase):
|
|||||||
speech_provider="cloud",
|
speech_provider="cloud",
|
||||||
wake_ack_text="",
|
wake_ack_text="",
|
||||||
followup_listen_timeout_ms=3000,
|
followup_listen_timeout_ms=3000,
|
||||||
|
barge_in_enabled=False,
|
||||||
),
|
),
|
||||||
transport=transport,
|
transport=transport,
|
||||||
wakeword=KeywordWakeWordProvider(),
|
wakeword=KeywordWakeWordProvider(),
|
||||||
@@ -771,6 +784,19 @@ class LiveRuntimeTests(unittest.TestCase):
|
|||||||
self.assertEqual(reporter.transcripts[-1], "打断问题")
|
self.assertEqual(reporter.transcripts[-1], "打断问题")
|
||||||
self.assertEqual(len(llm.calls), 2)
|
self.assertEqual(len(llm.calls), 2)
|
||||||
|
|
||||||
|
def test_barge_in_echo_guard_is_capped_and_only_used_once_per_reply(self) -> None:
|
||||||
|
runtime, _, _, _, _ = make_runtime(["第一问"])
|
||||||
|
runtime.controller._barge_in_reply_guard_used = False
|
||||||
|
segment = AudioSegment(b"\x01\x00" * 16000, 16000, 1, 0, 1000)
|
||||||
|
|
||||||
|
first_guard = runtime.controller._effective_barge_in_echo_guard_ms(segment)
|
||||||
|
second_guard = runtime.controller._effective_barge_in_echo_guard_ms(segment)
|
||||||
|
min_speech = runtime.controller._effective_barge_in_min_speech_ms()
|
||||||
|
|
||||||
|
self.assertEqual(first_guard, 120)
|
||||||
|
self.assertEqual(second_guard, 0)
|
||||||
|
self.assertLessEqual(min_speech, runtime.config.interrupt_target_latency_ms)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user