[真实全双工运行时]:完成run-agent-live可打断语音闭环,包含持续监听、软件回声抑制和播放取消验证
This commit is contained in:
@@ -147,6 +147,26 @@ class TurnController:
|
||||
except ProviderError as exc:
|
||||
return self._recover(exc, turn_id)
|
||||
|
||||
def run_agent_turn(self, turn_id: int) -> TurnResult:
|
||||
self._states = []
|
||||
try:
|
||||
self._event(
|
||||
WAKE_LISTENING,
|
||||
PipelineState.LISTENING,
|
||||
"监听中:请直接说话",
|
||||
turn_id=turn_id,
|
||||
)
|
||||
user_text = self._capture_and_transcribe(
|
||||
turn_id,
|
||||
state_message="录音中:正在听取问题",
|
||||
no_speech_timeout_ms=60 * 60 * 1000,
|
||||
)
|
||||
if isinstance(user_text, ProviderError):
|
||||
return self._recover(user_text, turn_id)
|
||||
return self._reply_to_user(user_text, turn_id)
|
||||
except ProviderError as exc:
|
||||
return self._recover(exc, turn_id)
|
||||
|
||||
def prepare_ack_audio(self) -> None:
|
||||
text = self.config.wake_ack_text.strip()
|
||||
if not text:
|
||||
@@ -493,7 +513,13 @@ class TurnController:
|
||||
def _end_conversation(self, turn_id: int, *, payload: dict[str, object] | None = None) -> None:
|
||||
self._event(CONTINUOUS_SESSION_ENDED, PipelineState.WAKE_LISTENING, "", turn_id=turn_id, payload=payload)
|
||||
self._play_end_chime()
|
||||
self._event(STANDBY_RESUMED, PipelineState.WAKE_LISTENING, "恢复待机:可继续唤醒", turn_id=turn_id)
|
||||
message = (
|
||||
"恢复监听:可直接说话"
|
||||
if self.config.assistant_mode == "full_duplex_agent"
|
||||
else "恢复待机:可继续唤醒"
|
||||
)
|
||||
state = PipelineState.LISTENING if self.config.assistant_mode == "full_duplex_agent" else PipelineState.WAKE_LISTENING
|
||||
self._event(STANDBY_RESUMED, state, message, turn_id=turn_id)
|
||||
|
||||
def _play_end_chime(self) -> None:
|
||||
if not self.config.end_chime_enabled:
|
||||
@@ -545,6 +571,7 @@ class TurnController:
|
||||
realtime_stt=self.realtime_stt,
|
||||
speaker_gate=self._barge_in_gate,
|
||||
assistant_profile=self._barge_in_gate.assistant_profile(segment),
|
||||
assistant_reference=segment,
|
||||
echo_guard_ms=self.config.barge_in_echo_guard_ms,
|
||||
min_speech_ms=self.config.barge_in_min_speech_ms,
|
||||
listen_interval_ms=self.config.barge_in_listen_interval_ms,
|
||||
@@ -697,9 +724,44 @@ class VoiceAssistantPipeline:
|
||||
self.shutdown()
|
||||
return RuntimeSummary(completed, failed, last_error=last_error)
|
||||
|
||||
def run_agent(self, *, once: bool = False, max_turns: int | None = None) -> RuntimeSummary:
|
||||
completed = 0
|
||||
failed = 0
|
||||
last_error: ProviderError | None = None
|
||||
self.load()
|
||||
self.transport.start_input(
|
||||
device_id=self.config.audio_input_device,
|
||||
sample_rate=self.config.sample_rate,
|
||||
channels=self.config.channels,
|
||||
)
|
||||
try:
|
||||
while True:
|
||||
turn_id = completed + failed + 1
|
||||
result = self.run_agent_turn(turn_id)
|
||||
completed += result.completed_turns
|
||||
if result.success:
|
||||
completed += 0 if result.completed_turns else 1
|
||||
else:
|
||||
failed += result.failed_turns or 1
|
||||
last_error = result.error
|
||||
if once:
|
||||
break
|
||||
if once and completed >= 1:
|
||||
break
|
||||
if max_turns is not None and completed >= max_turns:
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
return RuntimeSummary(completed, failed, interrupted=True, last_error=last_error)
|
||||
finally:
|
||||
self.shutdown()
|
||||
return RuntimeSummary(completed, failed, last_error=last_error)
|
||||
|
||||
def run_turn(self, turn_id: int) -> TurnResult:
|
||||
return self.controller.run_turn(turn_id)
|
||||
|
||||
def run_agent_turn(self, turn_id: int) -> TurnResult:
|
||||
return self.controller.run_agent_turn(turn_id)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.transport.stop()
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import math
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
@@ -11,8 +12,7 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .models import AudioFrame, AudioSegment, ProviderError
|
||||
from .protocols import AudioTransport, RealtimeSttProvider, RealtimeTranscriptSession
|
||||
from .stt import is_valid_transcript_text
|
||||
from .protocols import AudioTransport, RealtimeSttProvider
|
||||
from .vad import cosine_similarity, extract_timbre_vector
|
||||
|
||||
|
||||
@@ -52,16 +52,30 @@ class BargeInSpeakerGate:
|
||||
def assistant_profile(self, segment: AudioSegment) -> TimbreProfile:
|
||||
return self._profile_from_segment(segment)
|
||||
|
||||
def accepts_candidate(self, frame: AudioFrame, assistant_profile: TimbreProfile) -> bool:
|
||||
def accepts_candidate(
|
||||
self,
|
||||
frame: AudioFrame,
|
||||
assistant_profile: TimbreProfile,
|
||||
assistant_reference: AudioSegment | None = None,
|
||||
reference_elapsed_ms: int | None = None,
|
||||
) -> bool:
|
||||
if not self.enabled:
|
||||
return True
|
||||
if assistant_reference is not None and _looks_like_render_echo(
|
||||
frame,
|
||||
assistant_reference,
|
||||
reference_elapsed_ms=reference_elapsed_ms,
|
||||
):
|
||||
return False
|
||||
candidate = self._profile_from_frame(frame)
|
||||
if not candidate.ready:
|
||||
return False
|
||||
return True
|
||||
if self._matches(candidate, assistant_profile, self.assistant_reject_threshold):
|
||||
return False
|
||||
if self._user_profile.ready:
|
||||
return self._matches(candidate, self._user_profile, self.user_similarity_threshold)
|
||||
if self._matches(candidate, self._user_profile, self.user_similarity_threshold):
|
||||
return True
|
||||
return True
|
||||
return True
|
||||
|
||||
def _profile_from_segment(self, segment: AudioSegment) -> TimbreProfile:
|
||||
@@ -99,6 +113,7 @@ class AsyncBargeInMonitor:
|
||||
realtime_stt: RealtimeSttProvider | None,
|
||||
speaker_gate: BargeInSpeakerGate,
|
||||
assistant_profile: TimbreProfile,
|
||||
assistant_reference: AudioSegment | None = None,
|
||||
echo_guard_ms: int,
|
||||
min_speech_ms: int,
|
||||
listen_interval_ms: int,
|
||||
@@ -108,6 +123,7 @@ class AsyncBargeInMonitor:
|
||||
self.realtime_stt = realtime_stt
|
||||
self.speaker_gate = speaker_gate
|
||||
self.assistant_profile = assistant_profile
|
||||
self.assistant_reference = assistant_reference
|
||||
self.echo_guard_ms = max(0, echo_guard_ms)
|
||||
self.min_speech_ms = max(0, min_speech_ms)
|
||||
self.listen_interval_ms = max(1, listen_interval_ms)
|
||||
@@ -141,10 +157,8 @@ class AsyncBargeInMonitor:
|
||||
return list(self._pending_frames)
|
||||
|
||||
def _run(self) -> None:
|
||||
realtime_session = self.realtime_stt.start_stream() if self.realtime_stt is not None else None
|
||||
started_at = time.monotonic()
|
||||
speech_ms = 0
|
||||
partial_seen = False
|
||||
candidate_frames: list[AudioFrame] = []
|
||||
try:
|
||||
while not self._shutdown_event.is_set() and not self.stop_event.is_set():
|
||||
@@ -161,18 +175,19 @@ class AsyncBargeInMonitor:
|
||||
speech_ms = 0
|
||||
candidate_frames = []
|
||||
continue
|
||||
if not self.speaker_gate.accepts_candidate(frame, self.assistant_profile):
|
||||
if not self.speaker_gate.accepts_candidate(
|
||||
frame,
|
||||
self.assistant_profile,
|
||||
self.assistant_reference,
|
||||
reference_elapsed_ms=elapsed_ms,
|
||||
):
|
||||
speech_ms = 0
|
||||
candidate_frames = []
|
||||
continue
|
||||
frame_ms = int(frame.metadata.get("duration_ms", 20))
|
||||
speech_ms += frame_ms
|
||||
candidate_frames.append(frame)
|
||||
if realtime_session is not None:
|
||||
transcript = realtime_session.accept_frame(frame)
|
||||
if transcript is not None and is_valid_transcript_text(transcript.normalized_text):
|
||||
partial_seen = True
|
||||
if speech_ms >= self.min_speech_ms and partial_seen:
|
||||
if speech_ms >= self.min_speech_ms:
|
||||
with self._lock:
|
||||
self._pending_frames = list(candidate_frames)
|
||||
self._interrupted = True
|
||||
@@ -180,9 +195,67 @@ class AsyncBargeInMonitor:
|
||||
return
|
||||
except ProviderError as exc:
|
||||
self.error = exc
|
||||
finally:
|
||||
if realtime_session is not None:
|
||||
realtime_session.finish()
|
||||
|
||||
|
||||
def _looks_like_render_echo(
|
||||
frame: AudioFrame,
|
||||
reference: AudioSegment,
|
||||
*,
|
||||
reference_elapsed_ms: int | None = None,
|
||||
) -> bool:
|
||||
if frame.metadata.get("speaker_id") == "assistant":
|
||||
return True
|
||||
if not frame.pcm or not reference.pcm:
|
||||
return False
|
||||
try:
|
||||
import numpy as np
|
||||
|
||||
candidate = np.frombuffer(frame.pcm, dtype=np.int16).astype(np.float32)
|
||||
render = np.frombuffer(reference.pcm, dtype=np.int16).astype(np.float32)
|
||||
if candidate.size == 0 or render.size == 0:
|
||||
return False
|
||||
if frame.channels > 1:
|
||||
candidate = candidate.reshape(-1, frame.channels).mean(axis=1)
|
||||
if reference.channels > 1:
|
||||
render = render.reshape(-1, reference.channels).mean(axis=1)
|
||||
candidate_rms = float(math.sqrt(float(np.mean(candidate * candidate)))) if candidate.size else 0.0
|
||||
render_rms = float(math.sqrt(float(np.mean(render * render)))) if render.size else 0.0
|
||||
if candidate_rms <= 1.0 or render_rms <= 1.0:
|
||||
return False
|
||||
width = min(candidate.size, render.size)
|
||||
candidate_window = candidate[-width:]
|
||||
render_window = _reference_window(render, width, reference.sample_rate, reference_elapsed_ms)
|
||||
corr = _normalized_corr(candidate_window, render_window)
|
||||
if corr >= 0.92:
|
||||
return True
|
||||
render_peak = float(np.max(np.abs(render_window))) if render_window.size else 0.0
|
||||
candidate_peak = float(np.max(np.abs(candidate_window))) if candidate_window.size else 0.0
|
||||
return corr >= 0.82 and candidate_peak <= render_peak * 1.15
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _reference_window(render: Any, width: int, sample_rate: int, elapsed_ms: int | None) -> Any:
|
||||
if elapsed_ms is None:
|
||||
return render[:width]
|
||||
center = int(max(0, elapsed_ms) * max(1, sample_rate) / 1000)
|
||||
start = max(0, min(max(0, render.size - width), center - width))
|
||||
return render[start : start + width]
|
||||
|
||||
|
||||
def _normalized_corr(left: Any, right: Any) -> float:
|
||||
try:
|
||||
import numpy as np
|
||||
|
||||
left = left - np.mean(left)
|
||||
right = right - np.mean(right)
|
||||
numerator = float(np.dot(left, right))
|
||||
denominator = float(np.linalg.norm(left) * np.linalg.norm(right))
|
||||
if denominator <= 1e-9:
|
||||
return 0.0
|
||||
return numerator / denominator
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
def ensure_interruptible_pcm(segment: AudioSegment) -> AudioSegment | None:
|
||||
|
||||
@@ -43,6 +43,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
help="Run planned full-duplex Agent voice assistant entry point",
|
||||
)
|
||||
agent_live.add_argument("--check-config", action="store_true", help="Validate and print full-duplex Agent config")
|
||||
agent_live.add_argument("--once", action="store_true", help="Run one completed full-duplex Agent turn and exit")
|
||||
simulate = subparsers.add_parser("simulate-live", help="Run live pipeline with simulated microphone frames")
|
||||
simulate.add_argument("--turns", type=int, default=2, help="Number of simulated turns. Default: 2")
|
||||
simulate.add_argument("--fixture", default=None, help="Replay simulated microphone frames from JSONL")
|
||||
@@ -231,16 +232,17 @@ def main(argv: list[str] | None = None) -> int:
|
||||
"browser_playwright_enabled": config.browser_playwright_enabled,
|
||||
"computer_control_enabled": config.computer_control_enabled,
|
||||
"turn_based_entry": "run-live",
|
||||
"full_duplex_runtime_ready": False,
|
||||
"full_duplex_runtime_ready": True,
|
||||
}
|
||||
if not args.check_config:
|
||||
data["code"] = "FULL_DUPLEX_RUNTIME_NOT_IMPLEMENTED"
|
||||
data["message"] = (
|
||||
"run-agent-live is the reserved full-duplex Agent entry point; "
|
||||
"runtime wiring will be implemented by later OpenSpec tasks."
|
||||
)
|
||||
print(json.dumps(data, ensure_ascii=False, sort_keys=True))
|
||||
return 0 if args.check_config else 1
|
||||
if args.check_config:
|
||||
print(json.dumps(data, ensure_ascii=False, sort_keys=True))
|
||||
return 0
|
||||
try:
|
||||
summary = build_live_runtime(config).run_agent(once=args.once)
|
||||
except ProviderError as exc:
|
||||
print(json.dumps({"ok": False, "code": exc.code.value, "message": exc.message}, ensure_ascii=False, sort_keys=True))
|
||||
return 1
|
||||
return 0 if summary.completed_turns > 0 or summary.interrupted else 1
|
||||
|
||||
if args.command == "simulate-live":
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user