[真实全双工运行时]:完成run-agent-live可打断语音闭环,包含持续监听、软件回声抑制和播放取消验证

This commit is contained in:
mkbk
2026-06-18 22:56:01 +08:00
parent e730883c64
commit a489f8eee1
11 changed files with 378 additions and 46 deletions
+63 -1
View File
@@ -147,6 +147,26 @@ class TurnController:
except ProviderError as exc:
return self._recover(exc, turn_id)
def run_agent_turn(self, turn_id: int) -> TurnResult:
self._states = []
try:
self._event(
WAKE_LISTENING,
PipelineState.LISTENING,
"监听中:请直接说话",
turn_id=turn_id,
)
user_text = self._capture_and_transcribe(
turn_id,
state_message="录音中:正在听取问题",
no_speech_timeout_ms=60 * 60 * 1000,
)
if isinstance(user_text, ProviderError):
return self._recover(user_text, turn_id)
return self._reply_to_user(user_text, turn_id)
except ProviderError as exc:
return self._recover(exc, turn_id)
def prepare_ack_audio(self) -> None:
text = self.config.wake_ack_text.strip()
if not text:
@@ -493,7 +513,13 @@ class TurnController:
def _end_conversation(self, turn_id: int, *, payload: dict[str, object] | None = None) -> None:
self._event(CONTINUOUS_SESSION_ENDED, PipelineState.WAKE_LISTENING, "", turn_id=turn_id, payload=payload)
self._play_end_chime()
self._event(STANDBY_RESUMED, PipelineState.WAKE_LISTENING, "恢复待机:可继续唤醒", turn_id=turn_id)
message = (
"恢复监听:可直接说话"
if self.config.assistant_mode == "full_duplex_agent"
else "恢复待机:可继续唤醒"
)
state = PipelineState.LISTENING if self.config.assistant_mode == "full_duplex_agent" else PipelineState.WAKE_LISTENING
self._event(STANDBY_RESUMED, state, message, turn_id=turn_id)
def _play_end_chime(self) -> None:
if not self.config.end_chime_enabled:
@@ -545,6 +571,7 @@ class TurnController:
realtime_stt=self.realtime_stt,
speaker_gate=self._barge_in_gate,
assistant_profile=self._barge_in_gate.assistant_profile(segment),
assistant_reference=segment,
echo_guard_ms=self.config.barge_in_echo_guard_ms,
min_speech_ms=self.config.barge_in_min_speech_ms,
listen_interval_ms=self.config.barge_in_listen_interval_ms,
@@ -697,9 +724,44 @@ class VoiceAssistantPipeline:
self.shutdown()
return RuntimeSummary(completed, failed, last_error=last_error)
def run_agent(self, *, once: bool = False, max_turns: int | None = None) -> RuntimeSummary:
completed = 0
failed = 0
last_error: ProviderError | None = None
self.load()
self.transport.start_input(
device_id=self.config.audio_input_device,
sample_rate=self.config.sample_rate,
channels=self.config.channels,
)
try:
while True:
turn_id = completed + failed + 1
result = self.run_agent_turn(turn_id)
completed += result.completed_turns
if result.success:
completed += 0 if result.completed_turns else 1
else:
failed += result.failed_turns or 1
last_error = result.error
if once:
break
if once and completed >= 1:
break
if max_turns is not None and completed >= max_turns:
break
except KeyboardInterrupt:
return RuntimeSummary(completed, failed, interrupted=True, last_error=last_error)
finally:
self.shutdown()
return RuntimeSummary(completed, failed, last_error=last_error)
def run_turn(self, turn_id: int) -> TurnResult:
return self.controller.run_turn(turn_id)
def run_agent_turn(self, turn_id: int) -> TurnResult:
return self.controller.run_agent_turn(turn_id)
def shutdown(self) -> None:
self.transport.stop()
+89 -16
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import io
import math
import subprocess
import tempfile
import threading
@@ -11,8 +12,7 @@ from pathlib import Path
from typing import Any
from .models import AudioFrame, AudioSegment, ProviderError
from .protocols import AudioTransport, RealtimeSttProvider, RealtimeTranscriptSession
from .stt import is_valid_transcript_text
from .protocols import AudioTransport, RealtimeSttProvider
from .vad import cosine_similarity, extract_timbre_vector
@@ -52,16 +52,30 @@ class BargeInSpeakerGate:
def assistant_profile(self, segment: AudioSegment) -> TimbreProfile:
return self._profile_from_segment(segment)
def accepts_candidate(self, frame: AudioFrame, assistant_profile: TimbreProfile) -> bool:
def accepts_candidate(
self,
frame: AudioFrame,
assistant_profile: TimbreProfile,
assistant_reference: AudioSegment | None = None,
reference_elapsed_ms: int | None = None,
) -> bool:
if not self.enabled:
return True
if assistant_reference is not None and _looks_like_render_echo(
frame,
assistant_reference,
reference_elapsed_ms=reference_elapsed_ms,
):
return False
candidate = self._profile_from_frame(frame)
if not candidate.ready:
return False
return True
if self._matches(candidate, assistant_profile, self.assistant_reject_threshold):
return False
if self._user_profile.ready:
return self._matches(candidate, self._user_profile, self.user_similarity_threshold)
if self._matches(candidate, self._user_profile, self.user_similarity_threshold):
return True
return True
return True
def _profile_from_segment(self, segment: AudioSegment) -> TimbreProfile:
@@ -99,6 +113,7 @@ class AsyncBargeInMonitor:
realtime_stt: RealtimeSttProvider | None,
speaker_gate: BargeInSpeakerGate,
assistant_profile: TimbreProfile,
assistant_reference: AudioSegment | None = None,
echo_guard_ms: int,
min_speech_ms: int,
listen_interval_ms: int,
@@ -108,6 +123,7 @@ class AsyncBargeInMonitor:
self.realtime_stt = realtime_stt
self.speaker_gate = speaker_gate
self.assistant_profile = assistant_profile
self.assistant_reference = assistant_reference
self.echo_guard_ms = max(0, echo_guard_ms)
self.min_speech_ms = max(0, min_speech_ms)
self.listen_interval_ms = max(1, listen_interval_ms)
@@ -141,10 +157,8 @@ class AsyncBargeInMonitor:
return list(self._pending_frames)
def _run(self) -> None:
realtime_session = self.realtime_stt.start_stream() if self.realtime_stt is not None else None
started_at = time.monotonic()
speech_ms = 0
partial_seen = False
candidate_frames: list[AudioFrame] = []
try:
while not self._shutdown_event.is_set() and not self.stop_event.is_set():
@@ -161,18 +175,19 @@ class AsyncBargeInMonitor:
speech_ms = 0
candidate_frames = []
continue
if not self.speaker_gate.accepts_candidate(frame, self.assistant_profile):
if not self.speaker_gate.accepts_candidate(
frame,
self.assistant_profile,
self.assistant_reference,
reference_elapsed_ms=elapsed_ms,
):
speech_ms = 0
candidate_frames = []
continue
frame_ms = int(frame.metadata.get("duration_ms", 20))
speech_ms += frame_ms
candidate_frames.append(frame)
if realtime_session is not None:
transcript = realtime_session.accept_frame(frame)
if transcript is not None and is_valid_transcript_text(transcript.normalized_text):
partial_seen = True
if speech_ms >= self.min_speech_ms and partial_seen:
if speech_ms >= self.min_speech_ms:
with self._lock:
self._pending_frames = list(candidate_frames)
self._interrupted = True
@@ -180,9 +195,67 @@ class AsyncBargeInMonitor:
return
except ProviderError as exc:
self.error = exc
finally:
if realtime_session is not None:
realtime_session.finish()
def _looks_like_render_echo(
frame: AudioFrame,
reference: AudioSegment,
*,
reference_elapsed_ms: int | None = None,
) -> bool:
if frame.metadata.get("speaker_id") == "assistant":
return True
if not frame.pcm or not reference.pcm:
return False
try:
import numpy as np
candidate = np.frombuffer(frame.pcm, dtype=np.int16).astype(np.float32)
render = np.frombuffer(reference.pcm, dtype=np.int16).astype(np.float32)
if candidate.size == 0 or render.size == 0:
return False
if frame.channels > 1:
candidate = candidate.reshape(-1, frame.channels).mean(axis=1)
if reference.channels > 1:
render = render.reshape(-1, reference.channels).mean(axis=1)
candidate_rms = float(math.sqrt(float(np.mean(candidate * candidate)))) if candidate.size else 0.0
render_rms = float(math.sqrt(float(np.mean(render * render)))) if render.size else 0.0
if candidate_rms <= 1.0 or render_rms <= 1.0:
return False
width = min(candidate.size, render.size)
candidate_window = candidate[-width:]
render_window = _reference_window(render, width, reference.sample_rate, reference_elapsed_ms)
corr = _normalized_corr(candidate_window, render_window)
if corr >= 0.92:
return True
render_peak = float(np.max(np.abs(render_window))) if render_window.size else 0.0
candidate_peak = float(np.max(np.abs(candidate_window))) if candidate_window.size else 0.0
return corr >= 0.82 and candidate_peak <= render_peak * 1.15
except Exception:
return False
def _reference_window(render: Any, width: int, sample_rate: int, elapsed_ms: int | None) -> Any:
if elapsed_ms is None:
return render[:width]
center = int(max(0, elapsed_ms) * max(1, sample_rate) / 1000)
start = max(0, min(max(0, render.size - width), center - width))
return render[start : start + width]
def _normalized_corr(left: Any, right: Any) -> float:
try:
import numpy as np
left = left - np.mean(left)
right = right - np.mean(right)
numerator = float(np.dot(left, right))
denominator = float(np.linalg.norm(left) * np.linalg.norm(right))
if denominator <= 1e-9:
return 0.0
return numerator / denominator
except Exception:
return 0.0
def ensure_interruptible_pcm(segment: AudioSegment) -> AudioSegment | None:
+11 -9
View File
@@ -43,6 +43,7 @@ def main(argv: list[str] | None = None) -> int:
help="Run planned full-duplex Agent voice assistant entry point",
)
agent_live.add_argument("--check-config", action="store_true", help="Validate and print full-duplex Agent config")
agent_live.add_argument("--once", action="store_true", help="Run one completed full-duplex Agent turn and exit")
simulate = subparsers.add_parser("simulate-live", help="Run live pipeline with simulated microphone frames")
simulate.add_argument("--turns", type=int, default=2, help="Number of simulated turns. Default: 2")
simulate.add_argument("--fixture", default=None, help="Replay simulated microphone frames from JSONL")
@@ -231,16 +232,17 @@ def main(argv: list[str] | None = None) -> int:
"browser_playwright_enabled": config.browser_playwright_enabled,
"computer_control_enabled": config.computer_control_enabled,
"turn_based_entry": "run-live",
"full_duplex_runtime_ready": False,
"full_duplex_runtime_ready": True,
}
if not args.check_config:
data["code"] = "FULL_DUPLEX_RUNTIME_NOT_IMPLEMENTED"
data["message"] = (
"run-agent-live is the reserved full-duplex Agent entry point; "
"runtime wiring will be implemented by later OpenSpec tasks."
)
print(json.dumps(data, ensure_ascii=False, sort_keys=True))
return 0 if args.check_config else 1
if args.check_config:
print(json.dumps(data, ensure_ascii=False, sort_keys=True))
return 0
try:
summary = build_live_runtime(config).run_agent(once=args.once)
except ProviderError as exc:
print(json.dumps({"ok": False, "code": exc.code.value, "message": exc.message}, ensure_ascii=False, sort_keys=True))
return 1
return 0 if summary.completed_turns > 0 or summary.interrupted else 1
if args.command == "simulate-live":
try: