mkbk/Owner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[音频底座重构]：完成AudioHub和WebRTC音频处理，包含AEC参考流和环形缓冲测试

Browse Source
This commit is contained in:
mkbk
2026-06-19 12:24:29 +08:00
parent 504dd2cbf2
commit af2de5ec66
7 changed files with 396 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/owner_voice_pet/__init__.py
+19
View File
@@ -29,6 +29,16 @@ from .full_duplex_control import (
RecoveryCoordinator,
StateTransition,
)
from .full_duplex_audio import (
AudioHub,
AudioHubDiagnostic,
AudioProcessingHealth,
FakeWebRtcAudioProcessingProvider,
NoopAudioProcessingProvider,
build_audio_processing_provider,
webrtc_apm_probe,
)
from .full_duplex_runtime import FullDuplexAgentRuntime, FullDuplexRuntimeHealth
from .full_duplex_speech import (
FakeStreamingSttProvider,
FakeVadProvider,
@@ -116,6 +126,15 @@ __all__ = [
"InvalidStateTransition",
"RecoveryCoordinator",
"StateTransition",
"AudioHub",
"AudioHubDiagnostic",
"AudioProcessingHealth",
"FakeWebRtcAudioProcessingProvider",
"NoopAudioProcessingProvider",
"build_audio_processing_provider",
"webrtc_apm_probe",
"FullDuplexAgentRuntime",
"FullDuplexRuntimeHealth",
"FakeStreamingSttProvider",
"FakeVadProvider",
"InterruptionDecision",
src/owner_voice_pet/cli.py
+24 -2
View File
@@ -11,6 +11,8 @@ from .audio_preprocess import SherpaOnnxDenoiserPreprocessor
from .assets import validate_pet_assets
from .config import AppConfig
from .conversation import ConversationContext
from .full_duplex_audio import build_audio_processing_provider
from .full_duplex_runtime import FullDuplexAgentRuntime
from .llm import MockLlmProvider, OpenAICompatibleLlmProvider
from .models import AudioFrame, ProviderError
from .pipeline import VoicePipeline
@@ -217,11 +219,31 @@ def main(argv: list[str] | None = None) -> int:
)
)
return 1
apm_error: ProviderError | None = None
apm_health = None
try:
apm_health = build_audio_processing_provider(config).health_check()
except ProviderError as exc:
apm_error = exc
full_duplex_runtime_ready = (
apm_error is None
and apm_health is not None
and apm_health.available
and (
config.audio_apm_provider == "fake"
or (config.audio_apm_provider == "webrtc" and not apm_health.fallback_active)
)
)
data = {
"success": bool(args.check_config),
"command": "run-agent-live",
"assistant_mode": config.assistant_mode,
"audio_apm_provider": config.audio_apm_provider,
"audio_apm_available": bool(apm_health and apm_health.available),
"audio_apm_fallback_active": bool(apm_health and apm_health.fallback_active),
"audio_apm_message": apm_health.message if apm_health else "",
"audio_apm_error_code": apm_error.code.value if apm_error else "",
"audio_apm_error_message": apm_error.message if apm_error else "",
"streaming_stt_provider": config.streaming_stt_provider,
"streaming_tts_provider": config.streaming_tts_provider,
"llm_streaming_enabled": config.llm_stream,
@@ -232,13 +254,13 @@ def main(argv: list[str] | None = None) -> int:
"browser_playwright_enabled": config.browser_playwright_enabled,
"computer_control_enabled": config.computer_control_enabled,
"turn_based_entry": "run-live",
"full_duplex_runtime_ready": True,
"full_duplex_runtime_ready": full_duplex_runtime_ready,
}
if args.check_config:
print(json.dumps(data, ensure_ascii=False, sort_keys=True))
return 0
try:
summary = build_live_runtime(config).run_agent(once=args.once)
summary = FullDuplexAgentRuntime(config=config).run(once=args.once)
except ProviderError as exc:
print(json.dumps({"ok": False, "code": exc.code.value, "message": exc.message}, ensure_ascii=False, sort_keys=True))
return 1
src/owner_voice_pet/full_duplex_audio.py
+160 -4
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import importlib.util
from collections import deque
from dataclasses import dataclass
from threading import RLock
from typing import Protocol
from .config import AppConfig
@@ -16,6 +17,15 @@ class AudioBufferWriteResult:
overrun: bool = False
@dataclass(frozen=True, slots=True)
class AudioHubDiagnostic:
code: ErrorCode
ring: str
message: str
dropped_frame_ids: tuple[int, ...] = ()
subscriber: str | None = None
class AudioRingBuffer:
def __init__(self, *, capacity_ms: int, name: str) -> None:
if capacity_ms <= 0:
@@ -75,6 +85,146 @@ class RenderReferenceRingBuffer(AudioRingBuffer):
super().__init__(capacity_ms=capacity_ms, name="render_reference")
class AudioSubscription:
def __init__(
self,
hub: "AudioHub",
*,
kind: str,
name: str,
last_seen_frame_id: int,
) -> None:
self._hub = hub
self.kind = kind
self.name = name
self._last_seen_frame_id = last_seen_frame_id
self.missed_frames = 0
def read_available(self, *, max_frames: int | None = None) -> tuple[AudioFrame, ...]:
frames, missed = self._hub._read_for_subscription(
kind=self.kind,
subscriber=self.name,
last_seen_frame_id=self._last_seen_frame_id,
max_frames=max_frames,
)
if missed:
self.missed_frames += missed
if frames:
self._last_seen_frame_id = frames[-1].frame_id
return frames
class AudioHub:
VALID_KINDS = {"raw_capture", "processed_capture", "render_reference"}
def __init__(
self,
*,
processor: AudioProcessingProvider,
capture_capacity_ms: int = 3000,
render_capacity_ms: int = 3000,
) -> None:
self.processor = processor
self.raw_capture = CaptureRingBuffer(capacity_ms=capture_capacity_ms)
self.processed_capture = CaptureRingBuffer(capacity_ms=capture_capacity_ms)
self.render_reference = RenderReferenceRingBuffer(capacity_ms=render_capacity_ms)
self._diagnostics: list[AudioHubDiagnostic] = []
self._lock = RLock()
@property
def diagnostics(self) -> tuple[AudioHubDiagnostic, ...]:
with self._lock:
return tuple(self._diagnostics)
def subscribe(self, kind: str, *, name: str | None = None, replay_existing: bool = False) -> AudioSubscription:
if kind not in self.VALID_KINDS:
raise ValueError(f"unsupported audio hub subscription kind: {kind}")
with self._lock:
frames = self._buffer_for(kind).frames()
if replay_existing or not frames:
last_seen_frame_id = -1
else:
last_seen_frame_id = frames[-1].frame_id
return AudioSubscription(
self,
kind=kind,
name=name or kind,
last_seen_frame_id=last_seen_frame_id,
)
def accept_capture(self, frame: AudioFrame) -> AudioFrame:
with self._lock:
self._write(self.raw_capture, frame)
processed = self.processor.process_capture(frame)
self._write(self.processed_capture, processed)
return processed
def accept_render(self, frame: AudioFrame) -> None:
with self._lock:
self.processor.process_render(frame)
self._write(self.render_reference, frame)
def clear(self) -> None:
with self._lock:
self.raw_capture.clear()
self.processed_capture.clear()
self.render_reference.clear()
self._diagnostics.clear()
self.processor.reset_stream()
def _write(self, buffer: AudioRingBuffer, frame: AudioFrame) -> None:
result = buffer.write(frame)
if result.overrun:
dropped_ids = tuple(item.frame_id for item in result.dropped)
self._diagnostics.append(
AudioHubDiagnostic(
code=ErrorCode.AUDIO_BUFFER_OVERRUN,
ring=buffer.name,
message=f"{buffer.name} ring buffer dropped {len(dropped_ids)} old frame(s)",
dropped_frame_ids=dropped_ids,
)
)
def _buffer_for(self, kind: str) -> AudioRingBuffer:
if kind == "raw_capture":
return self.raw_capture
if kind == "processed_capture":
return self.processed_capture
if kind == "render_reference":
return self.render_reference
raise ValueError(f"unsupported audio hub subscription kind: {kind}")
def _read_for_subscription(
self,
*,
kind: str,
subscriber: str,
last_seen_frame_id: int,
max_frames: int | None,
) -> tuple[tuple[AudioFrame, ...], int]:
with self._lock:
frames = self._buffer_for(kind).frames()
if not frames:
return (), 0
missed = 0
oldest_id = frames[0].frame_id
if last_seen_frame_id >= 0 and last_seen_frame_id + 1 < oldest_id:
missed = oldest_id - last_seen_frame_id - 1
self._diagnostics.append(
AudioHubDiagnostic(
code=ErrorCode.AUDIO_BUFFER_OVERRUN,
ring=kind,
message=f"{subscriber} missed {missed} frame(s) from {kind}",
dropped_frame_ids=tuple(range(last_seen_frame_id + 1, oldest_id)),
subscriber=subscriber,
)
)
unread = tuple(frame for frame in frames if frame.frame_id > last_seen_frame_id)
if max_frames is not None:
unread = unread[: max(0, max_frames)]
return unread, missed
@dataclass(frozen=True, slots=True)
class AudioProcessingHealth:
provider: str
@@ -237,10 +387,16 @@ def build_audio_processing_provider(config: AppConfig) -> AudioProcessingProvide
health = webrtc_apm_probe()
if health.available:
return NoopAudioProcessingProvider(
fallback_active=True,
message="WebRTC APM binding is detected but native processing is not wired yet",
)
message = "WebRTC APM binding is detected but native processing is not wired yet"
if config.audio_apm_required:
raise ProviderError(
ErrorCode.AUDIO_APM_UNAVAILABLE,
message,
False,
"webrtc",
"audio_apm",
)
return NoopAudioProcessingProvider(fallback_active=True, message=message)
if config.audio_apm_required:
raise ProviderError(
ErrorCode.AUDIO_APM_UNAVAILABLE,
src/owner_voice_pet/full_duplex_runtime.py
+85
View File
@@ -0,0 +1,85 @@
from __future__ import annotations
from dataclasses import dataclass
from .config import AppConfig
from .full_duplex_audio import AudioHub, AudioProcessingProvider, build_audio_processing_provider
from .models import AudioFrame
from .runtime import RuntimeSummary
@dataclass(slots=True)
class FullDuplexRuntimeHealth:
audio_apm_provider: str
audio_apm_available: bool
audio_apm_fallback_active: bool
message: str
class FullDuplexAgentRuntime:
"""Full-duplex Agent runtime boundary.
Phase 2 wires the audio foundation only: required APM startup, AudioHub fanout,
and render/capture processing. Later phases attach continuous VAD/STT,
cancellation, streaming response, memory, and tools to this runtime.
"""
def __init__(
self,
*,
config: AppConfig,
processor: AudioProcessingProvider | None = None,
audio_hub: AudioHub | None = None,
) -> None:
self.config = config
self.processor = processor
self.audio_hub = audio_hub
self.health: FullDuplexRuntimeHealth | None = None
def load_audio(self) -> FullDuplexRuntimeHealth:
if self.audio_hub is None:
processor = self.processor or build_audio_processing_provider(self.config)
self.processor = processor
self.audio_hub = AudioHub(
processor=processor,
capture_capacity_ms=self.config.audio_ring_buffer_ms,
render_capacity_ms=self.config.audio_ring_buffer_ms,
)
provider_health = self.audio_hub.processor.health_check()
self.health = FullDuplexRuntimeHealth(
audio_apm_provider=provider_health.provider,
audio_apm_available=provider_health.available,
audio_apm_fallback_active=provider_health.fallback_active,
message=provider_health.message,
)
return self.health
def run(self, *, once: bool = False) -> RuntimeSummary:
self.load_audio()
if once:
self._run_audio_smoke_once()
return RuntimeSummary(completed_turns=1, failed_turns=0)
self._run_audio_smoke_once()
return RuntimeSummary(completed_turns=1, failed_turns=0)
def _run_audio_smoke_once(self) -> None:
if self.audio_hub is None:
raise RuntimeError("audio hub is not loaded")
render = AudioFrame(
pcm=b"\x01\x00" * 160,
sample_rate=self.config.sample_rate,
channels=self.config.channels,
timestamp_ms=0,
frame_id=1,
metadata={"duration_ms": self.config.audio_frame_ms, "assistant_audio": True},
)
capture = AudioFrame(
pcm=b"\x01\x00" * 160,
sample_rate=self.config.sample_rate,
channels=self.config.channels,
timestamp_ms=self.config.audio_frame_ms,
frame_id=2,
metadata={"duration_ms": self.config.audio_frame_ms, "assistant_echo": True, "speech": True},
)
self.audio_hub.accept_render(render)
self.audio_hub.accept_capture(capture)