mkbk/Owner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[主说话人端点]：完成音色消失结束录音，包含临时音色画像、端点配置和回归测试

Browse Source
This commit is contained in:
mkbk
2026-06-17 21:33:33 +08:00
parent f9da304568
commit da91be6e5c
8 changed files with 372 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/owner_voice_pet/__init__.py
+2 -1
View File
@@ -19,7 +19,7 @@ from .models import (
)
from .transport import AudioRingBuffer, FileReplayTransport, MemoryAudioTransport
from .wakeword import KeywordWakeWordProvider, SherpaOnnxKeywordWakeWordProvider
from .vad import EnergyVadProvider, HybridVadProvider, VadRecorder
from .vad import EnergyVadProvider, HybridVadProvider, PrimarySpeakerVadRecorder, VadRecorder
from .stt import CloudAsrSttProvider, MetadataSttProvider, SherpaOnnxSttProvider, is_valid_transcript_text
from .conversation import ConversationContext
from .llm import MockLlmProvider, OpenAICompatibleLlmProvider
@@ -45,6 +45,7 @@ __all__ = [
"SherpaOnnxKeywordWakeWordProvider",
"EnergyVadProvider",
"HybridVadProvider",
"PrimarySpeakerVadRecorder",
"VadRecorder",
"CloudAsrSttProvider",
"MetadataSttProvider",
src/owner_voice_pet/config.py
+80
View File
@@ -26,6 +26,12 @@ class AppConfig:
wake_kws_score: float = 1.0
wake_ack_text: str = "我在"
post_playback_drain_ms: int = 50
pipeline_mode: str = "live_turn_based"
endpoint_mode: str = "primary_speaker"
speaker_profile_ms: int = 600
speaker_absent_ms: int = 300
speaker_similarity_threshold: float = 0.70
speaker_min_rms: float = 0.012
vad_provider: str = "hybrid"
vad_threshold: float = 0.5
vad_min_duration_ms: int = 250
@@ -37,6 +43,7 @@ class AppConfig:
tts_model: str = "mimo-v2.5-tts"
tts_voice: str = "mimo_default"
speech_models_dir: Path = Path("models")
context_mode: str = "session_memory"
context_max_messages: int = 12
context_max_chars: int = 12000
@@ -67,6 +74,14 @@ class AppConfig:
wake_kws_score=float(get("WAKE_KWS_SCORE", "1.0") or "1.0"),
wake_ack_text=get("WAKE_ACK_TEXT", "我在") or "我在",
post_playback_drain_ms=int(get("POST_PLAYBACK_DRAIN_MS", "50") or "50"),
pipeline_mode=(get("PIPELINE_MODE", "live_turn_based") or "live_turn_based").lower(),
endpoint_mode=(get("ENDPOINT_MODE", "primary_speaker") or "primary_speaker").lower(),
speaker_profile_ms=int(get("SPEAKER_PROFILE_MS", "600") or "600"),
speaker_absent_ms=int(get("SPEAKER_ABSENT_MS", "300") or "300"),
speaker_similarity_threshold=float(
get("SPEAKER_SIMILARITY_THRESHOLD", "0.70") or "0.70"
),
speaker_min_rms=float(get("SPEAKER_MIN_RMS", "0.012") or "0.012"),
vad_provider=(get("VAD_PROVIDER", "hybrid") or "hybrid").lower(),
vad_threshold=float(get("VAD_THRESHOLD", "0.5") or "0.5"),
vad_min_duration_ms=int(get("VAD_MIN_DURATION_MS", "250") or "250"),
@@ -78,6 +93,7 @@ class AppConfig:
tts_model=get("TTS_MODEL", "mimo-v2.5-tts") or "mimo-v2.5-tts",
tts_voice=get("TTS_VOICE", "mimo_default") or "mimo_default",
speech_models_dir=Path(get("SPEECH_MODELS_DIR", "models") or "models"),
context_mode=(get("CONTEXT_MODE", "session_memory") or "session_memory").lower(),
context_max_messages=int(get("CONTEXT_MAX_MESSAGES", "12") or "12"),
context_max_chars=int(get("CONTEXT_MAX_CHARS", "12000") or "12000"),
)
@@ -178,6 +194,60 @@ class AppConfig:
"startup",
)
)
if self.pipeline_mode not in {"live_turn_based"}:
errors.append(
ProviderError(
ErrorCode.CONFIG_MISSING_VALUE,
"OWNER_PIPELINE_MODE must be live_turn_based",
False,
"config",
"startup",
)
)
if self.endpoint_mode not in {"primary_speaker", "vad"}:
errors.append(
ProviderError(
ErrorCode.CONFIG_MISSING_VALUE,
"OWNER_ENDPOINT_MODE must be primary_speaker or vad",
False,
"config",
"startup",
)
)
for name, value in {
"OWNER_SPEAKER_PROFILE_MS": self.speaker_profile_ms,
"OWNER_SPEAKER_ABSENT_MS": self.speaker_absent_ms,
}.items():
if value <= 0:
errors.append(
ProviderError(
ErrorCode.CONFIG_MISSING_VALUE,
f"{name} must be positive",
False,
"config",
"startup",
)
)
if not 0 < self.speaker_similarity_threshold <= 1:
errors.append(
ProviderError(
ErrorCode.CONFIG_MISSING_VALUE,
"OWNER_SPEAKER_SIMILARITY_THRESHOLD must be in (0, 1]",
False,
"config",
"startup",
)
)
if self.speaker_min_rms <= 0:
errors.append(
ProviderError(
ErrorCode.CONFIG_MISSING_VALUE,
"OWNER_SPEAKER_MIN_RMS must be positive",
False,
"config",
"startup",
)
)
if self.vad_provider not in {"hybrid", "local", "energy"}:
errors.append(
ProviderError(
@@ -224,6 +294,16 @@ class AppConfig:
"startup",
)
)
if self.context_mode not in {"session_memory"}:
errors.append(
ProviderError(
ErrorCode.CONFIG_MISSING_VALUE,
"OWNER_CONTEXT_MODE must be session_memory",
False,
"config",
"startup",
)
)
return errors
def api_url(self, path: str) -> str:
src/owner_voice_pet/runtime.py
+19 -8
View File
@@ -33,7 +33,7 @@ from .protocols import AudioTransport, LlmProvider, SttProvider, TtsProvider, Wa
from .stt import CloudAsrSttProvider, SherpaOnnxSttProvider, is_valid_transcript_text
from .transport import SoundDeviceAudioTransport
from .tts import CloudTtsProvider, MacSayTtsProvider, SentenceBuffer
from .vad import EnergyVadProvider, HybridVadProvider, SherpaOnnxVadProvider, VadRecorder
from .vad import EnergyVadProvider, HybridVadProvider, PrimarySpeakerVadRecorder, SherpaOnnxVadProvider, VadRecorder
from .wakeword import SherpaOnnxKeywordWakeWordProvider
@@ -345,6 +345,23 @@ def build_live_runtime(config: AppConfig, reporter: RuntimeReporter | None = Non
vad_provider = SherpaOnnxVadProvider(config.speech_models_dir, threshold=config.vad_threshold)
else:
vad_provider = EnergyVadProvider()
recorder_cls = PrimarySpeakerVadRecorder if config.endpoint_mode == "primary_speaker" else VadRecorder
recorder_kwargs = {
"provider": vad_provider,
"min_duration_ms": config.vad_min_duration_ms,
"end_silence_ms": config.vad_end_silence_ms,
"no_speech_timeout_ms": config.vad_no_speech_timeout_ms,
"max_recording_ms": config.vad_max_recording_ms,
}
if recorder_cls is PrimarySpeakerVadRecorder:
recorder_kwargs.update(
{
"speaker_profile_ms": config.speaker_profile_ms,
"speaker_absent_ms": config.speaker_absent_ms,
"similarity_threshold": config.speaker_similarity_threshold,
"min_rms": config.speaker_min_rms,
}
)
return VoiceAssistantPipeline(
config=config,
transport=SoundDeviceAudioTransport(output_device=config.audio_output_device),
@@ -355,13 +372,7 @@ def build_live_runtime(config: AppConfig, reporter: RuntimeReporter | None = Non
threshold=config.wake_kws_threshold,
score=config.wake_kws_score,
),
vad_recorder=VadRecorder(
vad_provider,
min_duration_ms=config.vad_min_duration_ms,
end_silence_ms=config.vad_end_silence_ms,
no_speech_timeout_ms=config.vad_no_speech_timeout_ms,
max_recording_ms=config.vad_max_recording_ms,
),
vad_recorder=recorder_cls(**recorder_kwargs),
stt=stt,
llm=OpenAICompatibleLlmProvider(config),
tts=tts,
src/owner_voice_pet/vad.py
+170
View File
@@ -204,6 +204,13 @@ class HybridVadProvider:
self.fallback.reset()
@dataclass(slots=True)
class SpeakerProfile:
vector: tuple[float, ...] | None = None
speaker_id: str | None = None
speech_ms: int = 0
@dataclass(slots=True)
class VadRecorder:
provider: Any
@@ -275,3 +282,166 @@ class VadRecorder:
self.reset()
self.provider.reset()
return segment
@dataclass(slots=True)
class PrimarySpeakerVadRecorder(VadRecorder):
speaker_profile_ms: int = 600
speaker_absent_ms: int = 300
similarity_threshold: float = 0.70
min_rms: float = 0.012
profile: SpeakerProfile = field(default_factory=SpeakerProfile, init=False)
profile_vectors: list[tuple[float, ...]] = field(default_factory=list, init=False)
primary_absent_ms: int = field(default=0, init=False)
def reset(self) -> None:
VadRecorder.reset(self)
self.profile = SpeakerProfile()
self.profile_vectors = []
self.primary_absent_ms = 0
def feed(self, frame: AudioFrame) -> AudioSegment | ProviderError | None:
if self.first_seen_ms is None:
self.first_seen_ms = frame.timestamp_ms
result = self.provider.analyze(frame)
frame_ms = int(frame.metadata.get("duration_ms", 20))
if result.is_speech:
if not self.started:
self.started = True
self.start_time_ms = frame.timestamp_ms
self.frames.append(frame)
self._update_profile(frame, frame_ms)
elif self.started:
self.frames.append(frame)
if not self.started:
elapsed = frame.timestamp_ms - self.first_seen_ms
if elapsed >= self.no_speech_timeout_ms:
return ProviderError(
ErrorCode.VAD_TIMEOUT_NO_SPEECH,
"no speech detected after wakeword",
True,
"primary-speaker-vad",
"vad",
)
return None
start_time = self.start_time_ms if self.start_time_ms is not None else frame.timestamp_ms
duration = frame.timestamp_ms - start_time
if duration >= self.max_recording_ms:
return self._build_segment("max_recording")
if self._profile_ready():
if self._matches_primary(frame):
self.primary_absent_ms = 0
else:
self.primary_absent_ms += frame_ms
if self.primary_absent_ms >= self.speaker_absent_ms and duration >= self.min_duration_ms:
return self._build_segment("primary_speaker_absent")
if result.silence_ms >= self.end_silence_ms and duration >= self.min_duration_ms:
return self._build_segment("silence")
return None
def _update_profile(self, frame: AudioFrame, frame_ms: int) -> None:
if self.profile.speech_ms >= self.speaker_profile_ms:
return
speaker_id = frame.metadata.get("speaker_id")
if speaker_id is not None:
if self.profile.speaker_id is None:
self.profile.speaker_id = str(speaker_id)
if str(speaker_id) == self.profile.speaker_id:
self.profile.speech_ms += frame_ms
return
vector = extract_timbre_vector(frame, min_rms=self.min_rms)
if vector is None:
return
self.profile_vectors.append(vector)
self.profile.speech_ms += frame_ms
if self.profile_vectors:
width = len(self.profile_vectors[0])
averaged = []
for index in range(width):
averaged.append(sum(item[index] for item in self.profile_vectors) / len(self.profile_vectors))
self.profile.vector = tuple(averaged)
def _profile_ready(self) -> bool:
minimum_ms = min(self.speaker_profile_ms, max(120, self.min_duration_ms))
return self.profile.speech_ms >= minimum_ms and (
self.profile.speaker_id is not None or self.profile.vector is not None
)
def _matches_primary(self, frame: AudioFrame) -> bool:
speaker_id = frame.metadata.get("speaker_id")
if self.profile.speaker_id is not None:
return speaker_id is not None and str(speaker_id) == self.profile.speaker_id
if self.profile.vector is None:
return True
vector = extract_timbre_vector(frame, min_rms=self.min_rms)
if vector is None:
return False
return cosine_similarity(self.profile.vector, vector) >= self.similarity_threshold
def extract_timbre_vector(frame: AudioFrame, *, min_rms: float) -> tuple[float, ...] | None:
if "timbre_vector" in frame.metadata:
raw = frame.metadata["timbre_vector"]
if isinstance(raw, (list, tuple)) and raw:
return tuple(float(item) for item in raw)
if not frame.pcm:
return None
try:
import numpy as np
samples = np.frombuffer(frame.pcm, dtype=np.int16).astype(np.float32)
if samples.size == 0:
return None
if frame.channels > 1:
samples = samples.reshape(-1, frame.channels).mean(axis=1)
normalized = samples / 32768.0
rms = float(np.sqrt(np.mean(normalized * normalized)))
if rms < min_rms:
return None
signs = np.signbit(normalized)
zcr = float(np.mean(signs[1:] != signs[:-1])) if normalized.size > 1 else 0.0
windowed = normalized * np.hanning(normalized.size)
spectrum = np.abs(np.fft.rfft(windowed))
total = float(np.sum(spectrum))
if total <= 1e-9:
return None
freqs = np.fft.rfftfreq(normalized.size, 1.0 / frame.sample_rate)
nyquist = max(frame.sample_rate / 2.0, 1.0)
centroid = float(np.sum(freqs * spectrum) / total) / nyquist
bandwidth = float(np.sqrt(np.sum(((freqs / nyquist - centroid) ** 2) * spectrum) / total))
cumulative = np.cumsum(spectrum)
rolloff_index = int(np.searchsorted(cumulative, 0.85 * cumulative[-1]))
rolloff = float(freqs[min(rolloff_index, freqs.size - 1)] / nyquist)
flatness = float(np.exp(np.mean(np.log(spectrum + 1e-9))) / (np.mean(spectrum) + 1e-9))
def band_ratio(low: float, high: float) -> float:
mask = (freqs >= low) & (freqs < high)
return float(np.sum(spectrum[mask]) / total)
return (
rms,
zcr,
centroid,
bandwidth,
rolloff,
flatness,
band_ratio(80, 500),
band_ratio(500, 2000),
band_ratio(2000, nyquist),
)
except Exception:
return None
def cosine_similarity(left: tuple[float, ...], right: tuple[float, ...]) -> float:
if len(left) != len(right):
return 0.0
numerator = sum(a * b for a, b in zip(left, right))
left_norm = sum(a * a for a in left) ** 0.5
right_norm = sum(b * b for b in right) ** 0.5
if left_norm <= 1e-9 or right_norm <= 1e-9:
return 0.0
return numerator / (left_norm * right_norm)