[播报文本净化]:完成TTS表情包过滤,包含emoji清理、上下文净化和回归测试

This commit is contained in:
mkbk
2026-06-18 12:45:31 +08:00
parent 25255f178e
commit 3408a30e25
8 changed files with 285 additions and 22 deletions
+126
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import math
import base64
import json
import re
import socket
import struct
import subprocess
@@ -18,6 +19,131 @@ from .config import AppConfig
from .models import AudioSegment, ErrorCode, ProviderError
_MARKDOWN_IMAGE_RE = re.compile(r"!\[[^\]]*]\([^)]*\)")
_SHORTCODE_EMOJI_RE = re.compile(r":(?:[A-Za-z][A-Za-z0-9_+\-]{1,31}):")
_ASCII_KAOMOJI_RE = re.compile(r"(?:\^_?\^|T_T|QAQ|QwQ|qwq|orz)")
_SYMBOL_KAOMOJI_RE = re.compile(r"[\(][^()()]{0,20}[\u00b0\u2500-\u2bff][^()()]{0,20}[\)][^\s,。!?,.!?]{0,10}")
_EMOJI_RANGES = (
(0x1F000, 0x1FAFF),
(0x2600, 0x27BF),
(0x2300, 0x23FF),
)
_EMOJI_JOINERS = {0x200D, 0x20E3}
_BRACKET_PAIRS = {"[": "]", "": "", "(": ")", "": ""}
_BRACKET_EMOTE_WORDS = {
"ok",
"doge",
"emoji",
"一笑",
"偷笑",
"傻笑",
"加油",
"发呆",
"吐舌",
"呲牙",
"",
"哭泣",
"",
"大哭",
"大笑",
"委屈",
"害羞",
"尴尬",
"开心",
"微笑",
"",
"惊讶",
"惊喜",
"惊恐",
"捂脸",
"抱拳",
"擦汗",
"",
"",
"流汗",
"流泪",
"滑稽",
"爱心",
"玫瑰",
"生气",
"白眼",
"点赞",
"破涕为笑",
"",
"笑哭",
"鼓掌",
"比心",
"亲亲",
"调皮",
"难过",
"高兴",
"鼓励",
"狗头",
}
def sanitize_tts_text(text: str) -> str:
clean = text.strip()
if not clean:
return ""
clean = _MARKDOWN_IMAGE_RE.sub("", clean)
clean = _SHORTCODE_EMOJI_RE.sub("", clean)
clean = _strip_bracket_emotes(clean)
clean = _SYMBOL_KAOMOJI_RE.sub("", clean)
clean = _ASCII_KAOMOJI_RE.sub("", clean)
clean = "".join(ch for ch in clean if not _is_emoji_char(ch))
return _normalize_spoken_text(clean)
def _strip_bracket_emotes(text: str) -> str:
result: list[str] = []
index = 0
while index < len(text):
ch = text[index]
close = _BRACKET_PAIRS.get(ch)
if close is None:
result.append(ch)
index += 1
continue
close_index = text.find(close, index + 1)
if close_index == -1 or close_index - index > 12:
result.append(ch)
index += 1
continue
content = text[index + 1 : close_index]
if _is_bracket_emote(content):
index = close_index + 1
continue
result.append(ch)
index += 1
return "".join(result)
def _is_bracket_emote(content: str) -> bool:
token = re.sub(r"\s+", "", content.strip()).lower()
if not token or len(token) > 8:
return False
if token in _BRACKET_EMOTE_WORDS or token.endswith("表情"):
return True
return all(_is_emoji_char(ch) for ch in token)
def _is_emoji_char(ch: str) -> bool:
codepoint = ord(ch)
if codepoint in _EMOJI_JOINERS or 0xFE00 <= codepoint <= 0xFE0F:
return True
return any(start <= codepoint <= end for start, end in _EMOJI_RANGES)
def _normalize_spoken_text(text: str) -> str:
clean = re.sub(r"[ \t]+", " ", text)
clean = re.sub(r"\s+([,。!?,.!?;:])", r"\1", clean)
clean = re.sub(r"([,])\s*([。!?!?])", r"\2", clean)
clean = re.sub(r"([。!?!?]){2,}", r"\1", clean)
clean = re.sub(r"\s{2,}", " ", clean)
return clean.strip(" \t\r\n,;:")
class SentenceBuffer:
def __init__(self, max_chars: int = 80) -> None:
self.max_chars = max_chars