[播报文本净化]:完成TTS表情包过滤,包含emoji清理、上下文净化和回归测试
This commit is contained in:
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import math
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
@@ -18,6 +19,131 @@ from .config import AppConfig
|
||||
from .models import AudioSegment, ErrorCode, ProviderError
|
||||
|
||||
|
||||
_MARKDOWN_IMAGE_RE = re.compile(r"!\[[^\]]*]\([^)]*\)")
|
||||
_SHORTCODE_EMOJI_RE = re.compile(r":(?:[A-Za-z][A-Za-z0-9_+\-]{1,31}):")
|
||||
_ASCII_KAOMOJI_RE = re.compile(r"(?:\^_?\^|T_T|QAQ|QwQ|qwq|orz)")
|
||||
_SYMBOL_KAOMOJI_RE = re.compile(r"[\((][^()()]{0,20}[\u00b0\u2500-\u2bff][^()()]{0,20}[\))][^\s,。!?,.!?]{0,10}")
|
||||
_EMOJI_RANGES = (
|
||||
(0x1F000, 0x1FAFF),
|
||||
(0x2600, 0x27BF),
|
||||
(0x2300, 0x23FF),
|
||||
)
|
||||
_EMOJI_JOINERS = {0x200D, 0x20E3}
|
||||
_BRACKET_PAIRS = {"[": "]", "【": "】", "(": ")", "(": ")"}
|
||||
_BRACKET_EMOTE_WORDS = {
|
||||
"ok",
|
||||
"doge",
|
||||
"emoji",
|
||||
"一笑",
|
||||
"偷笑",
|
||||
"傻笑",
|
||||
"加油",
|
||||
"发呆",
|
||||
"吐舌",
|
||||
"呲牙",
|
||||
"哭",
|
||||
"哭泣",
|
||||
"困",
|
||||
"大哭",
|
||||
"大笑",
|
||||
"委屈",
|
||||
"害羞",
|
||||
"尴尬",
|
||||
"开心",
|
||||
"微笑",
|
||||
"怒",
|
||||
"惊讶",
|
||||
"惊喜",
|
||||
"惊恐",
|
||||
"捂脸",
|
||||
"抱拳",
|
||||
"擦汗",
|
||||
"晕",
|
||||
"汗",
|
||||
"流汗",
|
||||
"流泪",
|
||||
"滑稽",
|
||||
"爱心",
|
||||
"玫瑰",
|
||||
"生气",
|
||||
"白眼",
|
||||
"点赞",
|
||||
"破涕为笑",
|
||||
"笑",
|
||||
"笑哭",
|
||||
"鼓掌",
|
||||
"比心",
|
||||
"亲亲",
|
||||
"调皮",
|
||||
"难过",
|
||||
"高兴",
|
||||
"鼓励",
|
||||
"狗头",
|
||||
}
|
||||
|
||||
|
||||
def sanitize_tts_text(text: str) -> str:
|
||||
clean = text.strip()
|
||||
if not clean:
|
||||
return ""
|
||||
clean = _MARKDOWN_IMAGE_RE.sub("", clean)
|
||||
clean = _SHORTCODE_EMOJI_RE.sub("", clean)
|
||||
clean = _strip_bracket_emotes(clean)
|
||||
clean = _SYMBOL_KAOMOJI_RE.sub("", clean)
|
||||
clean = _ASCII_KAOMOJI_RE.sub("", clean)
|
||||
clean = "".join(ch for ch in clean if not _is_emoji_char(ch))
|
||||
return _normalize_spoken_text(clean)
|
||||
|
||||
|
||||
def _strip_bracket_emotes(text: str) -> str:
|
||||
result: list[str] = []
|
||||
index = 0
|
||||
while index < len(text):
|
||||
ch = text[index]
|
||||
close = _BRACKET_PAIRS.get(ch)
|
||||
if close is None:
|
||||
result.append(ch)
|
||||
index += 1
|
||||
continue
|
||||
close_index = text.find(close, index + 1)
|
||||
if close_index == -1 or close_index - index > 12:
|
||||
result.append(ch)
|
||||
index += 1
|
||||
continue
|
||||
content = text[index + 1 : close_index]
|
||||
if _is_bracket_emote(content):
|
||||
index = close_index + 1
|
||||
continue
|
||||
result.append(ch)
|
||||
index += 1
|
||||
return "".join(result)
|
||||
|
||||
|
||||
def _is_bracket_emote(content: str) -> bool:
|
||||
token = re.sub(r"\s+", "", content.strip()).lower()
|
||||
if not token or len(token) > 8:
|
||||
return False
|
||||
if token in _BRACKET_EMOTE_WORDS or token.endswith("表情"):
|
||||
return True
|
||||
return all(_is_emoji_char(ch) for ch in token)
|
||||
|
||||
|
||||
def _is_emoji_char(ch: str) -> bool:
|
||||
codepoint = ord(ch)
|
||||
if codepoint in _EMOJI_JOINERS or 0xFE00 <= codepoint <= 0xFE0F:
|
||||
return True
|
||||
return any(start <= codepoint <= end for start, end in _EMOJI_RANGES)
|
||||
|
||||
|
||||
def _normalize_spoken_text(text: str) -> str:
|
||||
clean = re.sub(r"[ \t]+", " ", text)
|
||||
clean = re.sub(r"\s+([,。!?,.!?;;::])", r"\1", clean)
|
||||
clean = re.sub(r"([,,])\s*([。!?!?])", r"\2", clean)
|
||||
clean = re.sub(r"([。!?!?]){2,}", r"\1", clean)
|
||||
clean = re.sub(r"\s{2,}", " ", clean)
|
||||
return clean.strip(" \t\r\n,,;;::")
|
||||
|
||||
|
||||
class SentenceBuffer:
|
||||
def __init__(self, max_chars: int = 80) -> None:
|
||||
self.max_chars = max_chars
|
||||
|
||||
Reference in New Issue
Block a user