This commit is contained in:
田岩
2025-12-02 21:04:46 +08:00
parent fee7149458
commit 73a3d2e914
68 changed files with 1712 additions and 234 deletions
+32
View File
@@ -0,0 +1,32 @@
# 服务端口
APP_PORT=8000
# 日志级别(DEBUG/INFO/WARNING/ERROR
LOG_LEVEL=INFO
# 日志文件路径
LOG_FILE=logs/app.log
# ASR服务配置
ASR_SERVICE_URL=http://localhost:5000/asr
ASR_TIMEOUT=30 # 超时时间(秒)
ASR_RETRY_TIMES=2 # 重试次数
# LLM服务配置
LLM_SERVICE_URL=http://localhost:6000/chat
LLM_TIMEOUT=60
LLM_RETRY_TIMES=2
LLM_MODEL=gpt-3.5-turbo # 可选:指定大模型版本
# TTS服务配置
TTS_SERVICE_URL=http://localhost:7000/tts
TTS_TIMEOUT=30
TTS_RETRY_TIMES=2
TTS_VOICE=female # 可选:指定语音类型
# WebSocket配置
WS_MAX_SIZE=10485760 # 最大消息大小(10MB
WS_PING_INTERVAL=30 # 心跳检测间隔(秒)
WS_PING_TIMEOUT=10 # 心跳超时时间(秒)
# 加密配置(ProtocolCodec依赖的密钥等)
ENCRYPT_KEY=your_secret_key_123
ENCRYPT_IV=your_iv_456
+32
View File
@@ -0,0 +1,32 @@
# 服务端口
APP_PORT=8000
# 日志级别(DEBUG/INFO/WARNING/ERROR
LOG_LEVEL=INFO
# 日志文件路径
LOG_FILE=logs/app.log
# ASR服务配置
ASR_SERVICE_URL=http://localhost:5000/asr
ASR_TIMEOUT=30 # 超时时间(秒)
ASR_RETRY_TIMES=2 # 重试次数
# LLM服务配置
LLM_SERVICE_URL=http://localhost:6000/chat
LLM_TIMEOUT=60
LLM_RETRY_TIMES=2
LLM_MODEL=gpt-3.5-turbo # 可选:指定大模型版本
# TTS服务配置
TTS_SERVICE_URL=http://localhost:7000/tts
TTS_TIMEOUT=30
TTS_RETRY_TIMES=2
TTS_VOICE=female # 可选:指定语音类型
# WebSocket配置
WS_MAX_SIZE=10485760 # 最大消息大小(10MB
WS_PING_INTERVAL=30 # 心跳检测间隔(秒)
WS_PING_TIMEOUT=10 # 心跳超时时间(秒)
# 加密配置(ProtocolCodec依赖的密钥等)
ENCRYPT_KEY=your_secret_key_123
ENCRYPT_IV=your_iv_456
+5
View File
@@ -0,0 +1,5 @@
# 默认忽略的文件
/shelf/
/workspace.xml
# 基于编辑器的 HTTP 客户端请求
/httpRequests/
+10
View File
@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/.venv" />
</content>
<orderEntry type="jdk" jdkName="audio-ai-chat" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
@@ -0,0 +1,12 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredIdentifiers">
<list>
<option value="javalang.tree" />
</list>
</option>
</inspection_tool>
</profile>
</component>
@@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
+7
View File
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Black">
<option name="sdkName" value="Python 3.13 (audio_ai_chat)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="audio-ai-chat" project-jdk-type="Python SDK" />
</project>
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/audio_ai_chat.iml" filepath="$PROJECT_DIR$/.idea/audio_ai_chat.iml" />
</modules>
</component>
</project>
+6
View File
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$/.." vcs="Git" />
</component>
</project>
+71
View File
@@ -0,0 +1,71 @@
# 语音AI对话系统(WebSocket版)
基于FastAPI实现,支持前端通过WebSocket推送语音数据,串联ASR(语音转文字)→ LLM(大模型对话)→ TTS(文字转语音)全流程,返回语音结果给前端。
## 快速启动
1. 创建Conda虚拟环境:`conda create -n audio-ai-chat python=3.10 -y`
2. 激活环境:`conda activate audio-ai-chat`
3. 安装依赖:`pip install -r requirements.txt`
4. 配置环境变量:复制`.env.example``.env`,修改服务地址、密钥等配置
5. 启动服务:`python run.py`
6. 接口文档:访问 `http://localhost:8000/docs`FastAPI自动生成)
## 目录说明
audio_ai_chat/ # 项目根目录(包名,Python可导入)
├── audio_ai_chat/ # 核心代码目录(与根目录同名,避免导入冲突)
│ ├── __init__.py # 包初始化文件(空文件即可)
│ ├── main.py # 项目启动类(FastAPI入口+WebSocket路由)
│ ├── config/ # 配置文件目录
│ │ ├── __init__.py
│ │ ├── settings.py # 核心配置(ASR/LLM/TTS服务地址、WS参数等)
│ │ └── logger.py # 日志配置(格式、输出路径、级别)
│ ├── core/ # 核心业务逻辑(调整服务调用方式)
│ │ ├── __init__.py
│ │ ├── websocket_handler.py# 调整为:通过工厂类获取服务实例
│ │ ├── asr/ # ASR服务目录(多版本实现)
│ │ │ ├── __init__.py
│ │ │ ├── base.py # ASR统一接口抽象类
│ │ │ ├── version1.py # ASR版本1(如:本地离线版)
│ │ │ ├── version2.py # ASR版本2(如:百度云ASR)
│ │ │ └── factory.py # ASR工厂类(根据配置创建实例)
│ │ ├── llm/ # LLM服务目录(多版本实现)
│ │ │ ├── __init__.py
│ │ │ ├── base.py # LLM统一接口抽象类
│ │ │ ├── openai_llm.py # LLM版本1OpenAI
│ │ │ ├── local_llm.py # LLM版本2(本地部署LLM
│ │ │ └── factory.py # LLM工厂类
│ │ └── tts/ # TTS服务目录(多版本实现)
│ │ ├── __init__.py
│ │ ├── base.py # TTS统一接口抽象类
│ │ ├── ali_tts.py # TTS版本1(阿里云)
│ │ ├── pyttsx3_tts.py # TTS版本2(本地pyttsx3
│ │ └── factory.py # TTS工厂类
│ ├── codec/ # 数据编解码目录(存放加密/解密逻辑)
│ │ ├── __init__.py
│ │ └── ProtocolCodec.py # 已有:WS数据加密/解密、协议编解码
│ ├── models/ # 数据模型目录(Pydantic/数据结构定义)
│ │ ├── __init__.py
│ │ └── ws_models.py # WebSocket消息结构(请求/响应模型、错误模型)
│ └── utils/ # 工具函数目录
│ ├── __init__.py
│ ├── exceptions.py # 自定义异常(如服务调用失败、解码失败)
│ └── helpers.py # 通用工具(日志封装、异步重试、数据格式转换)
├── logs/ # 日志输出目录(自动创建)
│ └── app.log # 应用日志文件(按配置滚动生成)
├── .env # 环境变量文件(敏感配置,不提交Git)
├── .env.example # 环境变量示例(提交Git,指导配置)
├── requirements.txt # 依赖包清单
├── README.md # 项目说明(启动方式、配置说明、接口文档)
└── run.py # 项目启动脚本(简化启动命令)
## 握手流程
前端 后端
| |
|--- 建立 WebSocket 连接 ---->|
|<--- 连接接受(101状态码)---|
| |
|--- 发送身份信息(user_id+token--->|
| |
|<--- 校验结果(成功/失败)---|
| |
|--- 发送业务数据(音频/文本)--->|
|<--- 推送业务结果(TTS/回复)---|
@@ -5,7 +5,7 @@ import gzip
from typing import Optional, Union, Dict, Any, List, Tuple
# -------------------------- 协议常量定义(与JS完全一致) --------------------------
# -------------------------- 协议常量定义 --------------------------
class ProtocolConst:
PROTOCOL_VERSION = 0b0001 # 协议版本(4位,字节0低4位,0~15)
HEADER_SIZE = 8 # 头部固定字节数(字节0~7
@@ -13,13 +13,14 @@ class ProtocolConst:
STRING_ENCODING = "utf-8" # 字符串默认编码
# -------------------------- 枚举定义(与JS码值完全对齐) --------------------------
# -------------------------- 枚举定义 --------------------------
class MessageType(IntEnum):
"""消息类型(4位,字节0高4位,0~15)"""
PING = 0b0001 # 心跳(支持空包体)
AUDIO_DATA = 0b0010 # 纯音频数据(pcm)
TEXT_MESSAGE = 0b0011 # 纯文本消息
CONTROL_CMD = 0b0100 # 控制指令(暂停/继续等)
CONTROL_CMD = 0b0100 # 控制指令
IDENTITY = 0b0101 # 身份校验包
# 预留12种类型(0b0100 ~ 0b1111
@@ -161,11 +162,11 @@ class ProtocolCodec:
return header + compressed_body
@staticmethod
def unpack(packet: bytes) -> Tuple[MessageType, SerializationType, CompressionType, int, Any]:
def unpack(packet: bytes) -> Tuple[MessageType, int, Any]:
"""
解析协议包与JS unpack 方法完全兼容
:param packet: 完整协议包头部 + 包体
:return: (消息类型, 序列化方式, 压缩方式, 顺序号, 原始包体数据)
:return: (消息类型, 顺序号, 原始包体数据) # 仅返回核心必要信息
"""
# 1. 校验包长度
if len(packet) < ProtocolConst.HEADER_SIZE:
@@ -193,7 +194,7 @@ class ProtocolCodec:
# 字节4~6:包体长度(24位大端序),字节7:保留位(忽略)
body_len = (header[4] << 16) | (header[5] << 8) | header[6]
# 校验版本和包体长度
# 校验版本和包体长度(校验逻辑保留,确保数据有效性)
if version != ProtocolConst.PROTOCOL_VERSION:
raise ValueError(
f"协议版本不匹配:收到v{version}0b{version:04b}),支持v{ProtocolConst.PROTOCOL_VERSION}0b{ProtocolConst.PROTOCOL_VERSION:04b}"
@@ -236,16 +237,17 @@ class ProtocolCodec:
else:
raise ValueError(f"不支持的序列化方式:{serialization}")
return msg_type, serialization, compression, sequence, original_body
# 仅返回核心必要信息:消息类型、顺序号、原始包体数据
return msg_type, sequence, original_body
# -------------------------- 使用示例(验证与JS兼容性 --------------------------
# -------------------------- 使用示例(验证修改后功能正常 --------------------------
if __name__ == "__main__":
# 示例1:PING消息(空包体,默认顺序号0)
ping_packet = ProtocolCodec.pack(MessageType.PING)
print(f"PING消息包长度:{len(ping_packet)}字节(仅头部)")
msg_type1, ser1, comp1, seq1, body1 = ProtocolCodec.unpack(ping_packet)
print(f"PING解析结果:类型={msg_type1.name}序列化={ser1.name},压缩={comp1.name}顺序号={seq1},包体={body1}\n")
msg_type1, seq1, body1 = ProtocolCodec.unpack(ping_packet)
print(f"PING解析结果:类型={msg_type1.name},顺序号={seq1},包体={body1}\n")
# 示例2:纯文本消息(STRING序列化,指定顺序号)
text_body = "Python与JS协议兼容测试(纯字符串)"
@@ -256,8 +258,8 @@ if __name__ == "__main__":
compression=CompressionType.NONE
)
print(f"文本消息包长度:{len(text_packet)}字节")
msg_type2, ser2, comp2, seq2, body2 = ProtocolCodec.unpack(text_packet)
print(f"文本解析结果:类型={msg_type2.name}序列化={ser2.name}顺序号={seq2},内容={body2}\n")
msg_type2, seq2, body2 = ProtocolCodec.unpack(text_packet)
print(f"文本解析结果:类型={msg_type2.name},顺序号={seq2},内容={body2}\n")
# 示例3:控制指令(JSON序列化)
control_body = {"cmd": ControlCommand.PAUSE.value, "reason": "用户主动暂停"}
@@ -267,8 +269,8 @@ if __name__ == "__main__":
sequence=1002
)
print(f"控制指令包长度:{len(control_packet)}字节")
msg_type3, ser3, comp3, seq3, body3 = ProtocolCodec.unpack(control_packet)
print(f"控制指令解析结果:类型={msg_type3.name}序列化={ser3.name}顺序号={seq3},内容={body3}\n")
msg_type3, seq3, body3 = ProtocolCodec.unpack(control_packet)
print(f"控制指令解析结果:类型={msg_type3.name},顺序号={seq3},内容={body3}\n")
# 示例4:音频数据(RAW序列化)
audio_body = b"\x00\x01\x02\x03\x04\x05" * 100 # 模拟PCM数据
@@ -278,8 +280,8 @@ if __name__ == "__main__":
sequence=1003
)
print(f"音频数据包长度:{len(audio_packet)}字节")
msg_type4, ser4, comp4, seq4, body4 = ProtocolCodec.unpack(audio_packet)
print(f"音频解析结果:类型={msg_type4.name}序列化={ser4.name}顺序号={seq4},数据长度={len(body4)}字节\n")
msg_type4, seq4, body4 = ProtocolCodec.unpack(audio_packet)
print(f"音频解析结果:类型={msg_type4.name},顺序号={seq4},数据长度={len(body4)}字节\n")
# 示例5:GZIP压缩测试(需JS端启用GZIP解压)
long_text_body = "这是一段很长的文本,用于测试GZIP压缩效果" * 100
@@ -290,5 +292,5 @@ if __name__ == "__main__":
compression=CompressionType.GZIP
)
print(f"GZIP压缩后包长度:{len(gzip_packet)}字节(原始文本长度:{len(long_text_body.encode())}字节)")
msg_type5, ser5, comp5, seq5, body5 = ProtocolCodec.unpack(gzip_packet)
print(f"GZIP解析结果:类型={msg_type5.name}压缩={comp5.name},内容前50字:{body5[:50]}...")
msg_type5, seq5, body5 = ProtocolCodec.unpack(gzip_packet)
print(f"GZIP解析结果:类型={msg_type5.name}顺序号={seq5},内容前50字:{body5[:50]}...")
@@ -0,0 +1,28 @@
from loguru import logger
import sys
from audio_ai_chat.config.settings import settings
# 移除默认日志输出
logger.remove()
# 添加控制台输出(开发环境)
logger.add(
sink=sys.stdout,
level=settings.LOG_LEVEL,
format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
enqueue=True, # 异步日志,提高性能
)
# 添加文件输出(按大小滚动,保留10个文件,每个文件最大50MB)
logger.add(
sink=settings.LOG_FILE,
level=settings.LOG_LEVEL,
format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}",
rotation="50 MB",
retention=10,
compression="zip",
enqueue=True,
)
# 导出logger供其他模块使用
__all__ = ["logger"]
@@ -0,0 +1,70 @@
from pydantic_settings import BaseSettings, SettingsConfigDict
from pydantic import Field
from pathlib import Path
ROOT_DIR = Path(__file__).parent.parent.parent
class Settings(BaseSettings):
# 应用配置
APP_PORT: int = Field(default=8000, description="服务端口")
LOG_LEVEL: str = Field(default="INFO", description="日志级别")
LOG_FILE: Path = Field(default=ROOT_DIR / "logs/app.log", description="日志文件路径")
# ASR服务配置
ASR_SERVICE_URL: str = Field(..., description="ASR服务地址")
ASR_TIMEOUT: int = Field(default=30, description="ASR超时时间(秒)")
ASR_RETRY_TIMES: int = Field(default=2, description="ASR重试次数")
# LLM服务配置
LLM_SERVICE_URL: str = Field(..., description="LLM服务地址")
LLM_TIMEOUT: int = Field(default=60, description="LLM超时时间(秒)")
LLM_RETRY_TIMES: int = Field(default=2, description="LLM重试次数")
LLM_MODEL: str = Field(default="default", description="LLM模型版本")
# TTS服务配置
TTS_SERVICE_URL: str = Field(..., description="TTS服务地址")
TTS_TIMEOUT: int = Field(default=30, description="TTS超时时间(秒)")
TTS_RETRY_TIMES: int = Field(default=2, description="TTS重试次数")
TTS_VOICE: str = Field(default="default", description="TTS语音类型")
# WebSocket配置
WS_MAX_SIZE: int = Field(default=10 * 1024 * 1024, description="WS最大消息大小(字节)")
WS_PING_INTERVAL: int = Field(default=30, description="WS心跳间隔(秒)")
WS_PING_TIMEOUT: int = Field(default=10, description="WS心跳超时(秒)")
# 加密配置
ENCRYPT_KEY: str = Field(..., description="数据加密密钥")
ENCRYPT_IV: str = Field(..., description="数据加密向量")
# -------------------------- 新增:服务版本配置 --------------------------
# ASR当前使用版本(对应ASR_REGISTRY中的key
ASR_CURRENT_VERSION: str = Field(default="local_v1", description="ASR服务当前版本")
# 本地ASR专属配置(仅local_v1版本使用)
LOCAL_ASR_MODEL_PATH: Path = Field(default=ROOT_DIR / "models/asr/local_model", description="本地ASR模型路径")
# 百度云ASR专属配置(仅baidu_v2版本使用)
BAIDU_ASR_API_KEY: str = Field(default="", description="百度云ASR API Key")
BAIDU_ASR_SECRET_KEY: str = Field(default="", description="百度云ASR Secret Key")
# LLM当前使用版本(对应LLM_REGISTRY中的key
LLM_CURRENT_VERSION: str = Field(default="local", description="LLM服务当前版本")
# OpenAI LLM专属配置(仅openai版本使用)
OPENAI_API_KEY: str = Field(default="", description="OpenAI API Key")
OPENAI_BASE_URL: str = Field(default="https://api.openai.com/v1", description="OpenAI接口地址")
# 本地LLM专属配置(仅local版本使用)
LOCAL_LLM_MODEL_PATH: Path = Field(default=ROOT_DIR / "models/llm/local_model", description="本地LLM模型路径")
# TTS当前使用版本(对应TTS_REGISTRY中的key
TTS_CURRENT_VERSION: str = Field(default="pyttsx3", description="TTS服务当前版本")
# 阿里云TTS专属配置(仅aliyun版本使用)
ALI_TTS_ACCESS_KEY: str = Field(default="", description="阿里云TTS Access Key")
ALI_TTS_ACCESS_SECRET: str = Field(default="", description="阿里云TTS Access Secret")
ALI_TTS_REGION_ID: str = Field(default="cn-hangzhou", description="阿里云TTS区域ID")
# 原有配置...WS_MAX_SIZE、ENCRYPT_KEY等)
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8")
settings = Settings()
# 确保模型目录存在(本地版本需要)
# settings.LOCAL_ASR_MODEL_PATH.parent.mkdir(parents=True, exist_ok=True)
# settings.LOCAL_LLM_MODEL_PATH.parent.mkdir(parents=True, exist_ok=True)
@@ -0,0 +1,25 @@
from abc import ABC, abstractmethod
from typing import Optional, Coroutine
from audio_ai_chat.config.settings import settings
class ASRBase(ABC):
"""ASR服务统一抽象接口"""
def __init__(self):
# 公共配置(所有ASR版本共享的超时、重试次数等)
self.timeout = settings.ASR_TIMEOUT
self.retry_times = settings.ASR_RETRY_TIMES
@abstractmethod
async def recognize(
self,
voice_data: bytes,
user_id: Optional[str] = None,
**kwargs # 兼容不同版本的额外参数
) -> str:
"""
语音识别核心方法(所有ASR版本必须实现)
:param voice_data: 语音二进制数据
:param user_id: 用户ID(可选)
:return: 识别后的文本
"""
pass
@@ -0,0 +1,26 @@
from typing import Type
from audio_ai_chat.config.settings import settings
from audio_ai_chat.utils.exceptions import ServiceCallError
from .base import ASRBase
# from .version1 import LocalOfflineASR
# from .version2 import BaiduASR
#
# # 注册所有ASR版本:key=配置中的版本名,value=对应的类
# ASR_REGISTRY: dict[str, Type[ASRBase]] = {
# "local_v1": LocalOfflineASR,
# "baidu_v2": BaiduASR,
# # 新增版本时,只需在这里注册:"新版本名": 新类名
# }
# class ASRFactory:
# """ASR服务工厂类:根据配置创建对应版本的实例"""
# @staticmethod
# def get_asr_client() -> ASRBase:
# # 从配置中获取当前指定的ASR版本
# current_version = settings.ASR_CURRENT_VERSION
# if current_version not in ASR_REGISTRY:
# raise ServiceCallError(
# f"不支持的ASR版本:{current_version},可选版本:{list(ASR_REGISTRY.keys())}"
# )
# # 创建并返回对应版本的实例
# return ASR_REGISTRY[current_version]()
@@ -0,0 +1,145 @@
from typing import Optional, Dict, Any, List
import asyncio
from audio_ai_chat.config.logger import logger
# from audio_ai_chat.core.llm.factory import LLMFactory
from audio_ai_chat.core.llm.base import LLMBase
class ConnectionContext:
"""
单个WebSocket连接的上下文管理器:封装用户信息、大模型Session、消息队列等资源
每个WebSocket连接对应一个该类实例,确保资源隔离
"""
def __init__(self, client_id: str):
"""
初始化连接上下文
:param client_id: WebSocket连接唯一标识(如id(websocket)
:param user_id: 用户唯一标识(从前端请求中获取)
"""
self.client_id = client_id # 连接唯一ID
self.created_at = asyncio.get_event_loop().time() # 连接创建时间
# 1. 大模型独立Session(每个连接创建一个新的LLM客户端实例)
# self.llm_session: LLMBase = LLMFactory.get_llm_client() # 独立Session
self.chat_history: List[Dict[str, str]] = [] # 该连接的对话历史([(user: "...", assistant: "..."), ...]
# 2. 异步消息队列(用于缓存TTS结果,有序推送给前端)
self.message_queue: asyncio.Queue[bytes] = asyncio.Queue()
# 3. 连接状态(可选:如是否正在处理请求、是否断开等)
self.is_active: bool = True
self.is_processing: bool = False
self.name = None
self.user_id = None
self.token = None
def set_user_info(self, token: str, user_id: str, name: str = "匿名用户"):
"""
二次设置用户信息(身份校验通过后调用)
:param token:
:param user_id: 用户唯一标识(必填)
:param name: 用户名(可选,默认匿名)
"""
if not user_id:
raise ValueError("user_id不能为空")
self.name = name
self.user_id = user_id
self.token = token
logger.debug(f"客户端 {self.client_id} 设置用户信息:user_id={user_id}, name={name}")
async def add_message_to_queue(self, message: bytes):
"""将TTS结果添加到消息队列(异步安全)"""
if not self.is_active:
raise ValueError(f"连接已断开(client_id={self.client_id}),无法添加消息")
await self.message_queue.put(message)
logger.debug(f"消息队列添加数据:client_id={self.client_id},队列长度={self.message_queue.qsize()}")
async def get_message_from_queue(self) -> Optional[bytes]:
"""从消息队列获取消息(异步阻塞,直到有消息或连接断开)"""
try:
# 超时时间:30秒(避免无限阻塞)
return await asyncio.wait_for(self.message_queue.get(), timeout=30.0)
except asyncio.TimeoutError:
logger.debug(f"消息队列超时:client_id={self.client_id},无新消息")
return None
def update_chat_history(self, user_text: str, assistant_text: str):
"""更新该连接的对话历史"""
self.chat_history.append({
"user": user_text,
"assistant": assistant_text
})
# 可选:限制历史长度(避免内存溢出)
if len(self.chat_history) > 50:
self.chat_history.pop(0) # 删除最早的历史
def close(self):
"""关闭连接上下文,释放资源"""
self.is_active = False
self.is_processing = False
# 清空消息队列(可选)
while not self.message_queue.empty():
try:
self.message_queue.get_nowait()
except asyncio.QueueEmpty:
break
logger.info(f"连接上下文已关闭:client_id={self.client_id}user_id={self.user_id}")
def __del__(self):
"""析构函数:确保资源释放"""
self.close()
class ConnectionManager:
"""
WebSocket连接全局管理器:维护所有活跃连接的上下文
提供创建、查询、删除连接上下文的接口(线程/异步安全)
"""
def __init__(self):
# 存储所有活跃连接:key=client_idint),value=ConnectionContext实例
self.connections: Dict[int, ConnectionContext] = {}
# 异步锁:确保多连接并发操作时的数据安全
self._lock = asyncio.Lock()
async def create_connection(self, client_id: int, user_id: Optional[str] = None) -> ConnectionContext:
"""创建新的连接上下文(线程安全)"""
async with self._lock:
# 避免重复创建(同一client_id不会重复连接)
if client_id in self.connections:
logger.warning(f"连接已存在:client_id={client_id},将覆盖旧连接")
self.connections[client_id].close()
# 创建新的连接上下文(包含独立LLM Session和消息队列)
context = ConnectionContext(client_id=client_id, user_id=user_id)
self.connections[client_id] = context
logger.info(
f"创建新连接上下文:client_id={client_id}user_id={user_id},当前活跃连接数={len(self.connections)}")
return context
async def get_connection(self, client_id: int) -> Optional[ConnectionContext]:
"""获取指定client_id的连接上下文(线程安全)"""
async with self._lock:
context = self.connections.get(client_id)
if context and not context.is_active:
# 清理已断开的连接
del self.connections[client_id]
return None
return context
async def remove_connection(self, client_id: int):
"""删除连接上下文(线程安全)"""
async with self._lock:
context = self.connections.pop(client_id, None)
if context:
context.close()
logger.info(f"移除连接上下文:client_id={client_id},当前活跃连接数={len(self.connections)}")
async def get_active_connections_count(self) -> int:
"""获取当前活跃连接数(线程安全)"""
async with self._lock:
# 过滤已断开的连接
self.connections = {k: v for k, v in self.connections.items() if v.is_active}
return len(self.connections)
@@ -0,0 +1,26 @@
from abc import ABC, abstractmethod
from typing import Optional, Coroutine
class LLMBase(ABC):
"""LLM服务统一抽象接口"""
def __init__(self):
self.timeout = settings.LLM_TIMEOUT
self.retry_times = settings.LLM_RETRY_TIMES
self.model = settings.LLM_MODEL # 模型版本(不同LLM可能支持不同模型)
@abstractmethod
async def chat(
self,
text: str,
user_id: Optional[str] = None,
history: Optional[list] = None, # 对话历史(部分LLM支持)
**kwargs
) -> str:
"""
大模型对话核心方法
:param text: 用户输入文本(ASR识别结果)
:param user_id: 用户ID(可选)
:param history: 对话历史(可选,格式:[(用户输入, 模型回答), ...])
:return: 模型生成的回答文本
"""
pass
@@ -0,0 +1,22 @@
from typing import Type
from audio_ai_chat.config.settings import settings
from audio_ai_chat.utils.exceptions import ServiceCallError
from .base import LLMBase
#
# from .openai_llm import OpenAILLM
# from .local_llm import LocalLLM
#
# LLM_REGISTRY: dict[str, Type[LLMBase]] = {
# "openai": OpenAILLM,
# "local": LocalLLM,
# }
#
# class LLMFactory:
# @staticmethod
# def get_llm_client() -> LLMBase:
# current_version = settings.LLM_CURRENT_VERSION
# if current_version not in LLM_REGISTRY:
# raise ServiceCallError(
# f"不支持的LLM版本:{current_version},可选版本:{list(LLM_REGISTRY.keys())}"
# )
# return LLM_REGISTRY[current_version]()
@@ -0,0 +1,26 @@
from abc import ABC, abstractmethod
from typing import Optional, Coroutine
class TTSBase(ABC):
"""TTS服务统一抽象接口"""
def __init__(self):
self.timeout = settings.TTS_TIMEOUT
self.retry_times = settings.TTS_RETRY_TIMES
self.voice = settings.TTS_VOICE # 语音类型
@abstractmethod
async def synthesize(
self,
text: str,
user_id: Optional[str] = None,
speed: float = 1.0, # 语速(默认1.0
**kwargs
) -> bytes:
"""
文本转语音核心方法
:param text: 待合成文本(LLM回答结果)
:param user_id: 用户ID(可选)
:param speed: 语速(0.5~2.0
:return: 语音二进制数据
"""
pass
@@ -0,0 +1,22 @@
from typing import Type
from audio_ai_chat.config.settings import settings
from audio_ai_chat.utils.exceptions import ServiceCallError
from .base import TTSBase
#
# from .ali_tts import AliTTS
# from .pyttsx3_tts import Pyttsx3TTS
#
# TTS_REGISTRY: dict[str, Type[TTSBase]] = {
# "aliyun": AliTTS,
# "pyttsx3": Pyttsx3TTS,
# }
#
# class TTSFactory:
# @staticmethod
# def get_tts_client() -> TTSBase:
# current_version = settings.TTS_CURRENT_VERSION
# if current_version not in TTS_REGISTRY:
# raise ServiceCallError(
# f"不支持的TTS版本:{current_version},可选版本:{list(TTS_REGISTRY.keys())}"
# )
# return TTS_REGISTRY[current_version]()
@@ -0,0 +1,302 @@
from fastapi import WebSocket
from typing import Dict, List, Optional
from pyexpat.errors import messages
from audio_ai_chat.config.logger import logger
# from audio_ai_chat.core.asr.factory import ASRFactory # 导入ASR工厂
# from audio_ai_chat.core.llm.factory import LLMFactory # 导入LLM工厂
# from audio_ai_chat.core.tts.factory import TTSFactory # 导入TTS工厂
from audio_ai_chat.config.settings import settings
from audio_ai_chat.utils.exceptions import ServiceCallError
from fastapi import WebSocket, WebSocketDisconnect
from audio_ai_chat.codec.ProtocolCodec import ProtocolCodec
from typing import Optional, Dict, Callable, Awaitable, List, Any, Coroutine
from dataclasses import dataclass, field
import sys
import json
import asyncio
import websockets
import uuid
import logging
from audio_ai_chat.core.connection import ConnectionContext
from audio_ai_chat.codec.ProtocolCodec import ProtocolCodec, MessageType
class WebSocketConnectionManager:
"""WebSocket连接管理器"""
def __init__(self):
# 活跃连接列表
self.active_connections: List[WebSocket] = []
# 关键映射:client_id -> ConnectionContext(快速获取用户专属上下文)
self.client_context_map: Dict[str, ConnectionContext] = {}
# self.asr_client = ASRFactory.get_asr_client()
# self.tts_client = TTSFactory.get_tts_client()
# 用户LLM会话存储
# self.user_llm_conversations: Dict[str, LLMConversation] = {}
# 全局唤醒事件
self.consume_wakeup = asyncio.Event()
async def connect(self, websocket: WebSocket) -> ConnectionContext:
"""
建立连接+身份校验(前端主动发送身份信息)
超时逻辑:5秒内未收到前端身份信息,自动关闭连接
"""
# 1. 接受连接
await websocket.accept()
self.active_connections.append(websocket)
# 生成唯一标识
client_id = str(id(websocket))
context = ConnectionContext(
client_id=client_id
)
logger.info(
f"连接 {client_id} 已接受,等待前端发送身份信息(5秒超时)...,当前连接数: {len(self.active_connections)}")
try:
# 2. 超时控制:5秒内未收到身份信息,关闭连接
try:
# 关键:用 receive_bytes 接收二进制包(而非 receive_json
ping_packet = await asyncio.wait_for(
websocket.receive_bytes(),
timeout=5.0 # 身份校验超时时间
)
except asyncio.TimeoutError:
# 发送超时错误响应(二进制格式)
# todo 告知前端失败
# await websocket.close(code=1008) # todo 错误码值待定
# self.active_connections.remove(websocket)
raise TimeoutError(f"连接 {client_id} 身份包超时")
# 3. 解包二进制包
msg_type, _, body = ProtocolCodec.unpack(ping_packet)
if msg_type != MessageType.IDENTITY:
raise ValueError(f"连接 {client_id} 首个包类型错误(期望1,实际{MessageType.IDENTITY}")
logger.debug(f"解包成功:, body={body}")
# 4. 提取并校验身份信息(核心:按需扩展校验逻辑)
identity_data = json.loads(body.decode("utf-8"))
user_id = identity_data.get("user_id")
token = identity_data.get("token")
name = identity_data.get("name")
# todo 校验身份信息 raise PermissionError(f"用户 {user_id} 身份校验失败")
context.set_user_info(token, user_id, name)
self.client_context_map[client_id] = context
# 5. 响应前端:校验成功
await context.add_message_to_queue(
message=ProtocolCodec.pack(MessageType.IDENTITY, {'messages': '身份校验成功,连接已就绪'}))
logger.info(f"用户 {user_id}{name})身份校验通过,连接就绪(client_id: {client_id}")
return context
except:
pass
# 初始化LLM会话
# self._init_llm_conversation(user_id)
# return conn_id, user_id, conn_id # conn_id 同时作为 tts_session_id
# def _init_llm_conversation(self, user_id: str):
# """初始化用户LLM会话"""
# if user_id not in self.user_llm_conversations:
# self.user_llm_conversations[user_id] = LLMConversation(
# user_id=user_id,
# scene_description="语音识别对话场景"
# )
def disconnect(self, websocket: WebSocket, conn_id: str):
"""断开连接并清理资源"""
if websocket in self.active_connections:
self.active_connections.remove(websocket)
logger.info(f"连接 {conn_id} 已断开,当前连接数: {len(self.active_connections)}")
# async def setup_tts_manager(self, result_queue: asyncio.Queue) -> TTSManager:
# """初始化TTS管理器"""
#
# def handle_tts_result(req_id: str, result: Dict[str, Any]):
# """TTS结果回调处理"""
# try:
# status = result.get("status")
# if status == "completed":
# audio_data = result.get("audio_data")
# if audio_data is not None and len(audio_data) > 0:
# # 转换为PCM格式
# pcm_data = (audio_data.astype(np.float32) * 32767).astype(np.int16)
# pcm_bytes = pcm_data.tobytes()
# result_queue.put_nowait(pcm_bytes)
# except Exception as e:
# logger.error(f"TTS结果处理失败: {str(e)}")
#
# self.tts_client = TTSFactory.get_tts_client()
# tts_manager = TTSManager()
# tts_manager.set_result_callback(handle_tts_result)
# tts_manager.set_playback_enabled(False)
# await tts_manager.initialize()
# return tts_manager
async def asr_result_callback(self, result: dict, websocket: WebSocket,
user_id: str, result_queue: asyncio.Queue):
"""ASR结果回调处理"""
try:
logger.info(f"ASR识别结果: {result}")
final_asr_text = result.get("text", "").strip()
# 转发ASR结果到前端队列
if final_asr_text:
print(f"插入ASR结果时队列大小: {result_queue.qsize()}")
self.consume_wakeup.set() # 唤醒消费协程
# 异步调用大模型
llm_conversation = self.user_llm_conversations.get(user_id)
if llm_conversation:
asyncio.create_task(
self.call_llm_and_send(
query=final_asr_text,
conversation=llm_conversation,
websocket=websocket
)
)
except Exception as e:
logger.error(f"ASR回调执行失败: {str(e)}")
# async def llm_stream_callback(self, chunk: str, tts_manager: TTSManager):
# """大模型流式回调处理"""
# if not chunk:
# return
# try:
# # 提交TTS合成请求
# await tts_manager.synthesize(chunk)
# await asyncio.sleep(0) # 让出调度权
# except Exception as e:
# logger.error(f"LLM流式回调处理失败: {str(e)}")
# async def call_llm_and_send(self, query: str, conversation: LLMConversation, websocket: WebSocket):
# """调用大模型并处理结果"""
# logger.info(f"调用大模型 - 用户({conversation.user_id}): {query}")
# try:
# conv_id, full_reply = await llm_client.send_message(
# query=query,
# conversation=conversation,
# stream_callback=self.llm_stream_callback,
# response_mode="streaming"
# )
# logger.info(f"大模型回复完成 - 会话ID: {conv_id}, 完整回复: {full_reply}")
# except Exception as e:
# logger.error(f"大模型调用失败: {str(e)}")
# if not websocket.client_state.disconnected:
# await websocket.send_json({
# "type": "llm_error",
# "data": {"error": str(e)}
# })
async def recv_frontend_data(self, websocket: WebSocket, asr_conn):
"""接收前端音频数据并推送到ASR"""
while not asr_conn.stop_event.is_set():
try:
raw_bytes = await websocket.receive_bytes()
# success = await push_audio_data(asr_conn, raw_bytes)
# if not success:
# logger.warning("音频数据插入ASR失败(队列满/连接失效)")
except WebSocketDisconnect:
logger.info("前端主动断开连接")
asr_conn.stop_event.set()
break
except Exception as e:
logger.error(f"接收前端数据失败: {str(e)}")
# asr_conn.stop_event.set()
# if not websocket.client_state.disconnected:
# await websocket.send_json({"error": f"接收数据失败: {str(e)}"})
# break
async def send_results(self, websocket: WebSocket, result_queue: asyncio.Queue, asr_conn):
"""从结果队列发送数据到前端"""
while True:
try:
# 等待队列数据或超时
result = await asyncio.wait_for(result_queue.get(), timeout=0.05)
if not websocket.client_state.disconnected:
await websocket.send_bytes(result)
except asyncio.TimeoutError:
if asr_conn.stop_event.is_set():
break
continue
except Exception as e:
logger.error(f"发送结果到前端失败: {str(e)}")
asr_conn.stop_event.set()
break
async def handle_connection(self, websocket: WebSocket):
"""处理单个WebSocket连接的完整生命周期"""
tts_manager = None
asr_conn = None
asr_task = None
result_queue = asyncio.Queue(maxsize=10000)
context = await self.connect(websocket)
print(context)
try:
pass
# 1. 建立连接
# 2. 初始化TTS
# tts_manager = await self.setup_tts_manager(result_queue)
# 3. 获取ASR连接
# asr_conn = await get_idle_asr_connection()
# if not asr_conn:
# await websocket.send_json({"error": "ASR服务暂时不可用", "text": ""})
# return
# 4. 启动ASR通信协程
# asr_callback = lambda res: self.asr_result_callback(res, websocket, user_id, result_queue)
# asr_task = asyncio.create_task(handle_asr_communication(asr_conn, asr_callback))
#
# # 5. 启动数据接收和发送协程
# task_recv = asyncio.create_task(self.recv_frontend_data(websocket, asr_conn))
# task_send = asyncio.create_task(self.send_results(websocket, result_queue, asr_conn))
#
# # 6. 等待任一任务完成
# done, pending = await asyncio.wait(
# [task_recv, task_send],
# return_when=asyncio.FIRST_COMPLETED
# )
except Exception as e:
logger.error(f"WebSocket连接处理异常: {str(e)}")
if asr_conn:
asr_conn.stop_event.set()
if not websocket.client_state.disconnected:
await websocket.send_json({"error": str(e)})
finally:
pass
# 7. 资源清理
# logger.info(f"开始清理连接 {conn_id} 的资源")
# # 停止ASR
# if asr_conn:
# asr_conn.stop_event.set()
#
# # 取消任务
# if asr_task and not asr_task.done():
# asr_task.cancel()
# try:
# await asr_task
# except asyncio.CancelledError:
# pass
#
# # 清理TTS
# if tts_manager:
# await tts_manager.cleanup() # 假设TTSManager有cleanup方法,无则忽略
#
# # 断开连接
# if websocket:
# self.disconnect(websocket, conn_id)
# try:
# await websocket.close()
# except Exception:
# pass
#
# logger.info(f"连接 {conn_id} 资源清理完成")
+56
View File
@@ -0,0 +1,56 @@
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.middleware.cors import CORSMiddleware
from audio_ai_chat.config.settings import settings
from audio_ai_chat.config.logger import logger
from audio_ai_chat.core.websocket_handler import WebSocketConnectionManager
# from audio_ai_chat.models.ws_models import WSRequest, WSResponse, WSError
from audio_ai_chat.codec.ProtocolCodec import ProtocolCodec # 已有加密类
from audio_ai_chat.utils.exceptions import CodecError, ServiceCallError
from contextlib import asynccontextmanager
from frontend_ws import frontend_websocket_handler
# FastAPI 启动时初始化 ASR 连接池
@asynccontextmanager
async def lifespan(app: FastAPI):
# 启动时执行(原 startup 逻辑)
print(' FastAPI 启动时初始化 ASR 连接池')
# await init_asr_pool()
yield # 应用运行中
# 关闭时执行(可选,比如清理连接池)
print("应用关闭,开始清理 ASR 连接池...")
# await close_asr_pool()
app = FastAPI(
title="语音AI对话系统",
description="整合前端/ASR/大模型/TTS的异步网关服务",
version="1.0",
lifespan=lifespan # 绑定生命周期
)
# 跨域配置(允许前端访问)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"], # 生产环境替换为具体前端域名
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# 初始化WebSocket连接管理器
ws_manager = WebSocketConnectionManager()
# 初始化编解码器(从配置读取密钥)
codec = ProtocolCodec()
@app.websocket("/ws/audio")
async def websocket_audio(websocket: WebSocket):
# 完全委托给frontend_websocket_handler处理
await ws_manager.handle_connection(websocket)
@app.get("/health")
async def health_check():
"""服务健康检查接口"""
return {"status": "healthy", "service": "audio-ai-chat"}
@@ -0,0 +1,15 @@
class AudioAIChatBaseException(Exception):
"""项目基础异常"""
pass
class CodecError(AudioAIChatBaseException):
"""编解码异常(加密/解密失败)"""
pass
class ServiceCallError(AudioAIChatBaseException):
"""服务调用异常(ASR/LLM/TTS调用失败)"""
pass
class ValidationError(AudioAIChatBaseException):
"""数据验证异常(请求参数不符合模型)"""
pass
@@ -0,0 +1,26 @@
from typing import Callable, Awaitable, TypeVar
from tenacity import retry, stop_after_attempt, wait_exponential
from audio_ai_chat.config.logger import logger
T = TypeVar("T")
def async_retry(
times: int = 3,
delay: int = 1,
max_delay: int = 5
) -> Callable[[Callable[..., Awaitable[T]]], Callable[..., Awaitable[T]]]:
"""异步重试装饰器(通用)"""
def decorator(func: Callable[..., Awaitable[T]]) -> Callable[..., Awaitable[T]]:
@retry(
stop=stop_after_attempt(times),
wait=wait_exponential(multiplier=delay, min=delay, max=max_delay),
reraise=True
)
async def wrapper(*args, **kwargs) -> T:
try:
return await func(*args, **kwargs)
except Exception as e:
logger.warning(f"函数 {func.__name__} 调用失败,将重试(剩余次数:{retry.statistics.get('attempt_number', 0)}):{str(e)}")
raise
return wrapper
return decorator
View File
+9
View File
@@ -0,0 +1,9 @@
fastapi>=0.104.1
uvicorn>=0.24.0 # FastAPI运行服务器
websockets>=12.0 # WebSocket支持
pydantic>=2.4.2 # 数据模型校验
python-dotenv>=1.0.0 # 读取.env文件
loguru>=0.7.2 # 日志工具(比原生logging更简洁)
requests>=2.31.0 # 同步HTTP请求(若ASR/LLM/TTS用HTTP接口)
aiohttp>=3.9.1 # 异步HTTP请求(推荐,契合FastAPI异步特性)
tenacity>=8.2.3 # 重试机制(服务调用失败自动重试)
+16
View File
@@ -0,0 +1,16 @@
# run.py
import uvicorn
from dotenv import load_dotenv
# 加载环境变量
load_dotenv()
if __name__ == "__main__":
uvicorn.run(
app="audio_ai_chat.main:app",
host="0.0.0.0",
port=8000,
reload=True,
log_level="info", # 关键:开启详细日志
access_log=True,
)
View File
+295
View File
@@ -0,0 +1,295 @@
from enum import IntEnum
import struct
import json
import gzip
from typing import Optional, Union, Dict, Any, List, Tuple
# -------------------------- 协议常量定义 --------------------------
class ProtocolConst:
PROTOCOL_VERSION = 0b0001 # 协议版本(4位,字节0低4位,0~15)
HEADER_SIZE = 8 # 头部固定字节数(字节0~7
MAX_BODY_SIZE = 0xFFFFFF # 最大包体大小(24位,≈16MB,字节4~6存储)
STRING_ENCODING = "utf-8" # 字符串默认编码
# -------------------------- 枚举定义 --------------------------
class MessageType(IntEnum):
"""消息类型(4位,字节0高4位,0~15)"""
PING = 0b0001 # 心跳(支持空包体)
AUDIO_DATA = 0b0010 # 纯音频数据(pcm)
TEXT_MESSAGE = 0b0011 # 纯文本消息
CONTROL_CMD = 0b0100 # 控制指令
# 预留12种类型(0b0100 ~ 0b1111
class SerializationType(IntEnum):
"""序列化方式(3位,字节1高3位,1~8)"""
RAW = 0b001 # 原始二进制(1
JSON = 0b010 # JSON格式(2
STRING = 0b011 # 直接字符串(3
# 预留5种方式(0b100 ~ 0b111
class CompressionType(IntEnum):
"""压缩方式(3位,字节1中3位,1~8"""
NONE = 0b001 # 无压缩(默认值1
GZIP = 0b010 # gzip压缩(2
# 预留6种方式(0b011 ~ 0b111
class ControlCommand(IntEnum):
"""控制指令类型(配合MessageType.CONTROL_CMD使用)"""
HEARTBEAT = 0b0001 # 心跳响应
PAUSE = 0b0010 # 暂停
RESUME = 0b0011 # 继续
STOP = 0b0100 # 停止
# -------------------------- 协议工具类(与JS协议结构一致) --------------------------
class ProtocolCodec:
@staticmethod
def pack(
msg_type: MessageType,
body: Union[bytes, str, Dict[str, Any], List[Any], None] = None,
sequence: int = 0,
serialization: Optional[SerializationType] = None,
compression: CompressionType = CompressionType.NONE
) -> bytes:
"""
封装协议包(与JS pack 方法完全兼容)
:param msg_type: 消息类型
:param body: 包体数据(PING消息可传None/空,其他类型必填)
:param sequence: 消息顺序号(0~65535,默认0)
:param serialization: 序列化方式(None时自动推导)
:param compression: 压缩方式(默认无压缩)
:return: 完整协议包(bytes
"""
# 校验顺序号范围(0~65535
if not isinstance(sequence, int) or not (0 <= sequence <= 0xFFFF):
raise ValueError(f"消息顺序号必须是0~65535的整数,当前传入:{sequence}")
# 特殊处理:PING消息支持空包体
if msg_type == MessageType.PING:
# PING消息强制RAW序列化
serialization = SerializationType.RAW
# 空包体转为空bytes
if body is None:
body = b""
if not isinstance(body, bytes):
raise TypeError(f"PING消息仅支持空包体或bytes类型,当前传入:{type(body)}")
else:
# 非PING消息包体不能为空
if body is None:
raise ValueError(f"非PING消息({msg_type.name})包体不能为空")
# 1. 自动推导序列化方式(非PING消息)
if serialization is None and msg_type != MessageType.PING:
if msg_type == MessageType.AUDIO_DATA:
serialization = SerializationType.RAW
elif msg_type == MessageType.TEXT_MESSAGE:
serialization = SerializationType.STRING
elif msg_type == MessageType.CONTROL_CMD:
serialization = SerializationType.JSON
else:
raise ValueError(f"不支持的消息类型:{msg_type}")
# 2. 校验枚举值范围
if not (0b001 <= serialization.value <= 0b111):
raise ValueError(f"序列化方式必须在1~8(0b001~0b111)范围内,当前传入:{serialization.value}")
if not (0b001 <= compression.value <= 0b111):
raise ValueError(f"压缩方式必须在1~8(0b001~0b111)范围内,当前传入:{compression.value}")
# 3. 序列化包体
serialized_body: bytes
if serialization == SerializationType.RAW:
if not isinstance(body, bytes):
raise TypeError("RAW序列化要求body必须是bytes类型")
serialized_body = body
elif serialization == SerializationType.STRING:
if not isinstance(body, str):
raise TypeError("STRING序列化要求body必须是str类型")
serialized_body = body.encode(ProtocolConst.STRING_ENCODING)
elif serialization == SerializationType.JSON:
if isinstance(body, str):
serialized_body = body.encode(ProtocolConst.STRING_ENCODING)
elif isinstance(body, (dict, list)):
serialized_body = json.dumps(body, ensure_ascii=False, separators=(',', ':')).encode(ProtocolConst.STRING_ENCODING)
else:
raise TypeError("JSON序列化要求body必须是str/dict/list类型")
else:
raise ValueError(f"不支持的序列化方式:{serialization}")
# 4. 压缩包体
compressed_body: bytes
if compression == CompressionType.GZIP:
compressed_body = gzip.compress(serialized_body)
elif compression == CompressionType.NONE:
compressed_body = serialized_body
else:
raise ValueError(f"不支持的压缩方式:{compression}")
# 5. 校验包体大小(24位最大支持0xFFFFFF字节)
body_len = len(compressed_body)
if body_len > ProtocolConst.MAX_BODY_SIZE:
raise OverflowError(
f"包体过大({body_len}字节),最大支持{ProtocolConst.MAX_BODY_SIZE}字节(≈16MB"
)
# 6. 构造头部(与JS头部结构完全一致)
# 字节0:消息类型(高4位) + 协议版本(低4位)
byte0 = ((msg_type.value & 0x0F) << 4) | (ProtocolConst.PROTOCOL_VERSION & 0x0F)
# 字节1:序列化方式(高3位) + 压缩方式(中3位) + 保留位(低2位)
byte1 = ((serialization.value & 0x07) << 5) | ((compression.value & 0x07) << 2) | 0x00
# 字节2~3:消息顺序号(16位大端序)
byte2_3 = struct.pack(">H", sequence)
# 字节4~6:包体长度(24位大端序)
byte4 = (body_len >> 16) & 0xFF
byte5 = (body_len >> 8) & 0xFF
byte6 = body_len & 0xFF
# 字节7:保留位(固定0x00
byte7 = 0x00
# 拼接头部
header = (
bytes([byte0, byte1]) +
byte2_3 +
bytes([byte4, byte5, byte6, byte7])
)
assert len(header) == ProtocolConst.HEADER_SIZE, f"头部长度错误:实际{len(header)}字节,预期{ProtocolConst.HEADER_SIZE}字节"
return header + compressed_body
@staticmethod
def unpack(packet: bytes) -> Tuple[MessageType, int, Any]:
"""
解析协议包(与JS unpack 方法完全兼容)
:param packet: 完整协议包(头部 + 包体)
:return: (消息类型, 顺序号, 原始包体数据) # 仅返回核心必要信息
"""
# 1. 校验包长度
if len(packet) < ProtocolConst.HEADER_SIZE:
raise ValueError(
f"包长度过短({len(packet)}字节),至少需要{ProtocolConst.HEADER_SIZE}字节头部"
)
# 2. 解析头部
header = packet[:ProtocolConst.HEADER_SIZE]
body = packet[ProtocolConst.HEADER_SIZE:]
# 字节0:消息类型(高4位) + 协议版本(低4位)
byte0 = header[0]
msg_type = MessageType((byte0 >> 4) & 0x0F)
version = byte0 & 0x0F
# 字节1:序列化方式(高3位) + 压缩方式(中3位) + 保留位(低2位)
byte1 = header[1]
serialization = SerializationType((byte1 >> 5) & 0x07)
compression = CompressionType((byte1 >> 2) & 0x07)
# 字节2~3:消息顺序号(16位大端序)
sequence = struct.unpack(">H", header[2:4])[0]
# 字节4~6:包体长度(24位大端序),字节7:保留位(忽略)
body_len = (header[4] << 16) | (header[5] << 8) | header[6]
# 校验版本和包体长度(校验逻辑保留,确保数据有效性)
if version != ProtocolConst.PROTOCOL_VERSION:
raise ValueError(
f"协议版本不匹配:收到v{version}0b{version:04b}),支持v{ProtocolConst.PROTOCOL_VERSION}0b{ProtocolConst.PROTOCOL_VERSION:04b}"
)
if len(body) != body_len:
raise ValueError(
f"包体长度不匹配:头部声明{body_len}字节,实际{len(body)}字节"
)
# 3. 解压包体
decompressed_body: bytes
if compression == CompressionType.GZIP:
try:
decompressed_body = gzip.decompress(body)
except Exception as e:
raise ValueError(f"GZIP解压失败:{str(e)}")
elif compression == CompressionType.NONE:
decompressed_body = body
else:
raise ValueError(f"不支持的压缩方式:{compression}")
# 4. 反序列化包体(空包体返回None)
original_body: Any
if len(decompressed_body) == 0:
original_body = None
elif serialization == SerializationType.RAW:
original_body = decompressed_body
elif serialization == SerializationType.STRING:
try:
original_body = decompressed_body.decode(ProtocolConst.STRING_ENCODING)
except UnicodeDecodeError:
raise ValueError(f"STRING反序列化失败:{ProtocolConst.STRING_ENCODING}解码错误")
elif serialization == SerializationType.JSON:
try:
original_body = json.loads(decompressed_body.decode(ProtocolConst.STRING_ENCODING))
except UnicodeDecodeError:
raise ValueError(f"JSON反序列化失败:{ProtocolConst.STRING_ENCODING}解码错误")
except json.JSONDecodeError as e:
raise ValueError(f"JSON反序列化失败:格式错误({str(e)}")
else:
raise ValueError(f"不支持的序列化方式:{serialization}")
# 仅返回核心必要信息:消息类型、顺序号、原始包体数据
return msg_type, sequence, original_body
# -------------------------- 使用示例(验证修改后功能正常) --------------------------
if __name__ == "__main__":
# 示例1:PING消息(空包体,默认顺序号0)
ping_packet = ProtocolCodec.pack(MessageType.PING)
print(f"PING消息包长度:{len(ping_packet)}字节(仅头部)")
msg_type1, seq1, body1 = ProtocolCodec.unpack(ping_packet)
print(f"PING解析结果:类型={msg_type1.name},顺序号={seq1},包体={body1}\n")
# 示例2:纯文本消息(STRING序列化,指定顺序号)
text_body = "Python与JS协议兼容测试(纯字符串)"
text_packet = ProtocolCodec.pack(
msg_type=MessageType.TEXT_MESSAGE,
body=text_body,
sequence=1001,
compression=CompressionType.NONE
)
print(f"文本消息包长度:{len(text_packet)}字节")
msg_type2, seq2, body2 = ProtocolCodec.unpack(text_packet)
print(f"文本解析结果:类型={msg_type2.name},顺序号={seq2},内容={body2}\n")
# 示例3:控制指令(JSON序列化)
control_body = {"cmd": ControlCommand.PAUSE.value, "reason": "用户主动暂停"}
control_packet = ProtocolCodec.pack(
msg_type=MessageType.CONTROL_CMD,
body=control_body,
sequence=1002
)
print(f"控制指令包长度:{len(control_packet)}字节")
msg_type3, seq3, body3 = ProtocolCodec.unpack(control_packet)
print(f"控制指令解析结果:类型={msg_type3.name},顺序号={seq3},内容={body3}\n")
# 示例4:音频数据(RAW序列化)
audio_body = b"\x00\x01\x02\x03\x04\x05" * 100 # 模拟PCM数据
audio_packet = ProtocolCodec.pack(
msg_type=MessageType.AUDIO_DATA,
body=audio_body,
sequence=1003
)
print(f"音频数据包长度:{len(audio_packet)}字节")
msg_type4, seq4, body4 = ProtocolCodec.unpack(audio_packet)
print(f"音频解析结果:类型={msg_type4.name},顺序号={seq4},数据长度={len(body4)}字节\n")
# 示例5:GZIP压缩测试(需JS端启用GZIP解压)
long_text_body = "这是一段很长的文本,用于测试GZIP压缩效果" * 100
gzip_packet = ProtocolCodec.pack(
msg_type=MessageType.TEXT_MESSAGE,
body=long_text_body,
sequence=1004,
compression=CompressionType.GZIP
)
print(f"GZIP压缩后包长度:{len(gzip_packet)}字节(原始文本长度:{len(long_text_body.encode())}字节)")
msg_type5, seq5, body5 = ProtocolCodec.unpack(gzip_packet)
print(f"GZIP解析结果:类型={msg_type5.name},顺序号={seq5},内容前50字:{body5[:50]}...")
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,285 @@
<template>
<view class="container">
<button @click="onStartRecord" :disabled="isRecording">开始录音</button>
<button @click="onStopRecord" :disabled="!isRecording">停止录音</button>
<view class="tip">{{ status }}</view>
<view class="tip">当前分贝{{ currentDecibels }}</view>
<yao-RecordFrame ref="recordFrame" @onFrameRecorded="frameRecorded" @currentDecibels="onCurrentDecibels"
@onStop="stopIt">
</yao-RecordFrame>
<sdx-StreamPlayer ref="aaa">xx</sdx-StreamPlayer>
<button @click="test">接通电话</button>
</view>
</template>
<script>
import {
ProtocolCodec,
MessageType,
SerializationType,
CompressionType,
ControlCommand,
ProtocolConst,
} from './ProtocolCodec'; // 确保协议文件也是 ESModule 格式(export 导出)
export default {
data() {
return {
status: "未录音",
currentDecibels: 0,
isRecording: false,
ws: null, // WebSocket 实例
audioContext: null, // 音频上下文
scriptProcessor: null, // 音频处理节点
audioBufferSource: null, // 音频源节点
frameBufferList: [], // 缓存音频帧
wsUrl: "ws://172.16.89.58:8000/ws/audio", // 替换为实际后端地址
};
},
onUnload() {
// 页面卸载时清理资源
this.stopRecordAndClean();
},
methods: {
test() {
this.ws = uni.connectSocket({
url: this.wsUrl,
fail: () => {
console.log('fail');
},
success: () => {
console.log('web');
this.ws.onOpen((res) => {
console.log('WebSocket连接已打开', res);
this.ws.onMessage((res) => {
let messageData = res.data;
// 处理不同类型的数据
if (typeof messageData === 'string') {
// 文本数据
try {
const parsedData = JSON.parse(messageData);
this.handleMessage(parsedData);
} catch (e) {
this.handleMessage(messageData);
}
} else if (messageData instanceof ArrayBuffer) {
// 二进制数据
try {
this.$refs.aaa.appendBuffer(messageData)
// const str = this.arrayBufferToStringCompat(messageData);
// console.log('转换后的字符串:', str);
} catch (e) {
console.log('二进制数据解析失败:', e);
}
}
})
});
},
fail: () => {
console.log('fail');
},
});
},
// 申请录音权限
async applyRecordPermission() {
try {
const res = await uni.requestPermissions({
scope: "scope.record"
});
const isGranted = res[0].grantStatus === 1;
if (!isGranted) {
uni.showToast({
title: "请授予录音权限",
icon: "none"
});
}
return isGranted;
} catch (e) {
console.error("申请权限失败:", e);
return false;
}
},
// ArrayBuffer 转字符串
// 兼容的 ArrayBuffer 转字符串方法
arrayBufferToStringCompat(buffer) {
// 方法1: 使用 String.fromCharCode 和 Uint8Array
const uint8Array = new Uint8Array(buffer);
let str = '';
for (let i = 0; i < uint8Array.length; i++) {
str += String.fromCharCode(uint8Array[i]);
}
return str;
// 方法2: 或者使用更简洁的方式
// return String.fromCharCode.apply(null, new Uint8Array(buffer));
},
// 开始录音
async onStartRecord() {
// 1. 申请权限
// const hasPermission = await this.applyRecordPermission();
// if (!hasPermission) return;
this.ws = uni.connectSocket({
url: this.wsUrl, //仅为示例,并非真实接口地址。
complete: () => {
console.log('complete');
},
success: () => {
console.log('web');
this.ws.onOpen((res) => {
console.log('WebSocket连接已打开', res);
this.ws.onMessage((res) => {
let messageData = res.data;
// 处理不同类型的数据
if (typeof messageData === 'string') {
// 文本数据
try {
const parsedData = JSON.parse(messageData);
this.handleMessage(parsedData);
} catch (e) {
this.handleMessage(messageData);
}
} else if (messageData instanceof ArrayBuffer) {
// 二进制数据
try {
this.$refs.aaa.appendBuffer(messageData)
// const str = this.arrayBufferToStringCompat(messageData);
// console.log('转换后的字符串:', str);
} catch (e) {
console.log('二进制数据解析失败:', e);
}
}
})
});
},
fail: () => {
console.log('fail');
},
});
// await this.initWebSocket();
// 3. 启动录音
try {
this.$refs.recordFrame.start({
sampleRate: 16000,
frameSize: 1024,
gain: 1.0,
onFrameRecorded: ({
isLastFrame,
frameBuffer
}) => {
// 处理帧数据(如实时上传/渲染)
// console.log('帧数据:', frameData);
this.ws.send({
data: ProtocolCodec.pack(MessageType.AUDIO_DATA, frameBuffer)
});
},
onDecibels: (decibels) => {
// 处理分贝数据(如实时更新UI
// console.log('当前分贝:', decibels);
this.currentDecibels = decibels;
}
});
this.isRecording = true;
this.status = "录音中...";
} catch (e) {
console.error("启动录音失败:", e);
this.status = "启动录音失败";
this.isRecording = false;
}
},
// 停止录音
onStopRecord() {
this.stopRecordAndClean();
},
// 停止录音并清理资源
stopRecordAndClean() {
if (this.isRecording) {
// 停止录音组件
this.$refs.recordFrame.stop();
this.isRecording = false;
this.status = "已停止录音";
}
// 关闭 WebSocket
if (this.ws) {
this.ws.close();
this.ws = null;
}
// 清理音频上下文
if (this.audioContext) {
this.audioContext.close();
this.audioContext = null;
this.scriptProcessor = null;
this.frameBufferList = [];
}
// 重置状态
this.currentDecibels = 0;
},
// 接收音频帧并处理
frameRecorded({
isLastFrame,
frameBuffer
}) {
// console.log("收到音频帧:", isLastFrame, frameBuffer.length);
// 2. 通过 WebSocket 发送给后端
if (this.ws) {
try {
this.ws.send({
data: frameBuffer
});
} catch (e) {
console.error("发送音频帧失败:", e);
}
}
},
// 监听分贝值
onCurrentDecibels(decibels) {
// this.currentDecibels = decibels.toFixed(2);
// console.log("当前分贝:", this.currentDecibels);
},
// 录音停止回调
stopIt(base64) {
this.stopRecordAndClean();
console.log("录音停止,最终音频Base64", base64?.substring(0, 50) + "...");
},
},
};
</script>
<style scoped>
.container {
padding: 20rpx;
}
button {
margin: 10rpx 0;
padding: 15rpx 30rpx;
background: #007aff;
color: white;
border: none;
border-radius: 8rpx;
}
button:disabled {
background: #ccc;
}
.tip {
margin: 15rpx 0;
font-size: 28rpx;
color: #333;
}
</style>
@@ -79,6 +79,7 @@
success: () => {
console.log('web');
this.ws.onOpen((res) => {
console.log('WebSocket连接已打开', res);
this.ws.onMessage((res) => {
let messageData = res.data;
+26 -211
View File
@@ -4,225 +4,40 @@
<button @click="onStopRecord" :disabled="!isRecording">停止录音</button>
<view class="tip">{{ status }}</view>
<view class="tip">当前分贝{{ currentDecibels }}</view>
<yao-RecordFrame ref="recordFrame" @onFrameRecorded="frameRecorded" @currentDecibels="onCurrentDecibels"
@onStop="stopIt">
</yao-RecordFrame>
<yao-RecordFrame
ref="recordFrame"
@onFrameRecorded="frameRecorded"
@currentDecibels="onCurrentDecibels"
@onStop="stopIt"
></yao-RecordFrame>
<sdx-StreamPlayer ref="aaa">xx</sdx-StreamPlayer>
<button @click="test">测试</button>
<button @click="callPhone">接通电话</button>
</view>
</template>
<script>
<script setup>
import { onMounted, onUnmounted } from 'vue';
import {
ProtocolCodec,
MessageType,
SerializationType,
CompressionType,
ControlCommand,
ProtocolConst,
ProtocolCodec,
MessageType,
SerializationType,
CompressionType,
ControlCommand,
ProtocolConst
} from './ProtocolCodec'; // 确保协议文件也是 ESModule 格式(export 导出)
export default {
data() {
return {
status: "未录音",
currentDecibels: 0,
isRecording: false,
ws: null, // WebSocket 实例
audioContext: null, // 音频上下文
scriptProcessor: null, // 音频处理节点
audioBufferSource: null, // 音频源节点
frameBufferList: [], // 缓存音频帧
wsUrl: "ws://172.16.89.58:8000/ws/audio", // 替换为实际后端地址
};
},
onUnload() {
// 页面卸载时清理资源
this.stopRecordAndClean();
},
methods: {
import { ref } from 'vue';
import useWebSocket from './useWebSocket';
const { initWebSocket, closeWebSocket } = useWebSocket();
test() {
// 组件挂载时初始化连接
onMounted(() => {
initWebSocket();
});
},
// 申请录音权限
async applyRecordPermission() {
try {
const res = await uni.requestPermissions({
scope: "scope.record"
});
const isGranted = res[0].grantStatus === 1;
if (!isGranted) {
uni.showToast({
title: "请授予录音权限",
icon: "none"
});
}
return isGranted;
} catch (e) {
console.error("申请权限失败:", e);
return false;
}
},
// ArrayBuffer 转字符串
// 兼容的 ArrayBuffer 转字符串方法
arrayBufferToStringCompat(buffer) {
// 方法1: 使用 String.fromCharCode 和 Uint8Array
const uint8Array = new Uint8Array(buffer);
let str = '';
for (let i = 0; i < uint8Array.length; i++) {
str += String.fromCharCode(uint8Array[i]);
}
return str;
// 方法2: 或者使用更简洁的方式
// return String.fromCharCode.apply(null, new Uint8Array(buffer));
},
// 开始录音
async onStartRecord() {
// 1. 申请权限
// const hasPermission = await this.applyRecordPermission();
// if (!hasPermission) return;
this.ws = uni.connectSocket({
url: this.wsUrl, //仅为示例,并非真实接口地址。
complete: () => {
console.log('complete');
},
success: () => {
console.log('web');
this.ws.onOpen((res) => {
console.log('WebSocket连接已打开', res);
this.ws.onMessage((res) => {
let messageData = res.data;
// 处理不同类型的数据
if (typeof messageData === 'string') {
// 文本数据
try {
const parsedData = JSON.parse(messageData);
this.handleMessage(parsedData);
} catch (e) {
this.handleMessage(messageData);
}
} else if (messageData instanceof ArrayBuffer) {
// 二进制数据
try {
this.$refs.aaa.appendBuffer(messageData)
// const str = this.arrayBufferToStringCompat(messageData);
// console.log('转换后的字符串:', str);
} catch (e) {
console.log('二进制数据解析失败:', e);
}
}
})
});
},
fail: () => {
console.log('fail');
},
});
// await this.initWebSocket();
// 3. 启动录音
try {
this.$refs.recordFrame.start({
sampleRate: 16000,
frameSize: 1024,
gain: 1.0,
onFrameRecorded: ({
isLastFrame,
frameBuffer
}) => {
// 处理帧数据(如实时上传/渲染)
// console.log('帧数据:', frameData);
this.ws.send({
data: ProtocolCodec.pack(MessageType.AUDIO_DATA, frameBuffer);
});
},
onDecibels: (decibels) => {
// 处理分贝数据(如实时更新UI
// console.log('当前分贝:', decibels);
this.currentDecibels = decibels;
}
});
this.isRecording = true;
this.status = "录音中...";
} catch (e) {
console.error("启动录音失败:", e);
this.status = "启动录音失败";
this.isRecording = false;
}
},
// 停止录音
onStopRecord() {
this.stopRecordAndClean();
},
// 停止录音并清理资源
stopRecordAndClean() {
if (this.isRecording) {
// 停止录音组件
this.$refs.recordFrame.stop();
this.isRecording = false;
this.status = "已停止录音";
}
// 关闭 WebSocket
if (this.ws) {
this.ws.close();
this.ws = null;
}
// 清理音频上下文
if (this.audioContext) {
this.audioContext.close();
this.audioContext = null;
this.scriptProcessor = null;
this.frameBufferList = [];
}
// 重置状态
this.currentDecibels = 0;
},
// 接收音频帧并处理
frameRecorded({
isLastFrame,
frameBuffer
}) {
// console.log("收到音频帧:", isLastFrame, frameBuffer.length);
// 2. 通过 WebSocket 发送给后端
if (this.ws) {
try {
this.ws.send({
data: frameBuffer
});
} catch (e) {
console.error("发送音频帧失败:", e);
}
}
},
// 监听分贝值
onCurrentDecibels(decibels) {
// this.currentDecibels = decibels.toFixed(2);
// console.log("当前分贝:", this.currentDecibels);
},
// 录音停止回调
stopIt(base64) {
this.stopRecordAndClean();
console.log("录音停止,最终音频Base64", base64?.substring(0, 50) + "...");
},
},
};
// 组件卸载时关闭连接(可选)
onUnmounted(() => {
closeWebSocket();
});
</script>
<style scoped>
@@ -0,0 +1,77 @@
import { ref } from 'vue';
// 在组件 setup 函数中使用
export default function useWebSocket() {
// 存储 WebSocket 实例
const ws = ref(null);
const wsUrl = 'ws://127.0.0.1:8000/ws/audio';
// 初始化 WebSocket 连接
const initWebSocket = () => {
ws.value = uni.connectSocket({
url: wsUrl,
success: () => {
console.log('web');
// 监听连接打开
ws.value.onOpen((res) => {
console.log('WebSocket连接已打开', res);
// 监听消息接收
ws.value.onMessage((res) => {
let messageData = res.data;
// 处理不同类型的数据
if (typeof messageData === 'string') {
// 文本数据
try {
const parsedData = JSON.parse(messageData);
handleMessage(parsedData);
} catch (e) {
handleMessage(messageData);
}
} else if (messageData instanceof ArrayBuffer) {
// 二进制数据
try {
// 注意:组合式 API 中需通过 ref 获取子组件实例
const aaaRef = ref(null); // 需在组件中声明 <component ref="aaaRef" />
aaaRef.value?.appendBuffer(messageData);
} catch (e) {
console.log('二进制数据解析失败:', e);
}
}
});
});
},
fail: () => {
console.log('fail');
},
});
};
// 消息处理函数(根据实际业务逻辑修改)
const handleMessage = (data) => {
// 原组件中的 handleMessage 逻辑迁移到这里
console.log('收到消息:', data);
};
// 辅助函数:ArrayBuffer 转字符串(如果后续需要使用)
const arrayBufferToStringCompat = (buffer) => {
const decoder = new TextDecoder('utf-8');
return decoder.decode(buffer);
};
// 关闭连接函数(可选,按需暴露)
const closeWebSocket = () => {
if (ws.value) {
ws.value.close();
console.log('WebSocket连接已关闭');
}
};
return {
ws,
initWebSocket,
closeWebSocket,
handleMessage
};
};
@@ -1,8 +1,8 @@
{
"hash": "030e727a",
"configHash": "c22f3258",
"lockfileHash": "c10b225f",
"browserHash": "77dd6173",
"hash": "56a2a6b1",
"configHash": "0d2436b6",
"lockfileHash": "1c743963",
"browserHash": "42eed156",
"optimized": {},
"chunks": {}
}