tianyan/aistream-test
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x

Browse Source
This commit is contained in:
田岩
2025-12-04 17:42:00 +08:00
parent 6065e73d5b
commit d97aaa770d
43 changed files with 691 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files
测试流式传输uniapp/pages/index/StreamPlayer.js
+201
View File
@@ -0,0 +1,201 @@
/**
* 支持安卓、iOS、H5的pcm格式，
* 如果想支持mp3，需要增加mp3相关解码合并处理等，个人测试效果不太好，所以没有加
*/
export class StreamPlayer {
constructor({
inputSampleRate = 16000,
numChannels = 1,
bitDepth = 16,
littleEndian = true,
pcmType = 'int',
callback = ()=>{}
} = {}) {
// 参数校验
if (![16, 32].includes(bitDepth)) throw new Error('bitDepth 必须是 16 或 32');
if (inputSampleRate <= 0) throw new Error('采样率必须大于 0');
if (!['int', 'float'].includes(pcmType)) throw new Error('pcmType 必须是 int 或 float');
this.audioContext = new(window.AudioContext || window.webkitAudioContext)();
this.inputSampleRate = inputSampleRate;
this.numChannels = numChannels;
this.bitDepth = bitDepth;
this.littleEndian = littleEndian;
this.pcmType = pcmType;
this.audioQueue = [];
this.playbackEndTime = 0;
this.currentSource = null;
this.lastBlockTail = null; // 用于交叉淡入淡出
this.callback = callback;
}
appendChunk(audioData) {
console.log('this.audioQueue', this.audioQueue.length);
this.audioQueue.push(audioData);
this._processQueue();
}
destroy() {
this.audioQueue = [];
this.playbackEndTime = 0;
if (this.currentSource) {
this.currentSource.stop();
this.currentSource = null;
}
if (this.audioContext) {
this.audioContext.close();
this.audioContext = null;
}
}
async _processQueue() {
if(this.audioQueue.length === 0) {
this.callback('ended')
return
}
if (!this.audioContext || this.currentSource) return;
const buffer = this._mergeArrayBuffers(this.audioQueue)
this.audioQueue = []
try {
const audioBuffer = this._convertPCM(buffer);
await this._schedulePlay(audioBuffer);
this._processQueue();
} catch (err) {
console.error('音频处理失败:', err.name, err.message, err.stack);
}
}
_convertPCM(buffer) {
// 1. 数据对齐：确保长度是样本大小的整数倍
const bytesPerSample = this.bitDepth / 8;
const requiredBytes = bytesPerSample * this.numChannels;
const validByteLength = Math.floor(buffer.byteLength / requiredBytes) * requiredBytes;
const validBuffer = buffer.slice(0, validByteLength);
const dataView = new DataView(validBuffer);
const numSamples = validByteLength / requiredBytes;
// 2. 创建音频缓冲区
const audioBuffer = this.audioContext.createBuffer(
this.numChannels,
numSamples,
this.inputSampleRate
);
// 3. 解析PCM数据
for (let ch = 0; ch < this.numChannels; ch++) {
const channelData = audioBuffer.getChannelData(ch);
for (let i = 0; i < numSamples; i++) {
// 计算字节位置（修正后的多声道索引）
const pos = (i * this.numChannels * bytesPerSample) + (ch * bytesPerSample);
let value;
// 根据位深和数据类型解析
if (this.bitDepth === 16) {
value = dataView.getInt16(pos, this.littleEndian) / 32768;
} else {
if (this.pcmType === 'int') {
value = dataView.getInt32(pos, this.littleEndian) / 2147483648;
} else {
value = dataView.getFloat32(pos, this.littleEndian);
}
}
channelData[i] = value;
}
// 4. 消除直流偏移
let sum = 0;
for (let i = 0; i < channelData.length; i++) {
sum += channelData[i];
}
const dcOffset = sum / channelData.length;
for (let i = 0; i < channelData.length; i++) {
channelData[i] -= dcOffset;
}
// 5. 块内淡入淡出（减少边界突变）
const fadeLength = Math.min(100, channelData.length);
for (let i = 0; i < fadeLength; i++) {
channelData[i] *= (i / fadeLength);
}
const startFadeOut = Math.max(0, channelData.length - fadeLength);
for (let i = startFadeOut; i < channelData.length; i++) {
const factor = 1 - (i - startFadeOut) / fadeLength;
channelData[i] *= factor;
}
}
// 6. 块间交叉淡入淡出（解决重音关键点）
const crossfadeLength = 50;
if (this.lastBlockTail && this.lastBlockTail.length === this.numChannels) {
for (let ch = 0; ch < this.numChannels; ch++) {
const channelData = audioBuffer.getChannelData(ch);
const prevTail = this.lastBlockTail[ch];
// 只处理有足够长度交叉的情况
if (prevTail.length >= crossfadeLength && channelData.length >= crossfadeLength) {
for (let i = 0; i < crossfadeLength; i++) {
const prevWeight = (crossfadeLength - i) / crossfadeLength;
const currWeight = i / crossfadeLength;
channelData[i] = channelData[i] * currWeight + prevTail[i] * prevWeight;
}
}
}
}
// 7. 保存当前块尾部用于下一次交叉
this.lastBlockTail = [];
for (let ch = 0; ch < this.numChannels; ch++) {
const channelData = audioBuffer.getChannelData(ch);
const tailStart = Math.max(0, channelData.length - crossfadeLength);
this.lastBlockTail[ch] = channelData.slice(tailStart);
}
return audioBuffer;
}
_schedulePlay(audioBuffer) {
return new Promise(resolve => {
const source = this.audioContext.createBufferSource();
source.buffer = audioBuffer;
source.connect(this.audioContext.destination);
const plannedStartTime = this.playbackEndTime;
const now = this.audioContext.currentTime;
let startTime = Math.max(plannedStartTime, now);
// 动态速率调整（示例，需根据实际测试调整阈值）
const timeDrift = plannedStartTime - now;
if (timeDrift < -0.02) {
source.playbackRate.value = Math.min(1.04, 1 - (timeDrift / audioBuffer.duration));
} else if (timeDrift > 0.02) {
source.playbackRate.value = Math.max(0.96, 1 - (timeDrift / audioBuffer.duration));
} else {
source.playbackRate.value = 1.0;
}
source.start(startTime);
this.playbackEndTime = startTime + (audioBuffer.duration / source.playbackRate.value);
source.onended = () => {
this.currentSource = null;
resolve();
};
this.currentSource = source;
});
}
// 合并多个 ArrayBuffer 的辅助函数
_mergeArrayBuffers(buffers) {
let totalLength = 0;
for (let buffer of buffers) {
totalLength += buffer.byteLength;
}
const result = new Uint8Array(totalLength);
let offset = 0;
for (let buffer of buffers) {
const view = new Uint8Array(buffer);
result.set(view, offset);
offset += view.length;
}
return result.buffer;
}
}
测试流式传输uniapp/pages/index/index.vue
+108 -46
View File
@@ -4,11 +4,23 @@
<button @click="onStopRecord" :disabled="!isRecording">停止录音</button>
<view class="tip">{{ status }}</view>
<view class="tip">当前分贝：{{ currentDecibels }}</view>
<yao-RecordFrame ref="recordFrame" @onFrameRecorded="frameRecorded" @currentDecibels="onCurrentDecibels"
@onStop="stopIt">
</yao-RecordFrame>
<sdx-StreamPlayer ref="aaa">xx</sdx-StreamPlayer>
<yao-RecordFrame
ref="recordFrame"
@onFrameRecorded="frameRecorded"
@currentDecibels="onCurrentDecibels"
@onStop="stopIt"
></yao-RecordFrame>
<!-- <sdx-StreamPlayer ref="aaa">xx</sdx-StreamPlayer> -->
<button @click="test">接通电话</button>
<view
ref="renderJSModule"
:pcmChunk="currentPCMChunk"
:change:pcmChunk="renderJS.appendPCMChunk"
:isStop="isStop"
:change:isStop="renderJS.stop"
type="renderjs"
module="renderJS"
></view>
</view>
</template>
@@ -19,12 +31,14 @@
SerializationType,
CompressionType,
ControlCommand,
ProtocolConst,
} from './ProtocolCodec';
ProtocolConst
} from './ProtocolCodec';
export default {
data() {
return {
status: "未录音",
currentPCMChunk: null,
isStop: false, // 停止播放指令
status: '未录音',
currentDecibels: 0,
isRecording: false,
ws: null, // WebSocket 实例
@@ -32,9 +46,10 @@
scriptProcessor: null, // 音频处理节点
audioBufferSource: null, // 音频源节点
frameBufferList: [], // 缓存音频帧
wsUrl: "ws://127.0.0.1:8000/ws/audio", // 替换为实际后端地址
// wsUrl: "ws://172.16.89.58:8000/ws/audio", // 替换为实际后端地址
// wsUrl: "ws://127.0.0.1:8000/ws/audio", // 替换为实际后端地址
// wsUrl: 'ws://172.16.89.58:8000/ws/audio' // 替换为实际后端地址
// wsUrl: "ws://25.64.32.157:9603/trapractice/voiceCallSocket/voiceCall?summary=3D072C9D242C96EA3F0FD647CB14A6F4B4BEA4493BB2FAAB065935FB405071B8A903128E488B6D06BEB23FD4E3D15EE6", // 替换为实际后端地址
wsUrl: "wss://aitstest.jlbank.com.cn:7001/trapractice/voiceCallSocket/voiceCall?summary=3D072C9D242C96EA3F0FD647CB14A6F44A8F9EF1C3D9726612B7339B566489139850DF3BE002DF99495FD60B34A5899D", // 替换为实际后端地址
};
},
onUnload() {
@@ -53,7 +68,7 @@
},
fail: () => {
console.log('fail');
},
}
});
this.ws.onOpen((res) => {
console.log('WebSocket连接已打开', res);
@@ -66,39 +81,41 @@
});
});
this.ws.onMessage((res) => {
const {
msgType,
body
} = ProtocolCodec.unpack(res.data)
const { msgType, body } = ProtocolCodec.unpack(res.data);
if (msgType === MessageType.IDENTITY) {
console.log('身份校验成功', body);
this.onStartRecord()
this.onStartRecord();
} else if (msgType === MessageType.AUDIO_DATA) {
this.$refs.aaa.appendBuffer(body)
}else {
// this.$refs.aaa.appendBuffer(body)
// this.currentPCMChunk = body;
const buffer = body;
if (buffer.byteLength < 2) return;
// ArrayBuffer转数字数组
const uint8 = new Uint8Array(buffer);
const numArray = Array.from(uint8);
this.currentPCMChunk = numArray; // 传递数组
} else {
console.log('其他消息', body);
}
})
});
},
// 申请录音权限
async applyRecordPermission() {
try {
const res = await uni.requestPermissions({
scope: "scope.record"
scope: 'scope.record'
});
const isGranted = res[0].grantStatus === 1;
if (!isGranted) {
uni.showToast({
title: "请授予录音权限",
icon: "none"
title: '请授予录音权限',
icon: 'none'
});
}
return isGranted;
} catch (e) {
console.error("申请权限失败：", e);
console.error('申请权限失败：', e);
return false;
}
},
@@ -124,25 +141,22 @@
sampleRate: 16000,
frameSize: 1024,
gain: 1.0,
onFrameRecorded: ({
isLastFrame,
frameBuffer
}) => {
try{
onFrameRecorded: ({ isLastFrame, frameBuffer }) => {
try {
this.ws.send({
data: ProtocolCodec.pack(MessageType.AUDIO_DATA, frameBuffer)
});
} catch(e){}
} catch (e) {}
},
onDecibels: (decibels) => {
this.currentDecibels = decibels;
}
});
this.isRecording = true;
this.status = "录音中...";
this.status = '录音中...';
} catch (e) {
console.error("启动录音失败：", e);
this.status = "启动录音失败";
console.error('启动录音失败：', e);
this.status = '启动录音失败';
this.isRecording = false;
}
},
@@ -158,7 +172,7 @@
// 停止录音组件
this.$refs.recordFrame.stop();
this.isRecording = false;
this.status = "已停止录音";
this.status = '已停止录音';
}
// 关闭 WebSocket
@@ -180,10 +194,7 @@
},
// 接收音频帧并处理
frameRecorded({
isLastFrame,
frameBuffer
}) {
frameRecorded({ isLastFrame, frameBuffer }) {
// console.log("收到音频帧：", isLastFrame, frameBuffer.length);
// 2. 通过 WebSocket 发送给后端
@@ -193,11 +204,9 @@
data: frameBuffer
});
} catch (e) {
console.error("发送音频帧失败：", e);
console.error('发送音频帧失败：', e);
}
}
},
// 监听分贝值
@@ -209,9 +218,62 @@
// 录音停止回调
stopIt(base64) {
this.stopRecordAndClean();
console.log("录音停止，最终音频Base64：", base64?.substring(0, 50) + "...");
},
},
console.log('录音停止，最终音频Base64：', base64?.substring(0, 50) + '...');
}
}
};
</script>
<script module="renderJS" lang="renderjs">
import { StreamPlayer } from ".//StreamPlayer";
let player = null;
export default {
mounted() {
const _this = this;
player = new StreamPlayer({
inputSampleRate: 44100, // 后端PCM采样率（如8000/24000）
numChannels: 1, // 声道数（单声道/双声道）
bitDepth: 16, // 位深（16/32bit）
littleEndian: true, // 端序（和后端一致）
pcmType: 'int', // 类型（int/float）
callback: _this.callback
});
},
methods: {
appendPCMChunk(numArray) {
if (!numArray || numArray.length === 0) return;
// 数字数组转回ArrayBuffer
const uint8 = new Uint8Array(numArray);
const arrayBuffer = uint8.buffer;
player.appendChunk(arrayBuffer);
},
// 停止播放（销毁播放器）
stop() {
if (player) {
player.destroy();
player = null;
// 重新初始化（方便后续再次播放）
const _this = this;
player = new StreamPlayer({
inputSampleRate: 16000,
numChannels: 1,
bitDepth: 16,
littleEndian: true,
pcmType: 'int',
callback: _this.callback
});
}
},
// 播放结束回调（通知主线程）
callback(e) {
if (e === 'ended') {
this.$ownerInstance.callMethod('changeStreamPlaying', { type: false });
}
}
}
};
</script>
@@ -238,4 +300,4 @@
font-size: 28rpx;
color: #333;
}
</style>
</style>