276 lines
7.7 KiB
JavaScript
276 lines
7.7 KiB
JavaScript
|
/**
|
|||
|
|
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights
|
|||
|
|
* Reserved. MIT License (https://opensource.org/licenses/MIT)
|
|||
|
|
*/
|
|||
|
|
/* 2022-2023 by zhaoming,mali aihealthx.com */
|
|||
|
|
|
|||
|
|
|
|||
|
|
// 连接; 定义socket连接类对象与语音对象
|
|||
|
|
var wsconnecter = new WebSocketConnectMethod({ msgHandle: getJsonMessage, stateHandle: getConnState });
|
|||
|
|
var audioBlob;
|
|||
|
|
var isfilemode = true; // if it is in file mode
|
|||
|
|
// 录音; 定义录音对象,wav格式
|
|||
|
|
var rec = Recorder({
|
|||
|
|
type: "pcm",
|
|||
|
|
bitRate: 16,
|
|||
|
|
sampleRate: 16000,
|
|||
|
|
onProcess: recProcess
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
var sampleBuf = new Int16Array();
|
|||
|
|
|
|||
|
|
var rec_text = ""; // for online rec asr result
|
|||
|
|
var offline_text = ""; // for offline rec asr result
|
|||
|
|
|
|||
|
|
|
|||
|
|
var file_ext = "";
|
|||
|
|
var file_sample_rate = 16000; //for wav file sample rate
|
|||
|
|
var file_data_array; // array to save file data
|
|||
|
|
var totalsend = 0;
|
|||
|
|
|
|||
|
|
addresschange();
|
|||
|
|
function addresschange() {
|
|||
|
|
var Uri = 'ws://192.168.0.232:10095';
|
|||
|
|
Uri = Uri.replace(/wss/g, "https");
|
|||
|
|
window.open(Uri, '_blank');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
var readWavInfo = function (bytes) {
|
|||
|
|
//读取wav文件头，统一成44字节的头
|
|||
|
|
if (bytes.byteLength < 44) {
|
|||
|
|
return null;
|
|||
|
|
};
|
|||
|
|
var wavView = bytes;
|
|||
|
|
var eq = function (p, s) {
|
|||
|
|
for (var i = 0; i < s.length; i++) {
|
|||
|
|
if (wavView[p + i] != s.charCodeAt(i)) {
|
|||
|
|
return false;
|
|||
|
|
};
|
|||
|
|
};
|
|||
|
|
return true;
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
if (eq(0, "RIFF") && eq(8, "WAVEfmt ")) {
|
|||
|
|
|
|||
|
|
var numCh = wavView[22];
|
|||
|
|
if (wavView[20] == 1 && (numCh == 1 || numCh == 2)) {//raw pcm 单或双声道
|
|||
|
|
var sampleRate = wavView[24] + (wavView[25] << 8) + (wavView[26] << 16) + (wavView[27] << 24);
|
|||
|
|
var bitRate = wavView[34] + (wavView[35] << 8);
|
|||
|
|
var heads = [wavView.subarray(0, 12)], headSize = 12;//head只保留必要的块
|
|||
|
|
//搜索data块的位置
|
|||
|
|
var dataPos = 0; // 44 或有更多块
|
|||
|
|
for (var i = 12, iL = wavView.length - 8; i < iL;) {
|
|||
|
|
if (wavView[i] == 100 && wavView[i + 1] == 97 && wavView[i + 2] == 116 && wavView[i + 3] == 97) {//eq(i,"data")
|
|||
|
|
heads.push(wavView.subarray(i, i + 8));
|
|||
|
|
headSize += 8;
|
|||
|
|
dataPos = i + 8; break;
|
|||
|
|
}
|
|||
|
|
var i0 = i;
|
|||
|
|
i += 4;
|
|||
|
|
i += 4 + wavView[i] + (wavView[i + 1] << 8) + (wavView[i + 2] << 16) + (wavView[i + 3] << 24);
|
|||
|
|
if (i0 == 12) {//fmt
|
|||
|
|
heads.push(wavView.subarray(i0, i));
|
|||
|
|
headSize += i - i0;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (dataPos) {
|
|||
|
|
var wavHead = new Uint8Array(headSize);
|
|||
|
|
for (var i = 0, n = 0; i < heads.length; i++) {
|
|||
|
|
wavHead.set(heads[i], n); n += heads[i].length;
|
|||
|
|
}
|
|||
|
|
return {
|
|||
|
|
sampleRate: sampleRate
|
|||
|
|
, bitRate: bitRate
|
|||
|
|
, numChannels: numCh
|
|||
|
|
, wavHead44: wavHead
|
|||
|
|
, dataPos: dataPos
|
|||
|
|
};
|
|||
|
|
};
|
|||
|
|
};
|
|||
|
|
};
|
|||
|
|
return null;
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
function upfileOnchange(files) {
|
|||
|
|
this.files = [files];
|
|||
|
|
var len = this.files.length;
|
|||
|
|
for (let i = 0; i < len; i++) {
|
|||
|
|
let fileAudio = new FileReader();
|
|||
|
|
fileAudio.readAsArrayBuffer(this.files[i]);
|
|||
|
|
file_ext = this.files[i].name.split('.').pop().toLowerCase();
|
|||
|
|
var audioblob;
|
|||
|
|
fileAudio.onload = function () {
|
|||
|
|
audioblob = fileAudio.result;
|
|||
|
|
file_data_array = audioblob;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
fileAudio.onerror = function (e) {
|
|||
|
|
console.log('error' + e);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// for wav file, we get the sample rate
|
|||
|
|
if (file_ext == "wav") {
|
|||
|
|
for (let i = 0; i < len; i++) {
|
|||
|
|
let fileAudio = new FileReader();
|
|||
|
|
fileAudio.readAsArrayBuffer(this.files[i]);
|
|||
|
|
fileAudio.onload = function () {
|
|||
|
|
audioblob = new Uint8Array(fileAudio.result);
|
|||
|
|
|
|||
|
|
var info = readWavInfo(audioblob);
|
|||
|
|
file_sample_rate = info.sampleRate;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function play_file() {
|
|||
|
|
var audioblob = new Blob([new Uint8Array(file_data_array)], { type: "audio/wav" });
|
|||
|
|
var audio_record = document.getElementById('audio_record');
|
|||
|
|
audio_record.src = (window.URL || webkitURL).createObjectURL(audioblob);
|
|||
|
|
audio_record.controls = true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function start_file_send() {
|
|||
|
|
sampleBuf = new Uint8Array(file_data_array);
|
|||
|
|
var chunk_size = 960; // for asr chunk_size [5, 10, 5]
|
|||
|
|
while (sampleBuf.length >= chunk_size) {
|
|||
|
|
sendBuf = sampleBuf.slice(0, chunk_size);
|
|||
|
|
totalsend = totalsend + sampleBuf.length;
|
|||
|
|
sampleBuf = sampleBuf.slice(chunk_size, sampleBuf.length);
|
|||
|
|
wsconnecter.wsSend(sendBuf);
|
|||
|
|
}
|
|||
|
|
stop();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function stop() {
|
|||
|
|
var chunk_size = new Array(5, 10, 5);
|
|||
|
|
var request = {
|
|||
|
|
"chunk_size": chunk_size,
|
|||
|
|
"wav_name": "h5",
|
|||
|
|
"is_speaking": false,
|
|||
|
|
"chunk_interval": 10,
|
|||
|
|
"mode": getAsrMode(),
|
|||
|
|
};
|
|||
|
|
if (sampleBuf.length > 0) {
|
|||
|
|
wsconnecter.wsSend(sampleBuf);
|
|||
|
|
sampleBuf = new Int16Array();
|
|||
|
|
}
|
|||
|
|
wsconnecter.wsSend(JSON.stringify(request));
|
|||
|
|
// 控件状态更新
|
|||
|
|
isRec = false;
|
|||
|
|
if (isfilemode == false) {
|
|||
|
|
//wait 3s for asr result
|
|||
|
|
setTimeout(function () {
|
|||
|
|
wsconnecter.wsStop();
|
|||
|
|
}, 3000);
|
|||
|
|
rec.stop(function (blob, duration) {
|
|||
|
|
var audioBlob = Recorder.pcm2wav(data = { sampleRate: 16000, bitRate: 16, blob: blob },
|
|||
|
|
function (theblob, duration) {
|
|||
|
|
console.log(theblob);
|
|||
|
|
var audio_record = document.getElementById('audio_record');
|
|||
|
|
audio_record.src = (window.URL || webkitURL).createObjectURL(theblob);
|
|||
|
|
audio_record.controls = true;
|
|||
|
|
}, function (msg) {
|
|||
|
|
console.log(msg);
|
|||
|
|
}
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
}, function (errMsg) {
|
|||
|
|
console.log("errMsg: " + errMsg);
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
// 停止连接
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function getAsrMode() {
|
|||
|
|
return 'offline';
|
|||
|
|
}
|
|||
|
|
function getHotwords() {
|
|||
|
|
return null
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
function handleWithTimestamp(tmptext, tmptime) {
|
|||
|
|
|
|||
|
|
if (tmptime == null || tmptime == "undefined" || tmptext.length <= 0) {
|
|||
|
|
return tmptext;
|
|||
|
|
}
|
|||
|
|
tmptext = tmptext.replace(/。|？|，|、|\?|\.|\ /g, ","); // in case there are a lot of "。"
|
|||
|
|
var words = tmptext.split(","); // split to chinese sentence or english words
|
|||
|
|
var jsontime = JSON.parse(tmptime); //JSON.parse(tmptime.replace(/\]\]\[\[/g, "],[")); // in case there are a lot segments by VAD
|
|||
|
|
var char_index = 0; // index for timestamp
|
|||
|
|
var text_withtime = "";
|
|||
|
|
for (var i = 0; i < words.length; i++) {
|
|||
|
|
if (words[i] == "undefined" || words[i].length <= 0) {
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
if (/^[a-zA-Z]+$/.test(words[i])) { // if it is english
|
|||
|
|
text_withtime = text_withtime + jsontime[char_index][0] / 1000 + ":" + words[i] + "\n";
|
|||
|
|
char_index = char_index + 1; //for english, timestamp unit is about a word
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
text_withtime = text_withtime + jsontime[char_index][0] / 1000 + ":" + words[i] + "\n";
|
|||
|
|
char_index = char_index + words[i].length; //for chinese, timestamp unit is about a char
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return text_withtime;
|
|||
|
|
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
// 语音识别结果; 对jsonMsg数据解析,将识别结果附加到编辑框中
|
|||
|
|
function getJsonMessage(jsonMsg) {
|
|||
|
|
var rectxt = "" + JSON.parse(jsonMsg.data)['text'];
|
|||
|
|
var asrmodel = JSON.parse(jsonMsg.data)['mode'];
|
|||
|
|
var is_final = JSON.parse(jsonMsg.data)['is_final'];
|
|||
|
|
var timestamp = JSON.parse(jsonMsg.data)['timestamp'];
|
|||
|
|
if (asrmodel == "2pass-offline" || asrmodel == "offline") {
|
|||
|
|
offline_text = offline_text + handleWithTimestamp(rectxt, timestamp); //rectxt; //.replace(/ +/g,"");
|
|||
|
|
rec_text = offline_text;
|
|||
|
|
} else {
|
|||
|
|
rec_text = rec_text + rectxt;
|
|||
|
|
}
|
|||
|
|
videoText = rec_text;
|
|||
|
|
if (is_final == true) {
|
|||
|
|
play_file();
|
|||
|
|
wsconnecter.wsStop();
|
|||
|
|
btnConnect.disabled = false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 连接状态响应
|
|||
|
|
function getConnState(connState) {
|
|||
|
|
if (connState === 0) start_file_send();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 识别启动、停止、清空操作
|
|||
|
|
function start() {
|
|||
|
|
var ret = wsconnecter.wsStart();//启动连接
|
|||
|
|
return ret == 1 ? 1 : 0;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function recProcess(buffer, powerLevel, bufferDuration, bufferSampleRate, newBufferIdx, asyncEnd) {
|
|||
|
|
if (isRec === true) {
|
|||
|
|
var data_48k = buffer[buffer.length - 1];
|
|||
|
|
var array_48k = new Array(data_48k);
|
|||
|
|
var data_16k = Recorder.SampleData(array_48k, bufferSampleRate, 16000).data;
|
|||
|
|
sampleBuf = Int16Array.from([...sampleBuf, ...data_16k]);
|
|||
|
|
var chunk_size = 960; // for asr chunk_size [5, 10, 5]
|
|||
|
|
while (sampleBuf.length >= chunk_size) {
|
|||
|
|
sendBuf = sampleBuf.slice(0, chunk_size);
|
|||
|
|
sampleBuf = sampleBuf.slice(chunk_size, sampleBuf.length);
|
|||
|
|
wsconnecter.wsSend(sendBuf);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function getUseITN() {
|
|||
|
|
return false;
|
|||
|
|
}
|