#endif
#include "dev_authentication.h"
#include "imsl_vad.h"
+#include "imsl_kws.h"
#include "aic_srv_tele_ex.h"
#include "power_supply.h"
#include "sysoff.h"
inbound_attributes_format_OPUS,
}inbound_attributes_format_type;
inbound_attributes_format_type inbound_attributes_format = inbound_attributes_format_PCM;
+#define OPUS_ENCODER_PACK_MS 60 //多少ms一帧数据
+#define OPUS_PERIOD_MS OPUS_PERIOD_60_MS
+#define OPUS_RB_LEN OPUS_ENCODER_PACK_MS*32*10//10贞数据
+bk_ring_buf_t *OPUS_rb_data = NULL;
AI_AMP_2_5_cfg vc_cfg;
volatile char app_audio_flag = 0;
volatile char audio_stop_flag = 0;
int32_t bk_audio_start_callback(void *p_param, uint32_t size);
+#define IS_KWS 0
+#define IS_VAD 1
+
#define VAD_RB_LEN 64*1024
bk_ring_buf_t *VAD_rb_data = NULL;
static u32 VAD_status = 0;//VAD状态
vc_cfg.VC_play_TYPE = VC_play_TYPE_PCM;//播放格式
vc_cfg.VC_record_samplerate = 16000;//录音采样率
vc_cfg.VC_play_samplerate = 16000;//播放采样率
- vc_cfg.is_VAD = 1;//=1打开vad打断
+ vc_cfg.is_VAD = IS_VAD;//=1打开vad打断
+ vc_cfg.is_KWS = IS_KWS;//=1打开本地语言打断
vc_cfg.speech_rate = 0;//语速配置
vc_cfg.vad_silence = 200;//vad断句时间 ms
vc_cfg.VC_TTS_volume = 50;//音量
if(inbound_attributes_format == inbound_attributes_format_OPUS)
{
cJSON_AddStringToObject(inbound_cfg, "codec", "opus");
- //cJSON_AddStringToObject(inbound_cfg, "format", "pcm");
+ cJSON_AddStringToObject(inbound_cfg, "format", "pcm");
}
else
cJSON_AddStringToObject(inbound_cfg, "format", "pcm");
}
-u16 opus_remain_buf[640];
-u32 opus_remain_size = 0;
+
static void AI_VC_send_task(void *thread_input)//websocket 发送线程
{
AI_VC_send_queue_t AI_VC_send_queue_data;
if(AI_client_VC && AI_VC_send_queue_data.data)
{
VC_pingping_timerout = 4;
- if(AI_VC_send_queue_data.AIC_WEBSOCKET_OPCODE ==AIC_WEBSOCKET_OPCODE_BINARY_FRAME &&inbound_attributes_format == inbound_attributes_format_OPUS)
- {
- #if SUPPORT_OPUS
- u8 *opus_data =aic_zalloc(AI_VC_send_queue_data.len);
- u8 *pcm_data =aic_zalloc(opus_remain_size+AI_VC_send_queue_data.len);
- //struct opus_dec_param dec_inparam;
- struct opus_enc_param enc_inparam;
- u32 opus_count = 0;
- u32 pcm_count = 0;
- if(opus_remain_size)//有剩余数据,先复制剩余数据
- {
- memcpy(pcm_data,opus_remain_buf,opus_remain_size);
- }
- memcpy(pcm_data+opus_remain_size,AI_VC_send_queue_data.data,AI_VC_send_queue_data.len);
- pcm_count = opus_remain_size+AI_VC_send_queue_data.len;
- opus_remain_size = 0;
- if(opus_data)
- {
-
- enc_inparam.input = (int16_t*)(pcm_data);//PCM数据
- enc_inparam.input_size = pcm_count;
- enc_inparam.output = (int8_t*)opus_data;
- enc_inparam.outbuffer_size = AI_VC_send_queue_data.len;
- ret = aic_opus_encode(&enc_inparam);
- if(enc_inparam.consumed_size<pcm_count)//有尾部数据没编码完
- {
- if(pcm_count-enc_inparam.consumed_size<=sizeof(opus_remain_buf))
- {
- opus_remain_size = pcm_count-enc_inparam.consumed_size;
- memcpy(opus_remain_buf,AI_VC_send_queue_data.data+enc_inparam.consumed_size,opus_remain_size);
-
- }else
- {
- bk_printf("ERROR opus编码剩余部分溢出了！！！！！！！！！！！！！！");
- }
- }
- opus_count = 0;
- for(int i=0;i<enc_inparam.ouputfrm_cnt;i++)
- {
-
- ret = send_websocket_message(AI_client_VC,opus_data+opus_count,enc_inparam.output_len_array[i],
- (WebSocketMessageType)AI_VC_send_queue_data.AIC_WEBSOCKET_OPCODE);
- opus_count += enc_inparam.output_len_array[i];
- }
- bk_printf("opus data send:opus_count=%d",opus_count);
- if(opus_data)
- aic_free(opus_data);
- if(pcm_data)
- aic_free(pcm_data);
-
- }else
- {
- bk_printf("opus alloc ERROR!!!!!!!!!!!");
- }
- #endif
- }else//正常pcm
+
{
ret = send_websocket_message(AI_client_VC,AI_VC_send_queue_data.data,AI_VC_send_queue_data.len,
(WebSocketMessageType)AI_VC_send_queue_data.AIC_WEBSOCKET_OPCODE);
bk_printf("AI_VC_read_task exit\n");
AI_VC_read_task_flag=3;
}
+
+u8 opus_remain_buf[OPUS_ENCODER_PACK_MS*32];
+u32 opus_remain_len = 0;
+u8 opus_data[OPUS_ENCODER_PACK_MS*16];
+void send_opus_end(void)
+{
+ opus_remain_len = 0;
+}
+//
+void send_opus_data(char *PCM_data,int data_len)
+{
+ #if SUPPORT_OPUS
+ int ret;
+
+ //struct opus_dec_param dec_inparam;
+ struct opus_enc_param enc_inparam;
+
+ if(opus_remain_len+data_len>=32*OPUS_ENCODER_PACK_MS)//够一帧数据
+ {
+ memcpy(opus_remain_buf+opus_remain_len,PCM_data,32*OPUS_ENCODER_PACK_MS-opus_remain_len);
+ bk_printf("send_opus_data:cpy1:偏移：%d，长度：%d",opus_remain_len,32*OPUS_ENCODER_PACK_MS-opus_remain_len);
+ enc_inparam.input = (int16_t*)(opus_remain_buf);//PCM数据
+ enc_inparam.input_size = 32*OPUS_ENCODER_PACK_MS;
+ enc_inparam.output = (int8_t*)opus_data;
+ enc_inparam.outbuffer_size = OPUS_ENCODER_PACK_MS*16;
+ ret = aic_opus_encode(&enc_inparam);
+ //enc_inparam.output_len_array[0];//长度
+ AI_VC_send_data(opus_data,enc_inparam.output_len_array[0],AIC_WEBSOCKET_OPCODE_BINARY_FRAME);//够一帧，发送数据
+ bk_printf("send_opus_data:send:data_len = %d",enc_inparam.output_len_array[0]);
+ memcpy(opus_remain_buf,PCM_data+32*OPUS_ENCODER_PACK_MS-opus_remain_len,data_len-32*OPUS_ENCODER_PACK_MS-opus_remain_len);
+ bk_printf("send_opus_data:cpy2:偏移：%d，长度：%d",32*OPUS_ENCODER_PACK_MS-opus_remain_len,data_len-32*OPUS_ENCODER_PACK_MS-opus_remain_len);
+ opus_remain_len = data_len-32*OPUS_ENCODER_PACK_MS-opus_remain_len;
+ }else
+ {
+ memcpy(opus_remain_buf+opus_remain_len,PCM_data,data_len);//缓存数据，等待够一帧数据
+ bk_printf("send_opus_data:cpy3:偏移：%d，长度：%d",opus_remain_len,data_len);
+ opus_remain_len+=data_len;
+ }
+
+ #endif
+
+}
u8 vrecord_buf[3500 + 4];
static void AI_VC_record_task(void *thread_input)//录音线程
{
if(vc_cfg.is_VAD == 1)//VAD逻辑：先判断有没有有效数据，触发了有效数据才发送
{
-
// 每次获得语音后调用, 语音长度最好小于100ms
// wave为u8*或void*指针时需要 / 2
run_vad((int16_t *)vrecord_buf, ret / 2);
audio_data(vrecord_buf,sizeof(vrecord_buf),21);
}else if(vad_status_t == 1 )//检测到说话
{
+
+ /*******************************************语言打断end************ */
if(VAD_status ==0)//第一次检测到说话
{
if(VAD_time !=0)
{
unsigned int r_len = bk_rb_read(VAD_rb_data,vad_r_tmp,2560);
+ audio_data(vad_r_tmp,r_len,21);
//AI_VC_send_data(vad_r_tmp,r_len,AIC_WEBSOCKET_OPCODE_BINARY_FRAME);//发送缓冲数据
AI_VC_put_record_data(vad_r_tmp,r_len);//入队缓冲数据
- audio_data(vad_r_tmp,r_len,21);
+ if(vc_cfg.is_KWS)
+ {
+ int KWS_time = aos_get_tickcount();
+ bk_printf("run_kws1: KWS_time:%d,data_len:%d",KWS_time,r_len);
+ run_kws((int16_t *)vad_r_tmp,r_len/2);
+ int kws_status = kws_detect();
+ bk_printf("run_kws2:end_time:%d ,kws_status:%d",aos_get_tickcount() - KWS_time,kws_status);
+ if (kws_status == 1) {
+ bk_printf("!!!!!!!!!!!!!!!!!!!!!!离线语音识别打断了!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n\n");
+ kws_clear();
+ }
+ }
+
}else
{
}
AI_VC_put_record_data(vrecord_buf,ret);
audio_data(vrecord_buf,ret,21);
+ if(vc_cfg.is_KWS)
+ {
+ int KWS_time = aos_get_tickcount();
+ bk_printf("run_kws1: KWS_time:%d,data_len:%d",KWS_time,ret);
+ run_kws((int16_t *)vrecord_buf,ret/2);
+ int kws_status = kws_detect();
+ bk_printf("run_kws2:end_time:%d ,kws_status:%d",aos_get_tickcount() - KWS_time,kws_status);
+ if (kws_status == 1) {
+ bk_printf("!!!!!!!!!!!!!!!!!!!!!!离线语音识别打断了!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n\n");
+ kws_clear();
+ }
+ }
//bk_printf("VAD:检测到说话了");
}else if(vad_status_t == 0 )//m没有说话
{
if(aos_recv_from_queue(AI_VC_record_tmp_queue,&AI_VC_play_record_data,AOS_NO_WAIT) == AOS_SUCCESS)//有有效录音数据
{
audio_data(AI_VC_play_record_data.data,AI_VC_play_record_data.len,22);
- AI_VC_send_data(AI_VC_play_record_data.data,AI_VC_play_record_data.len,AIC_WEBSOCKET_OPCODE_BINARY_FRAME);
+ if(inbound_attributes_format == inbound_attributes_format_OPUS)
+ {
+ send_opus_data(AI_VC_play_record_data.data,AI_VC_play_record_data.len);
+ }else
+ AI_VC_send_data(AI_VC_play_record_data.data,AI_VC_play_record_data.len,AIC_WEBSOCKET_OPCODE_BINARY_FRAME);
aic_free(AI_VC_play_record_data.data);
}else
break;
bk_printf("VAD:发送停止音频");
bk_printf("VAD:vad_status_t:%d",vad_status_t);
vad_end_flag = 0;
+ if(inbound_attributes_format == inbound_attributes_format_OPUS)
+ send_opus_end();
}
}
sizeof(AI_VC_audio_queue_t),
400);
}
- //创建VAD回溯缓冲buf
- if(vc_cfg.is_VAD == 1 && VAD_rb_data == NULL)
- {
- VAD_rb_data = bk_rb_create(VAD_RB_LEN);
- if(!VAD_rb_data )
- {
- bk_printf("VAD_rb_data error!申请失败");
- }
- }
+
}
//关闭录音
// aos_delete_queue(AI_VC_record_tmp_queue);
// AI_VC_record_tmp_queue = NULL;
}
- //删除VAD回溯缓冲buf
- if(VAD_rb_data != NULL)
- {
- bk_rb_destroy(&VAD_rb_data);
- }
+
}
static void AI_VC_1S_timeout(void *param)
play_data_len = 0;
if(app_audio_flag == 0)
{
+ bk_printf("app_audio_start 1");
app_audio_flag = 1;
status = app_audio_start(APP_AUDIO_TYPE_STREAM_DUPLEX, ¶m, bk_audio_start_callback);//初始化音频
bk_printf("app_audio_start status=%d",status);
if(inbound_attributes_format == inbound_attributes_format_OPUS)
{
#if SUPPORT_OPUS
+ bk_printf("opus_init start");
//int err;
if (opus_encoder_h == NULL)
{
}
opus_encoder_h->channel_cnt = 1;
opus_encoder_h->sample_rate = 16000;
- opus_encoder_h->period_ms = OPUS_PERIOD_20_MS;
+ opus_encoder_h->period_ms = OPUS_PERIOD_MS;
opus_encoder_h->bitrate = 16000;
status = aic_opus_init_encoder(opus_encoder_h);
if (status)
//opus_decoder_h = opus_decoder_create(16000, 1, &err);
}
+ if(OPUS_rb_data == NULL)
+ OPUS_rb_data = bk_rb_create(OPUS_RB_LEN);
+ if(!OPUS_rb_data )
+ {
+ bk_printf("OPUS_rb_data error!申请失败");
+ }
+ bk_printf("opus_init end");
#endif
}
conf.vad_log_level = 2;
init_vad(&conf);
VAD_status = 0;
-
+ //创建VAD回溯缓冲buf
+ if(VAD_rb_data == NULL)
+ VAD_rb_data = bk_rb_create(VAD_RB_LEN);
+ if(!VAD_rb_data )
+ {
+ bk_printf("VAD_rb_data error!申请失败");
+ ret = -1;
+ goto AI_VC_start_RET;
+ }
+
+ }
+ if(vc_cfg.is_KWS == 1)
+ {
+ struct KwsConfig kws_conf;
+ kws_conf.min_activate_len = 10;
+ kws_conf.active_threshold = 32;
+ kws_conf.max_activate_gap = 160;
+ kws_conf.kws_log_level = 0;
+ int kws_status = init_kws(&kws_conf);
+ bk_printf("init_kws:kws_status:%d",kws_status);
}
VC_record_Start();
VC_Play_Start();//
if(websocket_connect_VC() !=0)//
{
- AI_VC_Close();
- play_audio_file("C:/xinhaoruo1.mp3");
+
ret = -1;
goto AI_VC_start_RET;
close_timerout = 0;
AI_VC_start_RET:
AI_VC_start_lock = 0;
+ if(ret != 0)
+ {
+ AI_VC_Close();
+ play_audio_file("C:/xinhaoruo1.mp3");
+ }
return ret;
}
char AI_VC_Close_lock = 0;
{
// 不使用或暂停使用VAD时调用
free_vad();
+
VAD_status = 0;
+ //删除VAD回溯缓冲buf
+ if(VAD_rb_data != NULL)
+ {
+ bk_rb_destroy(&VAD_rb_data);
+ VAD_rb_data= NULL;
+ }
+ }
+ if(vc_cfg.is_KWS == 1)
+ {
+ free_kws();
}
-
bk_printf("AI_VC_Close!!!!!!!!!!!!");
//销毁录音线程
aic_free(opus_decoder_h);
opus_decoder_h = NULL;
}
+ if(OPUS_rb_data != NULL)
+ {
+ bk_rb_destroy(&OPUS_rb_data);
+ OPUS_rb_data= NULL;
+ }
#endif
}
bk_printf("app_audio_stop1");
}
}
}
+
static void AI_AMP_2_5_task(void *thread_input)//做逻辑线程
{
char count = 30;
app_audio_set_master_volume(15*vc_cfg.VC_TTS_volume/100);
play_audio_file_event("C:/R_POWER_1.mp3");
+
+ // volatile int32_t a = 1626;
+ // volatile int32_t b = 1536;
+ // volatile int32_t res = 0;
+ // int ai = 0;
+ // u32 time1 = aos_get_tickcount();
+ // for (ai = 0; ai < 245000; ai++) {
+ // res += (a + ai) * (b + ai);
+ // }
+ // u32 time2 = aos_get_tickcount();
+ // bk_printf("run int32 takes : %d\n", time2 - time1);
+
+ // volatile int64_t a1 = 1626;
+ // volatile int64_t b1 = 1536;
+ // volatile int64_t res1 = 0;
+ // // int ai = 0;
+ // time1 = aos_get_tickcount();
+ // for (ai = 0; ai < 245000; ai++) {
+ // res1 += (a1 + ai) * (b1 + ai);
+ // }
+ // time2 = aos_get_tickcount();
+ // bk_printf("run int64 takes : %d\n", time2 - time1);
+
+ // volatile float a2 = 1626.0f;
+ // volatile float b2 = 1536.0f;
+ // volatile float res2 = 0;
+ // // int ai = 0;
+ // time1 = aos_get_tickcount();
+ // for (ai = 0; ai < 245000; ai++) {
+ // res2 += (a2 + ai) * (b2 + ai);
+ // }
+ // time2 = aos_get_tickcount();
+ // bk_printf("run float takes : %d\n", time2 - time1);
+
+
while(count --)
{
pdp_activated = aic_is_default_pdp_activated();
current_card_id = aic_srv_tele_sim_get_current_card_sync();
aic_srv_tele_radio_get_current_slot_imei_sync(vc_cfg.dev_imei);
bk_printf("IMEI:%s",vc_cfg.dev_imei);
+ memset(vc_cfg.ICCID,0,sizeof(vc_cfg.ICCID));
aic_srv_tele_sim_get_iccid_sync(current_card_id,vc_cfg.ICCID);
bk_printf("vc_cfg.ICCID:%s",vc_cfg.ICCID);
dev_authentication_init();
--- /dev/null
+#include <stdint.h>
+struct KwsConfig
+{
+ int min_activate_len; // 最小检测多少ms的激活才算检测到关键词 !!! 必须小于 max_activate_gap 和 200
+ int max_activate_gap; // 关键词激活点最长持续时间 比如 100ms 时检测到关键词但是未达到最短长度，直到100ms + max_activate_gap 这个时间点使激活失效
+ int active_threshold; // 激活阈值
+
+ int kws_log_level; // kws log打印控制， 0 不打印，1 INFO，2 INFO DEBUG
+};
+
+/*
+初始化kws解码器，默认配置见下方的example
+*/
+int init_kws(struct KwsConfig* conf);
+void run_kws(const int16_t* data, const int data_len);
+int kws_detect();
+void kws_clear();
+void free_kws();
+
+/* example 只使用kws
+
+// 初始化
+struct KwsConfig kws_conf;
+kws_conf.min_activate_len = 10;
+kws_conf.active_threshold = 32;
+kws_conf.max_activate_gap = 160;
+kws_conf.kws_log_level = 0;
+
+int kws_status = init_kws(&kws_conf);
+
+// 在每次接收语音时调用 注意 如果是void* 或 int8* 的指针，长度 / 2
+// 运行结束后获取kws状态
+run_kws(wave, chunk_size / 2);
+int kws_status = kws_detect();
+if (kws_status == 1) {
+ kws_clear();
+}
+
+// 不使用kws或kws暂时停用时调用
+free_kws();
+
+*/
+/* example 配合vad使用
+// 初始化
+
+
+struct VadConfig vad_conf;
+vad_conf.sil_threshold = 63;
+vad_conf.energy_threshold = 5000;
+vad_conf.min_speech_length = 240;
+vad_conf.silence_detect_length = 300;
+vad_conf.rule1_min_tailing_silence = 480;
+vad_conf.rule2_min_tailing_silence = 960;
+vad_conf.rule3_min_tailing_silence = 960;
+vad_conf.vad_log_level = 2;
+
+int vad_status = init_vad(&vad_conf);
+
+struct KwsConfig kws_conf;
+kws_conf.min_activate_len = 10;
+kws_conf.active_threshold = 32;
+kws_conf.max_activate_gap = 160;
+kws_conf.kws_log_level = 0;
+
+int kws_status = init_kws(&kws_conf);
+
+const int chunk_size = 480; // 这里的chunk size每次送入kws语音的点数 比如 30ms 就是 30 * 16
+const int chunk_cache = 9; // 由于vad存在一定延时，需要cache一些chunk防止丢失头部语音
+const int cache_size = chunk_cache * chunk_size; // chunk_cache 的值需要根据chunk_size的值变化 9*480/chunk_size
+int16_t wave_cache[cache_size] = {0}; // !!!!!!!!!注意我这边是int16_t
+int prev_vad_status = 0; // 记录vad状态，只在vad刚检测到语音时使用cache语音
+
+
+// 每次接收到语音时
+run_vad(wave + i, chunk_size);
+
+if (vad_status == 1) {
+ if (prev_vad_status == 0 || prev_vad_status == 2) {
+ for (j = 0; j < chunk_cache; j++) {
+ u32 time4 = timer_get_ms();
+ run_kws(wave_cache + j * chunk_size, chunk_size);
+ u32 time5 = timer_get_ms();
+ printf("run kws takes %d ms\n", time5 - time4);
+ }
+ }
+ run_kws(wave + i, chunk_size);
+ int kws_status = kws_detect();
+ if (kws_status == 1) {
+ printf("!!!!!!!!!!!!!!!!!!!!!!keyword detected!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n");
+ kws_clear();
+ }
+} else if (vad_status == 2) {
+ vad_clear();
+}
+
+prev_vad_status = vad_status;
+memcpy(wave_cache, wave_cache + chunk_size, sizeof(int16_t) * (cache_size - chunk_size));
+memcpy(wave_cache + (cache_size - chunk_size), wave + i, sizeof(int16_t) * chunk_size);
+
+
+// kws暂停使用时
+free_kws();
+
+// vad暂停使用时
+free_vad();
+
+*/
projects / mirrors / FX01.git / commitdiff