#include "bk_websocket.h"
#include "AI_AMP_2_5.h"
#include "bk_key.h"
+#include "opus/guixin/opus_interface.h"
+#include "dev_authentication.h"
+// #include "opus/include/opus.h"
+typedef enum
+{
+ inbound_attributes_format_PCM,
+ inbound_attributes_format_OPUS,
+}inbound_attributes_format_type;
+inbound_attributes_format_type inbound_attributes_format = inbound_attributes_format_PCM;
+char VC_mode = 0;//0：短按连续问答模式 1：长按录音模式
#define ai_VC_task_priority 35
static aos_thread_id AI_AMP_task_id = 0;
static aos_queue_ptr AI_AMP_task_queue = NULL;
static u32 VC_END_play_timerout = 0;
static char *AI_TTS_str = NULL;
static aos_timer_ptr AI_VC_1S_timer = NULL;
+// OpusEncoder *opus_encoder_h = NULL;
+// OpusDecoder *opus_decoder_h = NULL;
+
+static struct opus_enc_init_param *opus_encoder_h = NULL;
+static struct opus_dec_init_param *opus_decoder_h = NULL;
static void AI_VC_send_satrt(void);
static aos_status AI_VC_send_data(char *data,u32 len,u8 AIC_WEBSOCKET_OPCODE);
VC_state_type VC_state = VC_State_None;
free(json_str);
cJSON_Delete(root);
}
+//开始事件
static void AI_VC_send_satrt(void)
{
// 创建根对象
// 创建 payload 对象
cJSON *payload = cJSON_CreateObject();
- cJSON_AddStringToObject(payload, "device_id", "125698745641");
+ cJSON_AddStringToObject(payload, "device_id", "12569874564165");
cJSON_AddStringToObject(payload, "user_agent", "AudioClient/1.0");
// 创建 dialog_attributes 对象
// 创建 inbound_attributes 对象
cJSON *inbound_attributes = cJSON_CreateObject();
cJSON_AddStringToObject(inbound_attributes, "provider", "volcengine");
- cJSON_AddBoolToObject(inbound_attributes, "disable_vad", 0);
+ if(VC_mode == 0)//连续对话模式
+ cJSON_AddBoolToObject(inbound_attributes, "disable_vad", 0);
+ else
+ cJSON_AddBoolToObject(inbound_attributes, "disable_vad", 1);
cJSON_AddNumberToObject(inbound_attributes, "vad_silence", 500);
// 创建 interrupt_options 对象
// 创建 inbound_attributes 下的 cfg 对象
cJSON *inbound_cfg = cJSON_CreateObject();
- cJSON_AddStringToObject(inbound_cfg, "format", "pcm");
+ if(inbound_attributes_format == inbound_attributes_format_OPUS)
+ {
+ cJSON_AddStringToObject(inbound_cfg, "codec", "opus");
+ //cJSON_AddStringToObject(inbound_cfg, "format", "pcm");
+ }
+ else
+ cJSON_AddStringToObject(inbound_cfg, "format", "pcm");
+
cJSON_AddNumberToObject(inbound_cfg, "sample_rate", 16000);
cJSON_AddNumberToObject(inbound_cfg, "bit_rate", 16);
cJSON_AddNumberToObject(inbound_cfg, "channel", 1);
// 释放内存
cJSON_Delete(root);
}
+//Audio.Create 开始发送音频
+static void AI_VC_send_audio_create(void)
+{
+ // 创建根对象
+ cJSON *root = cJSON_CreateObject();
+ if (root == NULL) {
+ bk_printf("Failed to create root object.\n");
+ return ;
+ }
+
+ // 设置 "event" 字段
+ cJSON_AddStringToObject(root, "event", "audio.create");
+
+ // 创建 "payload" 对象
+ cJSON *payload = cJSON_CreateObject();
+ if (payload == NULL) {
+ bk_printf("Failed to create payload object.\n");
+ cJSON_Delete(root);
+ return ;
+ }
+ // 将 "payload" 对象添加到根对象中
+ cJSON_AddItemToObject(root, "payload", payload);
+
+ // 将 JSON 对象转换为字符串
+ char *json_str = cJSON_Print(root);
+ if (json_str == NULL) {
+ bk_printf("Failed to print JSON object.\n");
+ cJSON_Delete(root);
+
+ }
+ AI_VC_send_data(json_str,strlen(json_str),AIC_WEBSOCKET_OPCODE_TEXT_FRAME);//发送数据
+
+
+ // 释放内存
+ free(json_str);
+ cJSON_Delete(root);
+}
+//Audio.Completed 结束发送音频
+static void AI_VC_send_audio_completed(void)
+{
+ // 创建根对象
+ cJSON *root = cJSON_CreateObject();
+ if (root == NULL) {
+ bk_printf("Failed to create root object.\n");
+ return ;
+ }
+
+ // 设置 "event" 字段
+ cJSON_AddStringToObject(root, "event", "audio.completed");
+
+ // 创建 "payload" 对象
+ cJSON *payload = cJSON_CreateObject();
+ if (payload == NULL) {
+ bk_printf("Failed to create payload object.\n");
+ cJSON_Delete(root);
+ return ;
+ }
+ // 将 "payload" 对象添加到根对象中
+ cJSON_AddItemToObject(root, "payload", payload);
+
+ // 将 JSON 对象转换为字符串
+ char *json_str = cJSON_Print(root);
+ if (json_str == NULL) {
+ bk_printf("Failed to print JSON object.\n");
+ cJSON_Delete(root);
+
+ }
+ AI_VC_send_data(json_str,strlen(json_str),AIC_WEBSOCKET_OPCODE_TEXT_FRAME);//发送数据
+ // 释放内存
+ free(json_str);
+ cJSON_Delete(root);
+}
//TTS播放
static void VC_TTS_Play(const char *str)
{
"AI_VC_send",
AI_VC_send_task,
&AI_VC_send_task_id,
- 4096u,
+ 1024*16,
0,
NULL,
ai_VC_task_priority,
return AOS_SUCCESS;
}
+u16 opus_remain_buf[640];
+u32 opus_remain_size = 0;
static void AI_VC_send_task(void *thread_input)//websocket 发送线程
{
AI_VC_send_queue_t AI_VC_send_queue_data;
bk_printf("AI_VC_send_task go......");
-
+ int ret = 0;
while(1)
{
if(AI_VC_send_queue && aos_recv_from_queue(AI_VC_send_queue,&AI_VC_send_queue_data,AOS_WAIT_FOREVER) == AOS_SUCCESS)
if(AI_client_VC && AI_VC_send_queue_data.data)
{
VC_pingping_timerout = 5;
- int ret = send_websocket_message(AI_client_VC,AI_VC_send_queue_data.data,AI_VC_send_queue_data.len,
- (WebSocketMessageType)AI_VC_send_queue_data.AIC_WEBSOCKET_OPCODE);
+ if(AI_VC_send_queue_data.AIC_WEBSOCKET_OPCODE ==AIC_WEBSOCKET_OPCODE_BINARY_FRAME &&inbound_attributes_format == inbound_attributes_format_OPUS)
+ {
+ u8 *opus_data =aic_zalloc(AI_VC_send_queue_data.len);
+ u8 *pcm_data =aic_zalloc(opus_remain_size+AI_VC_send_queue_data.len);
+ struct opus_dec_param dec_inparam;
+ struct opus_enc_param enc_inparam;
+ u32 opus_count = 0;
+ u32 pcm_count = 0;
+ if(opus_remain_size)//有剩余数据,先复制剩余数据
+ {
+ memcpy(pcm_data,opus_remain_buf,opus_remain_size);
+ }
+ memcpy(pcm_data+opus_remain_size,AI_VC_send_queue_data.data,AI_VC_send_queue_data.len);
+ pcm_count = opus_remain_size+AI_VC_send_queue_data.len;
+ opus_remain_size = 0;
+ if(opus_data)
+ {
+ enc_inparam.input = (int16_t*)(pcm_data);//PCM数据
+ enc_inparam.input_size = pcm_count;
+ enc_inparam.output = (int8_t*)opus_data;
+ enc_inparam.outbuffer_size = AI_VC_send_queue_data.len;
+ ret = aic_opus_encode(&enc_inparam);
+ if(enc_inparam.consumed_size<pcm_count)//有尾部数据没编码完
+ {
+ if(pcm_count-enc_inparam.consumed_size<=sizeof(opus_remain_buf))
+ {
+ opus_remain_size = pcm_count-enc_inparam.consumed_size;
+ memcpy(opus_remain_buf,AI_VC_send_queue_data.data+enc_inparam.consumed_size,opus_remain_size);
+
+ }else
+ {
+ bk_printf("ERROR opus编码剩余部分溢出了！！！！！！！！！！！！！！");
+ }
+ }
+ opus_count = 0;
+ for(int i=0;i<enc_inparam.ouputfrm_cnt;i++)
+ {
+
+ ret = send_websocket_message(AI_client_VC,opus_data+opus_count,enc_inparam.output_len_array[i],
+ (WebSocketMessageType)AI_VC_send_queue_data.AIC_WEBSOCKET_OPCODE);
+ opus_count += enc_inparam.output_len_array[i];
+ }
+ bk_printf("opus data send:opus_count=%d",opus_count);
+ //测试:
+ // memset(pcm_data,0,AI_VC_send_queue_data.len);
+ // dec_inparam.input = opus_data;
+ // dec_inparam.inputfrm_count = enc_inparam.ouputfrm_cnt;
+ // memcpy(dec_inparam.input_len_array , enc_inparam.output_len_array, sizeof(dec_inparam.input_len_array));
+ // dec_inparam.output = (int16_t*)pcm_data;
+ // dec_inparam.outbuffer_size = AI_VC_send_queue_data.len;
+ // ret = aic_opus_decode(&dec_inparam);
+ // audio_data(pcm_data,dec_inparam.outputfrms_size,25);
+ ////aos_sleep(10);
+ //bk_printf("OPUS:inputfrm_count:%d,opus_remain_size:%d,pcm_len:%d,dec_pcm_len:%d,opus_count:%d",\
+ // dec_inparam.inputfrm_count,opus_remain_size,AI_VC_send_queue_data.len,dec_inparam.outputfrms_size,opus_count);
+ if(opus_data)
+ aic_free(opus_data);
+ if(pcm_data)
+ aic_free(pcm_data);
+ }else
+ {
+ bk_printf("opus alloc ERROR!!!!!!!!!!!");
+ }
+
+ }else//正常pcm
+ {
+ ret = send_websocket_message(AI_client_VC,AI_VC_send_queue_data.data,AI_VC_send_queue_data.len,
+ (WebSocketMessageType)AI_VC_send_queue_data.AIC_WEBSOCKET_OPCODE);
+ }
+
if(AI_VC_send_queue_data.AIC_WEBSOCKET_OPCODE ==AIC_WEBSOCKET_OPCODE_TEXT_FRAME )
bk_printf("VCVC:send ret=%d,len=%d,str=%s",ret,AI_VC_send_queue_data.len,AI_VC_send_queue_data.data);
// else if(AI_VC_send_queue_data.AIC_WEBSOCKET_OPCODE ==AIC_WEBSOCKET_OPCODE_BINARY_FRAME )
bk_printf("VCVC:read TEXT:status:%d,data:%s\n",status,recv_buff);
if(strstr(recv_buff,"\"started\""))//开始
{
- VC_state = VC_State_Listening;
+ if(VC_mode == 0)//连续对话模式
+ VC_state = VC_State_Listening;
if(AI_TTS_str)
{
AI_client_VC_synthesis(AI_TTS_str);
aic_free(AI_TTS_str);
AI_TTS_str = NULL;
VC_state = VC_State_TTSStart;
+ }else
+ {
+ if(VC_mode == 1)//长按模式
+ AI_VC_send_audio_create();
}
- }else if(strstr(recv_buff,"\"vad.active\""))//vad触发
+ }else if(strstr(recv_buff,"\"asr.started\""))//asr开始
+ {
+ if(VC_mode == 1)//长按模式
+ VC_state = VC_State_Listening;
+ }
+ else if(strstr(recv_buff,"\"vad.active\""))//vad触发
{
if(VC_state >= VC_State_TTSStart)//要关闭播放声音
{
VC_END_play_timerout --;
if(VC_END_play_timerout<=0)
{
- //bk_msg_printf("VC_END_play_timerout1=%d",VC_END_play_timerout);
- VC_state = VC_State_Listening;//重新开始了
+ if(VC_mode == 0)
+ VC_state = VC_State_Listening;//重新开始了
+ else
+ AI_AMP_MSG_put(0,0,VC_CLOSE);
//AI_AMP_MSG_put(0,0,VC_RECORD_START);//开启录音
- //bk_msg_printf("VC_END_play_timerout2=%d",VC_END_play_timerout);
+ bk_msg_printf("VC_END_play_timerout=%d",VC_END_play_timerout);
}
}
if(AI_VC_play_queue)
VC_state = VC_State_None;
VC_record_Start() ;
AI_VC_start();
+
+
}
void AI_key_release(void)
{
+
bk_printf("AI_key_release");
}
void AI_key_long_press(void)
{
+ AI_VC_Close();
bk_printf("AI_key_long_press");
}
void pwk_key_release(void)
{
bk_printf("pwk_key_release");
- AI_VC_Close();
+ if(VC_mode == 1)//长按模式
+ AI_VC_send_audio_completed();
+
}
+
void AI_VC_start(void)
{
int ret = 0;
AOS_AUTO_ACTIVATE, AOS_BE_PERIOD);
aic_assert(AI_VC_1S_timer);
}
+ if(inbound_attributes_format == inbound_attributes_format_OPUS)
+ {
+ //int err;
+ if (opus_encoder_h == NULL)
+ {
+ opus_encoder_h = (opus_enc_init_param_t *)aic_zalloc(sizeof(opus_enc_init_param_t));
+ if(!opus_encoder_h)
+ {
+ bk_printf("opus_encoder_h alloc error");
+ return ;
+ }
+ opus_encoder_h->channel_cnt = 1;
+ opus_encoder_h->sample_rate = 16000;
+ opus_encoder_h->period_ms = OPUS_PERIOD_20_MS;
+ opus_encoder_h->bitrate = 16000;
+ ret = aic_opus_init_encoder(opus_encoder_h);
+ if (ret)
+ {
+ bk_printf("aic_opus_init_encoder error");
+ return ;
+ }
+ //opus_encoder_h = opus_encoder_create(16000, 1, OPUS_APPLICATION_VOIP, &err);
+
+ }
+
+ if (opus_decoder_h == NULL)
+ {
+
+ opus_decoder_h = (opus_dec_init_param_t *)aic_zalloc(sizeof(opus_dec_init_param_t));
+ if(!opus_decoder_h)
+ {
+ bk_printf("opus_decoder_h alloc error");
+ return;
+ }
+ opus_decoder_h->channel_cnt = 1;
+ opus_decoder_h->sample_rate = 16000;
+ ret = aic_opus_init_decoder(opus_decoder_h);
+ if (ret)
+ {
+ bk_printf("aic_opus_init_decoder error");
+ return ;
+ }
+
+ //opus_decoder_h = opus_decoder_create(16000, 1, &err);
+ }
+ }
+
+
+
VC_record_Start();
VC_Play_Start();//音频调试1
websocket_connect_VC();//音频调试1
AI_VC_1S_timer = NULL;
}
close_timerout = 0;
+ if(inbound_attributes_format == inbound_attributes_format_OPUS)
+ {
+ // if(opus_encoder_h)
+ // opus_encoder_destroy(opus_encoder_h);
+ // opus_encoder_h = NULL;
+ // if(opus_decoder_h)
+ // opus_decoder_destroy(opus_decoder_h);
+ // opus_decoder_h = NULL;
+ if(aic_opus_deinit_encoder() == 0)
+ {
+ if(opus_encoder_h)
+ aic_free(opus_encoder_h);
+ opus_encoder_h = NULL;
+ }
+ if(aic_opus_deinit_decoder() == 0)
+ {
+ if(opus_decoder_h)
+ aic_free(opus_decoder_h);
+ opus_decoder_h = NULL;
+ }
+ }
+
app_audio_stop(APP_AUDIO_TYPE_STREAM_DUPLEX, NULL);//要在销毁线程之后
}
aos_status AI_AMP_MSG_put(const char *data,u32 len,AI_AMP_MSG_Type MSG_Type)
bk_printf("AI_AMP_2_5_init 等待注网..... \n");
aos_sleep(1000);
}
+ active_md5_sign_demo();
aic_network_enable_pdp_nic_capdump();//打开网络调试log
char *tts_str = "你好！我是小瑞";
VC_TTS_Play(tts_str);//开启vc播放tts
ai_uart_handle = uart_open(1);
aic_assert(ai_uart_handle);
+
+
/* change baudrate */
uart_set_baud_rate(ai_uart_handle, 921600);
//创建AI_AMP_task队列
projects / mirrors / FX01.git / commitdiff