From dd4803bf4ed68415bbd67810fecb5213b6bc3f50 Mon Sep 17 00:00:00 2001 From: hcj <1224667075@qq.com> Date: Tue, 22 Jul 2025 13:46:18 +0800 Subject: [PATCH] =?utf8?q?=E4=BF=AE=E6=94=B9ai=E6=89=93=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit --- .../slip/cat1Service/cat1_ai_server.cpp | 498 +++++++++++++++++- .../slip/cat1Service/cat1_at_cmd.cpp | 4 +- .../peripheral/slip/cat1Service/cat1_ws.h | 1 + .../slip/cat1Service/dev_authentication.cpp | 46 ++ .../slip/cat1Service/dev_authentication.h | 1 + 5 files changed, 525 insertions(+), 25 deletions(-) diff --git a/code/sdk/drivers/peripheral/slip/cat1Service/cat1_ai_server.cpp b/code/sdk/drivers/peripheral/slip/cat1Service/cat1_ai_server.cpp index 7c24fe1ce..619cb1647 100644 --- a/code/sdk/drivers/peripheral/slip/cat1Service/cat1_ai_server.cpp +++ b/code/sdk/drivers/peripheral/slip/cat1Service/cat1_ai_server.cpp @@ -51,6 +51,8 @@ #include "dev_authentication.h" #include "bk_websocket.h" + + #ifdef __cplusplus extern "C" { #endif /* __cplusplus */ @@ -60,7 +62,9 @@ extern "C" { #include #include #include +#include "bts_br_gap.h" +#define AI_RESULT_STR_LEN 512 extern bool read_dev_token(char *data_token); @@ -120,7 +124,7 @@ uint8_t uapi_get_tts_break(void) return tts_break; } -char result_str_asr[256]={0}; +char result_str_asr[AI_RESULT_STR_LEN]={0}; static uint8_t type_ai=0; @@ -129,10 +133,11 @@ uint8_t chat_reconnect=0; void uapi_UI_start_asr(uint8_t type) { - memset_s(result_str_asr,256,0,256); + memset_s(result_str_asr,AI_RESULT_STR_LEN,0,AI_RESULT_STR_LEN); // uapi_stop_tts(); + type_ai=type; if(chat_client) { @@ -175,13 +180,19 @@ void uapi_UI_stop_start_asr_timeout(void) void uapi_UI_stop_asr(void) { + // SendMsgToUIService(UI_AI_JUMP_TO_RESULT_VIEW, 0, nullptr, 0);//先跳转到结果界面 + + // if(strlen(result_str_asr)) + // { + // send_ai_txt(0,result_str_asr); + // ble_set_ai_Question_str(result_str_asr); + + // memset_s(result_str_asr,AI_RESULT_STR_LEN,0,AI_RESULT_STR_LEN); + // } + if(stop_AudioCaptureProcess_AI()) { - // if(strlen(result_str_asr)) - // { - // send_ai_txt(0,result_str_asr); - // memset_s(result_str_asr,256,0,256); - // } + } } @@ -271,9 +282,13 @@ bool chat_result_analysis(char *buf) { // memset_s(content_str_chat,64,0,64); // memcpy_s(content_str_chat,64,content_str,strlen(content_str)); + + char *content_str_chat = (char *)malloc(strlen(content_str)+1); strcpy(content_str_chat,content_str); ble_set_ai_Answer_str(content_str_chat,2); + + // uapi_start_tts_single_txt(content_str_chat); // static uint8_t test=1; // if(test) @@ -328,7 +343,7 @@ bool chat_ws_url_init(char * put_buf,uint8_t type) } else if(type_ai==AI_TYPE_DE) { - sprintf(put_buf,"ws://amp.raisound.com/chat/ws?dt=%s&platform=huoshan&model=ep-20250206162546-g74w6",token); + sprintf(put_buf,"ws://amp.raisound.com/chat/ws?dt=%s&platform=huoshan&model=ep-20250205103849-582m7",token); } else { @@ -482,9 +497,14 @@ void bk_websocket_demo(void *data) { int len= receive_websocket_message(chat_client,(unsigned char *)recv_buf,1024,&type); if(len==-1 || type == WS_CLOSE) { + printf("disconnect_server chat_client"); break; } chat_result_analysis(recv_buf); + if(chat_reconnect==1) + { + break; + } printf("bk_websocket_demo %s",recv_buf); } @@ -594,21 +614,7 @@ void uapi_stop_asr(void) -void send_ai_bin(uint8_t *buf,uint32_t len) -{ - WEARABLE_LOGE(WEARABLE_LOG_MODULE_APP,"send_ai_bin %d\n",len); - - if (send_websocket_message(asr_client,( uint8_t *)buf,len,WS_BINARY)==0) - { - WEARABLE_LOGE(WEARABLE_LOG_MODULE_APP,"send_websocket_message succ 0\n"); - msleep(100); - } - else - { - WEARABLE_LOGE(WEARABLE_LOG_MODULE_APP,"send_websocket_message fail 0\n"); - } -} void record_test(void) { @@ -710,7 +716,7 @@ bool aer_result_analysis(char *buf) memset_s(result_str_asr,256,0,256); memcpy_s(result_str_asr,256,result_str,strlen(result_str)); SendMsgToUIService(UI_AI_JUMP_TO_RESULT_VIEW, 0, nullptr, 0);//先跳转到结果界面 - + ble_set_ai_Question_str(result_str_asr); // stop_AudioCaptureProcess_AI(); send_ai_txt(0,result_str_asr); @@ -1402,7 +1408,7 @@ if(asr_client) } - +errcode_t voice_chat_server_init(void); bool init_ai_task(uint8_t type) { @@ -1417,8 +1423,11 @@ bool init_ai_task(uint8_t type) } type_ai=type; uapi_tts_paly_init(); + tts_server_init(); chat_server_init(); + + // voice_chat_server_init(); // is_init_ai=0; } @@ -1468,9 +1477,450 @@ bool pre_AI_init(void) +///////////////////////////////////////////////////////////////////////////////////////////////// + +WebSocketClient *voice_chat_client=NULL; + +uint8_t is_bk_websocket_voice_chat=0; + + +void send_ai_bin(uint8_t *buf,uint32_t len) +{ + WEARABLE_LOGE(WEARABLE_LOG_MODULE_APP,"send_ai_bin %d\n",len); + // voice_chat_client + + // if (send_websocket_message(asr_client,( uint8_t *)buf,len,WS_BINARY)==0) + if (send_websocket_message(asr_client,( uint8_t *)buf,len,WS_BINARY)==0) + { + WEARABLE_LOGE(WEARABLE_LOG_MODULE_APP,"send_websocket_message succ 0\n"); + msleep(50); + } + else + { + WEARABLE_LOGE(WEARABLE_LOG_MODULE_APP,"send_websocket_message fail 0\n"); + } + +} + + +typedef enum +{ + AI_STATE_0=0, + Listening, + Responding, + Thinking, + +}AI_VOICE_STATE; + +uint8_t ai_voice_chat_state=0; + + + + +// { +// "event": "start", +// "payload": { +// "device_id": "123123123", // 设备ID +// "user_agent": "AudioClient/1.0", // 浏览器 UA +// "simple_response": false, // true 只返回完整的LLM文本 默认false 返回增量LLM文本 +// // "dialog_attributes": { // 智能体配置 +// // "provider": "coze", // 切换服务驱动 +// // "cfg": { +// // "disable_empty_prompt": true, // 拦截 asr 识别出来的空文本发送到大模型 +// // "custom_variables": { // coze 自定义参数 +// // "name": "小瑞" +// // }, +// // "greeting": { // 问候语 配置 +// // "enabled": true, // 是否开启问候语 +// // "content": "" // 问候语内容 coze 不填写默认使用 coze 平台设置的 +// // }, +// // "bot_option": { +// // "use_workflow": true, // 使用对话流方式 coze 推荐使用对话流 +// // "bot_id":"7462617881076351002", // 使用对话流 bot_id 需要和 workflow_id处于同一个工作空间即可 +// // "workflow_id":"7462621757327196187", // +// // } +// // } +// // }, +// "outbound_attributes": { // TTS输出音频配置 +// "provider": "volcengine", // TTS服务驱动 +// "cfg": { +// "format": "pcm", // 音频输出格式 +// "sample_rate": 16000, // 音频输出采样率 +// "bit_rate": 16, // 比特率 +// "channel": 1, // 声道 1 单声道 +// "volume": 100, // 音量 0-100 +// "speech_rate": 0, // 语速 [-50,100] 默认 0 +// "chunk_interval_ms": 100, // 配置每段音频入队间隔时间 100ms +// "voice": "zh_female_linjianvhai_moon_bigtts" // 音色 +// "voice_source": "private", // 音色来源。目前定义 private 复刻音色。 system 使用系统默认音色，不提交这个参数兼容 system +// } +// }, +// "inbound_attributes": { // ASR输入音频配置 +// "provider": "volcengine", // ASR服务驱动 +// "disable_vad": false, // 开启vad 自动断句 （开启后可以有客户端处理回音消除后持续发送音频由服务器判断是否转写和发送LLM） +// "vad_silence": 400, // 设置vad 静音阈值 （毫秒） 连续静音超过 阈值自动切换状态 +// "interrupt_options": { +// "concatenate_results": true, // 是否在打断后拼接前后的ASR结果 +// "thinking_time_only":true // 是否只在思考时间允许打断 思考时间(ASR结束--->第一帧音频合成) +// "keyword_interrupt": true, // 启用打断词功能 （打断词只有在思考以及之后才能打断） +// "interrupt_keywords": "",// 配置打断词列表 +// }, +// "cfg": { +// "format": "pcm", // 音频输入格式 +// "sample_rate": 16000, // 音频输入采样率 目前只支持 16000 +// "bit_rate": 16, // 比特率 目前只支持 16bit +// "channel": 1 // 声道 目前只支持单声道 +// "enable_punctuation": true, // 开启标点 +// "enable_itn":true, // 开启itn +// "keywords": ["张三"] // 热词 +// } +// }, +// // 注意： 如果走智能体 端侧插件 commands 不传 + +// } +// } + + +bool start_ai_voice_chat() +{ + unsigned char deviceId[20] = {0}; + memset(deviceId,0,6); + + bluetooth_get_local_addr(&deviceId[0],6); + + char deviceId_temp[20] = {0}; + sprintf(deviceId_temp,"%02X%02x%02X%02x%02X%02x",deviceId[5],deviceId[4],deviceId[3],deviceId[2],deviceId[1],deviceId[0]); + + + time_t timestamp; + timestamp = time(NULL); + long long timestamp_ms = (long long)timestamp ; + + char* timestamp_str_2=(char *)malloc(20); + memset(timestamp_str_2,0,20); + sprintf(timestamp_str_2, "%lld", timestamp_ms); + + char md5_buf[33]={0}; + active_md5_buf((char *)timestamp_str_2,md5_buf); + printf("md5_buf:%s",md5_buf); + + + char jsonString[800]={0}; + sprintf(jsonString, "{ \ + \"header\": { \ + \"trace_id\": \"%s\", \ + \"event\":\"Start\" \ + }, \ + \"payload\": { \ + \"device_id\": \"%s\", \ + \"user_agent\": \"python\", \ + \"dialog_id\":\"\", \ + \"in_format\": \"pcm\", \ + \"out_format\": \"mp3\", \ + \"in_sample_rate\": 16000, \ + \"volume\": \"10\", \ + \"sample_rate\": 16000, \ + \"voice\": \"zh_female_shuangkuaisisi_moon_bigtts\", \ + \"voice_detection\": \"disable\" \ + } \ + } ",md5_buf,deviceId_temp) + ; + + if (send_websocket_message(voice_chat_client,( uint8_t *)jsonString,strlen(jsonString),WS_TEXT)==0) + { + WEARABLE_LOGE(WEARABLE_LOG_MODULE_APP,"send_websocket_message succ %s\n",jsonString); + return true; + } + + else + { + WEARABLE_LOGE(WEARABLE_LOG_MODULE_APP,"send_websocket_message erro\n"); + return false; + } + +} + + +// {"event":"Started","msg":"success","payload": +// {"dialog_id":"b0444aed86774e47a142a967e5668a8b"}, +// "status":20000000,"task_id":"d5e447826a4f899694d87d0571998d0f"} + +// {"header":{"trace_id":"ailaqbo0r6o","event":"SendSpeech"},"payload":{"dialog_id":"b0444aed86774e47a142a967e5668a8b"}} + +// {"event":"SpeechEnded","msg":"success", +// "payload":{"dialog_id":"b0444aed86774e47a142a967e5668a8b"}, +// "status":20000000,"task_id":"d5e447826a4f899694d87d0571998d0f"} + + +char dialog_id_str_buf[128]={0}; + + +void start_send_voice(void) +{ + char *jsonString = (char *)malloc(150);// [512] ={0}; + memset(jsonString,0,150); + + sprintf(jsonString, + "{\"header\":{\"trace_id\":\"ailaqbo0r6o\",\"event\":\"SendSpeech\"},\ + \"payload\":{\"dialog_id\":\"%s\"}}" + + ,dialog_id_str_buf); + + printf("start_send_voice : %s\r\n",jsonString); + + if (send_websocket_message(voice_chat_client,( uint8_t *)jsonString,strlen(jsonString),WS_TEXT)==0) + { + startAudioCaptureTask(); + WEARABLE_LOGE(WEARABLE_LOG_MODULE_APP,"send_websocket_message succ 0\n"); + + } + + free(jsonString); +} + +// {"header":{"trace_id":"ailaqbo0r6o","event":"RequestToSpeak"},"payload":{"dialog_id":"b0444aed86774e47a142a967e5668a8b"}} + +void back_voice_RequestToSpeak(void) +{ + char *jsonString = (char *)malloc(150);// [512] ={0}; + + sprintf(jsonString, + + "{\"header\":{\"trace_id\":\"ailaqbo0r6o\",\"event\":\"RequestToSpeak\"},\ + \"payload\":{\"dialog_id\":\"%s\"}}" + + ,dialog_id_str_buf); + + printf("uapi_start_tts : %s\r\n",jsonString); + + if (send_websocket_message(voice_chat_client,( uint8_t *)jsonString,strlen(jsonString),WS_TEXT)==0) + WEARABLE_LOGE(WEARABLE_LOG_MODULE_APP,"send_websocket_message succ 0\n"); + free(jsonString); +} + + +// {"event":"DialogStateChanged", +// "msg":"success","payload": +// {"dialog_id":"b0444aed86774e47a142a967e5668a8b","state":"Thinking"}, +// "status":20000000,"task_id":"d5e447826a4f899694d87d0571998d0f"} + +// {"event":"DialogStateChanged", +// "msg":"success","payload": +// {"dialog_id":"b0444aed86774e47a142a967e5668a8b", +// "state":"Listening"},"status":20000000,"task_id":"d5e447826a4f899694d87d0571998d0f"} + +// {"event":"SpeechContent","msg":"success", +// "payload":{"dialog_id":"b0444aed86774e47a142a967e5668a8b","finished":true,"text":"你好"}, +// "status":20000000,"task_id":"d5e447826a4f899694d87d0571998d0f"} + +bool voice_chat_analysis(char *buf) +{ + cJSON* root = cJSON_Parse((const char *)buf); + if (root == NULL) { + WEARABLE_LOGE(WEARABLE_LOG_MODULE_APP,"解析失败\n"); + return false; + } + char *event_str=NULL; + char *dialog_id_str=NULL; + char *state_str=NULL; + char *text_str=NULL; + + cJSON* event = cJSON_GetObjectItem(root, "event"); + if (event && cJSON_IsString(event)) { + event_str= event->valuestring; + } + + + cJSON* payload = cJSON_GetObjectItem(root, "payload"); + if(payload) + { + cJSON* dialog_id = cJSON_GetObjectItem(payload, "dialog_id"); + if (dialog_id && cJSON_IsString(dialog_id)) { + dialog_id_str= dialog_id->valuestring; + memset(dialog_id_str_buf,0,128); + memcpy(dialog_id_str_buf,dialog_id_str,strlen(dialog_id_str)); + } + + cJSON* state = cJSON_GetObjectItem(payload, "state"); + if (state && cJSON_IsString(state)) { + state_str= dialog_id->valuestring; + + if(strcmp("Responding",state_str)==0) + { + ai_voice_chat_state=Responding; + } + else if (strcmp("Listening",state_str)==0) + { + ai_voice_chat_state=Listening; + } + else if (strcmp("Thinking",state_str)==0) + { + ai_voice_chat_state=Thinking; + } + } + + cJSON* text = cJSON_GetObjectItem(payload, "text"); + + if (text && cJSON_IsString(state)) { + state_str= dialog_id->valuestring; + } + + } + + if(strcmp("Started",event_str)==0) + { + + } + + if(strcmp("SpeechEnded",event_str)==0) + { + stop_AudioCaptureProcess_AI(); + } + + + + cJSON_Delete(root); + return true; +} + +void voice_chat_websocket(void *data) { + + // while (1) + uapi_cat1_keep_wakeup(1); + + WebSocketMessageType type; + bool state = true; + + is_bk_websocket_tts=1; + + uint8_t is_raisound_url=0; + + { + // ws://amp.raisound.com/asr/ws + char url_buf[124]={0}; + // if(tts_ws_url_init(url_buf)) + // { + // printf("%s\r\n",url_buf); + // is_raisound_url=1; + // } + // else + // { + // is_raisound_url=0; + // } + + uint8_t client_count=3; + + + + while (client_count) + { + if(is_raisound_url==1) + { + voice_chat_client = init_websocket_client(url_buf,NULL,1); + } + else + { + // http://118.145.158.207:22080ws://118.145.158.207:22080/voice-chat/ws + // /voice-chat/v2/ws + voice_chat_client = init_websocket_client("ws://118.145.158.207:22080/voice-chat/v2/ws?platform=huoshan",NULL,0); + } + + + if (voice_chat_client== NULL) + { + printf("init_websocket_client fail %d\r\n",client_count); + client_count--; + msleep(1000); + continue ; + } + else + { + printf("tts_client init_websocket_client succ \r\n"); + break; + } + } + if (voice_chat_client== NULL){ + is_bk_websocket_tts=0; + return; + } + + + char* voice_chat_client_buf=(char *)malloc(1024*4); + + start_ai_voice_chat(); + + while (disconnect_server==0) + { + + + memset_s(voice_chat_client_buf,1024*4,0,1024*4); + int len=receive_websocket_message(voice_chat_client,(unsigned char *)voice_chat_client_buf,1024*4,&type); + // if(type==WS_CONT || + if( type==WS_CLOSE || len==-1)//|| type==WS_CONT ) + { + printf("WS_lwip_close bk_websocket_tts\r\n"); + break; + } + // printf("bk_websocket_tts bk_websocket_tts %d\r\n",type); + if(type==WS_BINARY) + { + if(bk_websocket_tts_tick==0) + { + set_AIStreamend(0); + bk_websocket_tts_tick = (uint32_t)osKernelGetTickCount(); + } + send_msg_data_into_Ai_msg_queue((uint8_t *)voice_chat_client_buf,len,200); + printf("receive_websocket_message voice_chat %d\n",len); + msleep(1); + + } + else if(type==WS_TEXT) + { + printf("receive_websocket_message voice_chat %s\n",voice_chat_client_buf); + voice_chat_analysis(voice_chat_client_buf); + // tts_paly(); + } + // aer_result_analysis(asr_recv_buf); + } + + bk_websocket_tts_tick=0; + lwip_close_websocket_client(voice_chat_client); + voice_chat_client=NULL; + + + // return; + } + is_bk_websocket_tts=0; + +} + + +errcode_t voice_chat_server_init(void) +{ + if(tts_client!=NULL || is_bk_websocket_tts) + { + return ERRCODE_FAIL; + } + WEARABLE_LOGE(WEARABLE_LOG_MODULE_APP,"tts_server_init !\r\n"); + app_task_definition_t taskcfg={0}; + taskcfg.create_queue_handle = NULL; + taskcfg.attr.name="tts_server_init"; + taskcfg.attr.priority = TASK_PRIORITY_APP; + taskcfg.attr.stack_size = 0x2000; + taskcfg.func = voice_chat_websocket; + taskcfg.use_mem = USE_EXTEND_MEM; + + errcode_t ret=uapi_create_osThread_custom(&taskcfg); + if(ret != ERRCODE_SUCC) + { + WEARABLE_LOGE(WEARABLE_LOG_MODULE_APP,"create cat1_service_file failed %d!\r\n",ret); + return ERRCODE_FAIL; + } + return ERRCODE_SUCC; +} diff --git a/code/sdk/drivers/peripheral/slip/cat1Service/cat1_at_cmd.cpp b/code/sdk/drivers/peripheral/slip/cat1Service/cat1_at_cmd.cpp index 71cf4bd74..4209627a4 100644 --- a/code/sdk/drivers/peripheral/slip/cat1Service/cat1_at_cmd.cpp +++ b/code/sdk/drivers/peripheral/slip/cat1Service/cat1_at_cmd.cpp @@ -549,7 +549,9 @@ static uint32_t start_asr(uint8_t *para, uint32_t paraLen, char *strCommonCmd) printf("CAT1TEST ATCMD Convert Command Failed\n"); return ERRCODE_FAIL; } - uapi_UI_start_asr(AI_TYPE_JIU); + // uapi_UI_start_asr(AI_TYPE_JIU); + + start_send_voice(); return 0; } diff --git a/code/sdk/drivers/peripheral/slip/cat1Service/cat1_ws.h b/code/sdk/drivers/peripheral/slip/cat1Service/cat1_ws.h index 8f2c5e53c..ade7af40c 100644 --- a/code/sdk/drivers/peripheral/slip/cat1Service/cat1_ws.h +++ b/code/sdk/drivers/peripheral/slip/cat1Service/cat1_ws.h @@ -33,6 +33,7 @@ bool pre_AI_init(void); //////////////////////////////////////////////////////////////////////// +void start_send_voice(void); void tts_server_deinit(void); diff --git a/code/sdk/drivers/peripheral/slip/cat1Service/dev_authentication.cpp b/code/sdk/drivers/peripheral/slip/cat1Service/dev_authentication.cpp index 15043bdf6..2932161dc 100644 --- a/code/sdk/drivers/peripheral/slip/cat1Service/dev_authentication.cpp +++ b/code/sdk/drivers/peripheral/slip/cat1Service/dev_authentication.cpp @@ -776,6 +776,52 @@ void read_file_put(void ) } +int active_md5_buf(char *in_buf,char*md5_str) +{ + + char md5_buf[16]={0}; + char *sign_buf = (char *)malloc(1024); + + if(!sign_buf) + return -10; + memset(sign_buf,0,1024); + + int sign_len = 0; + strcpy(sign_buf,in_buf); + + sign_len = strlen(in_buf); + + + mbedtls_md5_context ctx; + + // 初始化MD5上下文 + mbedtls_md5_init(&ctx); + + // 开始计算MD5哈希 + mbedtls_md5_starts(&ctx); + // 更新要计算哈希的数据 + mbedtls_md5_update(&ctx, (const unsigned char *)sign_buf, sign_len); + + // 完成哈希计算 + mbedtls_md5_finish(&ctx, (unsigned char *)md5_buf); + + // printf("md5_buf\r\n"); + // for (int i = 0; i < 16; i++) { + // printf("%02X\n", (uint8_t)md5_buf[i]); + // } + // printf("md5_buf end\r\n"); + + // 清理MD5上下文 + mbedtls_md5_free(&ctx); + for (int i = 0; i < 16; i++) { + sprintf(&md5_str[i * 2], "%02x", (uint8_t)md5_buf[i]); + } + md5_str[32] = '\0'; + + free(sign_buf); + return 0; +} + #ifdef __cplusplus } diff --git a/code/sdk/drivers/peripheral/slip/cat1Service/dev_authentication.h b/code/sdk/drivers/peripheral/slip/cat1Service/dev_authentication.h index c606a0cda..f184feea0 100644 --- a/code/sdk/drivers/peripheral/slip/cat1Service/dev_authentication.h +++ b/code/sdk/drivers/peripheral/slip/cat1Service/dev_authentication.h @@ -40,6 +40,7 @@ bool aut_login(void);//设备激活; void system_init(void); +int active_md5_buf(char *in_buf,char*md5_str); #ifdef __cplusplus } -- 2.17.1