From b5b14fe9cbf6ffffdde86e0417b8e211c1cce185 Mon Sep 17 00:00:00 2001 From: hcj <1224667075@qq.com> Date: Mon, 29 Sep 2025 18:26:45 +0800 Subject: [PATCH] =?utf8?q?=E5=A2=9E=E5=8A=A0=E6=92=AD=E6=94=BE=E8=B6=85?= =?utf8?q?=E6=97=B6=E6=97=B6=E9=97=B4=EF=BC=8C=E4=B8=8A=E6=8A=A5=E9=80=9F?= =?utf8?q?=E5=BA=A6=E5=8F=AF=E9=80=9A=E8=BF=87=E5=8F=82=E6=95=B0=E8=B0=83?= =?utf8?q?=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit --- code/sdk/zhihui/ai/include/ai_media.h | 19 +++ .../zhihui/ai/media/ai_media_service_ext.cpp | 41 ++++++- .../zhihui/leiniao/imsl_ln/leiniao_api_at.c | 2 +- code/sdk/zhihui/leiniao/imsl_ln/ln_api.c | 2 +- code/sdk/zhihui/leiniao/imsl_ln/ln_api.h | 3 +- .../zhihui/leiniao/imsl_ln/ln_api_websocket.c | 116 +++++++++++++----- 6 files changed, 150 insertions(+), 33 deletions(-) diff --git a/code/sdk/zhihui/ai/include/ai_media.h b/code/sdk/zhihui/ai/include/ai_media.h index d070e434d..89eb326dc 100644 --- a/code/sdk/zhihui/ai/include/ai_media.h +++ b/code/sdk/zhihui/ai/include/ai_media.h @@ -110,6 +110,25 @@ int32_t old_Start_AIVoice(AIVoiceDataCallback dataCallback, void* userData, int32_t AI_StartOggEncapsulation(void); int32_t AI_EndOggEncapsulation(void); +/* + * @brief 音频数据写入回调函数类型 + * @param size 音频数据大小 + */ +typedef void (*AI_WriteAudioDataCallback)(uint32_t size); + +/* + * @brief 注册音频数据写入回调 + * @param callback 回调函数指针 + * @return 0 成功，-1 失败 + */ +int32_t AI_RegisterWriteAudioDataCallback(AI_WriteAudioDataCallback callback); + +/* + * @brief 取消注册音频数据写入回调 + * @return 0 成功，-1 失败 + */ +int32_t AI_UnregisterWriteAudioDataCallback(void); + /* * @brief 写入音频数据到输出流 * @param data 音频数据指针 diff --git a/code/sdk/zhihui/ai/media/ai_media_service_ext.cpp b/code/sdk/zhihui/ai/media/ai_media_service_ext.cpp index 03fa4e301..9a80ffecd 100644 --- a/code/sdk/zhihui/ai/media/ai_media_service_ext.cpp +++ b/code/sdk/zhihui/ai/media/ai_media_service_ext.cpp @@ -136,6 +136,9 @@ static const size_t AI_AUDIO_CHUNK_SIZE = 1024; // 固定音频块大小：1KB static MediaMutexHandle g_fileCacheMutex = nullptr; static long g_fileCacheOffset = 0; // 文件读取偏移量 +// 音频数据写入回调相关变量 +static AI_WriteAudioDataCallback g_writeAudioDataCallback = nullptr; + // OGG相关内存管理函数 static void MediaMemFree(void **src) { @@ -178,6 +181,7 @@ static void AI_InitFileCacheMutex() } } + /** * @brief 将音频数据写入文件缓存 * @param msg 音频消息 @@ -1057,7 +1061,7 @@ static int32_t RunAIVoice(AIVoiceContext *context) if (msg.data != nullptr && msg.size > 0) { // 写入音频数据到输出流 AI_WriteAudioData(msg.data,msg.size); - // struct timespec ts; + // struct timespec ts; // AudioStreamGetTimestamp(&ts); // printf("AudioStreamGetTimestamp %d\r\n",ts.tv_sec); // 释放消息数据内存 @@ -1563,6 +1567,36 @@ static void ExampleAIVoiceDataCallback(const uint8_t* data, uint32_t size, void* // uint32_t queueSize, queueCapacity; // AI_GetQueueStatus(&queueSize, &queueCapacity); +/** + * @brief 注册音频数据写入回调 + * @param callback 回调函数指针 + * @return 0 成功，-1 失败 + */ +int32_t AI_RegisterWriteAudioDataCallback(AI_WriteAudioDataCallback callback) +{ + if (callback == nullptr) { + MEDIA_ERR_LOG("Invalid callback parameter"); + return -1; + } + + g_writeAudioDataCallback = callback; + + MEDIA_INFO_LOG("Audio data write callback registered successfully"); + return 0; +} + +/** + * @brief 取消注册音频数据写入回调 + * @return 0 成功，-1 失败 + */ +int32_t AI_UnregisterWriteAudioDataCallback(void) +{ + g_writeAudioDataCallback = nullptr; + + MEDIA_INFO_LOG("Audio data write callback unregistered successfully"); + return 0; +} + /** * @brief 写入音频数据到输出流 * @param data 音频数据指针 @@ -1618,6 +1652,11 @@ int32_t AI_WriteAudioData(const uint8_t* data, uint32_t size) goto AI_WRITE; } + // 调用注册的回调函数 + if (g_writeAudioDataCallback != nullptr) { + g_writeAudioDataCallback(size); + } + // MEDIA_INFO_LOG("Successfully wrote %u bytes of audio data to output stream", size); return 0; } diff --git a/code/sdk/zhihui/leiniao/imsl_ln/leiniao_api_at.c b/code/sdk/zhihui/leiniao/imsl_ln/leiniao_api_at.c index 66aade480..f02f43686 100644 --- a/code/sdk/zhihui/leiniao/imsl_ln/leiniao_api_at.c +++ b/code/sdk/zhihui/leiniao/imsl_ln/leiniao_api_at.c @@ -704,7 +704,7 @@ static uint32_t ln_ws_start_at(uint8_t *para, uint32_t paraLen, char *strCommonC VAD_UNUSED(paraLen); VAD_UNUSED(strCommonCmd); printf("[LN][WS_START]\n"); - int ret = ln_ws_start(NULL); + int ret = ln_ws_start(NULL,2000); if (ret != 0) { printf("[LN][WS_START] failed: %d\n", ret); diff --git a/code/sdk/zhihui/leiniao/imsl_ln/ln_api.c b/code/sdk/zhihui/leiniao/imsl_ln/ln_api.c index f6df7e583..7ba7c2afc 100644 --- a/code/sdk/zhihui/leiniao/imsl_ln/ln_api.c +++ b/code/sdk/zhihui/leiniao/imsl_ln/ln_api.c @@ -1329,7 +1329,7 @@ errcode_t uapi_ln_http_server_init(void) printf("create ln_send_service failed %d!\r\n", ret); return ERRCODE_FAIL; } - ln_ws_start(ln_ws_process_message); + ln_ws_start(ln_ws_process_message,2000); ln_init_required_dirs(); ln_load_init_config(); diff --git a/code/sdk/zhihui/leiniao/imsl_ln/ln_api.h b/code/sdk/zhihui/leiniao/imsl_ln/ln_api.h index 0036dfbcf..7f352c1fa 100644 --- a/code/sdk/zhihui/leiniao/imsl_ln/ln_api.h +++ b/code/sdk/zhihui/leiniao/imsl_ln/ln_api.h @@ -314,6 +314,7 @@ typedef enum { WS_CMD_RANDOM_STORY_RESPONSE, /* 随机故事响应 (RandomStoryResponse) */ WS_CMD_CONNECTED, /* WebSocket连接成功 */ WS_CMD_DISCONNECTED, /* WebSocket断连 */ + WS_CMD_AUDIO_PLAYBACK_END, /* 音频播放结束 */ WS_CMD_MAX /* 最大命令 */ } ws_command_t; @@ -347,7 +348,7 @@ typedef enum { } ln_audio_chat_command_t; /* WebSocket接口函数 */ -int ln_ws_start(ws_message_callback_t_ln callback); +int ln_ws_start(ws_message_callback_t_ln callback, uint32_t audio_timeout_ticks); int ln_ws_send_text(const char *text); int ln_ws_send_audio_data(const uint8_t *audio_data, uint32_t audio_size, bool finished); int ln_ws_send_random_story_request(const char *story_theme); diff --git a/code/sdk/zhihui/leiniao/imsl_ln/ln_api_websocket.c b/code/sdk/zhihui/leiniao/imsl_ln/ln_api_websocket.c index dab65924e..cd14dbafe 100644 --- a/code/sdk/zhihui/leiniao/imsl_ln/ln_api_websocket.c +++ b/code/sdk/zhihui/leiniao/imsl_ln/ln_api_websocket.c @@ -9,6 +9,7 @@ #include "soc_osal.h" #include "cmsis_os2.h" #include "mbedtls/base64.h" +#include "ai_media.h" // Added for AI_RegisterWriteAudioDataCallback /* * - g_ws_client_fd 当前 WebSocket 客户端fd @@ -69,6 +70,42 @@ static void ln_ws_process_base64_data(const char *base64_data, size_t length); static void ln_ws_process_incremental_base64_data(const char *new_data, size_t new_length); static void ln_ws_incomplete_callback(const char *raw_data, size_t length, bool isBinary); + +uint16_t send_audio=0; + +// 音频聊天相关结构体 +typedef struct { + char chat_id[64]; // 聊天ID + uint32_t message_counter; // 消息计数器 + bool is_active; // 是否活跃 +} audio_chat_context_t; + +// 全局音频聊天上下文 +static audio_chat_context_t g_audio_chat_ctx = {0}; + +// 音频聊天结束标志 +static bool g_audio_chat_ending = false; + + +// 当前对话匹配标志 +static bool g_chat_id_matched = false; // 当前对话ID是否匹配 + +// 音频播放计时相关变量 +static uint32_t g_last_audio_write_tick = 0; // 最后一次音频写入的tick +static bool g_audio_playback_active = false; // 音频播放是否活跃 +static uint32_t g_audio_timeout_ticks = 1500; // 音频超时tick数（默认1.5秒） + +// 函数声明 +static void generate_chat_id(char *chat_id, size_t size); +static void generate_message_id(char *message_id, size_t size, const char *chat_id, uint32_t counter); +static void reset_chat_id_match(void); +static void set_chat_id_matched(void); +static bool is_chat_id_matched(void); +static bool is_active_chat_response(const char *chat_id, const char *response_id); +static void ln_ws_audio_write_callback(uint32_t size); + + + /* 握手成功回调：标记连接就绪 */ static void ln_ws_on_open(int client) { @@ -180,6 +217,26 @@ static int ln_ws_heartbeat_callback(void) // 每30秒发送一次心跳 static int heartbeat_counter = 0; heartbeat_counter++; + + // 检查音频播放结束 + if (g_audio_playback_active) { + uint32_t current_tick = osKernelGetTickCount(); + uint32_t elapsed_ticks = current_tick - g_last_audio_write_tick; + + printf("[LN][WS] Heartbeat callback: audio playback active, elapsed ticks: %u\n", elapsed_ticks); + + // 如果超过超时时间没有音频写入，认为播放结束 + if (elapsed_ticks >= g_audio_timeout_ticks) { + printf("[LN][WS] Audio playback timeout detected, triggering end event\n"); + g_audio_playback_active = false; + + // 触发播放结束事件 + if (g_ws_message_callback != NULL) { + g_ws_message_callback(NULL, WS_CMD_AUDIO_PLAYBACK_END); + } + } + } + if (heartbeat_counter >= 15) { // 30秒 = 30次 * 1秒 heartbeat_counter = 0; ln_ws_send_ping(); @@ -235,34 +292,6 @@ int ln_ws_send_ping(void) } - -uint16_t send_audio=0; - -// 音频聊天相关结构体 -typedef struct { - char chat_id[64]; // 聊天ID - uint32_t message_counter; // 消息计数器 - bool is_active; // 是否活跃 -} audio_chat_context_t; - -// 全局音频聊天上下文 -static audio_chat_context_t g_audio_chat_ctx = {0}; - -// 音频聊天结束标志 -static bool g_audio_chat_ending = false; - - -// 当前对话匹配标志 -static bool g_chat_id_matched = false; // 当前对话ID是否匹配 - -// 函数声明 -static void generate_chat_id(char *chat_id, size_t size); -static void generate_message_id(char *message_id, size_t size, const char *chat_id, uint32_t counter); -static void reset_chat_id_match(void); -static void set_chat_id_matched(void); -static bool is_chat_id_matched(void); -static bool is_active_chat_response(const char *chat_id, const char *response_id); - // 生成聊天ID static void generate_chat_id(char *chat_id, size_t size) { @@ -317,6 +346,16 @@ static bool is_active_chat_response(const char *chat_id, const char *response_id return is_chat_id_matched(); } +// 音频数据写入回调函数 +static void ln_ws_audio_write_callback(uint32_t size) +{ + printf("[LN][WS] Audio data written successfully: %u bytes\n", size); + + // 更新最后一次音频写入tick + g_last_audio_write_tick = osKernelGetTickCount(); + g_audio_playback_active = true; +} + // 发送音频数据到WebSocket int ln_ws_send_audio_data(const uint8_t *audio_data, uint32_t audio_size, bool finished) { @@ -662,8 +701,9 @@ static void ln_ws_incomplete_callback(const char *raw_data, size_t length, bool * 启动：开启监督线程与心跳机制（心跳线程首次连接成功后会自然启动） * - 若已在运行则直接返回成功 * - callback: 消息处理回调函数，可以为NULL使用默认处理 + * - audio_timeout_ticks: 音频播放超时tick数，0表示使用默认值 */ -int ln_ws_start(ws_message_callback_t_ln callback) +int ln_ws_start(ws_message_callback_t_ln callback, uint32_t audio_timeout_ticks) { if (g_ws_running) { return 0; @@ -672,6 +712,14 @@ int ln_ws_start(ws_message_callback_t_ln callback) g_ws_reconnect_count = 0; /* 启动时从0开始 */ g_ws_connected = WS_STATE_DISCONNECTED; /* 启动时重置连接状态 */ + // 设置音频超时tick数 + if (audio_timeout_ticks > 0) { + g_audio_timeout_ticks = audio_timeout_ticks; + printf("[LN][WS] Audio timeout ticks set to: %u\n", g_audio_timeout_ticks); + } else { + printf("[LN][WS] Using default audio timeout ticks: %u\n", g_audio_timeout_ticks); + } + // 注册回调函数 g_ws_message_callback = callback; if (callback != NULL) { @@ -697,6 +745,9 @@ int ln_ws_start(ws_message_callback_t_ln callback) old_Start_AIVoice(AIVoiceDataCallback_ln,NULL,NULL,NULL,false); + // 注册音频数据写入回调 + AI_RegisterWriteAudioDataCallback(ln_ws_audio_write_callback); + cat1_api_keep_wakeup(1); cat1_api_wakeup_cat1(1); @@ -858,6 +909,10 @@ int ln_ws_send_voice_chat_interrupt(const char *reason) int ln_ws_stop(void) { g_ws_running = 0; + + // 取消注册音频数据写入回调 + AI_UnregisterWriteAudioDataCallback(); + int ret = uapi_ws_stop(); g_ws_connected = WS_STATE_DISCONNECTED; @@ -1274,6 +1329,9 @@ static void ln_ws_default_message_handler(const ws_data_msg_t *data_msg, ws_comm case WS_CMD_DISCONNECTED: printf("[LN][WS] Default: Processing WebSocket Disconnected event\n"); break; + case WS_CMD_AUDIO_PLAYBACK_END: + printf("[LN][WS] Default: Processing Audio Playback End event\n"); + break; case WS_CMD_PONG: if (data_msg) { printf("[LN][WS] Default: Processing Pong message\n"); -- 2.17.1