From a0d34f98ce474f50e7bc4fb93695a7347bfcbdf6 Mon Sep 17 00:00:00 2001 From: hcj <1224667075@qq.com> Date: Wed, 3 Jun 2026 14:49:58 +0800 Subject: [PATCH] =?utf8?q?=E5=A2=9E=E5=8A=A0ct=5Fai?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit --- code/sdk/zhihui/CT_AI/ct_ai_websocket.c | 283 +++++++++++++++++++++--- 1 file changed, 249 insertions(+), 34 deletions(-) diff --git a/code/sdk/zhihui/CT_AI/ct_ai_websocket.c b/code/sdk/zhihui/CT_AI/ct_ai_websocket.c index 8815ff831..c87aff0f2 100644 --- a/code/sdk/zhihui/CT_AI/ct_ai_websocket.c +++ b/code/sdk/zhihui/CT_AI/ct_ai_websocket.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "cJSON.h" // Added for JSON parsing #include "custom_utils.h" #include "soc_osal.h" @@ -39,6 +40,11 @@ extern void ln_http_download_deinit(void); #define ENABLE_BINARY_DATA_INTERRUPT_CHECK 1 // 启用二进制数据间隔检查 #define ENABLE_AUDIO_FILE_WRITE_CT_AI 0 // 启用音频文件写入功能(根据chat_id生成文件) +/* S2C Opus 下行:落盘调试(opus 原始帧 + 解码 PCM);仅 hasNext=false 时写文件,中间分片只缓存 */ +#ifndef CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE +#define CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE 0 +#endif + // 主线程循环间隔时长(毫秒) #define CT_AI_WS_MAIN_THREAD_INTERVAL_MS 500 @@ -103,12 +109,17 @@ static const char *g_log_level_str[] = { #define CT_AI_S2C_OPUS_USE_DECODE_QUEUE 0 #endif -/* S2C 下行:按 hasNext 聚合整段 PCM,仅在 hasNext=false 时一次性写入播放队列(仅解码线程持锁修改) */ +/* S2C 下行播放:worker 模式按 hasNext 聚合后入队;inline 模式解码后立即入队 */ static uint8_t *g_s2c_pcm_accum_buf = NULL; static size_t g_s2c_pcm_accum_len = 0; static size_t g_s2c_pcm_accum_cap = 0; static osMutexId_t g_s2c_pcm_accum_mutex = NULL; +#if CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE +static void ct_ai_ws_s2c_opus_record_free_inner(void); +static void ct_ai_ws_s2c_pcm_record_free_inner(void); +#endif + static void ct_ai_ws_s2c_pcm_accum_free_inner(void) { free(g_s2c_pcm_accum_buf); @@ -122,9 +133,17 @@ static void ct_ai_ws_s2c_pcm_accum_free(void) if (g_s2c_pcm_accum_mutex != NULL) { (void)osMutexAcquire(g_s2c_pcm_accum_mutex, osWaitForever); ct_ai_ws_s2c_pcm_accum_free_inner(); +#if CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE + ct_ai_ws_s2c_opus_record_free_inner(); + ct_ai_ws_s2c_pcm_record_free_inner(); +#endif (void)osMutexRelease(g_s2c_pcm_accum_mutex); } else { ct_ai_ws_s2c_pcm_accum_free_inner(); +#if CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE + ct_ai_ws_s2c_opus_record_free_inner(); + ct_ai_ws_s2c_pcm_record_free_inner(); +#endif } } @@ -173,6 +192,143 @@ static void ct_ai_ws_s2c_pcm_accum_flush_to_queue_inner(void) ct_ai_ws_s2c_pcm_accum_free_inner(); } +static void ct_ai_ws_s2c_pcm_send_to_queue_immediate(const uint8_t *pcm_pos, size_t pcm_bytes) +{ + const uint32_t pcm_chunk_size = 1024U; + + while (pcm_bytes > 0U) { + uint32_t send_len = (pcm_bytes > (size_t)pcm_chunk_size) ? pcm_chunk_size : (uint32_t)pcm_bytes; + + AI_SendAudioDataToQueue(pcm_pos, send_len); + pcm_pos += send_len; + pcm_bytes -= (size_t)send_len; + } +} + +#if CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE +static uint8_t *g_s2c_opus_record_buf = NULL; +static size_t g_s2c_opus_record_len = 0; +static size_t g_s2c_opus_record_cap = 0; +static uint8_t *g_s2c_pcm_record_buf = NULL; +static size_t g_s2c_pcm_record_len = 0; +static size_t g_s2c_pcm_record_cap = 0; +static uint32_t g_s2c_opus_record_file_index = 0; + +static void ct_ai_ws_s2c_opus_record_free_inner(void) +{ + free(g_s2c_opus_record_buf); + g_s2c_opus_record_buf = NULL; + g_s2c_opus_record_len = 0; + g_s2c_opus_record_cap = 0; +} + +static void ct_ai_ws_s2c_pcm_record_free_inner(void) +{ + free(g_s2c_pcm_record_buf); + g_s2c_pcm_record_buf = NULL; + g_s2c_pcm_record_len = 0; + g_s2c_pcm_record_cap = 0; +} + +static int ct_ai_ws_s2c_byte_record_append(uint8_t **buf, size_t *len, size_t *cap, const uint8_t *data, + size_t data_len) +{ + if (data == NULL || data_len == 0U) { + return 0; + } + + size_t new_len = *len + data_len; + if (new_len > *cap) { + size_t new_cap = *cap ? *cap : 4096U; + + while (new_cap < new_len) { + if (new_cap > (SIZE_MAX / 2U)) { + return -1; + } + new_cap *= 2U; + } + uint8_t *nb = (uint8_t *)realloc(*buf, new_cap); + if (nb == NULL) { + return -1; + } + *buf = nb; + *cap = new_cap; + } + + memcpy(*buf + *len, data, data_len); + *len = new_len; + return 0; +} + +static int ct_ai_ws_s2c_opus_record_append_inner(const uint8_t *opus, size_t opus_bytes) +{ + return ct_ai_ws_s2c_byte_record_append(&g_s2c_opus_record_buf, &g_s2c_opus_record_len, &g_s2c_opus_record_cap, + opus, opus_bytes); +} + +static int ct_ai_ws_s2c_pcm_record_append_inner(const uint8_t *pcm, size_t pcm_bytes) +{ + return ct_ai_ws_s2c_byte_record_append(&g_s2c_pcm_record_buf, &g_s2c_pcm_record_len, &g_s2c_pcm_record_cap, pcm, + pcm_bytes); +} + +static void ct_ai_ws_s2c_opus_record_write_files_inner(int sample_rate) +{ + if (g_s2c_opus_record_len == 0U && g_s2c_pcm_record_len == 0U) { + return; + } + + time_t now = time(NULL); + struct tm tm_info; + + (void)localtime_r(&now, &tm_info); + { + uint32_t idx = g_s2c_opus_record_file_index++; + + if (g_s2c_opus_record_len > 0U) { + char opus_path[160]; + + (void)snprintf(opus_path, sizeof(opus_path), + "/user/ct_ai_s2c_%04d%02d%02d_%02d%02d%02d_%03u_%d.opus", + tm_info.tm_year + 1900, tm_info.tm_mon + 1, tm_info.tm_mday, tm_info.tm_hour, tm_info.tm_min, + tm_info.tm_sec, (unsigned)idx, sample_rate); + FILE *fp = fopen(opus_path, "wb"); + if (fp != NULL) { + (void)fwrite(g_s2c_opus_record_buf, 1U, g_s2c_opus_record_len, fp); + (void)fclose(fp); + CT_AI_WS_INFO("s2c record: opus %zu bytes -> %s\n", g_s2c_opus_record_len, opus_path); + } else { + CT_AI_WS_ERROR("s2c record: fopen opus failed %s\n", opus_path); + } + } + + if (g_s2c_pcm_record_len > 0U) { + char pcm_path[160]; + + (void)snprintf(pcm_path, sizeof(pcm_path), + "/user/ct_ai_s2c_%04d%02d%02d_%02d%02d%02d_%03u_%d.pcm", + tm_info.tm_year + 1900, tm_info.tm_mon + 1, tm_info.tm_mday, tm_info.tm_hour, tm_info.tm_min, + tm_info.tm_sec, (unsigned)idx, sample_rate); + FILE *fp = fopen(pcm_path, "wb"); + if (fp != NULL) { + (void)fwrite(g_s2c_pcm_record_buf, 1U, g_s2c_pcm_record_len, fp); + (void)fclose(fp); + CT_AI_WS_INFO("s2c record: pcm %zu bytes -> %s\n", g_s2c_pcm_record_len, pcm_path); + } else { + CT_AI_WS_ERROR("s2c record: fopen pcm failed %s\n", pcm_path); + } + } + } +} + +static void ct_ai_ws_s2c_opus_record_flush_inner(int sample_rate) +{ + ct_ai_ws_s2c_opus_record_write_files_inner(sample_rate); + ct_ai_ws_s2c_opus_record_free_inner(); + ct_ai_ws_s2c_pcm_record_free_inner(); +} +#endif /* CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE */ + #if CT_AI_S2C_OPUS_USE_DECODE_QUEUE typedef struct ct_ai_s2c_opus_job { @@ -254,22 +410,26 @@ static void ct_ai_s2c_opus_worker_main(void *argument) int pcm_samples = 0; int dec_ret = -1; + uint8_t *opus_bin = NULL; + size_t opus_bin_len = 0U; if (job->b64_chunk != NULL && job->b64_len > 0U) { size_t dec_cap = job->b64_len; - uint8_t *opus_bin = (uint8_t *)malloc(dec_cap); + + opus_bin = (uint8_t *)malloc(dec_cap); if (opus_bin == NULL) { CT_AI_WS_ERROR("opus worker: malloc(base64 decode buf) failed\n"); dec_ret = -1; } else { - size_t opus_bin_len = 0; int t0 = (int)osKernelGetTickCount(); int b64_r = mbedtls_base64_decode(opus_bin, dec_cap, &opus_bin_len, (const unsigned char *)job->b64_chunk, job->b64_len); int t1 = (int)osKernelGetTickCount(); + tick_b64 = t1 - t0; if (b64_r == 0 && opus_bin_len > 0U) { int t2 = (int)osKernelGetTickCount(); + dec_ret = opus_frame_to_pcm_decode_packet(opus_bin, (int)opus_bin_len, job->sample_rate, 1, g_opus_debug_pcm_buf, OPUS_DEBUG_PCM_BUF_SAMPLES, &pcm_samples); int t3 = (int)osKernelGetTickCount(); @@ -277,7 +437,6 @@ static void ct_ai_s2c_opus_worker_main(void *argument) } else { dec_ret = -1; } - free(opus_bin); } } @@ -289,23 +448,45 @@ static void ct_ai_s2c_opus_worker_main(void *argument) if (job->b64_chunk != NULL && job->b64_len > 0U) { if (dec_ret == 0 && pcm_samples > 0) { size_t pcm_bytes = (size_t)pcm_samples * sizeof(short); +#if CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE + if (opus_bin_len > 0U && + ct_ai_ws_s2c_opus_record_append_inner(opus_bin, opus_bin_len) != 0) { + CT_AI_WS_ERROR("opus worker: opus record append failed\n"); + } + if (ct_ai_ws_s2c_pcm_record_append_inner((const uint8_t *)g_opus_debug_pcm_buf, pcm_bytes) != 0) { + CT_AI_WS_ERROR("opus worker: pcm record append failed\n"); + } +#endif if (ct_ai_ws_s2c_pcm_accum_append_inner((const uint8_t *)g_opus_debug_pcm_buf, pcm_bytes) != 0) { CT_AI_WS_ERROR("opus worker: pcm accum append/realloc failed\n"); ct_ai_ws_s2c_pcm_accum_free_inner(); +#if CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE + ct_ai_ws_s2c_opus_record_free_inner(); + ct_ai_ws_s2c_pcm_record_free_inner(); +#endif } } else { CT_AI_WS_ERROR("opus worker: decode fail ret=%d pcm_samples=%d\n", dec_ret, pcm_samples); ct_ai_ws_s2c_pcm_accum_free_inner(); +#if CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE + ct_ai_ws_s2c_opus_record_free_inner(); + ct_ai_ws_s2c_pcm_record_free_inner(); +#endif } } if (!job->has_next) { ct_ai_ws_s2c_pcm_accum_flush_to_queue_inner(); +#if CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE + ct_ai_ws_s2c_opus_record_flush_inner(job->sample_rate); +#endif } if (g_s2c_pcm_accum_mutex != NULL) { (void)osMutexRelease(g_s2c_pcm_accum_mutex); } + free(opus_bin); + opus_bin = NULL; int tp1 = (int)osKernelGetTickCount(); tick_pcm_path = tp1 - tp0; @@ -425,6 +606,10 @@ static void ct_ai_ws_s2c_opus_worker_deinit(void) if (g_s2c_pcm_accum_mutex != NULL) { (void)osMutexAcquire(g_s2c_pcm_accum_mutex, osWaitForever); ct_ai_ws_s2c_pcm_accum_free_inner(); +#if CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE + ct_ai_ws_s2c_opus_record_free_inner(); + ct_ai_ws_s2c_pcm_record_free_inner(); +#endif (void)osMutexRelease(g_s2c_pcm_accum_mutex); osMutexDelete(g_s2c_pcm_accum_mutex); g_s2c_pcm_accum_mutex = NULL; @@ -512,53 +697,83 @@ static void ct_ai_ws_s2c_opus_worker_deinit(void) (void)osMutexAcquire(g_s2c_pcm_accum_mutex, osWaitForever); ct_ai_ws_s2c_pcm_accum_free_inner(); +#if CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE + ct_ai_ws_s2c_opus_record_free_inner(); + ct_ai_ws_s2c_pcm_record_free_inner(); +#endif (void)osMutexRelease(g_s2c_pcm_accum_mutex); osMutexDelete(g_s2c_pcm_accum_mutex); g_s2c_pcm_accum_mutex = NULL; } -/* WS 回调线程内联解码:不依赖 hasNext,解码成功后立即按块写入播放队列(与 flush 一致每片最多 1024 字节) */ +/* WS 回调线程内联解码:解码成功后立即写入播放队列;落盘按 hasNext 聚合 */ static void ct_ai_ws_s2c_opus_decode_on_ws_thread(const char *b64_chunk, size_t b64_len, int sample_rate, bool has_next) { - (void)has_next; - int pcm_samples = 0; int dec_ret = -1; + uint8_t *opus_bin = NULL; + size_t opus_bin_len = 0U; - if (b64_chunk != NULL && b64_len > 0U) { - uint8_t *opus_bin = (uint8_t *)malloc(b64_len); - if (opus_bin == NULL) { - CT_AI_WS_ERROR("opus inline: malloc(base64 decode buf) failed\n"); - dec_ret = -1; + if (b64_chunk == NULL || b64_len == 0U) { +#if CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE + if (!has_next && g_s2c_pcm_accum_mutex != NULL) { + (void)osMutexAcquire(g_s2c_pcm_accum_mutex, osWaitForever); + ct_ai_ws_s2c_opus_record_flush_inner(sample_rate); + (void)osMutexRelease(g_s2c_pcm_accum_mutex); + } +#endif + return; + } + + opus_bin = (uint8_t *)malloc(b64_len); + if (opus_bin == NULL) { + CT_AI_WS_ERROR("opus inline: malloc(base64 decode buf) failed\n"); + dec_ret = -1; + } else { + int b64_r = mbedtls_base64_decode(opus_bin, b64_len, &opus_bin_len, (const unsigned char *)b64_chunk, b64_len); + if (b64_r == 0 && opus_bin_len > 0U) { + dec_ret = opus_frame_to_pcm_decode_packet(opus_bin, (int)opus_bin_len, sample_rate, 1, g_opus_debug_pcm_buf, + OPUS_DEBUG_PCM_BUF_SAMPLES, &pcm_samples); } else { - size_t opus_bin_len = 0U; - int b64_r = mbedtls_base64_decode(opus_bin, b64_len, &opus_bin_len, (const unsigned char *)b64_chunk, b64_len); - if (b64_r == 0 && opus_bin_len > 0U) { - dec_ret = opus_frame_to_pcm_decode_packet(opus_bin, (int)opus_bin_len, sample_rate, 1, g_opus_debug_pcm_buf, - OPUS_DEBUG_PCM_BUF_SAMPLES, &pcm_samples); - } else { - dec_ret = -1; - } - free(opus_bin); + dec_ret = -1; } } - if (b64_chunk != NULL && b64_len > 0U) { - if (dec_ret == 0 && pcm_samples > 0) { - const uint8_t *pcm_pos = (const uint8_t *)g_opus_debug_pcm_buf; - size_t pcm_bytes = (size_t)pcm_samples * sizeof(short); - const uint32_t pcm_chunk_size = 1024U; + if (dec_ret == 0 && pcm_samples > 0) { + size_t pcm_bytes = (size_t)pcm_samples * sizeof(short); - while (pcm_bytes > 0U) { - uint32_t send_len = (pcm_bytes > (size_t)pcm_chunk_size) ? pcm_chunk_size : (uint32_t)pcm_bytes; - AI_SendAudioDataToQueue(pcm_pos, send_len); - pcm_pos += send_len; - pcm_bytes -= (size_t)send_len; - } - } else { - CT_AI_WS_ERROR("opus inline: decode fail ret=%d pcm_samples=%d\n", dec_ret, pcm_samples); + ct_ai_ws_s2c_pcm_send_to_queue_immediate((const uint8_t *)g_opus_debug_pcm_buf, pcm_bytes); + +#if CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE + if (g_s2c_pcm_accum_mutex != NULL) { + (void)osMutexAcquire(g_s2c_pcm_accum_mutex, osWaitForever); } + if (opus_bin_len > 0U && ct_ai_ws_s2c_opus_record_append_inner(opus_bin, opus_bin_len) != 0) { + CT_AI_WS_ERROR("opus inline: opus record append failed\n"); + } + if (ct_ai_ws_s2c_pcm_record_append_inner((const uint8_t *)g_opus_debug_pcm_buf, pcm_bytes) != 0) { + CT_AI_WS_ERROR("opus inline: pcm record append failed\n"); + } + if (!has_next) { + ct_ai_ws_s2c_opus_record_flush_inner(sample_rate); + } + if (g_s2c_pcm_accum_mutex != NULL) { + (void)osMutexRelease(g_s2c_pcm_accum_mutex); + } +#endif + } else if (b64_chunk != NULL && b64_len > 0U) { + CT_AI_WS_ERROR("opus inline: decode fail ret=%d pcm_samples=%d\n", dec_ret, pcm_samples); +#if CT_AI_S2C_OPUS_RECORD_SAVE_TO_FILE + if (g_s2c_pcm_accum_mutex != NULL) { + (void)osMutexAcquire(g_s2c_pcm_accum_mutex, osWaitForever); + ct_ai_ws_s2c_opus_record_free_inner(); + ct_ai_ws_s2c_pcm_record_free_inner(); + (void)osMutexRelease(g_s2c_pcm_accum_mutex); + } +#endif } + + free(opus_bin); } #endif /* CT_AI_S2C_OPUS_USE_DECODE_QUEUE */ -- 2.17.1