From: hcj <1224667075@qq.com> Date: Thu, 27 Nov 2025 10:21:44 +0000 (+0800) Subject: 修改vad数据输入,采用直接dmp中的怕pcm数据 X-Git-Url: http://git.snicker.com.cn/css/static/gitweb.css?a=commitdiff_plain;h=2e7581363a22418cefc554fcd59965fac7008702;p=mirrors%2FFD230.git 修改vad数据输入,采用直接dmp中的怕pcm数据 --- diff --git a/code/sdk/drivers/drivers/driver/audio/source/core/include/core_module.h b/code/sdk/drivers/drivers/driver/audio/source/core/include/core_module.h index 2c52f429e..c7d99a5bc 100644 --- a/code/sdk/drivers/drivers/driver/audio/source/core/include/core_module.h +++ b/code/sdk/drivers/drivers/driver/audio/source/core/include/core_module.h @@ -31,7 +31,7 @@ extern "C" { #define MOD_AOP_MAX_NUM 2 #define MOD_TRACK_MAX_NUM 2 #define MOD_ADEC_MAX_NUM 2 -#define MOD_AENC_MAX_NUM 1 +#define MOD_AENC_MAX_NUM 2 #define MOD_ANC_MAX_NUM 0 #define MOD_ANPU_MAX_NUM 0 #define MOD_HAID_MAX_NUM 1 diff --git a/code/sdk/middleware/services/media/hal/audio/plugins/brandy/common/include/plugin_common.h b/code/sdk/middleware/services/media/hal/audio/plugins/brandy/common/include/plugin_common.h index 7afbe27ec..53d8460a9 100644 --- a/code/sdk/middleware/services/media/hal/audio/plugins/brandy/common/include/plugin_common.h +++ b/code/sdk/middleware/services/media/hal/audio/plugins/brandy/common/include/plugin_common.h @@ -34,6 +34,7 @@ extern "C" { typedef enum { AUDIO_MOD_AI = 0, + AUDIO_MOD_AI_SEA, AUDIO_MOD_ADP, AUDIO_MOD_ADEC, AUDIO_MOD_AENC, @@ -102,6 +103,8 @@ int32_t GetParamByValue(uint8_t *param, uint32_t paramLen, const char *value, uint32_t valueLen, uint32_t configLen); int32_t SetSeaEnable(td_handle seaHandle, const char *value, uint32_t valueLen); + +bool isMultipleSeaOpen(void); #endif int32_t ExtraParamsInvoke(AudioHandle portHandle, struct InvokeAttr *invokeAttr, diff --git a/code/sdk/middleware/services/media/hal/audio/plugins/brandy/common/src/plugin_common.c b/code/sdk/middleware/services/media/hal/audio/plugins/brandy/common/src/plugin_common.c index 0126bbcb0..16d43ef93 100644 --- a/code/sdk/middleware/services/media/hal/audio/plugins/brandy/common/src/plugin_common.c +++ b/code/sdk/middleware/services/media/hal/audio/plugins/brandy/common/src/plugin_common.c @@ -40,7 +40,9 @@ extern "C" { #define WAIT_CREATE_AENC_SUCCESS_US 10000 static bool g_audioSdkIsInited = false; -static bool g_isCanCreateAenc = true; +static int32_t g_isCreateAencCount = 0; +static int32_t g_subPortSeaOpenCount = 0; +td_handle g_seaHandle = 0; static MediaHalMutexHandle g_audioSdkInitLock = NULL; #ifdef SUPPORT_NV static audio_vqe_param_t g_vqeParam = { 0 }; @@ -425,8 +427,22 @@ static int32_t SetSeaAfeAttr(td_handle seaHandle) return MEDIA_HAL_OK; } +bool isMultipleSeaOpen(void) +{ + printf("%s %d\r\n","g_subPortSeaOpenCount",g_subPortSeaOpenCount); + + return g_subPortSeaOpenCount > 1; +} + int32_t AudioSDKSeaOpen(td_handle *seaHandle, struct PortPluginAttr inputAttr) { + MediaHalMutexLock(g_audioSdkInitLock); + if (g_subPortSeaOpenCount > 0) { + *seaHandle = g_seaHandle; + g_subPortSeaOpenCount++; + MediaHalMutexUnLock(g_audioSdkInitLock); + return MEDIA_HAL_OK; + } int32_t ret; uapi_sea_attr seaAttr = {}; uapi_sea_eng_sel seaEng = {}; @@ -462,23 +478,38 @@ int32_t AudioSDKSeaOpen(td_handle *seaHandle, struct PortPluginAttr inputAttr) if (ret != MEDIA_HAL_OK) { MEDIA_HAL_LOGW(MODULE_NAME, "set sea param failed"); } + g_seaHandle = *seaHandle; + g_subPortSeaOpenCount++; MEDIA_HAL_LOGD(MODULE_NAME, "success"); + printf("AudioSDKSeaOpen %s %d\r\n","g_subPortSeaOpenCount",g_subPortSeaOpenCount); + MediaHalMutexUnLock(g_audioSdkInitLock); return MEDIA_HAL_OK; SEA_UNLOAD_ENGINE: (void)uapi_sea_unload_engine(UAPI_SEA_LIB_SEE, "imedia_2mic"); + MediaHalMutexUnLock(g_audioSdkInitLock); return ret; } int32_t AudioSDKSeaClose(td_handle seaHandle) { + MediaHalMutexLock(g_audioSdkInitLock); int32_t ret; + g_subPortSeaOpenCount--; + if (g_subPortSeaOpenCount > 0) { + MediaHalMutexUnLock(g_audioSdkInitLock); + return MEDIA_HAL_OK; + } + printf("AudioSDKSeaClose %s %d\r\n","g_subPortSeaOpenCount",g_subPortSeaOpenCount); + MEDIA_HAL_DOFUNC_TIME(ret = uapi_sea_destroy(seaHandle)); CHK_FAILED_NO_RETURN(ret, MEDIA_HAL_OK, "destroy sea failed"); MEDIA_HAL_DOFUNC_TIME(ret = uapi_sea_unload_engine(UAPI_SEA_LIB_SEE, "imedia_2mic")); CHK_FAILED_NO_RETURN(ret, MEDIA_HAL_OK, "unload sea engine failed"); MEDIA_HAL_LOGD(MODULE_NAME, "success"); + g_seaHandle = 0; + MediaHalMutexUnLock(g_audioSdkInitLock); return MEDIA_HAL_OK; } @@ -487,8 +518,7 @@ static int32_t WaitCreateAnec(void) int32_t cnt = 0; while (cnt < WAIT_CREATE_AENC_LOOP_CNT) { MediaHalMutexLock(g_audioSdkInitLock); - if (g_isCanCreateAenc) { - g_isCanCreateAenc = false; + if (g_isCreateAencCount < 0x2) { MediaHalMutexUnLock(g_audioSdkInitLock); return MEDIA_HAL_OK; } @@ -508,10 +538,9 @@ int32_t AudioSDKAencOpen(td_handle *aencHandle, uapi_aenc_attr aencAttr) return ret; } MEDIA_HAL_DOFUNC_TIME(ret = uapi_aenc_create(aencHandle)); - if (ret != MEDIA_HAL_OK) { - MEDIA_HAL_LOGE(MODULE_NAME, "uapi_aenc_create failed"); - g_isCanCreateAenc = true; - return ret; + if (ret != MEDIA_HAL_OK) { + MEDIA_HAL_LOGE(MODULE_NAME, "uapi_aenc_create failed"); + return ret; } MEDIA_HAL_LOGI(MODULE_NAME, "codec_id: 0x%x", aencAttr.codec_id); @@ -520,22 +549,24 @@ int32_t AudioSDKAencOpen(td_handle *aencHandle, uapi_aenc_attr aencAttr) MEDIA_HAL_LOGE(MODULE_NAME, "uapi_aenc_set_attr failed: 0x%x, codec id: 0x%x", ret, aencAttr.codec_id); goto CLOSE; } -#ifndef MELODY_AUDIO_SUPPORT - if (aencAttr.codec_id == UAPI_ACODEC_ID_OPUS) { - uint32_t bitRate = OPUS_ENC_BIT_RATE; - ret = uapi_aenc_set_param(*aencHandle, UAPI_AENC_PARAM_BIT_RATE, &bitRate, sizeof(uint32_t)); - if (ret != MEDIA_HAL_OK) { - MEDIA_HAL_LOGE(MODULE_NAME, "uapi set aenc send cmd bit Rate faild, ret: 0x%x", ret); - goto CLOSE; - } - } +#ifndef MELODY_AUDIO_SUPPORT + if (aencAttr.codec_id == UAPI_ACODEC_ID_OPUS) { + uint32_t bitRate = OPUS_ENC_BIT_RATE; + ret = uapi_aenc_set_param(*aencHandle, UAPI_AENC_PARAM_BIT_RATE, &bitRate, sizeof(uint32_t)); + if (ret != MEDIA_HAL_OK) { + MEDIA_HAL_LOGE(MODULE_NAME, "uapi set aenc send cmd bit Rate faild, ret: 0x%x", ret); + goto CLOSE; + } + } #endif + MediaHalMutexLock(g_audioSdkInitLock); + g_isCreateAencCount++; + MediaHalMutexUnLock(g_audioSdkInitLock); MEDIA_HAL_LOGD(MODULE_NAME, "success"); return MEDIA_HAL_OK; CLOSE: MEDIA_HAL_DOFUNC_TIME((void)uapi_aenc_destroy(*aencHandle)); - g_isCanCreateAenc = true; return ret; } @@ -545,7 +576,7 @@ int32_t AudioSDKAencClose(td_handle aencHandle) MEDIA_HAL_DOFUNC_TIME(ret = uapi_aenc_destroy(aencHandle)); CHK_FAILED_RETURN(ret, MEDIA_HAL_OK, ret, "uapi_aenc_destroy failed"); MediaHalMutexLock(g_audioSdkInitLock); - g_isCanCreateAenc = true; + g_isCreateAencCount--; MediaHalMutexUnLock(g_audioSdkInitLock); MEDIA_HAL_LOGD(MODULE_NAME, "success"); return MEDIA_HAL_OK; @@ -559,6 +590,13 @@ int32_t AudioSDKAttach(AudioSDKModuleType modType, td_handle srcHandle, td_handl MEDIA_HAL_DOFUNC_TIME(ret = uapi_ai_attach_output(srcHandle, dstHandle)); CHK_FAILED_RETURN(ret, MEDIA_HAL_OK, ret, "uapi_ai_attach_output"); break; + case AUDIO_MOD_AI_SEA: + if (g_subPortSeaOpenCount <= 1) { // Sea only needs to be bound once. + printf("AudioSDKAttach %X %X\r\n",srcHandle,dstHandle); + MEDIA_HAL_DOFUNC_TIME(ret = uapi_ai_attach_output(srcHandle, dstHandle)); + CHK_FAILED_RETURN(ret, MEDIA_HAL_OK, ret, "uapi_ai_attach_output"); + } + break; case AUDIO_MOD_ADP: MEDIA_HAL_DOFUNC_TIME(ret = uapi_adp_attach_output(srcHandle, dstHandle)); CHK_FAILED_RETURN(ret, MEDIA_HAL_OK, ret, "uapi_adp_attach_output failed"); @@ -596,6 +634,14 @@ int32_t AudioSDKDetach(AudioSDKModuleType modType, td_handle srcHandle, td_handl MEDIA_HAL_DOFUNC_TIME(ret = uapi_ai_detach_output(srcHandle, dstHandle)); CHK_FAILED_RETURN(ret, MEDIA_HAL_OK, ret, "uapi_ai_detach_output failed"); break; + case AUDIO_MOD_AI_SEA: + if (g_subPortSeaOpenCount <= 1) { // Sea only needs to be unbound once. + printf("AudioSDKDetach AUDIO_MOD_AI_SEA %X %X\r\n",srcHandle,dstHandle); + + MEDIA_HAL_DOFUNC_TIME(ret = uapi_ai_detach_output(srcHandle, dstHandle)); + CHK_FAILED_RETURN(ret, MEDIA_HAL_OK, ret, "uapi_ai_detach_output failed"); + } + break; case AUDIO_MOD_ADP: MEDIA_HAL_DOFUNC_TIME(ret = uapi_adp_detach_output(srcHandle, dstHandle)); CHK_FAILED_RETURN(ret, MEDIA_HAL_OK, ret, "uapi_adp_detach_output failed"); @@ -609,6 +655,8 @@ int32_t AudioSDKDetach(AudioSDKModuleType modType, td_handle srcHandle, td_handl CHK_FAILED_RETURN(ret, MEDIA_HAL_OK, ret, "uapi_aenc_detach_output failed"); break; case AUDIO_MOD_SEA: + printf("AudioSDKDetach AUDIO_MOD_SEA %X %X\r\n",srcHandle,dstHandle); + MEDIA_HAL_DOFUNC_TIME(ret = uapi_sea_detach_output(srcHandle, UAPI_SEA_OUTPUT_ASR_SRC, dstHandle)); CHK_FAILED_RETURN(ret, MEDIA_HAL_OK, ret, "uapi_sea_detach_output failed"); break; @@ -766,19 +814,19 @@ int32_t GetParamByValue(uint8_t *param, uint32_t paramLen, const char *value, uint32_t valueLen, uint32_t configLen) { uint32_t prefixlen = 0; - for (uint32_t i = 0; i < valueLen; i++) { - if (value[i] == ':') { - prefixlen = i + 1; - break; - } + for (uint32_t i = 0; i < valueLen; i++) { + if (value[i] == ':') { + prefixlen = i + 1; + break; + } } - if (prefixlen == 0) { + if (prefixlen == 0) { MEDIA_HAL_LOGE(MODULE_NAME, "prefix is not found."); - return MEDIA_HAL_ERR; + return MEDIA_HAL_ERR; } - if (memcpy_s(param, paramLen, value + prefixlen, configLen) != EOK) { + if (memcpy_s(param, paramLen, value + prefixlen, configLen) != EOK) { MEDIA_HAL_LOGE(MODULE_NAME, "set spk cali coef memcpy_s failed!"); - return MEDIA_HAL_ERR; + return MEDIA_HAL_ERR; } return MEDIA_HAL_OK; } diff --git a/code/sdk/middleware/services/media/hal/audio/plugins/brandy/primary/primary_sub_port_in.c b/code/sdk/middleware/services/media/hal/audio/plugins/brandy/primary/primary_sub_port_in.c index da2a6614e..1ca8cf7d0 100644 --- a/code/sdk/middleware/services/media/hal/audio/plugins/brandy/primary/primary_sub_port_in.c +++ b/code/sdk/middleware/services/media/hal/audio/plugins/brandy/primary/primary_sub_port_in.c @@ -71,7 +71,7 @@ typedef struct { /* attr context */ bool isRaw; struct PortPluginAttr inputAttr; -} SubPortTrackInContext; + } SubPortTrackInContext; typedef struct { #ifdef AUDIO_SDK_SUPPORT @@ -80,7 +80,7 @@ typedef struct { td_handle reserved; } SubPortInContext; -static bool g_subPortTrackInOpened = false; +static uint32_t g_subPortTrackInOpened = 0; static MediaHalMutexHandle g_subPortTrackInLock = NULL; typedef struct { @@ -154,7 +154,7 @@ static int32_t SubPortTrackInInit(SubPortTrackInContext *portTrackInCtx, const s portTrackInCtx->common.SetVolume = AudioPrimarySubPortInSetVolume; portTrackInCtx->common.GetVolume = AudioPrimarySubPortInGetVolume; portTrackInCtx->common.Invoke = AudioPrimarySubPortInInvoke; - MEDIA_HAL_LOGD(MODULE_NAME, "success"); + MEDIA_HAL_LOGD(MODULE_NAME, "success"); return MEDIA_HAL_OK; } @@ -163,7 +163,7 @@ static int32_t AIDevOpen(td_handle *aiHandle, enum AudioCategory audioCategory) { MediaHalInitStaticMutexLock(&g_subPortTrackInLock); MediaHalMutexLock(g_subPortTrackInLock); - if (!g_subPortTrackInOpened) { + if (g_subPortTrackInOpened == 0) { int32_t ret; uapi_ai_attr aiAttr = {}; uapi_ai_port aiPort; @@ -199,7 +199,7 @@ static int32_t AIDevOpen(td_handle *aiHandle, enum AudioCategory audioCategory) MediaHalMutexUnLock(g_subPortTrackInLock); return ret; } -uapi_ai_gain gain = {}; + uapi_ai_gain gain = {}; MEDIA_HAL_DOFUNC_TIME(ret = uapi_ai_get_volume(*aiHandle, &gain)); CHK_FAILED_RETURN(ret, MEDIA_HAL_OK, ret, "uapi_ai_get_volume failed"); #ifdef BOARD_FD230 @@ -207,13 +207,12 @@ uapi_ai_gain gain = {}; #elif defined(BORAD_N9) || defined(BORAD_W9) gain.integer = gain.integer + 10; // 前端mic灵敏度不同可通过此处调增,增加4db增益 默认gain.integer = gain.integer+4 #elif defined(LEINIAO) - gain.integer = gain.integer + 8; // 前端mic灵敏度不同可通过此处调增,增加4db增益 默认gain.integer = gain.integer+4 + gain.integer = gain.integer +3; // 前端mic灵敏度不同可通过此处调增,增加4db增益 默认gain.integer = gain.integer+4 #endif MEDIA_HAL_DOFUNC_TIME(ret = uapi_ai_set_volume(*aiHandle, &gain)); - CHK_FAILED_RETURN(ret, MEDIA_HAL_OK, ret, "uapi_ai_set_volume failed"); - - g_subPortTrackInOpened = true; + CHK_FAILED_NO_RETURN(ret, MEDIA_HAL_OK, "uapi_ai_set_volume failed"); } + g_subPortTrackInOpened++; MediaHalMutexUnLock(g_subPortTrackInLock); MEDIA_HAL_LOGD(MODULE_NAME, "success"); @@ -226,7 +225,8 @@ static int32_t AIDevClose(td_handle aiHandle) { MediaHalInitStaticMutexLock(&g_subPortTrackInLock); MediaHalMutexLock(g_subPortTrackInLock); - if (g_subPortTrackInOpened) { + g_subPortTrackInOpened--; + if (g_subPortTrackInOpened == 0) { int32_t ret; MEDIA_HAL_DOFUNC_TIME(ret = uapi_ai_close(aiHandle)); if (ret != MEDIA_HAL_OK) { @@ -234,8 +234,6 @@ static int32_t AIDevClose(td_handle aiHandle) MediaHalMutexUnLock(g_subPortTrackInLock); return ret; } - - g_subPortTrackInOpened = false; } MediaHalMutexUnLock(g_subPortTrackInLock); @@ -269,7 +267,7 @@ static int32_t AudioSDKOpenInternal(SubPortTrackInContext *portTrackInCtx) aencAttr.param.interleaved = true; if (inputAttrTemp.type == AUDIO_IN_MEDIA) { aencAttr.frame_mode_output = TD_TRUE; - } + } aencAttr.param.channels = inputAttrTemp.channelCount; aencAttr.param.bit_depth = inputAttrTemp.bitWidth; aencAttr.param.sample_rate = inputAttrTemp.sampleRate; @@ -529,13 +527,13 @@ static int32_t CheckParam(SubPortTrackInContext *portTrackInCtx, uint8_t *buffer MEDIA_HAL_LOGE(MODULE_NAME, "input buffer too small, actual: %llu vs except: %u", requestBytes, audioStream.size); return MEDIA_HAL_INVALID_PARAM; - } - + } + if (requestBytes > SIZE_MAX) { MEDIA_HAL_LOGE(MODULE_NAME, "input buffer size overflow"); return MEDIA_HAL_INVALID_PARAM; } - + if (memcpy_s(buffer, (size_t)requestBytes, audioStream.data, audioStream.size) != EOK) { MEDIA_HAL_LOGE(MODULE_NAME, "memcpy_s failed"); return MEDIA_HAL_ERR; @@ -569,11 +567,11 @@ int32_t AudioPrimarySubPortInAcquireFrame(AudioHandle trackHandle, uint8_t *buff *replyBytes = audioStream.size; ret = CheckParam(portTrackInCtx, buffer, requestBytes, audioStream); if (ret != MEDIA_HAL_OK) { - MEDIA_HAL_LOGE(MODULE_NAME, "param is invalid"); + MEDIA_HAL_LOGE(MODULE_NAME, "param is invalid"); (void)uapi_adp_release_stream(portTrackInCtx->outputAdpHandle, &audioStream); return ret; } - + ret = uapi_adp_release_stream(portTrackInCtx->outputAdpHandle, &audioStream); if (ret != MEDIA_HAL_OK) { MEDIA_HAL_LOGE(MODULE_NAME, "release stream failed: 0x%x", ret); @@ -898,7 +896,7 @@ FREE_TRACK_IN: int32_t AudioPrimarySubPortInDestroyTrack(AudioHandle portHandle, AudioHandle trackHandle) { #ifdef AUDIO_SDK_SUPPORT - SubPortTrackInContext *portTrackInCtx = (SubPortTrackInContext *)trackHandle; + SubPortTrackInContext *portTrackInCtx = (SubPortTrackInContext *)trackHandle; int32_t ret = AIDetach(portTrackInCtx); CHK_FAILED_NO_RETURN(ret, MEDIA_HAL_OK, "AIDetach failed"); diff --git a/code/sdk/tools/pkg/bin/brandy/dsp/max/dsp_main.bin b/code/sdk/tools/pkg/bin/brandy/dsp/max/dsp_main.bin index 492ce9e98..5c93e9147 100644 Binary files a/code/sdk/tools/pkg/bin/brandy/dsp/max/dsp_main.bin and b/code/sdk/tools/pkg/bin/brandy/dsp/max/dsp_main.bin differ diff --git a/code/sdk/zhihui/ai/include/ai_media.h b/code/sdk/zhihui/ai/include/ai_media.h index 89eb326dc..cb4f7b50a 100644 --- a/code/sdk/zhihui/ai/include/ai_media.h +++ b/code/sdk/zhihui/ai/include/ai_media.h @@ -105,6 +105,7 @@ typedef struct { int32_t old_Start_AIVoice(AIVoiceDataCallback dataCallback, void* userData, const AIVoiceAudioParams* inputAudioParams, const AIVoiceAudioParams* outputAudioParams, + const AIVoiceAudioParams* secondStreamInParams, bool useOggEncapsulation); int32_t AI_StartOggEncapsulation(void); @@ -175,6 +176,34 @@ bool AI_GetInterruptFlag(void); */ int32_t AI_ForceInterrupt(void); +/* + * @brief 设置第二个音频上行流参数 + * @param audioParams 第二个上行流音频参数,如果为nullptr则禁用第二个上行流 + * @return 0 成功,-1 失败 + */ +int32_t AI_SetSecondStreamInParams(const AIVoiceAudioParams* audioParams); + +/* + * @brief 获取第二个音频上行流参数 + * @param audioParams 输出参数:第二个上行流音频参数 + * @return 0 成功,-1 失败 + */ +int32_t AI_GetSecondStreamInParams(AIVoiceAudioParams* audioParams); + +/* + * @brief 检查第二个音频上行流是否启用 + * @return true 已启用,false 未启用或服务未运行 + */ +bool AI_IsSecondStreamInEnabled(void); + +/* + * @brief 从第二个音频上行流获取音频数据缓冲区 + * @param data 音频数据缓冲区指针 + * @param size 输入参数:缓冲区大小,输出参数:实际读取的数据大小 + * @return 0 成功,-1 失败 + */ +int32_t AI_SecondStreamInObtainBuffer(uint8_t *data, uint32_t *size); + #ifdef __cplusplus }; #endif diff --git a/code/sdk/zhihui/ai/media/ai_media_service_ext.cpp b/code/sdk/zhihui/ai/media/ai_media_service_ext.cpp index 65f5334ac..14de8a835 100644 --- a/code/sdk/zhihui/ai/media/ai_media_service_ext.cpp +++ b/code/sdk/zhihui/ai/media/ai_media_service_ext.cpp @@ -125,6 +125,13 @@ typedef struct { CaptureOggContext oggContext; // OGG封装上下文 bool useOggEncapsulation; // 是否使用OGG封装 bool oggEncapsulationStarted; // OGG封装是否已开始 + + // 第二个上行流(用于额外的音频输入) + std::shared_ptr secondStreamIn; // 第二个上行流句柄 + AudioCodecFormat secondInputAudioFormat; // 第二个输入音频格式 + uint32_t secondInputChannelCount; // 第二个输入声道数 + uint32_t secondInputSampleRate; // 第二个输入采样率 + bool useSecondStreamIn; // 是否使用第二个上行流 } AIVoiceContext; static MediaMutexHandle g_aiVoiceContextMutex = nullptr; @@ -720,6 +727,161 @@ static MediaMutexHandle GetGlobalMutex(void) static std::shared_ptr g_streamIn = nullptr; static std::shared_ptr g_streamOut = nullptr; +/** + * @brief 初始化第二个音频上行流 + * @param context AI语音上下文 + * @return 0 成功,-1 失败 + */ +static int32_t InitSecondAudioStreamIn(AIVoiceContext *context) +{ + if (context == nullptr) { + MEDIA_ERR_LOG("AI voice context is null"); + return -1; + } + + if (!context->useSecondStreamIn) { + MEDIA_INFO_LOG("Second audio stream in not enabled"); + return 0; + } + + // 创建第二个上行流实例 + context->secondStreamIn = std::make_shared(); + if (context->secondStreamIn == nullptr || context->secondStreamIn.get() == nullptr) { + MEDIA_ERR_LOG("Failed to create second audio stream in"); + return -1; + } + + // 配置第二个上行流参数 + CapturerInputConfig secondCaptureConfig = {}; + secondCaptureConfig.audioFormat = context->secondInputAudioFormat; + secondCaptureConfig.channelCount = context->secondInputChannelCount; + secondCaptureConfig.linkDir = AUDIO_UP_LINK; + secondCaptureConfig.sampleRate = context->secondInputSampleRate; + secondCaptureConfig.sessionID = context->sessionId; + secondCaptureConfig.streamType = AUDIO_STREAM_VOICE_ASSISTANT; + + // 初始化第二个上行流 + int32_t ret = context->secondStreamIn->Init(secondCaptureConfig); + if (ret != 0) { + MEDIA_ERR_LOG("Second audio stream in init failed: 0x%x", ret); + context->secondStreamIn.reset(); + context->secondStreamIn = nullptr; + return -1; + } + + MEDIA_INFO_LOG("Second audio stream in initialized successfully"); + return 0; +} + +/** + * @brief 启动第二个音频上行流 + * @param context AI语音上下文 + * @return 0 成功,-1 失败 + */ +static int32_t StartSecondAudioStreamIn(AIVoiceContext *context) +{ + if (context == nullptr || context->secondStreamIn == nullptr) { + return 0; // 如果未启用或未初始化,直接返回成功 + } + + int32_t ret = context->secondStreamIn->Start(); + if (ret != 0) { + MEDIA_ERR_LOG("Second audio stream in start failed: 0x%x", ret); + return -1; + } + + MEDIA_INFO_LOG("Second audio stream in started successfully"); + return 0; +} + +/** + * @brief 停止第二个音频上行流 + * @param context AI语音上下文 + * @return 0 成功,-1 失败 + */ +static int32_t StopSecondAudioStreamIn(AIVoiceContext *context) +{ + if (context == nullptr || context->secondStreamIn == nullptr) { + return 0; // 如果未启用或未初始化,直接返回成功 + } + + int32_t ret = context->secondStreamIn->Stop(); + if (ret != 0) { + MEDIA_ERR_LOG("Second audio stream in stop failed: 0x%x", ret); + return -1; + } + + MEDIA_INFO_LOG("Second audio stream in stopped successfully"); + return 0; +} + +/** + * @brief 反初始化第二个音频上行流 + * @param context AI语音上下文 + * @return 0 成功,-1 失败 + */ +static int32_t DeinitSecondAudioStreamIn(AIVoiceContext *context) +{ + if (context == nullptr || context->secondStreamIn == nullptr) { + return 0; // 如果未启用或未初始化,直接返回成功 + } + + context->secondStreamIn->Release(); + context->secondStreamIn.reset(); + context->secondStreamIn = nullptr; + + MEDIA_INFO_LOG("Second audio stream in deinitialized successfully"); + return 0; +} + +/** + * @brief 停止第二个音频上行流(线程安全版本,使用全局上下文) + * @return 0 成功,-1 失败 + */ +static int32_t StopSecondAudioStreamInSafe(void) +{ + int32_t ret = 0; + MediaMutexHandle mutex = GetGlobalMutex(); + if (mutex == nullptr) { + return -1; + } + + MediaMutexLock(mutex); + if (g_aiVoiceContext != nullptr) { + ret = StopSecondAudioStreamIn(g_aiVoiceContext); + if (ret != 0) { + MEDIA_ERR_LOG("second audio stream in stop failed"); + } + } + MediaMutexUnLock(mutex); + + return ret; +} + +/** + * @brief 反初始化第二个音频上行流(线程安全版本,使用全局上下文) + * @return 0 成功,-1 失败 + */ +static int32_t DeinitSecondAudioStreamInSafe(void) +{ + int32_t ret = 0; + MediaMutexHandle mutex = GetGlobalMutex(); + if (mutex == nullptr) { + return -1; + } + + MediaMutexLock(mutex); + if (g_aiVoiceContext != nullptr) { + ret = DeinitSecondAudioStreamIn(g_aiVoiceContext); + if (ret != 0) { + MEDIA_ERR_LOG("second audio stream in deinit failed"); + } + } + MediaMutexUnLock(mutex); + + return ret; +} + // 注意:g_aiVoiceContext 已在上方定义,这里不需要重复定义 @@ -795,6 +957,11 @@ static void DeinitAIVoiceResources(AIVoiceContext *context) MEDIA_INFO_LOG("OGG context cleaned up"); } + // 清理第二个上行流 + if (context->secondStreamIn != nullptr) { + DeinitSecondAudioStreamIn(context); + } + // 清理队列 if (context->inputQueue != NULL) { // 清空队列中的所有消息 @@ -879,6 +1046,22 @@ static int32_t LocalAIVoiceStart(AIVoiceContext *context) MEDIA_ERR_LOG("audio stream in start failed"); } + // 初始化第二个上行流(如果启用) + if (context->useSecondStreamIn) { + ret = InitSecondAudioStreamIn(context); + if (ret != 0) { + MEDIA_ERR_LOG("Failed to init second audio stream in"); + // 不返回错误,继续执行 + } else { + // 启动第二个上行流(如果初始化成功) + ret = StartSecondAudioStreamIn(context); + if (ret != 0) { + MEDIA_ERR_LOG("Failed to start second audio stream in"); + // 不返回错误,继续执行 + } + } + } + ret = AudioStreamOutPlay(); if (ret != 0) { MEDIA_ERR_LOG("audio stream out play failed"); @@ -900,15 +1083,26 @@ static int32_t LocalAIVoiceStop(void) if (ret != 0) { MEDIA_ERR_LOG("audio stream in stop failed"); } + ret = AudioStreamInDeinit(); + if (ret != 0) { + MEDIA_ERR_LOG("audio stream in release failed"); + } - ret = AudioStreamOutStop(); + // 停止第二个上行流 + ret = StopSecondAudioStreamInSafe(); if (ret != 0) { - MEDIA_ERR_LOG("audio stream out stop failed"); + MEDIA_ERR_LOG("second audio stream in stop failed"); } - ret = AudioStreamInDeinit(); + // 反初始化第二个上行流 + ret = DeinitSecondAudioStreamInSafe(); if (ret != 0) { - MEDIA_ERR_LOG("audio stream in release failed"); + MEDIA_ERR_LOG("second audio stream in deinit failed"); + } + + ret = AudioStreamOutStop(); + if (ret != 0) { + MEDIA_ERR_LOG("audio stream out stop failed"); } ret = AudioStreamOutDeinit(); @@ -1284,6 +1478,7 @@ static void AIVoiceThread(void *priv) int32_t old_Start_AIVoice(AIVoiceDataCallback dataCallback, void* userData, const AIVoiceAudioParams* inputAudioParams, const AIVoiceAudioParams* outputAudioParams, + const AIVoiceAudioParams* secondStreamInParams, bool useOggEncapsulation) { MEDIA_ERR_LOG("StartAIVoice"); @@ -1356,6 +1551,24 @@ int32_t old_Start_AIVoice(AIVoiceDataCallback dataCallback, void* userData, MEDIA_INFO_LOG("OGG encapsulation disabled by parameter"); } + // 设置第二个上行流参数(如果传入参数则启用,否则不启用) + if (secondStreamInParams != nullptr) { + g_aiVoiceContext->useSecondStreamIn = true; + g_aiVoiceContext->secondInputAudioFormat = (AudioCodecFormat)secondStreamInParams->audioFormat; + g_aiVoiceContext->secondInputChannelCount = secondStreamInParams->channelCount; + g_aiVoiceContext->secondInputSampleRate = secondStreamInParams->sampleRate; + g_aiVoiceContext->secondStreamIn = nullptr; + MEDIA_INFO_LOG("Second stream in enabled with params: format=%d, channels=%u, sampleRate=%u", + secondStreamInParams->audioFormat, secondStreamInParams->channelCount, secondStreamInParams->sampleRate); + } else { + g_aiVoiceContext->useSecondStreamIn = false; + g_aiVoiceContext->secondInputAudioFormat = DEFAULT_AUDIO_FORMAT; + g_aiVoiceContext->secondInputChannelCount = DEFAULT_CHANNEL_COUNT; + g_aiVoiceContext->secondInputSampleRate = DEFAULT_SAMPLE_RATE; + g_aiVoiceContext->secondStreamIn = nullptr; + MEDIA_INFO_LOG("Second stream in disabled (null parameter)"); + } + MediaMutexUnLock(mutex); MediaThreadJoin(&g_aiVoiceContext->taskHandle); @@ -1990,6 +2203,149 @@ int32_t AI_ForceInterrupt(void) return 0; } +/** + * @brief 设置第二个音频上行流参数 + * @param audioParams 第二个上行流音频参数,如果为nullptr则禁用第二个上行流 + * @return 0 成功,-1 失败 + */ +int32_t AI_SetSecondStreamInParams(const AIVoiceAudioParams* audioParams) +{ + MediaMutexHandle mutex = GetGlobalMutex(); + if (mutex == nullptr) { + MEDIA_ERR_LOG("Failed to get global mutex"); + return -1; + } + + MediaMutexLock(mutex); + if (g_aiVoiceContext == nullptr) { + MEDIA_ERR_LOG("AI voice context is null, please start AI voice first"); + MediaMutexUnLock(mutex); + return -1; + } + + if (audioParams == nullptr) { + // 禁用第二个上行流 + g_aiVoiceContext->useSecondStreamIn = false; + MEDIA_INFO_LOG("Second audio stream in disabled"); + } else { + // 设置第二个上行流参数 + g_aiVoiceContext->useSecondStreamIn = true; + g_aiVoiceContext->secondInputAudioFormat = (AudioCodecFormat)audioParams->audioFormat; + g_aiVoiceContext->secondInputChannelCount = audioParams->channelCount; + g_aiVoiceContext->secondInputSampleRate = audioParams->sampleRate; + MEDIA_INFO_LOG("Second audio stream in params set: format=%d, channels=%u, sampleRate=%u", + audioParams->audioFormat, audioParams->channelCount, audioParams->sampleRate); + } + + MediaMutexUnLock(mutex); + return 0; +} + +/** + * @brief 获取第二个音频上行流参数 + * @param audioParams 输出参数:第二个上行流音频参数 + * @return 0 成功,-1 失败 + */ +int32_t AI_GetSecondStreamInParams(AIVoiceAudioParams* audioParams) +{ + if (audioParams == nullptr) { + MEDIA_ERR_LOG("Invalid output parameter"); + return -1; + } + + MediaMutexHandle mutex = GetGlobalMutex(); + if (mutex == nullptr) { + MEDIA_ERR_LOG("Failed to get global mutex"); + return -1; + } + + MediaMutexLock(mutex); + if (g_aiVoiceContext == nullptr) { + MEDIA_ERR_LOG("AI voice context is null, please start AI voice first"); + MediaMutexUnLock(mutex); + return -1; + } + + audioParams->audioFormat = (int32_t)g_aiVoiceContext->secondInputAudioFormat; + audioParams->channelCount = g_aiVoiceContext->secondInputChannelCount; + audioParams->sampleRate = g_aiVoiceContext->secondInputSampleRate; + + MediaMutexUnLock(mutex); + return 0; +} + +/** + * @brief 检查第二个音频上行流是否启用 + * @return true 已启用,false 未启用或服务未运行 + */ +bool AI_IsSecondStreamInEnabled(void) +{ + bool enabled = false; + + MediaMutexHandle mutex = GetGlobalMutex(); + if (mutex == nullptr) { + return false; + } + + MediaMutexLock(mutex); + if (g_aiVoiceContext != nullptr) { + enabled = g_aiVoiceContext->useSecondStreamIn; + } + MediaMutexUnLock(mutex); + + return enabled; +} + +/** + * @brief 从第二个音频上行流获取音频数据缓冲区 + * @param data 音频数据缓冲区指针 + * @param size 输入参数:缓冲区大小,输出参数:实际读取的数据大小 + * @return 0 成功,-1 失败 + */ +int32_t AI_SecondStreamInObtainBuffer(uint8_t *data, uint32_t *size) +{ + if (data == nullptr || size == nullptr || *size == 0) { + MEDIA_ERR_LOG("Invalid parameters for second stream in obtain buffer"); + return -1; + } + + // 先获取流指针(需要锁保护) + std::shared_ptr secondStreamIn = nullptr; + MediaMutexHandle mutex = GetGlobalMutex(); + if (mutex != nullptr) { + MediaMutexLock(mutex); + if (g_aiVoiceContext != nullptr && + g_aiVoiceContext->useSecondStreamIn && + g_aiVoiceContext->secondStreamIn != nullptr) { + secondStreamIn = g_aiVoiceContext->secondStreamIn; + } + MediaMutexUnLock(mutex); + } + + if (secondStreamIn == nullptr) { + MEDIA_ERR_LOG("Second audio stream in is not available"); + return -1; + } + + // 读取数据(不需要锁) + CapturerBuffer buffer; + buffer.data = data; + buffer.size = *size; + int32_t ret = secondStreamIn->ObtainBuffer(buffer, false); + + if (ret < 0) { + MEDIA_ERR_LOG("Second stream in obtain buffer failed: %d", ret); + return -1; + } + else + { + // MEDIA_INFO_LOG("Second stream in obtain buffer %d\r\n",buffer.size); + } + + *size = static_cast(ret); + return 0; +} + #ifdef __cplusplus }; #endif diff --git a/code/sdk/zhihui/ai_new/ai_V2.5/ai_v2_5_main.cpp b/code/sdk/zhihui/ai_new/ai_V2.5/ai_v2_5_main.cpp index a462d56f3..64b50e16d 100644 --- a/code/sdk/zhihui/ai_new/ai_V2.5/ai_v2_5_main.cpp +++ b/code/sdk/zhihui/ai_new/ai_V2.5/ai_v2_5_main.cpp @@ -352,7 +352,7 @@ int ai_service_start(void) .sampleRate = AI_DEFAULT_SAMPLE_RATE }; - old_Start_AIVoice(AIVoiceDataCallback_V25,nullptr, nullptr,nullptr,false); + old_Start_AIVoice(AIVoiceDataCallback_V25,nullptr, nullptr,nullptr,nullptr,false); return 0; } diff --git a/code/sdk/zhihui/ai_old/websocket_server_example.cpp b/code/sdk/zhihui/ai_old/websocket_server_example.cpp index 4e6ebf2de..40cbbe3ff 100644 --- a/code/sdk/zhihui/ai_old/websocket_server_example.cpp +++ b/code/sdk/zhihui/ai_old/websocket_server_example.cpp @@ -482,7 +482,11 @@ bool chat_url_callback(ws_server_type_t type, char *url_buf, uint32_t url_size, snprintf(url_buf, url_size, "ws://118.145.158.207:22080/chat/ws?platform=moonshot"); } else if (g_chat_ai_type == 5) { // AI_TYPE_DE snprintf(url_buf, url_size, "ws://118.145.158.207:22080/chat/ws?platform=huoshan&model=ep-20250205103849-582m7"); - } else { + } + else if (g_chat_ai_type == 6) { // AI_TYPE_DE + snprintf(url_buf, url_size, "ws://118.145.158.207:22080/chat/ws?platform=telechat"); + } + else { snprintf(url_buf, url_size, "ws://118.145.158.207:22080/chat/ws?platform=aliyun"); } } @@ -612,7 +616,7 @@ errcode_t start_all_websocket_servers(uint8_t chat_ai_type) } // ai_media_init(); - old_Start_AIVoice(AIVoiceDataCallback_,nullptr,nullptr,nullptr,false); + old_Start_AIVoice(AIVoiceDataCallback_,nullptr,nullptr,nullptr,nullptr,false); /* 不进行默认回退注册,未注册时仅打印错误日志 */ diff --git a/code/sdk/zhihui/imsl/src/vad_server.c b/code/sdk/zhihui/imsl/src/vad_server.c index 8fd27ed75..b333bc584 100644 --- a/code/sdk/zhihui/imsl/src/vad_server.c +++ b/code/sdk/zhihui/imsl/src/vad_server.c @@ -128,6 +128,8 @@ static void vad_mp3_pcm_callback(const int16_t *pcm_data, int samples, int chann #endif static uint8_t dmp_auth=0; +uint32_t dmp_auth_max_count=10; + /** * @brief 处理 PCM 数据,直接送给 VAD(简化版) * @note VAD 库支持可变长度输入(推荐 < 100ms),无需固定切分 @@ -141,8 +143,8 @@ static errcode_t vad_process_pcm_data(const int16_t *pcm_data, size_t sample_cou } pcm_call_count++; - - LOGIN_SUCC: + +LOGIN_SUCC: if(dmp_auth==0) { @@ -421,7 +423,7 @@ errcode_t vad_server_init(vad_status_callback_t callback) struct VadConfig vad_config = { .vad_log_level = 1, /* INFO 日志 */ .sil_threshold = 63, /* 静音阈值 */ - .energy_threshold = 80000, /* 能量阈值 */ + .energy_threshold = 50000, /* 能量阈值 */ .min_speech_length = 240, /* 最短语音长度 (ms) */ .silence_detect_length = 300, /* 静音检测长度 (ms) */ .rule1_min_tailing_silence = 480, /* 规则1静音时间 (ms) */ diff --git a/code/sdk/zhihui/leiniao/imsl_ln/leiniao_api_at.c b/code/sdk/zhihui/leiniao/imsl_ln/leiniao_api_at.c index 4fb245df0..da38acf41 100644 --- a/code/sdk/zhihui/leiniao/imsl_ln/leiniao_api_at.c +++ b/code/sdk/zhihui/leiniao/imsl_ln/leiniao_api_at.c @@ -256,7 +256,7 @@ static uint32_t ln_ws_start_at(uint8_t *para, uint32_t paraLen, char *strCommonC LN_UNUSED(paraLen); LN_UNUSED(strCommonCmd); printf("[LN][WS_START]\n"); - int ret = ln_ws_start(NULL,2000); + int ret = ln_ws_start(NULL,1500); if (ret != 0) { printf("[LN][WS_START] failed: %d\n", ret); diff --git a/code/sdk/zhihui/leiniao/imsl_ln/ln_api_websocket.c b/code/sdk/zhihui/leiniao/imsl_ln/ln_api_websocket.c index 4c305f392..b1261cd24 100644 --- a/code/sdk/zhihui/leiniao/imsl_ln/ln_api_websocket.c +++ b/code/sdk/zhihui/leiniao/imsl_ln/ln_api_websocket.c @@ -701,7 +701,7 @@ int ln_ws_start_audio_chat(ln_audio_chat_command_t command_type) LN_WS_WARN("Audio chat already active: %s\n", g_audio_chat_ctx.chat_id); return 0; } - ln_ws_send_voice_chat_interrupt("新对话开始"); + ln_ws_send_voice_chat_interrupt("新对话开始"); AI_ForceInterrupt(); @@ -782,17 +782,39 @@ static void AIVoiceDataCallback_ln(const uint8_t* data, uint32_t size, void* use // 更上层、更清晰、天然只处理录音数据 #if ENABLE_VAD_SERVER if (g_vad_server_enabled && g_vad_server_initialized && vad_server_is_running()) { + + #ifdef VAD_LOCAL_PM vad_audio_format_t format = VAD_AUDIO_FORMAT_MP3; // 默认 MP3 +errcode_t ret = vad_server_send_audio(format, data, size, false); + + #else + vad_audio_format_t format = VAD_AUDIO_FORMAT_PCM; // 默认 MP3 + if (data && size > 0) { - errcode_t ret = vad_server_send_audio(format, data, size, false); + + uint8_t *pcm_data=(uint8_t *) malloc(1024*3); + uint32_t len=1024*3; + + errcode_t ret= AI_SecondStreamInObtainBuffer(pcm_data,&len); + + if(ret==0 && len>0) + { + ret = vad_server_send_audio(format, pcm_data, len, false); +// AI_SendAudioDataToQueue(pcm_data, len); + if (ret != ERRCODE_SUCC) { /* 静默失败,避免过多日志 */ } else { // LN_WS_DEBUG("VAD: Sent %u bytes (format=%d, is_last=%d)\n", // size, format, false); } + } + free(pcm_data); + + } + #endif // 如果是结束标志,打印日志 if (false) { @@ -1141,6 +1163,12 @@ void ln_ws_audio_init(void) .channelCount = 1, .sampleRate = 16000 }; + + AIVoiceAudioParams audioParams_in_vad = { + .audioFormat = PCM, + .channelCount = 1, + .sampleRate = 16000 + }; AIVoiceAudioParams audioParams_out = { .audioFormat = MP3, @@ -1155,7 +1183,7 @@ void ln_ws_audio_init(void) #endif uapi_cat1_add_cat1_vote(); - old_Start_AIVoice(AIVoiceDataCallback_ln,NULL,&audioParams_in,&audioParams_out,false); + old_Start_AIVoice(AIVoiceDataCallback_ln,NULL,&audioParams_in,&audioParams_out,&audioParams_in_vad,false); // 设置初始化标志位 g_audio_initialized = true; @@ -1213,6 +1241,9 @@ void ln_ws_audio_deinit(void) #endif uapi_cat1_remove_cat1_vote(); + g_audio_playback_active = false; + g_audio_chat_ctx.finish_state = LN_CHAT_NOT_ENDED; + // 清除初始化标志位 g_audio_initialized = false; @@ -1562,7 +1593,7 @@ int ln_ws_send_voice_chat_interrupt(const char *reason) if (ret == 0) { LN_WS_DEBUG("VoiceChatInterrupt sent successfully\n"); - if (need_reconnect || g_audio_chat_ctx.finish_state == LN_CHAT_SENT_END_NO_REPLY) { + if (need_reconnect || (g_audio_chat_ctx.finish_state == LN_CHAT_SENT_END_NO_REPLY)) { // 如果检测到频繁数据接收,直接重连 LN_WS_INFO("Triggering reconnection due to frequent binary data reception\n"); if (ln_ws_reconnect() != 0) {