From: snickersun Date: Thu, 4 Jun 2026 06:53:06 +0000 (+0800) Subject: 优化vad(独立线程),opus编解码, 解决延迟等交互问题。 X-Git-Url: http://git.snicker.com.cn/css/static/gitweb.css?a=commitdiff_plain;h=07003373dd3af861e7821696635209427286ad71;p=mirrors%2FASR1603_G5.git 优化vad(独立线程),opus编解码, 解决延迟等交互问题。 --- diff --git a/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/default_config.py b/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/default_config.py index 86d70307..95d7012b 100644 --- a/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/default_config.py +++ b/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/default_config.py @@ -200,10 +200,19 @@ VC_IDLE_DEEP_SLEEP_ENABLE = 1 # 本地 imsl 断句静音(ms);映射 rule2/3;可被 storage vad_silence 覆盖 VAD_SILENCE_DEFAULT = 1000 +# 本地 VAD 判句末后追加采集的尾音(ms),到点再发 audio.completed;防句尾渐弱音被关闸截断。0=立即收尾 +LOCAL_VAD_ENDPOINT_HANGOVER_MS = 200 +# 本地 imsl VAD 能量阈值:越低越灵敏(易触发说话),越高越钝;imsl 推荐 10000。 +# 现场若本地 VAD 不出 speech start/end(全靠 asr_silence_fallback 收尾),调低(如 6000) +VAD_ENERGY_THRESHOLD = 10000 # 长对话:单轮最长录音(ms)、asr.completed 后续说 merge 窗(ms);0=不限制 +# 说明:MAX_TURN_MS 解决"场景②一直说话太久"——强制提交截断输入; +# NO_SPEECH_MS 解决"场景①开麦后一直没说话"——停录回 idle 省电,不提交。两者动作不同。 LONG_DIALOGUE_MAX_TURN_MS = 25000 -LONG_DIALOGUE_ASR_CONTINUE_MS = 2000 +LONG_DIALOGUE_ASR_CONTINUE_MS = 1000 +# 场景①:开麦后本轮始终无本地 VAD 说话起点且无服务端 asr.result,超此(ms)停录回 idle;0=不限制 +LONG_DIALOGUE_NO_SPEECH_MS = 12000 # llm 已起后连续 canceled 达此次数则冻结续说、停麦等回复(避免一直录无 TTS) LONG_DIALOGUE_LLM_CANCEL_MAX = 2 @@ -252,6 +261,17 @@ VC_OPUS_DOWNLINK_ENABLED = True # 下行:服务端 opus 包解码为 PCM # Opus 帧时长(ms):合法 2.5/5/10/20/40/60;16kHz 单声道。上行按此分帧(60ms=1920字节PCM)。 VC_OPUS_FRAME_MS = 60 +# 对话中途传输瞬断(ECONNRESET 等)自动重连: +# True=对话进行中瞬断时静默重连+重开麦续录,不立刻播报错音;30s 窗口内超 MAX 次(抖动/真断网)才回退报错音。 +# 服务端 Start 不带 dialog_id,重连=新会话,无法续接服务端上下文,仅端侧重开麦。 +VC_RESUME_RECONNECT_ENABLED = True +VC_RESUME_RECONNECT_MAX = 3 + +# 本地 VAD 线程化:True=VAD 检测移到独立 worker 线程(配合 modVad GIL 释放可与录音/编码/发送并行), +# 录音线程读 PCM 后立刻走上行、不再被 VAD(~50ms)阻塞;VAD 结果滞后约 1 帧供端侧断句判定。 +# 默认关:先验证重连+GIL 这版稳定,再单独打开验证 VAD 线程化(一次只引入一个并发变量)。 +VC_VAD_THREAD_ENABLED = True + # ============================================================================= # 9. 本地指令与固定话术 # ============================================================================= diff --git a/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/products/configs/k8v3/default_config.py b/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/products/configs/k8v3/default_config.py index 86d70307..95d7012b 100644 --- a/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/products/configs/k8v3/default_config.py +++ b/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/products/configs/k8v3/default_config.py @@ -200,10 +200,19 @@ VC_IDLE_DEEP_SLEEP_ENABLE = 1 # 本地 imsl 断句静音(ms);映射 rule2/3;可被 storage vad_silence 覆盖 VAD_SILENCE_DEFAULT = 1000 +# 本地 VAD 判句末后追加采集的尾音(ms),到点再发 audio.completed;防句尾渐弱音被关闸截断。0=立即收尾 +LOCAL_VAD_ENDPOINT_HANGOVER_MS = 200 +# 本地 imsl VAD 能量阈值:越低越灵敏(易触发说话),越高越钝;imsl 推荐 10000。 +# 现场若本地 VAD 不出 speech start/end(全靠 asr_silence_fallback 收尾),调低(如 6000) +VAD_ENERGY_THRESHOLD = 10000 # 长对话:单轮最长录音(ms)、asr.completed 后续说 merge 窗(ms);0=不限制 +# 说明:MAX_TURN_MS 解决"场景②一直说话太久"——强制提交截断输入; +# NO_SPEECH_MS 解决"场景①开麦后一直没说话"——停录回 idle 省电,不提交。两者动作不同。 LONG_DIALOGUE_MAX_TURN_MS = 25000 -LONG_DIALOGUE_ASR_CONTINUE_MS = 2000 +LONG_DIALOGUE_ASR_CONTINUE_MS = 1000 +# 场景①:开麦后本轮始终无本地 VAD 说话起点且无服务端 asr.result,超此(ms)停录回 idle;0=不限制 +LONG_DIALOGUE_NO_SPEECH_MS = 12000 # llm 已起后连续 canceled 达此次数则冻结续说、停麦等回复(避免一直录无 TTS) LONG_DIALOGUE_LLM_CANCEL_MAX = 2 @@ -252,6 +261,17 @@ VC_OPUS_DOWNLINK_ENABLED = True # 下行:服务端 opus 包解码为 PCM # Opus 帧时长(ms):合法 2.5/5/10/20/40/60;16kHz 单声道。上行按此分帧(60ms=1920字节PCM)。 VC_OPUS_FRAME_MS = 60 +# 对话中途传输瞬断(ECONNRESET 等)自动重连: +# True=对话进行中瞬断时静默重连+重开麦续录,不立刻播报错音;30s 窗口内超 MAX 次(抖动/真断网)才回退报错音。 +# 服务端 Start 不带 dialog_id,重连=新会话,无法续接服务端上下文,仅端侧重开麦。 +VC_RESUME_RECONNECT_ENABLED = True +VC_RESUME_RECONNECT_MAX = 3 + +# 本地 VAD 线程化:True=VAD 检测移到独立 worker 线程(配合 modVad GIL 释放可与录音/编码/发送并行), +# 录音线程读 PCM 后立刻走上行、不再被 VAD(~50ms)阻塞;VAD 结果滞后约 1 帧供端侧断句判定。 +# 默认关:先验证重连+GIL 这版稳定,再单独打开验证 VAD 线程化(一次只引入一个并发变量)。 +VC_VAD_THREAD_ENABLED = True + # ============================================================================= # 9. 本地指令与固定话术 # ============================================================================= diff --git a/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/source/modVad.c b/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/source/modVad.c index 7c21009a..47303095 100644 --- a/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/source/modVad.c +++ b/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/source/modVad.c @@ -114,6 +114,7 @@ static mp_obj_t vad_run_pcm(mp_obj_t data, mp_obj_t data_len_obj) int byte_len; int sample_len; const int16_t *samples; + int vad_st; vad_ensure_init(NULL); mp_get_buffer_raise(data, &bufinfo, MP_BUFFER_READ); @@ -129,8 +130,12 @@ static mp_obj_t vad_run_pcm(mp_obj_t data, mp_obj_t data_len_obj) return mp_obj_new_int(0); } samples = (const int16_t *)bufinfo.buf; + /* 纯 C 计算期间释放 GIL,避免阻塞录音/收发等其它线程(samples 指向调用方持有的不可变 bytes,GC 安全) */ + MP_THREAD_GIL_EXIT(); run_vad(samples, sample_len); - return mp_obj_new_int(vad_detect()); + vad_st = vad_detect(); + MP_THREAD_GIL_ENTER(); + return mp_obj_new_int(vad_st); } STATIC mp_obj_t mp_vad_process(mp_obj_t self_in, mp_obj_t data, mp_obj_t data_len) @@ -221,7 +226,9 @@ STATIC mp_obj_t mp_vad_free(size_t n_args, const mp_obj_t *args) } return mp_const_none; } -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_vad_free_obj, 0, 0, mp_vad_free); +/* 作为 locals_dict 方法调用 vad.free() 时,self 会作为第 1 个位置参数传入, + * 故需允许 1 个参数(否则报 "takes 0 positional arguments but 1 were given")。 */ +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_vad_free_obj, 0, 1, mp_vad_free); STATIC const mp_rom_map_elem_t VAD_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_process), MP_ROM_PTR(&mp_vad_process_obj) }, diff --git a/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/source/modopus.c b/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/source/modopus.c index 8dae3b3b..121812d6 100644 --- a/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/source/modopus.c +++ b/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/source/modopus.c @@ -97,8 +97,12 @@ STATIC mp_obj_t opus_codec_encode(mp_obj_t self_in, mp_obj_t pcm_in) } /* 帧长可变:2.5/5/10/20/40/60ms@16k(=40/80/160/320/640/960 样本),非法长度 opus_encode 返回 <0 */ uint8_t out[OPUS_CODEC_MAX_PKT]; - mp_int_t n = opus_encode(self->enc, (const opus_int16 *)pcm.buf, (int)(pcm.len / 2), - out, OPUS_CODEC_MAX_PKT); + mp_int_t n; + /* 纯 C 编码期间释放 GIL,避免阻塞 VAD/收发线程(pcm.buf 指向调用方持有的不可变 bytes,out 为 C 栈,GC 安全) */ + MP_THREAD_GIL_EXIT(); + n = opus_encode(self->enc, (const opus_int16 *)pcm.buf, (int)(pcm.len / 2), + out, OPUS_CODEC_MAX_PKT); + MP_THREAD_GIL_ENTER(); if (n < 0) { mp_raise_msg_varg(&mp_type_OSError, MP_ERROR_TEXT("opus_encode %d"), (int)n); } @@ -119,8 +123,12 @@ STATIC mp_obj_t opus_codec_decode(mp_obj_t self_in, mp_obj_t pkt_in) } /* 帧长可变:按包内信息解码,输出实际样本数(可为 60ms=960 样本) */ opus_int16 pcm[OPUS_CODEC_MAX_FRAME]; - mp_int_t n = opus_decode(self->dec, pkt.buf, (opus_int32)pkt.len, pcm, - OPUS_CODEC_MAX_FRAME, 0); + mp_int_t n; + /* 纯 C 解码期间释放 GIL,避免阻塞录音/VAD 线程(pkt.buf 指向调用方持有的不可变 bytes,pcm 为 C 栈,GC 安全) */ + MP_THREAD_GIL_EXIT(); + n = opus_decode(self->dec, pkt.buf, (opus_int32)pkt.len, pcm, + OPUS_CODEC_MAX_FRAME, 0); + MP_THREAD_GIL_ENTER(); if (n < 0) { mp_raise_msg_varg(&mp_type_OSError, MP_ERROR_TEXT("opus_decode %d"), (int)n); } diff --git a/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/vc_session.py b/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/vc_session.py index 943fa33c..763d611e 100644 --- a/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/vc_session.py +++ b/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/vc_session.py @@ -363,6 +363,94 @@ class AI_VC(VCSessionIoMixin, VCSessionMediaMixin, Abstract): except Exception: pass + def _vc_local_vad_asr_fallback_tick(self): + """local_vad endpoint 模式下 fallback:若本地 VAD 未检测到语音终点,但服务端 ASR + 已有结果且超过 merge 窗仍无新结果,视为句末,主动发 audio.completed。 + 防止 VAD 灵敏度不足时录音无限持续直到 turn_cap。""" + try: + if not self._vc_local_vad_endpoint(): + return + if getattr(self, "VC_record_state_flag", 0) < 2: + return + if getattr(self, "record_data_save_flag", 0) != 0: + return + if getattr(self, "_server_vad_silence", 0) == 1: + return + if getattr(self, "_ptt_audio_created", 0) != 1: + return + if getattr(self, "_turn_has_llm_created", 0) == 1: + return + last_ms = getattr(self, "_vc_last_asr_result_ms", 0) + if not last_ms: + return + elapsed = utime.ticks_diff(utime.ticks_ms(), last_ms) + # 比 merge 窗多留 500ms,避免 ASR 网络延迟误判 + threshold = self._vc_asr_continue_merge_ms() + 500 + if elapsed < threshold: + return + print("[VAD][local] asr_silence_fallback elapsed={}ms -> audio.completed".format(elapsed)) + try: + if hasattr(self, "_vc_opus_uplink_flush"): + self._vc_opus_uplink_flush() + except Exception: + pass + self._vc_local_vad_on_endpoint() + except Exception: + pass + + def _vc_build_vad(self): + """构造 imsl VAD 对象并记录授权模式(self._vad_init_mode)。 + 失败返回 None。init_vad 内部完成联网授权:有网授权成功=2,否则回退试用=1。""" + try: + sil_ms = int(self.vad_silence) + # energy_threshold 越低越灵敏(易触发 speech),越高越钝;imsl 推荐 10000。 + # 现场本地 VAD 不出 speech start/end 多为此值相对麦增益偏高,可调低试。 + energy = cfg_int("VAD_ENERGY_THRESHOLD", 10000) + vad = VAD.VAD( + self.VC_record_samplerate, 1, self.VAD_data_len, + energy, 240, sil_ms, 480) + mode = vad.init_mode() + self._vad_init_mode = mode + print("[VAD] init_mode=", mode, "trial=", VAD.MODE_TRIAL, + "sil_ms=", sil_ms, "energy=", energy, + "debug_only=", self._local_vad_debug_only, + "endpoint=", not self._local_vad_debug_only) + return vad + except Exception as e: + print("[VAD] init fail:", e) + return None + + def _vc_vad_try_activate(self, reason=""): + """开机网络就绪后联网激活 VAD:若当前为试用模式且有网络,free 后重建以触发联网授权。 + 一旦拿到授权(mode=2)即置 _vad_activate_done,后续不再重试。无网/失败保留试用,下次再试。""" + try: + if getattr(self, "is_VAD", 0) != 1 or self.vad is None: + return + if getattr(self, "_vad_activate_done", 0) == 1: + return + if getattr(self, "_vad_init_mode", 0) == 2: + self._vad_activate_done = 1 + return + # 网络可用性判断:CSQ 有效即认为可联网授权 + try: + csq = net.csqQueryPoll() + except Exception: + csq = -1 + if csq is None or csq == -1 or csq == 99: + return # 无网,下次录音再试 + # 有网才执行激活;不论结果只做一次,避免每轮录音重复 free+重建 + self._vad_activate_done = 1 + self.vad.free() + new_vad = self._vc_build_vad() + if new_vad is not None: + self.vad = new_vad + if self._vad_init_mode == 2: + print("[VAD] activate ok (licensed) reason=", reason) + else: + print("[VAD] activate still trial reason=", reason, "csq=", csq) + except Exception as e: + print("[VAD] activate fail:", e) + def _long_dialogue_cancel_preserve_speech(self): """长对话 canceled:非用户打断时,录音中或 asr 续说窗内均保持/恢复上行。""" if not self._vc_is_long_dialogue_mode(): @@ -498,6 +586,51 @@ class AI_VC(VCSessionIoMixin, VCSessionMediaMixin, Abstract): except Exception: pass + def _vc_no_speech_open_ms(self): + """场景①:开麦后一直无有效语音的停录超时(ms);0=不限制。""" + return cfg_int("LONG_DIALOGUE_NO_SPEECH_MS", 12000) + + def _vc_no_speech_idle_tick(self): + """场景①:长对话开麦后,本轮始终无本地 VAD 说话起点且无 asr.result, + 判定无人接话→停录回 idle 省电(会话保持,不 force_commit、不喂 LLM)。 + 与 turn_cap(场景②说太久强制提交) 区分:一旦检测到语音即不再触发本逻辑。""" + try: + if not self._vc_is_long_dialogue_mode(): + return + if getattr(self, "_vc_no_speech_idle", 0) == 1: + return # 本轮已触发,等待异步停录完成 + if getattr(self, "record_data_save_flag", 0) != 0: + return + if getattr(self, "VC_record_state_flag", 0) < 2: + return + # 本轮有任何语音迹象 → 交给正常断句/turn_cap,不属"无人说话" + if getattr(self, "_vc_turn_had_speech", 0) == 1: + return + if getattr(self, "_vc_last_asr_result_ms", 0): + return + if getattr(self, "_turn_has_llm_created", 0) == 1: + return + if getattr(self, "_vc_local_vad_last_state", 0) == 1: + return + max_ms = self._vc_no_speech_open_ms() + if max_ms <= 0: + return + start = getattr(self, "_vc_turn_wall_start_ms", 0) + if not start: + return + if utime.ticks_diff(utime.ticks_ms(), start) < max_ms: + return + print("[VC][no_speech] stop->idle after {}ms (no speech/asr)".format(max_ms)) + self._vc_no_speech_idle = 1 # 让 VC_Speech_End èµ° idle 灯态,且防本轮重复触发 + self._server_vad_silence = 1 + try: + self._uplink_mark_end("no_speech") + except Exception: + pass + self.request_uplink_stop("no_speech") # 异步停录;不 force_commit、不提交静音 + except Exception: + pass + def _long_dialogue_media_busy(self): """长对话:TTS/思考/播报中(按键须走打断,勿误走 continue)。""" try: @@ -2553,20 +2686,12 @@ class AI_VC(VCSessionIoMixin, VCSessionMediaMixin, Abstract): # ARG_silence_start_threshold, // 静音开始阈值(ms) → imsl rule1 self.vad = None self._vc_vad_pcm_tail = bytearray() + self._vad_init_mode = 0 # 0失败 1试用 2授权 + self._vad_activate_done = 0 # 已成功联网授权(mode=2)后置1,不再重试 if self.is_VAD == 1: - try: - sil_ms = int(self.vad_silence) - self.vad = VAD.VAD( - self.VC_record_samplerate, 1, self.VAD_data_len, - 10000, 240, sil_ms, 480) - mode = self.vad.init_mode() - print("[VAD] init_mode=", mode, "trial=", VAD.MODE_TRIAL, - "sil_ms=", sil_ms, "debug_only=", self._local_vad_debug_only, - "endpoint=", not self._local_vad_debug_only) - except Exception as e: - print("[VAD] init fail:", e) + self.vad = self._vc_build_vad() + if self.vad is None: self.is_VAD = 0 - self.vad = None else: print("[VAD] local vad disabled VC_mode=", getattr(self, "VC_mode", -1)) diff --git a/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/vc_session_io.py b/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/vc_session_io.py index ed72f6eb..b27ca991 100644 --- a/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/vc_session_io.py +++ b/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/vc_session_io.py @@ -2,7 +2,7 @@ """AI_VC mixin:VC WebSocket 建链、收发线程、协议 JSON 分发。""" from vc_session_imports import * # noqa: F401,F403 from vc_notify import NotifyScheduler -from vc_utils import cfg_bool, cfg_attr +from vc_utils import cfg_bool, cfg_attr, cfg_int # import * 不导入以下划线开头的符号;_thread 必须显式引入(否则 start_new_thread NameError) import _thread @@ -275,6 +275,18 @@ class VCSessionIoMixin: self.websocket_deconnect_VC() except Exception: pass + def _vc_ws_close_is_clean(self): + """判断 WS 关闭是否为服务端正常收尾(tts.done 已在 10s 内收到)。 + 是则抑制"VC连接断开了"播报,避免打扰正常对话流程。""" + wall_ms = getattr(self, "_tts_completed_wall_ms", 0) + if not wall_ms: + return False + try: + elapsed = utime.ticks_diff(utime.ticks_ms(), wall_ms) + return 0 <= elapsed <= 10000 + except Exception: + return False + def websocket_error(self, TTS_str=None, play_weak_signal_tone=False): """play_weak_signal_tone=True 时播 xinhaoruo1(弱网/无卡类);连接/协议失败请传 False,避免误报「信号弱」。""" if getattr(self, "VC_mode", 0) != 1: @@ -327,6 +339,46 @@ class VCSessionIoMixin: print("websocket_error4") + def _vc_on_transport_error(self, tts_str, reason="transport"): + """对话中途传输瞬断(ECONNRESET 等)入口: + - 对话进行中(_vc_session_started==1)且未在关闭:静默重连+重开麦(复用 websocket_error(None) + 的 Close→AI_VC_start 重连链,count<3 自带风暴保护),不立刻播报错音; + - 30s 窗口内瞬断超 VC_RESUME_RECONNECT_MAX 次(抖动/真断网) 或非对话中/已禁用: + 回退原 websocket_error(tts_str) 播报错音并按原逻辑处理。 + 服务端 Start 不带 dialog_id,重连=新会话,无法续接服务端上下文,仅端侧重开麦续录。""" + try: + # 去抖:断线瞬间发送队列里多帧会连续触发,6s 内只处理一次, + # 避免后续帧在首次已 Close(session=0) 后误走播报错音分支。 + now = utime.ticks_ms() + last = getattr(self, "_vc_transport_err_last_ms", 0) + if last and utime.ticks_diff(now, last) < cfg_int("VC_RESUME_DEBOUNCE_MS", 6000): + print("[VC][resume] debounce skip dup transport error") + return + self._vc_transport_err_last_ms = now + in_session = getattr(self, "_vc_session_started", 0) == 1 + closing = getattr(self, "Close_flag", 0) == 1 + if (not cfg_bool("VC_RESUME_RECONNECT_ENABLED", True)) or (not in_session) or closing: + self.websocket_error(tts_str) + return + win = getattr(self, "_vc_resume_win_ms", 0) + if (not win) or utime.ticks_diff(now, win) > 30000: + self._vc_resume_win_ms = now + self._vc_resume_cnt = 0 + self._vc_resume_cnt = getattr(self, "_vc_resume_cnt", 0) + 1 + if self._vc_resume_cnt > cfg_int("VC_RESUME_RECONNECT_MAX", 3): + print("[VC][resume] storm guard cnt={} reason={} -> spoken error".format( + self._vc_resume_cnt, reason)) + self.websocket_error(tts_str) + return + print("[VC][resume] silent reconnect reason={} cnt={} was_rec={}".format( + reason, self._vc_resume_cnt, getattr(self, "VC_record_state_flag", 0))) + self.websocket_error(None) + except Exception as e: + usys.print_exception(e) + try: + self.websocket_error(tts_str) + except Exception: + pass def websocket_deconnect_VC(self): #print("websocket_deconnect_VC {}!!!!!!!".format(self.AI_client_VC_isconnect)) print("websocket_deconnect_VC!!!!!!!") @@ -442,7 +494,9 @@ class VCSessionIoMixin: frame = bytes(buf[off:off + fb]) off += fb try: + _enc_t0 = utime.ticks_us() pkt = codec.encode(frame) + _enc_us = utime.ticks_diff(utime.ticks_us(), _enc_t0) except Exception as e: print("[VC][opus_up] encode fail in={}".format(fb)) usys.print_exception(e) @@ -451,8 +505,13 @@ class VCSessionIoMixin: if l <= 0: continue self._opus_up_cnt = getattr(self, "_opus_up_cnt", 0) + 1 + self._opus_up_us_sum = getattr(self, "_opus_up_us_sum", 0) + _enc_us + if _enc_us > getattr(self, "_opus_up_us_max", 0): + self._opus_up_us_max = _enc_us if self._opus_up_cnt <= 5 or (self._opus_up_cnt % 50) == 0: - print("[VC][opus_up] #{} in={} out={}".format(self._opus_up_cnt, fb, l)) + _avg = self._opus_up_us_sum // self._opus_up_cnt + print("[VC][opus_up] #{} in={} out={} enc={}us avg={}us max={}us".format( + self._opus_up_cnt, fb, l, _enc_us, _avg, self._opus_up_us_max)) if self.AI_client_VC_isconnect != 1 or self.AI_client_VC is None: self._uplink_note_tx_fail(l, "opus_enqueue_ws_down") self._opus_up_buf = bytearray(buf[off:]) @@ -462,6 +521,34 @@ class VCSessionIoMixin: self.AI_VC_send_data_queue.put([AI_client_VC_num, pkt]) self._opus_up_buf = bytearray(buf[off:]) if off < total else bytearray() + def _vc_opus_uplink_flush(self): + """VAD endpoint 时将 Opus 上行缓冲中不足一帧的剩余 PCM 补零到完整帧后编码发出, + 避免句末语音被截断。必须在 silence_gate(_server_vad_silence=1) 前调用。""" + if not self._vc_opus_uplink_on(): + return + buf = getattr(self, "_opus_up_buf", None) + if not buf or len(buf) == 0: + return + fb = self._vc_opus_frame_bytes() + remaining = len(buf) % fb + if remaining == 0: + return # 恰好整帧,正常流程会处理 + pad_len = fb - remaining + try: + codec = self._vc_opus_up_codec() + padded = bytes(buf[-remaining:]) + bytes(pad_len) + pkt = codec.encode(padded) + if pkt and len(pkt) > 0: + if self.AI_client_VC_isconnect == 1 and self.AI_client_VC is not None: + self._uplink_mark_start(len(pkt)) + self._uplink_bytes += len(pkt) + self.AI_VC_send_data_queue.put([AI_client_VC_num, pkt]) + print("[VC][opus_up] flush in={} pad={} out={}".format( + remaining, pad_len, len(pkt))) + except Exception as e: + usys.print_exception(e) + self._opus_up_buf = bytearray(buf[: len(buf) - remaining]) + def AI_client_VC_send(self,data_str): # 上行 opus:PCM(bytes) 累积分帧编码后入队;开关关闭时完全走原 PCM 逻辑 if not isinstance(data_str, str) and self._vc_opus_uplink_on(): @@ -616,7 +703,7 @@ class VCSessionIoMixin: try: if self.AI_client_VC_isconnect != 0: print("VC发送数据错误,退出") - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"VC发送出错了,请重试"}) + EventMesh.publish("publish_AI_btnInfo",{"func":self._vc_on_transport_error,"data":"VC发送出错了,请重试"}) self.ud_error(["VC",self.VC_taskid]) except Exception as e: #网络连接异常 usys.print_exception(e) @@ -667,13 +754,13 @@ class VCSessionIoMixin: self.close_timerout = 0 if recv_data == "websocket close": # 服务器关闭连接 - print("VC 服务器关闭连接!!!!!!!!!!!!!!!!!!!!!!!!!!!") - #self.AI_client_VC_isconnect = 0 - #self.websocket_deconnect_VC() - #EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_deconnect_asr}) - # #EventMesh.publish("publish_AI_btnInfo",{"func":utime.sleep_ms,'data':(1500)}) - #EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_connect_asr}) - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"VC连接断开了,请重试"}) + _ws_close_clean = self._vc_ws_close_is_clean() + print("[VC] ws server close clean={}".format(_ws_close_clean)) + _ws_err_str = None if _ws_close_clean else "VC连接断开了,请重试" + if _ws_err_str is None: + EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":None}) + else: + EventMesh.publish("publish_AI_btnInfo",{"func":self._vc_on_transport_error,"data":_ws_err_str}) self.ud_error(["VC",self.VC_taskid]) break elif recv_data == "": # EOF:链路断开/对端关闭 @@ -681,10 +768,15 @@ class VCSessionIoMixin: sid = getattr(self, "_cur_sid", "") or "-" except Exception: sid = "-" - print("[VC] ws EOF (link down) VC_mode={} sid={}".format( - getattr(self, "VC_mode", -1), sid + _ws_eof_clean = self._vc_ws_close_is_clean() + print("[VC] ws EOF (link down) VC_mode={} sid={} clean={}".format( + getattr(self, "VC_mode", -1), sid, _ws_eof_clean )) - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"VC连接断开了,请重试"}) + _ws_err_str = None if _ws_eof_clean else "VC连接断开了,请重试" + if _ws_err_str is None: + EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":None}) + else: + EventMesh.publish("publish_AI_btnInfo",{"func":self._vc_on_transport_error,"data":_ws_err_str}) self.ud_error(["VC",self.VC_taskid]) break try: @@ -693,21 +785,30 @@ class VCSessionIoMixin: _tl = recv_data.strip().lower() if _tl in ("ping", "pong"): continue + # llm.delta 流式包:端侧不展示文本,VAD 模式下处理为空操作。 + # 服务端会在 LLM 生成期间狂发 delta,若每条都 ujson.loads+dispatch, + # recv 线程会被拖住 2~3s,导致其后的 tts.started 晚到、播放迟迟不开始。 + # 这里在解析前用子串快速识别并跳过(idle 计时已在解析前刷新,安全)。 + if self.is_VAD != 0 and ('"llm.delta"' in recv_data): + continue jsload_data = ujson.loads(recv_data) try: jsload_data = vc_proto_normalize_server(jsload_data) except Exception: pass + try: + _rev = jsload_data.get("event", "") + except Exception: + _rev = "" if VC_WS_IO_LOG: - print("[VC] recv raw:{}".format(recv_data)) + # llm.delta 流式包数量多,逐条 print 完整 raw 会拖慢 recv 线程(~124ms/条), + # 导致 tts.started 被堵在 delta 队列后晚到数秒;这里跳过 delta 的 raw 打印 + if _rev != "llm.delta": + print("[VC] recv raw:{}".format(recv_data)) else: - try: - _rev = jsload_data.get("event", "") - if _rev != "llm.delta": - print("[VC] rx event={} sid={}".format( - _rev, jsload_data.get("sid", "") or "-")) - except Exception: - pass + if _rev != "llm.delta": + print("[VC] rx event={} sid={}".format( + _rev, jsload_data.get("sid", "") or "-")) #print("AI_client_VC_recv jsload_data = {}".format(jsload_data)) try: self._cur_sid = jsload_data.get("sid", "") or "" @@ -893,6 +994,8 @@ class VCSessionIoMixin: try: self.request_uplink_stop("asr.completed/config_cmd") + # synthesis 在新会话 server.started 后播出;需跳过首次自动开麦,等 synthesis TTS 播完再开 + self._skip_started_uplink_once = 1 EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_deconnect_VC}) EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_VC_start}) except Exception as e: @@ -1188,23 +1291,45 @@ class VCSessionIoMixin: print("新的AI对话不允许以前的播放数据:") break + # 打断/丢弃判断提前:在 Opus 解码之前决定是否丢帧 + # 丢弃时直接 continue,跳过解码+sleep,让 recv 线程快速排空 WS 缓冲 + _drop_pcm = False + try: + if ( + hasattr(self, "_long_dialogue_should_drop_inbound_tts") + and self._vc_is_long_dialogue_mode() + ): + _drop_pcm = self._long_dialogue_should_drop_inbound_tts() + elif getattr(self, "is_ruppt_tts", 0) == 1: + _drop_pcm = True + except Exception: + _drop_pcm = getattr(self, "is_ruppt_tts", 0) == 1 + if _drop_pcm: + # 跳过昂贵的 opus 解码+文件写+入播放队列,快速丢弃被打断后的残留帧; + # 但必须保留 sleep 让出 CPU,否则连续 continue 会饿死上行发送线程导致 TX_FAIL + utime.sleep_ms(1) + continue + # 下行 opus:每个二进制帧=一个 opus 包,解码为 PCM 再走原播放/保存路径 if not isinstance(recv_data, str) and self._vc_opus_downlink_on(): try: _ilen = len(recv_data) + _dec_t0 = utime.ticks_us() recv_data = self._vc_opus_down_codec().decode(recv_data) + _dec_us = utime.ticks_diff(utime.ticks_us(), _dec_t0) self._opus_dn_cnt = getattr(self, "_opus_dn_cnt", 0) + 1 + self._opus_dn_us_sum = getattr(self, "_opus_dn_us_sum", 0) + _dec_us + if _dec_us > getattr(self, "_opus_dn_us_max", 0): + self._opus_dn_us_max = _dec_us if self._opus_dn_cnt <= 5 or (self._opus_dn_cnt % 50) == 0: - print("[VC][opus_dn] #{} in={} out={}".format( - self._opus_dn_cnt, _ilen, len(recv_data))) + _avg = self._opus_dn_us_sum // self._opus_dn_cnt + print("[VC][opus_dn] #{} in={} out={} dec={}us avg={}us max={}us".format( + self._opus_dn_cnt, _ilen, len(recv_data), _dec_us, _avg, self._opus_dn_us_max)) except Exception as e: print("[VC][opus_dn] decode fail in={}".format(_ilen)) usys.print_exception(e) continue - # if VC_AUDIO_FRAME_LOG: - # print("下发音频数据1:{}".format(len(recv_data))) - if self.play_data_save_flag == 1: if self.play_fd ==None: try: @@ -1214,39 +1339,20 @@ class VCSessionIoMixin: self.play_fd = open('/usr/play_data.pcm','w') if self.play_fd !=None: self.play_fd.write(recv_data) - # print("下发音频数据2:{}".format(len(recv_data))) - #if self.VC_state >= VC_State_TTSStart: - #self.put_record_data([recv_data,""]) - _drop_pcm = False - try: - if ( - hasattr(self, "_long_dialogue_should_drop_inbound_tts") - and self._vc_is_long_dialogue_mode() - ): - _drop_pcm = self._long_dialogue_should_drop_inbound_tts() - elif getattr(self, "is_ruppt_tts", 0) == 1: - _drop_pcm = True - except Exception: - _drop_pcm = getattr(self, "is_ruppt_tts", 0) == 1 - if not _drop_pcm: - if self.VC_play_TYPE == VC_play_TYPE_PCM: - if self.play_pcm != None: - try: - #print("play_pcm.stream_write !!!") + if self.VC_play_TYPE == VC_play_TYPE_PCM: + if self.play_pcm != None: + try: + buf_len = self.play_pcm.stream_get_buf_len() + count_tmp = 1000 + while buf_len < len(recv_data): + utime.sleep_ms(1) + count_tmp-=1 + if count_tmp<=0 or self.play_pcm == None: + break buf_len = self.play_pcm.stream_get_buf_len() - count_tmp = 1000 - while buf_len < len(recv_data): - utime.sleep_ms(1) - count_tmp-=1 - if count_tmp<=0 or self.play_pcm == None: - break - buf_len = self.play_pcm.stream_get_buf_len() - #print("buf已满,等待释放:buf_len:{},count_tmp:{}".format(buf_len,count_tmp)) - except Exception as e: - usys.print_exception(e) - EventMesh.publish("publish_AI_btnInfo",{"func":self.put_record_data,"data":[recv_data,""]}) - else: - print("被打断不播放") + except Exception as e: + usys.print_exception(e) + EventMesh.publish("publish_AI_btnInfo",{"func":self.put_record_data,"data":[recv_data,""]}) utime.sleep_ms(1) except Exception as e: usys.print_exception(e) @@ -1263,7 +1369,7 @@ class VCSessionIoMixin: usys.print_exception(e) #self.websocket_error() print("VC VC :读数据报错,退出") - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"VC出错了,请重试"}) + EventMesh.publish("publish_AI_btnInfo",{"func":self._vc_on_transport_error,"data":"VC出错了,请重试"}) self.ud_error(["VC",self.VC_taskid]) break self.AI_client_VC_isconnect = 0 diff --git a/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/vc_session_media.py b/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/vc_session_media.py index 0a9a2b6b..ba360c38 100644 --- a/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/vc_session_media.py +++ b/2.sdk/project/7PIN_1_5/sdk/services/microPython/ports/quectel/core/vc_session_media.py @@ -486,7 +486,11 @@ class VCSessionMediaMixin: return self._vc_vad_ready() and not self._vc_local_vad_debug_only() def _vc_vad_feed_record_pcm(self, record_buf, record_len): - """录音 PCM 按 VAD_data_len 切块送 imsl process,返回 0/1/2(2=句末)。""" + """录音 PCM 按 VAD_data_len 切块实时逐帧送 imsl process,返回 0/1/2(2=句末)。 + 每帧独立判断: + - st==2(endpoint):立即 break,不再处理剩余帧,保证及时响应; + - st==1(speech):峰值保留,不因后续静音帧归零; + - st==0(silence):不重置已有 speech 状态,由 imsl 内部静音累积决定 endpoint。""" if not self._vc_vad_ready() or record_len <= 0: return 0 frame_len = int(getattr(self, "VAD_data_len", 640)) @@ -512,10 +516,10 @@ class VCSessionMediaMixin: except Exception: pass vad_state = 2 + break # endpoint:立即退出,剩余帧留在 tail 等下次(录音停后会被丢弃) elif st == 1: vad_state = 1 - elif vad_state != 2: - vad_state = 0 + # st==0 时不重置 vad_state(峰值保留) return vad_state def _vc_local_vad_send_audio_create(self): @@ -556,6 +560,12 @@ class VCSessionMediaMixin: self._vc_asr_completed_ms = utime.ticks_ms() except Exception: pass + # silence_gate 前先把 Opus 缓冲中不足一帧的剩余 PCM 补零发出,避免句末截断 + try: + if hasattr(self, "_vc_opus_uplink_flush"): + self._vc_opus_uplink_flush() + except Exception: + pass if self._vc_long_dialogue_defer_vad_stop(): self._vc_defer_vad_uplink_stop = 1 self._server_vad_silence = 1 @@ -568,6 +578,51 @@ class VCSessionMediaMixin: self._uplink_mark_end("local.vad.end") self._vc_local_vad_send_audio_completed(defer_stop=False) + def _vc_endpoint_hangover_ms(self): + """本地 VAD 句末后追加采集的尾音时长(ms);0=不延迟立即收尾。""" + return cfg_int("LOCAL_VAD_ENDPOINT_HANGOVER_MS", 200) + + def _vc_local_vad_arm_endpoint_hangover(self): + """本地 VAD 判句末:先进入尾音 hangover(不关上行,继续录发 N ms), + 到点再真正收尾(audio.completed),避免句尾渐弱音被立即关闸截断。 + hangover 期间用户又开口则取消收尾;再次判句末则立即收尾。""" + ms = self._vc_endpoint_hangover_ms() + if ms <= 0 or not self._vc_is_long_dialogue_mode(): + self._vc_local_vad_on_endpoint() + return + if getattr(self, "_vc_endpoint_hangover_until", 0): + # 已在 hangover 中又判句末 → 直接收尾,不再延长 + self._vc_endpoint_hangover_until = 0 + self._vc_local_vad_on_endpoint() + return + self._vc_endpoint_hangover_until = utime.ticks_ms() + ms + print("[VAD][local] endpoint -> hangover {}ms".format(ms)) + + def _vc_endpoint_hangover_tick(self): + """录音线程周期检查:hangover 到点收尾;期间用户又开口则取消。""" + try: + until = getattr(self, "_vc_endpoint_hangover_until", 0) + if not until: + return + if getattr(self, "VC_record_state_flag", 0) < 2: + self._vc_endpoint_hangover_until = 0 + return + if getattr(self, "_server_vad_silence", 0) == 1: + # 已被其它路径收尾(如 fallback),不重复发 audio.completed + self._vc_endpoint_hangover_until = 0 + return + if getattr(self, "_vc_local_vad_last_state", 0) == 1: + # 尾音窗内用户又开口 → 取消收尾,继续录这一句 + self._vc_endpoint_hangover_until = 0 + print("[VAD][local] hangover canceled (speech resumed)") + return + if utime.ticks_diff(utime.ticks_ms(), until) < 0: + return + self._vc_endpoint_hangover_until = 0 + self._vc_local_vad_on_endpoint() + except Exception: + pass + def _vc_local_vad_resume_uplink_merge(self, reason="local.pause"): """句中短停后续说:开 silence_gate + 新 audio.create,同轮 turn 墙钟不重置。""" if not self._vc_local_vad_endpoint(): @@ -588,8 +643,9 @@ class VCSessionMediaMixin: if vad_state == 1: prev = int(getattr(self, "_vc_local_vad_last_state", 0)) if prev != 1: - print("[VAD][local] speech start") + print(">>>>>>>>>> [VAD][local] SPEECH START (人声起点) <<<<<<<<<<") self._vc_local_vad_last_state = 1 + self._vc_turn_had_speech = 1 if self._vc_local_vad_endpoint(): if self._vc_asr_continue_merge_active(): self._vc_local_vad_resume_uplink_merge("asr.completed") @@ -604,17 +660,25 @@ class VCSessionMediaMixin: return 1 if vad_state == 2: frames = int(getattr(self, "VAD_count", 0)) - print("[VAD][local] speech end frames={}".format(frames)) + print(">>>>>>>>>> [VAD][local] SPEECH END (人声结束) frames={} <<<<<<<<<<".format(frames)) self._vc_local_vad_last_state = 0 self.VAD_count = 0 if not self._vc_local_vad_debug_only(): - self._vc_local_vad_on_endpoint() + self._vc_local_vad_arm_endpoint_hangover() return 0 return 0 def _vc_vad_reset_feed(self): self._vc_vad_pcm_tail = bytearray() self._vc_local_vad_last_state = 0 + # 每轮录音开始时清零上一轮的 ASR 时间戳,防止 asr_silence_fallback 用旧时间戳 + # 在新轮还未收到任何 asr.result 时就提前触发 + self._vc_last_asr_result_ms = 0 + # 场景① no_speech 判据:本轮是否有过说话起点 / 是否已触发停录回 idle + self._vc_turn_had_speech = 0 + self._vc_no_speech_idle = 0 + # 句末尾音 hangover 截止时刻(0=未在 hangover) + self._vc_endpoint_hangover_until = 0 if self._vc_vad_ready(): self.vad.custom_vad_reset() @@ -1201,7 +1265,10 @@ class VCSessionMediaMixin: self._vc_handle_asr_empty("ptt_release/silent") else: self._ptt_await_server = 0 - if ( + if getattr(self, "_vc_no_speech_idle", 0) == 1: + # 场景①:开麦后一直无有效语音,停录回 idle 省电(会话保持,不提交) + self._vc_publish_led("idle", "VC_Speech_End/no_speech") + elif ( getattr(self, "_vc_session_started", 0) == 1 and getattr(self, "Close_flag", 0) == 0 and getattr(self, "AI_client_VC_isconnect", 0) == 1 @@ -1219,7 +1286,63 @@ class VCSessionMediaMixin: except Exception: pass print("[VC] Speech_End record_state={}".format(self.VC_record_state_flag)) + def _vc_vad_worker_thread(self): + """VAD 独立 worker:阻塞取录音线程投递的 PCM,跑 imsl run_vad/vad_detect(å·² GIL 释放, + 可与录音/编码/发送并行)。VAD 内部状态(tail/last_state/VAD_count)仅本线程访问,避免竞态。 + 结果写 _vad_async_state 供录音线程读。""" + print("[VC][vad] worker start") + self._vad_worker_lock = 1 + while True: + try: + if self._vad_worker_lock == 2: + break + self._vad_worker_lock = 1 + buf = self._vad_in_queue.get() # 空则阻塞,put(None) 唤醒退出 + if self._vad_worker_lock == 2: + break + if buf is None: + continue + _t0 = utime.ticks_us() + self._vad_async_state = self._vc_vad_on_record_pcm(buf, len(buf)) + _us = utime.ticks_diff(utime.ticks_us(), _t0) + self._vad_proc_cnt = getattr(self, "_vad_proc_cnt", 0) + 1 + self._vad_proc_us_sum = getattr(self, "_vad_proc_us_sum", 0) + _us + if _us > getattr(self, "_vad_proc_us_max", 0): + self._vad_proc_us_max = _us + if self._vad_proc_cnt <= 5 or (self._vad_proc_cnt % 50) == 0: + _avg = self._vad_proc_us_sum // self._vad_proc_cnt + print("[VC][vad][thr] #{} len={} state={} vad={}us avg={}us max={}us".format( + self._vad_proc_cnt, len(buf), self._vad_async_state, _us, _avg, self._vad_proc_us_max)) + except Exception as e: + usys.print_exception(e) + utime.sleep_ms(2) + self._vad_worker_lock = 0 + self._vad_worker_id = 0 + print("[VC][vad] worker exit") + def _vc_vad_worker_close(self): + if getattr(self, "_vad_worker_id", 0) != 0: + count = 50 + while True: + count -= 1 + if self._vad_worker_lock == 1: + self._vad_worker_lock = 2 + # 唤醒阻塞在 get() 的 worker + try: + self._vad_in_queue.put(None) + except Exception: + pass + if self._vad_worker_lock == 0 or count <= 0: + break + utime.sleep_ms(1) + self._vad_worker_lock = 0 + if getattr(self, "_vad_worker_id", 0) != 0: + try: + _thread.stop_thread(self._vad_worker_id) + except Exception as e: + usys.print_exception(e) + self._vad_worker_id = 0 def VC_record_thread_close(self): + self._vc_vad_worker_close() if self.VC_record_thread_id != 0: count =50 while True: @@ -1257,7 +1380,27 @@ class VCSessionMediaMixin: AI_record_tmp_queue_len = 0 vad_state = 0 self.VAD_count = 0 + # 开机网络就绪后联网激活 VAD(试用→授权):录音启动时 WS 已连通,网络可用,只需成功一次 + try: + if hasattr(self, "_vc_vad_try_activate"): + self._vc_vad_try_activate("record_start") + except Exception: + pass self._vc_vad_reset_feed() + # VAD 线程化:开则启动 worker,录音线程只投递 PCM、不内联跑 VAD + self._vad_thread_on = cfg_bool("VC_VAD_THREAD_ENABLED", False) + if self._vad_thread_on: + self._vad_async_state = 0 + if getattr(self, "_vad_in_queue", None) is None: + self._vad_in_queue = Queue(16) + else: + self._vad_in_queue.clear() + if getattr(self, "_vad_worker_id", 0) == 0: + self._vad_worker_lock = 0 + _thread.stack_size(24*1024) + self._vad_worker_id = _thread.start_new_thread(self._vc_vad_worker_thread, ()) + _thread.stack_size(8*1024) + print("[VC][vad] worker_id:{}".format(self._vad_worker_id)) while True: try: if self.VC_record_thread_lock == 2: @@ -1308,7 +1451,25 @@ class VCSessionMediaMixin: #self.record_fd = None # VAD 检测:录音 PCM → imsl run_vad/vad_detect(custom_vad_process 封装) if self._vc_vad_ready(): - vad_state = self._vc_vad_on_record_pcm(Record_read_buf, Record_len) + if getattr(self, "_vad_thread_on", 0): + # 线程化:投递 PCM 拷贝给 worker(满则丢弃不阻塞),读最近一次结果(滞后约1帧) + try: + self._vad_in_queue.put(bytes(Record_read_buf[:Record_len])) + except Exception: + pass + vad_state = int(getattr(self, "_vad_async_state", 0)) + else: + _vad_t0 = utime.ticks_us() + vad_state = self._vc_vad_on_record_pcm(Record_read_buf, Record_len) + _vad_us = utime.ticks_diff(utime.ticks_us(), _vad_t0) + self._vad_proc_cnt = getattr(self, "_vad_proc_cnt", 0) + 1 + self._vad_proc_us_sum = getattr(self, "_vad_proc_us_sum", 0) + _vad_us + if _vad_us > getattr(self, "_vad_proc_us_max", 0): + self._vad_proc_us_max = _vad_us + if self._vad_proc_cnt <= 5 or (self._vad_proc_cnt % 50) == 0: + _vad_avg = self._vad_proc_us_sum // self._vad_proc_cnt + print("[VC][vad] #{} len={} state={} vad={}us avg={}us max={}us".format( + self._vad_proc_cnt, Record_len, vad_state, _vad_us, _vad_avg, self._vad_proc_us_max)) else: vad_state = 0 if self._vc_tts_pipeline_busy(): @@ -1320,13 +1481,19 @@ class VCSessionMediaMixin: else: try: + if hasattr(self, "_vc_endpoint_hangover_tick"): + self._vc_endpoint_hangover_tick() if hasattr(self, "_vc_long_dialogue_turn_cap_tick"): self._vc_long_dialogue_turn_cap_tick() + if hasattr(self, "_vc_no_speech_idle_tick"): + self._vc_no_speech_idle_tick() if hasattr(self, "_vc_asr_completed_stop_tick"): self._vc_asr_completed_stop_tick() + if hasattr(self, "_vc_local_vad_asr_fallback_tick"): + self._vc_local_vad_asr_fallback_tick() except Exception: pass - if not self._vc_vad_ready(): + if (not self._vc_vad_ready()) and not getattr(self, "_vad_thread_on", 0): self._vc_vad_pcm_tail = bytearray() if self.VC_record_state_flag <=1 :#没发送前先存起来 self.AI_record_tmp_queue.put(read_buf)