# 本地 imsl 断句静音(ms);映射 rule2/3;可被 storage vad_silence 覆盖
VAD_SILENCE_DEFAULT = 1000
+# 本地 VAD 判句末后追加采集的尾音(ms),到点再发 audio.completed;防句尾渐弱音被关闸截断。0=立即收尾
+LOCAL_VAD_ENDPOINT_HANGOVER_MS = 200
+# 本地 imsl VAD 能量阈值:越低越灵敏(易触发说话),越高越钝;imsl 推荐 10000。
+# 现场若本地 VAD 不出 speech start/end(全靠 asr_silence_fallback 收尾),调低(如 6000)
+VAD_ENERGY_THRESHOLD = 10000
# 长对话:单轮最长录音(ms)、asr.completed 后续说 merge 窗(ms);0=不限制
+# 说明:MAX_TURN_MS 解决"场景②一直说话太久"——强制提交截断输入;
+# NO_SPEECH_MS 解决"场景①开麦后一直没说话"——停录回 idle 省电,不提交。两者动作不同。
LONG_DIALOGUE_MAX_TURN_MS = 25000
-LONG_DIALOGUE_ASR_CONTINUE_MS = 2000
+LONG_DIALOGUE_ASR_CONTINUE_MS = 1000
+# 场景①:开麦后本轮始终无本地 VAD 说话起点且无服务端 asr.result,超此(ms)停录回 idle;0=不限制
+LONG_DIALOGUE_NO_SPEECH_MS = 12000
# llm 已起后连续 canceled 达此次数则冻结续说、停麦等回复(避免一直录无 TTS)
LONG_DIALOGUE_LLM_CANCEL_MAX = 2
# Opus 帧时长(ms):合法 2.5/5/10/20/40/60;16kHz 单声道。上行按此分帧(60ms=1920字节PCM)。
VC_OPUS_FRAME_MS = 60
+# 对话中途传输瞬断(ECONNRESET 等)自动重连:
+# True=对话进行中瞬断时静默重连+重开麦续录,不立刻播报错音;30s 窗口内超 MAX 次(抖动/真断网)才回退报错音。
+# 服务端 Start 不带 dialog_id,重连=新会话,无法续接服务端上下文,仅端侧重开麦。
+VC_RESUME_RECONNECT_ENABLED = True
+VC_RESUME_RECONNECT_MAX = 3
+
+# 本地 VAD 线程化:True=VAD 检测移到独立 worker 线程(配合 modVad GIL 释放可与录音/编码/发送并行),
+# 录音线程读 PCM 后立刻走上行、不再被 VAD(~50ms)阻塞;VAD 结果滞后约 1 帧供端侧断句判定。
+# 默认关:先验证重连+GIL 这版稳定,再单独打开验证 VAD 线程化(一次只引入一个并发变量)。
+VC_VAD_THREAD_ENABLED = True
+
# =============================================================================
# 9. 本地指令与固定话术
# =============================================================================
# 本地 imsl 断句静音(ms);映射 rule2/3;可被 storage vad_silence 覆盖
VAD_SILENCE_DEFAULT = 1000
+# 本地 VAD 判句末后追加采集的尾音(ms),到点再发 audio.completed;防句尾渐弱音被关闸截断。0=立即收尾
+LOCAL_VAD_ENDPOINT_HANGOVER_MS = 200
+# 本地 imsl VAD 能量阈值:越低越灵敏(易触发说话),越高越钝;imsl 推荐 10000。
+# 现场若本地 VAD 不出 speech start/end(全靠 asr_silence_fallback 收尾),调低(如 6000)
+VAD_ENERGY_THRESHOLD = 10000
# 长对话:单轮最长录音(ms)、asr.completed 后续说 merge 窗(ms);0=不限制
+# 说明:MAX_TURN_MS 解决"场景②一直说话太久"——强制提交截断输入;
+# NO_SPEECH_MS 解决"场景①开麦后一直没说话"——停录回 idle 省电,不提交。两者动作不同。
LONG_DIALOGUE_MAX_TURN_MS = 25000
-LONG_DIALOGUE_ASR_CONTINUE_MS = 2000
+LONG_DIALOGUE_ASR_CONTINUE_MS = 1000
+# 场景①:开麦后本轮始终无本地 VAD 说话起点且无服务端 asr.result,超此(ms)停录回 idle;0=不限制
+LONG_DIALOGUE_NO_SPEECH_MS = 12000
# llm 已起后连续 canceled 达此次数则冻结续说、停麦等回复(避免一直录无 TTS)
LONG_DIALOGUE_LLM_CANCEL_MAX = 2
# Opus 帧时长(ms):合法 2.5/5/10/20/40/60;16kHz 单声道。上行按此分帧(60ms=1920字节PCM)。
VC_OPUS_FRAME_MS = 60
+# 对话中途传输瞬断(ECONNRESET 等)自动重连:
+# True=对话进行中瞬断时静默重连+重开麦续录,不立刻播报错音;30s 窗口内超 MAX 次(抖动/真断网)才回退报错音。
+# 服务端 Start 不带 dialog_id,重连=新会话,无法续接服务端上下文,仅端侧重开麦。
+VC_RESUME_RECONNECT_ENABLED = True
+VC_RESUME_RECONNECT_MAX = 3
+
+# 本地 VAD 线程化:True=VAD 检测移到独立 worker 线程(配合 modVad GIL 释放可与录音/编码/发送并行),
+# 录音线程读 PCM 后立刻走上行、不再被 VAD(~50ms)阻塞;VAD 结果滞后约 1 帧供端侧断句判定。
+# 默认关:先验证重连+GIL 这版稳定,再单独打开验证 VAD 线程化(一次只引入一个并发变量)。
+VC_VAD_THREAD_ENABLED = True
+
# =============================================================================
# 9. 本地指令与固定话术
# =============================================================================
int byte_len;
int sample_len;
const int16_t *samples;
+ int vad_st;
vad_ensure_init(NULL);
mp_get_buffer_raise(data, &bufinfo, MP_BUFFER_READ);
return mp_obj_new_int(0);
}
samples = (const int16_t *)bufinfo.buf;
+ /* 纯 C 计算期间释放 GIL,避免阻塞录音/收发等其它线程(samples 指向调用方持有的不可变 bytes,GC 安全) */
+ MP_THREAD_GIL_EXIT();
run_vad(samples, sample_len);
- return mp_obj_new_int(vad_detect());
+ vad_st = vad_detect();
+ MP_THREAD_GIL_ENTER();
+ return mp_obj_new_int(vad_st);
}
STATIC mp_obj_t mp_vad_process(mp_obj_t self_in, mp_obj_t data, mp_obj_t data_len)
}
return mp_const_none;
}
-STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_vad_free_obj, 0, 0, mp_vad_free);
+/* 作为 locals_dict 方法调用 vad.free() 时,self 会作为第 1 个位置参数传入,
+ * 故需允许 1 个参数(否则报 "takes 0 positional arguments but 1 were given")。 */
+STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_vad_free_obj, 0, 1, mp_vad_free);
STATIC const mp_rom_map_elem_t VAD_locals_dict_table[] = {
{ MP_ROM_QSTR(MP_QSTR_process), MP_ROM_PTR(&mp_vad_process_obj) },
}\r
/* 帧长可变:2.5/5/10/20/40/60ms@16k(=40/80/160/320/640/960 样本),非法长度 opus_encode 返回 <0 */\r
uint8_t out[OPUS_CODEC_MAX_PKT];\r
- mp_int_t n = opus_encode(self->enc, (const opus_int16 *)pcm.buf, (int)(pcm.len / 2),\r
- out, OPUS_CODEC_MAX_PKT);\r
+ mp_int_t n;\r
+ /* 纯 C 编码期间释放 GIL,避免阻塞 VAD/收发线程(pcm.buf 指向调用方持有的不可变 bytes,out 为 C 栈,GC 安全) */\r
+ MP_THREAD_GIL_EXIT();\r
+ n = opus_encode(self->enc, (const opus_int16 *)pcm.buf, (int)(pcm.len / 2),\r
+ out, OPUS_CODEC_MAX_PKT);\r
+ MP_THREAD_GIL_ENTER();\r
if (n < 0) {\r
mp_raise_msg_varg(&mp_type_OSError, MP_ERROR_TEXT("opus_encode %d"), (int)n);\r
}\r
}\r
/* 帧长可变:按包内信息解码,输出实际样本数(可为 60ms=960 样本) */\r
opus_int16 pcm[OPUS_CODEC_MAX_FRAME];\r
- mp_int_t n = opus_decode(self->dec, pkt.buf, (opus_int32)pkt.len, pcm,\r
- OPUS_CODEC_MAX_FRAME, 0);\r
+ mp_int_t n;\r
+ /* 纯 C 解码期间释放 GIL,避免阻塞录音/VAD 线程(pkt.buf 指向调用方持有的不可变 bytes,pcm 为 C 栈,GC 安全) */\r
+ MP_THREAD_GIL_EXIT();\r
+ n = opus_decode(self->dec, pkt.buf, (opus_int32)pkt.len, pcm,\r
+ OPUS_CODEC_MAX_FRAME, 0);\r
+ MP_THREAD_GIL_ENTER();\r
if (n < 0) {\r
mp_raise_msg_varg(&mp_type_OSError, MP_ERROR_TEXT("opus_decode %d"), (int)n);\r
}\r
except Exception:
pass
+ def _vc_local_vad_asr_fallback_tick(self):
+ """local_vad endpoint 模式下 fallback:若本地 VAD 未检测到语音终点,但服务端 ASR
+ 已有结果且超过 merge 窗仍无新结果,视为句末,主动发 audio.completed。
+ 防止 VAD 灵敏度不足时录音无限持续直到 turn_cap。"""
+ try:
+ if not self._vc_local_vad_endpoint():
+ return
+ if getattr(self, "VC_record_state_flag", 0) < 2:
+ return
+ if getattr(self, "record_data_save_flag", 0) != 0:
+ return
+ if getattr(self, "_server_vad_silence", 0) == 1:
+ return
+ if getattr(self, "_ptt_audio_created", 0) != 1:
+ return
+ if getattr(self, "_turn_has_llm_created", 0) == 1:
+ return
+ last_ms = getattr(self, "_vc_last_asr_result_ms", 0)
+ if not last_ms:
+ return
+ elapsed = utime.ticks_diff(utime.ticks_ms(), last_ms)
+ # 比 merge 窗多留 500ms,避免 ASR 网络延迟误判
+ threshold = self._vc_asr_continue_merge_ms() + 500
+ if elapsed < threshold:
+ return
+ print("[VAD][local] asr_silence_fallback elapsed={}ms -> audio.completed".format(elapsed))
+ try:
+ if hasattr(self, "_vc_opus_uplink_flush"):
+ self._vc_opus_uplink_flush()
+ except Exception:
+ pass
+ self._vc_local_vad_on_endpoint()
+ except Exception:
+ pass
+
+ def _vc_build_vad(self):
+ """构造 imsl VAD 对象并记录授权模式(self._vad_init_mode)。
+ 失败返回 None。init_vad 内部完成联网授权:有网授权成功=2,否则回退试用=1。"""
+ try:
+ sil_ms = int(self.vad_silence)
+ # energy_threshold 越低越灵敏(易触发 speech),越高越钝;imsl 推荐 10000。
+ # 现场本地 VAD 不出 speech start/end 多为此值相对麦增益偏高,可调低试。
+ energy = cfg_int("VAD_ENERGY_THRESHOLD", 10000)
+ vad = VAD.VAD(
+ self.VC_record_samplerate, 1, self.VAD_data_len,
+ energy, 240, sil_ms, 480)
+ mode = vad.init_mode()
+ self._vad_init_mode = mode
+ print("[VAD] init_mode=", mode, "trial=", VAD.MODE_TRIAL,
+ "sil_ms=", sil_ms, "energy=", energy,
+ "debug_only=", self._local_vad_debug_only,
+ "endpoint=", not self._local_vad_debug_only)
+ return vad
+ except Exception as e:
+ print("[VAD] init fail:", e)
+ return None
+
+ def _vc_vad_try_activate(self, reason=""):
+ """开机网络就绪后联网激活 VAD:若当前为试用模式且有网络,free 后重建以触发联网授权。
+ 一旦拿到授权(mode=2)即置 _vad_activate_done,后续不再重试。无网/失败保留试用,下次再试。"""
+ try:
+ if getattr(self, "is_VAD", 0) != 1 or self.vad is None:
+ return
+ if getattr(self, "_vad_activate_done", 0) == 1:
+ return
+ if getattr(self, "_vad_init_mode", 0) == 2:
+ self._vad_activate_done = 1
+ return
+ # 网络可用性判断:CSQ 有效即认为可联网授权
+ try:
+ csq = net.csqQueryPoll()
+ except Exception:
+ csq = -1
+ if csq is None or csq == -1 or csq == 99:
+ return # 无网,下次录音再试
+ # 有网才执行激活;不论结果只做一次,避免每轮录音重复 free+重建
+ self._vad_activate_done = 1
+ self.vad.free()
+ new_vad = self._vc_build_vad()
+ if new_vad is not None:
+ self.vad = new_vad
+ if self._vad_init_mode == 2:
+ print("[VAD] activate ok (licensed) reason=", reason)
+ else:
+ print("[VAD] activate still trial reason=", reason, "csq=", csq)
+ except Exception as e:
+ print("[VAD] activate fail:", e)
+
def _long_dialogue_cancel_preserve_speech(self):
"""长对话 canceled:非用户打断时,录音中或 asr 续说窗内均保持/恢复上行。"""
if not self._vc_is_long_dialogue_mode():
except Exception:
pass
+ def _vc_no_speech_open_ms(self):
+ """场景①:开麦后一直无有效语音的停录超时(ms);0=不限制。"""
+ return cfg_int("LONG_DIALOGUE_NO_SPEECH_MS", 12000)
+
+ def _vc_no_speech_idle_tick(self):
+ """场景①:长对话开麦后,本轮始终无本地 VAD 说话起点且无 asr.result,
+ 判定无人接话→停录回 idle 省电(会话保持,不 force_commit、不喂 LLM)。
+ 与 turn_cap(场景②说太久强制提交) 区分:一旦检测到语音即不再触发本逻辑。"""
+ try:
+ if not self._vc_is_long_dialogue_mode():
+ return
+ if getattr(self, "_vc_no_speech_idle", 0) == 1:
+ return # 本轮已触发,等待异步停录完成
+ if getattr(self, "record_data_save_flag", 0) != 0:
+ return
+ if getattr(self, "VC_record_state_flag", 0) < 2:
+ return
+ # 本轮有任何语音迹象 → 交给正常断句/turn_cap,不属"无人说话"
+ if getattr(self, "_vc_turn_had_speech", 0) == 1:
+ return
+ if getattr(self, "_vc_last_asr_result_ms", 0):
+ return
+ if getattr(self, "_turn_has_llm_created", 0) == 1:
+ return
+ if getattr(self, "_vc_local_vad_last_state", 0) == 1:
+ return
+ max_ms = self._vc_no_speech_open_ms()
+ if max_ms <= 0:
+ return
+ start = getattr(self, "_vc_turn_wall_start_ms", 0)
+ if not start:
+ return
+ if utime.ticks_diff(utime.ticks_ms(), start) < max_ms:
+ return
+ print("[VC][no_speech] stop->idle after {}ms (no speech/asr)".format(max_ms))
+ self._vc_no_speech_idle = 1 # 让 VC_Speech_End 走 idle 灯态,且防本轮重复触发
+ self._server_vad_silence = 1
+ try:
+ self._uplink_mark_end("no_speech")
+ except Exception:
+ pass
+ self.request_uplink_stop("no_speech") # 异步停录;不 force_commit、不提交静音
+ except Exception:
+ pass
+
def _long_dialogue_media_busy(self):
"""长对话:TTS/思考/播报中(按键须走打断,勿误走 continue)。"""
try:
# ARG_silence_start_threshold, // 静音开始阈值(ms) → imsl rule1
self.vad = None
self._vc_vad_pcm_tail = bytearray()
+ self._vad_init_mode = 0 # 0失败 1试用 2授权
+ self._vad_activate_done = 0 # 已成功联网授权(mode=2)后置1,不再重试
if self.is_VAD == 1:
- try:
- sil_ms = int(self.vad_silence)
- self.vad = VAD.VAD(
- self.VC_record_samplerate, 1, self.VAD_data_len,
- 10000, 240, sil_ms, 480)
- mode = self.vad.init_mode()
- print("[VAD] init_mode=", mode, "trial=", VAD.MODE_TRIAL,
- "sil_ms=", sil_ms, "debug_only=", self._local_vad_debug_only,
- "endpoint=", not self._local_vad_debug_only)
- except Exception as e:
- print("[VAD] init fail:", e)
+ self.vad = self._vc_build_vad()
+ if self.vad is None:
self.is_VAD = 0
- self.vad = None
else:
print("[VAD] local vad disabled VC_mode=", getattr(self, "VC_mode", -1))
"""AI_VC mixin:VC WebSocket 建链、收发线程、协议 JSON 分发。"""
from vc_session_imports import * # noqa: F401,F403
from vc_notify import NotifyScheduler
-from vc_utils import cfg_bool, cfg_attr
+from vc_utils import cfg_bool, cfg_attr, cfg_int
# import * 不导入以下划线开头的符号;_thread 必须显式引入(否则 start_new_thread NameError)
import _thread
self.websocket_deconnect_VC()
except Exception:
pass
+ def _vc_ws_close_is_clean(self):
+ """判断 WS 关闭是否为服务端正常收尾(tts.done 已在 10s 内收到)。
+ 是则抑制"VC连接断开了"播报,避免打扰正常对话流程。"""
+ wall_ms = getattr(self, "_tts_completed_wall_ms", 0)
+ if not wall_ms:
+ return False
+ try:
+ elapsed = utime.ticks_diff(utime.ticks_ms(), wall_ms)
+ return 0 <= elapsed <= 10000
+ except Exception:
+ return False
+
def websocket_error(self, TTS_str=None, play_weak_signal_tone=False):
"""play_weak_signal_tone=True 时播 xinhaoruo1(弱网/无卡类);连接/协议失败请传 False,避免误报「信号弱」。"""
if getattr(self, "VC_mode", 0) != 1:
print("websocket_error4")
+ def _vc_on_transport_error(self, tts_str, reason="transport"):
+ """对话中途传输瞬断(ECONNRESET 等)入口:
+ - 对话进行中(_vc_session_started==1)且未在关闭:静默重连+重开麦(复用 websocket_error(None)
+ 的 Close→AI_VC_start 重连链,count<3 自带风暴保护),不立刻播报错音;
+ - 30s 窗口内瞬断超 VC_RESUME_RECONNECT_MAX 次(抖动/真断网) 或非对话中/已禁用:
+ 回退原 websocket_error(tts_str) 播报错音并按原逻辑处理。
+ 服务端 Start 不带 dialog_id,重连=新会话,无法续接服务端上下文,仅端侧重开麦续录。"""
+ try:
+ # 去抖:断线瞬间发送队列里多帧会连续触发,6s 内只处理一次,
+ # 避免后续帧在首次已 Close(session=0) 后误走播报错音分支。
+ now = utime.ticks_ms()
+ last = getattr(self, "_vc_transport_err_last_ms", 0)
+ if last and utime.ticks_diff(now, last) < cfg_int("VC_RESUME_DEBOUNCE_MS", 6000):
+ print("[VC][resume] debounce skip dup transport error")
+ return
+ self._vc_transport_err_last_ms = now
+ in_session = getattr(self, "_vc_session_started", 0) == 1
+ closing = getattr(self, "Close_flag", 0) == 1
+ if (not cfg_bool("VC_RESUME_RECONNECT_ENABLED", True)) or (not in_session) or closing:
+ self.websocket_error(tts_str)
+ return
+ win = getattr(self, "_vc_resume_win_ms", 0)
+ if (not win) or utime.ticks_diff(now, win) > 30000:
+ self._vc_resume_win_ms = now
+ self._vc_resume_cnt = 0
+ self._vc_resume_cnt = getattr(self, "_vc_resume_cnt", 0) + 1
+ if self._vc_resume_cnt > cfg_int("VC_RESUME_RECONNECT_MAX", 3):
+ print("[VC][resume] storm guard cnt={} reason={} -> spoken error".format(
+ self._vc_resume_cnt, reason))
+ self.websocket_error(tts_str)
+ return
+ print("[VC][resume] silent reconnect reason={} cnt={} was_rec={}".format(
+ reason, self._vc_resume_cnt, getattr(self, "VC_record_state_flag", 0)))
+ self.websocket_error(None)
+ except Exception as e:
+ usys.print_exception(e)
+ try:
+ self.websocket_error(tts_str)
+ except Exception:
+ pass
def websocket_deconnect_VC(self):
#print("websocket_deconnect_VC {}!!!!!!!".format(self.AI_client_VC_isconnect))
print("websocket_deconnect_VC!!!!!!!")
frame = bytes(buf[off:off + fb])
off += fb
try:
+ _enc_t0 = utime.ticks_us()
pkt = codec.encode(frame)
+ _enc_us = utime.ticks_diff(utime.ticks_us(), _enc_t0)
except Exception as e:
print("[VC][opus_up] encode fail in={}".format(fb))
usys.print_exception(e)
if l <= 0:
continue
self._opus_up_cnt = getattr(self, "_opus_up_cnt", 0) + 1
+ self._opus_up_us_sum = getattr(self, "_opus_up_us_sum", 0) + _enc_us
+ if _enc_us > getattr(self, "_opus_up_us_max", 0):
+ self._opus_up_us_max = _enc_us
if self._opus_up_cnt <= 5 or (self._opus_up_cnt % 50) == 0:
- print("[VC][opus_up] #{} in={} out={}".format(self._opus_up_cnt, fb, l))
+ _avg = self._opus_up_us_sum // self._opus_up_cnt
+ print("[VC][opus_up] #{} in={} out={} enc={}us avg={}us max={}us".format(
+ self._opus_up_cnt, fb, l, _enc_us, _avg, self._opus_up_us_max))
if self.AI_client_VC_isconnect != 1 or self.AI_client_VC is None:
self._uplink_note_tx_fail(l, "opus_enqueue_ws_down")
self._opus_up_buf = bytearray(buf[off:])
self.AI_VC_send_data_queue.put([AI_client_VC_num, pkt])
self._opus_up_buf = bytearray(buf[off:]) if off < total else bytearray()
+ def _vc_opus_uplink_flush(self):
+ """VAD endpoint 时将 Opus 上行缓冲中不足一帧的剩余 PCM 补零到完整帧后编码发出,
+ 避免句末语音被截断。必须在 silence_gate(_server_vad_silence=1) 前调用。"""
+ if not self._vc_opus_uplink_on():
+ return
+ buf = getattr(self, "_opus_up_buf", None)
+ if not buf or len(buf) == 0:
+ return
+ fb = self._vc_opus_frame_bytes()
+ remaining = len(buf) % fb
+ if remaining == 0:
+ return # 恰好整帧,正常流程会处理
+ pad_len = fb - remaining
+ try:
+ codec = self._vc_opus_up_codec()
+ padded = bytes(buf[-remaining:]) + bytes(pad_len)
+ pkt = codec.encode(padded)
+ if pkt and len(pkt) > 0:
+ if self.AI_client_VC_isconnect == 1 and self.AI_client_VC is not None:
+ self._uplink_mark_start(len(pkt))
+ self._uplink_bytes += len(pkt)
+ self.AI_VC_send_data_queue.put([AI_client_VC_num, pkt])
+ print("[VC][opus_up] flush in={} pad={} out={}".format(
+ remaining, pad_len, len(pkt)))
+ except Exception as e:
+ usys.print_exception(e)
+ self._opus_up_buf = bytearray(buf[: len(buf) - remaining])
+
def AI_client_VC_send(self,data_str):
# 上行 opus:PCM(bytes) 累积分帧编码后入队;开关关闭时完全走原 PCM 逻辑
if not isinstance(data_str, str) and self._vc_opus_uplink_on():
try:
if self.AI_client_VC_isconnect != 0:
print("VC发送数据错误,退出")
- EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"VC发送出错了,请重试"})
+ EventMesh.publish("publish_AI_btnInfo",{"func":self._vc_on_transport_error,"data":"VC发送出错了,请重试"})
self.ud_error(["VC",self.VC_taskid])
except Exception as e: #网络连接异常
usys.print_exception(e)
self.close_timerout = 0
if recv_data == "websocket close":
# 服务器关闭连接
- print("VC 服务器关闭连接!!!!!!!!!!!!!!!!!!!!!!!!!!!")
- #self.AI_client_VC_isconnect = 0
- #self.websocket_deconnect_VC()
- #EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_deconnect_asr})
- # #EventMesh.publish("publish_AI_btnInfo",{"func":utime.sleep_ms,'data':(1500)})
- #EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_connect_asr})
- EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"VC连接断开了,请重试"})
+ _ws_close_clean = self._vc_ws_close_is_clean()
+ print("[VC] ws server close clean={}".format(_ws_close_clean))
+ _ws_err_str = None if _ws_close_clean else "VC连接断开了,请重试"
+ if _ws_err_str is None:
+ EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":None})
+ else:
+ EventMesh.publish("publish_AI_btnInfo",{"func":self._vc_on_transport_error,"data":_ws_err_str})
self.ud_error(["VC",self.VC_taskid])
break
elif recv_data == "": # EOF:链路断开/对端关闭
sid = getattr(self, "_cur_sid", "") or "-"
except Exception:
sid = "-"
- print("[VC] ws EOF (link down) VC_mode={} sid={}".format(
- getattr(self, "VC_mode", -1), sid
+ _ws_eof_clean = self._vc_ws_close_is_clean()
+ print("[VC] ws EOF (link down) VC_mode={} sid={} clean={}".format(
+ getattr(self, "VC_mode", -1), sid, _ws_eof_clean
))
- EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"VC连接断开了,请重试"})
+ _ws_err_str = None if _ws_eof_clean else "VC连接断开了,请重试"
+ if _ws_err_str is None:
+ EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":None})
+ else:
+ EventMesh.publish("publish_AI_btnInfo",{"func":self._vc_on_transport_error,"data":_ws_err_str})
self.ud_error(["VC",self.VC_taskid])
break
try:
_tl = recv_data.strip().lower()
if _tl in ("ping", "pong"):
continue
+ # llm.delta 流式包:端侧不展示文本,VAD 模式下处理为空操作。
+ # 服务端会在 LLM 生成期间狂发 delta,若每条都 ujson.loads+dispatch,
+ # recv 线程会被拖住 2~3s,导致其后的 tts.started 晚到、播放迟迟不开始。
+ # 这里在解析前用子串快速识别并跳过(idle 计时已在解析前刷新,安全)。
+ if self.is_VAD != 0 and ('"llm.delta"' in recv_data):
+ continue
jsload_data = ujson.loads(recv_data)
try:
jsload_data = vc_proto_normalize_server(jsload_data)
except Exception:
pass
+ try:
+ _rev = jsload_data.get("event", "")
+ except Exception:
+ _rev = ""
if VC_WS_IO_LOG:
- print("[VC] recv raw:{}".format(recv_data))
+ # llm.delta 流式包数量多,逐条 print 完整 raw 会拖慢 recv 线程(~124ms/条),
+ # 导致 tts.started 被堵在 delta 队列后晚到数秒;这里跳过 delta 的 raw 打印
+ if _rev != "llm.delta":
+ print("[VC] recv raw:{}".format(recv_data))
else:
- try:
- _rev = jsload_data.get("event", "")
- if _rev != "llm.delta":
- print("[VC] rx event={} sid={}".format(
- _rev, jsload_data.get("sid", "") or "-"))
- except Exception:
- pass
+ if _rev != "llm.delta":
+ print("[VC] rx event={} sid={}".format(
+ _rev, jsload_data.get("sid", "") or "-"))
#print("AI_client_VC_recv jsload_data = {}".format(jsload_data))
try:
self._cur_sid = jsload_data.get("sid", "") or ""
try:
self.request_uplink_stop("asr.completed/config_cmd")
+ # synthesis 在新会话 server.started 后播出;需跳过首次自动开麦,等 synthesis TTS 播完再开
+ self._skip_started_uplink_once = 1
EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_deconnect_VC})
EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_VC_start})
except Exception as e:
print("新的AI对话不允许以前的播放数据:")
break
+ # 打断/丢弃判断提前:在 Opus 解码之前决定是否丢帧
+ # 丢弃时直接 continue,跳过解码+sleep,让 recv 线程快速排空 WS 缓冲
+ _drop_pcm = False
+ try:
+ if (
+ hasattr(self, "_long_dialogue_should_drop_inbound_tts")
+ and self._vc_is_long_dialogue_mode()
+ ):
+ _drop_pcm = self._long_dialogue_should_drop_inbound_tts()
+ elif getattr(self, "is_ruppt_tts", 0) == 1:
+ _drop_pcm = True
+ except Exception:
+ _drop_pcm = getattr(self, "is_ruppt_tts", 0) == 1
+ if _drop_pcm:
+ # 跳过昂贵的 opus 解码+文件写+入播放队列,快速丢弃被打断后的残留帧;
+ # 但必须保留 sleep 让出 CPU,否则连续 continue 会饿死上行发送线程导致 TX_FAIL
+ utime.sleep_ms(1)
+ continue
+
# 下行 opus:每个二进制帧=一个 opus 包,解码为 PCM 再走原播放/保存路径
if not isinstance(recv_data, str) and self._vc_opus_downlink_on():
try:
_ilen = len(recv_data)
+ _dec_t0 = utime.ticks_us()
recv_data = self._vc_opus_down_codec().decode(recv_data)
+ _dec_us = utime.ticks_diff(utime.ticks_us(), _dec_t0)
self._opus_dn_cnt = getattr(self, "_opus_dn_cnt", 0) + 1
+ self._opus_dn_us_sum = getattr(self, "_opus_dn_us_sum", 0) + _dec_us
+ if _dec_us > getattr(self, "_opus_dn_us_max", 0):
+ self._opus_dn_us_max = _dec_us
if self._opus_dn_cnt <= 5 or (self._opus_dn_cnt % 50) == 0:
- print("[VC][opus_dn] #{} in={} out={}".format(
- self._opus_dn_cnt, _ilen, len(recv_data)))
+ _avg = self._opus_dn_us_sum // self._opus_dn_cnt
+ print("[VC][opus_dn] #{} in={} out={} dec={}us avg={}us max={}us".format(
+ self._opus_dn_cnt, _ilen, len(recv_data), _dec_us, _avg, self._opus_dn_us_max))
except Exception as e:
print("[VC][opus_dn] decode fail in={}".format(_ilen))
usys.print_exception(e)
continue
- # if VC_AUDIO_FRAME_LOG:
- # print("下发音频数据1:{}".format(len(recv_data)))
-
if self.play_data_save_flag == 1:
if self.play_fd ==None:
try:
self.play_fd = open('/usr/play_data.pcm','w')
if self.play_fd !=None:
self.play_fd.write(recv_data)
- # print("下发音频数据2:{}".format(len(recv_data)))
- #if self.VC_state >= VC_State_TTSStart:
- #self.put_record_data([recv_data,""])
- _drop_pcm = False
- try:
- if (
- hasattr(self, "_long_dialogue_should_drop_inbound_tts")
- and self._vc_is_long_dialogue_mode()
- ):
- _drop_pcm = self._long_dialogue_should_drop_inbound_tts()
- elif getattr(self, "is_ruppt_tts", 0) == 1:
- _drop_pcm = True
- except Exception:
- _drop_pcm = getattr(self, "is_ruppt_tts", 0) == 1
- if not _drop_pcm:
- if self.VC_play_TYPE == VC_play_TYPE_PCM:
- if self.play_pcm != None:
- try:
- #print("play_pcm.stream_write !!!")
+ if self.VC_play_TYPE == VC_play_TYPE_PCM:
+ if self.play_pcm != None:
+ try:
+ buf_len = self.play_pcm.stream_get_buf_len()
+ count_tmp = 1000
+ while buf_len < len(recv_data):
+ utime.sleep_ms(1)
+ count_tmp-=1
+ if count_tmp<=0 or self.play_pcm == None:
+ break
buf_len = self.play_pcm.stream_get_buf_len()
- count_tmp = 1000
- while buf_len < len(recv_data):
- utime.sleep_ms(1)
- count_tmp-=1
- if count_tmp<=0 or self.play_pcm == None:
- break
- buf_len = self.play_pcm.stream_get_buf_len()
- #print("buf已满,等待释放:buf_len:{},count_tmp:{}".format(buf_len,count_tmp))
- except Exception as e:
- usys.print_exception(e)
- EventMesh.publish("publish_AI_btnInfo",{"func":self.put_record_data,"data":[recv_data,""]})
- else:
- print("被打断不播放")
+ except Exception as e:
+ usys.print_exception(e)
+ EventMesh.publish("publish_AI_btnInfo",{"func":self.put_record_data,"data":[recv_data,""]})
utime.sleep_ms(1)
except Exception as e:
usys.print_exception(e)
usys.print_exception(e)
#self.websocket_error()
print("VC VC :读数据报错,退出")
- EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"VC出错了,请重试"})
+ EventMesh.publish("publish_AI_btnInfo",{"func":self._vc_on_transport_error,"data":"VC出错了,请重试"})
self.ud_error(["VC",self.VC_taskid])
break
self.AI_client_VC_isconnect = 0
return self._vc_vad_ready() and not self._vc_local_vad_debug_only()
def _vc_vad_feed_record_pcm(self, record_buf, record_len):
- """录音 PCM 按 VAD_data_len 切块送 imsl process,返回 0/1/2(2=句末)。"""
+ """录音 PCM 按 VAD_data_len 切块实时逐帧送 imsl process,返回 0/1/2(2=句末)。
+ 每帧独立判断:
+ - st==2(endpoint):立即 break,不再处理剩余帧,保证及时响应;
+ - st==1(speech):峰值保留,不因后续静音帧归零;
+ - st==0(silence):不重置已有 speech 状态,由 imsl 内部静音累积决定 endpoint。"""
if not self._vc_vad_ready() or record_len <= 0:
return 0
frame_len = int(getattr(self, "VAD_data_len", 640))
except Exception:
pass
vad_state = 2
+ break # endpoint:立即退出,剩余帧留在 tail 等下次(录音停后会被丢弃)
elif st == 1:
vad_state = 1
- elif vad_state != 2:
- vad_state = 0
+ # st==0 时不重置 vad_state(峰值保留)
return vad_state
def _vc_local_vad_send_audio_create(self):
self._vc_asr_completed_ms = utime.ticks_ms()
except Exception:
pass
+ # silence_gate 前先把 Opus 缓冲中不足一帧的剩余 PCM 补零发出,避免句末截断
+ try:
+ if hasattr(self, "_vc_opus_uplink_flush"):
+ self._vc_opus_uplink_flush()
+ except Exception:
+ pass
if self._vc_long_dialogue_defer_vad_stop():
self._vc_defer_vad_uplink_stop = 1
self._server_vad_silence = 1
self._uplink_mark_end("local.vad.end")
self._vc_local_vad_send_audio_completed(defer_stop=False)
+ def _vc_endpoint_hangover_ms(self):
+ """本地 VAD 句末后追加采集的尾音时长(ms);0=不延迟立即收尾。"""
+ return cfg_int("LOCAL_VAD_ENDPOINT_HANGOVER_MS", 200)
+
+ def _vc_local_vad_arm_endpoint_hangover(self):
+ """本地 VAD 判句末:先进入尾音 hangover(不关上行,继续录发 N ms),
+ 到点再真正收尾(audio.completed),避免句尾渐弱音被立即关闸截断。
+ hangover 期间用户又开口则取消收尾;再次判句末则立即收尾。"""
+ ms = self._vc_endpoint_hangover_ms()
+ if ms <= 0 or not self._vc_is_long_dialogue_mode():
+ self._vc_local_vad_on_endpoint()
+ return
+ if getattr(self, "_vc_endpoint_hangover_until", 0):
+ # 已在 hangover 中又判句末 → 直接收尾,不再延长
+ self._vc_endpoint_hangover_until = 0
+ self._vc_local_vad_on_endpoint()
+ return
+ self._vc_endpoint_hangover_until = utime.ticks_ms() + ms
+ print("[VAD][local] endpoint -> hangover {}ms".format(ms))
+
+ def _vc_endpoint_hangover_tick(self):
+ """录音线程周期检查:hangover 到点收尾;期间用户又开口则取消。"""
+ try:
+ until = getattr(self, "_vc_endpoint_hangover_until", 0)
+ if not until:
+ return
+ if getattr(self, "VC_record_state_flag", 0) < 2:
+ self._vc_endpoint_hangover_until = 0
+ return
+ if getattr(self, "_server_vad_silence", 0) == 1:
+ # 已被其它路径收尾(如 fallback),不重复发 audio.completed
+ self._vc_endpoint_hangover_until = 0
+ return
+ if getattr(self, "_vc_local_vad_last_state", 0) == 1:
+ # 尾音窗内用户又开口 → 取消收尾,继续录这一句
+ self._vc_endpoint_hangover_until = 0
+ print("[VAD][local] hangover canceled (speech resumed)")
+ return
+ if utime.ticks_diff(utime.ticks_ms(), until) < 0:
+ return
+ self._vc_endpoint_hangover_until = 0
+ self._vc_local_vad_on_endpoint()
+ except Exception:
+ pass
+
def _vc_local_vad_resume_uplink_merge(self, reason="local.pause"):
"""句中短停后续说:开 silence_gate + 新 audio.create,同轮 turn 墙钟不重置。"""
if not self._vc_local_vad_endpoint():
if vad_state == 1:
prev = int(getattr(self, "_vc_local_vad_last_state", 0))
if prev != 1:
- print("[VAD][local] speech start")
+ print(">>>>>>>>>> [VAD][local] SPEECH START (人声起点) <<<<<<<<<<")
self._vc_local_vad_last_state = 1
+ self._vc_turn_had_speech = 1
if self._vc_local_vad_endpoint():
if self._vc_asr_continue_merge_active():
self._vc_local_vad_resume_uplink_merge("asr.completed")
return 1
if vad_state == 2:
frames = int(getattr(self, "VAD_count", 0))
- print("[VAD][local] speech end frames={}".format(frames))
+ print(">>>>>>>>>> [VAD][local] SPEECH END (人声结束) frames={} <<<<<<<<<<".format(frames))
self._vc_local_vad_last_state = 0
self.VAD_count = 0
if not self._vc_local_vad_debug_only():
- self._vc_local_vad_on_endpoint()
+ self._vc_local_vad_arm_endpoint_hangover()
return 0
return 0
def _vc_vad_reset_feed(self):
self._vc_vad_pcm_tail = bytearray()
self._vc_local_vad_last_state = 0
+ # 每轮录音开始时清零上一轮的 ASR 时间戳,防止 asr_silence_fallback 用旧时间戳
+ # 在新轮还未收到任何 asr.result 时就提前触发
+ self._vc_last_asr_result_ms = 0
+ # 场景① no_speech 判据:本轮是否有过说话起点 / 是否已触发停录回 idle
+ self._vc_turn_had_speech = 0
+ self._vc_no_speech_idle = 0
+ # 句末尾音 hangover 截止时刻(0=未在 hangover)
+ self._vc_endpoint_hangover_until = 0
if self._vc_vad_ready():
self.vad.custom_vad_reset()
self._vc_handle_asr_empty("ptt_release/silent")
else:
self._ptt_await_server = 0
- if (
+ if getattr(self, "_vc_no_speech_idle", 0) == 1:
+ # 场景①:开麦后一直无有效语音,停录回 idle 省电(会话保持,不提交)
+ self._vc_publish_led("idle", "VC_Speech_End/no_speech")
+ elif (
getattr(self, "_vc_session_started", 0) == 1
and getattr(self, "Close_flag", 0) == 0
and getattr(self, "AI_client_VC_isconnect", 0) == 1
except Exception:
pass
print("[VC] Speech_End record_state={}".format(self.VC_record_state_flag))
+ def _vc_vad_worker_thread(self):
+ """VAD 独立 worker:阻塞取录音线程投递的 PCM,跑 imsl run_vad/vad_detect(已 GIL 释放,
+ 可与录音/编码/发送并行)。VAD 内部状态(tail/last_state/VAD_count)仅本线程访问,避免竞态。
+ 结果写 _vad_async_state 供录音线程读。"""
+ print("[VC][vad] worker start")
+ self._vad_worker_lock = 1
+ while True:
+ try:
+ if self._vad_worker_lock == 2:
+ break
+ self._vad_worker_lock = 1
+ buf = self._vad_in_queue.get() # 空则阻塞,put(None) 唤醒退出
+ if self._vad_worker_lock == 2:
+ break
+ if buf is None:
+ continue
+ _t0 = utime.ticks_us()
+ self._vad_async_state = self._vc_vad_on_record_pcm(buf, len(buf))
+ _us = utime.ticks_diff(utime.ticks_us(), _t0)
+ self._vad_proc_cnt = getattr(self, "_vad_proc_cnt", 0) + 1
+ self._vad_proc_us_sum = getattr(self, "_vad_proc_us_sum", 0) + _us
+ if _us > getattr(self, "_vad_proc_us_max", 0):
+ self._vad_proc_us_max = _us
+ if self._vad_proc_cnt <= 5 or (self._vad_proc_cnt % 50) == 0:
+ _avg = self._vad_proc_us_sum // self._vad_proc_cnt
+ print("[VC][vad][thr] #{} len={} state={} vad={}us avg={}us max={}us".format(
+ self._vad_proc_cnt, len(buf), self._vad_async_state, _us, _avg, self._vad_proc_us_max))
+ except Exception as e:
+ usys.print_exception(e)
+ utime.sleep_ms(2)
+ self._vad_worker_lock = 0
+ self._vad_worker_id = 0
+ print("[VC][vad] worker exit")
+ def _vc_vad_worker_close(self):
+ if getattr(self, "_vad_worker_id", 0) != 0:
+ count = 50
+ while True:
+ count -= 1
+ if self._vad_worker_lock == 1:
+ self._vad_worker_lock = 2
+ # 唤醒阻塞在 get() 的 worker
+ try:
+ self._vad_in_queue.put(None)
+ except Exception:
+ pass
+ if self._vad_worker_lock == 0 or count <= 0:
+ break
+ utime.sleep_ms(1)
+ self._vad_worker_lock = 0
+ if getattr(self, "_vad_worker_id", 0) != 0:
+ try:
+ _thread.stop_thread(self._vad_worker_id)
+ except Exception as e:
+ usys.print_exception(e)
+ self._vad_worker_id = 0
def VC_record_thread_close(self):
+ self._vc_vad_worker_close()
if self.VC_record_thread_id != 0:
count =50
while True:
AI_record_tmp_queue_len = 0
vad_state = 0
self.VAD_count = 0
+ # 开机网络就绪后联网激活 VAD(试用→授权):录音启动时 WS 已连通,网络可用,只需成功一次
+ try:
+ if hasattr(self, "_vc_vad_try_activate"):
+ self._vc_vad_try_activate("record_start")
+ except Exception:
+ pass
self._vc_vad_reset_feed()
+ # VAD 线程化:开则启动 worker,录音线程只投递 PCM、不内联跑 VAD
+ self._vad_thread_on = cfg_bool("VC_VAD_THREAD_ENABLED", False)
+ if self._vad_thread_on:
+ self._vad_async_state = 0
+ if getattr(self, "_vad_in_queue", None) is None:
+ self._vad_in_queue = Queue(16)
+ else:
+ self._vad_in_queue.clear()
+ if getattr(self, "_vad_worker_id", 0) == 0:
+ self._vad_worker_lock = 0
+ _thread.stack_size(24*1024)
+ self._vad_worker_id = _thread.start_new_thread(self._vc_vad_worker_thread, ())
+ _thread.stack_size(8*1024)
+ print("[VC][vad] worker_id:{}".format(self._vad_worker_id))
while True:
try:
if self.VC_record_thread_lock == 2:
#self.record_fd = None
# VAD 检测:录音 PCM → imsl run_vad/vad_detect(custom_vad_process 封装)
if self._vc_vad_ready():
- vad_state = self._vc_vad_on_record_pcm(Record_read_buf, Record_len)
+ if getattr(self, "_vad_thread_on", 0):
+ # 线程化:投递 PCM 拷贝给 worker(满则丢弃不阻塞),读最近一次结果(滞后约1帧)
+ try:
+ self._vad_in_queue.put(bytes(Record_read_buf[:Record_len]))
+ except Exception:
+ pass
+ vad_state = int(getattr(self, "_vad_async_state", 0))
+ else:
+ _vad_t0 = utime.ticks_us()
+ vad_state = self._vc_vad_on_record_pcm(Record_read_buf, Record_len)
+ _vad_us = utime.ticks_diff(utime.ticks_us(), _vad_t0)
+ self._vad_proc_cnt = getattr(self, "_vad_proc_cnt", 0) + 1
+ self._vad_proc_us_sum = getattr(self, "_vad_proc_us_sum", 0) + _vad_us
+ if _vad_us > getattr(self, "_vad_proc_us_max", 0):
+ self._vad_proc_us_max = _vad_us
+ if self._vad_proc_cnt <= 5 or (self._vad_proc_cnt % 50) == 0:
+ _vad_avg = self._vad_proc_us_sum // self._vad_proc_cnt
+ print("[VC][vad] #{} len={} state={} vad={}us avg={}us max={}us".format(
+ self._vad_proc_cnt, Record_len, vad_state, _vad_us, _vad_avg, self._vad_proc_us_max))
else:
vad_state = 0
if self._vc_tts_pipeline_busy():
else:
try:
+ if hasattr(self, "_vc_endpoint_hangover_tick"):
+ self._vc_endpoint_hangover_tick()
if hasattr(self, "_vc_long_dialogue_turn_cap_tick"):
self._vc_long_dialogue_turn_cap_tick()
+ if hasattr(self, "_vc_no_speech_idle_tick"):
+ self._vc_no_speech_idle_tick()
if hasattr(self, "_vc_asr_completed_stop_tick"):
self._vc_asr_completed_stop_tick()
+ if hasattr(self, "_vc_local_vad_asr_fallback_tick"):
+ self._vc_local_vad_asr_fallback_tick()
except Exception:
pass
- if not self._vc_vad_ready():
+ if (not self._vc_vad_ready()) and not getattr(self, "_vad_thread_on", 0):
self._vc_vad_pcm_tail = bytearray()
if self.VC_record_state_flag <=1 :#没发送前先存起来
self.AI_record_tmp_queue.put(read_buf)