From: p15692025051 <846603105@qq.com> Date: Wed, 19 Mar 2025 09:46:01 +0000 (+0800) Subject: VAD调试前备份 X-Git-Url: http://git.snicker.com.cn/?a=commitdiff_plain;h=cf3e12e22f58d8a1599c690443b5c65ad8179e89;p=mirrors%2FASR1603_G5.git VAD调试前备份 --- diff --git a/2.sdk/project/7PIN/sdk/services/microPython/ports/quectel/core/ai_VC.py b/2.sdk/project/7PIN/sdk/services/microPython/ports/quectel/core/ai_VC.py index 4fec50dd..82e7b17e 100644 --- a/2.sdk/project/7PIN/sdk/services/microPython/ports/quectel/core/ai_VC.py +++ b/2.sdk/project/7PIN/sdk/services/microPython/ports/quectel/core/ai_VC.py @@ -30,14 +30,13 @@ import machine import dataCall import constants import versions - +#import VAD #dev_type = "G10" dev_type = "7PIN_BK"#小瑞 #dev_type = "7PIN"#柒牌 #dev_type = 'momo'#食梦貘 #dev_type = 'sjgq'#四季甘泉 #dev_type = 'szbst'#深圳百事通 -#dev_type = 'rabbit' class AI_led(Abstract): def __init__(self): if dev_type == "G10": @@ -172,7 +171,7 @@ audio_buf = ['chenggong', 'didianguanji', 'IMEIhao', 'jianquancuowu', 'key_promp class AI_dev(Abstract): # AI def __init__(self): - self.is_wdt = 1 #是否开启看门狗 + self.is_wdt = 0 #是否开启看门狗 self.VBATT_tab = {"charge_full":4300,"100" :4250 , "90":4179 , "80":4070, "60": 3883, "40":3770, "20":3699, "10":3652,"5":3600,} self.KeyAI_status = 0 @@ -223,6 +222,8 @@ class AI_dev(Abstract): EventMesh.subscribe("audio_montage_play",self.audio_montage_play) EventMesh.subscribe("paly_iccid_imei",self.paly_iccid_imei) EventMesh.subscribe("paly_device_res_error",self.paly_device_res_error) + EventMesh.subscribe("Power_off",self.pwk_key_Long_Press) + print("AI_dev:2") self.pk = PowerKey() self.pk.powerKeyEventRegister(self.powerKey_callback) @@ -355,7 +356,7 @@ class AI_dev(Abstract): self.aud.stopAll() Power.powerDown() #print("AI_key_release") - EventMesh.publish("pwk_key_long_press") + #EventMesh.publish("pwk_key_long_press") #EventMesh.publish("publish_AI_btnInfo",{"func":self.pwk_key_Long_Press}) if self.pwk_key_status == 1:#松开 #print("pwk_key_timer_cb:{}".format(self.pwk_key_timer_count)) @@ -437,7 +438,7 @@ class AI_dev(Abstract): print("AI_key_release") EventMesh.publish("AI_key_release") - def pwk_key_Long_Press(self): + def pwk_key_Long_Press(self,event=None,msg=None): self.check_rest_timer_count = 0 print("pwk_key_long_press!!!!!!!") print("pwk_key_long_press!!!!!!!") @@ -452,7 +453,7 @@ class AI_dev(Abstract): self.aud.stopAll() Power.powerDown() #print("AI_key_release") - EventMesh.publish("pwk_key_long_press") + #EventMesh.publish("pwk_key_long_press") def pwk_key_release(self): self.check_rest_timer_count = 0 @@ -1165,16 +1166,18 @@ class AI_VC(Abstract): self.is_long_record = 0 #是否开启长录音模式（语音打断模式） self.ai_OTA = OTA() self.is_auth = 0 #是否开启dmp鉴权 - #self.VC_mode = 0 #0：短按连续问答模式 1：长按录音模式 + #self.VC_mode = 1 #0：短按连续问答模式 1：长按录音模式 self.VC_mode = EventMesh.publish("get_storage_config","VC_mode")# if self.VC_mode == None : self.VC_mode = 0 self.is_up_error= 0 #是否上传错误id + self.Close_flag =0 #=1时，连续对话播放完音频就退出 + self.is_llm_commands = 1#=1用大模型判断命令，=0用服务器本地插件判断命令 self.error_data = [] self.asr_taskid = "" self.VC_taskid = "" self.VC_server = 86 #0柒牌正式服务器，1：测试服务器1,2：测试服务器2 - + self.server_ip = EventMesh.publish("get_storage_config","server_ip")# if self.server_ip == None: if self.VC_server == 1:# @@ -1187,8 +1190,12 @@ class AI_VC(Abstract): self.server_ip="amp-wss.raisound.com" elif self.VC_server == 6:# self.server_ip="118.145.158.207:22080" + elif self.VC_server == 7:# + self.server_ip="118.145.158.207:22080" + elif self.VC_server == 8:# + self.server_ip="115.190.94.208:22080" elif self.VC_server == 86:# - self.server_ip="118.145.158.207:22086" + self.server_ip="118.145.158.207:22086" else: #self.server_ip="sevenpin.qipai.com.cn" self.server_ip="118.145.158.207:22082" @@ -1199,9 +1206,7 @@ class AI_VC(Abstract): elif dev_type == "sjgq": self.server_ip="118.145.158.207:22084" - elif dev_type == "szbst" or dev_type == 'rabbit': - self.server_ip="118.145.158.207:22086" - #self.server_ip="115.190.94.208:22080" + #开机提示音 self.power_on_tone = EventMesh.publish("get_storage_config","power_on_tone")# if dev_type != "7PIN": @@ -1211,15 +1216,12 @@ class AI_VC(Abstract): self.power_on_tone = "已开机，您好！我是北科瑞讯智能 AI bot_name" elif dev_type == "7PIN_BK": self.power_on_tone = "小瑞来了！我们一起来愉快地聊天吧！" - elif dev_type == 'rabbit': - self.power_on_tone = "兔兔来了！我们一起来愉快地聊天吧！" - elif dev_type == "szbst": - self.power_on_tone = "深圳百事通来了！请问您有什么需要我帮忙的吗？" - elif dev_type == "momo": self.power_on_tone = "DuloDulo，我是上古神兽食梦貘，主人可以叫我momo，我来自《山海经》，是一只以梦为食的神秘灵兽，每当颜色朦胧、人们入睡时，我就会悄悄飞到人们的床头，用我长长的鼻子把噩梦统统吞噬掉，只留下甜甜的美梦。从今往后，就由我守护在主人身边，一起度过每一个星河璀璨的夜晚。" elif dev_type == "sjgq": self.power_on_tone = "您好！我是巴马四季甘泉的智能助手。我可以为你解答各种各样的问题，像是有关巴马四季甘泉知识库、中基长寿科学产品、卫斯曼院士国际先进医疗中心的问题，还有日常生活与工作、旅游信息、健康咨询、教育与学习、情感交流、全球资讯、世界文化等方面的问题哦。你有没有这方面的问题想要问我呀？" + elif dev_type == "szbst": + self.power_on_tone = "深圳百事通来了！请问您有什么需要我帮忙的吗？" else: self.power_on_tone = "您好！我是世界首创柒牌男装AI生活好帮手-bot_name， 很高兴为您服务，我可以帮您解答柒牌公司知识库、日常生活与工作、旅游信息、健康与运动咨询、教育与学习、情感交流、全球资讯、世界文化等问题！请问您有什么需要我帮忙的吗？" @@ -1228,39 +1230,49 @@ class AI_VC(Abstract): self.VC_play_TYPE = VC_play_TYPE_PCM #0:PCM 1:mp3 self.VC_record_samplerate = 16000 self.VC_play_samplerate = 16000 - + # ARG_sample_rate, // 采样率 + # ARG_channel, // 通道数 + # ARG_frame_size, // 每帧字节数 + # ARG_energy_threshold, // 能量阈值 + # ARG_voice_time, // 语音持续时间阈值(ms) + # ARG_silence_time, // 静音持续时间阈值(ms) + # ARG_silence_start_threshold, // 静音开始阈值(ms) + #self.vad = VAD.VAD(self.VC_record_samplerate ,1,6400,300,50,300,100) + self.speech_rate = EventMesh.publish("get_storage_config","speech_rate")#语速配置 + if self.speech_rate == None: + self.speech_rate = 0 self.VC_TTS_voice_type = EventMesh.publish("get_storage_config","VC_TTS_voice_type")#音色配置 if self.VC_TTS_voice_type == None or ("男" in self.VC_TTS_voice_type) == False: - if dev_type == "G10" or dev_type == "7PIN_BK"or dev_type == "szbst"or dev_type == 'rabbit': + if dev_type == "G10" or dev_type == "7PIN_BK"or dev_type == "szbst": self.VC_TTS_voice_type = { "女":{ "bot_name":"小瑞", - "voice":"huoshan::zh_female_wanwanxiaohe_moon_bigtts" , + "voice":"zh_female_wanwanxiaohe_moon_bigtts" , "lang":"zh-cn" }, "男":{ "bot_name":"小瑞", - "voice":"huoshan::zh_male_wennuanahu_moon_bigtts" , + "voice":"zh_male_wennuanahu_moon_bigtts" , "lang":"zh-cn" }, "ç«¥":{ "bot_name":"小瑞", - "voice":"huoshan::ICL_zh_female_huoponvhai_tob" , + "voice":"ICL_zh_female_huoponvhai_tob" , "lang":"zh-cn" }, "粤":{ "bot_name":"小瑞", - "voice":"huoshan::zh_female_wanqudashu_moon_bigtts" , + "voice":"zh_female_wanqudashu_moon_bigtts" , "lang":"zh-hk" }, "四川":{ "bot_name":"小瑞", - "voice":"huoshan::zh_female_daimengchuanmei_moon_bigtts", + "voice":"zh_female_daimengchuanmei_moon_bigtts", "lang":"&lang=zh-cn-sc" }, "个性":{ "bot_name":"小瑞", - "voice":"huoshan::zh_male_sunwukong_mars_bigtts" , + "voice":"zh_male_sunwukong_mars_bigtts" , "lang":"zh-cn" } @@ -1269,17 +1281,17 @@ class AI_VC(Abstract): self.VC_TTS_voice_type = { "女":{ "bot_name":"食梦貘", - "voice":"huoshan::zh_female_shaoergushi_mars_bigtts" , + "voice":"zh_female_shaoergushi_mars_bigtts" , "lang":"zh-cn" }, "男":{ "bot_name":"食梦貘", - "voice":"huoshan::zh_male_yangguangqingnian_mars_bigtts" , + "voice":"zh_male_yangguangqingnian_mars_bigtts" , "lang":"zh-cn" }, "ç«¥":{ "bot_name":"食梦貘", - "voice":"huoshan::ICL_zh_female_huoponvhai_tob" , + "voice":"ICL_zh_female_huoponvhai_tob" , "lang":"zh-cn" }, } @@ -1287,17 +1299,17 @@ class AI_VC(Abstract): self.VC_TTS_voice_type = { "女":{ "bot_name":"小巴", - "voice":"huoshan::zh_female_wanwanxiaohe_moon_bigtts" , + "voice":"zh_female_wanwanxiaohe_moon_bigtts" , "lang":"zh-cn" }, "男":{ "bot_name":"小巴", - "voice":"huoshan::zh_male_wennuanahu_moon_bigtts" , + "voice":"zh_male_wennuanahu_moon_bigtts" , "lang":"zh-cn" }, "ç«¥":{ "bot_name":"小巴", - "voice":"huoshan::ICL_zh_female_huoponvhai_tob" , + "voice":"ICL_zh_female_huoponvhai_tob" , "lang":"zh-cn" }, } @@ -1349,12 +1361,12 @@ class AI_VC(Abstract): #EventMesh.publish("set_storage_config",["VC_TTS_voice_type",self.VC_TTS_voice_type]) - try: self.VC_TTS_voice_key = EventMesh.publish("get_storage_config","VC_TTS_voice_key")#音色配置 if self.VC_TTS_voice_key == None : - if dev_type == "7PIN_BK" or dev_type == 'rabbit': + if dev_type == "7PIN_BK": self.VC_TTS_voice_key = "ç«¥" + #self.VC_TTS_voice_key = "男" elif dev_type == "momo": self.VC_TTS_voice_key = "男" else: @@ -1367,7 +1379,171 @@ class AI_VC(Abstract): self.VC_TTS_volume = EventMesh.publish("get_storage_config","VC_TTS_volume")#音量配置 except Exception as e: usys.print_exception(e) - + #配置词 + self.commands = [ + { + "command": "切换音色",#命令 + "patterns": [ #多种问法，说法 + "声音改成", "声音改为", "改成", "改为", "换成", "切换成", "变成", "声音" + "使用", "设置声音为", "把声音改成", "声音切换到", "用","声音切换的" + ], + "exact_matches": ["声音改成", "声音改为", "改成", "改为"],#精确匹配的完整短语 + "has_param": True,# 是否包含参数 + "param_regexp": r"(?P(男|难|女|ç«¥|同|粤|广东话|粤语|四川话|川普|四川口音))",#参数提取正则表达式 + "require_full_match": False,#是否要求完全匹配 + "negative_patterns": [],#否定模式（避免误触发） + "keyword_threshold": 0.6#关键词匹配阈值 + }, + { + "command": "音量加大", + "patterns": [ + "音量加大","音量大", "声音大" ,"声音加大","听不清楚","声音太小","太小声了" + ], + "exact_matches": [], + "has_param": False, + "param_regexp": "", + "require_full_match": False, + "negative_patterns": ["声音太大"], + "keyword_threshold": 1 + }, + { + "command": "音量减小", + "patterns": [ + "音量减小","音量小", "声音小","你太吵了","声音太大","太大声了" + ], + "exact_matches": [], + "has_param": False, + "param_regexp": "", + "require_full_match": False, + "negative_patterns": ["声音太小"], + "keyword_threshold": 0.8 + }, + { + "command": "关机", + "patterns": [ + "关机", "关闭电源" + ], + "exact_matches": ["关机", "关闭电源"], + "has_param": False, + "param_regexp": "", + "require_full_match": True, + "negative_patterns": [], + "keyword_threshold": 0.8 + }, + { + "command": "进入工厂模式", + "patterns": [ + "进入工厂模式" + ], + "exact_matches": ["进入工厂模式"], + "has_param": False, + "param_regexp": "", + "require_full_match": True, + "negative_patterns": [], + "keyword_threshold": 0.8 + }, + { + + "command": "再见", + "patterns": [ + "再见","拜拜","我要走了" + ], + "exact_matches": [], + "has_param": False, + "param_regexp": "", + "require_full_match": False, + "negative_patterns": [], + "keyword_threshold": 0.8 + }, + { + + "command": "查询信号", + "patterns": [ + "查询信号","信号怎样","信号是多少","信号怎么样" + ], + "exact_matches": [], + "has_param": False, + "param_regexp": "", + "require_full_match": False, + "negative_patterns": ["查询"], + "keyword_threshold": 0.8 + }, + { + + "command": "查询卡号", + "patterns": [ + "查询卡号","卡号","iccid号","ICCID号" + ], + "exact_matches": [], + "has_param": False, + "param_regexp": "", + "require_full_match": False, + "negative_patterns": ["查询"], + "keyword_threshold": 0.8 + }, + { + + "command": "查询设备号", + "patterns": [ + "设备号","查询设备号","查询IMEI号" + ], + "exact_matches": [], + "has_param": False, + "param_regexp": "", + "require_full_match": False, + "negative_patterns": ["查询"], + "keyword_threshold": 0.8 + }, + { + "command": "OTA升级", + "patterns": [ + "OTA升级" + ], + "exact_matches": ["OTA升级"], + "has_param": False, + "param_regexp": "", + "require_full_match": True, + "negative_patterns": [], + "keyword_threshold": 1 + }, + { + "command": "查询服务器地址", + "patterns": [ + "查询服务器地址" + ], + "exact_matches": [], + "has_param": False, + "param_regexp": "", + "require_full_match": False, + "negative_patterns": ["查询"], + "keyword_threshold": 0.9 + } + , + { + "command": "切换对话模式", + "patterns": [ + "切换对话模式","切换兑换模式" + ], + "exact_matches": [], + "has_param": False, + "param_regexp": "", + "require_full_match": False, + "negative_patterns": [], + "keyword_threshold": 0.8 + }, + { + "command": "查询版本", + "patterns": [ + "查询版本号","软件版本" + ], + "exact_matches": [], + "has_param": False, + "param_regexp": "", + "require_full_match": False, + "negative_patterns": ["查询"], + "keyword_threshold": 0.6 + }, + ] if self.VC_TTS_volume == None: self.VC_TTS_volume = 100 self.AI_audio_queue_thread_id = 0 @@ -1439,7 +1615,8 @@ class AI_VC(Abstract): self.audio_play_state = 1 #音频播放完状态 self.close_timerout = 0 self.LocalRespondingEnded_timerout = 0 - + + self.VC_Play_flag = 0 self.play_flag = 0 self.VC_record_state_flag = 0 self.AI_key_long_press_flag = 0 @@ -1492,7 +1669,7 @@ class AI_VC(Abstract): EventMesh.publish("set_storage_config",["powerRestart",0]) else: - + pass self.AI_TTS_str = self.power_on_tone.replace("bot_name", self.bot_name) print("开机提示：{}".format(self.AI_TTS_str)) EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_VC_start}) @@ -1605,6 +1782,8 @@ class AI_VC(Abstract): print("AI_key_press") self.get_config_timer_count = 0 self.key_state = 1 + self.Close() + #self.aud.aud_tone_play(10,200)#播放按键音 self.AI_TTS_str = ""#"你好，有什么吩咐" self.AI_ASR_str = "" @@ -1618,33 +1797,37 @@ class AI_VC(Abstract): self.VC_Speech_Start() self.AI_VC_start() + self.chenggong_tone() # self.Close() # self.AI_VC_start() self.AI_key_long_press_flag = 0 + def AI_key_release(self, event, msg):#按键弹起 print("AI_key_release123") self.key_state = 0 if self.AI_key_long_press_flag == 1 :#有长按 #self.AI_client_VC_end_speech() - EventMesh.publish("publish_AI_btnInfo",{"func":utime.sleep_ms,'data':(1000)}) - EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_client_asr_StopTranscription_requset}) + #EventMesh.publish("publish_AI_btnInfo",{"func":utime.sleep_ms,'data':(1000)}) + #EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_client_asr_StopTranscription_requset}) #self.AI_client_asr_StopTranscription_requset()#请求停止 if self.VC_mode == 1 :#长按单次问答模式 - if self.AI_ASR_str_tmp != "": - print("AI_key_release:asr_str:{}".format(self.AI_ASR_str)) - EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_VC_start}) + # if self.AI_ASR_str_tmp != "": + print("AI_key_release:asr_str:{}".format(self.AI_ASR_str)) + # EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_VC_start}) + self.AI_client_VC_end_speech() + pass else : if self.VC_mode == 1 :#长按单次问答模式 - # try: - # EventMesh.publish("publish_AI_btnInfo",{"func":self.Close}) - # except Exception as e: - # usys.print_exception(e) - # EventMesh.publish("publish_AI_btnInfo",{"func":self.key_prompt}) #提示请您按住中间的按钮开始讲话，讲完松开即可 - self.AI_TTS_str = "请您按住中间的按钮开始讲话，讲完松开即可 " - EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_VC_start}) - + try: + EventMesh.publish("publish_AI_btnInfo",{"func":self.Close}) + except Exception as e: + usys.print_exception(e) + EventMesh.publish("publish_AI_btnInfo",{"func":self.key_prompt}) #提示请您按住中间的按钮开始讲话，讲完松开即可 + #self.AI_TTS_str = "请您按住中间的按钮开始讲话，讲完松开即可 " + #EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_VC_start}) + pass #self.VC_Speech_End() def AI_key_long_press(self, event, msg):#按键长按 @@ -1661,472 +1844,7 @@ class AI_VC(Abstract): def VC_error(self, event, msg):#错误播报 self.websocket_error() #EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error}) -#****************************************ASR连接********************************************************************** - def websocket_connect_asr(self): - print("websocket_connect_asr:{} e:{}!!!!!!!!!!!!!!!!!!!!!".format(self.AI_client_asr_isconnect,self.AI_client_asr_debug)) - self.AI_ASR_text_repet_count = 0 - self.AI_ASR_check_timeout = 0 - self.AI_ASR_error_timeout = 0 - self.AI_ASR_text_tmp = "" - if self.AI_client_asr_isconnect == 1: - return - # 创建ASR客户端 - try: - signal = net.csqQueryPoll() - if signal == -1 or sim.getStatus() != 1:#网络正常 - self.websocket_error() - return - - # self.AI_client_asr = uwebsocket.Client.connect('wss://nls-gateway-cn-beijing.aliyuncs.com/ws/v1',headers=Headers, debug=False) - req_str = "" - platform = "platform=huoshan" - #platform= "platform=xfyun-ita" - platform= "platform=xfyun-bigmodel-ita" - lang = "lang=zh-cn"#普通话测试环境 - try: - if "loang" in self.VC_TTS_voice_type[self.VC_TTS_voice_key]: - lang = self.VC_TTS_voice_type[self.VC_TTS_voice_key]["lang"] - else: - print("not lang!!!!!!!!!!!!!!!!!!!!!!") - except Exception as e: - usys.print_exception(e) - req_str = "asr/ws?is_auth={}&{}&{}".format(self.is_auth,platform,lang) - print("req_str:{}".format(req_str)) - token = EventMesh.publish("get_dev_token") - print("token:{}".format(token)) - url = 'ws://{}/{}&dt={}'.format(self.server_ip,req_str,token) - print("websocket_connect_asr URL:{}".format(url)) - self.AI_client_asr = uwebsocket.Client.connect(url, debug=True)# - #self.AI_WS_connect_Queue.put(self.AI_client_asr) - - except Exception as e: #网络连接异常 - usys.print_exception(e) - - print("websocket_connect_asr error!!!\r\n") - #self.connect_timerout_popup(2) - str_error = str(e) - if str_error.find("[Errno 22] EINVAL")!= -1: - self.websocket_error() - print("[Errno 22] EINVAL!!!\r\n") - print("[Errno 22] EINVAL!!!\r\n") - print("[Errno 22] EINVAL!!!\r\n") - print("[Errno 22] EINVAL!!!\r\n") - EventMesh.publish("set_storage_config",["powerRestart",1])#调试临时关闭 - utime.sleep_ms(1000) - self.aud.stopPlayStream() - self.aud.stopAll() - Power.powerRestart() - - self.websocket_error() - - return - if self.AI_client_asr == None: - EventMesh.publish_async("update_dev_token",thread_size = 8192)#获取token - return - self.AI_client_asr_debug+=1 - utime.sleep_ms(10) - self.AI_client_asr.settimeout(5) - #开启发送线程 - if self.AI_ASR_send_data_queue_thread_id == 0: - _thread.stack_size(self.Audio_buf_size+8*1024) - self.AI_ASR_send_data_queue_thread_id = _thread.start_new_thread(self.AI_ASR_send_data_queue_thread, ()) - _thread.stack_size(8*1024) - print("AI_VC_start :AI_ASR_send_data_queue_thread_id :{} ".format(self.AI_ASR_send_data_queue_thread_id)) - print("websocket_connect_asr 222!!!\r\n") - self.AI_client_asr_recv_thread_close()#关闭读线程 - # 创建线程接收数据 - _thread.stack_size(self.Audio_buf_size+8*1024) - self.AI_client_asr_recv_id = _thread.start_new_thread(self.AI_client_asr_recv, (self.AI_client_asr,)) - _thread.stack_size(8*1024) - self.AI_client_asr_isconnect = 1 - self.AI_ASR_timer.start(1000, 1, self.AI_ASR_timer_cb) - self.AI_client_asr_start_requset()#开始请求 - print("websocket_connect_asr:OK!!!!!!!!!!!!!!!!!!!!!") - - def websocket_deconnect_asr(self): - print("websocket_deconnect_asr {}!!!!!!!".format(self.AI_client_asr_isconnect)) - print("websocket_deconnect_asr!!!!!!!") - self.AI_ASR_text_repet_count = 0 - self.AI_ASR_text_tmp = "" - self.AI_ASR_timer.stop() - #self.AI_ASR_send_data_queue_thread_close()#关闭发送线程 - self.AI_client_asr_recv_thread_close()#关闭读线程 - if self.AI_client_asr_isconnect == 1 or self.AI_client_asr!= None: - try: - self.AI_client_asr_isconnect = 0 - count = 500 - while True:#等待发完数据 - count-=1 - if count <=0 or self.AI_ASR_send_data_queue_thread_lock == 0: - break - print("等待发送完成：{}".format(count)) - if self.AI_ASR_send_data_queue.empty()!=True: - self.AI_ASR_send_data_queue.clear() - utime.sleep_ms(1) - - try: - self.AI_client_asr_StopTranscription_requset()#发送停止ASR - self.AI_client_asr.close() - except Exception as e: - usys.print_exception(e) - utime.sleep_ms(200)#再尝试关闭一次 - try: - self.AI_client_asr.close() - - except Exception as e: - usys.print_exception(e) - self.AI_client_asr_debug-=1 - try: - self.AI_client_asr._close() - except Exception as e: - usys.print_exception(e) - self.AI_client_asr = None - except Exception as e: - usys.print_exception(e) - print("websocket_deconnect_asr {}!!!!!!!".format(self.AI_client_asr_isconnect)) - - #websocket发送数据 - def AI_client_asr_send(self,data_str): - # if isinstance(data_str, str):#是字符串 - # print("ASR ASR AI_client_asr_send:{}".format(data_str)) - # pass - # else: - # pass - # #print("ASR ASR AI_client_asr_send:len:{}".format(len(data_str))) - self.AI_ASR_send_data_queue.put([AI_client_asr_num,data_str]) - - def AI_client_asr_ping(self): - pass - if self.AI_client_asr_isconnect == 1: - #self.AI_client_asr.write_frame(0x09,b'ping ping') - self.AI_client_asr_send("ping") - def set_AI_ASR_flag(self,data): - self.AI_ASR_flag = data - def AI_ASR_timer_cb(self, *args): # - EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_ASR_timer_cb_}) - - def AI_ASR_timer_cb_(self): - if self.AI_ASR_check_timeout > 0: - print("AI_ASR_check_timeout:{}".format(self.AI_ASR_check_timeout)) - self.AI_ASR_check_timeout-=1 - if self.AI_ASR_check_timeout <=0: - self.AI_ASR_check_timeout = 0 - self.AI_ASR_error_timeout = 3#发送停止后3秒后没正式结果就提示 - #EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_client_asr_StopTranscription_requset}) - - if self.AI_ASR_error_timeout>0: - self.AI_ASR_error_timeout-=1 - print("AI_ASR_error_timeout:{}".format(self.AI_ASR_error_timeout)) - if self.AI_ASR_error_timeout <= 0: - self.AI_ASR_error_timeout = 0 - #if self.AI_ASR_str == "": - if self.AI_ASR_str_tmp == "" and self.AI_ASR_str == "" and self.AI_ASR_text_tmp == "": - self.AI_TTS_str = "没有识别到您说话,请重试" - else : - self.AI_ASR_str = self.AI_ASR_str_tmp + self.AI_ASR_str+self.AI_ASR_text_tmp - self.AI_ASR_str_tmp = self.AI_ASR_str - print("AI_ASR_error_timeout2:重连:{}!!!!!!!!!!\r\n".format(self.AI_ASR_str) ) - print("AI_ASR_error_timeout2:重连:{}!!!!!!!!!!\r\n".format(self.AI_ASR_str) ) - print("AI_ASR_error_timeout2:重连:{}!!!!!!!!!!\r\n".format(self.AI_ASR_str) ) - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_deconnect_asr}) - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_connect_asr}) #重连ASR - #EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_End}) #重连ASR - EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_VC_start}) - EventMesh.publish("publish_AI_btnInfo",{"func":self.set_AI_ASR_flag,"data":1})#设置以打断过一次 - if self.VC_record_state_flag > 1: - self.VC_record_state_flag =1 - pass - #关闭ASR数据发送线程 - def AI_ASR_send_data_queue_thread_close(self): - if self.AI_ASR_send_data_queue_thread_id != 0: - count =500 - while True: - count -= 1 - if self.AI_ASR_send_data_queue_thread_lock == 0 or count <=0: - break - else: - self.AI_ASR_send_data_queue_thread_lock = 2 - print("AI_ASR_send_data_queue_thread going:{}...".format(count)) - utime.sleep_ms(1) - self.AI_ASR_send_data_queue_thread_lock = 0 - print("AI_ASR_send_data_queue_thread_close") - try: - _thread.stop_thread(self.AI_ASR_send_data_queue_thread_id) - except Exception as e: - usys.print_exception(e) - self.AI_ASR_send_data_queue_thread_id = 0 - #发送数据队列 - def AI_ASR_send_data_queue_thread(self): - print("AI_ASR_send_data_queue_thread:start") - count_t = 0 - while True: - if self.AI_ASR_send_data_queue_thread_lock == 2: - print("AI_ASR_send_data_queue_thread:该退出了") - break - if self.AI_ASR_send_data_queue.empty() :#不能去掉，去掉会卡死 - self.AI_ASR_send_data_queue_thread_lock = 0 - count_t+=1 - utime.sleep_ms(1) - else: - #data[0]是websocket连接对象编号，data[1]是数据 - #print("AI_ASR_send_data_queue_thread:get") - - self.AI_ASR_send_data_queue_thread_lock = 1 - data = self.AI_ASR_send_data_queue.get() - #print("AI_ASR_send_data_queue_thread:num{} len{}".format(data[0],len(data[1]))) - if data != None: - try: - # 发送数据 - if data[0] == AI_client_asr_num: - self.AI_ASR_send_buf_len -= len(data[1]) - self.asr_pingping_timerout = 0 - if self.AI_client_asr != None and self.AI_client_asr_isconnect == 1: - self.AI_client_asr.send(data[1]) - if isinstance(data[1], str):#是字符串 - print("ASR ASR: send_data:{}".format(data[1])) - # else: - # print("ASR ASR: send_data!:{}".format(len(data[1]))) - else: - if isinstance(data[1], str):#是字符串 - print("ASR ASR: send_data error!!!:{}".format(data[1])) - else: - print("ASR ASR: send_data error!!!:{}".format(len(data[1]))) - - except Exception as e: - usys.print_exception(e) - if data[0] == AI_client_asr_num: - if self.AI_client_asr_isconnect != 0: - print("ASR发送数据错误，退出") - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"ASR发送出错了，请重试"}) - self.ud_error(["asr",self.asr_taskid]) - #EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_connect_asr}) #重连 - utime.sleep_ms(1) - self.AI_ASR_send_data_queue_thread_lock = 0 - - #ASR关闭读线程 - def AI_client_asr_recv_thread_close(self): - if self.AI_client_asr_recv_id != 0: - count =500 - while True: - count -= 1 - if self.AI_client_asr_recv_lock == 1: - self.AI_client_asr_recv_lock = 2 - if self.AI_client_asr_recv_lock == 0 or count <=0: - break - else: - print("AI_client_asr_recv_thread going:{}...".format(count)) - utime.sleep_ms(1) - self.AI_client_asr_recv_lock = 0 - print("AI_client_asr_recv_thread_close") - try: - _thread.stop_thread(self.AI_client_asr_recv_id) - except Exception as e: - usys.print_exception(e) - self.AI_client_asr_recv_id = 0 - - #Ai ASR websocket返回数据 - def AI_client_asr_recv(self,cli): - while True: - try: - # 死循环接收数据 - self.AI_client_asr_recv_lock = 0 - utime.sleep_ms(1) - recv_data = cli.recv() - if self.AI_client_asr_recv_lock == 2:#要关闭线程了 - print("AI_client_asr_recv 该退出了") - break - self.AI_client_asr_recv_lock = 1 - self.close_timerout = 0#服务器有返回清零退出时间 - if recv_data == "websocket close": - # 服务器关闭连接 - print("ASR 服务器主动关闭连接!!!!!!!!!!!!!!!!!!!!!!!!!!!") - - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"ASR连接断开了，请重试"}) - self.ud_error(["asr",self.asr_taskid]) - break - elif recv_data == "":#客户端关闭连接 - print("ASR ASR :客户端关闭连接") - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"ASR连接断开了，请重试"}) - self.ud_error(["asr",self.asr_taskid]) - break - try: - if isinstance(recv_data, str): - jsload_data = ujson.loads(recv_data) - print("ASR ASR recv_data: {}".format(jsload_data)) - if jsload_data["status"] != 20000000:#返回错误 - print("ASR返回非20000000状态码，退出") - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"ASR出错了，请重试"}) - self.ud_error(["asr",self.asr_taskid]) - break - - if jsload_data["task_id"] != None: - self.asr_taskid = jsload_data["task_id"] - if jsload_data["event"] != None: - if jsload_data["event"] == "TranscriptionStarted": #开始录音 - self.VC_record_state_flag = 2 - self.VC_state = VC_State_SpeechStart - EventMesh.publish("open_led_blue") - self.AI_ASR_error_timeout = 0 - - pass - - elif jsload_data["event"] == "Changed" or jsload_data["event"] == "SentenceEnd" : #开始录音Changed - # if self.VC_record_state_flag == 2: - # - - self.AI_ASR_timer_count = 0 - if jsload_data["result"]["result"]: - #self.AI_listen_text = str(jsload_data["result"]["result"]) - text= str(jsload_data["result"]["result"]) - print("ASR recv self.VC_state :{}".format(self.VC_state)) - if self.is_long_record == 0: - if self.VC_state >= VC_State_TTSStart:#开始播音后不响应asr结果 - print("asr continue") - continue - EventMesh.publish("open_led_blue") - - if jsload_data["result"]["is_end"] == True: #asr结束 - self.AI_ASR_check_timeout = 0 - self.AI_ASR_error_timeout = 0 - self.AI_ASR_text_repet_count = 0 - self.AI_ASR_text_tmp = "" - self.VC_record_state_flag = 4 - ret= self.check_text(text) - if ret == 0:#配置指令 - - if self.AI_VC_send_data_queue.empty()!=True: - self.AI_VC_send_data_queue.clear() - if self.AI_ASR_send_data_queue.empty()!=True: - self.AI_ASR_send_data_queue.clear() - self.AI_ASR_send_buf_len = 0 - EventMesh.publish("open_led_all") - try: - - EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_End}) - #print("AI_client_asr_recv VC_Speech_End") - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_deconnect_VC}) - #EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_deconnect_asr}) - EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_VC_start}) - except Exception as e: - usys.print_exception(e) - continue - elif ret == 1:#不处理 - pass - else:#正常asr - if self.VC_mode == 0 :#连续问答模式 - self.AI_ASR_str = text# - if self.AI_ASR_flag == 0:#d第一次识别，没有打断 - self.AI_ASR_str_tmp = self.AI_ASR_str - else:#第二次识别，打断了 - self.AI_ASR_str = self.AI_ASR_str_tmp+text - print("ASR ASR 打断 续集 AI_ASR_str {}".format(self.AI_ASR_str)) - self.AI_ASR_str_tmp = self.AI_ASR_str - if self.is_long_record != 0: - #if self.VC_state >= VC_State_TTSStart:#如果已经播放了，关闭VC连接，重新打开，处理音频数据太多问题 - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_deconnect_VC})#关闭vc连接 - self.AI_ASR_flag = 1 - print("asr recv :asr_str:{}".format(self.AI_ASR_str)) - #EventMesh.publish("publish_AI_btnInfo",{"func":utime.sleep_ms,"data":(500)}) #等一秒 - EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_VC_start}) - elif self.VC_mode == 1:#单对话模式 - self.AI_ASR_str += text# - self.AI_ASR_str_tmp = self.AI_ASR_str - if self.key_state == 0:#按键已经弹起 - self.AI_ASR_str_tmp ="" - print("asr recv :asr_str:{}".format(self.AI_ASR_str)) - #EventMesh.publish("publish_AI_btnInfo",{"func":utime.sleep_ms,"data":(500)}) #等一秒 - EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_VC_start}) - self.VC_state = VC_State_SpeechEnd - elif jsload_data["result"]["is_end"] == False: - - self.VC_state = VC_State_SpeechIng - if self.AI_ASR_text_tmp != "" and self.AI_ASR_text_tmp == text: - pass - # self.AI_ASR_text_repet_count+=1 - # if self.AI_ASR_text_repet_count>=40:#重复30次下发一样结果，就报错 - # print("asr 错误，无正式结果:{}".format(text)) - # #EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error})# - # EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"ASR出错了，请重试"}) - else: - self.AI_ASR_check_timeout = 5 #5秒没更新结果，强制stop - self.AI_ASR_error_timeout = 0 - self.AI_ASR_text_repet_count = 0 - if self.VC_mode == 0 :#连续问答模式 - self.AI_ASR_text_tmp = text - self.VC_record_state_flag = 3 - if self.AI_ASR_flag ==1 :#如果已经识别过一次asr - self.AI_ASR_flag = 2 - print("asr 打断,关闭VC连接") - self.AI_ASR_str = "" - if self.is_long_record != 0: - self.VC_Play_End() - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_deconnect_VC})#关闭vc连接 - # EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_connect_VC})#连接服务器 - # EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_client_VC_start_requset})#请求 - EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_VC_start}) - EventMesh.publish("publish_AI_btnInfo",{"func":EventMesh.publish,"data":"open_led_blue"})# - elif self.VC_mode ==1:#单对话模式 - self.AI_ASR_str_tmp ="" - - except Exception as e: - usys.print_exception(e) - - except Exception as e: - error_str = str(e) - print("ASR ASR:e:{}".format(error_str)) - #usys.print_exception(e) - if error_str.find("[Errno 110]")!= -1: - #print("ASR ASR:recv TIMEDOUT 正常") - pass - else: - usys.print_exception(e) - print("{}".format(cli)) - #self.websocket_error() - print("ASR读数据报错，退出") - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"ASR出错了，请重试"})# - self.ud_error(["asr",self.asr_taskid]) - break - self.AI_client_asr_isconnect = 0 - self.AI_client_asr_recv_id = 0 - self.AI_client_asr_recv_lock = 0 - print("AI_client_asr_recv END!!!!!!!!!!!!!!!!!!!!!!!!!!!") - #ASR开始会话 - def AI_client_asr_start_requset(self): - self.session_id = self.get_session_id() - json_data = \ - { - "header":{ - "event":"StartTranscription", - "trace_id":self.get_trace_id() - }, - "payload":{ - "format":"pcm", - "sample_rate":self.VC_record_samplerate, - "session_id": self.session_id , - } - - } - if self.VC_record_TYPE == VC_record_TYPE_AMR : - json_data["payload"]["format"]='amr' - pass - self.AI_ASR_error_timeout = 5 - self.AI_client_asr_send(ujson.dumps(json_data)) - #ASR终止 - def AI_client_asr_StopTranscription_requset(self): - print("AI_client_asr_StopTranscription_requset!!!!!") - json_data = \ - { - "header":{ - "event":"StopTranscription", - "trace_id":self.get_trace_id() - } - } - - #self.AI_client_asr_send(ujson.dumps(json_data)) - if self.AI_client_asr != None and self.AI_client_asr_isconnect == 1: - self.AI_client_asr.send(ujson.dumps(json_data)) - #开始录音 def VC_Speech_Start(self): try: @@ -2137,9 +1855,7 @@ class AI_VC(Abstract): return if self.VC_record_state_flag == 0: - if dev_type != "7PIN" : - EventMesh.publish("publish_AI_btnInfo",{"func":self.chenggong_tone}) - self.websocket_connect_asr() + self.VC_record_thread_close()#关闭录音线程 _thread.stack_size(self.Audio_buf_size+8*1024) self.VC_record_thread_id = _thread.start_new_thread(self.VC_record_thread,()) @@ -2169,14 +1885,15 @@ class AI_VC(Abstract): # print("self.record_pcm.stream_read_start:ret:{}".format(ret)) except Exception as e: usys.print_exception(e) - - self.VC_record_state_flag = 1 + if self.VC_mode == 1 : + self.VC_record_state_flag = 1 + else: + self.VC_record_state_flag = 2 except Exception as e: self.websocket_error() self.VC_record_state_flag = 1 self.VC_Speech_End()#关闭录音 - usys.print_exception(e) - + usys.print_exception(e) #停止录音 def VC_Speech_End(self): try: @@ -2218,10 +1935,7 @@ class AI_VC(Abstract): self.VC_record_thread_close()#关闭录音线程 if self.AI_record_tmp_queue.empty() != True: self.AI_record_tmp_queue.clear() - try: - self.websocket_deconnect_asr() - except Exception as e: - usys.print_exception(e) + self.VC_record_state_flag = 0 print("VC_Speech_End VC_record_state_flag:{}!!!!!!!!".format(self.VC_record_state_flag)) #关闭录音线程 @@ -2272,7 +1986,7 @@ class AI_VC(Abstract): Record_len = 0 if self.VC_record_TYPE == VC_record_TYPE_PCM:#pcm Record_len = self.record.stream_read(Record_read_buf, self.Audio_buf_size) - utime.sleep_ms(20) + #utime.sleep_ms(60) else : @@ -2283,6 +1997,8 @@ class AI_VC(Abstract): #print("VC_record_thread:{}".format(Record_len)) if Record_len > 0: read_buf = bytes(Record_read_buf[:Record_len]) + # ret = self.vad.custom_vad_process(Record_read_buf,Record_len) + # print("VAD检测：datalen:{},结果：{}".format(Record_len,ret)) if self.record_data_save_flag == 1: if self.record_fd == None: try: @@ -2297,8 +2013,7 @@ class AI_VC(Abstract): # if count >150*1024: # self.record_fd.close() #self.record_fd = None - # if self.VC_record_state_flag >= 2: - # self.AI_client_asr_send(read_buf) + if self.VC_record_state_flag <=1 :#没发送前先存起来 self.AI_record_tmp_queue.put(read_buf) @@ -2309,8 +2024,9 @@ class AI_VC(Abstract): if self.AI_record_tmp_queue.empty() == True: break else: - self.AI_client_asr_send(self.AI_record_tmp_queue.get()) #发送之前存的数据 - self.AI_client_asr_send(read_buf) + pass + self.AI_client_VC_send(self.AI_record_tmp_queue.get()) #发送之前存的数据 + self.AI_client_VC_send(read_buf) #print("VC_record_thread:{}".format(len(read_buf))) #count+=len(read_buf) @@ -2361,17 +2077,11 @@ class AI_VC(Abstract): platform = "platform=huoshan&is_boot=1" #platform = "platform=raisound" #platform = "platform=aliyun" - elif dev_type == "7PIN_BK" : + elif dev_type == "7PIN_BK": platform = "platform=huoshan" #platform = "platform=raisound&bot_type=dify" - #platform = "platform=raisound&bot_type=fourseason_water" - #platform = "platform=raisound&bot_type=szbst" - + platform = "platform=raisound&bot_type=fourseason_water" pass - elif dev_type == "szbst": - platform = "platform=raisound&bot_type=szbst" - elif dev_type == 'rabbit': - platform = "platform=raisound&bot_type=rabbit" elif dev_type == 'momo':# #22080/voice-chat/v2.1/ws?platform=raisound&bot_type=deepseek platform = "platform=raisound&bot_type=shimeng_mo" @@ -2396,14 +2106,19 @@ class AI_VC(Abstract): print("token:{}".format(token)) if dev_type == 'momo' :# req_str = "voice-chat/v2.1/ws?is_auth={}&{}&{}".format(self.is_auth,platform,lang) - elif dev_type == '7PIN_BK'or dev_type == "szbst" or dev_type == 'rabbit': - req_str = "voice-chat/v2.1/ws?is_auth={}&{}&{}".format(self.is_auth,platform,lang) - #req_str = "voice-chat/ws?is_auth={}&{}&{}".format(self.is_auth,platform,lang) + elif dev_type == '7PIN_BK': + #req_str = "voice-chat/v2.1/ws?is_auth={}&{}&{}".format(self.is_auth,platform,lang) + req_str = "voice-chat/ws?is_auth={}&{}&{}".format(self.is_auth,platform,lang) elif dev_type == "sjgq": req_str = "22085/voice-chat/ws?is_auth={}&{}&{}".format(self.is_auth,platform,lang) + elif dev_type == "G10": + req_str = "vc/chat?is_auth={}".format(self.is_auth) else: req_str = "voice-chat/ws?is_auth={}&{}&{}".format(self.is_auth,platform,lang) + req_str = "vc/chat?is_auth={}".format(self.is_auth) url = 'ws://{}/{}&dt={}'.format(self.server_ip,req_str,token) + #url = 'ws://115.190.94.208:22080/vc/coze?dt={}'.format(token) + print("VC VC:url{}".format(url)) self.AI_client_VC = uwebsocket.Client.connect(url, debug=True) #self.AI_WS_connect_Queue.put(self.AI_client_VC) @@ -2530,7 +2245,7 @@ class AI_VC(Abstract): if self.AI_client_VC_isconnect == 1: #self.AI_client_VC.write_frame(0x09,b'ping ping') self.AI_client_VC_send("ping") - #print("websocket ping ping") + print("VC: ping ping") #websocket发送数据 def AI_client_VC_send(self,data_str): @@ -2593,14 +2308,16 @@ class AI_VC(Abstract): #print("AI_VC_send_data_queue_thread:len:{}".format(len(data[1]))) #print("AI_VC_send_data_queue_thread on line asr recv :{} vc recv:{}".format(_thread.threadIsRunning(self.AI_client_asr_recv_id),_thread.threadIsRunning(self.AI_client_VC_recv_id))) if data[0] == AI_client_VC_num: - self.VC_pingping_timerout = 0 + if isinstance(data[1], str):#是字符串 + self.VC_pingping_timerout = 0 print("VC VC: send_data:{}".format(data[1])) - #print("VC: send_data len:{}".format(len(data[1]))) + else: + print("VC: send_data len:{}".format(len(data[1]))) if self.AI_client_VC != None and self.AI_client_VC_isconnect == 1: self.AI_client_VC.send(data[1]) else: - print("VC VC: send_data:ERROR:{}".format(data[1])) + print("VC VC: send_data:ERROR:{}".format(len(data[1]))) #print("VC: send_data end") #print("VC: send_data end") @@ -2645,14 +2362,13 @@ class AI_VC(Abstract): #Ai VC websocket返回数据 def AI_client_VC_recv(self,cli): count = 0 - + asr_led_flag = 0 + tts_led_flag = 0 while True: try: # 死循环接收数据 self.AI_client_VC_recv_lock = 0 - if self.is_VC_ACK == 1: - self.AI_client_VC_Read() - #EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_client_VC_Read}) + count+=1 if self.VC_state >= VC_State_TTSStart and (self.tts_in_data_len - self.tts_out_data_len > 64*1024 or count>5): count = 0 @@ -2687,133 +2403,119 @@ class AI_VC(Abstract): print("VC VC:recv_data:{}".format(recv_data)) jsload_data = ujson.loads(recv_data) #print("AI_client_VC_recv jsload_data = {}".format(jsload_data)) - if jsload_data["status"] != 20000000:#返回错误 - print("VC VC :服务器返回非20000000状态码，退出") - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"VC出错了，请重试"}) - self.ud_error(["VC",self.VC_taskid]) - break - if jsload_data["task_id"] != None: - self.VC_taskid = jsload_data["task_id"] - if jsload_data["event"] == "Started" :#会话开始 - payload = jsload_data["payload"] - self.VC_dialog_id = payload["dialog_id"] - self.VC_task_id = jsload_data["task_id"] + #self.LocalRespondingEnded_timerout = 0 + if jsload_data["event"] == "started" :#会话开始 + #payload = jsload_data["payload"] + # self.VC_dialog_id = payload["dialog_id"] + # self.VC_task_id = jsload_data["task_id"] + EventMesh.publish("open_led_blue") print("VC:Started!!!!!!!!\r\n") - elif jsload_data["event"] == "SpeechContent" :#asr转译 - print("VC:SpeechContent!!!!!!!!\r\n") - payload = jsload_data["payload"] - - - if(payload['finished'] == True):#asr结束 - self.VC_state = VC_State_SpeechEnd - #self.VC_Speech_End()#停止录音 - text = payload["text"] - if self.check_text(text) == 0: - if self.AI_VC_send_data_queue.empty()!=True: - self.AI_VC_send_data_queue.clear() - if self.AI_ASR_send_data_queue.empty()!=True: - self.AI_ASR_send_data_queue.clear() - self.AI_ASR_send_buf_len = 0 - EventMesh.publish("open_led_all") - try: - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_deconnect_VC}) - EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_End}) - - # EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_deconnect_asr}) - EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_VC_start}) - except Exception as e: - usys.print_exception(e) + if self.AI_TTS_str == "": + #self.AI_client_VC_send_speech()#开始上传语音 + EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_client_VC_send_speech}) + else: + EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_client_VC_synthesis}) - pass - else: - pass - - elif jsload_data["event"] == "SpeechEnded" :#服务器下发ASR说话结束 - self.VC_state = VC_State_SpeechEnd - elif jsload_data["event"] == "RespondingContent" :#chat回答 + elif jsload_data["event"] == "asr.started" :#asr开始 + self.VC_state = VC_State_SpeechStart + if self.VC_mode == 1 : + self.VC_record_state_flag = 2 + EventMesh.publish("open_led_blue") + elif jsload_data["event"] == "asr.result" :#asr结果 + self.AI_ASR_str = jsload_data["payload"]["content"] + if asr_led_flag == 0: + asr_led_flag = 1 + EventMesh.publish("close_led") + else : + asr_led_flag = 0 + EventMesh.publish("open_led_blue") + elif jsload_data["event"] == "asr.completed" :#asr 终止的标识 + #self.VC_Speech_End() + #EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_End}) + if self.record_data_save_flag == 1: + if self.record_fd : + self.record_fd.close() + self.record_fd = None + # if dev_type != "7PIN" : + #EventMesh.publish("publish_AI_btnInfo",{"func":self.send_tone}) + elif jsload_data["event"] == "vad.silence" :#VAD 终止的标识 + pass + #EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_End}) + elif jsload_data["event"] == "llm.created" :#会话开始 + EventMesh.publish("open_led_red_green")# + self.VC_state = VC_State_Thinking + # EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Play_Start}) pass + elif jsload_data["event"] == "tts.started" :#tts开始 - elif jsload_data["event"] == "RespondingStarted" :#音频开始下发 - self.play_flag = 0 - self.VC_state = VC_State_TTSStart - if self.is_long_record == 0: - EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_End})# - else: - self.AI_ASR_str_tmp = "" - self.AI_ASR_str = "" + EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_End}) EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Play_Start}) - self.VC_Thinking_timerout = 0#关闭超时重试 - # self.VC_Play_Start() - EventMesh.publish("open_led_green")#绿色 - #self.tts_fd = open('/usr/tts_tesr.pcm','w') - if self.play_data_save_flag == 1: - try: - uos.remove('/usr/play_data.pcm') - except Exception as e: - usys.print_exception(e) - self.play_fd = open('/usr/play_data.pcm','w') + #self.VC_Play_Start() + EventMesh.publish("open_led_green")#绿色 + elif jsload_data["event"] == "tts.completed" :#tts结束 + EventMesh.publish("open_led_green")#绿色 + EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Play_Start}) + self.LocalRespondingEnded_timerout = 8 + self.VC_state = VC_State_TTSEnd pass - elif jsload_data["event"] == "RespondingEnded" :#音频下发结束 - #if self.VC_state == VC_State_TTSStart: + # self.AI_client_VC_send_speech()#开始上传语音 + # self.VC_record_state_flag = 2 + elif jsload_data["event"] == "completed" :#会话结束 + #EventMesh.publish("open_led_green")#绿色 - print("RespondingEnded!!!!!!" ) - self.VC_state = VC_State_TTSEnd - self.LocalRespondingEnded_timerout = 10 #最长倒数10秒发送音频结束或者终止对话 - if self.VC_mode == 0 :#连续问答模式 - #EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_End})#调试临时打开 - if dev_type != "7PIN_BK": - EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_Start})#开始录音 - pass - if self.play_data_save_flag == 1: - if self.play_fd !=None: - self.play_fd.close() - #self.tts_fd.close() - #pass - elif jsload_data["event"] == "DialogStateChanged" :#状态切换事件 - payload = jsload_data["payload"] - if payload["state"] == "Listening":#可以说了 - self.VC_state = VC_State_Listening - if self.AI_TTS_str != "" or self.AI_ASR_str != "":#有tts数据,则请求TTS数据，没有就请求开始上传语音 - self.AI_client_VC_RequestToRespond() - else: - pass - #self.VC_Speech_Start()#开启线程获取录音数据 - if self.is_long_record == 0: - EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_Start})# - #self.AI_client_VC_send_speech()#开始上传语音 - - #EventMesh.publish("open_led_red") - if payload["state"] == "Thinking":#思考 - self.VC_state = VC_State_Thinking - if self.VC_Thinking_timerout == 0: - self.VC_Thinking_timerout = 15#倒计时15秒，如果没有音频数据返回就重试 - EventMesh.publish("open_led_red_green")#黄色 - if payload["state"] == "Responding":#开始音频播放 - #self.VC_state = VC_State_Responding# - pass - elif jsload_data["event"] == "RequestAccepted" :#请求上传语音被允许 - #self.VC_state = VC_State_Responding - #self.AI_client_VC_send_speech()#开始上传语音 + self.LocalRespondingEnded_timerout = 8 + self.VC_state = VC_State_End pass + # self.AI_client_VC_send_speech()#开始上传语音 + # self.VC_record_state_flag = 2 + elif jsload_data["event"] == "command" :#命令匹配 + ret = -1 + try: + if self.is_llm_commands ==1: + if len(jsload_data["payload"]["functions"])>0: + + cmd = jsload_data["payload"]["functions"][0]["function"]["arguments"] + print("llm:cmd:{}".format(cmd)) + cmd = ujson.loads(cmd) + ret = self.check_text_llm(cmd) + else: + + cmd = jsload_data["payload"] + print("server:cmd:{}".format(cmd)) + ret = self.check_text(cmd) + except Exception as e: + usys.print_exception(e) + if ret == -1: + if self.AI_TTS_str =="": + self.AI_TTS_str = "抱歉，没找到对应操作，可以换个说法试试" + #EventMesh.publish("publish_AI_btnInfo",{"func":utime.sleep_ms,'data':(10)}) + EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_End}) + EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_deconnect_VC}) + EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_VC_start}) else:#下发音频数据 #self.AI_ASR_str = ""#下发音频后清掉asr结果 + if tts_led_flag == 0: + tts_led_flag = 1 + EventMesh.publish("open_led_green")#绿色 + else: + tts_led_flag = 0 + EventMesh.publish("close_led")# print("下发音频数据1:{}".format(len(recv_data))) if self.play_data_save_flag == 1: if self.play_fd !=None: self.play_fd.write(recv_data) # print("下发音频数据2:{}".format(len(recv_data))) - if self.VC_state >= VC_State_TTSStart: + #if self.VC_state >= VC_State_TTSStart: #self.put_record_data([recv_data,""]) - EventMesh.publish("publish_AI_btnInfo",{"func":self.put_record_data,"data":[recv_data,""]}) + EventMesh.publish("publish_AI_btnInfo",{"func":self.put_record_data,"data":[recv_data,""]}) utime.sleep_ms(1) except Exception as e: usys.print_exception(e) except Exception as e: - error_str = str(e) #print("VC VC:e:{}".format(error_str)) if error_str.find("[Errno 110] ETIMEDOUT")!= -1: @@ -2835,13 +2537,14 @@ class AI_VC(Abstract): ret=self.AI_audio_queue.put(data) self.tts_in_data_len += len(data[0]) #print("下发音频数据2:{},{}".format(len(data[0]),self.tts_in_data_len)) - def check_text(self,text): - if len(text)>60: - return -1 + def check_text(self,cmd): + # if len(text)>60: + # return -1 try: - if isinstance(text, str): - if text.find("声音") != -1 and text.find("改") != -1: + text = cmd["command"] + if text.find("切换音色") != -1 : try: + text = cmd["params"]["cmd_parameter"] for key in self.VC_TTS_voice_type: if text.find(key) != -1 : self.VC_TTS_voice_key = key @@ -3005,13 +2708,225 @@ class AI_VC(Abstract): self.AI_TTS_str = "当前已是最新版本" return 0 - if len(text)<12 and (text.find("再见") != -1 or text.find("拜拜") != -1): + if len(text)<20 and (text.find("再见") != -1 or text.find("拜拜") != -1 or text.find("关闭对话") != -1): + #EventMesh.publish("publish_AI_btnInfo",{"func":self.Close}) + self.AI_TTS_str = "好嘞，再见，我们下次再聊" + self.Close_flag = 1 + return 0 + if len(text)<12 and (text.find("关机") != -1 ): EventMesh.publish("publish_AI_btnInfo",{"func":self.Close}) + EventMesh.publish("Power_off") return 1 except Exception as e: usys.print_exception(e) return -1 + def check_text_llm(self,cmd): + try: + text = cmd["command"]["chinese_description"] + if text.find("切换音色") != -1: + text = cmd["command"]["params_json"] + try: + for key in self.VC_TTS_voice_type: + if text.find(key) != -1 : + self.VC_TTS_voice_key = key + EventMesh.publish("set_storage_config",["VC_TTS_voice_key",self.VC_TTS_voice_key]) + self.AI_TTS_str = "了解，声音已切换" + return 0 + except Exception as e: + usys.print_exception(e) + if text.find("男") != -1 : + if "男" in self.VC_TTS_voice_type: + self.VC_TTS_voice_key = "男" + EventMesh.publish("set_storage_config",["VC_TTS_voice_key",self.VC_TTS_voice_key]) + + self.AI_TTS_str = "了解，声音已切换" + return 0 + if text.find("女") != -1 : + if "女" in self.VC_TTS_voice_type: + self.VC_TTS_voice_key = "女" + EventMesh.publish("set_storage_config",["VC_TTS_voice_key",self.VC_TTS_voice_key]) + + self.AI_TTS_str = "了解，声音已切换" + return 0 + if text.find("同声") != -1 or text.find("ç«¥") != -1 : + if "ç«¥" in self.VC_TTS_voice_type: + self.VC_TTS_voice_key = "ç«¥" + EventMesh.publish("set_storage_config",["VC_TTS_voice_key",self.VC_TTS_voice_key]) + + self.AI_TTS_str = "了解，声音已切换" + return 0 + if text.find("粤") != -1 or text.find("越野") != -1 or text.find("广东话") != -1: + if "粤" in self.VC_TTS_voice_type: + self.VC_TTS_voice_key = "粤" + EventMesh.publish("set_storage_config",["VC_TTS_voice_key",self.VC_TTS_voice_key]) + + self.AI_TTS_str = "了解，声音已切换" + return 0 + if text.find("四川") != -1 : + if "四川" in self.VC_TTS_voice_type: + self.VC_TTS_voice_key = "四川" + EventMesh.publish("set_storage_config",["VC_TTS_voice_key",self.VC_TTS_voice_key]) + + self.AI_TTS_str = "了解，声音已切换" + return 0 + if text.find("个性") != -1 : + if "个性" in self.VC_TTS_voice_type: + self.VC_TTS_voice_key = "个性" + EventMesh.publish("set_storage_config",["VC_TTS_voice_key",self.VC_TTS_voice_key]) + + self.AI_TTS_str = "了解，声音已切换" + return 0 + if text.find("英语") != -1 : + if "英语" in self.VC_TTS_voice_type: + self.VC_TTS_voice_key = "英语" + EventMesh.publish("set_storage_config",["VC_TTS_voice_key",self.VC_TTS_voice_key]) + + self.AI_TTS_str = "了解，声音已切换" + return 0 + self.AI_TTS_str = "很抱歉，没找到对应声音" + return 0 + if text.find("切换对话模式") != -1: + if self.VC_mode == 0: + self.VC_mode = 1 + self.AI_TTS_str = "了解，已切换到单对话模式" + else: + self.VC_mode = 0 + self.AI_TTS_str = "了解，已切换到长对话模式" + EventMesh.publish("set_storage_config",["VC_mode",self.VC_mode]) + + + return 0 + if text.find("音量调整") != -1: + text = cmd["command"]["params_json"] + text = ujson.loads(text) + volume = text["n"] + if volume >100 or volume<0: + self.AI_TTS_str = "音量调整超出范围了，只能0-100哦" + else: + self.VC_TTS_volume = volume + self.aud.setVolume(self.VC_TTS_volume //10) + EventMesh.publish("set_storage_config",["VC_TTS_volume",self.VC_TTS_volume]) + self.AI_TTS_str = "了解，音量已经调整,当前音量是{}".format(self.VC_TTS_volume) + return 0 + if text.find("音量增大") != -1: + self.VC_TTS_volume += 10 + if self.VC_TTS_volume > 100: + + self.AI_TTS_str = "音量已经是最大啦" + self.VC_TTS_volume = 100 + else: + self.aud.setVolume(self.VC_TTS_volume //10) + EventMesh.publish("set_storage_config",["VC_TTS_volume",self.VC_TTS_volume]) + + self.AI_TTS_str = "了解，音量已经调大了,当前音量是{}".format(self.VC_TTS_volume) + + return 0 + if text.find("音量减小") != -1: + self.VC_TTS_volume -= 10 + if self.VC_TTS_volume < 0: + self.VC_TTS_volume = 0 + #EventMesh.publish("audio_montage_play",["yinliangyizuixiao.mp3"]) + #self.aud.play(1,0,'U:/static/yinliangyizuixiao.mp3')#音量已最小 + self.AI_TTS_str = "音量已经是最小啦" + else: + self.aud.setVolume(self.VC_TTS_volume //10) + EventMesh.publish("set_storage_config",["VC_TTS_volume",self.VC_TTS_volume]) + self.AI_TTS_str = "好嘞，音量已经调小了,当前音量是{}".format(self.VC_TTS_volume) + return 0 + + if text.find("语速增大") != -1: + self.speech_rate += 10 + if self.speech_rate > 50: + + self.AI_TTS_str = "语速已经是最快啦" + self.speech_rate = 50 + else: + + EventMesh.publish("set_storage_config",["speech_rate",self.speech_rate]) + + self.AI_TTS_str = "明白，语速已经调快了,当前语速是{}".format(self.speech_rate) + + return 0 + if text.find("语速减小") != -1: + self.speech_rate -= 10 + if self.speech_rate < -50: + self.speech_rate = -50 + #EventMesh.publish("audio_montage_play",["yinliangyizuixiao.mp3"]) + #self.aud.play(1,0,'U:/static/yinliangyizuixiao.mp3')#音量已最小 + self.AI_TTS_str = "语速已经是最慢啦" + else: + EventMesh.publish("set_storage_config",["speech_rate",self.speech_rate]) + + self.AI_TTS_str = "明白，语速已经调慢了,当前语速是{}".format(self.speech_rate) + return 0 + + if text.find("进入工厂模式") != -1 : + self.factory_mode() + return 1 + if text.find("查询") != -1: + if text.find("信号") != -1 : + signal = net.csqQueryPoll() + + self.AI_TTS_str = "当前信号{}".format(signal) + return 0 + if text.find("电量") != -1 : + battery = EventMesh.publish("get_battery") + print("电量查询：{}".format(battery)) + #EventMesh.publish("audio_montage_play",["dianliangshengyu{}.mp3".format(battery)]) + #self.aud.play(1,0,'U:/static/dianliangshengyu{}.mp3'.format(battery))#电量剩余 + self.AI_TTS_str = "电量剩余百分之{}".format(battery) + return 0 + if text.find("电压") != -1 : + Vbatt = Power.getVbatt() + #EventMesh.publish("audio_montage_play",["dianliangshengyu{}.mp3".format(battery)]) + #self.aud.play(1,0,'U:/static/dianliangshengyu{}.mp3'.format(battery))#电量剩余 + self.AI_TTS_str = "当前电池电压{}".format(Vbatt) + return 0 + + if text.find("卡号") != -1 : + iccid = sim.getIccid() + if iccid != -1 and len(iccid) > 5: + print("iccid{}".format(iccid)) + self.AI_TTS_str = "当前卡号为:{}".format(iccid) + else: + self.AI_TTS_str = "卡号查询失败" + return 0 + if text.find("设备号") != -1 : + imei = modem.getDevImei() + if imei != -1 and len(imei) > 5: + print("IMEI：{}".format(imei)) + self.AI_TTS_str = "当前设备号为:{}".format(imei) + else: + self.AI_TTS_str = "设备号查询失败" + return 0 + if text.find("版本") != -1 : + sourceVersion = versions.Build_Time + self.AI_TTS_str ="Mr-Seven-"+sourceVersion + return 0 + if text.find("服务器地址") != -1 : + self.AI_TTS_str ="服务器地址"+self.server_ip + return 0 + if text.find("升级") != -1 and (text.find("OTA") != -1 or text.find("ota") != -1): + if self.ai_OTA.min_sys_ota()==1: + self.AI_TTS_str = "有新版本，准备开始升级，升级过程会多次重启设备，请保证流量充足及电量充足" + else: + self.AI_TTS_str = "当前已是最新版本" + return 0 + + if len(text)<20 and (text.find("再见") != -1 or text.find("拜拜") != -1 or text.find("关闭对话") != -1): + #EventMesh.publish("publish_AI_btnInfo",{"func":self.Close}) + self.AI_TTS_str = "好嘞，再见，我们下次再聊" + self.Close_flag = 1 + return 0 + if len(text)<12 and (text.find("关机") != -1 ): + EventMesh.publish("publish_AI_btnInfo",{"func":self.Close}) + EventMesh.publish("Power_off") + return 1 + except Exception as e: + usys.print_exception(e) + self.AI_TTS_str = cmd["command"]["fail_prompt"] + return -1 #获取trace_id def get_trace_id(self): t = utime.mktime(utime.localtime()) @@ -3051,24 +2966,25 @@ class AI_VC(Abstract): def VC_Play_Start(self): self.tts_in_data_len = 0 self.tts_out_data_len = 0 - try: - if self.VC_play_TYPE == VC_play_TYPE_PCM: - if self.VC_play_samplerate == 8000: - pass - elif self.VC_play_samplerate == 16000: + if self.VC_Play_flag == 0: + try: + if self.VC_play_TYPE == VC_play_TYPE_PCM: if self.play_pcm == None: - self.play_pcm = audio.Audio.PCM(0,1,16000,1,1,2) + self.play_pcm = audio.Audio.PCM(0,1,self.VC_play_samplerate,1,1,2) #self.play_pcm.setVolume(11) ret = self.play_pcm.stream_write_start() print("self.record_pcm.stream_write_start:ret:{}".format(ret)) - elif self.VC_play_TYPE == VC_play_TYPE_MP3: - pass - except Exception as e: - - usys.print_exception(e) - EventMesh.publish("publish_AI_btnInfo",{"func":self.websocket_error,"data":"接口出错了。请重试"}) + elif self.VC_play_TYPE == VC_play_TYPE_MP3: + pass + self.VC_Play_flag = 1 + except Exception as e: + self.VC_Play_End() + usys.print_exception(e) + + #停止播放声音 def VC_Play_End(self): + self.VC_Play_flag = 0 if self.VC_play_TYPE == VC_play_TYPE_PCM: if self.VC_play_samplerate == 8000: self.aud.stopPlayStream() @@ -3112,17 +3028,26 @@ class AI_VC(Abstract): if self.VC_mode == 1 :#长按单次问答模式 EventMesh.publish("publish_AI_btnInfo",{"func":self.Close}) else: - EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_Start})#开始录音 + EventMesh.publish("open_led_blue") + if self.Close_flag == 1: + self.Close_flag = 0 + EventMesh.publish("publish_AI_btnInfo",{"func":self.Close}) + else: + #utime.sleep_ms(1000) + EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Play_End}) + #EventMesh.publish("publish_AI_btnInfo",{"func":self.chenggong_tone}) + EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_client_VC_send_speech})#开始录音 #通知服务器播放结束 - self.AI_client_VC_LocalRespondingEnded() + #self.AI_client_VC_LocalRespondingEnded() + self.VC_pingping_timerout+=1 if self.VC_pingping_timerout>3: self.VC_pingping_timerout=0 self.AI_client_VC_ping() - self.asr_pingping_timerout += 1 - if self.asr_pingping_timerout>3: - self.asr_pingping_timerout=0 - self.AI_client_asr_ping() + # self.asr_pingping_timerout += 1 + # if self.asr_pingping_timerout>3: + # self.asr_pingping_timerout=0 + # self.AI_client_asr_ping() if self.VC_Thinking_timerout > 0 and self.VC_Thinking_timerout < 100:#VC Thinking超时后重新开始 print("VC_Thinking_timerout:{}!!!!!!!!!!!!!!!!!!!!!".format(self.VC_Thinking_timerout)) self.VC_Thinking_timerout -=1 @@ -3150,39 +3075,45 @@ class AI_VC(Abstract): # if self.AI_client_VC_isconnect == 0:#没连接的时候请求开始 self.websocket_connect_VC()#连接服务器 - self.AI_client_VC_start_requset() + #utime.sleep_ms(50) else: print("self.VC_state :{}".format(self.VC_state )) - if self.VC_state > VC_State_SpeechStart: - #客户端打断对话 - print("打断对话") - # self.websocket_deconnect_VC() - # self.websocket_connect_VC()#连接服务器 - # self.AI_client_VC_start_requset() - self.AI_client_VC_RequestToSpeak() - elif self.VC_state == VC_State_Listening: - #直接chat - print("直接chat") - self.AI_client_VC_RequestToRespond() - return - else: - print("异常状态") + # if self.VC_state > VC_State_SpeechStart: + # #客户端打断对话 + # print("打断对话") + # # self.websocket_deconnect_VC() + # # self.websocket_connect_VC()#连接服务器 + # # self.AI_client_VC_start_requset() + # #self.AI_client_VC_RequestToSpeak() + # elif self.VC_state == VC_State_Listening: + # #直接chat + # print("直接chat") + # # self.AI_client_VC_RequestToRespond() + # return + # else: + # print("异常状态") #self.AI_client_VC_RequestToSpeak() - self.websocket_deconnect_VC() - self.websocket_connect_VC()#连接服务器 - self.AI_client_VC_start_requset() + # self.websocket_deconnect_VC() + # self.websocket_connect_VC()#连接服务器 + # self.AI_client_VC_start_requset() if self.AI_client_VC_isconnect == 0: return + self.AI_client_VC_start_requset() print("AI_VC_start3") self.VC_state = VC_State_None if self.AI_TTS_str == "":#没有tts要播放的时候才开始录音 #开始asr录音 - self.VC_Speech_Start() + if self.VC_mode == 1:#单对话模式 + self.VC_Speech_Start()#开启线程获取录音数据 + else : + self.VC_Speech_Start()#开启线程获取录音数据 + self.VC_record_state_flag = 2 # pass print("AI_VC_start4") - + #self.VC_Play_Start() + #EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Play_Start}) #开始线程 # if self.AI_VC_send_data_queue_thread_id == 0: @@ -3209,39 +3140,77 @@ class AI_VC(Abstract): voice = self.VC_TTS_voice_type[self.VC_TTS_voice_key]["voice"] except Exception as e: usys.print_exception(e) + #voice="huoshan::zh_female_dongmanxiaoyang_moon_bigtts" + json_data = \ { - "header":{ - "trace_id":self.get_trace_id(), - "event":"Start" + "header": { + "event": "Start", + "trace_id": self.get_trace_id(), + }, + "payload": { + "device_id": modem.getDevImei(), + "user_agent": "AudioClient/1.0", + "dialog_attributes": { + #"provider": "doubao", + "provider": "coze", + + "cfg": { + "disable_empty_prompt" : True,#拦截空字符请求大模型 + "custom_variables": { + "name": self.bot_name, + }, + # "greeting":{ + # "enabled":True, + # "content":"小瑞来陪你聊天了！" + # } + "bot_option": { + "use_workflow": True, + "bot_id":"7481164663863459850",#鸭子 + "workflow_id":"7480485093845991443", + # "bot_id":"7462617881076351002",#语言助手 + # "workflow_id":"7462621757327196187", + # "bot_id":"7469964258592931852",#深圳百事通 + # "workflow_id":"7469968083063603241", + } + } }, - "payload":{ - "dialog_id":self.VC_dialog_id, - "device_id":modem.getDevImei(), - "user_agent":"python", - "in_format":"pcm", - "out_format":"mp3", - "in_sample_rate":self.VC_record_samplerate, - "sample_rate":self.VC_play_samplerate, - "space_time": 10, - "volume": str(self.VC_TTS_volume), - "voice":voice,#self.VC_TTS_voice_type[str(self.VC_TTS_voice_key)], - "voice_detection":"enabled", - "simple_response":True, - "session_id": self.session_id , - "custom_variables":{ - "bot_name":"{}".format(self.bot_name) + "outbound_attributes": { + "provider": "volcengine", + "cfg": { + "format": "pcm", + "sample_rate": self.VC_play_samplerate, + "bit_rate": 16, + "channel": 1, + "volume": self.VC_TTS_volume, + #"voice": "7426720361732980745", + "voice": voice, + "speech_rate": self.speech_rate,#语速 + } + }, + "inbound_attributes": { + #"provider": "xun-fei",# + "provider": "volcengine", + "disable_vad":True, + "vad_silence":300, + "interrupt_options":{ + "concatenate_results":True, + "thinking_time_only": True, }, + "cfg": { + "format": "pcm", + "sample_rate": self.VC_record_samplerate, + "bit_rate": 16, + "channel": 1 + } }, - + "commands":self.commands, + } } - if self.VC_play_TYPE == VC_play_TYPE_PCM: - json_data["payload"]["out_format"]='pcm' - if self.VC_record_TYPE == VC_record_TYPE_AMR : - json_data["payload"]["in_format"]='amr' - pass - if self.VC_mode == 1: - json_data["payload"]["voice_detection"]='disable' + if self.is_llm_commands ==1: + json_data["payload"]["commands"] = [] + if self.VC_mode == 0:#长对话模式打开vad + json_data["payload"]["inbound_attributes"]["disable_vad"] = False self.VC_state = VC_State_None self.AI_client_VC_send(ujson.dumps(json_data)) @@ -3249,80 +3218,39 @@ class AI_VC(Abstract): def AI_client_VC_send_speech(self): json_data = \ { - "header":{ - "trace_id":self.get_trace_id(), - "event":"SendSpeech" - }, - "payload": - { - "dialog_id":self.VC_dialog_id, - "session_id": self.session_id , - } + "header": { + "event": "audio.create", + + }, } - self.AI_client_VC_send(ujson.dumps(json_data)) - self.VC_Speech_Start()#开启线程获取录音数据 + + if self.VC_mode == 1:#单对话模式 + self.AI_client_VC_send(ujson.dumps(json_data)) + EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_Start}) + #self.VC_Speech_Start()#开启线程获取录音数据 + else : + #EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Play_End}) + EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_Start}) + + # self.VC_Play_End() + # self.VC_Speech_Start()#开启线程获取录音数据 + + + self.VC_state = VC_State_SpeechStart #结束上传语音 def AI_client_VC_end_speech(self): json_data = \ { - "header":{ - "trace_id":self.get_trace_id(), - "event":"StopSpeech" - }, - "payload":{ - "dialog_id":self.VC_dialog_id, - "session_id": self.session_id , - } + "header": { + "event": "audio.completed", + + }, } self.AI_client_VC_send(ujson.dumps(json_data)) EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_Speech_End}) - - #通知服务器开始播放 - def AI_client_VC_LocalRespondingStarted(self): - json_data = \ - { - "header":{ - "trace_id":self.get_trace_id(), - "event":"LocalRespondingStarted" - }, - "payload":{ - "dialog_id":self.VC_dialog_id, - "session_id": self.session_id , - } - } - self.AI_client_VC_send(ujson.dumps(json_data)) - #通知服务器播放结束 - def AI_client_VC_LocalRespondingEnded(self): - json_data = \ - { - "header":{ - "trace_id":self.get_trace_id(), - "event":"LocalRespondingEnded" - }, - "payload":{ - "dialog_id":self.VC_dialog_id, - "session_id": self.session_id , - } - } - EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_init}) - EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_client_VC_send,"data":ujson.dumps(json_data)}) - # json_data = \ - # { - # "header":{ - # "trace_id":self.get_trace_id(), - # "event":"AAAAAAAAAAAAAA1" - # }, - # "payload":{ - # "dialog_id":self.VC_dialog_id, - # "session_id": self.session_id , - # } - # } - # EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_client_VC_send,"data":ujson.dumps(json_data)}) - # utime.sleep_ms(1) - # self.AI_client_VC_send(ujson.dumps(json_data)) - self.VC_state = VC_State_End + #客户端打断对话 def AI_client_VC_RequestToSpeak(self): json_data = \ @@ -3340,66 +3268,27 @@ class AI_VC(Abstract): EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_init}) EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_client_VC_send,"data":ujson.dumps(json_data)}) # utime.sleep_ms(1) - #请求TTS或者chat事件 - def AI_client_VC_RequestToRespond(self): - self.session_id = self.get_session_id()#调试临时打开 + #TTS播放 + def AI_client_VC_synthesis(self): json_data = \ { - "header":{ - "trace_id":self.get_trace_id(), - "event":"RequestToRespond" - }, - "payload":{ - "dialog_id":self.VC_dialog_id, - "session_id": self.session_id , - "type":"transcript",#prompt是跳过asr，transcript是跳过asr和大模型 - "text":"" - } + "event": "synthesis", + "payload": { + "text": self.AI_TTS_str + } } - - print("AI_client_VC_RequestToRespond!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + self.AI_TTS_str = "" + print("AI_client_VC_synthesis!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") # if self.VC_state == VC_State_RequestToRespond: # print("AI_client_VC_RequestToRespond 重复 tts_str:{}\nasr_str:{}".format(self.AI_TTS_str,self.AI_ASR_str)) - self.VC_state = VC_State_RequestToRespond - if self.AI_TTS_str != "": - json_data["payload"]["text"] = self.AI_TTS_str - json_data["payload"]["type"] = "transcript" - EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_init}) - EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_client_VC_send,"data":ujson.dumps(json_data)}) - self.AI_TTS_str = "" - elif self.AI_ASR_str != "": - json_data["payload"]["text"] = self.AI_ASR_str - json_data["payload"]["type"] = "prompt" - EventMesh.publish("publish_AI_btnInfo",{"func":self.VC_init}) - EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_client_VC_send,"data":ujson.dumps(json_data)}) - if dev_type != "7PIN" : - EventMesh.publish("publish_AI_btnInfo",{"func":self.send_tone}) - # - #self.AI_ASR_str = "" - # utime.sleep_ms(1) + self.AI_client_VC_send(ujson.dumps(json_data)) + def chenggong_tone(self): EventMesh.publish("audio_montage_play",["chenggong.mp3"]) def send_tone(self): EventMesh.publish("audio_montage_play",["send_tone.mp3"]) #给服务器回应ack - def AI_client_VC_Read(self): - json_data = \ - { - "header":{ - "trace_id":self.get_trace_id(), - "event":"Read" - }, - "payload":{ - "dialog_id":self.VC_dialog_id - } - } - print("AI_client_VC_Read!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - - #for i in range(6): - #EventMesh.publish("publish_AI_btnInfo",{"func":self.AI_client_VC_send,"data":ujson.dumps(json_data)}) - # utime.sleep_ms(1) - if self.AI_client_VC != None and self.AI_client_VC_isconnect == 1: - self.AI_client_VC.send(ujson.dumps(json_data)) + ############################################ #关闭音频播放线程 @@ -3426,6 +3315,9 @@ class AI_VC(Abstract): def AI_audio_queue_thread(self): print("AI_audio_queue_thread start") count = 0 + if self.AI_audio_queue.empty()!=True: + self.AI_audio_queue.clear() + print("音频数据还有剩余！！！！！！！！！！！！") while True: try: if self.AI_audio_queue_thread_lock == 2: @@ -3447,21 +3339,23 @@ class AI_VC(Abstract): self.close_timerout = 0 if self.play_flag == 0: self.play_flag =1 - print("AI_client_VC_LocalRespondingStarted!!!") - self.AI_client_VC_LocalRespondingStarted()#通知服务器开始播放 #self.AI_client_tts_data_len=len(data) #print("audio_queue:len:{}".format(len(data[0]))) + if len(data[0]) > 10: + for i in range(5): + if self.VC_Play_flag == 0: + self.VC_Play_Start() + utime.sleep_ms(2) + else: + break try: if self.VC_play_TYPE == VC_play_TYPE_PCM: - if self.VC_play_samplerate == 8000: - tmp= self.aud.playStream(1,data[0] ) - elif self.VC_play_samplerate == 16000: - - - + # if self.VC_play_samplerate == 8000: + # tmp= self.aud.playStream(1,data[0] ) + # else: write_buf = bytearray(data[0]) - #print("play_pcm.stream_write1!!") + print("play_pcm.stream_write1 len:{}".format(len(write_buf))) if self.play_pcm != None: #print("play_pcm.stream_write !!!") ret = self.play_pcm.stream_write(write_buf,len(write_buf)) @@ -3528,10 +3422,7 @@ class AI_VC(Abstract): self.tts_in_data_len = 0 self.tts_out_data_len = 0 #必须先销毁线程，不然线程会容易报错 - try: - self.AI_ASR_send_data_queue_thread_close()#销毁ASR发送线程 - except Exception as e: - usys.print_exception(e) + try: self.AI_audio_queue_thread_close()#销毁音频线程 except Exception as e: @@ -3542,19 +3433,20 @@ class AI_VC(Abstract): usys.print_exception(e) self.VC_Speech_End() + print("ai_VC Close1.1!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") self.VC_init() - print("ai_VC Close1.2!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - try: - self.websocket_deconnect_asr() - except Exception as e: - usys.print_exception(e) try: self.websocket_deconnect_VC() except Exception as e: usys.print_exception(e) + + + + + print("ai_VC Close1.3!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") print("ai_VC Close2!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") @@ -3575,8 +3467,7 @@ class AI_VC(Abstract): if self.AI_VC_send_data_queue.empty()!=True: self.AI_VC_send_data_queue.clear() - if self.AI_ASR_send_data_queue.empty()!=True: - self.AI_ASR_send_data_queue.clear() + self.AI_ASR_send_buf_len = 0 #self.AI_WS_connect_Queue.put(self.AI_client_VC) @@ -3609,8 +3500,7 @@ class AI_VC(Abstract): self._1S_timer.stop() if self.AI_VC_send_data_queue.empty()!=True: self.AI_VC_send_data_queue.clear() - if self.AI_ASR_send_data_queue.empty()!=True: - self.AI_ASR_send_data_queue.clear() + self.AI_ASR_send_buf_len = 0 if self.AI_audio_queue.empty()!=True: self.AI_audio_queue.clear()