From: liangzhenying <867174124@qq.com> Date: Tue, 27 Jan 2026 09:34:22 +0000 (+0800) Subject: 支持北科AI X-Git-Tag: RaisoundVad~1^2~2 X-Git-Url: http://git.snicker.com.cn/?a=commitdiff_plain;h=05c2d08fe34053a9981c7c7feee06280b13e15c0;p=mirrors%2FShenYin.git 支持北科AI --- diff --git a/ShenYin/ShenYin.xcodeproj/project.pbxproj b/ShenYin/ShenYin.xcodeproj/project.pbxproj index 5bc2888..528ea2c 100644 --- a/ShenYin/ShenYin.xcodeproj/project.pbxproj +++ b/ShenYin/ShenYin.xcodeproj/project.pbxproj @@ -744,7 +744,7 @@ CODE_SIGN_ENTITLEMENTS = ShenYin/ShenYin.entitlements; CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 1003; + CURRENT_PROJECT_VERSION = 1004; DEVELOPMENT_TEAM = G23X75CJ67; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", @@ -776,7 +776,7 @@ "@executable_path/Frameworks", ); LIBRARY_SEARCH_PATHS = "$(inherited)"; - MARKETING_VERSION = 1.0.3; + MARKETING_VERSION = 1.0.4; PRODUCT_BUNDLE_IDENTIFIER = com.deepchords.app; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; @@ -800,7 +800,7 @@ CODE_SIGN_ENTITLEMENTS = ShenYin/ShenYin.entitlements; CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 1003; + CURRENT_PROJECT_VERSION = 1004; DEVELOPMENT_TEAM = G23X75CJ67; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", @@ -832,7 +832,7 @@ "@executable_path/Frameworks", ); LIBRARY_SEARCH_PATHS = "$(inherited)"; - MARKETING_VERSION = 1.0.3; + MARKETING_VERSION = 1.0.4; PRODUCT_BUNDLE_IDENTIFIER = com.deepchords.app; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; diff --git a/ShenYin/ShenYin/Launch/SceneDelegate.swift b/ShenYin/ShenYin/Launch/SceneDelegate.swift index b55a14d..32c3493 100644 --- a/ShenYin/ShenYin/Launch/SceneDelegate.swift +++ b/ShenYin/ShenYin/Launch/SceneDelegate.swift @@ -17,10 +17,11 @@ class SceneDelegate: UIResponder, UIWindowSceneDelegate { guard let _ = (scene as? UIWindowScene) else { return } App.shared.window = window + AppLaunch.start() // window?.rootViewController = BaseTC() // MapManager.shared.setupSdk() - window?.rootViewController = UINavigationController(rootViewController: RSChatVC()) -// AppLaunch.start() +// window?.rootViewController = UINavigationController(rootViewController: RSChatVC()) + } func sceneDidDisconnect(_ scene: UIScene) { diff --git a/ShenYin/ShenYin/Sdk/JoySdk/tool/CCDevice.swift b/ShenYin/ShenYin/Sdk/JoySdk/tool/CCDevice.swift index a7a45aa..00ae57b 100644 --- a/ShenYin/ShenYin/Sdk/JoySdk/tool/CCDevice.swift +++ b/ShenYin/ShenYin/Sdk/JoySdk/tool/CCDevice.swift @@ -7,6 +7,7 @@ import JoyGemAI class CCDevice{ static var isJoy = true + static var showRaisound = true static func initSdk(){ if CCDevice.isJoy { JoySdkManager.shared.setupSdk() diff --git a/ShenYin/ShenYin/Sdk/LogManager.swift b/ShenYin/ShenYin/Sdk/LogManager.swift index 8735333..f61e054 100644 --- a/ShenYin/ShenYin/Sdk/LogManager.swift +++ b/ShenYin/ShenYin/Sdk/LogManager.swift @@ -9,7 +9,7 @@ import Foundation /// 日记管理 class Logger { - static var isDebug: Bool = true + static var isDebug: Bool = false fileprivate static let shared = Logger() private var fileHandle:FileHandle? diff --git a/ShenYin/ShenYin/ShenYin-Bridging-Header.h b/ShenYin/ShenYin/ShenYin-Bridging-Header.h index 6fb9b0d..d74e715 100644 --- a/ShenYin/ShenYin/ShenYin-Bridging-Header.h +++ b/ShenYin/ShenYin/ShenYin-Bridging-Header.h @@ -10,5 +10,5 @@ #import "PcmSound.h" #import #import "JLBleAssistManager.h" -//#import "OpusTool.h" -//#import "OpusManager.h" +#import "AiMouseAudioRecord.h" +#import "AiMouseRecordManager.h" diff --git a/ShenYin/ShenYin/UI/Base/BaseTC.swift b/ShenYin/ShenYin/UI/Base/BaseTC.swift index 8360d5a..3e4cd97 100644 --- a/ShenYin/ShenYin/UI/Base/BaseTC.swift +++ b/ShenYin/ShenYin/UI/Base/BaseTC.swift @@ -23,7 +23,6 @@ class BaseTC:UITabBarController{ let homeVC = HomeDeviceVC() let modeVC = ModeVC() let sleepVC = SleepVC() - let customerVC = CustomerVC() let mineVC = MineVC() let item0 = UITabBarItem(title: "AI" + " " + "智能".l10n , image: UIImage(named: "tab_0_normal")?.withRenderingMode(.alwaysOriginal), selectedImage: UIImage(named: "tab_0_selected")?.withRenderingMode(.alwaysOriginal)) @@ -38,13 +37,19 @@ class BaseTC:UITabBarController{ let nc2 = BaseNC(rootViewController: sleepVC) nc2.tabBarItem = item2 - let item3 = UITabBarItem(title: "客服".l10n, image: UIImage(named: "tab_3_normal")?.withRenderingMode(.alwaysOriginal), selectedImage: UIImage(named: "tab_3_selected")?.withRenderingMode(.alwaysOriginal)) - let nc3 = BaseNC(rootViewController: customerVC) - nc3.tabBarItem = item3 + var nc3 = BaseNC(rootViewController: CustomerVC()) + nc3.tabBarItem = UITabBarItem(title: "客服".l10n, image: UIImage(named: "tab_3_normal")?.withRenderingMode(.alwaysOriginal), selectedImage: UIImage(named: "tab_3_selected")?.withRenderingMode(.alwaysOriginal)) let item4 = UITabBarItem(title: "我的".l10n, image: UIImage(named: "tab_4_normal")?.withRenderingMode(.alwaysOriginal), selectedImage: UIImage(named: "tab_4_selected")?.withRenderingMode(.alwaysOriginal)) let nc4 = BaseNC(rootViewController: mineVC) nc4.tabBarItem = item4 + + if CCDevice.showRaisound { + nc3 = BaseNC(rootViewController: RSChatVC()) + nc3.tabBarItem = UITabBarItem(title: "AI".l10n, image: UIImage(named: "tab_2_normal")?.withRenderingMode(.alwaysOriginal), selectedImage: UIImage(named: "tab_2_selected")?.withRenderingMode(.alwaysOriginal)) + } + + self.viewControllers = [nc0,nc1,nc2,nc3,nc4] let appearance = UITabBarAppearance() diff --git a/ShenYin/ShenYin/UI/Raisound/MSVoiceChat.swift b/ShenYin/ShenYin/UI/Raisound/MSVoiceChat.swift index 53793b4..cf9afb3 100644 --- a/ShenYin/ShenYin/UI/Raisound/MSVoiceChat.swift +++ b/ShenYin/ShenYin/UI/Raisound/MSVoiceChat.swift @@ -7,16 +7,8 @@ // import Foundation - -/// 语音文件识别回调 -//public typealias MSSttCallBack = (_ text: String?,_ error:MSError?) -> Void -///// 事件回调 -//public typealias MSSttEventCallBack = (_ model:MSSttEvent) -> Void - /// 连接状态回调 public typealias MSConnectStateCallBack = (_ state:MSConnectState) -> Void -/// 对话回调 -public typealias MSChatCallBack = (_ model: MSChatModel) -> Void /// 语音回复回调 public typealias MSVoiceChatTtsCallBack = (_ data: Data) -> Void /// 事件回调 @@ -27,30 +19,17 @@ public class MSVoiceChat { var socket:WebSocket? var timer:Timer? // /// 事件回调 - // public var eventCallBack:MSVoiceChatEventCallBack? - /// 对话回调 - public var callBack:MSChatCallBack? + public var eventCallBack:MSVoiceChatEventCallBack? /// 语音数据回调 public var ttsCallBack:MSVoiceChatTtsCallBack? /// 连接状态回调 public var connectStateCallBack:MSConnectStateCallBack? - - /// 当前处在的事件状态 - public var currentEvent:MSVoiceChatEvent? - /// 九天pid - public var jiutian_pid:String = "" - /// 九天did - public var jiutian_did:String = "" + /// 设备ID/用户ID var deviceId:String = "" - - /// 任务id - var taskId:String = "" - var dialogId:String = "" - - /// 最后的聊天内容 - var lastChatModel:MSChatModel? + + /// 连接状态 public var connectState:MSConnectState{ @@ -67,8 +46,10 @@ public class MSVoiceChat { /// 开始任务 public func start(){ +// cancelSpeedch() + reset() openWebSocket() - setupHeartBeat() +// setupHeartBeat() } func setupHeartBeat(){ @@ -88,13 +69,6 @@ public class MSVoiceChat { if connectState == .connected { socket?.close(10100,reason: "正常结束") } - if let model = lastChatModel,model.type != "end"{ - lastChatModel = nil - let endModel = MSChatModel() - endModel.type = "end" - callBack?(endModel) - } - currentEvent = nil; } /// 重置聊天对话 public func reset(){ @@ -111,7 +85,7 @@ public class MSVoiceChat { "payload": [ "device_id": RSAuth.shared.deviceID, // 设备ID "user_agent": "AudioClient/1.0", // 浏览器 UA - "simple_response": true, // true 只返回完整的LLM文本 默认false 返回增量LLM文本 + "simple_response": false, // true 只返回完整的LLM文本 默认false 返回增量LLM文本 "outbound_attributes": [ // TTS输出音频配置 "cfg": [ "format": "pcm", // 音频输出格式 @@ -125,7 +99,7 @@ public class MSVoiceChat { ] ], "inbound_attributes": [ // ASR输入音频配置 - "disable_vad": false, // 开启vad 自动断句 （开启后可以有客户端处理回音消除后持续发送音频由服务器判断是否转写和发送LLM） + "disable_vad": true, // 开启vad 自动断句 （开启后可以有客户端处理回音消除后持续发送音频由服务器判断是否转写和发送LLM） "vad_silence": 400, // 设置vad 静音阈值 （毫秒） 连续静音超过 阈值自动切换状态 "interrupt_options": [ "concatenate_results": true, // 是否在打断后拼接前后的ASR结果 @@ -140,9 +114,9 @@ public class MSVoiceChat { "channel": 1, // 声道 目前只支持单声道 "enable_punctuation": true, // 开启标点 "enable_itn": true, // 开启itn - "keywords": [ - "张三" - ] // 热词 +// "keywords": [ +// "张三" +// ] // 热词 ] ] ] @@ -156,7 +130,7 @@ public class MSVoiceChat { let params:[String:Any] = [ "event": "audio.create", - "payload": {} +// "payload": [] ] let cmd = params.jsonString() ?? "" Logger.debug("send" ,cmd) @@ -168,7 +142,7 @@ public class MSVoiceChat { let params:[String:Any] = [ "event": "audio.completed", - "payload": {} +// "payload": [] ] let cmd = params.jsonString() ?? "" Logger.debug("send" ,cmd) @@ -179,107 +153,17 @@ public class MSVoiceChat { let params:[String:Any] = [ "event": "cancel", - "payload": {} - ] - let cmd = params.jsonString() ?? "" - Logger.debug("send" ,cmd) - socket?.send(text: cmd) - } - /// 发送内容 - /// - Parameter content: 内容 - public func send(content:String){ - requestToSpeak() - - let params:[String:Any] = [ - "header": [ - "trace_id": self.taskId, - "event": "RequestToRespond" - ], - "payload": [ - "dialog_id": dialogId, - "type": "prompt", // prompt 直接将文本提交给大模型 transcript 直接将文本转成音频 - "text": content, - "session_id": self.taskId - ] +// "payload": [] ] - let cmd = params.jsonString() ?? "" Logger.debug("send" ,cmd) socket?.send(text: cmd) } - - /// 打断，请求说话 - public func requestToSpeak(){ - if socket?.readyState == .open{ - sendSpeakCMD() - } - else{ - //还没连接 - Logger.debug("还没连接！！！！！！！！！！！") - self.openWebSocket() - DispatchQueue.main.asyncAfter(deadline: .now() + 2) { - self.sendSpeakCMD() - } - } - } - - func sendSpeakCMD(){ - Logger.debug("开始请求说话") - let params:[String:Any] = - [ - "header":[ - "trace_id":self.taskId, - "event":"RequestToSpeak" - ], - "payload":[ - "dialog_id":dialogId, - "session_id": self.taskId, - ] - ] - let cmd = params.jsonString() ?? "" - Logger.debug("send" ,cmd) - socket?.send(text: cmd) - - let tmpModel = MSChatModel() - tmpModel.type = "start" - tmpModel.content = nil - lastChatModel = tmpModel - callBack?(tmpModel) - } - /// 本地开始播放 - public func startPlay(){ - let params:[String:Any] = - [ - "header":[ - "trace_id":self.taskId, - "event":"LocalRespondingStarted" - ], - "payload":[ - "dialog_id":dialogId, - "session_id": self.taskId, - ] - ] - let cmd = params.jsonString() ?? "" - Logger.debug("send" ,cmd) - socket?.send(text: cmd) - } - - /// 本地结束播放 - public func endPlay(){ - let params:[String:Any] = - [ - "event": "audio.played", - "payload": {} - ] - let cmd = params.jsonString() ?? "" - Logger.debug("send" ,cmd) - socket?.send(text: cmd) - - } + /// 发送语音数据，format pcm，采样率 16000 /// - Parameter data: 语音数据 func sendVoice(pcmData:Data){ - if !RSAuth.shared.deviceToken.isEmpty { + if RSAuth.shared.deviceToken.isEmpty { Logger.debug("还未鉴权") return } @@ -290,13 +174,7 @@ public class MSVoiceChat { extension MSVoiceChat{ private func openWebSocket(){ - taskId = "\(Int(Date().timeIntervalSince1970)*1000)".md5().lowercased() - currentEvent = nil; - - - let url = RSApi.wsBaseURL + "token=\(RSAuth.shared.deviceToken)&dt=\(RSAuth.shared.deviceID)" - - + let url = RSApi.wsBaseURL + "dt=\(RSAuth.shared.deviceToken)" Logger.debug(url) socket = WebSocket(url) socket?.delegate = self @@ -326,37 +204,18 @@ extension MSVoiceChat:WebSocketDelegate{ guard let data = text.data,let model = JSON.decode(MSVoiceChatEvent.self, from: data) else{ return } + if model.event == "started"{ + beginSpeedch() + } + eventCallBack?(model) if model.event == "TaskFailed"{ closeWebSocket() return } - else if model.event == "Started",let dialogId = model.payload?.dialog_id{ - self.dialogId = dialogId - } - else if model.event == "DialogStateChanged"{ - currentEvent = model - } - - if model.event == "RespondingContent",let payload = model.payload,let text = payload.text{ - if let finish = payload.finished,finish == false { - let tmpModel = MSChatModel() - tmpModel.type = "middle" - tmpModel.content = text - lastChatModel = tmpModel - callBack?(tmpModel) - } - else{ - let tmpModel = MSChatModel() - tmpModel.type = "end" - tmpModel.content = nil - lastChatModel = tmpModel - callBack?(tmpModel) - } - } } func webSocketMessageData(_ data: Data){ - // Logger.debug("MSVoiceChat receive data ",data) + Logger.debug("MSVoiceChat receive data ",data) ttsCallBack?(data) } diff --git a/ShenYin/ShenYin/UI/Raisound/MSVoiceChatModel.swift b/ShenYin/ShenYin/UI/Raisound/MSVoiceChatModel.swift index f3ff160..05a0b4e 100644 --- a/ShenYin/ShenYin/UI/Raisound/MSVoiceChatModel.swift +++ b/ShenYin/ShenYin/UI/Raisound/MSVoiceChatModel.swift @@ -17,48 +17,31 @@ public enum MSConnectState: Int { /// voiceChat事件 public class MSVoiceChatEvent:Codable{ /// 事件 - /// Started - /// DialogStateChanged - /// SpeechContent - /// RespondingContent - /// RespondingStarted - /// RespondingEnded - /// LocalRespondingStarted - /// LocalRespondingEnded - /// TaskFailed + /// started + /// vad.active + /// asr.started + /// asr.result + /// vad.silence + /// vad.active + /// asr.completed + /// llm.created + /// llm.delta + /// llm.completed + /// llm.done + /// tts.started + /// tts.completed public var event:String = "" /// - public var msg:String = "" - /// 状态码 20000000 - public var status:Int = 0 - /// 任务id - public var task_id:String = "" + public var sid:String = "" /// 内容 public var payload:MSVoiceChatPayLoad? } public class MSVoiceChatPayLoad:Codable{ /// 会话id - public var dialog_id:String? - /// Listening 可以说话 - /// Thinking 正在思考 - /// Responding 正在回复 - public var state:String? - + public var dialog_id:String? = "" /// 是否结束 public var finished:Bool? /// 识别的文字 - public var text:String? = "" -} - -/// 对话模型 -public class MSChatModel:Codable{ - /// 类型 start middle end - public var type:String = "" - /// 内容 - public var content:String? - - /// 深度思考，deepseek才有 - public var reasoning_content:String? - + public var content:String? = "" } diff --git a/ShenYin/ShenYin/UI/Raisound/RSChatCell.swift b/ShenYin/ShenYin/UI/Raisound/RSChatCell.swift new file mode 100644 index 0000000..005c674 --- /dev/null +++ b/ShenYin/ShenYin/UI/Raisound/RSChatCell.swift @@ -0,0 +1,102 @@ +// +// RSChatCell.swift +// ShenYin +// +// Created by liangshi on 2026/1/27. +// +class RSChatCellModel{ + var id = "" + var content = "" + var isSend = false + init(id: String = "",content: String = "", isSend: Bool = false) { + self.content = content + self.isSend = isSend + self.id = id + } +} +class RSChatCell:BaseTableViewCell{ + let bgView = UIView() + let originalLabel = UILabel() + let icon = UIImageView() + + override init(style: UITableViewCell.CellStyle, reuseIdentifier: String?) { + super.init(style: style, reuseIdentifier: reuseIdentifier) + + } + + required init?(coder: NSCoder) { + fatalError("init(coder:) has not been implemented") + } + + override func setupViews() { + contentView.backgroundColor = .color(.bgColor) + + bgView.do { v in + v.yc.setCornerRadius(8) + } + originalLabel.do { v in + v.font = .regular(14) + v.textColor = .color(.first) + v.numberOfLines = 0 + v.textAlignment = .left + } + icon.do { v in + v.yc.setCornerRadius(18) + v.backgroundColor = .color(.cellBg) + } + contentView.addSubviews([bgView,icon]) + + bgView.addSubviews([originalLabel]) + } + + override func setupLayout() { + bgView.snp.makeConstraints { make in + make.top.bottom.equalToSuperview().inset(10) + make.width.lessThanOrEqualTo(kScreenWid - 36) + make.left.equalToSuperview().inset(18) + } + + originalLabel.snp.makeConstraints { make in + make.top.bottom.equalToSuperview().inset(14) + make.left.right.equalToSuperview().inset(12) + make.width.lessThanOrEqualTo(kScreenWid - 140) + make.width.greaterThanOrEqualTo(20) + make.height.greaterThanOrEqualTo(19) + } + } + + var data:RSChatCellModel!{ + didSet{ + originalLabel.text = data.content + if data.isSend { + bgView.backgroundColor = CCDevice.isJoy ? .color(.theme) : UIColor.color(.cellBg) + originalLabel.textColor = .white + + icon.snp.remakeConstraints { make in + make.size.equalTo(36) + make.top.equalToSuperview().inset(16) + make.right.equalToSuperview().inset(18) + } + bgView.snp.remakeConstraints { make in + make.top.bottom.equalToSuperview().inset(10) + make.right.equalToSuperview().inset(60) + } + icon.kf.setImage(with: CacheManager.shared.user.info.avatar.yc.asUrl) + } + else{ + bgView.backgroundColor = .color(.cellBg) + originalLabel.textColor = .color(.first) + icon.image = CCDevice.isJoy ? UIImage(named: "logo_2"):UIImage(named: "logo_4") + icon.snp.remakeConstraints { make in + make.size.equalTo(36) + make.top.equalToSuperview().inset(16) + make.left.equalToSuperview().inset(18) + } + bgView.snp.remakeConstraints { make in + make.top.bottom.equalToSuperview().inset(10) + make.left.equalToSuperview().inset(60) + } + } + } + } +} diff --git a/ShenYin/ShenYin/UI/Raisound/RSChatVC.swift b/ShenYin/ShenYin/UI/Raisound/RSChatVC.swift index ba6159d..be16014 100644 --- a/ShenYin/ShenYin/UI/Raisound/RSChatVC.swift +++ b/ShenYin/ShenYin/UI/Raisound/RSChatVC.swift @@ -5,16 +5,157 @@ // Created by liangshi on 2026/1/26. // + import UIKit +import AVFAudio class RSChatVC:BaseVC{ + let toolBar = ChatRecordBar() let tableView = UITableView(frame: CGRectZero, style: .plain) - var dataList:[Any] = [] - override func viewDidLoad() { - super.viewDidLoad() + let tipsLabel = UILabel() + let recordMamager = AiMouseRecordManager() + + var dataList:[RSChatCellModel] = [] + + override func viewDidAppear(_ animated: Bool) { + super.viewDidAppear(animated) + PcmSound.sharedInstance().delegate = self + RSManager.shared.start() + } + override func viewDidDisappear(_ animated: Bool) { + super.viewDidDisappear(animated) + CCAiManager.stopPlay() } override func setupViews() { - self.navBar.title = "客服".l10n - RSManager.shared.start() + navBar.leftView.isHidden = true + navBar.title = "AI" + + tableView.backgroundColor = .clear + tableView.delegate = self + tableView.dataSource = self + tableView.estimatedRowHeight = 60 + tableView.separatorStyle = .none + tableView.register(ChatCell.self, forCellReuseIdentifier: RSChatCell.reuseID) + + view.addSubviews([tableView,toolBar,tipsLabel]) + toolBar.delegate = self + + tipsLabel.do { v in + v.text = "内容由AI生成，请仔细甄别".l10n + v.font = .regular(10) + v.textColor = .hexString("#919191") + } + } + override func setupLayout() { + toolBar.snp.makeConstraints { make in + make.width.centerX.equalToSuperview() + make.height.equalTo(52) + make.bottom.equalToSuperview().inset(kTabbarHei + 20) + } + tipsLabel.snp.makeConstraints { make in + make.top.equalTo(toolBar.snp.bottom).offset(4) + make.centerX.equalToSuperview() + } + tableView.snp.makeConstraints { make in + make.top.equalToSuperview().inset(kNavBtm) + make.left.right.equalToSuperview() + make.bottom.equalTo(toolBar.snp.top) + } + } + + override func setupData() { + recordMamager.recordDataCallBack = {[weak self] data in + guard let self = self else {return} + if RSManager.shared.chat.connectState == .connected { + RSManager.shared.chat.sendVoice(pcmData: data) + } + else{ + //断开连接,不发送数据 + } + } + + RSManager.shared.chat.eventCallBack = {[weak self] event in + guard let self = self else {return} + if event.event == "started" { + print("开始已任务，可以开始录音") + //2、开启录音 + recordMamager.start() + } + else if event.event == "asr.result",let text = event.payload?.content,let id = event.payload?.dialog_id{ + if let model = dataList.last,model.id == id,model.isSend == true { + model.content = text + } + else{ + let model = RSChatCellModel(id: id, content: text, isSend: true) + dataList.append(model) + } + refreshData() + } + else if event.event == "llm.delta",let text = event.payload?.content,let id = event.payload?.dialog_id{ + if let model = dataList.last,model.id == id,model.isSend == false { + model.content += text + } + else{ + let model = RSChatCellModel(id: id, content: text, isSend: false) + dataList.append(model) + } + refreshData() + } + else if event.event == "llm.completed",let text = event.payload?.content,let id = event.payload?.dialog_id{ + + } + } + RSManager.shared.chat.ttsCallBack = { data in + PcmSound.sharedInstance().feed(data) + } + + } + + func refreshData(){ + tableView.reloadData() + if dataList.count > 0 { + tableView.scrollToRow(at: IndexPath(row: dataList.count - 1, section: 0), at: .none, animated: true) + } + } +} + +extension RSChatVC:ChatRecordBarDelegate{ + func recordBarCancel(_ toolBar: ChatRecordBar) { + recordMamager.stop() + RSManager.shared.chat.cancelSpeedch() + if let item = dataList.last,item.content.isEmpty{ + dataList.removeLast() + refreshData() + } + } + + func recordBarStart(_ toolBar: ChatRecordBar) { + CCAiManager.stopPlay() + RSManager.shared.chat.start() } + func recordBarStop(_ toolBar: ChatRecordBar) { + recordMamager.stop() + RSManager.shared.chat.endSpeedch() + CCAiManager.preparePlayBlueTooth() + } +} + +extension RSChatVC:UITableViewDelegate,UITableViewDataSource { + func tableView(_ tableView: UITableView, numberOfRowsInSection section: Int) -> Int { + return dataList.count + } + + func tableView(_ tableView: UITableView, cellForRowAt indexPath: IndexPath) -> UITableViewCell { + let cell = RSChatCell.cell(withTableView: tableView, indexPath: indexPath) + cell.data = dataList[indexPath.row] + return cell + } + func tableView(_ tableView: UITableView, didSelectRowAt indexPath: IndexPath) { + + } +} +extension RSChatVC : PcmSoundDelegate { + func remaining(_ frames: UInt) { + Logger.debug("[frame:\(frames)]"); + } } diff --git a/ShenYin/ShenYin/UI/Raisound/RSManager.swift b/ShenYin/ShenYin/UI/Raisound/RSManager.swift index 15ac44c..df1c74f 100644 --- a/ShenYin/ShenYin/UI/Raisound/RSManager.swift +++ b/ShenYin/ShenYin/UI/Raisound/RSManager.swift @@ -42,7 +42,7 @@ class RSManager{ } else{ Logger.debug(token ?? "") - self?.chat.start() +// self?.chat.start() } } } diff --git a/ShenYin/ShenYin/UI/Raisound/record/AiMouseAudioRecord.h b/ShenYin/ShenYin/UI/Raisound/record/AiMouseAudioRecord.h new file mode 100644 index 0000000..a5b1b43 --- /dev/null +++ b/ShenYin/ShenYin/UI/Raisound/record/AiMouseAudioRecord.h @@ -0,0 +1,38 @@ +// +// AiMouseAudioRecord.h +// AVDemo +// +// Created by incar on 2024/3/27. +// + +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface AiMouseAudioConfig : NSObject ++ (instancetype)defaultConfig; + +@property (nonatomic, assign) NSUInteger channels; // 声道数，default: 2。 +@property (nonatomic, assign) NSUInteger sampleRate; // 采样率，default: 44100。 +@property (nonatomic, assign) NSUInteger bitDepth; // 量化位深，default: 16。 +@end + + + + + +@interface AiMouseAudioRecord : NSObject ++ (instancetype)new NS_UNAVAILABLE; +- (instancetype)init NS_UNAVAILABLE; +- (instancetype)initWithConfig:(AiMouseAudioConfig *)config; + +@property (nonatomic, strong, readonly) AiMouseAudioConfig *config; +@property (nonatomic, copy) void (^sampleBufferOutputCallBack)(CMSampleBufferRef sample); // 音频采集数据回调。 +@property (nonatomic, copy) void (^errorCallBack)(NSError *error); // 音频采集错误回调。 + +- (void)startRunning; // 开始采集音频数据。 +- (void)stopRunning; // 停止采集音频数据。 +@end + +NS_ASSUME_NONNULL_END diff --git a/ShenYin/ShenYin/UI/Raisound/record/AiMouseAudioRecord.m b/ShenYin/ShenYin/UI/Raisound/record/AiMouseAudioRecord.m new file mode 100644 index 0000000..cd19cfb --- /dev/null +++ b/ShenYin/ShenYin/UI/Raisound/record/AiMouseAudioRecord.m @@ -0,0 +1,268 @@ +// +// AiMouseAudioRecord.m +// AVDemo +// +// Created by incar on 2024/3/27. +// + +#import "AiMouseAudioRecord.h" +#import +#import + +@implementation AiMouseAudioConfig ++ (instancetype)defaultConfig { + AiMouseAudioConfig *config = [[self alloc] init]; + config.channels = 1; + config.sampleRate = 16000; + config.bitDepth = 16; + + return config; +} + +@end + +@interface AiMouseAudioRecord () +@property (nonatomic, assign) AudioComponentInstance audioCaptureInstance; // 音频采集实例。 +@property (nonatomic, assign) AudioStreamBasicDescription audioFormat; // 音频采集参数。 +@property (nonatomic, strong, readwrite) AiMouseAudioConfig *config; +@property (nonatomic, strong) dispatch_queue_t captureQueue; +@property (nonatomic, assign) BOOL isError; +@end + +@implementation AiMouseAudioRecord + +#pragma mark - Lifecycle +- (instancetype)initWithConfig:(AiMouseAudioConfig *)config { + self = [super init]; + if (self) { + _config = config; + _captureQueue = dispatch_queue_create("com.KeyFrameKit.audioCapture", DISPATCH_QUEUE_SERIAL); + } + + return self; +} + +- (void)dealloc { + // 清理音频采集实例。 + if (_audioCaptureInstance) { + AudioOutputUnitStop(_audioCaptureInstance); + AudioComponentInstanceDispose(_audioCaptureInstance); + _audioCaptureInstance = nil; + } +} + +#pragma mark - Action +- (void)startRunning { + if (self.isError) { + return; + } + + __weak typeof(self) weakSelf = self; + dispatch_async(_captureQueue, ^{ + if (!weakSelf.audioCaptureInstance) { + NSError *error = nil; + // 第一次 startRunning 时创建音频采集实例。 + [weakSelf setupAudioCaptureInstance:&error]; + if (error) { + // 捕捉并回调创建音频实例时的错误。 + [weakSelf callBackError:error]; + return; + } + } + + // 开始采集。 + OSStatus startStatus = AudioOutputUnitStart(weakSelf.audioCaptureInstance); + if (startStatus != noErr) { + // 捕捉并回调开始采集时的错误。 + [weakSelf callBackError:[NSError errorWithDomain:NSStringFromClass([AiMouseAudioRecord class]) code:startStatus userInfo:nil]]; + } + }); +} + +- (void)stopRunning { + if (self.isError) { + return; + } + + __weak typeof(self) weakSelf = self; + dispatch_async(_captureQueue, ^{ + if (weakSelf.audioCaptureInstance) { + // 停止采集。 + OSStatus stopStatus = AudioOutputUnitStop(weakSelf.audioCaptureInstance); + if (stopStatus != noErr) { + // 捕捉并回调停止采集时的错误。 + [weakSelf callBackError:[NSError errorWithDomain:NSStringFromClass([AiMouseAudioRecord class]) code:stopStatus userInfo:nil]]; + } + } + }); +} + +#pragma mark - Utility +- (void)setupAudioCaptureInstance:(NSError **)error { + // 1、设置音频组件描述。 + AudioComponentDescription acd = { + .componentType = kAudioUnitType_Output, + //.componentSubType = kAudioUnitSubType_VoiceProcessingIO, // 回声消除模式 + .componentSubType = kAudioUnitSubType_RemoteIO, + .componentManufacturer = kAudioUnitManufacturer_Apple, + .componentFlags = 0, + .componentFlagsMask = 0, + }; + + // 2、查找符合指定描述的音频组件。 + AudioComponent component = AudioComponentFindNext(NULL, &acd); + + // 3、创建音频组件实例。 + OSStatus status = AudioComponentInstanceNew(component, &_audioCaptureInstance); + if (status != noErr) { + *error = [NSError errorWithDomain:NSStringFromClass(self.class) code:status userInfo:nil]; + return; + } + + // 4、设置实例的属性：可读写。0 不可读写，1 可读写。 + UInt32 flagOne = 1; + AudioUnitSetProperty(_audioCaptureInstance, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Input, 1, &flagOne, sizeof(flagOne)); + + // 5、设置实例的属性：音频参数，如：数据格式、声道数、采样位深、采样率等。 + AudioStreamBasicDescription asbd = {0}; + asbd.mFormatID = kAudioFormatLinearPCM; // 原始数据为 PCM，采用声道交错格式。 + asbd.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagsNativeEndian | kAudioFormatFlagIsPacked; + asbd.mChannelsPerFrame = (UInt32) self.config.channels; // 每帧的声道数 + asbd.mFramesPerPacket = 1; // 每个数据包帧数 + asbd.mBitsPerChannel = (UInt32) self.config.bitDepth; // 采样位深 + asbd.mBytesPerFrame = asbd.mChannelsPerFrame * asbd.mBitsPerChannel / 8; // 每帧字节数 (byte = bit / 8) + asbd.mBytesPerPacket = asbd.mFramesPerPacket * asbd.mBytesPerFrame; // 每个包的字节数 + asbd.mSampleRate = self.config.sampleRate; // 采样率 + self.audioFormat = asbd; + status = AudioUnitSetProperty(_audioCaptureInstance, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 1, &asbd, sizeof(asbd)); + if (status != noErr) { + *error = [NSError errorWithDomain:NSStringFromClass(self.class) code:status userInfo:nil]; + return; + } + + // 6、设置实例的属性：数据回调函数。 + AURenderCallbackStruct cb; + cb.inputProcRefCon = (__bridge void *) self; + cb.inputProc = audioBufferCallBack; + status = AudioUnitSetProperty(_audioCaptureInstance, kAudioOutputUnitProperty_SetInputCallback, kAudioUnitScope_Global, 1, &cb, sizeof(cb)); + if (status != noErr) { + *error = [NSError errorWithDomain:NSStringFromClass(self.class) code:status userInfo:nil]; + return; + } + + // 7、初始化实例。 + status = AudioUnitInitialize(_audioCaptureInstance); + if (status != noErr) { + *error = [NSError errorWithDomain:NSStringFromClass(self.class) code:status userInfo:nil]; + return; + } +} + +- (void)callBackError:(NSError *)error { + self.isError = YES; + if (error && self.errorCallBack) { + dispatch_async(dispatch_get_main_queue(), ^{ + self.errorCallBack(error); + }); + } +} + ++ (CMSampleBufferRef)sampleBufferFromAudioBufferList:(AudioBufferList)buffers inTimeStamp:(const AudioTimeStamp *)inTimeStamp inNumberFrames:(UInt32)inNumberFrames description:(AudioStreamBasicDescription)description { + CMSampleBufferRef sampleBuffer = NULL; // 待生成的 CMSampleBuffer 实例的引用。 + + // 1、创建音频流的格式描述信息。 + CMFormatDescriptionRef format = NULL; + OSStatus status = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &description, 0, NULL, 0, NULL, NULL, &format); + if (status != noErr) { + CFRelease(format); + return nil; + } + + // 2、处理音频帧的时间戳信息。 + mach_timebase_info_data_t info = {0, 0}; + mach_timebase_info(&info); + uint64_t time = inTimeStamp->mHostTime; + // 转换为纳秒。 + time *= info.numer; + time /= info.denom; + // PTS。 + CMTime presentationTime = CMTimeMake(time, 1000000000.0f); + // 对于音频，PTS 和 DTS 是一样的。 + CMSampleTimingInfo timing = {CMTimeMake(1, description.mSampleRate), presentationTime, presentationTime}; + + // 3、创建 CMSampleBuffer 实例。 + status = CMSampleBufferCreate(kCFAllocatorDefault, NULL, false, NULL, NULL, format, (CMItemCount) inNumberFrames, 1, &timing, 0, NULL, &sampleBuffer); + if (status != noErr) { + CFRelease(format); + return nil; + } + + // 4、创建 CMBlockBuffer 实例。其中数据拷贝自 AudioBufferList，并将 CMBlockBuffer 实例关联到 CMSampleBuffer 实例。 + status = CMSampleBufferSetDataBufferFromAudioBufferList(sampleBuffer, kCFAllocatorDefault, kCFAllocatorDefault, 0, &buffers); + if (status != noErr) { + CFRelease(format); + return nil; + } + + CFRelease(format); + return sampleBuffer; +} + +#pragma mark - Capture CallBack +static OSStatus audioBufferCallBack(void *inRefCon, + AudioUnitRenderActionFlags *ioActionFlags, + const AudioTimeStamp *inTimeStamp, + UInt32 inBusNumber, + UInt32 inNumberFrames, + AudioBufferList *ioData) { + @autoreleasepool { + AiMouseAudioRecord *capture = (__bridge AiMouseAudioRecord *) inRefCon; + if (!capture) { + return -1; + } + + // 1、创建 AudioBufferList 空间，用来接收采集回来的数据。 + AudioBuffer buffer; + buffer.mData = NULL; + buffer.mDataByteSize = 0; + // 采集的时候设置了数据格式是 kAudioFormatLinearPCM，即声道交错格式，所以即使是双声道这里也设置 mNumberChannels 为 1。 + // 对于双声道的数据，会按照采样位深 16 bit 每组，一组接一组地进行两个声道数据的交错拼装。 + buffer.mNumberChannels = 1; + AudioBufferList buffers; + buffers.mNumberBuffers = 1; + buffers.mBuffers[0] = buffer; + + // 2、获取音频 PCM 数据，存储到 AudioBufferList 中。 + // 这里有几个问题要说明清楚： + // 1）每次回调会过来多少数据？ + // 按照上面采集音频参数的设置：PCM 为声道交错格式、每帧的声道数为 2、采样位深为 16 bit。这样每帧的字节数是 4 字节（左右声道各 2 字节）。 + // 返回数据的帧数是 inNumberFrames。这样一次回调回来的数据字节数是多少就是：mBytesPerFrame(4) * inNumberFrames。 + // 2）这个数据回调的频率跟音频采样率有关系吗？ + // 这个数据回调的频率与音频采样率（上面设置的 mSampleRate 44100）是没关系的。声道数、采样位深、采样率共同决定了设备单位时间里采样数据的大小，这些数据是会缓冲起来，然后一块一块的通过这个数据回调给我们，这个回调的频率是底层一块一块给我们数据的速度，跟采样率无关。 + // 3）这个数据回调的频率是多少？ + // 这个数据回调的间隔是 [AVAudioSession sharedInstance].preferredIOBufferDuration，频率即该值的倒数。我们可以通过 [[AVAudioSession sharedInstance] setPreferredIOBufferDuration:1 error:nil] 设置这个值来控制回调频率。 + OSStatus status = AudioUnitRender(capture.audioCaptureInstance, + ioActionFlags, + inTimeStamp, + inBusNumber, + inNumberFrames, + &buffers); + + // 3、数据封装及回调。 + if (status == noErr) { + // 使用工具方法将数据封装为 CMSampleBuffer。 + CMSampleBufferRef sampleBuffer = [AiMouseAudioRecord sampleBufferFromAudioBufferList:buffers inTimeStamp:inTimeStamp inNumberFrames:inNumberFrames description:capture.audioFormat]; + // 回调数据。 + if (capture.sampleBufferOutputCallBack) { + capture.sampleBufferOutputCallBack(sampleBuffer); + } + if (sampleBuffer) { + CFRelease(sampleBuffer); + } + } + + return status; + } +} + +@end diff --git a/ShenYin/ShenYin/UI/Raisound/record/AiMouseRecordManager.h b/ShenYin/ShenYin/UI/Raisound/record/AiMouseRecordManager.h new file mode 100644 index 0000000..ee1d1fe --- /dev/null +++ b/ShenYin/ShenYin/UI/Raisound/record/AiMouseRecordManager.h @@ -0,0 +1,20 @@ +// +// AiMouseRecordManager.h +// AVDemo +// +// Created by incar on 2024/3/27. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface AiMouseRecordManager : NSObject +@property (nonatomic, copy) void (^recordDataCallBack)(NSData *data); + +- (void)start; + +- (void)stop; +@end + +NS_ASSUME_NONNULL_END diff --git a/ShenYin/ShenYin/UI/Raisound/record/AiMouseRecordManager.m b/ShenYin/ShenYin/UI/Raisound/record/AiMouseRecordManager.m new file mode 100644 index 0000000..098568c --- /dev/null +++ b/ShenYin/ShenYin/UI/Raisound/record/AiMouseRecordManager.m @@ -0,0 +1,116 @@ +#import "AiMouseRecordManager.h" +#import +#import "AiMouseAudioRecord.h" + +@interface AiMouseRecordManager() +@property (nonatomic, strong) AiMouseAudioConfig *audioConfig; +@property (nonatomic, strong) AiMouseAudioRecord *audioCapture; +//@property (nonatomic, strong) NSFileHandle *fileHandle; +@end + +@implementation AiMouseRecordManager +#pragma mark - Property +- (AiMouseAudioConfig *)audioConfig { + if (!_audioConfig) { + _audioConfig = [AiMouseAudioConfig defaultConfig]; + } + + return _audioConfig; +} + +- (AiMouseAudioRecord *)audioCapture { + if (!_audioCapture) { + __weak typeof(self) weakSelf = self; + _audioCapture = [[AiMouseAudioRecord alloc] initWithConfig:self.audioConfig]; + _audioCapture.errorCallBack = ^(NSError* error) { + NSLog(@"KFAudioCapture error: %zi %@", error.code, error.localizedDescription); + }; + // 音频采集数据回调。在这里将 PCM 数据写入文件。 + _audioCapture.sampleBufferOutputCallBack = ^(CMSampleBufferRef sampleBuffer) { + if (sampleBuffer) { + // 1、获取 CMBlockBuffer，这里面封装着 PCM 数据。 + CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer); + size_t lengthAtOffsetOutput, totalLengthOutput; + char *dataPointer; + + // 2、从 CMBlockBuffer 中获取 PCM 数据存储到文件中。 + CMBlockBufferGetDataPointer(blockBuffer, 0, &lengthAtOffsetOutput, &totalLengthOutput, &dataPointer); + weakSelf.recordDataCallBack([NSData dataWithBytes:dataPointer length:totalLengthOutput]); + // [weakSelf.fileHandle writeData:[NSData dataWithBytes:dataPointer length:totalLengthOutput]]; + } + }; + } + + return _audioCapture; +} + +//- (NSFileHandle *)fileHandle { +// if (!_fileHandle) { +// NSString *audioPath = [[NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES) lastObject] stringByAppendingPathComponent:@"test.pcm"]; +// NSLog(@"PCM file path: %@", audioPath); +// [[NSFileManager defaultManager] removeItemAtPath:audioPath error:nil]; +// [[NSFileManager defaultManager] createFileAtPath:audioPath contents:nil attributes:nil]; +// _fileHandle = [NSFileHandle fileHandleForWritingAtPath:audioPath]; +// } +// +// return _fileHandle; +//} +//- (void)dealloc { +// if (_fileHandle) { +// [_fileHandle closeFile]; +// } +//} +-(instancetype)init{ + if (self = [super init]) { + [self setupAudioSession]; + } + return self; +} + +#pragma mark - Action +- (void)start { + // 在开始录音前重新配置音频会话，以应对外部改变 + [self setupAudioSession]; + [self.audioCapture startRunning]; +} + +- (void)stop { + [self.audioCapture stopRunning]; +} + +- (void)setupAudioSession { + NSError *error = nil; + + // 1、获取音频会话实例。 + AVAudioSession *session = [AVAudioSession sharedInstance]; + + // 检查当前会话是否已经激活，如果已激活先暂停再重新配置 + // if ([session isActive]) { + // [session setActive:NO error:nil]; // 先停用当前会话 + // } + + // 2、设置分类和选项。 + [session setCategory:AVAudioSessionCategoryPlayAndRecord withOptions:AVAudioSessionCategoryOptionMixWithOthers | AVAudioSessionCategoryOptionDefaultToSpeaker error:&error]; + if (error) { + NSLog(@"AVAudioSession setCategory error: %@", error.localizedDescription); + error = nil; + return; + } + + // 3、设置模式。 + [session setMode:AVAudioSessionModeVideoRecording error:&error]; + if (error) { + NSLog(@"AVAudioSession setMode error: %@", error.localizedDescription); + error = nil; + return; + } + + // 4、激活会话。 + [session setActive:YES error:&error]; + if (error) { + NSLog(@"AVAudioSession setActive error: %@", error.localizedDescription); + error = nil; + return; + } +} +@end diff --git a/ShenYin/ShenYin/ViewController.swift b/ShenYin/ShenYin/ViewController.swift index 8a3af60..0d795e7 100644 --- a/ShenYin/ShenYin/ViewController.swift +++ b/ShenYin/ShenYin/ViewController.swift @@ -8,7 +8,6 @@ import UIKit class ViewController: UIViewController { - override func viewDidLoad() { super.viewDidLoad() // Do any additional setup after loading the view.