1. 下载插件：motionverse官网地址：概述 · Motionverse 接口文档 (deepscience.cn)
2. 按照官方文档新建Unity工程：对接说明 · Motionverse 接口文档 (deepscience.cn)
3. 通过我们自己的ASR，将语音转换为文本，文本通过大语言模型（chatgpt、文心一言以及其他大语言模型）生成的结果，直接通过生成式AI技术，把文本转化为AI智能体的声音、动作和表情，和大语言模型完美连接，只需要在获取到文本的时候调用以下代码即可：
代码示例：
DriveTask task = new DriveTask(); task.player = player; task.text = question; NLPDrive.GetDrive(task);
其中，player即为挂载motionverse插件中Player脚本的对象，question即为大语言模型获取到的答案。
还可以自己生成语音，通过语音链接调用以下代码：
DriveTask task = new DriveTask(); task.player = player; task.text = audioUrl; AudioUrlDrive.GetDrive(task);其中，player即为挂载motionverse插件中Player脚本的对象，audioUrl即为语音链接。
4. 新建脚本AskManager，并挂载到场景中（可新建空物体），脚本代码如下：
using LitJson; using Motionverse; using MotionverseSDK; using System; using System.Collections; using System.Text; using UnityEngine; using UnityEngine.Networking; using UnityEngine.UI; public class AskManager : MonoBehaviour { public Player player; public Dropdown dropdown; public InputField inputField; public Button btnSend; public string chatGptKey = ""; private string chatUrl = "https://chatgpt.kazava.io/v1/chat/completions"; public string wenXinAPI = ""; public string wenXinSECRET = ""; private string wenXinToken = ""; private int curSelectNLP = 0; // Start is called before the first frame update private void Awake() { for (int i = 0; i < Display.displays.Length; i++) { Display.displays[i].Activate(); } } void Start() { dropdown.onValueChanged.AddListener((value) => { curSelectNLP = value; }); StartCoroutine(GetWenxinToken()); btnSend.onClick.AddListener(() => { if (string.IsNullOrEmpty(inputField.text)) { Debug.Log("请输入内容！"); } else { GetAnswer(inputField.text); } }); } public void GetAnswer(string q) { StartCoroutine(RealAnswer(q)); } public IEnumerator RealAnswer(string question) { switch (curSelectNLP) { case 0: DriveTask task = new DriveTask(); task.player = player; task.text = question; NLPDrive.GetDrive(task); break; case 1: StartCoroutine(RequestChat(question)); break; case 2: StartCoroutine(ChatCompletions(question)); break; default: break; } Invoke("restartRecording", 1); yield return null; } IEnumerator GetWenxinToken() { string url =$"https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id={wenXinAPI}&client_secret={wenXinSECRET}"; UnityWebRequest webRequest = UnityWebRequest.Get(url); webRequest.timeout = 5000; yield return webRequest.SendWebRequest(); if (webRequest.result == UnityWebRequest.Result.Success) { string response = webRequest.downloadHandler.text; var result = JsonUtility.FromJson<AccessTokenResponse>(response); wenXinToken = result.access_token; } else { Debug.LogError("Failed to get access token: " + webRequest.error); } } IEnumerator ChatCompletions(string content) { string url = $"https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/completions?access_token={wenXinToken}"; WenXinPostData postData = new WenXinPostData(); postData.messages = new WenXinMessage[1]; postData.messages[0] = new WenXinMessage(); postData.messages[0].content = content + "30字以内"; string data = JsonMapper.ToJson(postData); UnityWebRequest webRequest = UnityWebRequest.Post(url, data, "application/json"); webRequest.timeout = 5000; yield return webRequest.SendWebRequest(); if (webRequest.result == UnityWebRequest.Result.Success) { string response = webRequest.downloadHandler.text; WenXinRequest requestData = JsonMapper.ToObject<WenXinRequest>(response); DriveTask task = new DriveTask(); task.player = player; task.text = requestData.result; TextDrive.GetDrive(task); } else { Debug.LogError("Chat completions request failed: " + webRequest.error); } } private IEnumerator RequestChat(string content) { using (UnityWebRequest webRequest = new UnityWebRequest(chatUrl, "POST")) { webRequest.SetRequestHeader("Content-Type", "application/json"); ChatGPTPostData postData = new ChatGPTPostData(); postData.key = chatGptKey; postData.messages = new PostMessage[1]; postData.messages[0] = new PostMessage(); postData.messages[0].content = content + "30字以内"; ; string data = JsonMapper.ToJson(postData); byte[] jsonToSend = new UTF8Encoding().GetBytes(data); webRequest.uploadHandler = new UploadHandlerRaw(jsonToSend); webRequest.downloadHandler = new DownloadHandlerBuffer(); yield return webRequest.SendWebRequest(); if (webRequest.result != UnityWebRequest.Result.Success) { Debug.LogError("ChatGPT request error: " + content + webRequest.error); } else { string response = webRequest.downloadHandler.text; ChatGPTRequestData requestData = JsonMapper.ToObject<ChatGPTRequestData>(response); DriveTask task = new DriveTask(); task.player = player; task.text = requestData.choices[0].message.content; TextDrive.GetDrive(task); } } } }
5. 新建脚本RealtimeAsrManager，代码如下：
using System; using System.Collections; using UnityEngine; using UnityEngine.UI; using UnityWebSocket; namespace Motionverse { public class RealtimeAsrManager : MonoBehaviour { private IWebSocket webSocket; private Status status = Status.FirstFrame; private bool lockReconnect = false; [HideInInspector] public static Action<string> Asr; [SerializeField] private GameObject TextBG; //百度 public int AsrAppId; public string AsrAppkey = ""; void Start() { CreateWebSocket(); RecorderManager.DataAvailable += OnDataAvailable; } private void OnDisable() { RecorderManager.DataAvailable -= OnDataAvailable; } void CreateWebSocket() { try { string uril = "wss://vop.baidu.com/realtime_asr?sn=" + Guid.NewGuid().ToString(); webSocket = new WebSocket(uril); InitHandle(); webSocket.ConnectAsync(); } catch (Exception e) { Debug.Log("websocket连接异常:" + e.Message); ReConnect(); } } private void InitHandle() { RemoveHandle(); webSocket.OnOpen += OnOpen; webSocket.OnMessage += OnMessage; webSocket.OnClose += OnClose; webSocket.OnError += OnError; } void RemoveHandle() { webSocket.OnOpen -= OnOpen; webSocket.OnMessage -= OnMessage; webSocket.OnClose -= OnClose; webSocket.OnError -= OnError; } //请求开始 private void OnDataAvailable(byte[] data) { if (webSocket == null || (webSocket != null && webSocket.ReadyState != WebSocketState.Open)) return; switch (status) { case Status.FirstFrame://握手 { var firstFrame = new FirstFrame(); firstFrame.data.appid= AsrAppId; firstFrame.data.appkey = AsrAppkey; webSocket.SendAsync(JsonUtility.ToJson(firstFrame)); status = Status.ContinueFrame; } break; case Status.ContinueFrame://开始发送 { if (data.Length > 0) { webSocket.SendAsync(data); } } break; case Status.LastFrame://关闭 { webSocket.SendAsync(JsonUtility.ToJson(new LastFrame())); } break; default: break; } } void ReConnect() { if (this.lockReconnect) return; this.lockReconnect = true; StartCoroutine(SetReConnect()); } private IEnumerator SetReConnect() { yield return new WaitForSeconds(1); CreateWebSocket(); lockReconnect = false; } #region WebSocket Event Handlers private void OnOpen(object sender, OpenEventArgs e) { status = Status.FirstFrame; } private void OnMessage(object sender, MessageEventArgs e) { var err_msg = Utils.GetJsonValue(e.Data, "err_msg"); var type = Utils.GetJsonValue(e.Data, "type"); if (err_msg == "OK" && type == "MID_TEXT") { var result = Utils.GetJsonValue(e.Data, "result"); TextBG.GetComponentInChildren<Text>().text = result; } if (err_msg == "OK" && type == "FIN_TEXT") { var result = Utils.GetJsonValue(e.Data, "result"); TextBG.GetComponentInChildren<Text>().text = result; if (result.Length > 1) { RecorderManager.Instance.EndRecording(); Asr?.Invoke(result); } } } private void OnClose(object sender, CloseEventArgs e) { Debug.Log("websocket关闭," + string.Format("Closed: StatusCode: {0}, Reason: {1}", e.StatusCode, e.Reason)); webSocket = null; ReConnect(); } private void OnError(object sender, ErrorEventArgs e) { if (e != null) Debug.Log("websocket连接异常:" + e.Message); webSocket = null; ReConnect(); } #endregion } }
6. 新建脚本RecorderManager，代码如下：
using UnityEngine; using System; using System.Collections; using UnityEngine.UI; using Unity.VisualScripting; namespace Motionverse { public class RecorderManager : Singleton<RecorderManager> { //标记是否有麦克风 private bool isHaveMic = false; //当前录音设备名称 private string currentDeviceName = string.Empty; //表示录音的最大时长 int recordMaxLength = 600; //录音频率,控制录音质量(16000) int recordFrequency = 16000; [HideInInspector] public static Action<byte[]> DataAvailable; [SerializeField] private GameObject micON; [SerializeField] private GameObject micOFF; [SerializeField] private Image micAmount; [SerializeField] private GameObject TextBG; private AudioClip saveAudioClip; int offsetSamples = 0; private void Start() { Debug.Log(Microphone.devices[0]); if (Microphone.devices.Length > 0) { isHaveMic = true; currentDeviceName = Microphone.devices[0]; StartCoroutine(GetAudioFrames()); StartRecording(); } } /// <summary> /// 开始录音 /// </summary> /// <returns></returns> public void StartRecording() //16000 { if (isHaveMic == false || Microphone.IsRecording(currentDeviceName)) { return; } micOFF.gameObject.SetActive(false); micON.gameObject.SetActive(true); TextBG.GetComponentInChildren<Text>().text = null; offsetSamples = 0; saveAudioClip = Microphone.Start(currentDeviceName, true, recordMaxLength, recordFrequency); } public void OnButtonClick() { if (Microphone.IsRecording(currentDeviceName)) { EndRecording(); } else { StartRecording(); } } public void EndRecording() { if (isHaveMic == false || !Microphone.IsRecording(currentDeviceName)) { return; } micOFF.gameObject.SetActive(true); micON.gameObject.SetActive(false); //结束录音 Microphone.End(currentDeviceName); } public bool IsRecording() { return Microphone.IsRecording(currentDeviceName); } IEnumerator GetAudioFrames() { while (true) { if (Microphone.IsRecording(currentDeviceName)) { int lenght = Microphone.GetPosition(currentDeviceName) * saveAudioClip.channels - offsetSamples; if (lenght > 0) { float[] samples = new float[lenght]; saveAudioClip.GetData(samples, offsetSamples); var samplesShort = new short[samples.Length]; for (var index = 0; index < samples.Length; index++) { samplesShort[index] = (short)(samples[index] * short.MaxValue); } byte[] binaryData = new byte[samplesShort.Length * 2]; Buffer.BlockCopy(samplesShort, 0, binaryData, 0, binaryData.Length); offsetSamples += lenght; DataAvailable?.Invoke(binaryData); } yield return new WaitForSeconds(0.16f); } else { yield return new WaitForSeconds(0.16f); byte[] binaryData = new byte[2]; DataAvailable?.Invoke(binaryData); } } } /// <summary> /// 获取毫秒级别的时间戳,用于计算按下录音时长 /// </summary> /// <returns></returns> public double GetTimestampOfNowWithMillisecond() { return (DateTime.Now.ToUniversalTime().Ticks - 621355968000000000) / 10000; } private void LateUpdate() { if (isHaveMic == true && Microphone.IsRecording(currentDeviceName)) { micAmount.fillAmount = Volume; } } public float Volume { get { if (Microphone.IsRecording(currentDeviceName)) { // 采样数 int sampleSize = 128; float[] samples = new float[sampleSize]; int startPosition = Microphone.GetPosition(currentDeviceName) - (sampleSize + 1); // 得到数据 if (startPosition < 0) return 0; saveAudioClip.GetData(samples, startPosition); // Getting a peak on the last 128 samples float levelMax = 0; for (int i = 0; i < sampleSize; ++i) { float wavePeak = samples[i]; if (levelMax < wavePeak) levelMax = wavePeak; } return levelMax; } return 0; } } void OnGUI() { GUIStyle guiStyle = GUIStyle.none; guiStyle.fontSize = 10; guiStyle.normal.textColor = Color.white; guiStyle.alignment = TextAnchor.UpperLeft; Rect tr = new Rect(0, 0, 100, 100); GUI.Label(tr, currentDeviceName, guiStyle); } } }
7. 新建空物体，挂载脚本RecorderManager和RealtimeAsrManager
8. 输入百度asr的appId和secretKey：
9. 输入GPTkey、问心一眼appId和secretKey：
10. 根据需求选择相应的NLP方式：
注意：如果缺少Singleton文件，可使用如下代码：
using UnityEngine; namespace Motionverse { public abstract class Singleton<T> : MonoBehaviour where T : Singleton<T> { private static T sInstance = null; public static T Instance { get { if (sInstance == null) { GameObject gameObject = new(typeof(T).FullName); sInstance = gameObject.AddComponent<T>(); } return sInstance; } } public static void Clear() { sInstance = null; } protected virtual void Awake() { if (sInstance != null) Debug.LogError(name + "error: already initialized", this); sInstance = (T)this; } } }若有收获，就点个赞吧
websocivajsonunitunitychatcodeappasrcliguiurltokengptdebugchatgptsrestemnlprtcclip语言模型rap大语言模型sshiconbaiducreateasoapiactionidectoscriptoauth生成式ai技术智能体生成式ai语音转换ai智能ppt制作软件
