3D 人脸渲染和 Web 音频同步：调整帧率、采样率、网络协议和代码实现

{"title":"如何调整帧率和音频的采样率等参数，来实现3D人脸通过UE渲染和web音频输出时如何保持同步？ 三方使用什么网络协议？\n\n请根据代码以完整实现同步问题：\n/ todo 标模音频驱动表情算法\nfunc (this NoticeDetectionCtl) OnModelVoiceDriver() {\n\tvar resMap protocol.STAReq\n\terr := this.UnMarshal(&resMap)\n\tif err != nil {\n\t logs.Error("OnModelVoiceDriver, %s, err = %v", string(this.Ctx.Input.RequestBody), err)\n\t this.OnError(errno.EC_InvalidArg, errno.EM_InvalidArg)\n\t return\n\t}\n\tif resMap.Input == "" {\n\t logs.Error("OnModelVoiceDriver,无效参数")\n\t this.OnError(errno.EC_InvalidArg, errno.EM_InvalidArg)\n\t return\n\t}\n\tlogs.Info("OnModelVoiceDriver, modelId=%d, userId=%d, gender=%d, flag=%d, input=%s", this.Session.ModelId, this.Session.Id, resMap.Gender, resMap.Flag, resMap.Input)\n\tcommonInfo := fmt.Sprintf("OnModelVoiceDriver OnSocket start")\n\n\t// 连接服务器\n\tconn, resp, err := websocket.DefaultDialer.Dial(appconf.TTSStaApi, nil)\n\tif err != nil {\n\t this.OnError(errno.EC_InnerError, errno.EM_InnerError)\n\t logs.Info("OnModelVoiceDriver OnSocket dial error: %v\n", err)\n\t return\n\t}\n\tdefer conn.Close()\n\n\t// 输出响应信息\n\tlogs.Info("OnModelVoiceDriver OnSocket ws resp: Status=%s, StatusCode=%d, Proto=%s\n",\n\t resp.Status, resp.StatusCode, resp.Proto)\n\n\tvar strSlice []string\n\t//构造请求数据\n\theader := ModelStaHeader(resMap.Gender)\n\tbody := ModelStaBody(resMap.Input)\n\n\t// 发送音频数据\n\tsendData := map[string]interface{}{"head": header, "data": body}\n\tsendDataBytes, _ := json.Marshal(sendData)\n\terr = conn.WriteMessage(websocket.TextMessage, sendDataBytes)\n\tif websocket.IsCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) {\n\t logs.Info("OnModelVoiceDriver Connection closed")\n\t return\n\t} else if err != nil {\n\t this.OnError(errno.EC_InnerError, errno.EM_InnerError)\n\t logs.Error("OnModelVoiceDriver Error sending audio data:", err)\n\t}\n\n\tvar bsFrames [][]int32\n\tvar orderStr [][]string\n\t//var builder strings.Builder\n\t// 接收服务端返回的数据流\n\tfor {\n\t _, msg, err := conn.ReadMessage()\n\t if websocket.IsCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) {\n\t logs.Info(" OnModelVoiceDriver Connection closed")\n\t return\n\t } else if err != nil {\n\t this.OnError(errno.EC_InnerError, errno.EM_InnerError)\n\t logs.Error("OnModelVoiceDriver read error: %v\n", err)\n\t return\n\t }\n\n\t var res protocol.ModelSTARes\n\t err = json.Unmarshal(msg, &res)\n\t if err != nil {\n\t this.OnError(errno.EC_InnerError, errno.EM_InnerError)\n\t logs.Error("receiveData, Parse JSON response and handle data,err: ", err)\n\t return\n\t }\n\n\t // Handle content type\n\t switch res.Code {\n\t case 200:\n\t // Decode base64 audio data and save to file\n\t if len(res.Data.VesAnimeResult.Order) == 0 {\n\t this.OnError(errno.EC_InnerError, errno.EM_InnerError)\n\t logs.Error("Failed to get VesAnimeResult.Order: %v\n", err)\n\t return\n\t }\n\t if len(res.Data.VesAnimeResult.Frame[0]) == 0 {\n\t this.OnError(errno.EC_InnerError, errno.EM_InnerError)\n\t logs.Error("res.Data.VesAnimeResult.Frame: %v\n", err)\n\t return\n\t }\n\t if res.Data.VesAnimeResult.Coefl == 0 {\n\t this.OnError(errno.EC_InnerError, errno.EM_InnerError)\n\t logs.Error("res.Data.VesAnimeResult.Coefl: %v\n", err)\n\t return\n\t }\n\t // 解析数据处理后给UE\n\t bsFrames = append(bsFrames, res.Data.VesAnimeResult.Frame...)\n\t orderStr = append(orderStr, res.Data.VesAnimeResult.Order)\n\t content, err := base64.StdEncoding.DecodeString(res.Data.Content)\n\t if err != nil {\n\t this.OnError(errno.EC_InnerError, errno.EM_InnerError)\n\t logs.Error("res.Data.Content decode error: %v", err)\n\t }\n\t strSlice = append(strSlice, string(content))\n\t //builder.WriteString(res.Data.Content)\n\t default:\n\t logs.Error("Received response code: %d\n", res.Code)\n\t this.OnError(errno.EC_InnerError, errno.EM_InnerError)\n\t }\n\t // 根据服务器返回的数据结束连接\n\t if res.Data.IsLast && res.Data.IsFinish {\n\t conn.Close()\n\t break\n\t }\n\t time.Sleep(10 * time.Millisecond)\n\n\t}\n\tif len(strSlice) == 0 {\n\t logs.Error("音频数据为空")\n\t this.OnError(errno.EC_InnerError, errno.EM_InnerError)\n\t}\n\tlogs.Info("OnModelVoiceDriver,orderStr： %v", orderStr)\n\tbyteSlice := FixFiles(strSlice)\n\twavStr := base64.StdEncoding.EncodeToString(byteSlice)\n\tHandelModelData(bsFrames, 10000, this.Token, commonInfo, resMap.Seq)\n\tlogs.Info("%s, finish", commonInfo)\n\ttime.Sleep(3 * time.Millisecond)\n\tif resMap.Flag == 1 {\n\t this.OnAddText(&resMap)\n\t}\n\tresult := make(map[string]interface{})\n\tif len(wavStr) != 0 {\n\t //result["audio"] = builder.String()\n\t result["audio"] = wavStr\n\t this.OnSuccess(result)\n\t} else {\n\t this.OnError(errno.EC_InnerError, errno.EM_InnerError)\n\t}\n}\n\n//todo 读取算法端数据处理给UE\nfunc HandelModelData(frames [][]int32, coefl float32, token string, commonInfo string, seq int) {\n\tvar finish bool = true\n\tvar sig chan int\n\t{\n\t var ok bool\n\t driverMutex.Lock()\n\t sig, ok = token2driver[token]\n\t if !ok {\n\t sig = make(chan int, 1)\n\t token2driver[token] = sig\n\t } else {\n\t logs.Info("%s, emmit sig flag, to quit last request", commonInfo)\n\t sig <- 0 // 退出前面的驱动\n\t }\n\t driverMutex.Unlock()\n\t}\n\n\tdefer func() {\n\t if finish {\n\t driverMutex.Lock()\n\t , ok := token2driver[token]\n\t if ok {\n\t close(sig)\n\t delete(token2driver, token)\n\t }\n\t driverMutex.Unlock()\n\t logs.Info("%s, finish", commonInfo)\n\t }\n\t}()\n\n\tapi := appconf.StreamInfoApi + "?token=" + token\n\tstatus, stream := util.OnGetHttpTimeout(api, 10)\n\tif status != 200 {\n\t logs.Error("%s, api = %v, status = %d, result = %s", commonInfo, api, status, string(stream))\n\t return\n\t}\n\tvar streamInfoRes protocol.StreamInfoRes\n\terr := json.Unmarshal(stream, &streamInfoRes)\n\tif err != nil {\n\t logs.Error("%s, err = %v", commonInfo, err)\n\t return\n\t}\n\tif streamInfoRes.Code != 200 {\n\t logs.Error("%s HandelModelData, err = %v", commonInfo, streamInfoRes)\n\t}\n\tsip := streamInfoRes.Data.Ip\n\tsport := 11111\n\tif sip == "" {\n\t logs.Error("%s, invalid ip", commonInfo)\n\t return\n\t}\n\tsocket, err := net.DialUDP("udp", nil, &net.UDPAddr{\n\t IP: net.ParseIP(sip),\n\t Port: sport,\n\t})\n\tif err != nil {\n\t logs.Error("%s, ip = %s, port = %d, err = %v", commonInfo, sip, sport, err)\n\t return\n\t}\n\tdefer socket.Close()\n\n\t = StartBSDriver(token, seq)\n\n\tlogs.Info("%s, sip = %s, sport = %d", commonInfo, sip, sport)\n\n\t{\n\t startReq := make(map[string]interface{})\n\t startReq["userIp"] = sip\n\t startData, _ := json.Marshal(&startReq)\n\t var nsent = 0\n\t for nsent < len(startData) {\n\t n, err := socket.Write(startData[nsent:])\n\t if err != nil {\n\t logs.Error("%s, ip = %s, port = %d, err = %v", commonInfo, sip, sport, err)\n\t }\n\t nsent += n\n\t }\n\t logs.Info("%s, ip = %s, port = %d, OnAudioDriver", commonInfo, sip, sport)\n\t}\n\nLoop:\n\tfor i := 0; i < len(frames); i++ {\n\t frame := make([]float32, 61)\n\t for j := 0; j < 51; j++ {\n\t if j == 1 || j == 4 || j == 11 || j == 40 || j == 41 || j == 42 || j == 43 || j == 44 || j == 45 || j == 8 || j == 9 || j == 10 {\n\t if frames[i][j] != 0 {\n\t frames[i][j] = 0\n\t }\n\t }\n\t frame[j] = float32(frames[i][j]) / coefl\n\t }\n\t select {\n\t case <-sig:\n\t logs.Info("%s, recv quit sig, quit now", commonInfo)\n\t finish = false\n\t break Loop\n\t default:\n\t data := MakeBSFrame(frame)\n\t var nsent = 0\n\t for nsent < len(data) {\n\t n, err := socket.Write(data[nsent:])\n\t if err != nil {\n\t fmt.Println("send, err =", err)\n\t }\n\t nsent += n\n\t }\n\t time.Sleep(20 * time.Millisecond)\n\t //time.Sleep(15time.Millisecond + 650*time.Microsecond)\n\t }\n\t}\n\t_ = StopBSDriver(token, seq)\n\treturn\n}