feat: 大模型接入，TTS 接入

src/renderer/components/layout/HeaderLayout.vue

@@ -10,7 +10,8 @@ const route: any = useRoute()
 const { settings } = useStore()
 const setting = storeToRefs(settings)
 
-settings.getSource()
+
+settings.tts === 'xf_tts' && settings.getSource()
 
 const handleRoute = (path: string): void => {
   router.push(path)
@@ -138,12 +139,20 @@ async function changeOpenDevTools() {
                 </template>
 
                 <v-text-field
-                  label="TTS 域名"
-                  :rules="[(value) => !!value || 'TTS 域名必填']"
+                  label="TTS 地址"
+                  :rules="[(value) => !!value || 'TTS 地址必填']"
                   hide-details="auto"
                   :model-value="setting.ttsHost"
                 ></v-text-field>
 
+                <v-text-field
+                  style="margin-top: 22px;"
+                  label="LLM 地址"
+                  :rules="[(value) => !!value || 'LLM 地址必填']"
+                  hide-details="auto"
+                  :model-value="setting.llmUrl"
+                ></v-text-field>
+
 
                 <v-switch
                   v-model="setting.isFullscreen.value"

src/renderer/plugins/tts/FetchTTS.ts

@@ -34,3 +34,23 @@ export async function audioAiTTS({
   if (res.code !== 200) throw new Error(JSON.stringify(res))
   return res.data
 }
+
+
+export async function localTTS({ url, text }: { url: string; text: string; }){
+  const resp = await fetch(url, {
+    headers: {
+      accept: 'application/json, text/plain, */*',
+      'content-type': 'application/json'
+    },
+    body: JSON.stringify({
+      text
+    }),
+    method: 'POST',
+    mode: 'cors'
+  })
+
+  const res = await resp.json()
+  if (res.results.length < 1) throw new Error(JSON.stringify(res))
+
+  return res.results
+}
\ No newline at end of file

src/renderer/screens/ShowPhoto.vue

@@ -7,7 +7,7 @@ import type {
   ServerMessageResult,
   Model
 } from '@/renderer/plugins/asr/index'
-import { audioAiTTS } from '../plugins/tts'
+import { audioAiTTS, localTTS } from '../plugins/tts'
 import useStore from '@/renderer/store'
 import flvjs from 'flv.js';
 
@@ -47,7 +47,7 @@ async function init(){
   canvasEle.width = img.naturalWidth;
   canvasEle.height = img.naturalHeight;
 
-  initPlayer(videoEle);
+  // initPlayer(videoEle);
 
   const fps = 1000 / 30;
   let lastTime = Date.now();
@@ -89,6 +89,7 @@ function draw(ctx: CanvasRenderingContext2D, img: HTMLImageElement, liveVideo?:
     }
   }
 
+// eslint-disable-next-line no-unused-vars
 async function initPlayer(videoEle: HTMLVideoElement){
   flvPlayer = flvjs.createPlayer({
     url: 'http://127.0.0.1:7001/live/movie.flv',
@@ -169,40 +170,23 @@ const inputContext: {
   audioContext2?: AudioContext
   scriptProcessorNode?: ScriptProcessorNode
   model?: Model
+  ws?: WebSocket;
 } = {}
 
 async function startAudioInput() {
   if (microphoneState.value === 'loading') return
 
   if (microphoneState.value === 'input') {
-    microphoneState.value = 'waitInput'
-    inputContext.mediaStream?.getTracks().forEach((track) => track.stop())
-    inputContext.audioContext?.close()
-    inputContext.audioContext2?.close()
-    inputContext.scriptProcessorNode && (inputContext.scriptProcessorNode.onaudioprocess = null)
-    inputContext.model?.terminate()
+    endAudioInput();
     return
   }
 
   microphoneState.value = 'loading'
 
   const { recognizer, channel } = await initVosk({
-    result: async (text) => {
-      console.log('----------------> text:', text)
-      const tone = settings.source.find(({ sourceId }) => settings.selectSource === sourceId)
-      if (!tone) return
-      const res = await audioAiTTS({
-        host: settings.ttsHost,
-        text,
-        speed: 3,
-        speaker: tone.sourceId,
-        provider: tone.provider
-      })
-
-      console.log('----------------> tts:', res)
-    },
+    result: onAsr,
     partialResult: (text) => {
-      console.log('----------------> partialResult:', text)
+      // console.log('----------------> partialResult:', text)
     }
   })
 
@@ -245,8 +229,101 @@ async function startAudioInput() {
 }
 
 function endAudioInput() {
-  console.log('----------------> end')
+  microphoneState.value = 'waitInput'
+  inputContext.mediaStream?.getTracks().forEach((track) => track.stop())
+  inputContext.audioContext?.close()
+  inputContext.audioContext2?.close()
+  inputContext.scriptProcessorNode && (inputContext.scriptProcessorNode.onaudioprocess = null)
+  inputContext.model?.terminate()
+  // inputContext.ws?.close()
 }
+
+async function onAsr(question: string) {
+  const ws = await initSocket();
+  inputContext.ws = ws;
+
+  let sliceAnswer = '';
+  let answer = '';
+  const answerArray: string[] = [];
+  let isTime = true;
+
+  ws.onmessage = (message) => {
+    try {
+      const { text, event }  = JSON.parse(message.data) as { event: string; message_num: number; text: string; }
+
+      if (event === 'stream_end'){
+        answerArray.push(sliceAnswer)
+        sliceAnswer = '';
+        inputContext.ws?.close();
+        console.log('----------------> answer: ', answer);
+        return;
+      }
+
+      answer += text;
+      isTime && console.time('sliceAnswer');
+      isTime = false;
+      sliceAnswer += text;
+
+      if (/[。，？！；,.?!;]/.test(text) && sliceAnswer.length >= 20) {
+        console.timeEnd('sliceAnswer');
+        answerArray.push(sliceAnswer)
+        runTTSTask(answerArray);
+        sliceAnswer = '';
+        isTime = true;
+      }
+
+    } catch (error) {
+      console.log('返回答案错误 -----> '+JSON.stringify(error))
+    }
+  }
+
+  console.log('----------------> Asr:', question)
+  ws.send(JSON.stringify({ prompt: question, historys_list: [] }))
+}
+
+function initSocket(): Promise<WebSocket>{
+  const ws = new WebSocket(settings.llmUrl);
+  return new Promise((resolve, reject) => {
+    ws.onopen = () => resolve(ws);
+    ws.onerror = reject;
+  });
+}
+
+let isTTSRunning = false;
+async function runTTSTask(tasks: string[]) {
+  if (isTTSRunning) return;
+  isTTSRunning = true;
+
+  try {
+      while (tasks.length) {
+        const task = tasks.shift()
+        if (!task) break;
+        console.time(task+' TTS: ');
+        const res = await localTTS({ url: settings.ttsHost, text: task });
+        console.log('----------------> TTS:', res);
+        console.timeEnd(task+' TTS: ');
+      }
+    } catch (error) {
+      console.error(error);
+    }
+
+    isTTSRunning = false;
+}
+
+// eslint-disable-next-line no-unused-vars
+async function xfTTS(text: string) {
+  const tone = settings.source.find(({ sourceId }) => settings.selectSource === sourceId)
+  if (!tone) return
+  const res = await audioAiTTS({
+    host: settings.ttsHost,
+    text,
+    speed: 3,
+    speaker: tone.sourceId,
+    provider: tone.provider
+  })
+  console.log('----------------> tts:', res)
+}
+
 </script>
 
 <template>
@@ -268,7 +345,6 @@ function endAudioInput() {
       size="x-large"
       :disabled="microphoneState === 'loading' || microphoneState === 'disabled'"
       @pointerdown="startAudioInput"
-      @pointerup="endAudioInput"
     >
       <v-icon v-if="microphoneState === 'waitInput'" icon="mdi-microphone"></v-icon>
       <v-icon v-if="microphoneState === 'loading'" icon="mdi-microphone-settings"></v-icon>

src/renderer/store/settings.ts

@@ -10,7 +10,7 @@ export type ISettings = {
   asr: 'vosk_asr' | 'xf_asr'
   voskModels: string[]
   voskSelectModel: string
-  tts: 'xf_tts' | 'local_tts',
+  tts: 'xf_tts' | 'local_tts'
   ttsHost: string
   source: {
     sourceName: string
@@ -21,8 +21,9 @@ export type ISettings = {
     sex: 1 | 0
   }[]
   selectSource: string
-  isFullscreen: 'yes' | 'no',
+  isFullscreen: 'yes' | 'no'
   isOpenDevTools: boolean
+  llmUrl: string
 }
 
 const useSettingsStore = defineStore('settings', {
@@ -51,7 +52,8 @@ const useSettingsStore = defineStore('settings', {
       source: [],
       selectSource: '',
       isFullscreen: 'no',
-      isOpenDevTools: false
+      isOpenDevTools: false,
+      llmUrl: 'ws://192.168.50.50:9001/api/v1/stream',
     }) as ISettings,
   getters: {},
   actions: {