Skip to content

C
CharIP-Electron
Commit 38df065c authored by ali's avatar ali
Browse files
Options

fix: 优化相关异常处理

parent dff91c47
Hide whitespace changes
Inline Side-by-side
Showing with 167 additions and 172 deletions
src/renderer/screens/ShowPhoto.vue
View file @ 38df065c
......@@ -50,10 +50,10 @@ onMounted(() => {
})
const errorSnackbar = ref<boolean>(false)
const errorMsg = ref<string>('');
const errorMsg = ref<string>('')
const showError = (msg: string) => {
errorSnackbar.value = true;
errorMsg.value = msg;
errorSnackbar.value = true
errorMsg.value = msg
}
function loadImg(): Promise<HTMLImageElement> {
......@@ -77,17 +77,7 @@ async function init() {
canvasEle.height = img.naturalHeight
const item = photo.list.find((i) => i.url === url)
photoRole = new PhotoRole(settings.liveHost, `${item?.liveUrl}`, canvasEle)
photoRole.on('asyncAnswer', async (ans) => {
if (ans.playState === 'playing') {
microphoneState.value = 'reply'
return
}
if (microphoneState.value === 'reply' && ans.playState === 'pause' && (await checkSteps())) {
microphoneState.value = 'input'
}
})
photoRole.on('asyncAnswer', onAsyncAnswer)
// initPlayer(videoEle);
......@@ -112,6 +102,17 @@ async function init() {
requestAnimationFrame(updateFrame)
}
async function onAsyncAnswer(ans: PhotoAnswer) {
if (ans.playState === 'playing') {
microphoneState.value = 'reply'
return
}
if (microphoneState.value === 'reply' && ans.playState === 'pause' && (await checkSteps())) {
microphoneState.value = 'input'
}
}
function draw(
ctx: CanvasRenderingContext2D,
img: HTMLImageElement,
......@@ -273,7 +274,7 @@ async function startVoskWsAudioInput() {
if (microphoneState.value === 'loading') return
if (microphoneState.value === 'input') {
endAudioInput()
await endAudioInput()
return
}
......@@ -378,6 +379,9 @@ async function endAudioInput() {
}
inputContext.asrPartial = ''
inputContext.answerArray.length = 0
inputContext.steps.length = 0
// @ts-ignore
photoRole?.off('asyncAnswer', onAsyncAnswer)
await photoRole?.destroy()
}
......@@ -417,73 +421,80 @@ async function onQ(question: string) {
microphoneState.value = 'loading'
const ws = await initLLMSocket()
const { pose, stepResolve, stepReject } = createStep()
const messageTimeout = setTimeout(async () => {
showError('llm：timeout!')
await endAudioInput()
microphoneState.value = 'waitInput'
}, 5000);
let sliceAnswer = ''
let answer = ''
let isTime = true
let sliceAnswerLength = 10
inputContext.ws = ws
inputContext.answerArray.length = 0
inputContext.steps.length = 0
inputContext.steps.push(pose)
photoRole!.answerArgs = new PhotoAnswer()
ws.onmessage = (message) => {
try {
const { text, event } = JSON.parse(message.data) as {
event: string
message_num: number
text: string
}
if (event === 'stream_end') {
inputContext.answerArray.push({ text: sliceAnswer, isLast: true })
sliceAnswer = ''
runTTSTask()
inputContext.ws?.close()
console.log('----------------> answer: ', answer)
stepResolve('chat')
return
}
try {
const ws = await initLLMSocket()
const { pose, stepResolve, stepReject } = createStep()
const messageTimeout = setTimeout(async () => {
showError('llm：timeout!')
await endAudioInput()
microphoneState.value = 'waitInput'
}, 10000)
let sliceAnswer = ''
let answer = ''
let isTime = true
let sliceAnswerLength = 10
inputContext.ws = ws
inputContext.answerArray.length = 0
inputContext.steps.length = 0
inputContext.steps.push(pose)
photoRole!.answerArgs = new PhotoAnswer()
photoRole!.on('asyncAnswer', onAsyncAnswer)
ws.onmessage = (message) => {
try {
const { text, event } = JSON.parse(message.data) as {
event: string
message_num: number
text: string
}
answer += text
photoRole!.answerArgs!.answer += answer
photoRole!.answerArgs!._typingAnswer.push(answer)
isTime && console.time('sliceAnswer')
isTime = false
clearTimeout(messageTimeout);
const textArr = text.split('')
for (let i = 0; i < textArr.length; i++) {
const t = textArr[i]
sliceAnswer += t
if (/[。，？！；,.?!;]/.test(t) && sliceAnswer.length >= sliceAnswerLength) {
console.timeEnd('sliceAnswer')
sliceAnswerLength = settings.llmToTTSSliceLength
if (event === 'stream_end') {
inputContext.answerArray.push({ text: sliceAnswer, isLast: true })
runTTSTask()
sliceAnswer = ''
isTime = true
runTTSTask()
inputContext.ws?.close()
console.log('----------------> answer: ', answer)
stepResolve('chat')
return
}
answer += text
photoRole!.answerArgs!.answer += answer
photoRole!.answerArgs!._typingAnswer.push(answer)
isTime && console.time('sliceAnswer')
isTime = false
clearTimeout(messageTimeout)
const textArr = text.split('')
for (let i = 0; i < textArr.length; i++) {
const t = textArr[i]
sliceAnswer += t
if (/[。，？！；,.?!;]/.test(t) && sliceAnswer.length >= sliceAnswerLength) {
console.timeEnd('sliceAnswer')
sliceAnswerLength = settings.llmToTTSSliceLength
inputContext.answerArray.push({ text: sliceAnswer, isLast: true })
runTTSTask()
sliceAnswer = ''
isTime = true
}
}
} catch (error) {
showError('llm：' + error)
endAudioInput().then(() => {
microphoneState.value = 'waitInput'
})
stepReject(JSON.stringify(error))
}
} catch (error) {
showError('llm：'+ error)
endAudioInput().then(() => {
microphoneState.value = 'waitInput'
})
stepReject(JSON.stringify(error))
}
}
ws.send(JSON.stringify({ prompt: question, historys_list: [] }))
ws.send(JSON.stringify({ prompt: question, historys_list: [] }))
} catch (error) {
console.error(error)
microphoneState.value = 'input'
showError(`llm：${JSON.stringify(error)}`)
}
}
let isTTSRunning = false
......@@ -525,7 +536,7 @@ async function runTTSTask() {
})
}
} catch (error) {
showError('tts：'+ error)
showError('tts：' + error)
endAudioInput().then(() => {
microphoneState.value = 'waitInput'
})
......@@ -577,7 +588,7 @@ async function down() {
try {
await startVoskWsAudioInput()
} catch (error) {
showError('start：'+ error)
showError('start：' + error)
endAudioInput().then(() => {
microphoneState.value = 'waitInput'
})
......@@ -631,22 +642,12 @@ async function down() {
</v-btn>
</div>
<v-snackbar
v-model="errorSnackbar"
multi-line
:timeout="3000"
>
{{ errorMsg }}
<template #actions>
<v-btn
color="red"
variant="text"
@click="errorSnackbar = false"
>
Close
</v-btn>
</template>
<v-snackbar v-model="errorSnackbar" multi-line :timeout="3000">
{{ errorMsg }}
<template #actions>
<v-btn color="red" variant="text" @click="errorSnackbar = false"> Close </v-btn>
</template>
</v-snackbar>
</template>
<style scoped>
......
src/renderer/screens/ShowVideo.vue
View file @ 38df065c
......@@ -35,8 +35,10 @@ const inputContext: {
voskWs?: WebSocket
asrPartial: string
playingAudio?: HTMLAudioElement
answerArray: string[]
} = {
asrPartial: ''
asrPartial: '',
answerArray: []
}
onMounted(() => {
......@@ -48,10 +50,10 @@ router.beforeEach((g) => {
})
const errorSnackbar = ref<boolean>(false)
const errorMsg = ref<string>('');
const errorMsg = ref<string>('')
const showError = (msg: string) => {
errorSnackbar.value = true;
errorMsg.value = msg;
errorSnackbar.value = true
errorMsg.value = msg
}
async function initVosk({
......@@ -275,6 +277,7 @@ function endAudioInput() {
videos[1].value?.pause()
videos[0].value?.pause()
isPlayRunning = false
inputContext.answerArray.length = 0
}
const canplay = () => {
......@@ -344,81 +347,82 @@ async function onQ(question: string) {
}
// 视频链接匹配不上，直接走大模型
const ws = await initLLMSocket()
const answerArray: string[] = []
const messageTimeout = setTimeout(() => {
showError('llm：timeout!')
endAudioInput()
microphoneState.value = 'waitInput'
}, 5000);
let sliceAnswer = ''
let answer = ''
let isTime = true
let sliceAnswerLength = 10
inputContext.ws = ws
ws.onmessage = (message) => {
if (microphoneState.value === 'input') {
return
}
try {
const { text, event } = JSON.parse(message.data) as {
event: string
message_num: number
text: string
try {
const ws = await initLLMSocket()
const messageTimeout = setTimeout(() => {
showError('llm：timeout!')
endAudioInput()
microphoneState.value = 'waitInput'
}, 10000)
let sliceAnswer = ''
let answer = ''
let isTime = true
let sliceAnswerLength = 10
inputContext.ws = ws
ws.onmessage = (message) => {
if (microphoneState.value === 'input') {
return
}
if (event === 'stream_end') {
answerArray.push(sliceAnswer)
runTTSTask(answerArray)
sliceAnswer = ''
try {
const { text, event } = JSON.parse(message.data) as {
event: string
message_num: number
text: string
}
answerArray.push(sliceAnswer)
sliceAnswer = ''
inputContext.ws?.close()
console.log('----------------> answer: ', answer)
return
}
if (event === 'stream_end') {
inputContext.answerArray.push(sliceAnswer)
runTTSTask()
sliceAnswer = ''
answer += text
isTime && console.time('sliceAnswer')
isTime = false
clearTimeout(messageTimeout);
const textArr = text.split('')
for (let i = 0; i < textArr.length; i++) {
const t = textArr[i]
sliceAnswer += t
if (/[。，？！；,.?!;]/.test(t) && sliceAnswer.length >= sliceAnswerLength) {
console.timeEnd('sliceAnswer')
sliceAnswerLength = settings.llmToTTSSliceLength
answerArray.push(sliceAnswer)
runTTSTask(answerArray)
inputContext.answerArray.push(sliceAnswer)
sliceAnswer = ''
isTime = true
inputContext.ws?.close()
console.log('----------------> answer: ', answer)
return
}
answer += text
isTime && console.time('sliceAnswer')
isTime = false
clearTimeout(messageTimeout)
const textArr = text.split('')
for (let i = 0; i < textArr.length; i++) {
const t = textArr[i]
sliceAnswer += t
if (/[。，？！；,.?!;]/.test(t) && sliceAnswer.length >= sliceAnswerLength) {
console.timeEnd('sliceAnswer')
sliceAnswerLength = settings.llmToTTSSliceLength
inputContext.answerArray.push(sliceAnswer)
runTTSTask()
sliceAnswer = ''
isTime = true
}
}
} catch (error) {
console.error(error)
showError(`message：${error}`)
microphoneState.value = 'waitInput'
}
} catch (error) {
console.error(error);
showError(`message：${error}`)
microphoneState.value = 'waitInput'
}
ws.send(JSON.stringify({ prompt: question, historys_list: [] }))
} catch (error) {
console.error(error)
microphoneState.value = 'input'
showError(`llm：${JSON.stringify(error)}`)
}
ws.send(JSON.stringify({ prompt: question, historys_list: [] }))
}
let isTTSRunning = false
async function runTTSTask(tasks: string[]) {
async function runTTSTask() {
if (isTTSRunning) return
isTTSRunning = true
microphoneState.value = 'loading'
try {
while (tasks.length) {
const task = tasks.shift()
while (inputContext.answerArray.length) {
const task = inputContext.answerArray.shift()
if (!task) break
if (task.trim().length < 1) continue
......@@ -560,22 +564,12 @@ async function down() {
</v-chip>
</div>
<v-snackbar
v-model="errorSnackbar"
multi-line
:timeout="3000"
>
{{ errorMsg }}
<template #actions>
<v-btn
color="red"
variant="text"
@click="errorSnackbar = false"
>
Close
</v-btn>
</template>
<v-snackbar v-model="errorSnackbar" multi-line :timeout="3000">
{{ errorMsg }}
<template #actions>
<v-btn color="red" variant="text" @click="errorSnackbar = false"> Close </v-btn>
</template>
</v-snackbar>
</template>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment