Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Contribute to GitLab
Sign in / Register
Toggle navigation
C
CharIP-Electron
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
ali
CharIP-Electron
Commits
9d261b2d
You need to sign in or sign up before continuing.
Commit
9d261b2d
authored
Nov 30, 2023
by
ali
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
feat: 大模型接入,TTS 接入
parent
781772bd
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
138 additions
and
31 deletions
+138
-31
HeaderLayout.vue
src/renderer/components/layout/HeaderLayout.vue
+12
-3
FetchTTS.ts
src/renderer/plugins/tts/FetchTTS.ts
+20
-0
ShowPhoto.vue
src/renderer/screens/ShowPhoto.vue
+101
-25
settings.ts
src/renderer/store/settings.ts
+5
-3
No files found.
src/renderer/components/layout/HeaderLayout.vue
View file @
9d261b2d
...
@@ -10,7 +10,8 @@ const route: any = useRoute()
...
@@ -10,7 +10,8 @@ const route: any = useRoute()
const
{
settings
}
=
useStore
()
const
{
settings
}
=
useStore
()
const
setting
=
storeToRefs
(
settings
)
const
setting
=
storeToRefs
(
settings
)
settings
.
getSource
()
settings
.
tts
===
'xf_tts'
&&
settings
.
getSource
()
const
handleRoute
=
(
path
:
string
):
void
=>
{
const
handleRoute
=
(
path
:
string
):
void
=>
{
router
.
push
(
path
)
router
.
push
(
path
)
...
@@ -138,12 +139,20 @@ async function changeOpenDevTools() {
...
@@ -138,12 +139,20 @@ async function changeOpenDevTools() {
</
template
>
</
template
>
<v-text-field
<v-text-field
label=
"TTS
域名
"
label=
"TTS
地址
"
:rules=
"[(value) => !!value || 'TTS
域名
必填']"
:rules=
"[(value) => !!value || 'TTS
地址
必填']"
hide-details=
"auto"
hide-details=
"auto"
:model-value=
"setting.ttsHost"
:model-value=
"setting.ttsHost"
></v-text-field>
></v-text-field>
<v-text-field
style=
"margin-top: 22px;"
label=
"LLM 地址"
:rules=
"[(value) => !!value || 'LLM 地址必填']"
hide-details=
"auto"
:model-value=
"setting.llmUrl"
></v-text-field>
<v-switch
<v-switch
v-model=
"setting.isFullscreen.value"
v-model=
"setting.isFullscreen.value"
...
...
src/renderer/plugins/tts/FetchTTS.ts
View file @
9d261b2d
...
@@ -34,3 +34,23 @@ export async function audioAiTTS({
...
@@ -34,3 +34,23 @@ export async function audioAiTTS({
if
(
res
.
code
!==
200
)
throw
new
Error
(
JSON
.
stringify
(
res
))
if
(
res
.
code
!==
200
)
throw
new
Error
(
JSON
.
stringify
(
res
))
return
res
.
data
return
res
.
data
}
}
export
async
function
localTTS
({
url
,
text
}:
{
url
:
string
;
text
:
string
;
}){
const
resp
=
await
fetch
(
url
,
{
headers
:
{
accept
:
'application/json, text/plain, */*'
,
'content-type'
:
'application/json'
},
body
:
JSON
.
stringify
({
text
}),
method
:
'POST'
,
mode
:
'cors'
})
const
res
=
await
resp
.
json
()
if
(
res
.
results
.
length
<
1
)
throw
new
Error
(
JSON
.
stringify
(
res
))
return
res
.
results
}
\ No newline at end of file
src/renderer/screens/ShowPhoto.vue
View file @
9d261b2d
...
@@ -7,7 +7,7 @@ import type {
...
@@ -7,7 +7,7 @@ import type {
ServerMessageResult
,
ServerMessageResult
,
Model
Model
}
from
'@/renderer/plugins/asr/index'
}
from
'@/renderer/plugins/asr/index'
import
{
audioAiTTS
}
from
'../plugins/tts'
import
{
audioAiTTS
,
localTTS
}
from
'../plugins/tts'
import
useStore
from
'@/renderer/store'
import
useStore
from
'@/renderer/store'
import
flvjs
from
'flv.js'
;
import
flvjs
from
'flv.js'
;
...
@@ -47,7 +47,7 @@ async function init(){
...
@@ -47,7 +47,7 @@ async function init(){
canvasEle
.
width
=
img
.
naturalWidth
;
canvasEle
.
width
=
img
.
naturalWidth
;
canvasEle
.
height
=
img
.
naturalHeight
;
canvasEle
.
height
=
img
.
naturalHeight
;
initPlayer
(
videoEle
);
//
initPlayer(videoEle);
const
fps
=
1000
/
30
;
const
fps
=
1000
/
30
;
let
lastTime
=
Date
.
now
();
let
lastTime
=
Date
.
now
();
...
@@ -89,6 +89,7 @@ function draw(ctx: CanvasRenderingContext2D, img: HTMLImageElement, liveVideo?:
...
@@ -89,6 +89,7 @@ function draw(ctx: CanvasRenderingContext2D, img: HTMLImageElement, liveVideo?:
}
}
}
}
// eslint-disable-next-line no-unused-vars
async
function
initPlayer
(
videoEle
:
HTMLVideoElement
){
async
function
initPlayer
(
videoEle
:
HTMLVideoElement
){
flvPlayer
=
flvjs
.
createPlayer
({
flvPlayer
=
flvjs
.
createPlayer
({
url
:
'http://127.0.0.1:7001/live/movie.flv'
,
url
:
'http://127.0.0.1:7001/live/movie.flv'
,
...
@@ -169,40 +170,23 @@ const inputContext: {
...
@@ -169,40 +170,23 @@ const inputContext: {
audioContext2
?:
AudioContext
audioContext2
?:
AudioContext
scriptProcessorNode
?:
ScriptProcessorNode
scriptProcessorNode
?:
ScriptProcessorNode
model
?:
Model
model
?:
Model
ws
?:
WebSocket
;
}
=
{}
}
=
{}
async
function
startAudioInput
()
{
async
function
startAudioInput
()
{
if
(
microphoneState
.
value
===
'loading'
)
return
if
(
microphoneState
.
value
===
'loading'
)
return
if
(
microphoneState
.
value
===
'input'
)
{
if
(
microphoneState
.
value
===
'input'
)
{
microphoneState
.
value
=
'waitInput'
endAudioInput
();
inputContext
.
mediaStream
?.
getTracks
().
forEach
((
track
)
=>
track
.
stop
())
inputContext
.
audioContext
?.
close
()
inputContext
.
audioContext2
?.
close
()
inputContext
.
scriptProcessorNode
&&
(
inputContext
.
scriptProcessorNode
.
onaudioprocess
=
null
)
inputContext
.
model
?.
terminate
()
return
return
}
}
microphoneState
.
value
=
'loading'
microphoneState
.
value
=
'loading'
const
{
recognizer
,
channel
}
=
await
initVosk
({
const
{
recognizer
,
channel
}
=
await
initVosk
({
result
:
async
(
text
)
=>
{
result
:
onAsr
,
console
.
log
(
'----------------> text:'
,
text
)
const
tone
=
settings
.
source
.
find
(({
sourceId
})
=>
settings
.
selectSource
===
sourceId
)
if
(
!
tone
)
return
const
res
=
await
audioAiTTS
({
host
:
settings
.
ttsHost
,
text
,
speed
:
3
,
speaker
:
tone
.
sourceId
,
provider
:
tone
.
provider
})
console
.
log
(
'----------------> tts:'
,
res
)
},
partialResult
:
(
text
)
=>
{
partialResult
:
(
text
)
=>
{
console
.
log
(
'----------------> partialResult:'
,
text
)
//
console.log('----------------> partialResult:', text)
}
}
})
})
...
@@ -245,8 +229,101 @@ async function startAudioInput() {
...
@@ -245,8 +229,101 @@ async function startAudioInput() {
}
}
function
endAudioInput
()
{
function
endAudioInput
()
{
console
.
log
(
'----------------> end'
)
microphoneState
.
value
=
'waitInput'
inputContext
.
mediaStream
?.
getTracks
().
forEach
((
track
)
=>
track
.
stop
())
inputContext
.
audioContext
?.
close
()
inputContext
.
audioContext2
?.
close
()
inputContext
.
scriptProcessorNode
&&
(
inputContext
.
scriptProcessorNode
.
onaudioprocess
=
null
)
inputContext
.
model
?.
terminate
()
// inputContext.ws?.close()
}
}
async
function
onAsr
(
question
:
string
)
{
const
ws
=
await
initSocket
();
inputContext
.
ws
=
ws
;
let
sliceAnswer
=
''
;
let
answer
=
''
;
const
answerArray
:
string
[]
=
[];
let
isTime
=
true
;
ws
.
onmessage
=
(
message
)
=>
{
try
{
const
{
text
,
event
}
=
JSON
.
parse
(
message
.
data
)
as
{
event
:
string
;
message_num
:
number
;
text
:
string
;
}
if
(
event
===
'stream_end'
){
answerArray
.
push
(
sliceAnswer
)
sliceAnswer
=
''
;
inputContext
.
ws
?.
close
();
console
.
log
(
'----------------> answer: '
,
answer
);
return
;
}
answer
+=
text
;
isTime
&&
console
.
time
(
'sliceAnswer'
);
isTime
=
false
;
sliceAnswer
+=
text
;
if
(
/
[
。,?!;,.?!;
]
/
.
test
(
text
)
&&
sliceAnswer
.
length
>=
20
)
{
console
.
timeEnd
(
'sliceAnswer'
);
answerArray
.
push
(
sliceAnswer
)
runTTSTask
(
answerArray
);
sliceAnswer
=
''
;
isTime
=
true
;
}
}
catch
(
error
)
{
console
.
log
(
'返回答案错误 -----> '
+
JSON
.
stringify
(
error
))
}
}
console
.
log
(
'----------------> Asr:'
,
question
)
ws
.
send
(
JSON
.
stringify
({
prompt
:
question
,
historys_list
:
[]
}))
}
function
initSocket
():
Promise
<
WebSocket
>
{
const
ws
=
new
WebSocket
(
settings
.
llmUrl
);
return
new
Promise
((
resolve
,
reject
)
=>
{
ws
.
onopen
=
()
=>
resolve
(
ws
);
ws
.
onerror
=
reject
;
});
}
let
isTTSRunning
=
false
;
async
function
runTTSTask
(
tasks
:
string
[])
{
if
(
isTTSRunning
)
return
;
isTTSRunning
=
true
;
try
{
while
(
tasks
.
length
)
{
const
task
=
tasks
.
shift
()
if
(
!
task
)
break
;
console
.
time
(
task
+
' TTS: '
);
const
res
=
await
localTTS
({
url
:
settings
.
ttsHost
,
text
:
task
});
console
.
log
(
'----------------> TTS:'
,
res
);
console
.
timeEnd
(
task
+
' TTS: '
);
}
}
catch
(
error
)
{
console
.
error
(
error
);
}
isTTSRunning
=
false
;
}
// eslint-disable-next-line no-unused-vars
async
function
xfTTS
(
text
:
string
)
{
const
tone
=
settings
.
source
.
find
(({
sourceId
})
=>
settings
.
selectSource
===
sourceId
)
if
(
!
tone
)
return
const
res
=
await
audioAiTTS
({
host
:
settings
.
ttsHost
,
text
,
speed
:
3
,
speaker
:
tone
.
sourceId
,
provider
:
tone
.
provider
})
console
.
log
(
'----------------> tts:'
,
res
)
}
</
script
>
</
script
>
<
template
>
<
template
>
...
@@ -268,7 +345,6 @@ function endAudioInput() {
...
@@ -268,7 +345,6 @@ function endAudioInput() {
size=
"x-large"
size=
"x-large"
:disabled=
"microphoneState === 'loading' || microphoneState === 'disabled'"
:disabled=
"microphoneState === 'loading' || microphoneState === 'disabled'"
@
pointerdown=
"startAudioInput"
@
pointerdown=
"startAudioInput"
@
pointerup=
"endAudioInput"
>
>
<v-icon
v-if=
"microphoneState === 'waitInput'"
icon=
"mdi-microphone"
></v-icon>
<v-icon
v-if=
"microphoneState === 'waitInput'"
icon=
"mdi-microphone"
></v-icon>
<v-icon
v-if=
"microphoneState === 'loading'"
icon=
"mdi-microphone-settings"
></v-icon>
<v-icon
v-if=
"microphoneState === 'loading'"
icon=
"mdi-microphone-settings"
></v-icon>
...
...
src/renderer/store/settings.ts
View file @
9d261b2d
...
@@ -10,7 +10,7 @@ export type ISettings = {
...
@@ -10,7 +10,7 @@ export type ISettings = {
asr
:
'vosk_asr'
|
'xf_asr'
asr
:
'vosk_asr'
|
'xf_asr'
voskModels
:
string
[]
voskModels
:
string
[]
voskSelectModel
:
string
voskSelectModel
:
string
tts
:
'xf_tts'
|
'local_tts'
,
tts
:
'xf_tts'
|
'local_tts'
ttsHost
:
string
ttsHost
:
string
source
:
{
source
:
{
sourceName
:
string
sourceName
:
string
...
@@ -21,8 +21,9 @@ export type ISettings = {
...
@@ -21,8 +21,9 @@ export type ISettings = {
sex
:
1
|
0
sex
:
1
|
0
}[]
}[]
selectSource
:
string
selectSource
:
string
isFullscreen
:
'yes'
|
'no'
,
isFullscreen
:
'yes'
|
'no'
isOpenDevTools
:
boolean
isOpenDevTools
:
boolean
llmUrl
:
string
}
}
const
useSettingsStore
=
defineStore
(
'settings'
,
{
const
useSettingsStore
=
defineStore
(
'settings'
,
{
...
@@ -51,7 +52,8 @@ const useSettingsStore = defineStore('settings', {
...
@@ -51,7 +52,8 @@ const useSettingsStore = defineStore('settings', {
source
:
[],
source
:
[],
selectSource
:
''
,
selectSource
:
''
,
isFullscreen
:
'no'
,
isFullscreen
:
'no'
,
isOpenDevTools
:
false
isOpenDevTools
:
false
,
llmUrl
:
'ws://192.168.50.50:9001/api/v1/stream'
,
})
as
ISettings
,
})
as
ISettings
,
getters
:
{},
getters
:
{},
actions
:
{
actions
:
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment