Skip to content

C
CharIP-Electron
Commit fd1558a6 authored by ali's avatar ali
Browse files
Options

feat: TTS 接口接入，完成各项功能支持配置功能

parent 90ce5548
Hide whitespace changes
Inline Side-by-side
Showing with 1198 additions and 1002 deletions
.vscode/settings.json
View file @ fd1558a6
...@@ -21,5 +21,5 @@ ...@@ -21,5 +21,5 @@
"cSpell.words": [ "cSpell.words": [
"Vosk" "Vosk"
], ],
"editor.inlineSuggest.showToolbar": "always" "editor.inlineSuggest.showToolbar": "onHover"
} }
README.md
View file @ fd1558a6
# chartIP-Electron # chartIP-Electron
\ No newline at end of file
buildAssets/builder/config.js
View file @ fd1558a6
...@@ -69,7 +69,7 @@ const baseConfig = { ...@@ -69,7 +69,7 @@ const baseConfig = {
oneClick: true oneClick: true
}, },
linux: { linux: {
executableName: 'vutron', executableName: 'chartIP',
icon: 'buildAssets/icons', icon: 'buildAssets/icons',
category: 'Utility', category: 'Utility',
target: [ target: [
......
src/main/IPCs.ts
View file @ fd1558a6
...@@ -5,8 +5,7 @@ import Constants from './utils/Constants' ...@@ -5,8 +5,7 @@ import Constants from './utils/Constants'
* IPC Communications * IPC Communications
* */ * */
export default class IPCs { export default class IPCs {
static browserWindows: Map<string, BrowserWindow[]> = new Map()
static browserWindows: Map<string, BrowserWindow[]> = new Map();
static initialize(window: BrowserWindow): void { static initialize(window: BrowserWindow): void {
// Get application version // Get application version
...@@ -20,41 +19,57 @@ export default class IPCs { ...@@ -20,41 +19,57 @@ export default class IPCs {
}) })
// open new window // open new window
ipcMain.on('openWindow', async (event, url: string, options: BrowserWindowConstructorOptions & { isCloseOther: boolean }) => { ipcMain.on(
const ops = Object.assign({}, { isCloseOther: true, frame: false, useContentSize: true, webPreferences: Constants.DEFAULT_WEB_PREFERENCES }, options); 'openWindow',
async (
event,
url: string,
options: BrowserWindowConstructorOptions & { isCloseOther: boolean }
) => {
const ops = Object.assign(
{},
{
isCloseOther: true,
frame: false,
useContentSize: true,
webPreferences: Constants.DEFAULT_WEB_PREFERENCES
},
options
)
if (IPCs.browserWindows.has(url) && ops.isCloseOther) { if (IPCs.browserWindows.has(url) && ops.isCloseOther) {
const wins = IPCs.browserWindows.get(url); const wins = IPCs.browserWindows.get(url)
wins?.forEach(w => !w.isDestroyed() && w.close()); wins?.forEach((w) => !w.isDestroyed() && w.close())
IPCs.browserWindows.set(url, []); IPCs.browserWindows.set(url, [])
} }
const win = new BrowserWindow(ops) const win = new BrowserWindow(ops)
win.setMenu(null) win.setMenu(null)
win.once('ready-to-show', (): void => { win.once('ready-to-show', (): void => {
win.setAlwaysOnTop(true) win.setAlwaysOnTop(true)
win.show() win.show()
win.focus() win.focus()
win.setAlwaysOnTop(false) win.setAlwaysOnTop(false)
}) })
win.webContents.on('did-frame-finish-load', (): void => { win.webContents.on('did-frame-finish-load', (): void => {
if (Constants.IS_DEV_ENV) { if (Constants.IS_DEV_ENV) {
win.webContents.openDevTools() win.webContents.openDevTools()
} }
}) })
await win.loadURL(`${Constants.APP_INDEX_URL_DEV}${url}`) await win.loadURL(`${Constants.APP_INDEX_URL_DEV}${url}`)
if (!IPCs.browserWindows.has(url)) { if (!IPCs.browserWindows.has(url)) {
IPCs.browserWindows.set(url, []); IPCs.browserWindows.set(url, [])
} }
IPCs.browserWindows.get(url)?.push(win); IPCs.browserWindows.get(url)?.push(win)
return win; return win
}) }
)
} }
} }
src/renderer/components/layout/DefaultLayout.vue
View file @ fd1558a6
<script setup lang="ts"> <script setup lang="ts">
import HeaderLayout from '@/renderer/components/layout/HeaderLayout.vue' import HeaderLayout from '@/renderer/components/layout/HeaderLayout.vue'
import { ref } from 'vue'; import { ref } from 'vue'
import { useRouter } from 'vue-router' import { useRouter } from 'vue-router'
const router = useRouter() const router = useRouter()
const isHeader = ref(true); const isHeader = ref(true)
router.beforeEach((guard) => { router.beforeEach((guard) => {
isHeader.value = typeof guard.meta.isHeader === 'boolean' ? (guard.meta.isHeader as boolean) : true; isHeader.value =
typeof guard.meta.isHeader === 'boolean' ? (guard.meta.isHeader as boolean) : true
}) })
</script> </script>
<template> <template>
......
src/renderer/components/layout/HeaderLayout.vue
View file @ fd1558a6
<script setup lang="tsx"> <script setup lang="tsx">
import { computed, ref } from 'vue'; import { computed, ref } from 'vue'
import { useRoute, useRouter } from 'vue-router' import { useRoute, useRouter } from 'vue-router'
import useStore from '@/renderer/store'; import useStore from '@/renderer/store'
import { storeToRefs } from 'pinia'; import { storeToRefs } from 'pinia'
import { audioAiTTS } from '@/renderer/plugins/tts' import { audioAiTTS } from '@/renderer/plugins/tts'
const router = useRouter() const router = useRouter()
const route: any = useRoute() const route: any = useRoute()
const { settings } = useStore(); const { settings } = useStore()
const setting = storeToRefs(settings); const setting = storeToRefs(settings)
settings.getSource(); settings.getSource()
const handleRoute = (path: string): void => { const handleRoute = (path: string): void => {
router.push(path) router.push(path)
...@@ -25,8 +25,8 @@ const asrItems = ref([ ...@@ -25,8 +25,8 @@ const asrItems = ref([
'vosk_asr', 'vosk_asr',
'xf_asr' 'xf_asr'
// 'Whisper Api' // 'Whisper Api'
]); ])
const asrSelect = ref(setting.asr); const asrSelect = ref(setting.asr)
const source = computed(() => { const source = computed(() => {
return setting.source.value.map(({ sourceId, sourceName, description, sex }) => { return setting.source.value.map(({ sourceId, sourceName, description, sex }) => {
...@@ -35,17 +35,22 @@ const source = computed(() => { ...@@ -35,17 +35,22 @@ const source = computed(() => {
value: sourceId, value: sourceId,
title: `${sourceName} - ${_sex} - ${description}` title: `${sourceName} - ${_sex} - ${description}`
} }
}); })
}); })
async function changeSource() { async function changeSource() {
const tone = setting.source.value.find(({ sourceId }) => setting.selectSource.value === sourceId); const tone = setting.source.value.find(({ sourceId }) => setting.selectSource.value === sourceId)
if (!tone) return; if (!tone) return
const res = await audioAiTTS({ host: settings.ttsHost, text: '你好，今天天气怎么样？', speed: 5.5, speaker: tone.sourceId, provider: tone.provider }); const res = await audioAiTTS({
host: settings.ttsHost,
console.log(res); text: '你好，今天天气怎么样？',
speed: 5.5,
speaker: tone.sourceId,
provider: tone.provider
})
console.log(res)
} }
</script> </script>
<template> <template>
<v-app-bar color="#d71b1b" density="compact" class="header"> <v-app-bar color="#d71b1b" density="compact" class="header">
...@@ -69,29 +74,20 @@ async function changeSource() { ...@@ -69,29 +74,20 @@ async function changeSource() {
<v-dialog width="600"> <v-dialog width="600">
<template #activator="{ props }"> <template #activator="{ props }">
<v-btn <v-btn v-bind="props" color="#fff" class="settings">
v-bind="props" <v-icon start icon="mdi-wrench"></v-icon>
color="#fff"
class="settings"
>
<v-icon
start
icon="mdi-wrench"
></v-icon>
配置 配置
</v-btn> </v-btn>
</template> </template>
<template #default="{ isActive }"> <template #default="{ isActive }">
<v-card title="配置"> <v-card title="配置">
<v-sheet width="500" class="mx-auto mt-6"> <v-sheet width="500" class="mx-auto mt-6">
<v-form ref="form"> <v-form ref="form">
<v-select <v-select
v-model="setting.asr.value" v-model="setting.asr.value"
:items="asrItems" :items="asrItems"
:rules="[v => !!v || '请选择 Asr']" :rules="[(v) => !!v || '请选择 Asr']"
label="语音识别（ASR）" label="语音识别（ASR）"
required required
></v-select> ></v-select>
...@@ -107,9 +103,7 @@ async function changeSource() { ...@@ -107,9 +103,7 @@ async function changeSource() {
<v-text-field <v-text-field
label="TTS 域名" label="TTS 域名"
:rules="[ :rules="[(value) => !!value || 'TTS 域名必填']"
value => !!value || 'TTS 域名必填',
]"
hide-details="auto" hide-details="auto"
:model-value="setting.ttsHost" :model-value="setting.ttsHost"
></v-text-field> ></v-text-field>
...@@ -118,26 +112,21 @@ async function changeSource() { ...@@ -118,26 +112,21 @@ async function changeSource() {
v-model="setting.selectSource.value" v-model="setting.selectSource.value"
class="mt-6" class="mt-6"
:items="source" :items="source"
:rules="[v => !!v || '请选择音色']" :rules="[(v) => !!v || '请选择音色']"
label="TTS 音色" label="TTS 音色"
required required
@update:model-value="changeSource" @update:model-value="changeSource"
></v-select> ></v-select>
</v-form> </v-form>
</v-sheet> </v-sheet>
<v-card-actions> <v-card-actions>
<v-spacer></v-spacer> <v-spacer></v-spacer>
<v-btn <v-btn text="关闭" @click="isActive.value = false"></v-btn>
text="关闭"
@click="isActive.value = false"
></v-btn>
</v-card-actions> </v-card-actions>
</v-card> </v-card>
</template> </template>
</v-dialog> </v-dialog>
</template> </template>
</v-app-bar> </v-app-bar>
</template> </template>
......
src/renderer/main.ts
View file @ fd1558a6
...@@ -7,8 +7,8 @@ import vuetify from '@/renderer/plugins/vuetify' ...@@ -7,8 +7,8 @@ import vuetify from '@/renderer/plugins/vuetify'
import i18n from '@/renderer/plugins/i18n' import i18n from '@/renderer/plugins/i18n'
import piniaPluginPersistedstate from 'pinia-plugin-persistedstate' import piniaPluginPersistedstate from 'pinia-plugin-persistedstate'
const pinia = createPinia(); const pinia = createPinia()
pinia.use(piniaPluginPersistedstate); pinia.use(piniaPluginPersistedstate)
// Add API key defined in contextBridge to window object type // Add API key defined in contextBridge to window object type
declare global { declare global {
......
src/renderer/plugins/asr/index.ts
View file @ fd1558a6
export * as Vosk from './vosk/vosk' export * as Vosk from './vosk/vosk'
export type * from './vosk/vosk' export type * from './vosk/vosk'
\ No newline at end of file
src/renderer/plugins/asr/vosk/interfaces.d.ts
View file @ fd1558a6
export interface ClientMessageLoad { export interface ClientMessageLoad {
action: "load"; action: 'load'
modelUrl: string; modelUrl: string
} }
export interface ClientMessageTerminate { export interface ClientMessageTerminate {
action: "terminate"; action: 'terminate'
} }
export interface ClientMessageRecognizerSet { export interface ClientMessageRecognizerSet {
action: "set"; action: 'set'
recognizerId: string; recognizerId: string
key: "words"; key: 'words'
value: boolean; value: boolean
} }
export interface ClientMessageGenericSet { export interface ClientMessageGenericSet {
action: "set"; action: 'set'
key: "logLevel"; key: 'logLevel'
value: number; value: number
} }
export declare type ClientMessageSet = ClientMessageRecognizerSet | ClientMessageGenericSet; export declare type ClientMessageSet = ClientMessageRecognizerSet | ClientMessageGenericSet
export interface ClientMessageAudioChunk { export interface ClientMessageAudioChunk {
action: "audioChunk"; action: 'audioChunk'
recognizerId: string; recognizerId: string
data: Float32Array; data: Float32Array
sampleRate: number; sampleRate: number
} }
export interface ClientMessageCreateRecognizer { export interface ClientMessageCreateRecognizer {
action: "create"; action: 'create'
recognizerId: string; recognizerId: string
sampleRate: number; sampleRate: number
grammar?: string; grammar?: string
} }
export interface ClientMessageRetrieveFinalResult { export interface ClientMessageRetrieveFinalResult {
action: "retrieveFinalResult"; action: 'retrieveFinalResult'
recognizerId: string; recognizerId: string
} }
export interface ClientMessageRemoveRecognizer { export interface ClientMessageRemoveRecognizer {
action: "remove"; action: 'remove'
recognizerId: string; recognizerId: string
} }
export declare type ClientMessage = ClientMessageTerminate | ClientMessageLoad | ClientMessageCreateRecognizer | ClientMessageAudioChunk | ClientMessageSet | ClientMessageRetrieveFinalResult | ClientMessageRemoveRecognizer; export declare type ClientMessage =
| ClientMessageTerminate
| ClientMessageLoad
| ClientMessageCreateRecognizer
| ClientMessageAudioChunk
| ClientMessageSet
| ClientMessageRetrieveFinalResult
| ClientMessageRemoveRecognizer
export declare namespace ClientMessage { export declare namespace ClientMessage {
function isTerminateMessage(message: ClientMessage): message is ClientMessageTerminate; function isTerminateMessage(message: ClientMessage): message is ClientMessageTerminate
function isLoadMessage(message: ClientMessage): message is ClientMessageLoad; function isLoadMessage(message: ClientMessage): message is ClientMessageLoad
function isSetMessage(message: ClientMessage): message is ClientMessageSet; function isSetMessage(message: ClientMessage): message is ClientMessageSet
function isAudioChunkMessage(message: ClientMessage): message is ClientMessageAudioChunk; function isAudioChunkMessage(message: ClientMessage): message is ClientMessageAudioChunk
function isRecognizerCreateMessage(message: ClientMessage): message is ClientMessageCreateRecognizer; function isRecognizerCreateMessage(
function isRecognizerRetrieveFinalResultMessage(message: ClientMessage): message is ClientMessageRetrieveFinalResult; message: ClientMessage
function isRecognizerRemoveMessage(message: ClientMessage): message is ClientMessageRemoveRecognizer; ): message is ClientMessageCreateRecognizer
function isRecognizerRetrieveFinalResultMessage(
message: ClientMessage
): message is ClientMessageRetrieveFinalResult
function isRecognizerRemoveMessage(
message: ClientMessage
): message is ClientMessageRemoveRecognizer
} }
export interface ServerMessageLoadResult { export interface ServerMessageLoadResult {
event: "load"; event: 'load'
result: boolean; result: boolean
} }
export interface ServerMessageError { export interface ServerMessageError {
event: "error"; event: 'error'
recognizerId?: string; recognizerId?: string
error: string; error: string
} }
export interface ServerMessageResult { export interface ServerMessageResult {
event: "result"; event: 'result'
recognizerId: string; recognizerId: string
result: { result: {
result: Array<{ result: Array<{
conf: number; conf: number
start: number; start: number
end: number; end: number
word: string; word: string
}>; }>
text: string; text: string
}; }
} }
export interface ServerMessagePartialResult { export interface ServerMessagePartialResult {
event: "partialresult"; event: 'partialresult'
recognizerId: string; recognizerId: string
result: { result: {
partial: string; partial: string
}; }
} }
export declare type ModelMessage = ServerMessageLoadResult | ServerMessageError; export declare type ModelMessage = ServerMessageLoadResult | ServerMessageError
export declare namespace ModelMessage { export declare namespace ModelMessage {
function isLoadResult(message: any): message is ServerMessageLoadResult; function isLoadResult(message: any): message is ServerMessageLoadResult
} }
export declare type RecognizerMessage = ServerMessagePartialResult | ServerMessageResult | ServerMessageError; export declare type RecognizerMessage =
export declare type RecognizerEvent = RecognizerMessage["event"]; | ServerMessagePartialResult
export declare type ServerMessage = ModelMessage | RecognizerMessage; | ServerMessageResult
| ServerMessageError
export declare type RecognizerEvent = RecognizerMessage['event']
export declare type ServerMessage = ModelMessage | RecognizerMessage
export declare namespace ServerMessage { export declare namespace ServerMessage {
function isRecognizerMessage(message: ServerMessage): message is RecognizerMessage; function isRecognizerMessage(message: ServerMessage): message is RecognizerMessage
function isResult(message: any): message is ServerMessageResult; function isResult(message: any): message is ServerMessageResult
function isPartialResult(message: any): message is ServerMessagePartialResult; function isPartialResult(message: any): message is ServerMessagePartialResult
} }
src/renderer/plugins/asr/vosk/model.d.ts
View file @ fd1558a6
import { ModelMessage, RecognizerEvent, RecognizerMessage } from "./interfaces"; import { ModelMessage, RecognizerEvent, RecognizerMessage } from './interfaces'
export * from "./interfaces"; export * from './interfaces'
export declare class Model extends EventTarget { export declare class Model extends EventTarget {
private modelUrl; private modelUrl
private worker; private worker
private _ready; private _ready
private messagePort; private messagePort
private logger; private logger
private recognizers; private recognizers
constructor(modelUrl: string, logLevel?: number); constructor(modelUrl: string, logLevel?: number)
private initialize; private initialize
private postMessage; private postMessage
private handleMessage; private handleMessage
on(event: ModelMessage["event"], listener: (message: ModelMessage) => void): void; on(event: ModelMessage['event'], listener: (message: ModelMessage) => void): void
registerPort(port: MessagePort): void; registerPort(port: MessagePort): void
private forwardMessage; private forwardMessage
get ready(): boolean; get ready(): boolean
terminate(): void; terminate(): void
setLogLevel(level: number): void; setLogLevel(level: number): void
registerRecognizer(recognizer: KaldiRecognizer): void; registerRecognizer(recognizer: KaldiRecognizer): void
unregisterRecognizer(recognizerId: string): void; unregisterRecognizer(recognizerId: string): void
/** /**
* KaldiRecognizer anonymous class * KaldiRecognizer anonymous class
*/ */
get KaldiRecognizer(): { get KaldiRecognizer(): {
new (sampleRate: number, grammar?: string): { new (
id: string; sampleRate: number,
on(event: RecognizerEvent, listener: (message: RecognizerMessage) => void): void; grammar?: string
setWords(words: boolean): void; ): {
acceptWaveform(buffer: AudioBuffer): void; id: string
acceptWaveformFloat(buffer: Float32Array, sampleRate: number): void; on(event: RecognizerEvent, listener: (message: RecognizerMessage) => void): void
retrieveFinalResult(): void; setWords(words: boolean): void
remove(): void; acceptWaveform(buffer: AudioBuffer): void
addEventListener(type: string, callback: EventListenerOrEventListenerObject | null, options?: boolean | AddEventListenerOptions | undefined): void; acceptWaveformFloat(buffer: Float32Array, sampleRate: number): void
dispatchEvent(event: Event): boolean; retrieveFinalResult(): void
removeEventListener(type: string, callback: EventListenerOrEventListenerObject | null, options?: boolean | EventListenerOptions | undefined): void; remove(): void
}; addEventListener(
}; type: string,
callback: EventListenerOrEventListenerObject | null,
options?: boolean | AddEventListenerOptions | undefined
): void
dispatchEvent(event: Event): boolean
removeEventListener(
type: string,
callback: EventListenerOrEventListenerObject | null,
options?: boolean | EventListenerOptions | undefined
): void
}
}
} }
export declare type KaldiRecognizer = InstanceType<Model["KaldiRecognizer"]>; export declare type KaldiRecognizer = InstanceType<Model['KaldiRecognizer']>
export declare function createModel(modelUrl: string, logLevel?: number): Promise<Model>; export declare function createModel(modelUrl: string, logLevel?: number): Promise<Model>
src/renderer/plugins/asr/vosk/utils/logging.d.ts
View file @ fd1558a6
export declare class Logger { export declare class Logger {
private logLevel; private logLevel
constructor(logLevel?: number); constructor(logLevel?: number)
getLogLevel(): number; getLogLevel(): number
setLogLevel(level: number): void; setLogLevel(level: number): void
error(message: string): void; error(message: string): void
warn(message: string): void; warn(message: string): void
info(message: string): void; info(message: string): void
verbose(message: string): void; verbose(message: string): void
debug(message: string): void; debug(message: string): void
} }
src/renderer/plugins/asr/vosk/vosk.d.ts
View file @ fd1558a6
export * from "./model"; export * from './model'
src/renderer/plugins/asr/vosk/vosk.js
View file @ fd1558a6
This source diff could not be displayed because it is too large. You can view the blob instead.
src/renderer/plugins/asr/vosk/worker.d.ts
View file @ fd1558a6
import * as VoskWasm from "./vosk-wasm"; import * as VoskWasm from './vosk-wasm'
export interface Recognizer { export interface Recognizer {
id: string; id: string
buffAddr?: number; buffAddr?: number
buffSize?: number; buffSize?: number
recognizer: VoskWasm.Recognizer; recognizer: VoskWasm.Recognizer
sampleRate: number; sampleRate: number
words?: boolean; words?: boolean
grammar?: string; grammar?: string
} }
export declare class RecognizerWorker { export declare class RecognizerWorker {
private Vosk; private Vosk
private model; private model
private recognizers; private recognizers
private logger; private logger
constructor(); constructor()
private handleMessage; private handleMessage
private load; private load
private allocateBuffer; private allocateBuffer
private freeBuffer; private freeBuffer
private createRecognizer; private createRecognizer
private setConfiguration; private setConfiguration
private processAudioChunk; private processAudioChunk
private retrieveFinalResult; private retrieveFinalResult
private removeRecognizer; private removeRecognizer
private terminate; private terminate
} }
src/renderer/plugins/tts/FetchTTS.ts
View file @ fd1558a6
export async function audioAiTTS({ host, text, speaker, speed, provider }: { host: string; text: string; speaker: string; speed: number; provider: number; }) { export async function audioAiTTS({
const resp = await fetch(`${ host }/api/live/audioAI`, { host,
"headers": { text,
"accept": "application/json, text/plain, */*", speaker,
"content-type": "application/json", speed,
}, provider
body: JSON.stringify({ }: {
type: 1, host: string
text, text: string
speaker, speaker: string
speed, speed: number
provider, provider: number
pause_points: [], }) {
sceneType: 1, const resp = await fetch(`${host}/api/live/audioAI`, {
gender: 1 headers: {
accept: 'application/json, text/plain, */*',
'content-type': 'application/json'
},
body: JSON.stringify({
type: 1,
text,
speaker,
speed,
provider,
pause_points: [],
sceneType: 1,
gender: 1
}), }),
method: "POST", method: 'POST',
mode: "cors", mode: 'cors'
}); })
const res = await resp.json(); const res = await resp.json()
if (res.code !== 200) throw new Error(JSON.stringify(res)); if (res.code !== 200) throw new Error(JSON.stringify(res))
return res.data; return res.data
} }
\ No newline at end of file
src/renderer/plugins/tts/index.ts
View file @ fd1558a6
export * from './FetchTTS' export * from './FetchTTS'
\ No newline at end of file
src/renderer/public/vosk/recognizer-processor.js
View file @ fd1558a6
class RecognizerAudioProcessor extends AudioWorkletProcessor { class RecognizerAudioProcessor extends AudioWorkletProcessor {
constructor(options) { constructor(options) {
super(options); super(options)
this.port.onmessage = this._processMessage.bind(this); this.port.onmessage = this._processMessage.bind(this)
} }
_processMessage(event) { _processMessage(event) {
// console.debug(`Received event ${JSON.stringify(event.data, null, 2)}`); // console.debug(`Received event ${JSON.stringify(event.data, null, 2)}`);
if (event.data.action === "init") { if (event.data.action === 'init') {
this._recognizerId = event.data.recognizerId; this._recognizerId = event.data.recognizerId
this._recognizerPort = event.ports[0]; this._recognizerPort = event.ports[0]
}
} }
}
process(inputs, outputs, parameters) {
const data = inputs[0][0]; process(inputs, outputs, parameters) {
if (this._recognizerPort && data) { const data = inputs[0][0]
// AudioBuffer samples are represented as floating point numbers between -1.0 and 1.0 whilst if (this._recognizerPort && data) {
// Kaldi expects them to be between -32768 and 32767 (the range of a signed int16) // AudioBuffer samples are represented as floating point numbers between -1.0 and 1.0 whilst
const audioArray = data.map((value) => value * 0x8000); // Kaldi expects them to be between -32768 and 32767 (the range of a signed int16)
const audioArray = data.map((value) => value * 0x8000)
this._recognizerPort.postMessage(
{ this._recognizerPort.postMessage(
action: "audioChunk", {
data: audioArray, action: 'audioChunk',
recognizerId: this._recognizerId, data: audioArray,
sampleRate, // Part of AudioWorkletGlobalScope recognizerId: this._recognizerId,
}, sampleRate // Part of AudioWorkletGlobalScope
{ },
transfer: [audioArray.buffer], {
} transfer: [audioArray.buffer]
);
} }
return true; )
} }
return true
}
} }
registerProcessor('recognizer-processor', RecognizerAudioProcessor) registerProcessor('recognizer-processor', RecognizerAudioProcessor)
\ No newline at end of file
src/renderer/screens/PhotoScreen.vue
View file @ fd1558a6
...@@ -5,11 +5,11 @@ ...@@ -5,11 +5,11 @@
// import { useCounterStore } from '@/renderer/store/counter' // import { useCounterStore } from '@/renderer/store/counter'
// import { storeToRefs } from 'pinia' // import { storeToRefs } from 'pinia'
import { onMounted, ref } from 'vue' import { onMounted, ref } from 'vue'
import useStore from '@/renderer/store'; import useStore from '@/renderer/store'
import { storeToRefs } from 'pinia'; import { storeToRefs } from 'pinia'
const { photo: usePhoto } = useStore(); const { photo: usePhoto } = useStore()
const photo = storeToRefs(usePhoto); const photo = storeToRefs(usePhoto)
// const { availableLocales } = useI18n() // const { availableLocales } = useI18n()
// const { counterIncrease } = useCounterStore() // const { counterIncrease } = useCounterStore()
...@@ -20,70 +20,90 @@ const photo = storeToRefs(usePhoto); ...@@ -20,70 +20,90 @@ const photo = storeToRefs(usePhoto);
onMounted((): void => { onMounted((): void => {
// languages.value = availableLocales // languages.value = availableLocales
// window.mainApi.receive('msgReceivedVersion', (event: Event, version: string) => { // window.mainApi.receive('msgReceivedVersion', (event: Event, version: string) => {
// appVersion.value = version // appVersion.value = version
// }) // })
// window.mainApi.send('msgRequestGetVersion') // window.mainApi.send('msgRequestGetVersion')
}) })
async function handleOpen(event: Event,url: string) { async function handleOpen(event: Event, url: string) {
await window.mainApi.send('openWindow', `#show?url=${url}`, { width: window.screen.width / 4 , height: window.screen.height }); await window.mainApi.send('openWindow', `#show?url=${url}`, {
width: window.screen.width / 4,
height: window.screen.height
})
} }
const validateURL = (url: string) => { const validateURL = (url: string) => {
const regex = /^(https?|ftp):\/\/([\w/\-?=%.]+\.[\w/\-?=%.]+)$/; const regex = /^(https?|ftp):\/\/([\w/\-?=%.]+\.[\w/\-?=%.]+)$/
return regex.test(url); return regex.test(url)
} }
const urlValue = ref(''); const urlValue = ref('')
const imgLoading = ref(false); const imgLoading = ref(false)
async function appendPhoto(url: string) { async function appendPhoto(url: string) {
urlValue.value = url; urlValue.value = url
if (!validateURL(url)) return '请输入正确的 url！如（url（https://xxx.png）' if (!validateURL(url)) return '请输入正确的 url！如（url（https://xxx.png）'
try { try {
imgLoading.value = true; imgLoading.value = true
const img = new Image(); const img = new Image()
img.src = url; img.src = url
await new Promise((resolve, reject) => { await new Promise((resolve, reject) => {
img.onload = resolve; img.onload = resolve
img.onerror = reject; img.onerror = reject
}); })
imgLoading.value = false; imgLoading.value = false
} catch (error) { } catch (error) {
imgLoading.value = false; imgLoading.value = false
return '图片加载失败！' return '图片加载失败！'
} }
photo.list.value.push({ url }); photo.list.value.push({ url })
urlValue.value = '' urlValue.value = ''
return true; return true
} }
function removePhoto(index: number) { function removePhoto(index: number) {
photo.list.value.splice(index, 1) photo.list.value.splice(index, 1)
} }
</script> </script>
<template> <template>
<v-container class="d-flex mt-6 pb-0"> <v-container class="d-flex mt-6 pb-0">
<v-text-field label="自定义照片 url（https://xxx.png）" :model-value="urlValue" :loading="imgLoading" :rules="[ v => appendPhoto(v) ]" validate-on="blur lazy" ></v-text-field> <v-text-field
label="自定义照片 url（https://xxx.png）"
:model-value="urlValue"
:loading="imgLoading"
:rules="[(v) => appendPhoto(v)]"
validate-on="blur lazy"
></v-text-field>
</v-container> </v-container>
<v-container class="d-flex flex-wrap"> <v-container class="d-flex flex-wrap">
<v-sheet v-for="(item, index) in photo.list.value" :key="item.url" v-ripple :elevation="3" width="200" class="d-flex spacing-playground pa-6 mr-4 mt-4" rounded > <v-sheet
<v-img v-for="(item, index) in photo.list.value"
:width="200" :key="item.url"
aspect-ratio="1/1" v-ripple
cover :elevation="3"
:src="item.url" width="200"
@click="handleOpen($event, item.url)" class="d-flex spacing-playground pa-6 mr-4 mt-4"
></v-img> rounded
<v-btn density="compact" elevation="1" icon="mdi-close" class="mt-n7" @click="removePhoto(index)"></v-btn> >
</v-sheet> <v-img
:width="200"
aspect-ratio="1/1"
cover
:src="item.url"
@click="handleOpen($event, item.url)"
></v-img>
<v-btn
density="compact"
elevation="1"
icon="mdi-close"
class="mt-n7"
@click="removePhoto(index)"
></v-btn>
</v-sheet>
</v-container> </v-container>
</template> </template>
src/renderer/screens/ShowPhoto.vue
View file @ fd1558a6
<script setup lang="ts"> <script setup lang="ts">
import { ref } from 'vue'; import { ref } from 'vue'
import { useRoute, useRouter } from 'vue-router' import { useRoute, useRouter } from 'vue-router'
import { Vosk } from '@/renderer/plugins/asr/index' import { Vosk } from '@/renderer/plugins/asr/index'
import type { ServerMessagePartialResult, ServerMessageResult, Model } from '@/renderer/plugins/asr/index' import type {
import { audioAiTTS } from '../plugins/tts'; ServerMessagePartialResult,
import useStore from '@/renderer/store'; ServerMessageResult,
Model
} from '@/renderer/plugins/asr/index'
import { audioAiTTS } from '../plugins/tts'
import useStore from '@/renderer/store'
const router = useRouter() const router = useRouter()
const route = useRoute(); const route = useRoute()
const { settings } = useStore(); const { settings } = useStore()
const sampleRate = 48000; const sampleRate = 48000
const recordVolume = ref(0); const recordVolume = ref(0)
router.beforeEach(g => { router.beforeEach((g) => {
if (!g.query.url) return router.push('/error'); if (!g.query.url) return router.push('/error')
}) })
const microphoneState = ref< 'waitInput' | 'input' | 'loading' | 'disabled'>('waitInput'); const microphoneState = ref<'waitInput' | 'input' | 'loading' | 'disabled'>('waitInput')
async function initVosk({ modelPath, result, partialResult }: { async function initVosk({
modelPath: string; modelPath,
result?: (string) => void; result,
partialResult?: (string) => void; partialResult
}) { }: {
const channel = new MessageChannel(); modelPath: string
const model = await Vosk.createModel(modelPath); result?: (string) => void
const recognizer = new model.KaldiRecognizer(sampleRate); partialResult?: (string) => void
}) {
model.registerPort(channel.port1); const channel = new MessageChannel()
recognizer.setWords(true); const model = await Vosk.createModel(modelPath)
const recognizer = new model.KaldiRecognizer(sampleRate)
recognizer.on('result', (message) => {
result && result((message as ServerMessageResult).result.text) model.registerPort(channel.port1)
}); recognizer.setWords(true)
recognizer.on('partialresult', (message) => {
partialResult && partialResult((message as ServerMessagePartialResult).result.partial) recognizer.on('result', (message) => {
}); result && result((message as ServerMessageResult).result.text)
})
return { recognizer, channel }; recognizer.on('partialresult', (message) => {
partialResult && partialResult((message as ServerMessagePartialResult).result.partial)
})
return { recognizer, channel }
} }
function analyzeMicrophoneVolume(stream: MediaStream, callback: (number) => void) { function analyzeMicrophoneVolume(stream: MediaStream, callback: (number) => void) {
const audioContext = new AudioContext(); const audioContext = new AudioContext()
const analyser = audioContext.createAnalyser(); const analyser = audioContext.createAnalyser()
const microphone = audioContext.createMediaStreamSource(stream); const microphone = audioContext.createMediaStreamSource(stream)
const recordEventNode = audioContext.createScriptProcessor(2048, 1, 1); const recordEventNode = audioContext.createScriptProcessor(2048, 1, 1)
const audioprocess = () => { const audioprocess = () => {
const array = new Uint8Array(analyser.frequencyBinCount); const array = new Uint8Array(analyser.frequencyBinCount)
analyser.getByteFrequencyData(array); analyser.getByteFrequencyData(array)
let values = 0; let values = 0
const length = array.length; const length = array.length
for (let i = 0; i < length; i++) { for (let i = 0; i < length; i++) {
values += array[i]; values += array[i]
} }
const average = values / length; const average = values / length
callback(Math.round(average)); callback(Math.round(average))
} }
analyser.smoothingTimeConstant = 0.8; analyser.smoothingTimeConstant = 0.8
analyser.fftSize = 1024; analyser.fftSize = 1024
microphone.connect(analyser); microphone.connect(analyser)
analyser.connect(recordEventNode); analyser.connect(recordEventNode)
recordEventNode.connect(audioContext.destination); recordEventNode.connect(audioContext.destination)
// recordEventNode.addEventListener('audioprocess', audioprocess); // recordEventNode.addEventListener('audioprocess', audioprocess);
recordEventNode.onaudioprocess = audioprocess; recordEventNode.onaudioprocess = audioprocess
inputContext.audioContext2 = audioContext; inputContext.audioContext2 = audioContext
inputContext.scriptProcessorNode = recordEventNode; inputContext.scriptProcessorNode = recordEventNode
} }
const inputContext: { mediaStream?: MediaStream; audioContext?: AudioContext; audioContext2?: AudioContext; scriptProcessorNode?: ScriptProcessorNode; model?: Model } = {}; const inputContext: {
mediaStream?: MediaStream
audioContext?: AudioContext
audioContext2?: AudioContext
scriptProcessorNode?: ScriptProcessorNode
model?: Model
} = {}
async function startAudioInput() { async function startAudioInput() {
if (microphoneState.value === 'loading') return; if (microphoneState.value === 'loading') return
if (microphoneState.value === 'input') { if (microphoneState.value === 'input') {
microphoneState.value = 'waitInput'; microphoneState.value = 'waitInput'
inputContext.mediaStream?.getTracks().forEach((track) => track.stop()); inputContext.mediaStream?.getTracks().forEach((track) => track.stop())
inputContext.audioContext?.close(); inputContext.audioContext?.close()
inputContext.audioContext2?.close(); inputContext.audioContext2?.close()
inputContext.scriptProcessorNode && (inputContext.scriptProcessorNode.onaudioprocess = null); inputContext.scriptProcessorNode && (inputContext.scriptProcessorNode.onaudioprocess = null)
inputContext.model?.terminate(); inputContext.model?.terminate()
return; return
} }
microphoneState.value = 'loading'; microphoneState.value = 'loading'
const { recognizer, channel } = await initVosk({ const { recognizer, channel } = await initVosk({
modelPath: new URL(`/vosk/models/${settings.voskSelectModel}`, import.meta.url).href, modelPath: new URL(`/vosk/models/${settings.voskSelectModel}`, import.meta.url).href,
result: async (text) => { result: async (text) => {
console.log('----------------> text:', text); console.log('----------------> text:', text)
const tone = settings.source.find(({ sourceId }) => settings.selectSource === sourceId); const tone = settings.source.find(({ sourceId }) => settings.selectSource === sourceId)
if (!tone) return; if (!tone) return
const res = await audioAiTTS({ host: settings.ttsHost, text, speed: 3, speaker: tone.sourceId, provider: tone.provider }); const res = await audioAiTTS({
host: settings.ttsHost,
console.log('----------------> tts:', res); text,
speed: 3,
speaker: tone.sourceId,
provider: tone.provider
})
console.log('----------------> tts:', res)
}, },
partialResult: text => { partialResult: (text) => {
console.log('----------------> partialResult:', text); console.log('----------------> partialResult:', text)
}, }
}); })
const mediaStream = await navigator.mediaDevices.getUserMedia({ const mediaStream = await navigator.mediaDevices.getUserMedia({
video: false, video: false,
audio: { audio: {
echoCancellation: true, echoCancellation: true,
noiseSuppression: true, noiseSuppression: true,
channelCount: 1, channelCount: 1,
sampleRate sampleRate
}, }
}); })
const audioContext = new AudioContext(); const audioContext = new AudioContext()
await audioContext.audioWorklet.addModule(new URL('/vosk/recognizer-processor.js', import.meta.url)) await audioContext.audioWorklet.addModule(
const recognizerProcessor = new AudioWorkletNode(audioContext, 'recognizer-processor', { channelCount: 1, numberOfInputs: 1, numberOfOutputs: 1 }); new URL('/vosk/recognizer-processor.js', import.meta.url)
recognizerProcessor.port.postMessage({action: 'init', recognizerId: recognizer.id}, [ channel.port2 ]) )
recognizerProcessor.connect(audioContext.destination); const recognizerProcessor = new AudioWorkletNode(audioContext, 'recognizer-processor', {
channelCount: 1,
const source = audioContext.createMediaStreamSource(mediaStream); numberOfInputs: 1,
source.connect(recognizerProcessor); numberOfOutputs: 1
})
recognizerProcessor.port.postMessage({ action: 'init', recognizerId: recognizer.id }, [
channel.port2
])
recognizerProcessor.connect(audioContext.destination)
const source = audioContext.createMediaStreamSource(mediaStream)
source.connect(recognizerProcessor)
await analyzeMicrophoneVolume(mediaStream, (val) => { await analyzeMicrophoneVolume(mediaStream, (val) => {
recordVolume.value = val; recordVolume.value = val
}); })
microphoneState.value = 'input'; microphoneState.value = 'input'
inputContext.mediaStream = mediaStream; inputContext.mediaStream = mediaStream
inputContext.audioContext = audioContext; inputContext.audioContext = audioContext
} }
function endAudioInput() { function endAudioInput() {
console.log('----------------> end'); console.log('----------------> end')
} }
</script> </script>
<template> <template>
<div style="width: 100%; height: 100%;" class="d-flex justify-center align-center"> <div style="width: 100%; height: 100%" class="d-flex justify-center align-center">
<v-img <v-img
v-if="route.query.url" v-if="route.query.url"
:width="'100%'" :width="'100%'"
aspect-ratio="1/1" aspect-ratio="1/1"
cover cover
:src="(route.query.url as string)" :src="route.query.url as string"
></v-img> ></v-img>
</div> </div>
<div class="voice"> <div class="voice">
<v-btn icon="" color="#fff" variant="elevated" size="x-large" :disabled="microphoneState === 'loading' || microphoneState ==='disabled'" @pointerdown="startAudioInput" @pointerup="endAudioInput"> <v-btn
icon=""
color="#fff"
variant="elevated"
size="x-large"
:disabled="microphoneState === 'loading' || microphoneState === 'disabled'"
@pointerdown="startAudioInput"
@pointerup="endAudioInput"
>
<v-icon v-if="microphoneState === 'waitInput'" icon="mdi-microphone"></v-icon> <v-icon v-if="microphoneState === 'waitInput'" icon="mdi-microphone"></v-icon>
<v-icon v-if="microphoneState === 'loading'" icon="mdi-microphone-settings"></v-icon> <v-icon v-if="microphoneState === 'loading'" icon="mdi-microphone-settings"></v-icon>
<v-icon v-if="microphoneState === 'disabled'" icon="mdi-microphone-off"></v-icon> <v-icon v-if="microphoneState === 'disabled'" icon="mdi-microphone-off"></v-icon>
<template v-if="microphoneState === 'input'"> <template v-if="microphoneState === 'input'">
<img width="30" height="30" src="/images/microphone-input.svg" alt="" srcset=""> <img width="30" height="30" src="/images/microphone-input.svg" alt="" srcset="" />
<div class="progress"> <div class="progress">
<span class="volume" :style="{ 'clip-path': `polygon(0 ${100 - recordVolume}%, 100% ${100 - recordVolume}%, 100% 100%, 0 100%)` }"></span> <span
class="volume"
:style="{
'clip-path': `polygon(0 ${100 - recordVolume}%, 100% ${
100 - recordVolume
}%, 100% 100%, 0 100%)`
}"
></span>
</div> </div>
</template> </template>
</v-btn> </v-btn>
</div> </div>
</template> </template>
<style scoped> <style scoped>
.voice { .voice {
display: flex; display: flex;
justify-content: center; justify-content: center;
position: fixed; position: fixed;
left: 0; left: 0;
right: 0; right: 0;
top: 70%; top: 70%;
margin: auto; margin: auto;
} }
.progress{
position: absolute;
top: 21px;
left: 28px;
width: 8px;
height: 16px;
overflow: hidden;
border-radius: 36%;
}
.progress .volume{
display: block;
width: 100%;
height: 100%;
background: #2FB84F;
border-radius: 36%;
}
.progress {
position: absolute;
top: 21px;
left: 28px;
width: 8px;
height: 16px;
overflow: hidden;
border-radius: 36%;
}
.progress .volume {
display: block;
width: 100%;
height: 100%;
background: #2fb84f;
border-radius: 36%;
}
</style> </style>
src/renderer/store/index.ts
View file @ fd1558a6
import useSettings from './settings'; import useSettings from './settings'
import usePhoto from './photo'; import usePhoto from './photo'
export default function useStore() { export default function useStore() {
return { return {
settings: useSettings(), settings: useSettings(),
photo: usePhoto() photo: usePhoto()
} }
} }
\ No newline at end of file
src/renderer/store/photo.ts
View file @ fd1558a6
...@@ -6,20 +6,19 @@ type IPhoto = { ...@@ -6,20 +6,19 @@ type IPhoto = {
const usePhotoStore = defineStore('photo', { const usePhotoStore = defineStore('photo', {
persist: true, persist: true,
state: () => ({ state: () =>
list: [ ({
{ list: [
url: 'https://resources.laihua.com/2023-11-2/93ffb6a7-ae93-4918-944e-877016ba266b.png' {
}, url: 'https://resources.laihua.com/2023-11-2/93ffb6a7-ae93-4918-944e-877016ba266b.png'
{ },
url: 'https://resources.laihua.com/2023-6-19/6fa9a127-2ce5-43ea-a543-475bf9354eda.png' {
} url: 'https://resources.laihua.com/2023-6-19/6fa9a127-2ce5-43ea-a543-475bf9354eda.png'
] }
} as IPhoto), ]
getters: { }) as IPhoto,
}, getters: {},
actions: { actions: {}
}
}) })
export default usePhotoStore export default usePhotoStore
\ No newline at end of file
src/renderer/store/settings.ts
View file @ fd1558a6
import { defineStore } from 'pinia' import { defineStore } from 'pinia'
export type ISettings = { export type ISettings = {
asr: 'vosk_asr' | 'xf_asr'; asr: 'vosk_asr' | 'xf_asr'
voskModels: string[]; voskModels: string[]
voskSelectModel: string; voskSelectModel: string
ttsHost: string; ttsHost: string
source: { sourceName: string; sourceId: string; provider: number; speaker: string; description: string; sex: 1 | 0 }[]; source: {
selectSource: string; sourceName: string
sourceId: string
provider: number
speaker: string
description: string
sex: 1 | 0
}[]
selectSource: string
} }
const useSettingsStore = defineStore('settings', { const useSettingsStore = defineStore('settings', {
persist: true, persist: true,
state: () => ({ state: () =>
asr: 'vosk_asr', ({
voskModels: [ asr: 'vosk_asr',
'vosk-model-small-ca-0.4.tar.gz', voskModels: [
'vosk-model-small-cn-0.3.tar.gz', 'vosk-model-small-ca-0.4.tar.gz',
'vosk-model-small-de-0.15.tar.gz', 'vosk-model-small-cn-0.3.tar.gz',
'vosk-model-small-en-in-0.4.tar.gz', 'vosk-model-small-de-0.15.tar.gz',
'vosk-model-small-en-us-0.15.tar.gz', 'vosk-model-small-en-in-0.4.tar.gz',
'vosk-model-small-es-0.3.tar.gz', 'vosk-model-small-en-us-0.15.tar.gz',
'vosk-model-small-fa-0.4.tar.gz', 'vosk-model-small-es-0.3.tar.gz',
'vosk-model-small-fr-pguyot-0.3.tar.gz', 'vosk-model-small-fa-0.4.tar.gz',
'vosk-model-small-it-0.4.tar.gz', 'vosk-model-small-fr-pguyot-0.3.tar.gz',
'vosk-model-small-pt-0.3.tar.gz', 'vosk-model-small-it-0.4.tar.gz',
'vosk-model-small-ru-0.4.tar.gz', 'vosk-model-small-pt-0.3.tar.gz',
'vosk-model-small-tr-0.3.tar.gz', 'vosk-model-small-ru-0.4.tar.gz',
'vosk-model-small-vn-0.3.tar.gz' 'vosk-model-small-tr-0.3.tar.gz',
], 'vosk-model-small-vn-0.3.tar.gz'
voskSelectModel: 'vosk-model-small-cn-0.3.tar.gz', ],
ttsHost: 'https://beta.laihua.com', voskSelectModel: 'vosk-model-small-cn-0.3.tar.gz',
source: [], ttsHost: 'https://beta.laihua.com',
selectSource: '' source: [],
} as ISettings), selectSource: ''
getters: { }) as ISettings,
}, getters: {},
actions: { actions: {
async getSource() { async getSource() {
const resp = await fetch(`${this.$state.ttsHost}/api/live/audioAI/source?platform=31`, { const resp = await fetch(`${this.$state.ttsHost}/api/live/audioAI/source?platform=31`, {
"method": "GET", method: 'GET',
"mode": "cors", mode: 'cors'
}); })
const res = await resp.json(); const res = await resp.json()
if (res.code !== 200) return; if (res.code !== 200) return
this.source = res.data; this.source = res.data
} }
} }
}) })
export default useSettingsStore; export default useSettingsStore
tsconfig.json
View file @ fd1558a6
...@@ -26,11 +26,5 @@ ...@@ -26,11 +26,5 @@
"path": "./tsconfig.node.json" "path": "./tsconfig.node.json"
} }
], ],
"exclude": [ "exclude": ["node_modules", "dist", "rollup.config.js", "*.json", "*.js"]
"node_modules",
"dist",
"rollup.config.js",
"*.json",
"*.js"
]
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment