Skip to content

C
CharIP-Electron
Commit fd1558a6 authored by ali's avatar ali
Browse files
Options

feat: TTS 接口接入，完成各项功能支持配置功能

parent 90ce5548
Show whitespace changes
Inline Side-by-side
Showing with 1198 additions and 1002 deletions
.vscode/settings.json
View file @ fd1558a6
......@@ -21,5 +21,5 @@
"cSpell.words": [
"Vosk"
],
"editor.inlineSuggest.showToolbar": "always"
"editor.inlineSuggest.showToolbar": "onHover"
}
buildAssets/builder/config.js
View file @ fd1558a6
......@@ -69,7 +69,7 @@ const baseConfig = {
oneClick: true
},
linux: {
executableName: 'vutron',
executableName: 'chartIP',
icon: 'buildAssets/icons',
category: 'Utility',
target: [
......
src/main/IPCs.ts
View file @ fd1558a6
......@@ -5,8 +5,7 @@ import Constants from './utils/Constants'
* IPC Communications
* */
export default class IPCs {
static browserWindows: Map<string, BrowserWindow[]> = new Map();
static browserWindows: Map<string, BrowserWindow[]> = new Map()
static initialize(window: BrowserWindow): void {
// Get application version
......@@ -20,13 +19,28 @@ export default class IPCs {
})
// open new window
ipcMain.on('openWindow', async (event, url: string, options: BrowserWindowConstructorOptions & { isCloseOther: boolean }) => {
const ops = Object.assign({}, { isCloseOther: true, frame: false, useContentSize: true, webPreferences: Constants.DEFAULT_WEB_PREFERENCES }, options);
ipcMain.on(
'openWindow',
async (
event,
url: string,
options: BrowserWindowConstructorOptions & { isCloseOther: boolean }
) => {
const ops = Object.assign(
{},
{
isCloseOther: true,
frame: false,
useContentSize: true,
webPreferences: Constants.DEFAULT_WEB_PREFERENCES
},
options
)
if (IPCs.browserWindows.has(url) && ops.isCloseOther) {
const wins = IPCs.browserWindows.get(url);
wins?.forEach(w => !w.isDestroyed() && w.close());
IPCs.browserWindows.set(url, []);
const wins = IPCs.browserWindows.get(url)
wins?.forEach((w) => !w.isDestroyed() && w.close())
IPCs.browserWindows.set(url, [])
}
const win = new BrowserWindow(ops)
......@@ -49,12 +63,13 @@ export default class IPCs {
await win.loadURL(`${Constants.APP_INDEX_URL_DEV}${url}`)
if (!IPCs.browserWindows.has(url)) {
IPCs.browserWindows.set(url, []);
IPCs.browserWindows.set(url, [])
}
IPCs.browserWindows.get(url)?.push(win);
IPCs.browserWindows.get(url)?.push(win)
return win;
})
return win
}
)
}
}
src/renderer/components/layout/DefaultLayout.vue
View file @ fd1558a6
<script setup lang="ts">
import HeaderLayout from '@/renderer/components/layout/HeaderLayout.vue'
import { ref } from 'vue';
import { ref } from 'vue'
import { useRouter } from 'vue-router'
const router = useRouter()
const isHeader = ref(true);
const isHeader = ref(true)
router.beforeEach((guard) => {
isHeader.value = typeof guard.meta.isHeader === 'boolean' ? (guard.meta.isHeader as boolean) : true;
isHeader.value =
typeof guard.meta.isHeader === 'boolean' ? (guard.meta.isHeader as boolean) : true
})
</script>
<template>
......
src/renderer/components/layout/HeaderLayout.vue
View file @ fd1558a6
<script setup lang="tsx">
import { computed, ref } from 'vue';
import { computed, ref } from 'vue'
import { useRoute, useRouter } from 'vue-router'
import useStore from '@/renderer/store';
import { storeToRefs } from 'pinia';
import useStore from '@/renderer/store'
import { storeToRefs } from 'pinia'
import { audioAiTTS } from '@/renderer/plugins/tts'
const router = useRouter()
const route: any = useRoute()
const { settings } = useStore();
const setting = storeToRefs(settings);
const { settings } = useStore()
const setting = storeToRefs(settings)
settings.getSource();
settings.getSource()
const handleRoute = (path: string): void => {
router.push(path)
......@@ -25,8 +25,8 @@ const asrItems = ref([
'vosk_asr',
'xf_asr'
// 'Whisper Api'
]);
const asrSelect = ref(setting.asr);
])
const asrSelect = ref(setting.asr)
const source = computed(() => {
return setting.source.value.map(({ sourceId, sourceName, description, sex }) => {
......@@ -35,17 +35,22 @@ const source = computed(() => {
value: sourceId,
title: `${sourceName} - ${_sex} - ${description}`
}
});
});
})
})
async function changeSource() {
const tone = setting.source.value.find(({ sourceId }) => setting.selectSource.value === sourceId);
if (!tone) return;
const res = await audioAiTTS({ host: settings.ttsHost, text: '你好，今天天气怎么样？', speed: 5.5, speaker: tone.sourceId, provider: tone.provider });
console.log(res);
const tone = setting.source.value.find(({ sourceId }) => setting.selectSource.value === sourceId)
if (!tone) return
const res = await audioAiTTS({
host: settings.ttsHost,
text: '你好，今天天气怎么样？',
speed: 5.5,
speaker: tone.sourceId,
provider: tone.provider
})
console.log(res)
}
</script>
<template>
<v-app-bar color="#d71b1b" density="compact" class="header">
......@@ -69,29 +74,20 @@ async function changeSource() {
<v-dialog width="600">
<template #activator="{ props }">
<v-btn
v-bind="props"
color="#fff"
class="settings"
>
<v-icon
start
icon="mdi-wrench"
></v-icon>
<v-btn v-bind="props" color="#fff" class="settings">
<v-icon start icon="mdi-wrench"></v-icon>
配置
</v-btn>
</template>
<template #default="{ isActive }">
<v-card title="配置">
<v-sheet width="500" class="mx-auto mt-6">
<v-form ref="form">
<v-select
v-model="setting.asr.value"
:items="asrItems"
:rules="[v => !!v || '请选择 Asr']"
:rules="[(v) => !!v || '请选择 Asr']"
label="语音识别（ASR）"
required
></v-select>
......@@ -107,9 +103,7 @@ async function changeSource() {
<v-text-field
label="TTS 域名"
:rules="[
value => !!value || 'TTS 域名必填',
]"
:rules="[(value) => !!value || 'TTS 域名必填']"
hide-details="auto"
:model-value="setting.ttsHost"
></v-text-field>
......@@ -118,26 +112,21 @@ async function changeSource() {
v-model="setting.selectSource.value"
class="mt-6"
:items="source"
:rules="[v => !!v || '请选择音色']"
:rules="[(v) => !!v || '请选择音色']"
label="TTS 音色"
required
@update:model-value="changeSource"
></v-select>
</v-form>
</v-sheet>
<v-card-actions>
<v-spacer></v-spacer>
<v-btn
text="关闭"
@click="isActive.value = false"
></v-btn>
<v-btn text="关闭" @click="isActive.value = false"></v-btn>
</v-card-actions>
</v-card>
</template>
</v-dialog>
</template>
</v-app-bar>
</template>
......
src/renderer/main.ts
View file @ fd1558a6
......@@ -7,8 +7,8 @@ import vuetify from '@/renderer/plugins/vuetify'
import i18n from '@/renderer/plugins/i18n'
import piniaPluginPersistedstate from 'pinia-plugin-persistedstate'
const pinia = createPinia();
pinia.use(piniaPluginPersistedstate);
const pinia = createPinia()
pinia.use(piniaPluginPersistedstate)
// Add API key defined in contextBridge to window object type
declare global {
......
src/renderer/plugins/asr/vosk/interfaces.d.ts
View file @ fd1558a6
export interface ClientMessageLoad {
action: "load";
modelUrl: string;
action: 'load'
modelUrl: string
}
export interface ClientMessageTerminate {
action: "terminate";
action: 'terminate'
}
export interface ClientMessageRecognizerSet {
action: "set";
recognizerId: string;
key: "words";
value: boolean;
action: 'set'
recognizerId: string
key: 'words'
value: boolean
}
export interface ClientMessageGenericSet {
action: "set";
key: "logLevel";
value: number;
action: 'set'
key: 'logLevel'
value: number
}
export declare type ClientMessageSet = ClientMessageRecognizerSet | ClientMessageGenericSet;
export declare type ClientMessageSet = ClientMessageRecognizerSet | ClientMessageGenericSet
export interface ClientMessageAudioChunk {
action: "audioChunk";
recognizerId: string;
data: Float32Array;
sampleRate: number;
action: 'audioChunk'
recognizerId: string
data: Float32Array
sampleRate: number
}
export interface ClientMessageCreateRecognizer {
action: "create";
recognizerId: string;
sampleRate: number;
grammar?: string;
action: 'create'
recognizerId: string
sampleRate: number
grammar?: string
}
export interface ClientMessageRetrieveFinalResult {
action: "retrieveFinalResult";
recognizerId: string;
action: 'retrieveFinalResult'
recognizerId: string
}
export interface ClientMessageRemoveRecognizer {
action: "remove";
recognizerId: string;
action: 'remove'
recognizerId: string
}
export declare type ClientMessage = ClientMessageTerminate | ClientMessageLoad | ClientMessageCreateRecognizer | ClientMessageAudioChunk | ClientMessageSet | ClientMessageRetrieveFinalResult | ClientMessageRemoveRecognizer;
export declare type ClientMessage =
| ClientMessageTerminate
| ClientMessageLoad
| ClientMessageCreateRecognizer
| ClientMessageAudioChunk
| ClientMessageSet
| ClientMessageRetrieveFinalResult
| ClientMessageRemoveRecognizer
export declare namespace ClientMessage {
function isTerminateMessage(message: ClientMessage): message is ClientMessageTerminate;
function isLoadMessage(message: ClientMessage): message is ClientMessageLoad;
function isSetMessage(message: ClientMessage): message is ClientMessageSet;
function isAudioChunkMessage(message: ClientMessage): message is ClientMessageAudioChunk;
function isRecognizerCreateMessage(message: ClientMessage): message is ClientMessageCreateRecognizer;
function isRecognizerRetrieveFinalResultMessage(message: ClientMessage): message is ClientMessageRetrieveFinalResult;
function isRecognizerRemoveMessage(message: ClientMessage): message is ClientMessageRemoveRecognizer;
function isTerminateMessage(message: ClientMessage): message is ClientMessageTerminate
function isLoadMessage(message: ClientMessage): message is ClientMessageLoad
function isSetMessage(message: ClientMessage): message is ClientMessageSet
function isAudioChunkMessage(message: ClientMessage): message is ClientMessageAudioChunk
function isRecognizerCreateMessage(
message: ClientMessage
): message is ClientMessageCreateRecognizer
function isRecognizerRetrieveFinalResultMessage(
message: ClientMessage
): message is ClientMessageRetrieveFinalResult
function isRecognizerRemoveMessage(
message: ClientMessage
): message is ClientMessageRemoveRecognizer
}
export interface ServerMessageLoadResult {
event: "load";
result: boolean;
event: 'load'
result: boolean
}
export interface ServerMessageError {
event: "error";
recognizerId?: string;
error: string;
event: 'error'
recognizerId?: string
error: string
}
export interface ServerMessageResult {
event: "result";
recognizerId: string;
event: 'result'
recognizerId: string
result: {
result: Array<{
conf: number;
start: number;
end: number;
word: string;
}>;
text: string;
};
conf: number
start: number
end: number
word: string
}>
text: string
}
}
export interface ServerMessagePartialResult {
event: "partialresult";
recognizerId: string;
event: 'partialresult'
recognizerId: string
result: {
partial: string;
};
partial: string
}
}
export declare type ModelMessage = ServerMessageLoadResult | ServerMessageError;
export declare type ModelMessage = ServerMessageLoadResult | ServerMessageError
export declare namespace ModelMessage {
function isLoadResult(message: any): message is ServerMessageLoadResult;
function isLoadResult(message: any): message is ServerMessageLoadResult
}
export declare type RecognizerMessage = ServerMessagePartialResult | ServerMessageResult | ServerMessageError;
export declare type RecognizerEvent = RecognizerMessage["event"];
export declare type ServerMessage = ModelMessage | RecognizerMessage;
export declare type RecognizerMessage =
| ServerMessagePartialResult
| ServerMessageResult
| ServerMessageError
export declare type RecognizerEvent = RecognizerMessage['event']
export declare type ServerMessage = ModelMessage | RecognizerMessage
export declare namespace ServerMessage {
function isRecognizerMessage(message: ServerMessage): message is RecognizerMessage;
function isResult(message: any): message is ServerMessageResult;
function isPartialResult(message: any): message is ServerMessagePartialResult;
function isRecognizerMessage(message: ServerMessage): message is RecognizerMessage
function isResult(message: any): message is ServerMessageResult
function isPartialResult(message: any): message is ServerMessagePartialResult
}
src/renderer/plugins/asr/vosk/model.d.ts
View file @ fd1558a6
import { ModelMessage, RecognizerEvent, RecognizerMessage } from "./interfaces";
export * from "./interfaces";
import { ModelMessage, RecognizerEvent, RecognizerMessage } from './interfaces'
export * from './interfaces'
export declare class Model extends EventTarget {
private modelUrl;
private worker;
private _ready;
private messagePort;
private logger;
private recognizers;
constructor(modelUrl: string, logLevel?: number);
private initialize;
private postMessage;
private handleMessage;
on(event: ModelMessage["event"], listener: (message: ModelMessage) => void): void;
registerPort(port: MessagePort): void;
private forwardMessage;
get ready(): boolean;
terminate(): void;
setLogLevel(level: number): void;
registerRecognizer(recognizer: KaldiRecognizer): void;
unregisterRecognizer(recognizerId: string): void;
private modelUrl
private worker
private _ready
private messagePort
private logger
private recognizers
constructor(modelUrl: string, logLevel?: number)
private initialize
private postMessage
private handleMessage
on(event: ModelMessage['event'], listener: (message: ModelMessage) => void): void
registerPort(port: MessagePort): void
private forwardMessage
get ready(): boolean
terminate(): void
setLogLevel(level: number): void
registerRecognizer(recognizer: KaldiRecognizer): void
unregisterRecognizer(recognizerId: string): void
/**
* KaldiRecognizer anonymous class
*/
get KaldiRecognizer(): {
new (sampleRate: number, grammar?: string): {
id: string;
on(event: RecognizerEvent, listener: (message: RecognizerMessage) => void): void;
setWords(words: boolean): void;
acceptWaveform(buffer: AudioBuffer): void;
acceptWaveformFloat(buffer: Float32Array, sampleRate: number): void;
retrieveFinalResult(): void;
remove(): void;
addEventListener(type: string, callback: EventListenerOrEventListenerObject | null, options?: boolean | AddEventListenerOptions | undefined): void;
dispatchEvent(event: Event): boolean;
removeEventListener(type: string, callback: EventListenerOrEventListenerObject | null, options?: boolean | EventListenerOptions | undefined): void;
};
};
new (
sampleRate: number,
grammar?: string
): {
id: string
on(event: RecognizerEvent, listener: (message: RecognizerMessage) => void): void
setWords(words: boolean): void
acceptWaveform(buffer: AudioBuffer): void
acceptWaveformFloat(buffer: Float32Array, sampleRate: number): void
retrieveFinalResult(): void
remove(): void
addEventListener(
type: string,
callback: EventListenerOrEventListenerObject | null,
options?: boolean | AddEventListenerOptions | undefined
): void
dispatchEvent(event: Event): boolean
removeEventListener(
type: string,
callback: EventListenerOrEventListenerObject | null,
options?: boolean | EventListenerOptions | undefined
): void
}
}
}
export declare type KaldiRecognizer = InstanceType<Model["KaldiRecognizer"]>;
export declare function createModel(modelUrl: string, logLevel?: number): Promise<Model>;
export declare type KaldiRecognizer = InstanceType<Model['KaldiRecognizer']>
export declare function createModel(modelUrl: string, logLevel?: number): Promise<Model>
src/renderer/plugins/asr/vosk/utils/logging.d.ts
View file @ fd1558a6
export declare class Logger {
private logLevel;
constructor(logLevel?: number);
getLogLevel(): number;
setLogLevel(level: number): void;
error(message: string): void;
warn(message: string): void;
info(message: string): void;
verbose(message: string): void;
debug(message: string): void;
private logLevel
constructor(logLevel?: number)
getLogLevel(): number
setLogLevel(level: number): void
error(message: string): void
warn(message: string): void
info(message: string): void
verbose(message: string): void
debug(message: string): void
}
src/renderer/plugins/asr/vosk/vosk.d.ts
View file @ fd1558a6
export * from "./model";
export * from './model'
src/renderer/plugins/asr/vosk/vosk.js
View file @ fd1558a6
This source diff could not be displayed because it is too large. You can view the blob instead.
src/renderer/plugins/asr/vosk/worker.d.ts
View file @ fd1558a6
import * as VoskWasm from "./vosk-wasm";
import * as VoskWasm from './vosk-wasm'
export interface Recognizer {
id: string;
buffAddr?: number;
buffSize?: number;
recognizer: VoskWasm.Recognizer;
sampleRate: number;
words?: boolean;
grammar?: string;
id: string
buffAddr?: number
buffSize?: number
recognizer: VoskWasm.Recognizer
sampleRate: number
words?: boolean
grammar?: string
}
export declare class RecognizerWorker {
private Vosk;
private model;
private recognizers;
private logger;
constructor();
private handleMessage;
private load;
private allocateBuffer;
private freeBuffer;
private createRecognizer;
private setConfiguration;
private processAudioChunk;
private retrieveFinalResult;
private removeRecognizer;
private terminate;
private Vosk
private model
private recognizers
private logger
constructor()
private handleMessage
private load
private allocateBuffer
private freeBuffer
private createRecognizer
private setConfiguration
private processAudioChunk
private retrieveFinalResult
private removeRecognizer
private terminate
}
src/renderer/plugins/tts/FetchTTS.ts
View file @ fd1558a6
export async function audioAiTTS({ host, text, speaker, speed, provider }: { host: string; text: string; speaker: string; speed: number; provider: number; }) {
const resp = await fetch(`${ host }/api/live/audioAI`, {
"headers": {
"accept": "application/json, text/plain, */*",
"content-type": "application/json",
export async function audioAiTTS({
host,
text,
speaker,
speed,
provider
}: {
host: string
text: string
speaker: string
speed: number
provider: number
}) {
const resp = await fetch(`${host}/api/live/audioAI`, {
headers: {
accept: 'application/json, text/plain, */*',
'content-type': 'application/json'
},
body: JSON.stringify({
type: 1,
......@@ -14,11 +26,11 @@ export async function audioAiTTS({ host, text, speaker, speed, provider }: { hos
sceneType: 1,
gender: 1
}),
method: "POST",
mode: "cors",
});
method: 'POST',
mode: 'cors'
})
const res = await resp.json();
if (res.code !== 200) throw new Error(JSON.stringify(res));
return res.data;
const res = await resp.json()
if (res.code !== 200) throw new Error(JSON.stringify(res))
return res.data
}
src/renderer/public/vosk/recognizer-processor.js
View file @ fd1558a6
class RecognizerAudioProcessor extends AudioWorkletProcessor {
constructor(options) {
super(options);
super(options)
this.port.onmessage = this._processMessage.bind(this);
this.port.onmessage = this._processMessage.bind(this)
}
_processMessage(event) {
// console.debug(`Received event ${JSON.stringify(event.data, null, 2)}`);
if (event.data.action === "init") {
this._recognizerId = event.data.recognizerId;
this._recognizerPort = event.ports[0];
if (event.data.action === 'init') {
this._recognizerId = event.data.recognizerId
this._recognizerPort = event.ports[0]
}
}
process(inputs, outputs, parameters) {
const data = inputs[0][0];
const data = inputs[0][0]
if (this._recognizerPort && data) {
// AudioBuffer samples are represented as floating point numbers between -1.0 and 1.0 whilst
// Kaldi expects them to be between -32768 and 32767 (the range of a signed int16)
const audioArray = data.map((value) => value * 0x8000);
const audioArray = data.map((value) => value * 0x8000)
this._recognizerPort.postMessage(
{
action: "audioChunk",
action: 'audioChunk',
data: audioArray,
recognizerId: this._recognizerId,
sampleRate, // Part of AudioWorkletGlobalScope
sampleRate // Part of AudioWorkletGlobalScope
},
{
transfer: [audioArray.buffer],
transfer: [audioArray.buffer]
}
);
)
}
return true;
return true
}
}
......
src/renderer/screens/PhotoScreen.vue
View file @ fd1558a6
......@@ -5,11 +5,11 @@
// import { useCounterStore } from '@/renderer/store/counter'
// import { storeToRefs } from 'pinia'
import { onMounted, ref } from 'vue'
import useStore from '@/renderer/store';
import { storeToRefs } from 'pinia';
import useStore from '@/renderer/store'
import { storeToRefs } from 'pinia'
const { photo: usePhoto } = useStore();
const photo = storeToRefs(usePhoto);
const { photo: usePhoto } = useStore()
const photo = storeToRefs(usePhoto)
// const { availableLocales } = useI18n()
// const { counterIncrease } = useCounterStore()
......@@ -20,61 +20,76 @@ const photo = storeToRefs(usePhoto);
onMounted((): void => {
// languages.value = availableLocales
// window.mainApi.receive('msgReceivedVersion', (event: Event, version: string) => {
// appVersion.value = version
// })
// window.mainApi.send('msgRequestGetVersion')
})
async function handleOpen(event: Event,url: string) {
await window.mainApi.send('openWindow', `#show?url=${url}`, { width: window.screen.width / 4 , height: window.screen.height });
async function handleOpen(event: Event, url: string) {
await window.mainApi.send('openWindow', `#show?url=${url}`, {
width: window.screen.width / 4,
height: window.screen.height
})
}
const validateURL = (url: string) => {
const regex = /^(https?|ftp):\/\/([\w/\-?=%.]+\.[\w/\-?=%.]+)$/;
return regex.test(url);
const regex = /^(https?|ftp):\/\/([\w/\-?=%.]+\.[\w/\-?=%.]+)$/
return regex.test(url)
}
const urlValue = ref('');
const imgLoading = ref(false);
const urlValue = ref('')
const imgLoading = ref(false)
async function appendPhoto(url: string) {
urlValue.value = url;
urlValue.value = url
if (!validateURL(url)) return '请输入正确的 url！如（url（https://xxx.png）'
try {
imgLoading.value = true;
const img = new Image();
img.src = url;
imgLoading.value = true
const img = new Image()
img.src = url
await new Promise((resolve, reject) => {
img.onload = resolve;
img.onerror = reject;
});
imgLoading.value = false;
img.onload = resolve
img.onerror = reject
})
imgLoading.value = false
} catch (error) {
imgLoading.value = false;
imgLoading.value = false
return '图片加载失败！'
}
photo.list.value.push({ url });
photo.list.value.push({ url })
urlValue.value = ''
return true;
return true
}
function removePhoto(index: number) {
photo.list.value.splice(index, 1)
}
</script>
<template>
<v-container class="d-flex mt-6 pb-0">
<v-text-field label="自定义照片 url（https://xxx.png）" :model-value="urlValue" :loading="imgLoading" :rules="[ v => appendPhoto(v) ]" validate-on="blur lazy" ></v-text-field>
<v-text-field
label="自定义照片 url（https://xxx.png）"
:model-value="urlValue"
:loading="imgLoading"
:rules="[(v) => appendPhoto(v)]"
validate-on="blur lazy"
></v-text-field>
</v-container>
<v-container class="d-flex flex-wrap">
<v-sheet v-for="(item, index) in photo.list.value" :key="item.url" v-ripple :elevation="3" width="200" class="d-flex spacing-playground pa-6 mr-4 mt-4" rounded >
<v-sheet
v-for="(item, index) in photo.list.value"
:key="item.url"
v-ripple
:elevation="3"
width="200"
class="d-flex spacing-playground pa-6 mr-4 mt-4"
rounded
>
<v-img
:width="200"
aspect-ratio="1/1"
......@@ -82,8 +97,13 @@ function removePhoto(index: number) {
:src="item.url"
@click="handleOpen($event, item.url)"
></v-img>
<v-btn density="compact" elevation="1" icon="mdi-close" class="mt-n7" @click="removePhoto(index)"></v-btn>
<v-btn
density="compact"
elevation="1"
icon="mdi-close"
class="mt-n7"
@click="removePhoto(index)"
></v-btn>
</v-sheet>
</v-container>
</template>
src/renderer/screens/ShowPhoto.vue
View file @ fd1558a6
<script setup lang="ts">
import { ref } from 'vue';
import { ref } from 'vue'
import { useRoute, useRouter } from 'vue-router'
import { Vosk } from '@/renderer/plugins/asr/index'
import type { ServerMessagePartialResult, ServerMessageResult, Model } from '@/renderer/plugins/asr/index'
import { audioAiTTS } from '../plugins/tts';
import useStore from '@/renderer/store';
import type {
ServerMessagePartialResult,
ServerMessageResult,
Model
} from '@/renderer/plugins/asr/index'
import { audioAiTTS } from '../plugins/tts'
import useStore from '@/renderer/store'
const router = useRouter()
const route = useRoute();
const { settings } = useStore();
const sampleRate = 48000;
const route = useRoute()
const { settings } = useStore()
const sampleRate = 48000
const recordVolume = ref(0);
const recordVolume = ref(0)
router.beforeEach(g => {
if (!g.query.url) return router.push('/error');
router.beforeEach((g) => {
if (!g.query.url) return router.push('/error')
})
const microphoneState = ref< 'waitInput' | 'input' | 'loading' | 'disabled'>('waitInput');
const microphoneState = ref<'waitInput' | 'input' | 'loading' | 'disabled'>('waitInput')
async function initVosk({ modelPath, result, partialResult }: {
modelPath: string;
result?: (string) => void;
partialResult?: (string) => void;
}) {
const channel = new MessageChannel();
const model = await Vosk.createModel(modelPath);
const recognizer = new model.KaldiRecognizer(sampleRate);
async function initVosk({
modelPath,
result,
partialResult
}: {
modelPath: string
result?: (string) => void
partialResult?: (string) => void
}) {
const channel = new MessageChannel()
const model = await Vosk.createModel(modelPath)
const recognizer = new model.KaldiRecognizer(sampleRate)
model.registerPort(channel.port1);
recognizer.setWords(true);
model.registerPort(channel.port1)
recognizer.setWords(true)
recognizer.on('result', (message) => {
result && result((message as ServerMessageResult).result.text)
});
})
recognizer.on('partialresult', (message) => {
partialResult && partialResult((message as ServerMessagePartialResult).result.partial)
});
})
return { recognizer, channel };
return { recognizer, channel }
}
function analyzeMicrophoneVolume(stream: MediaStream, callback: (number) => void) {
const audioContext = new AudioContext();
const analyser = audioContext.createAnalyser();
const microphone = audioContext.createMediaStreamSource(stream);
const recordEventNode = audioContext.createScriptProcessor(2048, 1, 1);
const audioContext = new AudioContext()
const analyser = audioContext.createAnalyser()
const microphone = audioContext.createMediaStreamSource(stream)
const recordEventNode = audioContext.createScriptProcessor(2048, 1, 1)
const audioprocess = () => {
const array = new Uint8Array(analyser.frequencyBinCount);
analyser.getByteFrequencyData(array);
let values = 0;
const length = array.length;
const array = new Uint8Array(analyser.frequencyBinCount)
analyser.getByteFrequencyData(array)
let values = 0
const length = array.length
for (let i = 0; i < length; i++) {
values += array[i];
values += array[i]
}
const average = values / length;
callback(Math.round(average));
const average = values / length
callback(Math.round(average))
}
analyser.smoothingTimeConstant = 0.8;
analyser.fftSize = 1024;
microphone.connect(analyser);
analyser.connect(recordEventNode);
recordEventNode.connect(audioContext.destination);
analyser.smoothingTimeConstant = 0.8
analyser.fftSize = 1024
microphone.connect(analyser)
analyser.connect(recordEventNode)
recordEventNode.connect(audioContext.destination)
// recordEventNode.addEventListener('audioprocess', audioprocess);
recordEventNode.onaudioprocess = audioprocess;
recordEventNode.onaudioprocess = audioprocess
inputContext.audioContext2 = audioContext;
inputContext.scriptProcessorNode = recordEventNode;
inputContext.audioContext2 = audioContext
inputContext.scriptProcessorNode = recordEventNode
}
const inputContext: { mediaStream?: MediaStream; audioContext?: AudioContext; audioContext2?: AudioContext; scriptProcessorNode?: ScriptProcessorNode; model?: Model } = {};
const inputContext: {
mediaStream?: MediaStream
audioContext?: AudioContext
audioContext2?: AudioContext
scriptProcessorNode?: ScriptProcessorNode
model?: Model
} = {}
async function startAudioInput() {
if (microphoneState.value === 'loading') return;
if (microphoneState.value === 'loading') return
if (microphoneState.value === 'input') {
microphoneState.value = 'waitInput';
inputContext.mediaStream?.getTracks().forEach((track) => track.stop());
inputContext.audioContext?.close();
inputContext.audioContext2?.close();
inputContext.scriptProcessorNode && (inputContext.scriptProcessorNode.onaudioprocess = null);
inputContext.model?.terminate();
return;
microphoneState.value = 'waitInput'
inputContext.mediaStream?.getTracks().forEach((track) => track.stop())
inputContext.audioContext?.close()
inputContext.audioContext2?.close()
inputContext.scriptProcessorNode && (inputContext.scriptProcessorNode.onaudioprocess = null)
inputContext.model?.terminate()
return
}
microphoneState.value = 'loading';
microphoneState.value = 'loading'
const { recognizer, channel } = await initVosk({
modelPath: new URL(`/vosk/models/${settings.voskSelectModel}`, import.meta.url).href,
result: async (text) => {
console.log('----------------> text:', text);
const tone = settings.source.find(({ sourceId }) => settings.selectSource === sourceId);
if (!tone) return;
const res = await audioAiTTS({ host: settings.ttsHost, text, speed: 3, speaker: tone.sourceId, provider: tone.provider });
console.log('----------------> tts:', res);
},
partialResult: text => {
console.log('----------------> partialResult:', text);
console.log('----------------> text:', text)
const tone = settings.source.find(({ sourceId }) => settings.selectSource === sourceId)
if (!tone) return
const res = await audioAiTTS({
host: settings.ttsHost,
text,
speed: 3,
speaker: tone.sourceId,
provider: tone.provider
})
console.log('----------------> tts:', res)
},
});
partialResult: (text) => {
console.log('----------------> partialResult:', text)
}
})
const mediaStream = await navigator.mediaDevices.getUserMedia({
video: false,
......@@ -109,64 +129,86 @@ async function startAudioInput() {
noiseSuppression: true,
channelCount: 1,
sampleRate
},
});
}
})
const audioContext = new AudioContext();
const audioContext = new AudioContext()
await audioContext.audioWorklet.addModule(new URL('/vosk/recognizer-processor.js', import.meta.url))
const recognizerProcessor = new AudioWorkletNode(audioContext, 'recognizer-processor', { channelCount: 1, numberOfInputs: 1, numberOfOutputs: 1 });
recognizerProcessor.port.postMessage({action: 'init', recognizerId: recognizer.id}, [ channel.port2 ])
recognizerProcessor.connect(audioContext.destination);
await audioContext.audioWorklet.addModule(
new URL('/vosk/recognizer-processor.js', import.meta.url)
)
const recognizerProcessor = new AudioWorkletNode(audioContext, 'recognizer-processor', {
channelCount: 1,
numberOfInputs: 1,
numberOfOutputs: 1
})
recognizerProcessor.port.postMessage({ action: 'init', recognizerId: recognizer.id }, [
channel.port2
])
recognizerProcessor.connect(audioContext.destination)
const source = audioContext.createMediaStreamSource(mediaStream);
source.connect(recognizerProcessor);
const source = audioContext.createMediaStreamSource(mediaStream)
source.connect(recognizerProcessor)
await analyzeMicrophoneVolume(mediaStream, (val) => {
recordVolume.value = val;
});
recordVolume.value = val
})
microphoneState.value = 'input';
microphoneState.value = 'input'
inputContext.mediaStream = mediaStream;
inputContext.audioContext = audioContext;
inputContext.mediaStream = mediaStream
inputContext.audioContext = audioContext
}
function endAudioInput() {
console.log('----------------> end');
console.log('----------------> end')
}
</script>
<template>
<div style="width: 100%; height: 100%;" class="d-flex justify-center align-center">
<div style="width: 100%; height: 100%" class="d-flex justify-center align-center">
<v-img
v-if="route.query.url"
:width="'100%'"
aspect-ratio="1/1"
cover
:src="(route.query.url as string)"
:src="route.query.url as string"
></v-img>
</div>
<div class="voice">
<v-btn icon="" color="#fff" variant="elevated" size="x-large" :disabled="microphoneState === 'loading' || microphoneState ==='disabled'" @pointerdown="startAudioInput" @pointerup="endAudioInput">
<v-btn
icon=""
color="#fff"
variant="elevated"
size="x-large"
:disabled="microphoneState === 'loading' || microphoneState === 'disabled'"
@pointerdown="startAudioInput"
@pointerup="endAudioInput"
>
<v-icon v-if="microphoneState === 'waitInput'" icon="mdi-microphone"></v-icon>
<v-icon v-if="microphoneState === 'loading'" icon="mdi-microphone-settings"></v-icon>
<v-icon v-if="microphoneState === 'disabled'" icon="mdi-microphone-off"></v-icon>
<template v-if="microphoneState === 'input'">
<img width="30" height="30" src="/images/microphone-input.svg" alt="" srcset="">
<img width="30" height="30" src="/images/microphone-input.svg" alt="" srcset="" />
<div class="progress">
<span class="volume" :style="{ 'clip-path': `polygon(0 ${100 - recordVolume}%, 100% ${100 - recordVolume}%, 100% 100%, 0 100%)` }"></span>
<span
class="volume"
:style="{
'clip-path': `polygon(0 ${100 - recordVolume}%, 100% ${
100 - recordVolume
}%, 100% 100%, 0 100%)`
}"
></span>
</div>
</template>
</v-btn>
</div>
</template>
<style scoped>
.voice {
.voice {
display: flex;
justify-content: center;
position: fixed;
......@@ -174,9 +216,9 @@ function endAudioInput() {
right: 0;
top: 70%;
margin: auto;
}
}
.progress{
.progress {
position: absolute;
top: 21px;
left: 28px;
......@@ -184,13 +226,12 @@ function endAudioInput() {
height: 16px;
overflow: hidden;
border-radius: 36%;
}
.progress .volume{
}
.progress .volume {
display: block;
width: 100%;
height: 100%;
background: #2FB84F;
background: #2fb84f;
border-radius: 36%;
}
}
</style>
src/renderer/store/index.ts
View file @ fd1558a6
import useSettings from './settings';
import usePhoto from './photo';
import useSettings from './settings'
import usePhoto from './photo'
export default function useStore() {
return {
......
src/renderer/store/photo.ts
View file @ fd1558a6
......@@ -6,7 +6,8 @@ type IPhoto = {
const usePhotoStore = defineStore('photo', {
persist: true,
state: () => ({
state: () =>
({
list: [
{
url: 'https://resources.laihua.com/2023-11-2/93ffb6a7-ae93-4918-944e-877016ba266b.png'
......@@ -15,11 +16,9 @@ const usePhotoStore = defineStore('photo', {
url: 'https://resources.laihua.com/2023-6-19/6fa9a127-2ce5-43ea-a543-475bf9354eda.png'
}
]
} as IPhoto),
getters: {
},
actions: {
}
}) as IPhoto,
getters: {},
actions: {}
})
export default usePhotoStore
src/renderer/store/settings.ts
View file @ fd1558a6
import { defineStore } from 'pinia'
export type ISettings = {
asr: 'vosk_asr' | 'xf_asr';
voskModels: string[];
voskSelectModel: string;
ttsHost: string;
source: { sourceName: string; sourceId: string; provider: number; speaker: string; description: string; sex: 1 | 0 }[];
selectSource: string;
asr: 'vosk_asr' | 'xf_asr'
voskModels: string[]
voskSelectModel: string
ttsHost: string
source: {
sourceName: string
sourceId: string
provider: number
speaker: string
description: string
sex: 1 | 0
}[]
selectSource: string
}
const useSettingsStore = defineStore('settings', {
persist: true,
state: () => ({
state: () =>
({
asr: 'vosk_asr',
voskModels: [
'vosk-model-small-ca-0.4.tar.gz',
......@@ -32,22 +40,21 @@ const useSettingsStore = defineStore('settings', {
ttsHost: 'https://beta.laihua.com',
source: [],
selectSource: ''
} as ISettings),
getters: {
},
}) as ISettings,
getters: {},
actions: {
async getSource() {
const resp = await fetch(`${this.$state.ttsHost}/api/live/audioAI/source?platform=31`, {
"method": "GET",
"mode": "cors",
});
method: 'GET',
mode: 'cors'
})
const res = await resp.json();
const res = await resp.json()
if (res.code !== 200) return;
this.source = res.data;
if (res.code !== 200) return
this.source = res.data
}
}
})
export default useSettingsStore;
export default useSettingsStore
tsconfig.json
View file @ fd1558a6
......@@ -26,11 +26,5 @@
"path": "./tsconfig.node.json"
}
],
"exclude": [
"node_modules",
"dist",
"rollup.config.js",
"*.json",
"*.js"
]
"exclude": ["node_modules", "dist", "rollup.config.js", "*.json", "*.js"]
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment