feat: 照片数字人渲染方式调整

.vscode/settings.json

@@ -19,7 +19,8 @@
     "editor.formatOnSave": true,
     "editor.tabSize": 2,
     "cSpell.words": [
+      "flvjs",
       "Vosk"
     ],
-    "editor.inlineSuggest.showToolbar": "onHover"
+    "editor.inlineSuggest.showToolbar": "always"
 }

src/renderer/screens/ShowPhoto.vue

+<!-- eslint-disable camelcase -->
 <script setup lang="ts">
-import { ref } from 'vue'
+import { onMounted, ref } from 'vue'
 import { useRoute, useRouter } from 'vue-router'
 import type {
   ServerMessagePartialResult,
@@ -8,6 +9,7 @@ import type {
 } from '@/renderer/plugins/asr/index'
 import { audioAiTTS } from '../plugins/tts'
 import useStore from '@/renderer/store'
+import flvjs from 'flv.js';
 
 const router = useRouter()
 const route = useRoute()
@@ -16,13 +18,98 @@ const sampleRate = 48000
 
 const recordVolume = ref(0)
 const url = route.query.url as string
+const microphoneState = ref<'waitInput' | 'input' | 'loading' | 'disabled'>('waitInput')
+const videoElement = ref<HTMLVideoElement | null>(null);
+const can = ref<HTMLCanvasElement | null>(null);
+let flvPlayer: flvjs.Player | null = null;
+
+onMounted(() => {
+  init();
+});
+
+function loadImg(): Promise<HTMLImageElement> {
+  const img = new Image();
+  img.src = url;
+  return new Promise((resolve, reject) => {
+    img.onload = () => resolve(img);
+    img.onerror = reject;
+  })
+}
+
+async function init(){
+  const img = await loadImg();
+  const videoEle = videoElement.value;
+  const canvasEle = can.value;
+  const ctx = canvasEle && canvasEle.getContext('2d')
+  if (!videoEle || !canvasEle || !ctx) return;
+
+  draw(ctx, img)
+  canvasEle.width = img.naturalWidth;
+  canvasEle.height = img.naturalHeight;
+
+  initPlayer(videoEle);
+
+  const fps = 1000 / 30;
+  let lastTime = Date.now();
+  const updateFrame = () => {
+    if (Date.now() - lastTime > fps ) {
+      draw(ctx, img, videoEle, {
+        "width": 579,
+        "height": 579,
+        "center": {
+          "x": 295,
+          "y": 168
+        },
+        "r_w": 304,
+        "r_h": 304
+      });
+      lastTime = Date.now();
+    }
+    requestAnimationFrame(updateFrame);
+  }
+  requestAnimationFrame(updateFrame);
+}
+
+function draw(ctx: CanvasRenderingContext2D, img: HTMLImageElement, liveVideo?: HTMLVideoElement, videoInfo?: {
+    center: {
+        x: number;
+        y: number;
+    };
+    width: number;
+    height: number;
+    r_w: number;
+    r_h: number;
+}) {
+    ctx.clearRect(0, 0, img.naturalWidth, img.naturalHeight);
+    ctx.drawImage(img, 0, 0, img.naturalWidth, img.naturalHeight);
+
+    if (liveVideo && videoInfo) {
+      const { center, r_w, r_h } = videoInfo;
+      ctx.drawImage(liveVideo, center.x - r_w / 2, center.y - r_h / 2, r_w, r_h);
+    }
+  }
+
+async function initPlayer(videoEle: HTMLVideoElement){
+  flvPlayer = flvjs.createPlayer({
+    url: 'http://127.0.0.1:7001/live/movie.flv',
+    type: "flv",
+    isLive: true,
+    cors: true
+    }, {
+    // enableWorker: true,
+    enableStashBuffer: false,
+    stashInitialSize: 128,
+  });
+
+  flvPlayer.attachMediaElement(videoEle);
+  flvPlayer.load();
+  await flvPlayer.play();
+}
 
 router.beforeEach((g) => {
   if (!g.query.url) return router.push('/error')
 })
 
-const microphoneState = ref<'waitInput' | 'input' | 'loading' | 'disabled'>('waitInput')
-
 async function initVosk({
   result,
   partialResult
@@ -168,7 +255,9 @@ function endAudioInput() {
     class="d-flex justify-center align-center"
     :style="{ background: '#000' }"
   >
-    <v-img v-if="url" aspect-ratio="9/16" :src="url"></v-img>
+    <!-- <v-img v-if="url" aspect-ratio="9/16" :src="url" @load="initPlayer()"></v-img> -->
+    <canvas id="can" ref="can" style="width: 100%; height: 100%;"></canvas>
+    <video id="videoElement" ref="videoElement" class="video-ele" :style="{ top: 0, left: 0, width: '250px', height: '250px' }"></video>
   </div>
 
   <div class="voice">
@@ -229,4 +318,9 @@ function endAudioInput() {
   background: #2fb84f;
   border-radius: 36%;
 }
+
+.video-ele {
+  position: absolute;
+  display: none;
+}
 </style>