Add builtin decoder.

BUILD.gn

@@ -4324,6 +4324,8 @@ v8_source_set("v8_base_without_compiler") {
 
     # Encoder
     "src/builtins/builtins-encoder.cc",
+    # Decoder
+    "src/builtins/builtins-decoder.cc",
 
     "src/builtins/builtins-number.cc",
     "src/builtins/builtins-object.cc",

src/builtins/builtins-decoder.cc

+
+#include <string>
+#include <dlfcn.h>
+#include <cstring>
+#include <vector>
+#include "builtins-decoder.h"
+#include "src/execution/isolate.h"
+#include "src/builtins/builtins.h"
+#include "src/builtins/builtins-utils-inl.h"
+
+namespace v8 {
+namespace internal {
+namespace {
+
+bool inited;
+void *loadedSyms;
+
+// Output size
+int lastFrameIDX;
+
+// Reference to external entities.
+void *decoder;
+void *resourcer;
+std::vector<void*> *encoders;
+
+/* Reference to external interfaces */
+// Encoder, for testing purposes.
+void* (*spawnEncoder)(const char *output, uint32_t width,
+                   uint32_t height, uint32_t bitrate,
+                   int fmt, int32_t framerate,
+                   const char *encoderName);
+void (*encoding)(void *encoder_, const uint8_t *buff, int length);
+bool (*encoderFailed)(void *encoder_);
+void (*closeEncoder)(void *encoder_);
+const char* (*outputOfEncoder)(void *encoder_);
+
+
+// Decoder
+void*     (*getDecoder)();
+double*   (*newVideoDec)(void *decoder, const char *path);
+bool      (*cropDec)(void *decoder, int index, double start, double end);
+bool      (*resizeDec)(void *decoder, int index, unsigned width, unsigned height);
+uint8_t*  (*getFrameDec)(void *decoder, int index, int *frameIDX, int *frameSize); // This interface make sure that must not return NULL.
+
+// Resourcer
+void* (*getResourcer)();
+int   (*fetchRe)(void *resourcer, const char *url, const char *fsPath);
+void  (*cleanup)(void *resourcer);
+}
+
+/* PRE:
+ * (1) Context is unintialized
+ * POST:
+ * (1) Initialize all of reference of external entities  */
+bool initContext() {
+    if (inited) return true;
+
+    loadedSyms = dlopen("libencoders.so", RTLD_LAZY);
+    if (!loadedSyms) {
+        return false;
+    }
+
+        spawnEncoder = reinterpret_cast<decltype(spawnEncoder)>(dlsym(loadedSyms, "spawnEncoder"));
+        if (spawnEncoder == NULL) {
+            return false;
+        }
+
+        encoding = reinterpret_cast<decltype(encoding)>(dlsym(loadedSyms, "encoding"));
+        if (encoding == NULL) {
+            return false;
+        }
+
+        encoderFailed = reinterpret_cast<decltype(encoderFailed)>(dlsym(loadedSyms, "encoderFailed"));
+        if (encoderFailed == NULL) {
+            return false;
+        }
+
+        closeEncoder = reinterpret_cast<decltype(closeEncoder)>(dlsym(loadedSyms, "closeEncoder"));
+        if (closeEncoder == NULL) {
+            return false;
+        }
+
+        outputOfEncoder = reinterpret_cast<decltype(outputOfEncoder)>(dlsym(loadedSyms, "outputOfEncoder"));
+        if (outputOfEncoder == NULL) {
+            return false;
+        }
+
+
+    // Initialize extern interfaces
+    getDecoder = reinterpret_cast<decltype(getDecoder)>(dlsym(loadedSyms, "getDecoder"));
+    if (getDecoder == NULL) {
+        return false;
+    }
+    newVideoDec = reinterpret_cast<decltype(newVideoDec)>(dlsym(loadedSyms, "newVideoDec"));
+    if (newVideoDec == NULL) {
+        return false;
+    }
+    cropDec = reinterpret_cast<decltype(cropDec)>(dlsym(loadedSyms, "cropDec"));
+    if (cropDec == NULL) {
+        return false;
+    }
+    resizeDec = reinterpret_cast<decltype(resizeDec)>(dlsym(loadedSyms, "resizeDec"));
+    if (resizeDec == NULL) {
+        return false;
+    }
+    getFrameDec = reinterpret_cast<decltype(getFrameDec)>(dlsym(loadedSyms, "getFrameDec"));
+    if (getFrameDec == NULL) {
+        return false;
+    }
+    getResourcer = reinterpret_cast<decltype(getResourcer)>(dlsym(loadedSyms, "getResourcer"));
+    if (getResourcer == NULL) {
+        return false;
+    }
+    fetchRe = reinterpret_cast<decltype(fetchRe)>(dlsym(loadedSyms, "fetchRe"));
+    if (fetchRe == NULL) {
+        return false;
+    }
+
+    // Initialize extern entities
+    decoder = getDecoder();
+    if (!decoder) {
+         return false;
+    }
+
+    resourcer = getResourcer();
+    if (!resourcer) {
+        return false;
+    }
+
+    encoders = new std::vector<void*>();
+
+    return true;
+}
+
+/* POST:
+ * (1) Download video file with a unique path
+ * (2) Add video file to decoder
+ * (3) Download file is keep tracked */
+BUILTIN(AddVideoURL) {
+    HandleScope scope(isolate);
+
+    if (!inited) {
+        if (!initContext()) {
+         THROW_NEW_ERROR_RETURN_FAILURE(
+            isolate, NewTypeError(MessageTemplate::kUnsupported));
+        }
+        inited = true;
+    }
+
+    Handle<Object> url = args.atOrUndefined(isolate, 1);
+    if (!url->IsString()) {
+        THROW_NEW_ERROR_RETURN_FAILURE(
+            isolate, NewTypeError(MessageTemplate::kInvalidArgument));
+    }
+
+    Handle<String> url_str;
+    ASSIGN_RETURN_FAILURE_ON_EXCEPTION(
+        isolate, url_str, Object::ToString(isolate, url));
+    std::string url_cppstr = url_str->ToCString().get();
+
+    std::string outpath = std::tmpnam(nullptr);
+    outpath += ".mp4";
+
+    if (resourcer == NULL) {
+        resourcer = getResourcer();
+    }
+
+    int errorCode = fetchRe(resourcer, url_cppstr.c_str(), outpath.c_str());
+    if (errorCode == 200) {
+        // Add video file to decoder.
+
+        // newVideoDec() return array with type double[6];
+        double *info = newVideoDec(decoder, outpath.c_str());
+
+        // info has 6 elements with type double,
+        // each of them own 8 bytes.
+        int size_bytes = 6 * 8;
+
+        MaybeHandle<JSArrayBuffer> infoBuffer = isolate->factory()->NewJSArrayBufferAndBackingStore(
+            size_bytes, InitializedFlag::kZeroInitialized);
+        if (infoBuffer.is_null()) {
+            THROW_NEW_ERROR_RETURN_FAILURE(
+                isolate, NewEvalError(MessageTemplate::kOutOfMemory));
+        }
+        Handle<JSTypedArray> infoBytes = isolate->factory()->NewJSTypedArray(
+            kExternalFloat64Array, infoBuffer.ToHandleChecked(), 0, size_bytes / 8);
+
+        std::shared_ptr<BackingStore> store = infoBytes->GetBuffer()->GetBackingStore();
+        double *buffer = reinterpret_cast<double*>(store->buffer_start());
+
+        for (int i = 0; i < 6; ++i) {
+            buffer[i] = info[i];
+        }
+
+        return *infoBytes;
+    } else {
+        return ReadOnlyRoots(isolate).undefined_value();
+    }
+
+}
+
+/* PARAS:
+ *  Width  : Num
+ *  Height : Num
+ *
+ * Ret:
+ *  Success: true
+ *  Fail:    false
+ *
+ * PRE: Inited & File-Added & Argument(1,2,3): Num
+ * POST: Resized  */
+BUILTIN(Resize) {
+    HandleScope scope(isolate);
+
+    if (!inited) return *isolate->factory()->ToBoolean(true);
+
+    Handle<Object> indexObj = args.atOrUndefined(isolate, 1);
+    Handle<Object> widthObj = args.atOrUndefined(isolate, 2);
+    Handle<Object> heightObj = args.atOrUndefined(isolate, 3);
+
+    if (!indexObj->IsNumber() || !widthObj->IsNumber() || !heightObj->IsNumber()) {
+        return *isolate->factory()->ToBoolean(false);
+    }
+
+    int index, width, height;
+
+    indexObj->ToInt32(&index);
+    widthObj->ToInt32(&width);
+    heightObj->ToInt32(&height);
+
+    return *isolate->factory()->ToBoolean(
+        resizeDec(decoder, index, width, height));
+}
+
+BUILTIN(Crop) {
+    HandleScope scope(isolate);
+
+    if (!inited) return *isolate->factory()->ToBoolean(true);
+
+    Handle<Object> indexObj = args.atOrUndefined(isolate, 1);
+    Handle<Object> startObj = args.atOrUndefined(isolate, 2);
+    Handle<Object> endObj = args.atOrUndefined(isolate, 3);
+
+    if (!indexObj->IsNumber() || !startObj->IsNumber() || !endObj->IsNumber()) {
+        return *isolate->factory()->ToBoolean(false);
+    }
+
+    int index;
+    double start = startObj->Number(), end = endObj->Number();
+
+    indexObj->ToInt32(&index);
+
+
+    return *isolate->factory()->ToBoolean(
+        cropDec(decoder, index, start, end));
+}
+
+BUILTIN(GetNextFrame) {
+    HandleScope scope(isolate);
+
+    if (!inited) return *isolate->factory()->ToBoolean(true);
+
+    Handle<Object> indexObj = args.atOrUndefined(isolate, 1);
+    if (!indexObj->IsNumber()) {
+        return ReadOnlyRoots(isolate).undefined_value();
+    }
+
+    int index, frameSize;
+    indexObj->ToInt32(&index);
+
+    uint8_t *frameData = getFrameDec(decoder, index, &lastFrameIDX, &frameSize);
+
+    MaybeHandle<JSArrayBuffer> frameBuffer = isolate->factory()->NewJSArrayBufferAndBackingStore(
+        frameSize, InitializedFlag::kZeroInitialized);
+    if (frameBuffer.is_null()) {
+        THROW_NEW_ERROR_RETURN_FAILURE(
+            isolate, NewEvalError(MessageTemplate::kOutOfMemory));
+    }
+    Handle<JSTypedArray> frameBytes = isolate->factory()->NewJSTypedArray(
+        kExternalUint8Array, frameBuffer.ToHandleChecked(), 0, frameSize);
+
+    std::shared_ptr<BackingStore> store = frameBytes->GetBuffer()->GetBackingStore();
+    char *buffer = reinterpret_cast<char*>(store->buffer_start());
+
+    // Memory referenced by 'frameData' is maintained by external library,
+    // this Builtin should not broken the memory return from external function.
+    std::memcpy(buffer, frameData, frameSize);
+
+    return *frameBytes;
+}
+
+BUILTIN(GetLastFrameIDX) {
+    HandleScope scope(isolate);
+    return *isolate->factory()->NewNumberFromInt(lastFrameIDX);
+}
+
+BUILTIN(Cleanup) {
+    HandleScope scope(isolate);
+
+    if (!inited) return ReadOnlyRoots(isolate).undefined_value();
+
+    cleanup(resourcer);
+    dlclose(loadedSyms);
+
+    return ReadOnlyRoots(isolate).undefined_value();
+}
+
+} // internal
+} // v8

src/builtins/builtins-definitions.h

@@ -631,6 +631,14 @@ namespace internal {
   CPP(ApplyAudioEffects)                                                       \
   CPP(AddAudioURL)                                                             \
                                                                                \
+  /* DECODER */                                                                \
+  CPP(AddVideoURL)                                                             \
+  CPP(Resize)                                                                  \
+  CPP(Crop)                                                                    \
+  CPP(GetNextFrame)                                                            \
+  CPP(Cleanup)                                                                 \
+  CPP(GetLastFrameIDX)                                                         \
+                                                                               \
   /* Web snapshots */                                                          \
   CPP(WebSnapshotSerialize)                                                    \
   CPP(WebSnapshotDeserialize)                                                  \

src/builtins/builtins-encoder.cc

@@ -43,7 +43,6 @@ int (*addInputFileAE)(void *effecter, const char *filename, const double volume,
                    bool isLoop);
 int (*doTranscodingAE)(void *effecter, const char *output, const double duration);
 int (*isWeakAE)(void *effecter);
-void (*removeInputsAE)(void *effecter);
 
 // AV Merger
 void* (*spawnMerger)(const char *output, const char *videoPath, const char *audioPath);
@@ -52,6 +51,7 @@ void (*merging)(void *merger_);
 // Resourcer
 void* (*getResourcer)();
 int   (*fetchRe)(void *resourcer, const char *url, const char *fsPath);
+void  (*cleanupRe)(void *resourcer);
 
 }
 
@@ -62,7 +62,7 @@ BUILTIN(InitEncodeContext) {
     if (!loadedSyms) {
         loadedSyms = dlopen("libencoders.so", RTLD_LAZY);
         if (!loadedSyms) {
-            printf("Failed to load libencoder.so");
+            printf("Failed to load libencoders.so\n");
             THROW_NEW_ERROR_RETURN_FAILURE(
                 isolate, NewEvalError(MessageTemplate::kInvalidArgument));
         }
@@ -137,13 +137,6 @@ BUILTIN(InitEncodeContext) {
                 isolate, NewEvalError(MessageTemplate::kInvalidArgument));
         }
 
-        removeInputsAE = reinterpret_cast<decltype(removeInputsAE)>(
-            dlsym(loadedSyms, "removeInputsAE"));
-        if (removeInputsAE == NULL) {
-             THROW_NEW_ERROR_RETURN_FAILURE(
-                isolate, NewEvalError(MessageTemplate::kInvalidArgument));
-        }
-
         getResourcer = reinterpret_cast<decltype(getResourcer)>(
             dlsym(loadedSyms, "getResourcer"));
         if (getResourcer == NULL) {
@@ -157,6 +150,13 @@ BUILTIN(InitEncodeContext) {
              THROW_NEW_ERROR_RETURN_FAILURE(
                 isolate, NewEvalError(MessageTemplate::kInvalidArgument));
         }
+
+        cleanupRe = reinterpret_cast<decltype(cleanupRe)>(
+            dlsym(loadedSyms, "cleanupRe"));
+        if (cleanupRe == NULL) {
+             THROW_NEW_ERROR_RETURN_FAILURE(
+                isolate, NewEvalError(MessageTemplate::kInvalidArgument));
+        }
     }
 
     // Width
@@ -283,7 +283,7 @@ BUILTIN(Close) {
     encoder = nullptr;
     inited = false;
 
-    if (audioEffecter) removeInputsAE(audioEffecter);
+    if (resourcer) cleanupRe(resourcer);
 
     return *isolate->factory()->ToBoolean(true);
 }

src/init/bootstrapper.cc

@@ -2794,25 +2794,47 @@ void Genesis::InitializeGlobal(Handle<JSGlobalObject> global_object,
     Handle<JSObject> mp4 =
         factory->NewJSObject(isolate_->object_function(), AllocationType::kOld);
     JSObject::AddProperty(isolate_, media_codex, "Mp4", mp4, DONT_ENUM);
-    SimpleInstallFunction(isolate_, mp4, "init",
+
+    Handle<JSObject> encoder =
+      factory->NewJSObject(isolate_->object_function(), AllocationType::kOld);
+    JSObject::AddProperty(isolate_, mp4, "Encoder", encoder, DONT_ENUM);
+
+    SimpleInstallFunction(isolate_, encoder, "init",
                           Builtin::kInitEncodeContext, 5, true);
-    SimpleInstallFunction(isolate_, mp4, "encode",
+    SimpleInstallFunction(isolate_, encoder, "encode",
                           Builtin::kEncode, 1, true);
-    SimpleInstallFunction(isolate_, mp4, "close",
+    SimpleInstallFunction(isolate_, encoder, "close",
                           Builtin::kClose, 0, true);
-    SimpleInstallFunction(isolate_, mp4, "getVideo",
+    SimpleInstallFunction(isolate_, encoder, "getVideo",
                           Builtin::kGetVideo, 0, true);
-    SimpleInstallFunction(isolate_, mp4, "save",
+    SimpleInstallFunction(isolate_, encoder, "save",
                           Builtin::kSave, 2, true);
-    SimpleInstallFunction(isolate_, mp4, "addAudioBuffer",
+    SimpleInstallFunction(isolate_, encoder, "addAudioBuffer",
                           Builtin::kAddAudioBuffer, 9, true);
-    SimpleInstallFunction(isolate_, mp4, "addAudioURL",
+    SimpleInstallFunction(isolate_, encoder, "addAudioURL",
                           Builtin::kAddAudioURL, 9, true);
-    SimpleInstallFunction(isolate_, mp4, "mergeMP4",
+    SimpleInstallFunction(isolate_, encoder, "mergeMP4",
                           Builtin::kMergeMP4, 2, true);
-    SimpleInstallFunction(isolate_, mp4, "applyAudioEffects",
+    SimpleInstallFunction(isolate_, encoder, "applyAudioEffects",
                           Builtin::kApplyAudioEffects, 1, true);
 
+    Handle<JSObject> decoder =
+      factory->NewJSObject(isolate_->object_function(), AllocationType::kOld);
+    JSObject::AddProperty(isolate_, mp4, "Decoder", decoder, DONT_ENUM);
+
+    SimpleInstallFunction(isolate_, decoder, "addVideoURL",
+                          Builtin::kAddVideoURL, 1, true);
+    SimpleInstallFunction(isolate_, decoder, "resize",
+                          Builtin::kResize, 3, true);
+    SimpleInstallFunction(isolate_, decoder, "crop",
+                          Builtin::kCrop, 3, true);
+    SimpleInstallFunction(isolate_, decoder, "getNextFrame",
+                          Builtin::kGetNextFrame, 1, true);
+    SimpleInstallFunction(isolate_, decoder, "cleanup",
+                          Builtin::kCleanup, 0, true);
+    SimpleInstallFunction(isolate_, decoder, "getLastFrameIDX",
+                          Builtin::kGetLastFrameIDX, 0, true);
+
     InstallToStringTag(isolate_, media_codex, "MediaCodex");
   }