dnn: introduce dnn operand (in c code) to hold operand infos within network

the info can be saved in dnn operand object without regenerating again and again,
and it is also needed for layer split/merge, and for memory reuse.

to make things step by step, this patch just focuses on c code,
the change within python script will be added later.
Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
Signed-off-by: Pedro Arthur <bygrandao@gmail.com>

libavfilter/dnn/dnn_backend_native.h

@@ -36,12 +36,60 @@ typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc;
 
 typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam;
 
+typedef enum {DOT_INPUT, DOT_INTERMEDIATE, DOT_OUTPUT} DNNOperandType;
+
 typedef struct Layer{
     DNNLayerType type;
-    float *output;
+    /**
+     * a layer can have multiple inputs and one output.
+     * 4 is just a big enough number for input operands (increase it if necessary),
+     * do not use 'int32_t *input_operand_indexes', so we don't worry about mem leaks.
+     */
+    int32_t input_operand_indexes[4];
+    int32_t output_operand_index;
     void *params;
 } Layer;
 
+typedef struct DnnOperand{
+    /**
+     * there are two memory layouts, NHWC or NCHW, so we use dims,
+     * dims[0] is Number.
+     */
+    int32_t dims[4];
+
+    /**
+     * input/output/intermediate operand of the network
+     */
+    DNNOperandType type;
+
+    /**
+     * support different kinds of data type such as float, half float, int8 etc,
+     * first support float now.
+     */
+    DNNDataType data_type;
+
+    /**
+     * NHWC if 1, otherwise NCHW.
+     * let's first support NHWC only, this flag is for extensive usage.
+     */
+    int8_t isNHWC;
+
+    /**
+     * to avoid possible memory leak, do not use char *name
+     */
+    char name[128];
+
+    /**
+     * data pointer with data length in bytes.
+     * usedNumbersLeft is only valid for intermediate operand,
+     * it means how many layers still depend on this operand,
+     * todo: the memory can be reused when usedNumbersLeft is zero.
+     */
+    void *data;
+    int32_t length;
+    int32_t usedNumbersLeft;
+}DnnOperand;
+
 typedef struct ConvolutionalParams{
     int32_t input_num, output_num, kernel_size;
     DNNActivationFunc activation;
@@ -63,6 +111,8 @@ typedef struct DepthToSpaceParams{
 typedef struct ConvolutionalNetwork{
     Layer *layers;
     int32_t layers_num;
+    DnnOperand *operands;
+    int32_t operands_num;
 } ConvolutionalNetwork;
 
 DNNModel *ff_dnn_load_model_native(const char *model_filename);
@@ -71,4 +121,6 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output
 
 void ff_dnn_free_model_native(DNNModel **model);
 
+int32_t calculate_operand_data_length(DnnOperand *operand);
+
 #endif

libavfilter/dnn/dnn_backend_native_layer_pad.c

@@ -48,12 +48,21 @@ static int after_get_buddy(int given, int border, LayerPadModeParam mode)
     }
 }
 
-void dnn_execute_layer_pad(const float *input, float *output, const LayerPadParams *params, int number, int height, int width, int channel)
+int dnn_execute_layer_pad(DnnOperand *operands, const int32_t *input_operand_indexes, int32_t output_operand_index,
+                           const LayerPadParams *params)
 {
     int32_t before_paddings;
     int32_t after_paddings;
+    float* output;
 
     // suppose format is <N, H, W, C>
+    int32_t input_operand_index = input_operand_indexes[0];
+    int number = operands[input_operand_index].dims[0];
+    int height = operands[input_operand_index].dims[1];
+    int width = operands[input_operand_index].dims[2];
+    int channel = operands[input_operand_index].dims[3];
+    const float *input = operands[input_operand_index].data;
+
     int new_number = number + params->paddings[0][0] + params->paddings[0][1];
     int new_height = height + params->paddings[1][0] + params->paddings[1][1];
     int new_width = width + params->paddings[2][0] + params->paddings[2][1];
@@ -67,6 +76,17 @@ void dnn_execute_layer_pad(const float *input, float *output, const LayerPadPara
     int new_wc_stride = new_c_stride * new_width;
     int new_hwc_stride = new_wc_stride * new_height;
 
+    DnnOperand *output_operand = &operands[output_operand_index];
+    output_operand->dims[0] = new_number;
+    output_operand->dims[1] = new_height;
+    output_operand->dims[2] = new_width;
+    output_operand->dims[3] = new_channel;
+    output_operand->length = calculate_operand_data_length(output_operand);
+    output_operand->data = av_realloc(output_operand->data, output_operand->length);
+    if (!output_operand->data)
+        return -1;
+    output = output_operand->data;
+
     // copy the original data
     for (int n = 0; n < number; n++) {
         for (int h = 0; h < height; h++) {
@@ -208,4 +228,6 @@ void dnn_execute_layer_pad(const float *input, float *output, const LayerPadPara
             }
         }
     }
+
+    return 0;
 }

libavfilter/dnn/dnn_backend_native_layer_pad.h

@@ -26,6 +26,7 @@
 #define AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_PAD_H
 
 #include <stdint.h>
+#include "dnn_backend_native.h"
 
 typedef enum {LPMP_CONSTANT, LPMP_REFLECT, LPMP_SYMMETRIC} LayerPadModeParam;
 
@@ -35,6 +36,7 @@ typedef struct LayerPadParams{
     float constant_values;
 } LayerPadParams;
 
-void dnn_execute_layer_pad(const float *input, float *output, const LayerPadParams *params, int number, int height, int width, int channel);
+int dnn_execute_layer_pad(DnnOperand *operands, const int32_t *input_operand_indexes, int32_t output_operand_index,
+                           const LayerPadParams *params);
 
 #endif

tests/dnn/Makefile

@@ -5,7 +5,7 @@ DNNTESTPROGS := $(DNNTESTPROGS:%=$(DNNTESTSDIR)/%-test$(EXESUF))
 -include $(wildcard $(DNNTESTOBJS:.o=.d))
 
 $(DNNTESTPROGS): %$(EXESUF): %.o $(FF_STATIC_DEP_LIBS)
-	$(LD) $(LDFLAGS) $(LDEXEFLAGS) $(LD_O) $(filter %.o,$^) $(FF_STATIC_DEP_LIBS) $(ELIBS)
+	$(LD) $(LDFLAGS) $(LDEXEFLAGS) $(LD_O) $(filter %.o,$^) $(FF_STATIC_DEP_LIBS) $(EXTRALIBS-avcodec) $(EXTRALIBS-avfilter) $(EXTRALIBS-avformat) $(EXTRALIBS-avutil) $(EXTRALIBS-swresample) $(EXTRALIBS)
 
 testclean::
 	$(RM) $(addprefix $(DNNTESTSDIR)/,$(CLEANSUFFIXES) *-test$(EXESUF))

tests/dnn/dnn-layer-pad-test.c

@@ -44,6 +44,8 @@ static int test_with_mode_symmetric(void)
     */
 
     LayerPadParams params;
+    DnnOperand operands[2];
+    int32_t input_indexes[1];
     float input[1*4*4*3] = {
         0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47
     };
@@ -57,8 +59,7 @@ static int test_with_mode_symmetric(void)
         27.0, 28.0, 29.0, 24.0, 25.0, 26.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, 33.0, 34.0, 35.0, 33.0, 34.0, 35.0, 30.0, 31.0, 32.0, 18.0, 19.0, 20.0, 15.0, 16.0, 17.0, 12.0,
         13.0, 14.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 21.0, 22.0, 23.0, 18.0, 19.0, 20.0
     };
-    float output[1*9*9*3];
-    memset(output, 0, sizeof(output));
+    float *output;
 
     params.mode = LPMP_SYMMETRIC;
     params.paddings[0][0] = 0;
@@ -70,15 +71,26 @@ static int test_with_mode_symmetric(void)
     params.paddings[3][0] = 0;
     params.paddings[3][1] = 0;
 
-    dnn_execute_layer_pad(input, output, &params, 1, 4, 4, 3);
+    operands[0].data = input;
+    operands[0].dims[0] = 1;
+    operands[0].dims[1] = 4;
+    operands[0].dims[2] = 4;
+    operands[0].dims[3] = 3;
+    operands[1].data = NULL;
 
-    for (int i = 0; i < sizeof(output) / sizeof(float); i++) {
+    input_indexes[0] = 0;
+    dnn_execute_layer_pad(operands, input_indexes, 1, &params);
+
+    output = operands[1].data;
+    for (int i = 0; i < sizeof(expected_output) / sizeof(float); i++) {
         if (fabs(output[i] - expected_output[i]) > EPSON) {
             printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]);
+            av_freep(&output);
             return 1;
         }
     }
 
+    av_freep(&output);
     return 0;
 
 }
@@ -102,6 +114,8 @@ static int test_with_mode_reflect(void)
     */
 
     LayerPadParams params;
+    DnnOperand operands[2];
+    int32_t input_indexes[1];
     float input[3*2*2*3] = {
         0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35
     };
@@ -110,8 +124,7 @@ static int test_with_mode_reflect(void)
         12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, 33.0, 34.0,
         35.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0
     };
-    float output[6*2*2*3];
-    memset(output, 0, sizeof(output));
+    float *output;
 
     params.mode = LPMP_REFLECT;
     params.paddings[0][0] = 1;
@@ -123,15 +136,26 @@ static int test_with_mode_reflect(void)
     params.paddings[3][0] = 0;
     params.paddings[3][1] = 0;
 
-    dnn_execute_layer_pad(input, output, &params, 3, 2, 2, 3);
+    operands[0].data = input;
+    operands[0].dims[0] = 3;
+    operands[0].dims[1] = 2;
+    operands[0].dims[2] = 2;
+    operands[0].dims[3] = 3;
+    operands[1].data = NULL;
+
+    input_indexes[0] = 0;
+    dnn_execute_layer_pad(operands, input_indexes, 1, &params);
 
-    for (int i = 0; i < sizeof(output) / sizeof(float); i++) {
+    output = operands[1].data;
+    for (int i = 0; i < sizeof(expected_output) / sizeof(float); i++) {
         if (fabs(output[i] - expected_output[i]) > EPSON) {
             printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]);
+            av_freep(&output);
             return 1;
         }
     }
 
+    av_freep(&output);
     return 0;
 
 }
@@ -155,6 +179,8 @@ static int test_with_mode_constant(void)
     */
 
     LayerPadParams params;
+    DnnOperand operands[2];
+    int32_t input_indexes[1];
     float input[1*2*2*3] = {
         0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
     };
@@ -163,8 +189,7 @@ static int test_with_mode_constant(void)
         728.0, 728.0, 0.0, 1.0, 2.0, 728.0, 728.0, 728.0, 3.0, 4.0, 5.0, 728.0, 728.0,
         728.0, 6.0, 7.0, 8.0, 728.0, 728.0, 728.0, 9.0, 10.0, 11.0, 728.0, 728.0
     };
-    float output[1*3*2*6];
-    memset(output, 0, sizeof(output));
+    float *output;
 
     params.mode = LPMP_CONSTANT;
     params.constant_values = 728;
@@ -177,15 +202,26 @@ static int test_with_mode_constant(void)
     params.paddings[3][0] = 1;
     params.paddings[3][1] = 2;
 
-    dnn_execute_layer_pad(input, output, &params, 1, 2, 2, 3);
+    operands[0].data = input;
+    operands[0].dims[0] = 3;
+    operands[0].dims[1] = 2;
+    operands[0].dims[2] = 2;
+    operands[0].dims[3] = 3;
+    operands[1].data = NULL;
+
+    input_indexes[0] = 0;
+    dnn_execute_layer_pad(operands, input_indexes, 1, &params);
 
-    for (int i = 0; i < sizeof(output) / sizeof(float); i++) {
+    output = operands[1].data;
+    for (int i = 0; i < sizeof(expected_output) / sizeof(float); i++) {
         if (fabs(output[i] - expected_output[i]) > EPSON) {
             printf("at index %d, output: %f, expected_output: %f\n", i, output[i], expected_output[i]);
+            av_freep(&output);
             return 1;
         }
     }
 
+    av_freep(&output);
     return 0;
 
 }