dnn: export operand info in python script and load in c code

Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
Signed-off-by: Pedro Arthur <bygrandao@gmail.com>

4 files changed, 142 insertions, 22 deletions

diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c
index 5d39353..8b05bec 100644
--- a/libavfilter/dnn/dnn_backend_native.c
+++ b/libavfilter/dnn/dnn_backend_native.c
@@ -72,7 +72,6 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
     ConvolutionalParams *conv_params;
     DepthToSpaceParams *depth_to_space_params;
     LayerPadParams *pad_params;
-    int32_t operand_index = 0;
 
     model = av_malloc(sizeof(DNNModel));
     if (!model){
@@ -93,9 +92,10 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
     }
     model->model = (void *)network;
 
-    avio_seek(model_file_context, file_size - 4, SEEK_SET);
+    avio_seek(model_file_context, file_size - 8, SEEK_SET);
     network->layers_num = (int32_t)avio_rl32(model_file_context);
-    dnn_size = 4;
+    network->operands_num = (int32_t)avio_rl32(model_file_context);
+    dnn_size = 8;
     avio_seek(model_file_context, 0, SEEK_SET);
 
     network->layers = av_mallocz(network->layers_num * sizeof(Layer));
@@ -105,11 +105,6 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
         return NULL;
     }
 
-    /**
-     * Operands should be read from model file, the whole change will be huge.
-     * to make things step by step, we first mock the operands, instead of reading from model file.
-     */
-    network->operands_num = network->layers_num + 1;
     network->operands = av_mallocz(network->operands_num * sizeof(DnnOperand));
     if (!network->operands){
         avio_closep(&model_file_context);
@@ -120,8 +115,6 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
     for (layer = 0; layer < network->layers_num; ++layer){
         layer_type = (int32_t)avio_rl32(model_file_context);
         dnn_size += 4;
-        network->layers[layer].input_operand_indexes[0] = operand_index++;
-        network->layers[layer].output_operand_index = operand_index;
         switch (layer_type){
         case CONV:
             conv_params = av_malloc(sizeof(ConvolutionalParams));
@@ -162,6 +155,9 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
             for (i = 0; i < conv_params->output_num; ++i){
                 conv_params->biases[i] = av_int2float(avio_rl32(model_file_context));
             }
+            network->layers[layer].input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
+            network->layers[layer].output_operand_index = (int32_t)avio_rl32(model_file_context);
+            dnn_size += 8;
             network->layers[layer].type = CONV;
             network->layers[layer].params = conv_params;
             break;
@@ -174,6 +170,9 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
             }
             depth_to_space_params->block_size = (int32_t)avio_rl32(model_file_context);
             dnn_size += 4;
+            network->layers[layer].input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
+            network->layers[layer].output_operand_index = (int32_t)avio_rl32(model_file_context);
+            dnn_size += 8;
             network->layers[layer].type = DEPTH_TO_SPACE;
             network->layers[layer].params = depth_to_space_params;
             break;
@@ -191,6 +190,9 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
                 pad_params->paddings[i][1] = avio_rl32(model_file_context);
                 dnn_size += 8;
             }
+            network->layers[layer].input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
+            network->layers[layer].output_operand_index = (int32_t)avio_rl32(model_file_context);
+            dnn_size += 8;
             network->layers[layer].type = MIRROR_PAD;
             network->layers[layer].params = pad_params;
             break;
@@ -201,6 +203,33 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
         }
     }
 
+    for (int32_t i = 0; i < network->operands_num; ++i){
+        DnnOperand *oprd;
+        int32_t name_len;
+        int32_t operand_index = (int32_t)avio_rl32(model_file_context);
+        dnn_size += 4;
+
+        oprd = &network->operands[operand_index];
+        name_len = (int32_t)avio_rl32(model_file_context);
+        dnn_size += 4;
+
+        avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
+        dnn_size += name_len;
+
+        oprd->type = (int32_t)avio_rl32(model_file_context);
+        dnn_size += 4;
+
+        oprd->data_type = (int32_t)avio_rl32(model_file_context);
+        dnn_size += 4;
+
+        for (int32_t dim = 0; dim < 4; ++dim) {
+            oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
+            dnn_size += 4;
+        }
+
+        oprd->isNHWC = 1;
+    }
+
     avio_closep(&model_file_context);
 
     if (dnn_size != file_size){
diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h
index 87b4394..08e7d15 100644
--- a/libavfilter/dnn/dnn_backend_native.h
+++ b/libavfilter/dnn/dnn_backend_native.h
@@ -36,7 +36,7 @@ typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc;
 
 typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam;
 
-typedef enum {DOT_INPUT, DOT_INTERMEDIATE, DOT_OUTPUT} DNNOperandType;
+typedef enum {DOT_INPUT = 1, DOT_OUTPUT = 2, DOT_INTERMEDIATE = DOT_INPUT | DOT_INPUT} DNNOperandType;
 
 typedef struct Layer{
     DNNLayerType type;
diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h
index c24df0e..057005f 100644
--- a/libavfilter/dnn_interface.h
+++ b/libavfilter/dnn_interface.h
@@ -32,7 +32,7 @@ typedef enum {DNN_SUCCESS, DNN_ERROR} DNNReturnType;
 
 typedef enum {DNN_NATIVE, DNN_TF} DNNBackendType;
 
-typedef enum {DNN_FLOAT, DNN_UINT8} DNNDataType;
+typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
 
 typedef struct DNNInputData{
     void *data;
diff --git a/tools/python/convert_from_tensorflow.py b/tools/python/convert_from_tensorflow.py
index cbc76a9..bab11a5 100644
--- a/tools/python/convert_from_tensorflow.py
+++ b/tools/python/convert_from_tensorflow.py
@@ -23,6 +23,37 @@ import sys, struct
 
 __all__ = ['convert_from_tensorflow']
 
+class Operand(object):
+    IOTYPE_INPUT = 1
+    IOTYPE_OUTPUT = 2
+    IOTYPE_INTERMEDIATE = IOTYPE_INPUT | IOTYPE_OUTPUT
+    DTYPE_FLOAT = 1
+    DTYPE_UINT8 = 4
+    index = 0
+    def __init__(self, name, dtype, dims):
+        self.name = name
+        self.dtype = dtype
+        self.dims = dims
+        self.iotype = 0
+        self.used_count = 0
+        self.index = Operand.index
+        Operand.index = Operand.index + 1
+        self.iotype2str = {Operand.IOTYPE_INPUT: 'in', Operand.IOTYPE_OUTPUT: 'out', Operand.IOTYPE_INTERMEDIATE: 'inout'}
+        self.dtype2str = {Operand.DTYPE_FLOAT: 'DT_FLOAT', Operand.DTYPE_UINT8: 'DT_UINT8'}
+
+    def add_iotype(self, iotype):
+        self.iotype = self.iotype | iotype
+        if iotype == Operand.IOTYPE_INPUT:
+            self.used_count = self.used_count + 1
+
+    def __str__(self):
+        return "{}: (name: {}, iotype: {}, dtype: {}, dims: ({},{},{},{}) used_count: {})".format(self.index,
+                            self.name, self.iotype2str[self.iotype], self.dtype2str[self.dtype],
+                            self.dims[0], self.dims[1], self.dims[2], self.dims[3], self.used_count)
+
+    def __lt__(self, other):
+        return self.index < other.index
+
 class TFConverter:
     def __init__(self, graph_def, nodes, outfile, dump4tb):
         self.graph_def = graph_def
@@ -37,8 +68,28 @@ class TFConverter:
         self.conv_paddings = {'VALID':0, 'SAME':1}
         self.converted_nodes = set()
         self.conv2d_scope_names = set()
+        self.conv2d_scopename_inputname_dict = {}
         self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3}
         self.mirrorpad_mode = {'CONSTANT':0, 'REFLECT':1, 'SYMMETRIC':2}
+        self.name_operand_dict = {}
+
+
+    def add_operand(self, name, type):
+        node = self.name_node_dict[name]
+        if name not in self.name_operand_dict:
+            dtype = node.attr['dtype'].type
+            if dtype == 0:
+                dtype = node.attr['T'].type
+            dims = [-1,-1,-1,-1]
+            if 'shape' in node.attr:
+                dims[0] = node.attr['shape'].shape.dim[0].size
+                dims[1] = node.attr['shape'].shape.dim[1].size
+                dims[2] = node.attr['shape'].shape.dim[2].size
+                dims[3] = node.attr['shape'].shape.dim[3].size
+            operand = Operand(name, dtype, dims)
+            self.name_operand_dict[name] = operand;
+        self.name_operand_dict[name].add_iotype(type)
+        return self.name_operand_dict[name].index
 
 
     def dump_for_tensorboard(self):
@@ -60,11 +111,10 @@ class TFConverter:
         # the BiasAdd name is possible be changed into the output name,
         # if activation is None, and BiasAdd.next is the last op which is Identity
         if conv2d_scope_name + '/BiasAdd' in self.edges:
-            activation = self.edges[conv2d_scope_name + '/BiasAdd'][0]
-            activation = activation.op
+            anode = self.edges[conv2d_scope_name + '/BiasAdd'][0]
         else:
-            activation = 'None'
-        return knode, bnode, dnode, activation
+            anode = None
+        return knode, bnode, dnode, anode
 
 
     def dump_conv2d_to_file(self, node, f):
@@ -73,16 +123,21 @@ class TFConverter:
         self.converted_nodes.add(node.name)
 
         scope_name = TFConverter.get_scope_name(node.name)
-        #knode for kernel, bnode for bias, dnode for dilation
-        knode, bnode, dnode, activation = self.get_conv2d_params(scope_name)
+        #knode for kernel, bnode for bias, dnode for dilation, anode for activation
+        knode, bnode, dnode, anode = self.get_conv2d_params(scope_name)
 
         if dnode is not None:
             dilation = struct.unpack('i', dnode.attr['value'].tensor.tensor_content[0:4])[0]
         else:
             dilation = 1
 
+        if anode is not None:
+            activation = anode.op
+        else:
+            activation = 'None'
+
         padding = node.attr['padding'].s.decode("utf-8")
-        # conv2d with dilation > 1 generates tens of nodes, not easy to parse them, so use tricky.
+        # conv2d with dilation > 1 generates tens of nodes, not easy to parse them, so use this tricky method.
         if dilation > 1 and scope_name + '/stack' in self.name_node_dict:
             if self.name_node_dict[scope_name + '/stack'].op == "Const":
                 padding = 'SAME'
@@ -107,6 +162,15 @@ class TFConverter:
             bias = btensor.tensor_content
         f.write(bias)
 
+        input_name = self.conv2d_scopename_inputname_dict[scope_name]
+        input_operand_index = self.add_operand(input_name, Operand.IOTYPE_INPUT)
+
+        if anode is not None:
+            output_operand_index = self.add_operand(anode.name, Operand.IOTYPE_OUTPUT)
+        else:
+            output_operand_index = self.add_operand(self.edges[bnode.name][0].name, Operand.IOTYPE_OUTPUT)
+        np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
+
 
     def dump_depth2space_to_file(self, node, f):
         assert(node.op == 'DepthToSpace')
@@ -114,6 +178,9 @@ class TFConverter:
         block_size = node.attr['block_size'].i
         np.array([self.op2code[node.op], block_size], dtype=np.uint32).tofile(f)
         self.converted_nodes.add(node.name)
+        input_operand_index = self.add_operand(node.input[0], Operand.IOTYPE_INPUT)
+        output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
+        np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
 
 
     def dump_mirrorpad_to_file(self, node, f):
@@ -127,6 +194,9 @@ class TFConverter:
         paddings = pnode.attr['value'].tensor.tensor_content
         f.write(paddings)
         self.converted_nodes.add(node.name)
+        input_operand_index = self.add_operand(node.input[0], Operand.IOTYPE_INPUT)
+        output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
+        np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
 
 
     def dump_layers_to_file(self, f):
@@ -147,10 +217,21 @@ class TFConverter:
                 self.dump_mirrorpad_to_file(node, f)
 
 
+    def dump_operands_to_file(self, f):
+            operands = sorted(self.name_operand_dict.values())
+            for operand in operands:
+                #print('{}'.format(operand))
+                np.array([operand.index, len(operand.name)], dtype=np.uint32).tofile(f)
+                f.write(operand.name.encode('utf-8'))
+                np.array([operand.iotype, operand.dtype], dtype=np.uint32).tofile(f)
+                np.array([operand.dims[0], operand.dims[1], operand.dims[2], operand.dims[3]], dtype=np.uint32).tofile(f)
+
+
     def dump_to_file(self):
         with open(self.outfile, 'wb') as f:
             self.dump_layers_to_file(f)
-            np.array([self.layer_number], dtype=np.uint32).tofile(f)
+            self.dump_operands_to_file(f)
+            np.array([self.layer_number, len(self.name_operand_dict)], dtype=np.uint32).tofile(f)
 
 
     def generate_name_node_dict(self):
@@ -212,19 +293,29 @@ class TFConverter:
         return name[0:index]
 
 
-    def generate_conv2d_scope_names(self):
+    def generate_conv2d_scope_info(self):
+        # conv2d is a sub block in graph, get the scope name
         for node in self.nodes:
             if node.op == 'Conv2D':
                 scope = TFConverter.get_scope_name(node.name)
                 self.conv2d_scope_names.add(scope)
 
+        # get the input name to the conv2d sub block
+        for node in self.nodes:
+            scope = TFConverter.get_scope_name(node.name)
+            if scope in self.conv2d_scope_names:
+                if node.op == 'Conv2D' or node.op == 'Shape':
+                    for inp in node.input:
+                        if TFConverter.get_scope_name(inp) != scope:
+                            self.conv2d_scopename_inputname_dict[scope] = inp
+
 
     def run(self):
         self.generate_name_node_dict()
         self.generate_output_names()
         self.remove_identity()
         self.generate_edges()
-        self.generate_conv2d_scope_names()
+        self.generate_conv2d_scope_info()
 
         if self.dump4tb:
             self.dump_for_tensorboard()