Merge pull request #39 from tobybreckon/tflite

pull tflite conversion and validation
tobybreckon · Dec 16, 2019 · dd00971 · dd00971
2 parents ef2e5db + d9568b9
commit dd00971
Show file tree

Hide file tree

Showing 9 changed files with 471 additions and 23 deletions.
diff --git a/README.md b/README.md
@@ -78,24 +78,34 @@ $ python superpixel-inceptionV1OnFire.py models/test.mp4
 
 ## Instructions to use pre-trained models with other frameworks:
 
-To convert the supplied pre-trained models from TFLearn checkpoint format to protocol buffer (.pb) format (used by [OpenCV](http://www.opencv.org) DNN, [TensorFlow](https://www.tensorflow.org/), ...) do:
+To convert the supplied pre-trained models from TFLearn checkpoint format to protocol buffer (.pb) format (used by [OpenCV](http://www.opencv.org) DNN, [TensorFlow](https://www.tensorflow.org/), ...) and
+also tflite (used with [TensorFlow](https://www.tensorflow.org/)) do:
 
 
 ```
 $ cd converter
-$ python firenet-to-protobuf.py
-$ python inceptionV1OnFire-to-protobuf.py
+$ python firenet-conversion.py
+$ python inceptionV1OnFire-conversion.py
 ```
 
-This creates three ```.pb``` files inside the ```converter``` directory (```firenet.pb``` / ```inceptionv1onfire.pb```/```sp-inceptionv1onfire.pb```) which can then be tested with the  [OpenCV](http://www.opencv.org) DNN module (for example, using OpenCV > 4.1.0-pre) from within the same directory:
+This creates a set of six ```.pb``` and ```.tflite``` files inside the ```converter``` directory (```firenet.xxx``` / ```inceptionv1onfire.xxx```/```sp-inceptionv1onfire.xxx``` for ```xxx``` in ```[pb, tflite]```). These files can then be validated  with the [OpenCV](http://www.opencv.org) DNN module (OpenCV > 4.1.0-pre) and [TensorFlow](https://www.tensorflow.org/) against the original (tflearn) from within the same directory, as follows:
 
 ```
-$ python test-pb-opencv.py
+$ python firenet-validation.py
+Load tflearn model from: ../models/FireNet ...OK
+Load protocolbuf (pb) model from: firenet.pb ...OK
+Load tflite model from: firenet.tflite ...OK
+Load test video from ../models/test.mp4 ...
+frame: 0        : TFLearn (original): [[9.999914e-01 8.576833e-06]]     : Tensorflow .pb (via opencv): [[9.999914e-01 8.576866e-06]]    : TFLite (via tensorflow): [[9.999914e-01 8.576899e-06]]: all equal test - PASS
+frame: 1        : TFLearn (original): [[9.999924e-01 7.609045e-06]]     : Tensorflow .pb (via opencv): [[9.999924e-01 7.608987e-06]]    : TFLite (via tensorflow): [[9.999924e-01 7.608980e-06]]: all equal test - PASS
+frame: 2        : TFLearn (original): [[9.999967e-01 3.373572e-06]]     : Tensorflow .pb (via opencv): [[9.999967e-01 3.373559e-06]]    : TFLite (via tensorflow): [[9.999967e-01 3.373456e-06]]: all equal test - PASS
+frame: 3        : TFLearn (original): [[9.999968e-01 3.165212e-06]]     : Tensorflow .pb (via opencv): [[9.999968e-01 3.165221e-06]]    : TFLite (via tensorflow): [[9.999968e-01 3.165176e-06]]: all equal test - PASS
+...
 ```
 
-(N.B. for the superpixel network, the test script just checks loading and inference with the ```.pb``` loaded model but does not supply an actual superpixel image - just any test image, hence inference fails to detect the fire correctly for the example only).
+This can be similarly repeated with the ```inceptionV1OnFire-validation.py``` and ```sp-inceptionV1OnFire-validation.py``` validation scripts (N.B. here the superpixel inceptionV1OnFire network is being validated against the whole image frame rather than superpixels just for simply showing consistent output between the original and converted models).
 
-**To convert to to other frameworks** (such as PyTorch, MXNet, Keras, ...) from this tensorflow format (protocol buffer, .pb): - please see the extensive deep neural network model conversion tools offered by the [MMdnn](https://github.com/Microsoft/MMdnn) project.
+**To convert to to other frameworks** (such as PyTorch, MXNet, Keras, ...) from these tensorflow formats: - please see the extensive deep neural network model conversion tools offered by the [MMdnn](https://github.com/Microsoft/MMdnn) project.
 
 ---
 

diff --git a/converter/converter.py b/converter/converter.py
@@ -1,7 +1,7 @@
 ################################################################################
 
 # Example : perform conversion from tflearn checkpoint format to TensorFlow
-# protocol buffer (.pb) binary format files (for import into other tools)
+# protocol buffer (.pb) binary format and also .tflite files (for import into other tools)
 
 # Copyright (c) 2019 Toby Breckon, Durham University, UK
 
@@ -19,9 +19,10 @@
 
 import tensorflow as tf
 from tensorflow.python.framework import graph_util
-from tensorflow.python.framework.graph_util import convert_variables_to_constants
+from tensorflow.compat.v1.graph_util import convert_variables_to_constants
 from tensorflow.tools.graph_transforms import TransformGraph
 from tensorflow.python.tools import optimize_for_inference_lib
+from tensorflow.compat.v1.lite import TFLiteConverter
 
 ################################################################################
 
@@ -36,14 +37,14 @@
 ################################################################################
 # convert a loaded model definition by loading a checkpoint from a given path
 # retaining the network between the specified input and output layers
-# outputs to pbfilename as a binary .pb protocol buffer format files
+# outputs to pbfilename as a binary .pb protocol buffer format file
 
 # e.g. for FireNet
 #    model = construct_firenet (224, 224, False)
 #    path = "models/FireNet/firenet"; # path to tflearn checkpoint including filestem
 #    input_layer_name = 'InputData/X'                  # input layer of network
 #    output_layer_name= 'FullyConnected_2/Softmax'     # output layer of network
-#    pbfilename = "firenet.pb"        # output pb format filename
+#    filename = "firenet.pb"                              # output filename
 
 def convert_to_pb(model, path, input_layer_name,  output_layer_name, pbfilename, verbose=False):
 
@@ -95,3 +96,21 @@ def convert_to_pb(model, path, input_layer_name,  output_layer_name, pbfilename,
   os.remove('checkpoint')
 
 ################################################################################
+# convert a  binary .pb protocol buffer format model to tflite format
+
+# e.g. for FireNet
+#    pbfilename = "firenet.pb"
+#    input_layer_name = 'InputData/X'                  # input layer of network
+#    output_layer_name= 'FullyConnected_2/Softmax'     # output layer of network
+
+def convert_to_tflite(pbfilename, input_layer_name,  output_layer_name):
+
+  input_tensor={input_layer_name:[1,224,224,3]}
+
+  print("[INFO] tflite model to " +  pbfilename.replace(".pb",".tflite") + " ...")
+
+  converter = tf.lite.TFLiteConverter.from_frozen_graph(pbfilename, [input_layer_name], [output_layer_name], input_tensor)
+  tflite_model = converter.convert()
+  open(pbfilename.replace(".pb",".tflite"), "wb").write(tflite_model)
+
+################################################################################
diff --git a/converter/firenet-to-protobuf.py → converter/firenet-conversion.py b/converter/firenet-to-protobuf.py → converter/firenet-conversion.py
@@ -1,7 +1,7 @@
 ################################################################################
 
 # Example : perform conversion of FireNet tflearn model to TensorFlow protocol
-# buffer (.pb) binary format files (for import into other tools, example OpenCV)
+# buffer (.pb) binary format and tflife format files (for import into other tools, example OpenCV)
 
 # Copyright (c) 2019 Toby Breckon, Durham University, UK
 
@@ -19,6 +19,7 @@
 
 from firenet import construct_firenet
 from converter import convert_to_pb
+from converter import convert_to_tflite
 
 ################################################################################
 
@@ -32,8 +33,9 @@
     path = "../models/FireNet/firenet"; # path to tflearn checkpoint including filestem
     input_layer_name = 'InputData/X'                  # input layer of network
     output_layer_name= 'FullyConnected_2/Softmax'     # output layer of network
-    pbfilename = "firenet.pb"        # output pb format filename
+    filename = "firenet.pb"                           # output pb format filename
 
-    convert_to_pb(model, path, input_layer_name,  output_layer_name, pbfilename)
+    convert_to_pb(model, path, input_layer_name,  output_layer_name, filename)
+    convert_to_tflite(filename, input_layer_name,  output_layer_name)
 
 ################################################################################
diff --git a/converter/firenet-validation.py b/converter/firenet-validation.py
@@ -0,0 +1,136 @@
+################################################################################
+
+# Example : perform validation of FireNet models in TFLearn, PB and TFLite formats
+
+# Copyright (c) 2019 - Toby Breckon, Durham University, UK
+
+# License : https://github.com/tobybreckon/fire-detection-cnn/blob/master/LICENSE
+
+################################################################################
+
+import cv2
+import os
+import sys
+import math
+
+################################################################################
+
+import tflearn
+from tflearn.layers.core import *
+from tflearn.layers.conv import *
+from tflearn.layers.normalization import *
+from tflearn.layers.estimator import regression
+
+################################################################################
+
+VALIDATE_TO_PRECISION_N = 5
+
+################################################################################
+
+sys.path.append('..')
+from firenet import construct_firenet
+
+################################################################################
+
+# tflearn - load model
+
+print("Load tflearn model from: ../models/FireNet ...", end = '')
+model_tflearn = construct_firenet (224, 224, training=False)
+model_tflearn.load(os.path.join("../models/FireNet", "firenet"),weights_only=True)
+print("OK")
+
+################################################################################
+
+# tf protocol buffer - load model (into opencv)
+
+print("Load protocolbuf (pb) model from: firenet.pb ...", end = '')
+tensorflow_pb_model = cv2.dnn.readNetFromTensorflow('firenet.pb')
+print("OK")
+
+################################################################################
+
+# tflite - load model
+
+print("Load tflite model from: firenet.tflite ...", end = '')
+tflife_model = tf.lite.Interpreter(model_path="firenet.tflite")
+tflife_model.allocate_tensors()
+print("OK")
+
+# Get input and output tensors.
+tflife_input_details = tflife_model.get_input_details()
+tflife_output_details = tflife_model.get_output_details()
+
+################################################################################
+
+# load video file
+
+video = cv2.VideoCapture("../models/test.mp4")
+print("Load test video from ../models/test.mp4 ...")
+
+# get video properties
+
+width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH))
+height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
+
+frame_counter = 0
+fail_counter = 0
+
+while (True):
+
+    # get video frame from file, handle end of file
+
+    ret, frame = video.read()
+    if not ret:
+        print("... end of video file reached")
+        break
+
+    print("frame: " + str(frame_counter),  end = '')
+    frame_counter = frame_counter + 1
+
+    # re-size image to network input size and perform prediction
+
+    # input to networks is: 224x224x3 colour image with channel ordering as {B,G,R}
+    # as is the opencv norm, not {R,G,B} and pixel value range 0->255 for each channel
+
+    small_frame = cv2.resize(frame, (224, 224), cv2.INTER_AREA)
+
+    ############################################################################
+
+    np.set_printoptions(precision=6)
+
+    # perform predictiion with tflearn model
+
+    output_tflearn = model_tflearn.predict([small_frame])
+    print("\t: TFLearn (original): ", end = '')
+    print(output_tflearn, end = '')
+
+    # perform prediction with protocolbuf model via opencv
+
+    tensorflow_pb_model.setInput(cv2.dnn.blobFromImage(small_frame, size=(224, 224), swapRB=False, crop=False))
+    output_tensorflow_pb = tensorflow_pb_model.forward()
+
+    print("\t: Tensorflow .pb (via opencv): ", end = '')
+    print(output_tensorflow_pb, end = '')
+
+    # perform prediction with tflite model via TensorFlow
+
+    tflife_input_data = np.reshape(np.float32(small_frame), (1, 224, 224, 3))
+    tflife_model.set_tensor(tflife_input_details[0]['index'], tflife_input_data)
+
+    tflife_model.invoke()
+
+    output_tflite = tflife_model.get_tensor(tflife_output_details[0]['index'])
+    print("\t: TFLite (via tensorflow): ", end = '')
+    print(output_tflite, end = '')
+
+    try:
+        np.testing.assert_almost_equal(output_tflearn, output_tensorflow_pb, VALIDATE_TO_PRECISION_N)
+        np.testing.assert_almost_equal(output_tflearn, output_tflite, 3)
+        print(": all equal test - PASS")
+    except AssertionError:
+        print(" all equal test - FAIL")
+        fail_counter = fail_counter +1
+
+################################################################################
+print("*** FINAL cross-model validation FAILS (for precision of " + str(VALIDATE_TO_PRECISION_N) + ") = " + str(fail_counter))
+################################################################################
diff --git a/converter/inceptionV1OnFire-to-protobuf.py → converter/inceptionV1OnFire-conversion.py b/converter/inceptionV1OnFire-to-protobuf.py → converter/inceptionV1OnFire-conversion.py
@@ -1,7 +1,7 @@
 ################################################################################
 
 # Example : perform conversion of inceptionV1OnFire tflearn model to TensorFlow protocol
-# buffer (.pb) format files (for import into other tools, example OpenCV DNN)
+# buffer (.pb) binary format and tflife format files (for import into other tools, example OpenCV)
 
 # Copyright (c) 2019 Toby Breckon, Durham University, UK
 
@@ -20,6 +20,7 @@
 
 from inceptionV1OnFire import construct_inceptionv1onfire
 from converter import convert_to_pb
+from converter import convert_to_tflite
 
 ################################################################################
 
@@ -28,21 +29,25 @@
     # construct and re-export model (so that is excludes the training layers)
 
     model = construct_inceptionv1onfire (224, 224, False)
-    print("[INFO] Constructed InceptionV1-OnFire ...")
+    print("[INFO] Constructed InceptionV1-OnFire (binary, full-frame)...")
 
     path = "../models/InceptionV1-OnFire/inceptiononv1onfire"; # path to tflearn checkpoint including filestem
     input_layer_name = 'InputData/X'            # input layer of network
     output_layer_name= 'FullyConnected/Softmax' # output layer of network
-    pbfilename = "inceptionv1onfire.pb"         # output pb format filename
+    filename = "inceptionv1onfire.pb"           # output pb format filename
 
-    convert_to_pb(model, path, input_layer_name,  output_layer_name, pbfilename)
+    convert_to_pb(model, path, input_layer_name,  output_layer_name, filename)
+    convert_to_tflite(filename, input_layer_name,  output_layer_name)
 
     tf.reset_default_graph()
 
+    print("[INFO] Constructed InceptionV1-OnFire (superpixel)...")
+
     model_sp = construct_inceptionv1onfire (224, 224, False)
     path_sp = "../models/SP-InceptionV1-OnFire/sp-inceptiononv1onfire"; # path to tflearn checkpoint including filestem
-    pbfilename_sp = "sp-inceptionv1onfire.pb"         # output pb format filename
+    filename_sp = "sp-inceptionv1onfire.pb"         # output filename
 
-    convert_to_pb(model_sp, path_sp, input_layer_name,  output_layer_name, pbfilename_sp)
+    convert_to_pb(model_sp, path_sp, input_layer_name,  output_layer_name, filename_sp)
+    convert_to_tflite(filename_sp, input_layer_name,  output_layer_name)
 
 ################################################################################