I am working on an object detection API implemented in Tensorflow 1.14 and OpenCV 4.1, where my task is to recognize personal protection equipment (PPE) worn by workers at various construction site gates.
We are using RTSP streams, where I am already using threading to minimize latency, but still some times the stream crashes. So I decided to restart the whole python script every n times of detection to prevent the whole thing crashing because of corrupted frames and so on, but Tensorflow is very slow with the loading of inference graph and such for the first time (for me it's ~20 seconds), which is unacceptable to wait for the workers to get inside the site at the gate. So now I am considering to just stop and then restart JUST the RTSP stream with OpenCV, which constantly feeds the inference machinery with frames for executing object detection on them.
Now I have not found any helpful threads on this topic, so that is why I am writing it here.
My code:
from threading import Thread
import cv2, time
import numpy as np
import os
import six.moves.urllib as urllib
import sys
import tarfile
from threading import Thread
import tensorflow as tf
import tensorflow.contrib.tensorrt as trt
import zipfile
from distutils.version import StrictVersion
from collections import defaultdict
from io import StringIO
from object_detection.utils import ops as utils_ops
import streamlit as st
import imageio
import shutil
from PIL import Image, ImageDraw
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
TF_CUDNN_USE_AUTOTUNE=0
if StrictVersion(tf.__version__) < StrictVersion('1.9.0'):
raise ImportError('Please upgrade your TensorFlow installation to v1.9.* or later!')
from object_detection.utils import label_map_util
from object_detection.utils import visualization_utils as vis_util
MODEL_NAME = './object_detection/inference_Strabag_2020_01_20_100k_mefelelo'
modeln = "inference_Strabag_2020_01_20_100k_mefelelo"
PATH_TO_FROZEN_GRAPH = MODEL_NAME + '/frozen_inference_graph.pb'
PATH_TO_LABELS = './object_detection/training/labelmap.pbtxt'
savedir1 = "/home/dome/web/html/"
#savedir2 ="/var/www/html/"
image_store1 = 0
image_store2 = 0
detection_graph = tf.Graph()
with detection_graph.as_default():
od_graph_def = tf.GraphDef()
with tf.gfile.GFile(PATH_TO_FROZEN_GRAPH, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def, name='')
category_index = label_map_util.create_category_index_from_labelmap(PATH_TO_LABELS, use_display_name=True)
with detection_graph.as_default():
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.35))) as sess:
# Get handles to input and output tensors
ops = tf.get_default_graph().get_operations()
all_tensor_names = {output.name for op in ops for output in op.outputs}
tensor_dict = {}
for key in ['num_detections', 'detection_boxes', 'detection_scores','detection_classes', 'detection_masks']:
tensor_name = key + ':0'
if tensor_name in all_tensor_names:
tensor_dict[key] = tf.get_default_graph().get_tensor_by_name(tensor_name)
global detected_counter1
global detected_counter2
global PASS_score1
global NOPASS_score1
global PASS_score2
global NOPASS_score2
global scoresumCounter1
global scoresumCounter2
global PASS_score_tmp
global NOPASS_score_tmp
global detection_index
global detboxesAreaTMP1
global detboxesAreaTMP2
global imagesgif1
global imagesgif2
resized_width = 426
resized_height = 240
rtsp_Url = 'not public'
frameST = st.empty()
def run_inference_for_single_image(image, graph):
if 'detection_masks' in tensor_dict:
# The following processing is only for single image
detection_boxes = tf.squeeze(tensor_dict['detection_boxes'], [0])
detection_masks = tf.squeeze(tensor_dict['detection_masks'], [0])
# Reframe is required to translate mask from box coordinates to image coordinates and fit the image size.
real_num_detection = tf.cast(tensor_dict['num_detections'][0], tf.int32)
detection_boxes = tf.slice(detection_boxes, [0, 0], [real_num_detection, -1])
detection_masks = tf.slice(detection_masks, [0, 0, 0], [real_num_detection, -1, -1])
detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks(
detection_masks, detection_boxes, image.shape[0], image.shape[1])
detection_masks_reframed = tf.cast(
tf.greater(detection_masks_reframed, 0.5), tf.uint8)
# Follow the convention by adding back the batch dimension
tensor_dict['detection_masks'] = tf.expand_dims(
detection_masks_reframed, 0)
image_tensor = tf.get_default_graph().get_tensor_by_name('image_tensor:0')
# Run inference
output_dict = sess.run(tensor_dict,
feed_dict={image_tensor: np.expand_dims(image, 0)})
# all outputs are float32 numpy arrays, so convert types as appropriate
output_dict['num_detections'] = int(output_dict['num_detections'][0])
output_dict['detection_classes'] = output_dict[
'detection_classes'][0].astype(np.uint8)
output_dict['detection_boxes'] = output_dict['detection_boxes'][0]
output_dict['detection_scores'] = output_dict['detection_scores'][0]
if 'detection_masks' in output_dict:
output_dict['detection_masks'] = output_dict['detection_masks'][0]
return output_dict
detection_graph = tf.Graph()
with detection_graph.as_default():
od_graph_def = tf.GraphDef()
with tf.gfile.GFile(PATH_TO_FROZEN_GRAPH, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def, name='')
category_index = label_map_util.create_category_index_from_labelmap(PATH_TO_LABELS, use_display_name=True)
global_def_tmp = 0
class VideoStreamWidget(object):
detected_counter1 = 0
detected_counter2 = 0
PASS_score1 = 0
NOPASS_score1 = 0
PASS_score2 = 0
NOPASS_score2 = 0
scoresumCounter1=0
scoresumCounter2=0
PASS_score_tmp =0
NOPASS_score_tmp=0
detection_index = 0
detboxesAreaTMP1 = []
detboxesAreaTMP2 = []
image_store1 = []
image_store2 = []
imagesgif1 = []
imagesgif2 = []
def __init__(self, src=rtsp_Url):
# Create a VideoCapture object
#self.pipline_r = 'rtspsrc location='+rtsp_Url+' ! decodebin ! videoconvert ! appsink'
self.capture = cv2.VideoCapture(src)
self.capture.set(cv2.CAP_PROP_BUFFERSIZE, 1)
self.capture.set(cv2.CAP_PROP_FPS, 4)
self.capture.set(cv2.CAP_PROP_FRAME_WIDTH,resized_width)
self.capture.set(cv2.CAP_PROP_FRAME_HEIGHT,resized_height)
self.FPS = 1/4
self.FPS_MS = int(self.FPS * 1000)
# Start the thread to read frames from the video stream
self.thread = Thread(target=self.update, args=())
self.thread.daemon = True
self.thread.start()
self.counter = 0
self.counter2=0
self.status = False
def update(self):
# Read the next frame from the stream in a different thread
while True:
if self.capture.isOpened():
#capture_buffer = np.empty(shape=(resized_width, resized_height, 3), dtype=np.uint8)
(self.status, self.frame) = self.capture.read()
self.frame = cv2.resize(self.frame,(resized_width,resized_height))
#print(self.status)
#time.sleep(self.FPS)
def show_frame(self):
# Display frames in main program
if self.counter == 0 :
#print(self.counter)
global detected_counter1
global detected_counter2
global PASS_score1
global NOPASS_score1
global PASS_score2
global NOPASS_score2
global scoresumCounter1
global scoresumCounter2
global PASS_score_tmp
global NOPASS_score_tmp
global detection_index
global detboxesAreaTMP1
global detboxesAreaTMP2
global out1
global out2
global fourcc
global image_store1
global image_store2
global imagesgif1
global imagesgif2
global end_time1
global start_time1
detected_counter1 = 0
detected_counter2 = 0
PASS_score1 = 0
NOPASS_score1 = 0
PASS_score2 = 0
NOPASS_score2 = 0
scoresumCounter1=0
scoresumCounter2=0
PASS_score_tmp =0
NOPASS_score_tmp=0
detection_index = 0
detboxesAreaTMP1 = []
detboxesAreaTMP2 = []
image_store1 = []
image_store2 = []
imagesgif1 = []
imagesgif2 = []
start_time1 = 0
end_time1 = 0
self.counter = 1
if self.status:
start_time = time.time()
if detected_counter1 == 0:
#out1 = cv2.VideoWriter(savedir1+'cegled-villa1-vidiTMP.mp4',fourcc, 4.0, (resized_width, resized_height))
imagesgif1 = []
if detected_counter2 == 0:
#out2 = cv2.VideoWriter(savedir1+'cegled-villa2-vidiTMP.mp4',fourcc, 4.0, (resized_width, resized_height))
imagesgif2 = []
#frameST.image(self.frame, channels="BGR")
image_np = cv2.resize(self.frame,(resized_width,resized_height))
image_origi = image_np
image_np_expanded = np.expand_dims(image_np, axis=0)
image_origi_expanded = np.expand_dims(image_origi, axis=0)
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')
num_detections = detection_graph.get_tensor_by_name('num_detections:0')
(detection_boxes, detection_scores, detection_classes, num_detections) = sess.run(
[detection_boxes, detection_scores, detection_classes, num_detections],
feed_dict={image_tensor: image_np_expanded})
detection_boxes = np.asarray(detection_boxes)
detection_classes = np.asarray(detection_classes)
detection_scores = np.asarray(detection_scores)
#print(end_time-start_time)
ARO = 0.5
scoreThreshold = 0.8
scoresum1 = 0
scoresum2 = 0
avg_count = 8
thresh = 0.1
detboxesArea1 = [[0 for i in range(8)] for i in range(10)]
detboxesArea2 = [[0 for i in range(8)] for i in range(10)]
if detected_counter1==0:
detboxesAreaTMP1 = [[0 for i in range(7)] for i in range(avg_count+2)]
image_store1 = [0]*(avg_count+2)
if detected_counter2==0:
detboxesAreaTMP2 = [[0 for i in range(7)] for i in range(avg_count+2)]
image_store2 = [0]*(avg_count+2)
for k in range(10):
if detection_scores[0][k] >scoreThreshold and resized_width*detection_boxes[0][k][1]>=resized_width/2.0- resized_width*0.1 :
detboxesArea1[k][0] = (resized_height*(detection_boxes[0][k][2]-detection_boxes[0][k][0])*resized_width*(detection_boxes[0][k][3]-detection_boxes[0][k][1]))
detboxesArea1[k][1] = k
detboxesArea1[k][2] = resized_width*detection_boxes[0][k][1] #xmin
detboxesArea1[k][3] = resized_height*detection_boxes[0][k][0] #ymin
detboxesArea1[k][4] = resized_width*detection_boxes[0][k][3] #xmax
detboxesArea1[k][5] = resized_height*detection_boxes[0][k][2] #ymax
detboxesArea1[k][6] = int(detection_classes[0][k])
detboxesArea1[k][7] = detection_scores[0][k]
if detection_scores[0][k] >scoreThreshold and resized_width*detection_boxes[0][k][3]<resized_width/2.0+resized_width*0.1 :
detboxesArea2[k][0] = (resized_height*(detection_boxes[0][k][2]-detection_boxes[0][k][0])*resized_width*(detection_boxes[0][k][3]-detection_boxes[0][k][1]))
detboxesArea2[k][1] = k
detboxesArea2[k][2] = resized_width*detection_boxes[0][k][1]
detboxesArea2[k][3] = resized_height*detection_boxes[0][k][0]
detboxesArea2[k][4] = resized_width*detection_boxes[0][k][3]
detboxesArea2[k][5] = resized_height*detection_boxes[0][k][2]
detboxesArea2[k][6] = int(detection_classes[0][k])
detboxesArea2[k][7] = detection_scores[0][k]
detboxesArea1 = sorted(detboxesArea1,reverse=True)
detboxesArea2 = sorted(detboxesArea2,reverse=True)
if detboxesArea1[0][3] > 0.05*resized_height and detboxesArea1[0][0] >0 :
detboxesAreaTMP1[detected_counter1][0] = detboxesArea1[0][0]
detboxesAreaTMP1[detected_counter1][1] = detboxesArea1[0][2] #xmin
detboxesAreaTMP1[detected_counter1][2] = detboxesArea1[0][3] #ymin
detboxesAreaTMP1[detected_counter1][3] = detboxesArea1[0][4] #xmax
detboxesAreaTMP1[detected_counter1][4] = detboxesArea1[0][5] #ymax
detboxesAreaTMP1[detected_counter1][5] = detected_counter1
image_store1[detected_counter1] = image_np
detbox1Switch = 1
if detboxesArea2[0][3] > 0.05*resized_height and detboxesArea2[0][0] >0:
detboxesAreaTMP2[detected_counter2][0] = detboxesArea2[0][0]
detboxesAreaTMP2[detected_counter2][1] = detboxesArea2[0][2]
detboxesAreaTMP2[detected_counter2][2] = detboxesArea2[0][3]
detboxesAreaTMP2[detected_counter2][3] = detboxesArea2[0][4]
detboxesAreaTMP2[detected_counter2][4] = detboxesArea2[0][5]
detboxesAreaTMP2[detected_counter2][5] = detected_counter2
image_store2[detected_counter2] = image_origi
detbox2Switch = 1
if detboxesArea1[0][6] == 1 :
PASS_score1 += detboxesArea1[0][7]
scoresum1 += detboxesArea1[0][7]
cv2.rectangle(image_np, (int(detboxesArea1[0][2]),int(detboxesArea1[0][3])),(int(detboxesArea1[0][4]),int(detboxesArea1[0][5])), (37,200,37), 8)
elif detboxesArea1[0][6] == 2:
NOPASS_score1 += detboxesArea1[0][7]
scoresum1 += detboxesArea1[0][7]
cv2.rectangle(image_np, (int(detboxesArea1[0][2]),int(detboxesArea1[0][3])),(int(detboxesArea1[0][4]),int(detboxesArea1[0][5])), (60,29,200), 8)
if detboxesArea2[0][6] == 1 :
PASS_score2 += detboxesArea2[0][7]
scoresum2 += detboxesArea2[0][7]
cv2.rectangle(image_origi, (int(detboxesArea2[0][2]),int(detboxesArea2[0][3])),(int(detboxesArea2[0][4]),int(detboxesArea2[0][5])), (37,200,37), 8)
elif detboxesArea2[0][6] == 2:
NOPASS_score2 += detboxesArea2[0][7]
scoresum2 += detboxesArea2[0][7]
cv2.rectangle(image_origi, (int(detboxesArea2[0][2]),int(detboxesArea2[0][3])),(int(detboxesArea2[0][4]),int(detboxesArea2[0][5])), (60,29,200), 8)
# Expand dimensions since the model expects images to have shape: [1, None, None, 3]
image_np_expanded = np.expand_dims(image_np, axis=0)
image_origi_expanded = np.expand_dims(image_origi, axis=0)
detected_counter1+=1
detected_counter2+=1
# Visualization of the results of a detection.
'''
vis_util.visualize_boxes_and_labels_on_image_array(
image_np,
np.squeeze(detection_boxes[0]),
np.squeeze(detection_classes[0]).astype(np.int32),
np.squeeze(detection_scores[0]),
category_index,
instance_masks=output_dict.get('detection_masks'),
use_normalized_coordinates=True,
line_thickness=8)
'''
#cv2.namedWindow('object_detection')
#cv2.imshow('object_detection',image_np)
if scoresum1 == 0:
scoresumCounter1+=1
if scoresum2 == 0:
scoresumCounter2+=1
if detected_counter1==avg_count and (avg_count-scoresumCounter1)>=int(avg_count/4.0):
timeframe = time.time()
detboxesAreaTMP1 = sorted(detboxesAreaTMP1,reverse=True)
if (PASS_score1)/(avg_count-scoresumCounter1) >= (NOPASS_score1)/(avg_count-scoresumCounter1):
cv2.rectangle(image_store1[detboxesAreaTMP1[0][5]], (int(detboxesAreaTMP1[0][1]),int(detboxesAreaTMP1[0][2])),(int(detboxesAreaTMP1[0][3]),int(detboxesAreaTMP1[0][4])), (37,152,37), 8)
cv2.rectangle(image_store1[detboxesAreaTMP1[0][5]], (int(resized_width/2.0)+5,5),(resized_width-5,resized_height-5), (0,255,0), 8)
start_time1 = time.time()
cv2.imwrite(savedir1+"cegled-villa1.jpg", image_store1[detboxesAreaTMP1[0][5]])
os.system('php-cgi -f /home/dome/web/html/_iras.php passed=1 device_id=cegled-villa1')
else:
cv2.rectangle(image_store1[detboxesAreaTMP1[0][5]], (int(detboxesAreaTMP1[0][1]),int(detboxesAreaTMP1[0][2])),(int(detboxesAreaTMP1[0][3]),int(detboxesAreaTMP1[0][4])), (60,29,182), 8)
cv2.rectangle(image_store1[detboxesAreaTMP1[0][5]], (int(resized_width/2.0)+5,5),(resized_width-5,resized_height-5), (0,0,255), 8)
cv2.imwrite(savedir1+"cegled-villa1.jpg", image_store1[detboxesAreaTMP1[0][5]])
os.system('php-cgi -f /home/dome/web/html/_iras.php passed=0 device_id=cegled-villa1')
PASS_score1 = 0
NOPASS_score1 = 0
detboxesAreaTMP1 = []
detected_counter1 = 0
scoresumCounter1 = 0
if detected_counter2==avg_count and (avg_count-scoresumCounter2)>=int(avg_count/4.0):
timeframe = time.time()
detboxesAreaTMP2 = sorted(detboxesAreaTMP2,reverse=True)
if (PASS_score2)/(avg_count-scoresumCounter2) >= (NOPASS_score2)/(avg_count-scoresumCounter2):
cv2.rectangle(image_store2[detboxesAreaTMP2[0][5]], (int(detboxesAreaTMP2[0][1]),int(detboxesAreaTMP2[0][2])),(int(detboxesAreaTMP2[0][3]),int(detboxesAreaTMP2[0][4])), (0,50,0), 8)
cv2.rectangle(image_store2[detboxesAreaTMP2[0][5]], (5,5),(int(resized_width/2.0)-5,resized_height-5), (0,255,0), 8)
cv2.imwrite(savedir1+"cegled-villa2.jpg", image_store2[detboxesAreaTMP2[0][5]])
os.system('php-cgi -f /home/dome/web/html/_iras.php passed=1 device_id=cegled-villa2')
else:
cv2.rectangle(image_store2[detboxesAreaTMP2[0][5]], (int(detboxesAreaTMP2[0][1]),int(detboxesAreaTMP2[0][2])),(int(detboxesAreaTMP2[0][3]),int(detboxesAreaTMP2[0][4])), (0,0,50), 8)
cv2.rectangle(image_store2[detboxesAreaTMP2[0][5]], (5,5),(int(resized_width/2.0)-5,resized_height-5), (0,0,255), 8)
cv2.imwrite(savedir1+"cegled-villa2.jpg", image_store2[detboxesAreaTMP2[0][5]])
os.system('php-cgi -f /home/dome/web/html/_iras.php passed=0 device_id=cegled-villa2')
detected_counter2=0
scoresumCounter2 = 0
PASS_score2 = 0
NOPASS_score2 = 0
detboxesAreaTMP2 = []
if (detected_counter1==avg_count and (avg_count-scoresumCounter1)<int(avg_count/4.0)) and (avg_count-scoresumCounter1)>0 :
PASS_score1 = 0
NOPASS_score1 = 0
detected_counter1=0
scoresumCounter1 = 0
detboxesAreaTMP1 = []
try:
cv2.rectangle(image_store1[detboxesAreaTMP1[0][5]], (int(detboxesAreaTMP1[0][1]),int(detboxesAreaTMP1[0][2])),(int(detboxesAreaTMP1[0][3]),int(detboxesAreaTMP1[0][4])), (60,29,182), 8)
cv2.rectangle(image_store1[detboxesAreaTMP1[0][5]], (int(resized_width/2.0)+5,5),(resized_width-5,resized_height-5), (0,0,255), 8)
cv2.imwrite(savedir1+"cegled-villa1.jpg", image_store1[detboxesAreaTMP1[0][5]])
except IndexError:
cv2.imwrite(savedir1+"cegled-villa1.jpg", image_np)
os.system('php-cgi -f /home/dome/web/html/_iras.php passed=0 device_id=cegled-villa1')
if (detected_counter2==avg_count and (avg_count-scoresumCounter2)<int(avg_count/4.0)) and (avg_count-scoresumCounter2)>0:
PASS_score2 = 0
NOPASS_score2 = 0
detected_counter2=0
scoresumCounter2 = 0
detboxesAreaTMP2 = []
try:
cv2.rectangle(image_store2[detboxesAreaTMP2[0][5]], (int(detboxesAreaTMP2[0][1]),int(detboxesAreaTMP2[0][2])),(int(detboxesAreaTMP2[0][3]),int(detboxesAreaTMP2[0][4])), (0,0,50), 8)
cv2.rectangle(image_store2[detboxesAreaTMP2[0][5]], (5,5),(int(resized_width/2.0)-5,resized_height-5), (0,0,255), 8)
cv2.imwrite(savedir1+"cegled-villa2.jpg", image_store2[detboxesAreaTMP2[0][5]])
except IndexError:
cv2.imwrite(savedir1+"cegled-villa2.jpg", image_np)
os.system('php-cgi -f /home/dome/web/html/_iras.php passed=0 device_id=cegled-villa2')
if detected_counter1 == avg_count and (avg_count-scoresumCounter1)==0:
detected_counter1 = 0
PASS_score1 = 0
NOPASS_score1 = 0
detected_counter1=0
scoresumCounter1 = 0
detboxesAreaTMP1 = []
if detected_counter2 == avg_count and (avg_count-scoresumCounter2)==0:
detected_counter2 = 0
PASS_score2 = 0
NOPASS_score2 = 0
detected_counter2=0
scoresumCounter2 = 0
detboxesAreaTMP2 = []
#cv2.namedWindow('object_detection')
end_time = time.time()
del image_origi
del image_np
# Press Q on keyboard to stop recording
key = cv2.waitKey(2)
if key == ord('q'):
self.capture.release()
cv2.destroyAllWindows()
exit(1)
if __name__ == '__main__':
stream_link = 'your stream link!'
video_stream_widget = VideoStreamWidget(rtsp_Url)
with detection_graph.as_default():
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.6))) as sess:
# Get handles to input and output tensors
ops = tf.get_default_graph().get_operations()
all_tensor_names = {output.name for op in ops for output in op.outputs}
tensor_dict = {}
for key in ['num_detections', 'detection_boxes', 'detection_scores','detection_classes', 'detection_masks']:
tensor_name = key + ':0'
if tensor_name in all_tensor_names:
tensor_dict[key] = tf.get_default_graph().get_tensor_by_name(tensor_name)
resfresherCounter = 0
imagesgif1 = []
imagesgif2 = []
while True:
resfresherCounter+=1
try:
video_stream_widget.show_frame()
except cv2.error as e:
print(e)
detected_counter1 = 0
detected_counter2 = 0
continue
