#!/usr/bin/env python3 """ Copyright 2022-2023 NXP SPDX-License-Identifier: Apache-2.0 This application allows ML-resource-constraint MPU systems (clients) to connect and run inferencing on an ML Gateway system (server) that has very high-performance ML capabilities. """ from threading import Thread import logging import os import socket import sys import tflite_runtime.interpreter as tflite import numpy as np import gi import subprocess import glob from gi.repository import Gtk, Gst, GObject, Gio, GLib sys.path.append("/home/root/.nxp-demo-experience/scripts/") import utils gi.require_version("Gtk", "3.0") gi.require_version("Gst", "1.0") def initialize(): """Initial package installation""" dwnwin = DownloadGUI() GLib.idle_add(dwnwin.show_all) """ GLib.idle_add(dwnwin.status_label.set_text, "\n\nTesting internet...") res = subprocess.getstatusoutput( "ping -c 1 8.8.8.8" )[0] if res != 0: GLib.idle_add( dwnwin.status_label.set_text, "\n\nInternet connection required!") return """ if FIRST_RUN: GLib.idle_add(dwnwin.status_label.set_text, "\n\nInstalling packages...") res = subprocess.getstatusoutput("pip3 --retries 0 install ssdpy")[0] if res != 0: GLib.idle_add(dwnwin.status_label.set_text, "\n\nPackage install failed!") return win.connect("destroy", Gtk.main_quit) GLib.idle_add(win.show_all) GLib.idle_add(dwnwin.destroy) FIRST_RUN = False BOARD = None try: from ssdpy import SSDPServer except ModuleNotFoundError: FIRST_RUN = True MAX_TRY = 4 SSDP_ADDRESS = "239.255.255.250" SSDP_PORT = 1900 SSDP_MX = 2 SSDP_ST = "imx-ml-server" MODEL = "mobilenet_ssd_v2_coco_quant_postprocess.tflite" DATA_SET = "coco_labels_nonum.txt" ML_MODEL = utils.download_file(MODEL) def access_ip(): """Returns IP address if found in network. SSDP is used to discover devices in local network using multicast SSDP address. SSDP uses NOTIFY to announce establishment information and M-Search to discover devices in network. MX : Maximum wait time(in seconds) ST : Search Target """ ssdp_request = ( "M-SEARCH * HTTP/1.1\r\n" + "HOST: {host}:{port}\r\n" + 'MAN: "ssdp:discover"\r\n' + "MX: {mx}\r\n" + "ST: {st}\r\n" + "\r\n" ) ssdp_request = ssdp_request.format( host=SSDP_ADDRESS, port=SSDP_PORT, st=SSDP_ST, mx=SSDP_MX ) # send an m-search request and collect responses sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.sendto(ssdp_request.encode("utf-8"), (SSDP_ADDRESS, SSDP_PORT)) sock.settimeout(1) global ssdp_address try: ssdp_response, ssdp_address = sock.recvfrom(150) except socket.timeout as timeout_error: erro = timeout_error.args[0] # timeout exception is setup if erro == "timed out": print("Receiver timed out, retry later") return False else: # got a response return ssdp_address def get_my_ip(): """Obtaining its own IP address""" sock_obj = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock_obj.connect(("8.8.8.8", 80)) self_ip = sock_obj.getsockname()[0] loopback = socket.gethostbyname(socket.gethostname()) if self_ip == loopback: print("ERROR: Connected to loopback device, try again") sys.exit(1) else: return self_ip def cast_ip(): """Handle incoming m-search request and send a response""" server = SSDPServer("imx-server", device_type=SSDP_ST) server.serve_forever() class DownloadGUI(Gtk.Window): """The main voice GUI application.""" def __init__(self): """Creates the loading window and then shows it""" super().__init__() self.set_default_size(450, 100) self.set_resizable(False) self.set_border_width(10) header = Gtk.HeaderBar() header.set_title("ML Gateway") header.set_subtitle("NNStreamer Demo") self.set_titlebar(header) quit_button = Gtk.Button() quit_icon = Gio.ThemedIcon(name="process-stop-symbolic") quit_image = Gtk.Image.new_from_gicon(quit_icon, Gtk.IconSize.BUTTON) quit_button.add(quit_image) header.pack_end(quit_button) quit_button.connect("clicked", Gtk.main_quit) self.main_grid = Gtk.Grid( row_homogeneous=False, column_homogeneous=True, column_spacing=15, row_spacing=15, ) self.main_grid.set_margin_end(10) self.main_grid.set_margin_start(10) self.status_label = Gtk.Label.new("\n\nSetting up...") self.main_grid.attach(self.status_label, 0, 0, 1, 1) self.add(self.main_grid) class ServerWindow(Gtk.Window): """Server Window""" def __init__(self): super().__init__() self.set_default_size(300, 200) self.set_resizable(False) self.set_border_width(15) self.set_position(Gtk.WindowPosition.CENTER) self.grid = Gtk.Grid( row_homogeneous=True, column_homogeneous=True, row_spacing=10, column_spacing=10, ) # obtain Server's IP address self.label1 = Gtk.Label() self.label1.set_markup(" Server's IP: ") self.label2 = Gtk.Label() self.ip_address = get_my_ip() self.label2.set_text(self.ip_address) self.header = Gtk.HeaderBar() self.header.set_show_close_button(False) self.header.props.title = "ML Server Setup" self.set_titlebar(self.header) self.quit_button = Gtk.Button() quit_icon = Gio.ThemedIcon(name="process-stop-symbolic") quit_image = Gtk.Image.new_from_gicon(quit_icon, Gtk.IconSize.BUTTON) self.quit_button.add(quit_image) self.header.pack_end(self.quit_button) self.quit_button.connect("clicked", Gtk.main_quit) cast_thread = Thread(target=cast_ip) cast_thread.daemon = True cast_thread.start() processors = ["NPU", "CPU"] self.processor_select = Gtk.ComboBoxText() self.processor_select.set_entry_text_column(0) self.processor_select.connect("changed", self.on_changed) for processor in processors: self.processor_select.append_text(processor) self.processor_select.set_active(0) self.label3 = Gtk.Label() self.label3.set_markup("Select NPU or CPU: ") self.button1 = Gtk.Button(label="Start Server") button1_label = self.button1.get_child() button1_label.set_markup("Start Server") self.button1.connect("clicked", self.start_server_thread) self.button1.set_focus_on_click(True) self.grid.attach(self.label1, 0, 0, 1, 1) self.grid.attach(self.label2, 1, 0, 1, 1) self.grid.attach(self.label3, 0, 2, 1, 1) self.grid.attach(self.processor_select, 1, 2, 1, 1) self.grid.attach(self.button1, 0, 6, 2, 1) self.add(self.grid) def check_file(folder_path): file = [f for f in os.listdir(folder_path) if f.endswith(".nb")] return True if len(file) else False if check_file("/home/root/.cache/demoexperience") is False: self.to_reduce_warmup() def to_reduce_warmup(self): """To reduce NPU warmup time""" ext_delegate = tflite.load_delegate("/usr/lib/libvx_delegate.so") # Load the TFLite model and allocate tensors. interpreter = tflite.Interpreter( model_path=ML_MODEL, num_threads=4, experimental_delegates=[ext_delegate] ) interpreter.allocate_tensors() # Get input and output tensors. input_details = interpreter.get_input_details() output_details = interpreter.get_output_details() # Test the model on random input data. input_shape = input_details[0]["shape"] input_data = np.array(np.random.random_sample(input_shape), dtype=np.uint8) interpreter.set_tensor(input_details[0]["index"], input_data) interpreter.invoke() # The function get_tensor() returns a copy of the tensor data. # Use tensor() in order to get a pointer to the tensor. interpreter.get_tensor(output_details[0]["index"]) def on_changed(self, widget): """Allows user to change processors""" pro = widget.get_active_text() if pro == "NPU": self.custom = "Delegate:External,ExtDelegateLib:libvx_delegate.so" elif pro == "CPU": self.custom = "NumThreads:4" def start_server_thread(self, widget): """Server Thread""" server_thread = Thread(target=ServerWindow.start_server, args=[server_window]) server_thread.daemon = True server_thread.start() def start_server(self): """Pipeline to start server""" self.button1.set_sensitive(False) self.processor_select.set_sensitive(False) # initializes state in background Gst.init(None) # mainloop allows to parse events and run operations in background self.main_loop = GLib.MainLoop() server_pipeline = "tensor_query_serversrc host={ip} ! video/x-raw,format=RGB" server_pipeline += ",framerate=0/1 ! tensor_converter ! " server_pipeline += ( "tensor_filter framework=tensorflow-lite model={model} custom={custom} " ) server_pipeline += "! tensor_query_serversink" server_pipeline = server_pipeline.format( ip=self.ip_address, model=ML_MODEL, custom=self.custom ) # creating the pipeline and launching it self.pipeline = Gst.parse_launch(server_pipeline) # message callback bus = self.pipeline.get_bus() bus.add_signal_watch() bus.connect("message", self.on_message) # by default pipelines are in NULL state, pipeline suppose to be set in running state monitor_status = self.pipeline.set_state(Gst.State.PLAYING) if monitor_status == Gst.StateChangeReturn.FAILURE: print("ERROR: Unable to set the pipeline to the playing state") sys.exit(1) self.button1.set_label(" Server is running ") button1_label = self.button1.get_child() button1_label.set_markup(" Server is running ") self.main_loop.run() # disconnecting the pipeline self.pipeline.set_state(Gst.State.NULL) bus.remove_signal_watch() def on_message(self, bus, message): """Callback for message. bus: pipeline bus message: message from pipeline """ mtype = message.type if mtype == Gst.MessageType.EOS: # Handle End of Stream print("End of stream") self.main_loop.quit() elif mtype == Gst.MessageType.ERROR: # Handle Errors err, debug = message.parse_error() print(err, debug) self.main_loop.quit() elif mtype == Gst.MessageType.WARNING: # Handle warnings err, debug = message.parse_warning() print(err, debug) elif mtype == Gst.MessageType.STREAM_START: logging.info("received start message") elif mtype == Gst.MessageType.QOS: data_format, processed, dropped = message.parse_qos_stats() format_str = Gst.Format.get_name(data_format) logging.debug( "[qos] format[%s] processed[%d] dropped[%d]", format_str, processed, dropped, ) class ClientWindow(Gtk.Window): """Client Window""" def __init__(self): super().__init__() self.set_default_size(300, 150) self.set_resizable(False) self.set_border_width(10) self.set_position(Gtk.WindowPosition.CENTER) self.grid = Gtk.Grid( row_homogeneous=True, column_homogeneous=True, row_spacing=15, column_spacing=30, ) self.label1 = Gtk.Label() self.label1.set_markup("Enter or Select Server IP: ") self.entry2 = Gtk.Entry.new() self.entry2.set_placeholder_text("Type here...") self.entry2.set_activates_default(False) self.entry2.set_sensitive(False) maximum_search = MAX_TRY addresses = [] attempt = 0 self.devices = [] for device in glob.glob("/dev/video*"): self.devices.append(device) source_label = Gtk.Label(label="Source") self.source_select = Gtk.ComboBoxText() for option in self.devices: self.source_select.append_text(option) self.source_select.set_active(0) while maximum_search != 0 and attempt <= 1: # write a loop to attempt x times if access_ip() is not False: if ssdp_address[0] not in addresses: addresses.append(ssdp_address[0]) maximum_search -= 1 attempt = attempt + 1 if len(addresses) != 0: self.server_ip = addresses[0] else: self.server_ip = "Not found!" self.radio_ip = Gtk.RadioButton() self.radio_ip.set_label(self.server_ip) self.radio_ip.connect("toggled", self.ip_changed) self.radio_savedip = Gtk.RadioButton(group=self.radio_ip) self.enter_ip = "Type IP address :" self.radio_savedip.set_label(self.enter_ip) self.radio_savedip.connect("toggled", self.ip_changed) self.button3 = Gtk.Button() self.button3.set_label("Connect to Server") button3_label = self.button3.get_child() button3_label.set_markup(" Connect to Server ") self.button3.connect("clicked", self.connect_to_server_thread) if len(addresses) != 0: self.radio_ip.set_active(True) else: self.radio_savedip.set_active(True) self.radio_ip.set_sensitive(False) self.header = Gtk.HeaderBar() self.header.set_show_close_button(False) self.header.props.title = "ML Client Setup" self.set_titlebar(self.header) quit_button = Gtk.Button() quit_icon = Gio.ThemedIcon(name="process-stop-symbolic") quit_image = Gtk.Image.new_from_gicon(quit_icon, Gtk.IconSize.BUTTON) quit_button.add(quit_image) self.header.pack_end(quit_button) quit_button.connect("clicked", Gtk.main_quit) self.grid.attach(source_label, 0, 0, 1, 1) self.grid.attach(self.source_select, 1, 0, 1, 1) self.grid.attach(self.label1, 0, 1, 1, 1) self.grid.attach(self.radio_ip, 0, 2, 1, 1) self.grid.attach(self.radio_savedip, 0, 3, 1, 1) self.grid.attach(self.entry2, 1, 3, 1, 1) self.grid.attach(self.button3, 0, 4, 2, 2) self.add(self.grid) def ip_changed(self, widget): """Check if the client wants to connect to different IP address""" if widget.get_active() is True: if widget.get_label() == self.enter_ip: self.entry2.set_activates_default(True) self.entry2.set_sensitive(True) else: self.entry2.set_activates_default(False) self.entry2.set_sensitive(False) def connect_to_server_thread(self, widget): """Thread to connect to server""" client_thread = Thread( target=ClientWindow.connect_to_server, args=[client_window] ) client_thread.daemon = True client_thread.start() def connect_to_server(self): """To start the connection to server""" self.button3.set_sensitive(False) # initialises state in background Gst.init(None) self.main_loop = GLib.MainLoop() ml_data_set = utils.download_file(DATA_SET) if self.entry2.get_activates_default() is True: self.server_ip = self.entry2.get_text() src = self.source_select.get_active_text() client_pipeline = "v4l2src device={dev} ! " client_pipeline += ( "video/x-raw,width=640,height=480,framerate=30/1 ! tee name=t t. ! " ) client_pipeline += "queue max-size-buffers=2 leaky=2 ! imxvideoconvert_g2d ! " client_pipeline += "video/x-raw,width=300,height=300,format=RGBA ! " client_pipeline += ( "videoconvert ! video/x-raw,format=RGB ! tensor_query_client " ) client_pipeline += "host={ip_client} dest-host={ip} ! tensor_decoder" client_pipeline += " mode=bounding_boxes option1=tf-ssd option2={label} " client_pipeline += "option3=0:1:2:3,50 option4=640:480 option5=300:300 !" client_pipeline += ( " mix. t. ! queue max-size-buffers=2 ! imxcompositor_g2d name=mix" ) client_pipeline += " sink_0::zorder=2 sink_1::zorder=1 ! waylandsink" client_pipeline = client_pipeline.format( dev=src, ip_client=get_my_ip(), ip=self.server_ip, label=ml_data_set ) # creating the pipeline and launching it self.pipeline = Gst.parse_launch(client_pipeline) monitor_status = self.pipeline.set_state(Gst.State.PLAYING) bus = self.pipeline.get_bus() bus.add_signal_watch() bus.connect("message", self.on_message) if monitor_status == Gst.StateChangeReturn.FAILURE: print("ERROR: Unable to set the pipeline to the playing state") sys.exit(1) self.button3.set_label("Connected to server") button3_label = self.button3.get_child() button3_label.set_markup(" Connected to server ") self.main_loop.run() # disconnecting the pipeline self.pipeline.set_state(Gst.State.NULL) bus.remove_signal_watch() def on_message(self, bus, message): """Callback for message. bus: pipeline bus message: message from pipeline """ mtype = message.type if mtype == Gst.MessageType.EOS: # Handle End of Stream print("End of stream") self.main_loop.quit() elif mtype == Gst.MessageType.ERROR: # Handle Errors err, debug = message.parse_error() print(err, debug) self.button3.set_label("GStreamer crashed!") button3_label = self.button3.get_child() button3_label.set_markup(" GStreamer crashed! ") self.main_loop.quit() elif mtype == Gst.MessageType.WARNING: # Handle warnings err, debug = message.parse_warning() print(err, debug) elif mtype == Gst.MessageType.STREAM_START: logging.info("received start message") elif mtype == Gst.MessageType.QOS: data_format, processed, dropped = message.parse_qos_stats() format_str = Gst.Format.get_name(data_format) logging.debug( "[qos] format[%s] processed[%d] dropped[%d]", format_str, processed, dropped, ) class DisplayWindow(Gtk.Window): """Main Window display""" def __init__(self): # Main window super().__init__() self.set_default_size(230, 180) self.set_resizable(False) self.set_border_width(20) self.set_position(Gtk.WindowPosition.CENTER) # Create widgets for server and client self.grid_display = Gtk.Grid( row_homogeneous=True, column_homogeneous=True, row_spacing=10, column_spacing=10, ) self.grid_display.set_margin_end(10) self.grid_display.set_margin_start(10) header = Gtk.HeaderBar() header.set_show_close_button(False) header.set_title("ML Server using NNStreamer") header.set_subtitle("ML Server on i.MX8M Plus") self.set_titlebar(header) quit_button = Gtk.Button() quit_icon = Gio.ThemedIcon(name="process-stop-symbolic") quit_image = Gtk.Image.new_from_gicon(quit_icon, Gtk.IconSize.BUTTON) quit_button.add(quit_image) header.pack_end(quit_button) quit_button.connect("clicked", Gtk.main_quit) self.server_button = Gtk.Button(label="Set up a server...") self.client_button = Gtk.Button(label="Set up a client...") self.server_button.connect("clicked", self.server_initiate) self.client_button.connect("clicked", self.client_initiate) global BOARD print(BOARD) if BOARD == "i.MX8MP": self.client_button.set_sensitive(False) else: self.server_button.set_sensitive(False) self.grid_display.attach(self.server_button, 0, 0, 1, 1) self.grid_display.attach(self.client_button, 0, 2, 1, 1) self.add(self.grid_display) def server_initiate(self, widget): """Initiate the server""" global server_window server_window = ServerWindow() server_window.connect("destroy", Gtk.main_quit) server_window.show_all() win.hide() def client_initiate(self, widget): """Initiate the client""" capture_ip_thread = Thread(target=access_ip) capture_ip_thread.start() global client_window client_window = ClientWindow() client_window.connect("destroy", Gtk.main_quit) client_window.show_all() win.hide() if __name__ == "__main__": os.environ["VIV_VX_CACHE_BINARY_GRAPH_DIR"] = "/home/root/.cache" "/demoexperience" os.environ["VIV_VX_ENABLE_CACHE_GRAPH_BINARY"] = "1" BOARD = subprocess.check_output(["cat", "/sys/devices/soc0/soc_id"]).decode( "utf-8" )[:-1] win = DisplayWindow() initialize_thread = Thread(target=initialize) initialize_thread.start() Gtk.main()