From 2600c1e35393d0f0e528ab64dd379fe7f6bb6488 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 16 Nov 2020 14:30:52 -0500 Subject: [PATCH 01/77] starting to bring over session, update licensing --- adafruit_minimqtt/adafruit_minimqtt.py | 80 ++++++++++++++++++++++- examples/minimqtt_simpletest_cpython | 88 ++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 examples/minimqtt_simpletest_cpython diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 6bfc60e0..e02f750e 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -27,13 +27,23 @@ `adafruit_minimqtt` ================================================================================ -MQTT Library for CircuitPython. +A minimal MQTT Library for CircuitPython. * Author(s): Brent Rubell Implementation Notes -------------------- +Adapted from https://github.com/micropython/micropython-lib/tree/master/umqtt.simple/umqtt + +micropython-lib consists of multiple modules from different sources and +authors. Each module comes under its own licensing terms. Short name of +a license can be found in a file within a module directory (usually +metadata.txt or setup.py). Complete text of each license used is provided +at https://github.com/micropython/micropython-lib/blob/master/LICENSE + +author='Paul Sokolovsky' +license='MIT' **Software and Dependencies:** * Adafruit CircuitPython firmware for the supported boards: @@ -102,6 +112,73 @@ def set_socket(sock, iface=None): _the_sock.set_interface(iface) +class Session: + """Session which shares sockets and SSL context.""" + def __init__(self, socket_pool, ssl_context=None): + self._socket_pool = socket_pool + self._ssl_context = ssl_context + # Hang onto open sockets so that we can reuse them. + self._open_sockets = {} + self._socket_free = {} + self._last_response = None + + def _free_socket(self, socket): + if socket not in self._open_sockets.values(): + raise RuntimeError("Socket not from active MQTT session.") + self._socket_free[socket] = True + + def _close_socket(self, sock): + sock.close() + del self._socket_free[sock] + key = None + for k in self._open_sockets: + if self._open_sockets[k] == sock: + key = k + break + if key: + del self._open_sockets[key] + + def _free_sockets(self): + free_sockets = [] + for sock in self._socket_free: + if self._socket_free[sock]: + free_sockets.append(sock) + for sock in free_sockets: + self._close_socket(sock) + + def _get_socket(self): + pass + + """ # Pre-existing get_socket + self._sock = _the_sock.socket() + self._sock.settimeout(15) + if self.port == 8883: + try: + if self.logger is not None: + self.logger.debug( + "Attempting to establish secure MQTT connection..." + ) + conntype = _the_interface.TLS_MODE + self._sock.connect((self.broker, self.port), conntype) + except RuntimeError as e: + raise MMQTTException("Invalid broker address defined.", e) from None + else: + try: + if self.logger is not None: + self.logger.debug( + "Attempting to establish insecure MQTT connection..." + ) + addr = _the_sock.getaddrinfo( + self.broker, self.port, 0, _the_sock.SOCK_STREAM + )[0] + self._sock.connect(addr[-1], _the_interface.TCP_MODE) + except RuntimeError as e: + raise MMQTTException("Invalid broker address defined.", e) from None + + """ + + + class MQTT: """MQTT Client for CircuitPython @@ -278,6 +355,7 @@ def connect(self, clean_session=True): :param bool clean_session: Establishes a persistent session. """ + # TODO: This will need to implement _get_socket in the future self._sock = _the_sock.socket() self._sock.settimeout(15) if self.port == 8883: diff --git a/examples/minimqtt_simpletest_cpython b/examples/minimqtt_simpletest_cpython new file mode 100644 index 00000000..a6b6f90f --- /dev/null +++ b/examples/minimqtt_simpletest_cpython @@ -0,0 +1,88 @@ +import socket + +import adafruit_minimqtt.adafruit_minimqtt as MQTT + +### WiFi ### + +# Get wifi details and more from a secrets.py file +try: + from secrets import secrets +except ImportError: + print("WiFi secrets are kept in secrets.py, please add them there!") + raise + +### Topic Setup ### + +# MQTT Topic +# Use this topic if you'd like to connect to a standard MQTT broker +mqtt_topic = "test/topic" + +# Adafruit IO-style Topic +# Use this topic if you'd like to connect to io.adafruit.com +# mqtt_topic = 'aio_user/feeds/temperature' + +### Code ### + +# Define callback methods which are called when events occur +# pylint: disable=unused-argument, redefined-outer-name +def connect(mqtt_client, userdata, flags, rc): + # This function will be called when the mqtt_client is connected + # successfully to the broker. + print("Connected to MQTT Broker!") + print("Flags: {0}\n RC: {1}".format(flags, rc)) + + +def disconnect(mqtt_client, userdata, rc): + # This method is called when the mqtt_client disconnects + # from the broker. + print("Disconnected from MQTT Broker!") + + +def subscribe(mqtt_client, userdata, topic, granted_qos): + # This method is called when the mqtt_client subscribes to a new feed. + print("Subscribed to {0} with QOS level {1}".format(topic, granted_qos)) + + +def unsubscribe(mqtt_client, userdata, topic, pid): + # This method is called when the mqtt_client unsubscribes from a feed. + print("Unsubscribed from {0} with PID {1}".format(topic, pid)) + + +def publish(mqtt_client, userdata, topic, pid): + # This method is called when the mqtt_client publishes data to a feed. + print("Published to {0} with PID {1}".format(topic, pid)) + + +mqtt_client = MQTT.session(socket) + +# Initialize MQTT interface with the esp interface +# MQTT.set_socket(socket, esp) + +# Set up a MiniMQTT Client +mqtt_client = MQTT.MQTT( + broker=secrets["broker"], username=secrets["user"], password=secrets["pass"] +) + +# Connect callback handlers to mqtt_client +mqtt_client.on_connect = connect +mqtt_client.on_disconnect = disconnect +mqtt_client.on_subscribe = subscribe +mqtt_client.on_unsubscribe = unsubscribe +mqtt_client.on_publish = publish + +print("Attempting to connect to %s" % mqtt_client.broker) +# TODO: Move config into connect call? +# broker=secrets["broker"], username=secrets["user"], password=secrets["pass"] +mqtt_client.connect() + +print("Subscribing to %s" % mqtt_topic) +mqtt_client.subscribe(mqtt_topic) + +print("Publishing to %s" % mqtt_topic) +mqtt_client.publish(mqtt_topic, "Hello Broker!") + +print("Unsubscribing from %s" % mqtt_topic) +mqtt_client.unsubscribe(mqtt_topic) + +print("Disconnecting from %s" % mqtt_client.broker) +mqtt_client.disconnect() From 1db4bad77343eb664f16cd29f416c74c45c5fcc3 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 16 Nov 2020 14:41:37 -0500 Subject: [PATCH 02/77] add _get_socket --- adafruit_minimqtt/adafruit_minimqtt.py | 79 ++++++++++++++++---------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index e02f750e..cc82b4b2 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -146,38 +146,59 @@ def _free_sockets(self): for sock in free_sockets: self._close_socket(sock) - def _get_socket(self): - pass - - """ # Pre-existing get_socket - self._sock = _the_sock.socket() - self._sock.settimeout(15) - if self.port == 8883: - try: - if self.logger is not None: - self.logger.debug( - "Attempting to establish secure MQTT connection..." - ) - conntype = _the_interface.TLS_MODE - self._sock.connect((self.broker, self.port), conntype) - except RuntimeError as e: - raise MMQTTException("Invalid broker address defined.", e) from None - else: - try: - if self.logger is not None: - self.logger.debug( - "Attempting to establish insecure MQTT connection..." - ) - addr = _the_sock.getaddrinfo( - self.broker, self.port, 0, _the_sock.SOCK_STREAM - )[0] - self._sock.connect(addr[-1], _the_interface.TCP_MODE) - except RuntimeError as e: - raise MMQTTException("Invalid broker address defined.", e) from None + def _get_socket(self, host, port, *, timeout=1): + key = (host, port) + if key in self._open_sockets: + sock = self._open_sockets[key] + if self._socket_free[sock]: + self._socket_free[sock] = False + return sock + if port == 8883 and not self._ssl_context: + raise RuntimeError( + "ssl_context must be set before using adafruit_minimqtt for secure mqtt" + ) + addr_info = self._socket_pool.getaddrinfo( + host, port, 0, self._socket_pool.SOCK_STREAM + )[0] + retry_count = 0 + sock = None + + while retry_count < 5 and sock is None: + if retry_count > 0: + if any(self._socket_free.items()): + self._free_sockets() + else: + raise RuntimeError("Unable to obtain free socket.") + retry_count += 1 - """ + try: + sock = self._socket_pool.socket( + addr_info[0], addr_info[1], addr_info[2] + ) + except OSError: + continue + connect_host = addr_info[-1][0] + if port == 8883: + sock = self._ssl_context.wrap_socket(sock, server_hostname=host) + connect_host = host + sock.settimeout(timeout) # socket read timeout + try: + sock.connect((connect_host, port)) + except MemoryError: + sock.close() + sock = None + except OSError: + sock.close() + sock = None + + if sock is None: + raise RuntimeError("Repeated socket failures") + + self._open_sockets[key] = sock + self._socket_free[sock] = False + return sock class MQTT: """MQTT Client for CircuitPython From 7d74dbfc7c36131d384029cdd10ee9c961172452 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 16 Nov 2020 14:59:49 -0500 Subject: [PATCH 03/77] BREAKING _init_, should match paho mqtt --- adafruit_minimqtt/adafruit_minimqtt.py | 98 ++++++++++---------------- 1 file changed, 37 insertions(+), 61 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index cc82b4b2..cc1b6419 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -99,7 +99,7 @@ class MMQTTException(Exception): def set_socket(sock, iface=None): - """Helper to set the global socket and optionally set the global network interface. + """Legacy API for setting the socket and network interface, use a `Session` instead. :param sock: socket object. :param iface: internet interface object @@ -201,49 +201,21 @@ def _get_socket(self, host, port, *, timeout=1): return sock class MQTT: - """MQTT Client for CircuitPython - - :param str broker: MQTT Broker URL or IP Address. - :param int port: Optional port definition, defaults to 8883. - :param str username: Username for broker authentication. - :param str password: Password for broker authentication. - :param network_manager: NetworkManager object, such as WiFiManager from ESPSPI_WiFiManager. - :param str client_id: Optional client identifier, defaults to a unique, generated string. - :param bool is_ssl: Sets a secure or insecure connection with the broker. - :param bool log: Attaches a logger to the MQTT client, defaults to logging level INFO. - :param int keep_alive: KeepAlive interval between the broker and the MiniMQTT client. + """MQTT Client for CircuitPython. + :param str client_id: Client identifier. If `client_id ` is + set to None, then an identifier string will be randomly generated. + :param bool clean_session: Broker removes all information about this client + after disconnection. If `False`, subscriptions and queued messages are retained + after the client disconnects. + :param str user_data: User defined data that is passed as userdata to callbacks. """ - # pylint: disable=too-many-arguments,too-many-instance-attributes, not-callable, invalid-name, no-member - def __init__( - self, - broker, - port=None, - username=None, - password=None, - client_id=None, - is_ssl=True, - log=False, - keep_alive=60, - ): - self._sock = None - self.broker = broker - # port/ssl - self.port = MQTT_TCP_PORT - if is_ssl: - self.port = MQTT_TLS_PORT - if port is not None: - self.port = port - # session identifiers - self.user = username - # [MQTT-3.1.3.5] - self.password = password - if ( - self.password is not None - and len(password.encode("utf-8")) > MQTT_TOPIC_LENGTH_LIMIT - ): - raise MMQTTException("Password length is too large.") + def __init__(self, client_id=None, clean_session=True, _user_data=None): + self._user_data = _user_data + self._clean_session = clean_session + + # Assign client identifier if client_id is not None: # user-defined client_id MAY allow client_id's > 23 bytes or # non-alpha-numeric characters @@ -256,27 +228,24 @@ def __init__( # generated client_id's enforce spec.'s length rules if len(self.client_id) > 23 or not self.client_id: raise ValueError("MQTT Client ID must be between 1 and 23 bytes") - self.keep_alive = keep_alive - self.user_data = None + # Set up logger instance self.logger = None - if log is True: - self.logger = logging.getLogger("log") - self.logger.setLevel(logging.INFO) - self._sock = None - self._is_connected = False - self._msg_size_lim = MQTT_MSG_SZ_LIM - self._pid = 0 - self._timestamp = 0 + # TODO: enable_logger() should be implemented instead of this! + self.logger = logging.getLogger("log") + self.logger.setLevel(logging.INFO) + + # List of subscribed topics, used for reconnections + self._subscribed_topics = [] + self._on_message_filtered = MQTTMatcher() + # LWT self._lw_topic = None self._lw_qos = 0 self._lw_topic = None self._lw_msg = None self._lw_retain = False - # List of subscribed topics, used for tracking - self._subscribed_topics = [] - self._on_message_filtered = MQTTMatcher() - # Server callbacks + + # Default callback methods self._on_message = None self.on_connect = None self.on_disconnect = None @@ -284,6 +253,13 @@ def __init__( self.on_subscribe = None self.on_unsubscribe = None + # Library defaults + self._is_connected = False + self._msg_size_lim = MQTT_MSG_SZ_LIM + self._pid = 0 + self._timestamp = 0 + + def __enter__(self): return self @@ -474,7 +450,7 @@ def connect(self, clean_session=True): self._is_connected = True result = rc[0] & 1 if self.on_connect is not None: - self.on_connect(self, self.user_data, result, rc[2]) + self.on_connect(self, self._user_data, result, rc[2]) return result def disconnect(self): @@ -490,7 +466,7 @@ def disconnect(self): self._is_connected = False self._subscribed_topics = None if self.on_disconnect is not None: - self.on_disconnect(self, self.user_data, 0) + self.on_disconnect(self, self._user_data, 0) def ping(self): """Pings the MQTT Broker to confirm if the broker is alive or if @@ -598,7 +574,7 @@ def publish(self, topic, msg, retain=False, qos=0): self._sock.send(pub_hdr_var) self._sock.send(msg) if qos == 0 and self.on_publish is not None: - self.on_publish(self, self.user_data, topic, self._pid) + self.on_publish(self, self._user_data, topic, self._pid) if qos == 1: while True: op = self._wait_for_msg() @@ -609,7 +585,7 @@ def publish(self, topic, msg, retain=False, qos=0): rcv_pid = rcv_pid[0] << 0x08 | rcv_pid[1] if pid == rcv_pid: if self.on_publish is not None: - self.on_publish(self, self.user_data, topic, rcv_pid) + self.on_publish(self, self._user_data, topic, rcv_pid) return def subscribe(self, topic, qos=0): @@ -694,7 +670,7 @@ def subscribe(self, topic, qos=0): raise MMQTTException("SUBACK Failure!") for t, q in topics: if self.on_subscribe is not None: - self.on_subscribe(self, self.user_data, t, q) + self.on_subscribe(self, self._user_data, t, q) self._subscribed_topics.append(t) return @@ -760,7 +736,7 @@ def unsubscribe(self, topic): ) for t in topics: if self.on_unsubscribe is not None: - self.on_unsubscribe(self, self.user_data, t, self._pid) + self.on_unsubscribe(self, self._user_data, t, self._pid) self._subscribed_topics.remove(t) return From afabde48f77cb01406d185624303bfbc83cb5707 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 16 Nov 2020 15:32:23 -0500 Subject: [PATCH 04/77] dont break init, leave alone for now --- adafruit_minimqtt/adafruit_minimqtt.py | 95 +++++++++++++++++--------- 1 file changed, 63 insertions(+), 32 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index cc1b6419..5ad49706 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -201,21 +201,57 @@ def _get_socket(self, host, port, *, timeout=1): return sock class MQTT: - """MQTT Client for CircuitPython. - :param str client_id: Client identifier. If `client_id ` is - set to None, then an identifier string will be randomly generated. - :param bool clean_session: Broker removes all information about this client - after disconnection. If `False`, subscriptions and queued messages are retained - after the client disconnects. - :param str user_data: User defined data that is passed as userdata to callbacks. +class MQTT: + """MQTT Client for CircuitPython + + :param str broker: MQTT Broker URL or IP Address. + :param int port: Optional port definition, defaults to 8883. + :param str username: Username for broker authentication. + :param str password: Password for broker authentication. + :param network_manager: NetworkManager object, such as WiFiManager from ESPSPI_WiFiManager. + :param str client_id: Optional client identifier, defaults to a unique, generated string. + :param bool is_ssl: Sets a secure or insecure connection with the broker. + :param bool log: Attaches a logger to the MQTT client, defaults to logging level INFO. + :param int keep_alive: KeepAlive interval between the broker and the MiniMQTT client. """ # pylint: disable=too-many-arguments,too-many-instance-attributes, not-callable, invalid-name, no-member - def __init__(self, client_id=None, clean_session=True, _user_data=None): - self._user_data = _user_data - self._clean_session = clean_session + def __init__( + self, + broker, + port=None, + username=None, + password=None, + client_id=None, + is_ssl=True, + log=False, + keep_alive=60, + ): + + self._sock = None + self.keep_alive = keep_alive + self._user_data = None + self._is_connected = False + self._msg_size_lim = MQTT_MSG_SZ_LIM + self._pid = 0 + self._timestamp = 0 - # Assign client identifier + self.broker = broker + self.user = username + self.password = password + if ( + self.password is not None + and len(password.encode("utf-8")) > MQTT_TOPIC_LENGTH_LIMIT + ): # [MQTT-3.1.3.5] + raise MMQTTException("Password length is too large.") + + self.port = MQTT_TCP_PORT + if is_ssl: + self.port = MQTT_TLS_PORT + if port is not None: + self.port = port + + # define client identifer if client_id is not None: # user-defined client_id MAY allow client_id's > 23 bytes or # non-alpha-numeric characters @@ -228,15 +264,12 @@ def __init__(self, client_id=None, clean_session=True, _user_data=None): # generated client_id's enforce spec.'s length rules if len(self.client_id) > 23 or not self.client_id: raise ValueError("MQTT Client ID must be between 1 and 23 bytes") - # Set up logger instance - self.logger = None - # TODO: enable_logger() should be implemented instead of this! - self.logger = logging.getLogger("log") - self.logger.setLevel(logging.INFO) - # List of subscribed topics, used for reconnections - self._subscribed_topics = [] - self._on_message_filtered = MQTTMatcher() + self.logger = None + if log is True: + # TODO: This should call an enable_logging() method + self.logger = logging.getLogger("log") + self.logger.setLevel(logging.INFO) # LWT self._lw_topic = None @@ -245,7 +278,11 @@ def __init__(self, client_id=None, clean_session=True, _user_data=None): self._lw_msg = None self._lw_retain = False - # Default callback methods + # List of subscribed topics, used for tracking + self._subscribed_topics = [] + self._on_message_filtered = MQTTMatcher() + + # Default topic callback methods self._on_message = None self.on_connect = None self.on_disconnect = None @@ -253,12 +290,6 @@ def __init__(self, client_id=None, clean_session=True, _user_data=None): self.on_subscribe = None self.on_unsubscribe = None - # Library defaults - self._is_connected = False - self._msg_size_lim = MQTT_MSG_SZ_LIM - self._pid = 0 - self._timestamp = 0 - def __enter__(self): return self @@ -450,7 +481,7 @@ def connect(self, clean_session=True): self._is_connected = True result = rc[0] & 1 if self.on_connect is not None: - self.on_connect(self, self._user_data, result, rc[2]) + self.on_connect(self, self.__user_data, result, rc[2]) return result def disconnect(self): @@ -466,7 +497,7 @@ def disconnect(self): self._is_connected = False self._subscribed_topics = None if self.on_disconnect is not None: - self.on_disconnect(self, self._user_data, 0) + self.on_disconnect(self, self.__user_data, 0) def ping(self): """Pings the MQTT Broker to confirm if the broker is alive or if @@ -574,7 +605,7 @@ def publish(self, topic, msg, retain=False, qos=0): self._sock.send(pub_hdr_var) self._sock.send(msg) if qos == 0 and self.on_publish is not None: - self.on_publish(self, self._user_data, topic, self._pid) + self.on_publish(self, self.__user_data, topic, self._pid) if qos == 1: while True: op = self._wait_for_msg() @@ -585,7 +616,7 @@ def publish(self, topic, msg, retain=False, qos=0): rcv_pid = rcv_pid[0] << 0x08 | rcv_pid[1] if pid == rcv_pid: if self.on_publish is not None: - self.on_publish(self, self._user_data, topic, rcv_pid) + self.on_publish(self, self.__user_data, topic, rcv_pid) return def subscribe(self, topic, qos=0): @@ -670,7 +701,7 @@ def subscribe(self, topic, qos=0): raise MMQTTException("SUBACK Failure!") for t, q in topics: if self.on_subscribe is not None: - self.on_subscribe(self, self._user_data, t, q) + self.on_subscribe(self, self.__user_data, t, q) self._subscribed_topics.append(t) return @@ -736,7 +767,7 @@ def unsubscribe(self, topic): ) for t in topics: if self.on_unsubscribe is not None: - self.on_unsubscribe(self, self._user_data, t, self._pid) + self.on_unsubscribe(self, self.__user_data, t, self._pid) self._subscribed_topics.remove(t) return From 10dfa0b64c176693c423330e1f3752f75115b1d5 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 16 Nov 2020 15:36:57 -0500 Subject: [PATCH 05/77] add username_pw_set, make username/pass private --- adafruit_minimqtt/adafruit_minimqtt.py | 31 +++++++++++++++++--------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 5ad49706..8f5ffa9c 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -200,7 +200,6 @@ def _get_socket(self, host, port, *, timeout=1): self._socket_free[sock] = False return sock -class MQTT: class MQTT: """MQTT Client for CircuitPython @@ -237,10 +236,10 @@ def __init__( self._timestamp = 0 self.broker = broker - self.user = username - self.password = password + self._username = username + self._password = password if ( - self.password is not None + self._password is not None and len(password.encode("utf-8")) > MQTT_TOPIC_LENGTH_LIMIT ): # [MQTT-3.1.3.5] raise MMQTTException("Password length is too large.") @@ -377,6 +376,18 @@ def _handle_on_message(self, client, topic, message): if not matched and self.on_message: # regular on_message self.on_message(client, topic, message) + def username_pw_set(self, username, password=None): + """Set client's username and an optional password. + :param str username: Username to use with your MQTT broker. + :param str password: Password to use with your MQTT broker. + + """ + if self._is_connected: + raise MMQTTException("This method must be called before connect().") + self._username = username + if password is not None: + self._password = password + # pylint: disable=too-many-branches, too-many-statements, too-many-locals def connect(self, clean_session=True): """Initiates connection with the MQTT Broker. @@ -421,8 +432,8 @@ def connect(self, clean_session=True): # Set up variable header and remaining_length remaining_length = 12 + len(self.client_id) - if self.user is not None: - remaining_length += 2 + len(self.user) + 2 + len(self.password) + if self._username is not None: + remaining_length += 2 + len(self._username) + 2 + len(self._password) var_header[6] |= 0xC0 if self.keep_alive: assert self.keep_alive < MQTT_TOPIC_LENGTH_LIMIT @@ -464,11 +475,11 @@ def connect(self, clean_session=True): # [MQTT-3.1.3-11] self._send_str(self._lw_topic) self._send_str(self._lw_msg) - if self.user is None: - self.user = None + if self._username is None: + self._username = None else: - self._send_str(self.user) - self._send_str(self.password) + self._send_str(self._username) + self._send_str(self._password) if self.logger is not None: self.logger.debug("Receiving CONNACK packet from broker") while True: From 8dae98f7d71fd159c13ff1e768daf2b61c0d5d18 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 16 Nov 2020 15:54:01 -0500 Subject: [PATCH 06/77] refactor logger class, add enable_logger, disable_logger --- adafruit_minimqtt/adafruit_minimqtt.py | 35 ++++++++++++++++++++------ 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 8f5ffa9c..30447d47 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -492,7 +492,7 @@ def connect(self, clean_session=True): self._is_connected = True result = rc[0] & 1 if self.on_connect is not None: - self.on_connect(self, self.__user_data, result, rc[2]) + self.on_connect(self, self._user_data, result, rc[2]) return result def disconnect(self): @@ -508,7 +508,7 @@ def disconnect(self): self._is_connected = False self._subscribed_topics = None if self.on_disconnect is not None: - self.on_disconnect(self, self.__user_data, 0) + self.on_disconnect(self, self._user_data, 0) def ping(self): """Pings the MQTT Broker to confirm if the broker is alive or if @@ -616,7 +616,7 @@ def publish(self, topic, msg, retain=False, qos=0): self._sock.send(pub_hdr_var) self._sock.send(msg) if qos == 0 and self.on_publish is not None: - self.on_publish(self, self.__user_data, topic, self._pid) + self.on_publish(self, self._user_data, topic, self._pid) if qos == 1: while True: op = self._wait_for_msg() @@ -627,7 +627,7 @@ def publish(self, topic, msg, retain=False, qos=0): rcv_pid = rcv_pid[0] << 0x08 | rcv_pid[1] if pid == rcv_pid: if self.on_publish is not None: - self.on_publish(self, self.__user_data, topic, rcv_pid) + self.on_publish(self, self._user_data, topic, rcv_pid) return def subscribe(self, topic, qos=0): @@ -712,7 +712,7 @@ def subscribe(self, topic, qos=0): raise MMQTTException("SUBACK Failure!") for t, q in topics: if self.on_subscribe is not None: - self.on_subscribe(self, self.__user_data, t, q) + self.on_subscribe(self, self._user_data, t, q) self._subscribed_topics.append(t) return @@ -778,7 +778,7 @@ def unsubscribe(self, topic): ) for t in topics: if self.on_unsubscribe is not None: - self.on_unsubscribe(self, self.__user_data, t, self._pid) + self.on_unsubscribe(self, self._user_data, t, self._pid) self._subscribed_topics.remove(t) return @@ -939,12 +939,33 @@ def mqtt_msg(self, msg_size): ### Logging ### def attach_logger(self, logger_name="log"): """Initializes and attaches a logger to the MQTTClient. - :param str logger_name: Name of the logger instance + NOTE: This method is replaced by enable_logger and + will be removed in a future release to + remove this lib's dependency from adafruit_logging. + """ self.logger = logging.getLogger(logger_name) self.logger.setLevel(logging.INFO) + def enable_logger(self, logger=None): + """Enables logging using the logging package. If a `logger` + is specified, then that object will be used. Otherwise, a `logger` + will be automatically created. + + """ + if logger: + self.logger = logger + else: + self.logger = logging.getLogger("log") + self.logger.setLevel(logging.INFO) + + def disable_logger(self): + """Disables logging.""" + if not self.logger: + raise ValueError("Can not disable logging - no logger enabled!") + self.logger = None + def set_logger_level(self, log_level): """Sets the level of the logger, if defined during init. From 722ecda67f12f7ac3f88e52241ee5432a2c6c68b Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 16 Nov 2020 15:55:21 -0500 Subject: [PATCH 07/77] hook logging --- adafruit_minimqtt/adafruit_minimqtt.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 30447d47..f7964280 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -266,9 +266,7 @@ def __init__( self.logger = None if log is True: - # TODO: This should call an enable_logging() method - self.logger = logging.getLogger("log") - self.logger.setLevel(logging.INFO) + self.enable_logger() # LWT self._lw_topic = None From 971903de1ca3097c2d87b28bbb2b2e49ac1218b3 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 16 Nov 2020 16:03:33 -0500 Subject: [PATCH 08/77] remove Session, socket pool impl. instead --- adafruit_minimqtt/adafruit_minimqtt.py | 89 -------------------------- 1 file changed, 89 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index f7964280..22e402cb 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -93,7 +93,6 @@ class MMQTTException(Exception): """MiniMQTT Exception class.""" - # pylint: disable=unnecessary-pass # pass @@ -112,94 +111,6 @@ def set_socket(sock, iface=None): _the_sock.set_interface(iface) -class Session: - """Session which shares sockets and SSL context.""" - def __init__(self, socket_pool, ssl_context=None): - self._socket_pool = socket_pool - self._ssl_context = ssl_context - # Hang onto open sockets so that we can reuse them. - self._open_sockets = {} - self._socket_free = {} - self._last_response = None - - def _free_socket(self, socket): - if socket not in self._open_sockets.values(): - raise RuntimeError("Socket not from active MQTT session.") - self._socket_free[socket] = True - - def _close_socket(self, sock): - sock.close() - del self._socket_free[sock] - key = None - for k in self._open_sockets: - if self._open_sockets[k] == sock: - key = k - break - if key: - del self._open_sockets[key] - - def _free_sockets(self): - free_sockets = [] - for sock in self._socket_free: - if self._socket_free[sock]: - free_sockets.append(sock) - for sock in free_sockets: - self._close_socket(sock) - - def _get_socket(self, host, port, *, timeout=1): - key = (host, port) - if key in self._open_sockets: - sock = self._open_sockets[key] - if self._socket_free[sock]: - self._socket_free[sock] = False - return sock - if port == 8883 and not self._ssl_context: - raise RuntimeError( - "ssl_context must be set before using adafruit_minimqtt for secure mqtt" - ) - addr_info = self._socket_pool.getaddrinfo( - host, port, 0, self._socket_pool.SOCK_STREAM - )[0] - retry_count = 0 - sock = None - - while retry_count < 5 and sock is None: - if retry_count > 0: - if any(self._socket_free.items()): - self._free_sockets() - else: - raise RuntimeError("Unable to obtain free socket.") - retry_count += 1 - - try: - sock = self._socket_pool.socket( - addr_info[0], addr_info[1], addr_info[2] - ) - except OSError: - continue - - connect_host = addr_info[-1][0] - if port == 8883: - sock = self._ssl_context.wrap_socket(sock, server_hostname=host) - connect_host = host - sock.settimeout(timeout) # socket read timeout - - try: - sock.connect((connect_host, port)) - except MemoryError: - sock.close() - sock = None - except OSError: - sock.close() - sock = None - - if sock is None: - raise RuntimeError("Repeated socket failures") - - self._open_sockets[key] = sock - self._socket_free[sock] = False - return sock - class MQTT: """MQTT Client for CircuitPython From 29bc57d60824553cb95ea88fc4762eafcd4e68a4 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 16 Nov 2020 16:40:15 -0500 Subject: [PATCH 09/77] Add socketpool, ssl context to constructor --- adafruit_minimqtt/adafruit_minimqtt.py | 12 ++++++++++++ examples/minimqtt_simpletest_cpython | 10 ++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 22e402cb..2a071363 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -123,6 +123,8 @@ class MQTT: :param bool is_ssl: Sets a secure or insecure connection with the broker. :param bool log: Attaches a logger to the MQTT client, defaults to logging level INFO. :param int keep_alive: KeepAlive interval between the broker and the MiniMQTT client. + :param socket socket_pool: A pool of socket resources available for the given radio. + :param ssl_context: SSL context for long-lived SSL connections. """ # pylint: disable=too-many-arguments,too-many-instance-attributes, not-callable, invalid-name, no-member @@ -136,7 +138,17 @@ def __init__( is_ssl=True, log=False, keep_alive=60, + socket_pool=None + ssl_context=None ): + # Socket Pool + if socket_pool is not None: + self._socket_pool = socket_pool + if ssl_context is not None: + self._ssl_context = ssl_context + # Hang onto open sockets so that we can reuse them + self._socket_free = {} + self._open_sockets = {} self._sock = None self.keep_alive = keep_alive diff --git a/examples/minimqtt_simpletest_cpython b/examples/minimqtt_simpletest_cpython index a6b6f90f..dc837622 100644 --- a/examples/minimqtt_simpletest_cpython +++ b/examples/minimqtt_simpletest_cpython @@ -53,14 +53,12 @@ def publish(mqtt_client, userdata, topic, pid): print("Published to {0} with PID {1}".format(topic, pid)) -mqtt_client = MQTT.session(socket) - -# Initialize MQTT interface with the esp interface -# MQTT.set_socket(socket, esp) - # Set up a MiniMQTT Client mqtt_client = MQTT.MQTT( - broker=secrets["broker"], username=secrets["user"], password=secrets["pass"] + broker=secrets["broker"], + username=secrets["user"], + password=secrets["pass"], + socket ) # Connect callback handlers to mqtt_client From c14b9907ba17e234c084018e126d85441e5e587b Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 16 Nov 2020 16:44:37 -0500 Subject: [PATCH 10/77] bring in socketpool helper methods --- adafruit_minimqtt/adafruit_minimqtt.py | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 2a071363..1c8575cd 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -210,6 +210,37 @@ def __init__( self.on_subscribe = None self.on_unsubscribe = None + # Socket helpers + def _free_socket(self, socket): + """Frees a socket for re-use.""" + if socket not in self._open_sockets.values(): + raise RuntimeError("Socket not from MQTT client.") + self._socket_free[socket] = True + + def _close_socket(self, socket) + """Closes a slocket.""" + socket.close() + del self._socket_free[socket] + key = None + for k in self._open_sockets: + if self._open_sockets[k] == socket: + key = k + break + if key: + del self._open_sockets[key] + + def _free_sockets(self): + """Closes all free sockets.""" + free_sockets = [] + for sock in self._socket_free: + if self._socket_free[sock]: + free_sockets.append(sock) + for sock in free_sockets: + self._close_socket(sock) + + def _get_socket(self, host, port, *, timeout=1): + pass + def __enter__(self): return self From 95efa9ed1f3855b425080b88649042bf29a7b603 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 16 Nov 2020 17:05:41 -0500 Subject: [PATCH 11/77] add _get_socket helper, replace code in connect, modify connect signature to be paho-mqtt friendly --- adafruit_minimqtt/adafruit_minimqtt.py | 100 +++++++++++++++++-------- 1 file changed, 69 insertions(+), 31 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 1c8575cd..d5b372e6 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -158,7 +158,7 @@ def __init__( self._pid = 0 self._timestamp = 0 - self.broker = broker + self._broker = broker self._username = username self._password = password if ( @@ -167,11 +167,11 @@ def __init__( ): # [MQTT-3.1.3.5] raise MMQTTException("Password length is too large.") - self.port = MQTT_TCP_PORT + self._port = MQTT_TCP_PORT if is_ssl: - self.port = MQTT_TLS_PORT + self._port = MQTT_TLS_PORT if port is not None: - self.port = port + self._port = port # define client identifer if client_id is not None: @@ -239,8 +239,57 @@ def _free_sockets(self): self._close_socket(sock) def _get_socket(self, host, port, *, timeout=1): - pass + key = (host, port) + if key in self._open_sockets: + sock = self._open_sockets[key] + if self._socket_free[sock]: + self._socket_free[sock] = False + return sock + if port == 8883 and not self._ssl_context: + raise RuntimeError( + "ssl_context must be set before using adafruit_mqtt for secure MQTT." + ) + addr_info = self._socket_pool.getaddrinfo( + host, port, 0, self._socket_pool.SOCK_STREAM + )[0] + retry_count = 0 + sock = None + while retry_count < 5 and sock is None: + if retry_count > 0: + if any(self._socket_free.items()): + self._free_sockets() + else: + raise RuntimeError("Sending request failed") + retry_count += 1 + + try: + sock = self._socket_pool.socket( + addr_info[0], addr_info[1], addr_info[2] + ) + except OSError: + continue + + connect_host = addr_info[-1][0] + if port == 1883: + sock = self._ssl_context.wrap_socket(sock, server_hostname=host) + connect_host = host + sock.settimeout(timeout) + + try: + sock.connect((connect_host, port)) + except MemoryError: + sock.close() + sock = None + except OSError: + sock.close() + sock = None + if sock is None: + raise RuntimeError("Repeated socket failures") + + self._open_sockets[key] = sock + self._socket_free[sock] = False + return sock def __enter__(self): return self @@ -341,36 +390,25 @@ def username_pw_set(self, username, password=None): self._password = password # pylint: disable=too-many-branches, too-many-statements, too-many-locals - def connect(self, clean_session=True): + def connect(self, clean_session=True, host=None, port=None, keepalive=None): """Initiates connection with the MQTT Broker. :param bool clean_session: Establishes a persistent session. + :param str host: Hostname or IP address of the remote broker. + :param int port: Network port of the remote broker. + :param int keepalive: Maximum period allowed for communication + with the broker, in seconds + """ - # TODO: This will need to implement _get_socket in the future - self._sock = _the_sock.socket() - self._sock.settimeout(15) - if self.port == 8883: - try: - if self.logger is not None: - self.logger.debug( - "Attempting to establish secure MQTT connection..." - ) - conntype = _the_interface.TLS_MODE - self._sock.connect((self.broker, self.port), conntype) - except RuntimeError as e: - raise MMQTTException("Invalid broker address defined.", e) from None - else: - try: - if self.logger is not None: - self.logger.debug( - "Attempting to establish insecure MQTT connection..." - ) - addr = _the_sock.getaddrinfo( - self.broker, self.port, 0, _the_sock.SOCK_STREAM - )[0] - self._sock.connect(addr[-1], _the_interface.TCP_MODE) - except RuntimeError as e: - raise MMQTTException("Invalid broker address defined.", e) from None + if host is not None: + self._broker = host + if port is not None: + self._port = port + + self.logger.debug("Attempting to establish MQTT connection...") + + # Attempt to get a new socket + self._sock = self._get_socket(host, port) # Fixed Header fixed_header = bytearray([0x10]) From d4002673cc4760c1f2fc6b18a314dc3c3b46cbe8 Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 18 Nov 2020 14:44:50 -0500 Subject: [PATCH 12/77] connects, try cpython native logging first --- adafruit_minimqtt/adafruit_minimqtt.py | 33 +++++++++++++++----------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index d5b372e6..42aa707b 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -54,7 +54,10 @@ import time from random import randint from micropython import const -import adafruit_logging as logging +try: + import logging +except: + import adafruit_logging as logging from .matcher import MQTTMatcher __version__ = "0.0.0-auto.0" @@ -138,7 +141,7 @@ def __init__( is_ssl=True, log=False, keep_alive=60, - socket_pool=None + socket_pool=None, ssl_context=None ): # Socket Pool @@ -158,7 +161,7 @@ def __init__( self._pid = 0 self._timestamp = 0 - self._broker = broker + self.broker = broker self._username = username self._password = password if ( @@ -167,11 +170,11 @@ def __init__( ): # [MQTT-3.1.3.5] raise MMQTTException("Password length is too large.") - self._port = MQTT_TCP_PORT + self.port = MQTT_TCP_PORT if is_ssl: - self._port = MQTT_TLS_PORT + self.port = MQTT_TLS_PORT if port is not None: - self._port = port + self.port = port # define client identifer if client_id is not None: @@ -212,13 +215,13 @@ def __init__( # Socket helpers def _free_socket(self, socket): - """Frees a socket for re-use.""" + """Frees a socket for re-use.""" if socket not in self._open_sockets.values(): raise RuntimeError("Socket not from MQTT client.") self._socket_free[socket] = True - def _close_socket(self, socket) - """Closes a slocket.""" + def _close_socket(self, socket): + """Closes a slocket.""" socket.close() del self._socket_free[socket] key = None @@ -249,6 +252,7 @@ def _get_socket(self, host, port, *, timeout=1): raise RuntimeError( "ssl_context must be set before using adafruit_mqtt for secure MQTT." ) + print(key) addr_info = self._socket_pool.getaddrinfo( host, port, 0, self._socket_pool.SOCK_STREAM )[0] @@ -270,7 +274,7 @@ def _get_socket(self, host, port, *, timeout=1): continue connect_host = addr_info[-1][0] - if port == 1883: + if port == 8883: sock = self._ssl_context.wrap_socket(sock, server_hostname=host) connect_host = host sock.settimeout(timeout) @@ -401,14 +405,15 @@ def connect(self, clean_session=True, host=None, port=None, keepalive=None): """ if host is not None: - self._broker = host + self.broker = host if port is not None: - self._port = port + self.port = port - self.logger.debug("Attempting to establish MQTT connection...") + if self.logger: + self.logger.debug("Attempting to establish MQTT connection...") # Attempt to get a new socket - self._sock = self._get_socket(host, port) + self._sock = self._get_socket(self.broker, self.port) # Fixed Header fixed_header = bytearray([0x10]) From 964ac3675e90734197e5b94ef8c8520f2c756f6f Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 18 Nov 2020 14:46:23 -0500 Subject: [PATCH 13/77] specify exception, cpython logging second for circuitpy clients --- adafruit_minimqtt/adafruit_minimqtt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 42aa707b..0b9a7267 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -54,10 +54,10 @@ import time from random import randint from micropython import const -try: - import logging -except: +try: # try circuitpython logging module import adafruit_logging as logging +except ImportError: # try cpython logging + import logging from .matcher import MQTTMatcher __version__ = "0.0.0-auto.0" From 966bae7f2245bbe28a3c0dc04685bf850fae7cf0 Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 18 Nov 2020 15:19:08 -0500 Subject: [PATCH 14/77] encode topic as bytes, wait_for_msg BlockingIOError --- adafruit_minimqtt/adafruit_minimqtt.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 0b9a7267..982cb07e 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -693,7 +693,8 @@ def subscribe(self, topic, qos=0): for t, q in topics: topic_size = len(t).to_bytes(2, "big") qos_byte = q.to_bytes(1, "big") - packet += topic_size + t + qos_byte + #print(packet, "\n", topic_size, "\n", t, "\n", qos_byte) + packet += topic_size + t.encode() + qos_byte if self.logger is not None: for t, q in topics: self.logger.debug("SUBSCRIBING to topic {0} with QoS {1}".format(t, q)) @@ -814,15 +815,19 @@ def loop(self): ) self.ping() self._timestamp = 0 - self._sock.settimeout(0.1) + #self._sock.settimeout(0.0) return self._wait_for_msg() - def _wait_for_msg(self, timeout=30): - """Reads and processes network events. - Returns response code if successful. - """ + def _wait_for_msg(self): + """Reads and processes network events.""" + self._sock.setblocking(False) + print('Sock Blocking: ', self._sock.getblocking()) + print('Sock timeout: ', self._sock.gettimeout()) res = self._sock.recv(1) - self._sock.settimeout(timeout) + print("Res: ", res) + + self._sock.setblocking(True) + if res in [None, b""]: return None if res == MQTT_PINGRESP: From f1e3fbe782e8e0994b45cdc610ae52b843305c93 Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 18 Nov 2020 16:00:29 -0500 Subject: [PATCH 15/77] fix BlockingIOerror, not calling on_message? --- adafruit_minimqtt/adafruit_minimqtt.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 982cb07e..0824cabf 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -815,28 +815,31 @@ def loop(self): ) self.ping() self._timestamp = 0 - #self._sock.settimeout(0.0) return self._wait_for_msg() def _wait_for_msg(self): """Reads and processes network events.""" + res = bytearray(1) #TODO: This should be a globally shared buffer for readinto + self._sock.setblocking(False) - print('Sock Blocking: ', self._sock.getblocking()) - print('Sock timeout: ', self._sock.gettimeout()) - res = self._sock.recv(1) + try: + self._sock.recv_into(res, 1) + except BlockingIOError: # fix for macOS Errno + return None print("Res: ", res) - self._sock.setblocking(True) - if res in [None, b""]: return None if res == MQTT_PINGRESP: sz = self._sock.recv(1)[0] + print("got ping!") assert sz == 0 return None if res[0] & 0xF0 != 0x30: return res[0] + print("obtaining sz") sz = self._recv_len() + print("sz: ", sz) topic_len = self._sock.recv(2) topic_len = (topic_len[0] << 8) | topic_len[1] topic = self._sock.recv(topic_len) @@ -847,6 +850,7 @@ def _wait_for_msg(self): pid = pid[0] << 0x08 | pid[1] sz -= 0x02 msg = self._sock.recv(sz) + print(msg, topic) self._handle_on_message(self, topic, str(msg, "utf-8")) if res[0] & 0x06 == 0x02: pkt = bytearray(b"\x40\x02\0\0") From a43035e7111acc112e2c9e9fda66db6ae5b234ff Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 18 Nov 2020 16:06:13 -0500 Subject: [PATCH 16/77] on_message works with subscriptions --- adafruit_minimqtt/adafruit_minimqtt.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 0824cabf..31804549 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -826,20 +826,16 @@ def _wait_for_msg(self): self._sock.recv_into(res, 1) except BlockingIOError: # fix for macOS Errno return None - print("Res: ", res) self._sock.setblocking(True) if res in [None, b""]: return None if res == MQTT_PINGRESP: sz = self._sock.recv(1)[0] - print("got ping!") assert sz == 0 return None if res[0] & 0xF0 != 0x30: return res[0] - print("obtaining sz") sz = self._recv_len() - print("sz: ", sz) topic_len = self._sock.recv(2) topic_len = (topic_len[0] << 8) | topic_len[1] topic = self._sock.recv(topic_len) @@ -850,7 +846,6 @@ def _wait_for_msg(self): pid = pid[0] << 0x08 | pid[1] sz -= 0x02 msg = self._sock.recv(sz) - print(msg, topic) self._handle_on_message(self, topic, str(msg, "utf-8")) if res[0] & 0x06 == 0x02: pkt = bytearray(b"\x40\x02\0\0") From 7cdcc32277ebff708448a84a6afe28efda4d3898 Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 18 Nov 2020 16:17:15 -0500 Subject: [PATCH 17/77] encode topic within unsubscribe --- adafruit_minimqtt/adafruit_minimqtt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 31804549..1624baff 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -755,7 +755,7 @@ def unsubscribe(self, topic): packet = MQTT_UNSUB + packet_length_byte + packet_id_bytes for t in topics: topic_size = len(t).to_bytes(2, "big") - packet += topic_size + t + packet += topic_size + t.encode() if self.logger is not None: for t in topics: self.logger.debug("UNSUBSCRIBING from topic {0}.".format(t)) From baa1fc43701cc210e49cb7388fe8b12f3717aa4d Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 18 Nov 2020 16:19:09 -0500 Subject: [PATCH 18/77] remove is not none statement from logger calls --- adafruit_minimqtt/adafruit_minimqtt.py | 30 +++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 1624baff..a93e9d78 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -319,7 +319,7 @@ def will_set(self, topic=None, payload=None, qos=0, retain=False): :param bool retain: Specifies if the payload is to be retained when it is published. """ - if self.logger is not None: + if self.logger: self.logger.debug("Setting last will properties") self._check_qos(qos) if self._is_connected: @@ -457,7 +457,7 @@ def connect(self, clean_session=True, host=None, port=None, keepalive=None): fixed_header.append(remaining_length) fixed_header.append(0x00) - if self.logger is not None: + if self.logger: self.logger.debug("Sending CONNECT to broker") self.logger.debug( "Fixed Header: {}\nVariable Header: {}".format(fixed_header, var_header) @@ -475,7 +475,7 @@ def connect(self, clean_session=True, host=None, port=None, keepalive=None): else: self._send_str(self._username) self._send_str(self._password) - if self.logger is not None: + if self.logger: self.logger.debug("Receiving CONNACK packet from broker") while True: op = self._wait_for_msg() @@ -494,10 +494,10 @@ def disconnect(self): """Disconnects the MiniMQTT client from the MQTT broker. """ self.is_connected() - if self.logger is not None: + if self.logger: self.logger.debug("Sending DISCONNECT packet to broker") self._sock.send(MQTT_DISCONNECT) - if self.logger is not None: + if self.logger: self.logger.debug("Closing socket") self._sock.close() self._is_connected = False @@ -510,10 +510,10 @@ def ping(self): there is an active network connection. """ self.is_connected() - if self.logger is not None: + if self.logger: self.logger.debug("Sending PINGREQ") self._sock.send(MQTT_PINGREQ) - if self.logger is not None: + if self.logger: self.logger.debug("Checking PINGRESP") while True: op = self._wait_for_msg(0.5) @@ -600,7 +600,7 @@ def publish(self, topic, msg, retain=False, qos=0): else: pub_hdr_fixed.append(remaining_length) - if self.logger is not None: + if self.logger: self.logger.debug( "Sending PUBLISH\nTopic: {0}\nMsg: {1}\ \nQoS: {2}\nRetain? {3}".format( @@ -695,7 +695,7 @@ def subscribe(self, topic, qos=0): qos_byte = q.to_bytes(1, "big") #print(packet, "\n", topic_size, "\n", t, "\n", qos_byte) packet += topic_size + t.encode() + qos_byte - if self.logger is not None: + if self.logger: for t, q in topics: self.logger.debug("SUBSCRIBING to topic {0} with QoS {1}".format(t, q)) self._sock.send(packet) @@ -756,11 +756,11 @@ def unsubscribe(self, topic): for t in topics: topic_size = len(t).to_bytes(2, "big") packet += topic_size + t.encode() - if self.logger is not None: + if self.logger: for t in topics: self.logger.debug("UNSUBSCRIBING from topic {0}.".format(t)) self._sock.send(packet) - if self.logger is not None: + if self.logger: self.logger.debug("Waiting for UNSUBACK...") while True: op = self._wait_for_msg() @@ -783,13 +783,13 @@ def reconnect(self, resub_topics=True): :param bool resub_topics: Resubscribe to previously subscribed topics. """ - if self.logger is not None: + if self.logger: self.logger.debug("Attempting to reconnect with MQTT broker") self.connect() - if self.logger is not None: + if self.logger: self.logger.debug("Reconnected with broker") if resub_topics: - if self.logger is not None: + if self.logger: self.logger.debug( "Attempting to resubscribe to previously subscribed topics." ) @@ -808,7 +808,7 @@ def loop(self): current_time = time.monotonic() if current_time - self._timestamp >= self.keep_alive: # Handle KeepAlive by expecting a PINGREQ/PINGRESP from the server - if self.logger is not None: + if self.logger: self.logger.debug( "KeepAlive period elapsed - \ requesting a PINGRESP from the server..." From 0c05e65bfc4c33cf970867986573a2e4f826d799 Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 18 Nov 2020 16:20:02 -0500 Subject: [PATCH 19/77] remove positional call to _wait_for_msg --- adafruit_minimqtt/adafruit_minimqtt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index a93e9d78..291d19eb 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -516,7 +516,7 @@ def ping(self): if self.logger: self.logger.debug("Checking PINGRESP") while True: - op = self._wait_for_msg(0.5) + op = self._wait_for_msg() if op == 208: ping_resp = self._sock.recv(2) if ping_resp[0] != 0x00: From 1a65b927dd5377bf1f9441780dcfbb693fc8171b Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 18 Nov 2020 17:15:44 -0500 Subject: [PATCH 20/77] pylint minimqtt, add simpletest for cpython --- adafruit_minimqtt/adafruit_minimqtt.py | 27 ++++--- examples/minimqtt_simpletest_cpython.py | 98 +++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 14 deletions(-) create mode 100644 examples/minimqtt_simpletest_cpython.py diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 291d19eb..1f83cb89 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -394,20 +394,22 @@ def username_pw_set(self, username, password=None): self._password = password # pylint: disable=too-many-branches, too-many-statements, too-many-locals - def connect(self, clean_session=True, host=None, port=None, keepalive=None): + def connect(self, clean_session=True, host=None, port=None, keep_alive=None): """Initiates connection with the MQTT Broker. :param bool clean_session: Establishes a persistent session. :param str host: Hostname or IP address of the remote broker. :param int port: Network port of the remote broker. - :param int keepalive: Maximum period allowed for communication + :param int keep_alive: Maximum period allowed for communication with the broker, in seconds """ - if host is not None: + if host: self.broker = host - if port is not None: + if port: self.port = port + if keep_alive: + self.keep_alive = keep_alive if self.logger: self.logger.debug("Attempting to establish MQTT connection...") @@ -427,7 +429,7 @@ def connect(self, clean_session=True, host=None, port=None, keepalive=None): # Set up variable header and remaining_length remaining_length = 12 + len(self.client_id) - if self._username is not None: + if self._username: remaining_length += 2 + len(self._username) + 2 + len(self._password) var_header[6] |= 0xC0 if self.keep_alive: @@ -458,10 +460,8 @@ def connect(self, clean_session=True, host=None, port=None, keepalive=None): fixed_header.append(0x00) if self.logger: - self.logger.debug("Sending CONNECT to broker") - self.logger.debug( - "Fixed Header: {}\nVariable Header: {}".format(fixed_header, var_header) - ) + self.logger.debug("Sending CONNECT to broker...") + self.logger.debug("Fixed Header: %x\nVariable Header: %x", fixed_header, var_header) self._sock.send(fixed_header) self._sock.send(var_header) # [MQTT-3.1.3-4] @@ -602,10 +602,9 @@ def publish(self, topic, msg, retain=False, qos=0): if self.logger: self.logger.debug( - "Sending PUBLISH\nTopic: {0}\nMsg: {1}\ - \nQoS: {2}\nRetain? {3}".format( + "Sending PUBLISH\nTopic: %s\nMsg: %x\ + \nQoS: %d\nRetain? %r", topic, msg, qos, retain - ) ) self._sock.send(pub_hdr_fixed) self._sock.send(pub_hdr_var) @@ -697,7 +696,7 @@ def subscribe(self, topic, qos=0): packet += topic_size + t.encode() + qos_byte if self.logger: for t, q in topics: - self.logger.debug("SUBSCRIBING to topic {0} with QoS {1}".format(t, q)) + self.logger.debug("SUBSCRIBING to topic %s with QoS %d", t, q) self._sock.send(packet) while True: op = self._wait_for_msg() @@ -758,7 +757,7 @@ def unsubscribe(self, topic): packet += topic_size + t.encode() if self.logger: for t in topics: - self.logger.debug("UNSUBSCRIBING from topic {0}.".format(t)) + self.logger.debug("UNSUBSCRIBING from topic %s", t) self._sock.send(packet) if self.logger: self.logger.debug("Waiting for UNSUBACK...") diff --git a/examples/minimqtt_simpletest_cpython.py b/examples/minimqtt_simpletest_cpython.py new file mode 100644 index 00000000..ba0d2094 --- /dev/null +++ b/examples/minimqtt_simpletest_cpython.py @@ -0,0 +1,98 @@ +import socket +import adafruit_minimqtt.adafruit_minimqtt as MQTT + +### Secrets Setup ### + +try: + from secrets import secrets +except ImportError: + print("Connection secrets are kept in secrets.py, please add them there!") + raise + +### Topic Setup ### + +# MQTT Topic +# Use this topic if you'd like to connect to a standard MQTT broker +#mqtt_topic = "test/topic" + +# Adafruit IO-style Topic +# Use this topic if you'd like to connect to io.adafruit.com +mqtt_topic = 'brubell/feeds/temperature' + +### Code ### + +# Keep track of client connection state +disconnect_client = False + +# Define callback methods which are called when events occur +# pylint: disable=unused-argument, redefined-outer-name +def connect(mqtt_client, userdata, flags, rc): + # This function will be called when the mqtt_client is connected + # successfully to the broker. + print("Connected to MQTT Broker!") + print("Flags: {0}\n RC: {1}".format(flags, rc)) + + +def disconnect(mqtt_client, userdata, rc): + # This method is called when the mqtt_client disconnects + # from the broker. + print("Disconnected from MQTT Broker!") + + +def subscribe(mqtt_client, userdata, topic, granted_qos): + # This method is called when the mqtt_client subscribes to a new feed. + print("Subscribed to {0} with QOS level {1}".format(topic, granted_qos)) + + +def unsubscribe(mqtt_client, userdata, topic, pid): + # This method is called when the mqtt_client unsubscribes from a feed. + print("Unsubscribed from {0} with PID {1}".format(topic, pid)) + + +def publish(mqtt_client, userdata, topic, pid): + # This method is called when the mqtt_client publishes data to a feed. + print("Published to {0} with PID {1}".format(topic, pid)) + + +def message(client, topic, message): + # Method callled when a client's subscribed feed has a new value. + global disconnect_client + print("New message on topic {0}: {1}".format(topic, message)) + + print("Unsubscribing from %s" % mqtt_topic) + mqtt_client.unsubscribe(mqtt_topic) + # Allow us to gracefully stop the `while True` loop + disconnect_client = True + +# Set up a MiniMQTT Client +mqtt_client = MQTT.MQTT( + broker=secrets['broker'], + port=secrets['port'], + username=secrets['aio_username'], + password=secrets['aio_key'], + socket_pool=socket +) + +# Connect callback handlers to mqtt_client +mqtt_client.on_connect = connect +mqtt_client.on_disconnect = disconnect +mqtt_client.on_subscribe = subscribe +mqtt_client.on_unsubscribe = unsubscribe +mqtt_client.on_publish = publish +mqtt_client.on_message = message + +print("Attempting to connect to %s" % mqtt_client.broker) +mqtt_client.connect() + +print("Subscribing to %s" % mqtt_topic) +mqtt_client.subscribe(mqtt_topic) + +print("Publishing to %s" % mqtt_topic) +mqtt_client.publish(mqtt_topic, "Hello Broker!") + +# Pump the loop until we receive a message on `mqtt_topic` +while disconnect_client == False: + mqtt_client.loop() + +print("Disconnecting from %s" % mqtt_client.broker) +mqtt_client.disconnect() From cb322ed5097949a22912f68504ce783630400e18 Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 18 Nov 2020 17:18:55 -0500 Subject: [PATCH 21/77] use secrets' username --- examples/minimqtt_simpletest_cpython.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/minimqtt_simpletest_cpython.py b/examples/minimqtt_simpletest_cpython.py index ba0d2094..a5408320 100644 --- a/examples/minimqtt_simpletest_cpython.py +++ b/examples/minimqtt_simpletest_cpython.py @@ -17,7 +17,7 @@ # Adafruit IO-style Topic # Use this topic if you'd like to connect to io.adafruit.com -mqtt_topic = 'brubell/feeds/temperature' +mqtt_topic = secrets["aio_username"] + '/feeds/temperature' ### Code ### From 1d78c0f5fcd3930f91674f9b2e6d577a02fed1b3 Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 18 Nov 2020 17:26:40 -0500 Subject: [PATCH 22/77] add cpython example for AIO, tested --- examples/minimqtt_adafruitio_cpython.py | 75 +++++++++++++++++++++++++ examples/minimqtt_simpletest_cpython.py | 2 +- 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 examples/minimqtt_adafruitio_cpython.py diff --git a/examples/minimqtt_adafruitio_cpython.py b/examples/minimqtt_adafruitio_cpython.py new file mode 100644 index 00000000..b4a853b1 --- /dev/null +++ b/examples/minimqtt_adafruitio_cpython.py @@ -0,0 +1,75 @@ +# Adafruit MiniMQTT Pub/Sub Example +# Written by Tony DiCola for Adafruit Industries +# Modified by Brent Rubell for Adafruit Industries +import time +import socket +import adafruit_minimqtt.adafruit_minimqtt as MQTT + +### Secrets File Setup ### + +try: + from secrets import secrets +except ImportError: + print("Connection secrets are kept in secrets.py, please add them there!") + raise + +### Feeds ### + +# Setup a feed named 'photocell' for publishing to a feed +photocell_feed = secrets["aio_username"] + "/feeds/photocell" + +# Setup a feed named 'onoff' for subscribing to changes +onoff_feed = secrets["aio_username"] + "/feeds/onoff" + +### Code ### + +# Define callback methods which are called when events occur +# pylint: disable=unused-argument, redefined-outer-name +def connected(client, userdata, flags, rc): + # This function will be called when the client is connected + # successfully to the broker. + print("Connected to Adafruit IO! Listening for topic changes on %s" % onoff_feed) + # Subscribe to all changes on the onoff_feed. + client.subscribe(onoff_feed) + + +def disconnected(client, userdata, rc): + # This method is called when the client is disconnected + print("Disconnected from Adafruit IO!") + + +def message(client, topic, message): + # This method is called when a topic the client is subscribed to + # has a new message. + print("New message on topic {0}: {1}".format(topic, message)) + + +# Set up a MiniMQTT Client +mqtt_client = MQTT.MQTT( + broker=secrets['broker'], + port=secrets['port'], + username=secrets['aio_username'], + password=secrets['aio_key'], + socket_pool=socket +) + +# Setup the callback methods above +mqtt_client.on_connect = connected +mqtt_client.on_disconnect = disconnected +mqtt_client.on_message = message + +# Connect the client to the MQTT broker. +print("Connecting to Adafruit IO...") +mqtt_client.connect() + +photocell_val = 0 +while True: + # Poll the message queue + mqtt_client.loop() + + # Send a new message + print("Sending photocell value: %d..." % photocell_val) + mqtt_client.publish(photocell_feed, photocell_val) + print("Sent!") + photocell_val += 1 + time.sleep(1) diff --git a/examples/minimqtt_simpletest_cpython.py b/examples/minimqtt_simpletest_cpython.py index a5408320..64aeea66 100644 --- a/examples/minimqtt_simpletest_cpython.py +++ b/examples/minimqtt_simpletest_cpython.py @@ -1,7 +1,7 @@ import socket import adafruit_minimqtt.adafruit_minimqtt as MQTT -### Secrets Setup ### +### Secrets File Setup ### try: from secrets import secrets From 4af0ad0d954c2d1194f4d0fd8333a3dbae9d8b8c Mon Sep 17 00:00:00 2001 From: brentru Date: Thu, 19 Nov 2020 14:24:16 -0500 Subject: [PATCH 23/77] socket exists, but ssl.SSLWantReadError persists? --- adafruit_minimqtt/adafruit_minimqtt.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 1f83cb89..4f50d4e0 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -145,10 +145,9 @@ def __init__( ssl_context=None ): # Socket Pool - if socket_pool is not None: + if socket_pool: self._socket_pool = socket_pool - if ssl_context is not None: - self._ssl_context = ssl_context + self._ssl_context = ssl_context # Hang onto open sockets so that we can reuse them self._socket_free = {} self._open_sockets = {} @@ -165,7 +164,7 @@ def __init__( self._username = username self._password = password if ( - self._password is not None + self._password and len(password.encode("utf-8")) > MQTT_TOPIC_LENGTH_LIMIT ): # [MQTT-3.1.3.5] raise MMQTTException("Password length is too large.") @@ -173,11 +172,11 @@ def __init__( self.port = MQTT_TCP_PORT if is_ssl: self.port = MQTT_TLS_PORT - if port is not None: + if port: self.port = port # define client identifer - if client_id is not None: + if client_id: # user-defined client_id MAY allow client_id's > 23 bytes or # non-alpha-numeric characters self.client_id = client_id @@ -252,7 +251,6 @@ def _get_socket(self, host, port, *, timeout=1): raise RuntimeError( "ssl_context must be set before using adafruit_mqtt for secure MQTT." ) - print(key) addr_info = self._socket_pool.getaddrinfo( host, port, 0, self._socket_pool.SOCK_STREAM )[0] From fbfb6ec506616a3cf5ffd544757c4aa05e6a2bf5 Mon Sep 17 00:00:00 2001 From: brentru Date: Thu, 19 Nov 2020 14:58:37 -0500 Subject: [PATCH 24/77] CircuitPython lacks select() module, use socket timeouttype instead of non-blocking SSL socket to avoid SSLWantWrite/SSLWantRead errors, https://docs.python.org/3/library/ssl.html#notes-on-non-blocking-sockets --- adafruit_minimqtt/adafruit_minimqtt.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 4f50d4e0..61dd523d 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -273,7 +273,8 @@ def _get_socket(self, host, port, *, timeout=1): connect_host = addr_info[-1][0] if port == 8883: - sock = self._ssl_context.wrap_socket(sock, server_hostname=host) + sock = self._ssl_context.wrap_socket(sock, + server_hostname=host) connect_host = host sock.settimeout(timeout) @@ -796,9 +797,11 @@ def reconnect(self, resub_topics=True): feed = subscribed_topics.pop() self.subscribe(feed) - def loop(self): + def loop(self, timeout=0.01): """Non-blocking message loop. Use this method to check incoming subscription messages. + :param float timeout: Set timeout in seconds for + polling the message queue. """ if self._timestamp == 0: self._timestamp = time.monotonic() @@ -812,17 +815,20 @@ def loop(self): ) self.ping() self._timestamp = 0 - return self._wait_for_msg() + return self._wait_for_msg(timeout) - def _wait_for_msg(self): + def _wait_for_msg(self, timeout=0.01): """Reads and processes network events.""" res = bytearray(1) #TODO: This should be a globally shared buffer for readinto - self._sock.setblocking(False) + self._sock.settimeout(timeout) try: self._sock.recv_into(res, 1) except BlockingIOError: # fix for macOS Errno return None + except self._socket_pool.timeout: + return None + self._sock.setblocking(True) if res in [None, b""]: return None From 8492a58f88465148268546fd58712c41208dee6c Mon Sep 17 00:00:00 2001 From: brentru Date: Thu, 19 Nov 2020 15:23:51 -0500 Subject: [PATCH 25/77] backwards compat. recv, adding shared buffer impl to save memory --- adafruit_minimqtt/adafruit_minimqtt.py | 28 +++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 61dd523d..23a21319 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -144,15 +144,15 @@ def __init__( socket_pool=None, ssl_context=None ): - # Socket Pool - if socket_pool: - self._socket_pool = socket_pool + + self._socket_pool = socket_pool self._ssl_context = ssl_context # Hang onto open sockets so that we can reuse them self._socket_free = {} self._open_sockets = {} - self._sock = None + self._backwards_compatible_sock = False + self.keep_alive = keep_alive self._user_data = None self._is_connected = False @@ -290,6 +290,8 @@ def _get_socket(self, host, port, *, timeout=1): if sock is None: raise RuntimeError("Repeated socket failures") + self._backwards_compatible_sock = not hasattr(sock, "recv_into") + self._open_sockets[key] = sock self._socket_free[sock] = False return sock @@ -819,15 +821,16 @@ def loop(self, timeout=0.01): def _wait_for_msg(self, timeout=0.01): """Reads and processes network events.""" - res = bytearray(1) #TODO: This should be a globally shared buffer for readinto + # attempt to recv from socket within `timeout` seconds self._sock.settimeout(timeout) try: + res = bytearray(1) #TODO: This should be a globally shared buffer for readinto self._sock.recv_into(res, 1) - except BlockingIOError: # fix for macOS Errno - return None except self._socket_pool.timeout: return None + except BlockingIOError: # fixes macOS socket Errno 35 + return None self._sock.setblocking(True) if res in [None, b""]: @@ -868,6 +871,17 @@ def _recv_len(self): return n sh += 7 + def _recv_into(self, buf, size=0): + """Backwards-compatible _recv_into implementation. + """ + if self._backwards_compatible_sock: + size = len(buf) if size == 0 else size + b = self._sock.recv(size) + read_size = len(b) + buf[:read_size] = b + return read_size + return self._sock.recv_into(buf, size) + def _send_str(self, string): """Packs and encodes a string to a socket. From 984c6004625658e7899fcbb713b43ae8e492efb3 Mon Sep 17 00:00:00 2001 From: brentru Date: Thu, 19 Nov 2020 15:46:30 -0500 Subject: [PATCH 26/77] add shared buffer, connect tested --- adafruit_minimqtt/adafruit_minimqtt.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 23a21319..89241915 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -212,6 +212,10 @@ def __init__( self.on_subscribe = None self.on_unsubscribe = None + # Shared buffer + self._rx_length = 0 + self._rx_buffer = bytearray(32) + # Socket helpers def _free_socket(self, socket): """Frees a socket for re-use.""" @@ -405,6 +409,7 @@ def connect(self, clean_session=True, host=None, port=None, keep_alive=None): with the broker, in seconds """ + buf = self._rx_buffer if host: self.broker = host if port: @@ -481,14 +486,14 @@ def connect(self, clean_session=True, host=None, port=None, keep_alive=None): while True: op = self._wait_for_msg() if op == 32: - rc = self._sock.recv(3) - assert rc[0] == 0x02 - if rc[2] != 0x00: - raise MMQTTException(CONNACK_ERRORS[rc[2]]) + self._recv_into(buf, 3) + assert buf[0] == 0x02 + if buf[2] != 0x00: + raise MMQTTException(CONNACK_ERRORS[buf[2]]) self._is_connected = True - result = rc[0] & 1 + result = buf[0] & 1 if self.on_connect is not None: - self.on_connect(self, self._user_data, result, rc[2]) + self.on_connect(self, self._user_data, result, buf[2]) return result def disconnect(self): From a907575b145d457b0b93c18c9c0ef3eb40914526 Mon Sep 17 00:00:00 2001 From: brentru Date: Thu, 19 Nov 2020 15:54:11 -0500 Subject: [PATCH 27/77] public command methods use shared rx buffer --- adafruit_minimqtt/adafruit_minimqtt.py | 39 +++++++++++++++----------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 89241915..b4d0b88d 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -505,7 +505,7 @@ def disconnect(self): self._sock.send(MQTT_DISCONNECT) if self.logger: self.logger.debug("Closing socket") - self._sock.close() + self._free_sockets() self._is_connected = False self._subscribed_topics = None if self.on_disconnect is not None: @@ -516,6 +516,7 @@ def ping(self): there is an active network connection. """ self.is_connected() + buf = self._rx_buffer if self.logger: self.logger.debug("Sending PINGREQ") self._sock.send(MQTT_PINGREQ) @@ -524,8 +525,8 @@ def ping(self): while True: op = self._wait_for_msg() if op == 208: - ping_resp = self._sock.recv(2) - if ping_resp[0] != 0x00: + self._recv_into(buf, 2) + if buf[0] != 0x00: raise MMQTTException("PINGRESP not returned from broker.") return @@ -579,6 +580,7 @@ def publish(self, topic, msg, retain=False, qos=0): 0 <= qos <= 1 ), "Quality of Service Level 2 is unsupported by this library." + buf = self._rx_buffer # fixed header. [3.3.1.2], [3.3.1.3] pub_hdr_fixed = bytearray([0x30 | retain | qos << 1]) @@ -621,13 +623,14 @@ def publish(self, topic, msg, retain=False, qos=0): while True: op = self._wait_for_msg() if op == 0x40: - sz = self._sock.recv(1) - assert sz == b"\x02" - rcv_pid = self._sock.recv(2) - rcv_pid = rcv_pid[0] << 0x08 | rcv_pid[1] - if pid == rcv_pid: + self._recv_into(buf, 1) + assert buf == b"\x02" + # rcv_pid = self._sock.recv(2) + self._recv_into(buf, 2) + buf = buf[0] << 0x08 | buf[1] + if pid == buf: if self.on_publish is not None: - self.on_publish(self, self._user_data, topic, rcv_pid) + self.on_publish(self, self._user_data, topic, buf) return def subscribe(self, topic, qos=0): @@ -686,6 +689,8 @@ def subscribe(self, topic, qos=0): self._check_qos(q) self._check_topic(t) topics.append((t, q)) + # Rx buffer + buf = self._rx_buffer # Assemble packet packet_length = 2 + (2 * len(topics)) + (1 * len(topics)) packet_length += sum(len(topic) for topic, qos in topics) @@ -707,9 +712,9 @@ def subscribe(self, topic, qos=0): while True: op = self._wait_for_msg() if op == 0x90: - rc = self._sock.recv(4) - assert rc[1] == packet[2] and rc[2] == packet[3] - if rc[3] == 0x80: + self._recv_into(buf, 4) + assert buf[1] == packet[2] and buf[2] == packet[3] + if buf[3] == 0x80: raise MMQTTException("SUBACK Failure!") for t, q in topics: if self.on_subscribe is not None: @@ -751,6 +756,8 @@ def unsubscribe(self, topic): raise MMQTTException( "Topic must be subscribed to before attempting unsubscribe." ) + # Rx buffer + buf = self._rx_buffer # Assemble packet packet_length = 2 + (2 * len(topics)) packet_length += sum(len(topic) for topic in topics) @@ -770,12 +777,12 @@ def unsubscribe(self, topic): while True: op = self._wait_for_msg() if op == 176: - return_code = self._sock.recv(3) - assert return_code[0] == 0x02 + self._recv_into(buf, 3) + assert buf[0] == 0x02 # [MQTT-3.32] assert ( - return_code[1] == packet_id_bytes[0] - and return_code[2] == packet_id_bytes[1] + buf[1] == packet_id_bytes[0] + and buf[2] == packet_id_bytes[1] ) for t in topics: if self.on_unsubscribe is not None: From 2ae61a459dd2842981a15b63f0e60c0a84646749 Mon Sep 17 00:00:00 2001 From: brentru Date: Fri, 20 Nov 2020 14:17:46 -0500 Subject: [PATCH 28/77] convert recv_len into recv_into --- adafruit_minimqtt/adafruit_minimqtt.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index b4d0b88d..dce560dd 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -876,10 +876,11 @@ def _wait_for_msg(self, timeout=0.01): def _recv_len(self): n = 0 sh = 0 + b = bytearray(1) while True: - b = self._sock.recv(1)[0] - n |= (b & 0x7F) << sh - if not b & 0x80: + self._recv_into(b, 1) + n |= (b[0] & 0x7F) << sh + if not b[0] & 0x80: return n sh += 7 From 6ebf23358fee3b0fbdcd69c9bb2ca6e25a3b603f Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 23 Nov 2020 15:57:03 -0500 Subject: [PATCH 29/77] recv->recv_into calls --- adafruit_minimqtt/adafruit_minimqtt.py | 36 +++++++++++++++++--------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index dce560dd..70b1a629 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -833,18 +833,23 @@ def loop(self, timeout=0.01): def _wait_for_msg(self, timeout=0.01): """Reads and processes network events.""" - # attempt to recv from socket within `timeout` seconds - self._sock.settimeout(timeout) + self._sock.settimeout(0) + try: - res = bytearray(1) #TODO: This should be a globally shared buffer for readinto - self._sock.recv_into(res, 1) - except self._socket_pool.timeout: - return None - except BlockingIOError: # fixes macOS socket Errno 35 + res = bytearray(1) + self._recv_into(res, 1) + print(res) + except OSError: return None - self._sock.setblocking(True) + # TODO: Re-implement these? + # except self._sock.timeout: + # return None + # except BlockingIOError: # fixes macOS socket Errno 35 + # return None + + self._sock.settimeout(0) if res in [None, b""]: return None if res == MQTT_PINGRESP: @@ -854,16 +859,23 @@ def _wait_for_msg(self, timeout=0.01): if res[0] & 0xF0 != 0x30: return res[0] sz = self._recv_len() - topic_len = self._sock.recv(2) + # topic length MSB & LSB + topic_len = bytearray(2) + self._recv_into(topic_len, 2) topic_len = (topic_len[0] << 8) | topic_len[1] - topic = self._sock.recv(topic_len) + topic = bytearray(topic_len) + self._recv_into(topic, topic_len) topic = str(topic, "utf-8") sz -= topic_len + 2 if res[0] & 0x06: - pid = self._sock.recv(2) + pid = bytearray(2) + self._recv_into(pid, 2) pid = pid[0] << 0x08 | pid[1] sz -= 0x02 - msg = self._sock.recv(sz) + + # TODO: Potentially resize buffer here if > 32bytes + msg = bytearray(sz) + self._recv_into(msg, len(msg)) self._handle_on_message(self, topic, str(msg, "utf-8")) if res[0] & 0x06 == 0x02: pkt = bytearray(b"\x40\x02\0\0") From 53563185c15c87c40fabece90bb9050fd96ccd3c Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 23 Nov 2020 16:29:44 -0500 Subject: [PATCH 30/77] resize the rx buffer dynamically --- adafruit_minimqtt/adafruit_minimqtt.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 70b1a629..5be5682a 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -376,15 +376,15 @@ def on_message(self): def on_message(self, method): self._on_message = method - def _handle_on_message(self, client, topic, message): + def _handle_on_message(self, client, topic): matched = False if topic is not None: for callback in self._on_message_filtered.iter_match(topic): - callback(client, topic, message) # on_msg with callback + callback(client, topic, self._rx_buffer.decode()) # on_msg with callback matched = True if not matched and self.on_message: # regular on_message - self.on_message(client, topic, message) + self.on_message(client, topic, self._rx_buffer.decode()) def username_pw_set(self, username, password=None): """Set client's username and an optional password. @@ -833,6 +833,7 @@ def loop(self, timeout=0.01): def _wait_for_msg(self, timeout=0.01): """Reads and processes network events.""" + buf = self._rx_buffer # attempt to recv from socket within `timeout` seconds self._sock.settimeout(0) @@ -873,10 +874,18 @@ def _wait_for_msg(self, timeout=0.01): pid = pid[0] << 0x08 | pid[1] sz -= 0x02 - # TODO: Potentially resize buffer here if > 32bytes - msg = bytearray(sz) - self._recv_into(msg, len(msg)) - self._handle_on_message(self, topic, str(msg, "utf-8")) + # if expected packet size is larger than buffer + if sz > len(buf): + # resize buffer to match expected size + new_size = sz + len(buf) + new_buf = bytearray(new_size) + new_buf[: len(buf)] = buf + buf = new_buf + self._rx_buffer = buf + # read incoming packet + self._recv_into(self._rx_buffer, sz) + + self._handle_on_message(self, topic) if res[0] & 0x06 == 0x02: pkt = bytearray(b"\x40\x02\0\0") struct.pack_into("!H", pkt, 2, pid) @@ -885,6 +894,7 @@ def _wait_for_msg(self, timeout=0.01): assert 0 return res[0] + def _recv_len(self): n = 0 sh = 0 From b9740e355e6f526f14159243e3ab5ef380b32d6c Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 23 Nov 2020 16:43:40 -0500 Subject: [PATCH 31/77] add example for esp32s2 --- ...cpython => minimqtt_simpletest_esp32s2.py} | 70 +++++++++++++++---- 1 file changed, 56 insertions(+), 14 deletions(-) rename examples/{minimqtt_simpletest_cpython => minimqtt_simpletest_esp32s2.py} (53%) diff --git a/examples/minimqtt_simpletest_cpython b/examples/minimqtt_simpletest_esp32s2.py similarity index 53% rename from examples/minimqtt_simpletest_cpython rename to examples/minimqtt_simpletest_esp32s2.py index dc837622..5dc37416 100644 --- a/examples/minimqtt_simpletest_cpython +++ b/examples/minimqtt_simpletest_esp32s2.py @@ -1,16 +1,40 @@ -import socket - +# adafruit_minimqtt usage with native wifi +import ssl +from random import randint +import adafruit_requests +import socketpool +import wifi import adafruit_minimqtt.adafruit_minimqtt as MQTT -### WiFi ### - -# Get wifi details and more from a secrets.py file +# Add a secrets.py to your filesystem that has a dictionary called secrets with "ssid" and +# "password" keys with your WiFi credentials. DO NOT share that file or commit it into Git or other +# source control. +# pylint: disable=no-name-in-module,wrong-import-order try: from secrets import secrets except ImportError: print("WiFi secrets are kept in secrets.py, please add them there!") raise +# Set your Adafruit IO Username and Key in secrets.py +# (visit io.adafruit.com if you need to create an account, +# or if you need your Adafruit IO key.) +aio_username = secrets["aio_username"] +aio_key = secrets["aio_key"] + +print("Connecting to %s"%secrets["ssid"]) +wifi.radio.connect(secrets["ssid"], secrets["password"]) +print("Connected to %s!"%secrets["ssid"]) + + +### Secrets File Setup ### + +try: + from secrets import secrets +except ImportError: + print("Connection secrets are kept in secrets.py, please add them there!") + raise + ### Topic Setup ### # MQTT Topic @@ -19,10 +43,13 @@ # Adafruit IO-style Topic # Use this topic if you'd like to connect to io.adafruit.com -# mqtt_topic = 'aio_user/feeds/temperature' +# mqtt_topic = secrets["aio_username"] + '/feeds/temperature' ### Code ### +# Keep track of client connection state +disconnect_client = False + # Define callback methods which are called when events occur # pylint: disable=unused-argument, redefined-outer-name def connect(mqtt_client, userdata, flags, rc): @@ -53,12 +80,27 @@ def publish(mqtt_client, userdata, topic, pid): print("Published to {0} with PID {1}".format(topic, pid)) +def message(client, topic, message): + # Method callled when a client's subscribed feed has a new value. + global disconnect_client + print("New message on topic {0}: {1}".format(topic, message)) + + print("Unsubscribing from %s" % mqtt_topic) + mqtt_client.unsubscribe(mqtt_topic) + # Allow us to gracefully stop the `while True` loop + disconnect_client = True + +# Create a socket pool +pool = socketpool.SocketPool(wifi.radio) + # Set up a MiniMQTT Client mqtt_client = MQTT.MQTT( - broker=secrets["broker"], - username=secrets["user"], - password=secrets["pass"], - socket + broker=secrets['broker'], + port=secrets['port'], + username=secrets['aio_username'], + password=secrets['aio_key'], + socket_pool=pool, + ssl_context= ssl.create_default_context() ) # Connect callback handlers to mqtt_client @@ -67,10 +109,9 @@ def publish(mqtt_client, userdata, topic, pid): mqtt_client.on_subscribe = subscribe mqtt_client.on_unsubscribe = unsubscribe mqtt_client.on_publish = publish +mqtt_client.on_message = message print("Attempting to connect to %s" % mqtt_client.broker) -# TODO: Move config into connect call? -# broker=secrets["broker"], username=secrets["user"], password=secrets["pass"] mqtt_client.connect() print("Subscribing to %s" % mqtt_topic) @@ -79,8 +120,9 @@ def publish(mqtt_client, userdata, topic, pid): print("Publishing to %s" % mqtt_topic) mqtt_client.publish(mqtt_topic, "Hello Broker!") -print("Unsubscribing from %s" % mqtt_topic) -mqtt_client.unsubscribe(mqtt_topic) +# Pump the loop until we receive a message on `mqtt_topic` +while disconnect_client == False: + mqtt_client.loop() print("Disconnecting from %s" % mqtt_client.broker) mqtt_client.disconnect() From e7c48e37ac89a4f9c8e2095e5b4846578b4671dd Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 23 Nov 2020 16:54:02 -0500 Subject: [PATCH 32/77] no more blocking error since we are using timeouts --- adafruit_minimqtt/adafruit_minimqtt.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 5be5682a..f09c8575 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -840,16 +840,9 @@ def _wait_for_msg(self, timeout=0.01): try: res = bytearray(1) self._recv_into(res, 1) - print(res) except OSError: return None - # TODO: Re-implement these? - # except self._sock.timeout: - # return None - # except BlockingIOError: # fixes macOS socket Errno 35 - # return None - self._sock.settimeout(0) if res in [None, b""]: return None From 5b29419129e5bceb69493af17f4431dd45d0b130 Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 25 Nov 2020 14:38:31 -0500 Subject: [PATCH 33/77] legacy socket compat, untested secure port --- adafruit_minimqtt/adafruit_minimqtt.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index f09c8575..482f9745 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -255,6 +255,11 @@ def _get_socket(self, host, port, *, timeout=1): raise RuntimeError( "ssl_context must be set before using adafruit_mqtt for secure MQTT." ) + + # Legacy API - use a default socket instead of socket pool + if self._socket_pool is None: + self._socket_pool = _the_sock + addr_info = self._socket_pool.getaddrinfo( host, port, 0, self._socket_pool.SOCK_STREAM )[0] @@ -835,7 +840,8 @@ def _wait_for_msg(self, timeout=0.01): """Reads and processes network events.""" buf = self._rx_buffer # attempt to recv from socket within `timeout` seconds - self._sock.settimeout(0) + # TODO: This line is inconsistent btween versions! + self._sock.settimeout(timeout) try: res = bytearray(1) From 0a3ee4fe5bf65a94ae322cd9e2a0000bb47eab15 Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 25 Nov 2020 14:45:30 -0500 Subject: [PATCH 34/77] add fake SSL context, legacy api --- adafruit_minimqtt/adafruit_minimqtt.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 482f9745..3a99c8a0 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -114,6 +114,15 @@ def set_socket(sock, iface=None): _the_sock.set_interface(iface) +class _FakeSSLContext: + def __init__(self, iface): + self._iface = iface + + def wrap_socket(self, socket, server_hostname=None): + """Return the same socket""" + # pylint: disable=unused-argument + return _FakeSSLSocket(socket, self._iface.TLS_MODE) + class MQTT: """MQTT Client for CircuitPython @@ -146,7 +155,15 @@ def __init__( ): self._socket_pool = socket_pool + # Legacy API - if we do not have a socket pool, use default socket + if self._socket_pool is None: + self._socket_pool = _the_sock + self._ssl_context = ssl_context + # Legacy API - if we do not have SSL context, fake it + if self._ssl_context is None: + self._ssl_context = _FakeSSLContext(_the_interface) + # Hang onto open sockets so that we can reuse them self._socket_free = {} self._open_sockets = {} From 869a1ffb7fd9164279ec88bf38d15c6a64ee8b4e Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 25 Nov 2020 14:57:59 -0500 Subject: [PATCH 35/77] add fakessl socket, working on esp32, cpython --- adafruit_minimqtt/adafruit_minimqtt.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 3a99c8a0..6d8039f9 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -50,10 +50,12 @@ https://github.com/adafruit/circuitpython/releases """ +import errno import struct import time from random import randint from micropython import const +# TODO: Remove logging dependency, pass in the logger instead! try: # try circuitpython logging module import adafruit_logging as logging except ImportError: # try cpython logging @@ -89,17 +91,17 @@ const(0x05): "Connection Refused - Unauthorized", } - _the_interface = None # pylint: disable=invalid-name _the_sock = None # pylint: disable=invalid-name - class MMQTTException(Exception): """MiniMQTT Exception class.""" # pylint: disable=unnecessary-pass # pass +# Legacy Socket API + def set_socket(sock, iface=None): """Legacy API for setting the socket and network interface, use a `Session` instead. @@ -113,6 +115,21 @@ def set_socket(sock, iface=None): _the_interface = iface _the_sock.set_interface(iface) +class _FakeSSLSocket: + def __init__(self, socket, tls_mode): + self._socket = socket + self._mode = tls_mode + self.settimeout = socket.settimeout + self.send = socket.send + self.recv = socket.recv + self.close = socket.close + + def connect(self, address): + """connect wrapper to add non-standard mode parameter""" + try: + return self._socket.connect(address, self._mode) + except RuntimeError as error: + raise OSError(errno.ENOMEM) from error class _FakeSSLContext: def __init__(self, iface): @@ -123,6 +140,7 @@ def wrap_socket(self, socket, server_hostname=None): # pylint: disable=unused-argument return _FakeSSLSocket(socket, self._iface.TLS_MODE) + class MQTT: """MQTT Client for CircuitPython From 2560c577732532d9dcb32b4ffa84b9e15ea84273 Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 25 Nov 2020 15:25:43 -0500 Subject: [PATCH 36/77] expand enable_logger, remove logging dependency for library (yay!) --- adafruit_minimqtt/adafruit_minimqtt.py | 52 +++++++++++--------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 6d8039f9..28e341f5 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -55,11 +55,6 @@ import time from random import randint from micropython import const -# TODO: Remove logging dependency, pass in the logger instead! -try: # try circuitpython logging module - import adafruit_logging as logging -except ImportError: # try cpython logging - import logging from .matcher import MQTTMatcher __version__ = "0.0.0-auto.0" @@ -140,7 +135,6 @@ def wrap_socket(self, socket, server_hostname=None): # pylint: disable=unused-argument return _FakeSSLSocket(socket, self._iface.TLS_MODE) - class MQTT: """MQTT Client for CircuitPython @@ -151,7 +145,6 @@ class MQTT: :param network_manager: NetworkManager object, such as WiFiManager from ESPSPI_WiFiManager. :param str client_id: Optional client identifier, defaults to a unique, generated string. :param bool is_ssl: Sets a secure or insecure connection with the broker. - :param bool log: Attaches a logger to the MQTT client, defaults to logging level INFO. :param int keep_alive: KeepAlive interval between the broker and the MiniMQTT client. :param socket socket_pool: A pool of socket resources available for the given radio. :param ssl_context: SSL context for long-lived SSL connections. @@ -166,7 +159,6 @@ def __init__( password=None, client_id=None, is_ssl=True, - log=False, keep_alive=60, socket_pool=None, ssl_context=None @@ -194,6 +186,7 @@ def __init__( self._msg_size_lim = MQTT_MSG_SZ_LIM self._pid = 0 self._timestamp = 0 + self.logger = None self.broker = broker self._username = username @@ -224,9 +217,6 @@ def __init__( if len(self.client_id) > 23 or not self.client_id: raise ValueError("MQTT Client ID must be between 1 and 23 bytes") - self.logger = None - if log is True: - self.enable_logger() # LWT self._lw_topic = None @@ -1021,43 +1011,34 @@ def mqtt_msg(self, msg_size): if msg_size < MQTT_MSG_MAX_SZ: self._msg_size_lim = msg_size - ### Logging ### - def attach_logger(self, logger_name="log"): - """Initializes and attaches a logger to the MQTTClient. - :param str logger_name: Name of the logger instance - NOTE: This method is replaced by enable_logger and - will be removed in a future release to - remove this lib's dependency from adafruit_logging. + ### Logging API ### - """ - self.logger = logging.getLogger(logger_name) - self.logger.setLevel(logging.INFO) + def enable_logger(self, logger, log_level=20): + """Enables library logging provided a `logger` object. - def enable_logger(self, logger=None): - """Enables logging using the logging package. If a `logger` - is specified, then that object will be used. Otherwise, a `logger` - will be automatically created. + :param logging.Logger logger: A python logger pacakge. + :param log_level: Numeric value of a logging level, defaults to `logging.INFO`. """ - if logger: - self.logger = logger - else: - self.logger = logging.getLogger("log") - self.logger.setLevel(logging.INFO) + self.logger = logging.getLogger("log") + self.logger.setLevel(log_level) def disable_logger(self): """Disables logging.""" if not self.logger: - raise ValueError("Can not disable logging - no logger enabled!") + raise ValueError("Can't disable logging - no logger enabled!") self.logger = None def set_logger_level(self, log_level): """Sets the level of the logger, if defined during init. + NOTE: This method is deprecated and will be removed. Use + `enable_logger`'s `log_level` as an alternative to this method. :param str log_level: Level of logging to output to the REPL. Acceptable options are ``DEBUG``, ``INFO``, ``WARNING``, or ``ERROR``. """ + print("This method ") if self.logger is None: raise MMQTTException( "No logger attached - did you create it during initialization?" @@ -1072,3 +1053,12 @@ def set_logger_level(self, log_level): self.logger.setLevel(logging.CRITICIAL) else: raise MMQTTException("Incorrect logging level provided!") + + def attach_logger(self, logger_name="log"): + """Initializes and attaches a logger to the MQTTClient. + :param str logger_name: Name of the logger instance + NOTE: This method is deprecated and will be removed. Use + `enable_logger` as an alternative to this method. + + """ + self.enable_logger() \ No newline at end of file From c1d3f06db4826d0d94cd12a65a7d837174a6c9c0 Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 25 Nov 2020 16:49:10 -0500 Subject: [PATCH 37/77] move examples into folders --- .../minimqtt_adafruitio_cellular.py | 0 .../minimqtt_simpletest_cellular.py | 0 .../minimqtt_adafruitio_cpython.py | 0 .../minimqtt_simpletest_cpython.py | 0 .../minimqtt_adafruitio_wifi.py | 0 .../{ => esp32spi}/minimqtt_certificate.py | 0 .../minimqtt_pub_sub_blocking.py | 0 ...nimqtt_pub_sub_blocking_topic_callbacks.py | 0 .../minimqtt_pub_sub_nonblocking.py | 0 .../minimqtt_pub_sub_pyportal.py | 0 examples/esp32spi/minimqtt_simpletest.py | 136 ++++++++++++++++++ .../{ => ethernet}/minimqtt_adafruitio_eth.py | 0 .../{ => ethernet}/minimqtt_simpletest_eth.py | 0 examples/minimqtt_simpletest.py | 122 ---------------- 14 files changed, 136 insertions(+), 122 deletions(-) rename examples/{ => cellular}/minimqtt_adafruitio_cellular.py (100%) rename examples/{ => cellular}/minimqtt_simpletest_cellular.py (100%) rename examples/{ => cpython}/minimqtt_adafruitio_cpython.py (100%) rename examples/{ => cpython}/minimqtt_simpletest_cpython.py (100%) rename examples/{ => esp32spi}/minimqtt_adafruitio_wifi.py (100%) rename examples/{ => esp32spi}/minimqtt_certificate.py (100%) rename examples/{ => esp32spi}/minimqtt_pub_sub_blocking.py (100%) rename examples/{ => esp32spi}/minimqtt_pub_sub_blocking_topic_callbacks.py (100%) rename examples/{ => esp32spi}/minimqtt_pub_sub_nonblocking.py (100%) rename examples/{ => esp32spi}/minimqtt_pub_sub_pyportal.py (100%) create mode 100644 examples/esp32spi/minimqtt_simpletest.py rename examples/{ => ethernet}/minimqtt_adafruitio_eth.py (100%) rename examples/{ => ethernet}/minimqtt_simpletest_eth.py (100%) delete mode 100644 examples/minimqtt_simpletest.py diff --git a/examples/minimqtt_adafruitio_cellular.py b/examples/cellular/minimqtt_adafruitio_cellular.py similarity index 100% rename from examples/minimqtt_adafruitio_cellular.py rename to examples/cellular/minimqtt_adafruitio_cellular.py diff --git a/examples/minimqtt_simpletest_cellular.py b/examples/cellular/minimqtt_simpletest_cellular.py similarity index 100% rename from examples/minimqtt_simpletest_cellular.py rename to examples/cellular/minimqtt_simpletest_cellular.py diff --git a/examples/minimqtt_adafruitio_cpython.py b/examples/cpython/minimqtt_adafruitio_cpython.py similarity index 100% rename from examples/minimqtt_adafruitio_cpython.py rename to examples/cpython/minimqtt_adafruitio_cpython.py diff --git a/examples/minimqtt_simpletest_cpython.py b/examples/cpython/minimqtt_simpletest_cpython.py similarity index 100% rename from examples/minimqtt_simpletest_cpython.py rename to examples/cpython/minimqtt_simpletest_cpython.py diff --git a/examples/minimqtt_adafruitio_wifi.py b/examples/esp32spi/minimqtt_adafruitio_wifi.py similarity index 100% rename from examples/minimqtt_adafruitio_wifi.py rename to examples/esp32spi/minimqtt_adafruitio_wifi.py diff --git a/examples/minimqtt_certificate.py b/examples/esp32spi/minimqtt_certificate.py similarity index 100% rename from examples/minimqtt_certificate.py rename to examples/esp32spi/minimqtt_certificate.py diff --git a/examples/minimqtt_pub_sub_blocking.py b/examples/esp32spi/minimqtt_pub_sub_blocking.py similarity index 100% rename from examples/minimqtt_pub_sub_blocking.py rename to examples/esp32spi/minimqtt_pub_sub_blocking.py diff --git a/examples/minimqtt_pub_sub_blocking_topic_callbacks.py b/examples/esp32spi/minimqtt_pub_sub_blocking_topic_callbacks.py similarity index 100% rename from examples/minimqtt_pub_sub_blocking_topic_callbacks.py rename to examples/esp32spi/minimqtt_pub_sub_blocking_topic_callbacks.py diff --git a/examples/minimqtt_pub_sub_nonblocking.py b/examples/esp32spi/minimqtt_pub_sub_nonblocking.py similarity index 100% rename from examples/minimqtt_pub_sub_nonblocking.py rename to examples/esp32spi/minimqtt_pub_sub_nonblocking.py diff --git a/examples/minimqtt_pub_sub_pyportal.py b/examples/esp32spi/minimqtt_pub_sub_pyportal.py similarity index 100% rename from examples/minimqtt_pub_sub_pyportal.py rename to examples/esp32spi/minimqtt_pub_sub_pyportal.py diff --git a/examples/esp32spi/minimqtt_simpletest.py b/examples/esp32spi/minimqtt_simpletest.py new file mode 100644 index 00000000..9292847e --- /dev/null +++ b/examples/esp32spi/minimqtt_simpletest.py @@ -0,0 +1,136 @@ +# adafruit_minimqtt usage with esp32spi wifi +import board +import busio +from digitalio import DigitalInOut +import neopixel +from adafruit_esp32spi import adafruit_esp32spi +import adafruit_esp32spi.adafruit_esp32spi_socket as socket +import adafruit_minimqtt.adafruit_minimqtt as MQTT + +# Add a secrets.py to your filesystem that has a dictionary called secrets with "ssid" and +# "password" keys with your WiFi credentials. DO NOT share that file or commit it into Git or other +# source control. +# pylint: disable=no-name-in-module,wrong-import-order +try: + from secrets import secrets +except ImportError: + print("WiFi secrets are kept in secrets.py, please add them there!") + raise + +# Set your Adafruit IO Username and Key in secrets.py +# (visit io.adafruit.com if you need to create an account, +# or if you need your Adafruit IO key.) +aio_username = secrets["aio_username"] +aio_key = secrets["aio_key"] + +# If you are using a board with pre-defined ESP32 Pins: +esp32_cs = DigitalInOut(board.ESP_CS) +esp32_ready = DigitalInOut(board.ESP_BUSY) +esp32_reset = DigitalInOut(board.ESP_RESET) + +# If you have an externally connected ESP32: +# esp32_cs = DigitalInOut(board.D9) +# esp32_ready = DigitalInOut(board.D10) +# esp32_reset = DigitalInOut(board.D5) + +spi = busio.SPI(board.SCK, board.MOSI, board.MISO) +esp = adafruit_esp32spi.ESP_SPIcontrol(spi, esp32_cs, esp32_ready, esp32_reset) + +print("Connecting to AP...") +while not esp.is_connected: + try: + esp.connect_AP(secrets["ssid"], secrets["password"]) + except RuntimeError as e: + print("could not connect to AP, retrying: ", e) + continue +print("Connected to", str(esp.ssid, "utf-8"), "\tRSSI:", esp.rssi) + +### Topic Setup ### + +# MQTT Topic +# Use this topic if you'd like to connect to a standard MQTT broker +mqtt_topic = "test/topic" + +# Adafruit IO-style Topic +# Use this topic if you'd like to connect to io.adafruit.com +# mqtt_topic = secrets["aio_username"] + '/feeds/temperature' + +### Code ### + +# Keep track of client connection state +disconnect_client = False + +# Define callback methods which are called when events occur +# pylint: disable=unused-argument, redefined-outer-name +def connect(mqtt_client, userdata, flags, rc): + # This function will be called when the mqtt_client is connected + # successfully to the broker. + print("Connected to MQTT Broker!") + print("Flags: {0}\n RC: {1}".format(flags, rc)) + + +def disconnect(mqtt_client, userdata, rc): + # This method is called when the mqtt_client disconnects + # from the broker. + print("Disconnected from MQTT Broker!") + + +def subscribe(mqtt_client, userdata, topic, granted_qos): + # This method is called when the mqtt_client subscribes to a new feed. + print("Subscribed to {0} with QOS level {1}".format(topic, granted_qos)) + + +def unsubscribe(mqtt_client, userdata, topic, pid): + # This method is called when the mqtt_client unsubscribes from a feed. + print("Unsubscribed from {0} with PID {1}".format(topic, pid)) + + +def publish(mqtt_client, userdata, topic, pid): + # This method is called when the mqtt_client publishes data to a feed. + print("Published to {0} with PID {1}".format(topic, pid)) + + +def message(client, topic, message): + # Method callled when a client's subscribed feed has a new value. + global disconnect_client + print("New message on topic {0}: {1}".format(topic, message)) + + print("Unsubscribing from %s" % mqtt_topic) + mqtt_client.unsubscribe(mqtt_topic) + # Allow us to gracefully stop the `while True` loop + disconnect_client = True + +socket.set_interface(esp) +MQTT.set_socket(socket, esp) + +# Set up a MiniMQTT Client +mqtt_client = MQTT.MQTT( + broker=secrets['broker'], + port=secrets['port'], + username=secrets['username'], + password=secrets['password'] +) + +# Connect callback handlers to mqtt_client +mqtt_client.on_connect = connect +mqtt_client.on_disconnect = disconnect +mqtt_client.on_subscribe = subscribe +mqtt_client.on_unsubscribe = unsubscribe +mqtt_client.on_publish = publish +mqtt_client.on_message = message + +print("Attempting to connect to %s" % mqtt_client.broker) +mqtt_client.connect() + +print("Subscribing to %s" % mqtt_topic) +mqtt_client.subscribe(mqtt_topic) + +print("Publishing to %s" % mqtt_topic) +mqtt_client.publish(mqtt_topic, "Hello Broker!") + +# Pump the loop until we receive a message on `mqtt_topic` +while disconnect_client == False: + mqtt_client.loop() + +print("Disconnecting from %s" % mqtt_client.broker) +mqtt_client.disconnect() diff --git a/examples/minimqtt_adafruitio_eth.py b/examples/ethernet/minimqtt_adafruitio_eth.py similarity index 100% rename from examples/minimqtt_adafruitio_eth.py rename to examples/ethernet/minimqtt_adafruitio_eth.py diff --git a/examples/minimqtt_simpletest_eth.py b/examples/ethernet/minimqtt_simpletest_eth.py similarity index 100% rename from examples/minimqtt_simpletest_eth.py rename to examples/ethernet/minimqtt_simpletest_eth.py diff --git a/examples/minimqtt_simpletest.py b/examples/minimqtt_simpletest.py deleted file mode 100644 index 2c08b99f..00000000 --- a/examples/minimqtt_simpletest.py +++ /dev/null @@ -1,122 +0,0 @@ -import board -import busio -from digitalio import DigitalInOut -import neopixel -from adafruit_esp32spi import adafruit_esp32spi -from adafruit_esp32spi import adafruit_esp32spi_wifimanager -import adafruit_esp32spi.adafruit_esp32spi_socket as socket - -import adafruit_minimqtt.adafruit_minimqtt as MQTT - -### WiFi ### - -# Get wifi details and more from a secrets.py file -try: - from secrets import secrets -except ImportError: - print("WiFi secrets are kept in secrets.py, please add them there!") - raise - -# If you are using a board with pre-defined ESP32 Pins: -esp32_cs = DigitalInOut(board.ESP_CS) -esp32_ready = DigitalInOut(board.ESP_BUSY) -esp32_reset = DigitalInOut(board.ESP_RESET) - -# If you have an externally connected ESP32: -# esp32_cs = DigitalInOut(board.D9) -# esp32_ready = DigitalInOut(board.D10) -# esp32_reset = DigitalInOut(board.D5) - -spi = busio.SPI(board.SCK, board.MOSI, board.MISO) -esp = adafruit_esp32spi.ESP_SPIcontrol(spi, esp32_cs, esp32_ready, esp32_reset) -"""Use below for Most Boards""" -status_light = neopixel.NeoPixel( - board.NEOPIXEL, 1, brightness=0.2 -) # Uncomment for Most Boards -"""Uncomment below for ItsyBitsy M4""" -# status_light = dotstar.DotStar(board.APA102_SCK, board.APA102_MOSI, 1, brightness=0.2) -# Uncomment below for an externally defined RGB LED -# import adafruit_rgbled -# from adafruit_esp32spi import PWMOut -# RED_LED = PWMOut.PWMOut(esp, 26) -# GREEN_LED = PWMOut.PWMOut(esp, 27) -# BLUE_LED = PWMOut.PWMOut(esp, 25) -# status_light = adafruit_rgbled.RGBLED(RED_LED, BLUE_LED, GREEN_LED) -wifi = adafruit_esp32spi_wifimanager.ESPSPI_WiFiManager(esp, secrets, status_light) - -### Topic Setup ### - -# MQTT Topic -# Use this topic if you'd like to connect to a standard MQTT broker -mqtt_topic = "test/topic" - -# Adafruit IO-style Topic -# Use this topic if you'd like to connect to io.adafruit.com -# mqtt_topic = 'aio_user/feeds/temperature' - -### Code ### - -# Define callback methods which are called when events occur -# pylint: disable=unused-argument, redefined-outer-name -def connect(client, userdata, flags, rc): - # This function will be called when the client is connected - # successfully to the broker. - print("Connected to MQTT Broker!") - print("Flags: {0}\n RC: {1}".format(flags, rc)) - - -def disconnect(client, userdata, rc): - # This method is called when the client disconnects - # from the broker. - print("Disconnected from MQTT Broker!") - - -def subscribe(client, userdata, topic, granted_qos): - # This method is called when the client subscribes to a new feed. - print("Subscribed to {0} with QOS level {1}".format(topic, granted_qos)) - - -def unsubscribe(client, userdata, topic, pid): - # This method is called when the client unsubscribes from a feed. - print("Unsubscribed from {0} with PID {1}".format(topic, pid)) - - -def publish(client, userdata, topic, pid): - # This method is called when the client publishes data to a feed. - print("Published to {0} with PID {1}".format(topic, pid)) - - -# Connect to WiFi -print("Connecting to WiFi...") -wifi.connect() -print("Connected!") - -# Initialize MQTT interface with the esp interface -MQTT.set_socket(socket, esp) - -# Set up a MiniMQTT Client -client = MQTT.MQTT( - broker=secrets["broker"], username=secrets["user"], password=secrets["pass"] -) - -# Connect callback handlers to client -client.on_connect = connect -client.on_disconnect = disconnect -client.on_subscribe = subscribe -client.on_unsubscribe = unsubscribe -client.on_publish = publish - -print("Attempting to connect to %s" % client.broker) -client.connect() - -print("Subscribing to %s" % mqtt_topic) -client.subscribe(mqtt_topic) - -print("Publishing to %s" % mqtt_topic) -client.publish(mqtt_topic, "Hello Broker!") - -print("Unsubscribing from %s" % mqtt_topic) -client.unsubscribe(mqtt_topic) - -print("Disconnecting from %s" % client.broker) -client.disconnect() From b3b08811e275341c862aa88e3f6413bb83f9851c Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 30 Nov 2020 17:38:06 -0500 Subject: [PATCH 38/77] Fix hang on _wait_for - attempt blocking read for cpython, s2. works with cpython insecure, does not work with cpython ssl sockets. Works with ESP32S2 insecure, not with SSL sockets --- adafruit_minimqtt/adafruit_minimqtt.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 28e341f5..373a36b0 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -861,20 +861,24 @@ def loop(self, timeout=0.01): self._timestamp = 0 return self._wait_for_msg(timeout) - def _wait_for_msg(self, timeout=0.01): + def _wait_for_msg(self, timeout=0.1): """Reads and processes network events.""" buf = self._rx_buffer - # attempt to recv from socket within `timeout` seconds - # TODO: This line is inconsistent btween versions! - self._sock.settimeout(timeout) + res = bytearray(1) + # Attempt to recv + self._sock.setblocking(False) try: - res = bytearray(1) - self._recv_into(res, 1) - except OSError: + self._sock.recv_into(res, 1) + print("Resp: ", res) + except BlockingIOError: # fix for macOS Errno + return None + except self._socket_pool.timeout: + print("timeout!") return None - self._sock.settimeout(0) + # Block while we parse the rest of the response + self._sock.setblocking(True) if res in [None, b""]: return None if res == MQTT_PINGRESP: From ee19319e20c5f9843c638453dc6232fafb897d8c Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 16 Dec 2020 12:34:50 -0500 Subject: [PATCH 39/77] switch to settimeout for socketpool consistency --- adafruit_minimqtt/adafruit_minimqtt.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 373a36b0..9d91dd3c 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -866,8 +866,7 @@ def _wait_for_msg(self, timeout=0.1): buf = self._rx_buffer res = bytearray(1) - # Attempt to recv - self._sock.setblocking(False) + self._sock.settimeout(0) try: self._sock.recv_into(res, 1) print("Resp: ", res) @@ -878,7 +877,7 @@ def _wait_for_msg(self, timeout=0.1): return None # Block while we parse the rest of the response - self._sock.setblocking(True) + self._sock.settimeout(None) if res in [None, b""]: return None if res == MQTT_PINGRESP: From 76ab54af0714c9d34a48fd0c2d3fdff48e426168 Mon Sep 17 00:00:00 2001 From: brentru Date: Wed, 16 Dec 2020 12:43:58 -0500 Subject: [PATCH 40/77] timeout instead of nonblocking to avoid cpython ssl error --- adafruit_minimqtt/adafruit_minimqtt.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 9d91dd3c..180fb89b 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -866,14 +866,12 @@ def _wait_for_msg(self, timeout=0.1): buf = self._rx_buffer res = bytearray(1) - self._sock.settimeout(0) + # Attempt to read + self._sock.settimeout(1) try: self._sock.recv_into(res, 1) - print("Resp: ", res) - except BlockingIOError: # fix for macOS Errno - return None + print("Resp: ", res) # TODO BR: Remove debugging except self._socket_pool.timeout: - print("timeout!") return None # Block while we parse the rest of the response From 65afbd583c4485b068e0060a57b29df4d8c9c4f3 Mon Sep 17 00:00:00 2001 From: brentru Date: Fri, 29 Jan 2021 16:17:35 -0500 Subject: [PATCH 41/77] addressing https://github.com/adafruit/Adafruit_CircuitPython_MiniMQTT/issues/58 --- adafruit_minimqtt/adafruit_minimqtt.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 180fb89b..44bc5aa1 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -870,9 +870,12 @@ def _wait_for_msg(self, timeout=0.1): self._sock.settimeout(1) try: self._sock.recv_into(res, 1) - print("Resp: ", res) # TODO BR: Remove debugging - except self._socket_pool.timeout: - return None + except OSError as error: + if error.errno == errno.ETIMEDOUT: + # raised by a socketpool + return None + else: + raise MMQTTException(error) # Block while we parse the rest of the response self._sock.settimeout(None) From cd8d9543dc189f920f50890cac3ef2a2e5ac233d Mon Sep 17 00:00:00 2001 From: brentru Date: Fri, 29 Jan 2021 16:44:21 -0500 Subject: [PATCH 42/77] pull in _sock_exact_recv and modify it for a cpython/socketpool implementation, tested with _connect. Removing the shared buffer overhead which may or may not work in favor of dynamically sizing rc buffer like in master --- adafruit_minimqtt/adafruit_minimqtt.py | 62 ++++++++++++++++++-------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 44bc5aa1..612bf671 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -1,6 +1,6 @@ # The MIT License (MIT) # -# Copyright (c) 2019 Brent Rubell for Adafruit Industries +# Copyright (c) 2019-2021 Brent Rubell for Adafruit Industries # # Original Work Copyright (c) 2016 Paul Sokolovsky, uMQTT # Modified Work Copyright (c) 2019 Bradley Beach, esp32spi_mqtt @@ -36,14 +36,6 @@ Adapted from https://github.com/micropython/micropython-lib/tree/master/umqtt.simple/umqtt -micropython-lib consists of multiple modules from different sources and -authors. Each module comes under its own licensing terms. Short name of -a license can be found in a file within a module directory (usually -metadata.txt or setup.py). Complete text of each license used is provided -at https://github.com/micropython/micropython-lib/blob/master/LICENSE - -author='Paul Sokolovsky' -license='MIT' **Software and Dependencies:** * Adafruit CircuitPython firmware for the supported boards: @@ -95,8 +87,7 @@ class MMQTTException(Exception): # pass -# Legacy Socket API - +# Legacy ESP32SPI Socket API def set_socket(sock, iface=None): """Legacy API for setting the socket and network interface, use a `Session` instead. @@ -237,8 +228,6 @@ def __init__( self.on_subscribe = None self.on_unsubscribe = None - # Shared buffer - self._rx_length = 0 self._rx_buffer = bytearray(32) # Socket helpers @@ -516,14 +505,14 @@ def connect(self, clean_session=True, host=None, port=None, keep_alive=None): while True: op = self._wait_for_msg() if op == 32: - self._recv_into(buf, 3) - assert buf[0] == 0x02 - if buf[2] != 0x00: - raise MMQTTException(CONNACK_ERRORS[buf[2]]) + rc = self._sock_exact_recv(3) + assert rc[0] == 0x02 + if rc[2] != 0x00: + raise MMQTTException(CONNACK_ERRORS[rc[2]]) self._is_connected = True - result = buf[0] & 1 + result = rc[0] & 1 if self.on_connect is not None: - self.on_connect(self, self._user_data, result, buf[2]) + self.on_connect(self, self._user_data, result, rc[2]) return result def disconnect(self): @@ -945,6 +934,41 @@ def _recv_into(self, buf, size=0): return read_size return self._sock.recv_into(buf, size) + def _sock_exact_recv(self, bufsize): + """Reads _exact_ number of bytes from the connected socket. Will only return + string with the exact number of bytes requested. + + The semantics of native socket receive is that it returns no more than the + specified number of bytes (i.e. max size). However, it makes no guarantees in + terms of the minimum size of the buffer, which could be 1 byte. This is a + wrapper for socket recv() to ensure that no less than the expected number of + bytes is returned or trigger a timeout exception. + :param int bufsize: number of bytes to receive + + """ + if not self._backwards_compatible_sock: + # CPython/Socketpool Impl. + rc = bytearray(bufsize) + self._sock.recv_into(rc, bufsize) + else: # ESP32SPI Impl. + stamp = time.monotonic() + read_timeout = self.keep_alive + rc = self._sock.recv(bufsize) + to_read = bufsize - len(rc) + assert to_read >= 0 + read_timeout = self.keep_alive + while to_read > 0: + recv = self._sock.recv(to_read) + to_read -= len(recv) + rc += recv + if time.monotonic() - stamp > read_timeout: + raise MMQTTException( + "Unable to receive {} bytes within {} seconds.".format( + to_read, read_timeout + ) + ) + return rc + def _send_str(self, string): """Packs and encodes a string to a socket. From cb16a27d09d8301c8e9b10be3a3e05610ce0f92a Mon Sep 17 00:00:00 2001 From: brentru Date: Fri, 29 Jan 2021 16:53:18 -0500 Subject: [PATCH 43/77] switch to _exact_recv on publish(), assert self._pid instead of unbound pid --- adafruit_minimqtt/adafruit_minimqtt.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 612bf671..8e4430de 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -642,14 +642,13 @@ def publish(self, topic, msg, retain=False, qos=0): while True: op = self._wait_for_msg() if op == 0x40: - self._recv_into(buf, 1) - assert buf == b"\x02" - # rcv_pid = self._sock.recv(2) - self._recv_into(buf, 2) - buf = buf[0] << 0x08 | buf[1] - if pid == buf: + sz = self._sock_exact_recv(1) + assert sz == b"\x02" + rcv_pid = self._sock_exact_recv(2) + rcv_pid = rcv_pid[0] << 0x08 | rcv_pid[1] + if self._pid == rcv_pid: if self.on_publish is not None: - self.on_publish(self, self._user_data, topic, buf) + self.on_publish(self, self._user_data, topic, rcv_pid) return def subscribe(self, topic, qos=0): From e2b7fa05ad28945d52148a804cad09cfe8ba9367 Mon Sep 17 00:00:00 2001 From: brentru Date: Fri, 29 Jan 2021 16:54:36 -0500 Subject: [PATCH 44/77] switch to _exact_recv on subscribe() calls --- adafruit_minimqtt/adafruit_minimqtt.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 8e4430de..8d0a5345 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -721,7 +721,6 @@ def subscribe(self, topic, qos=0): for t, q in topics: topic_size = len(t).to_bytes(2, "big") qos_byte = q.to_bytes(1, "big") - #print(packet, "\n", topic_size, "\n", t, "\n", qos_byte) packet += topic_size + t.encode() + qos_byte if self.logger: for t, q in topics: @@ -730,9 +729,9 @@ def subscribe(self, topic, qos=0): while True: op = self._wait_for_msg() if op == 0x90: - self._recv_into(buf, 4) - assert buf[1] == packet[2] and buf[2] == packet[3] - if buf[3] == 0x80: + rc = self._sock_exact_recv(4) + assert rc[1] == packet[2] and rc[2] == packet[3] + if rc[3] == 0x80: raise MMQTTException("SUBACK Failure!") for t, q in topics: if self.on_subscribe is not None: From f0907a446775894027244f60497c45358d6ef84b Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 11:27:58 -0500 Subject: [PATCH 45/77] update ping, unsubscribe. add timeout kwarg to loop --- adafruit_minimqtt/adafruit_minimqtt.py | 67 ++++++++++++++------------ 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 8d0a5345..b2a043c8 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -516,38 +516,42 @@ def connect(self, clean_session=True, host=None, port=None, keep_alive=None): return result def disconnect(self): - """Disconnects the MiniMQTT client from the MQTT broker. - """ + """Disconnects the MiniMQTT client from the MQTT broker.""" self.is_connected() - if self.logger: + if self.logger is not None: self.logger.debug("Sending DISCONNECT packet to broker") - self._sock.send(MQTT_DISCONNECT) - if self.logger: + try: + self._sock.send(MQTT_DISCONNECT) + except RuntimeError as e: + if self.logger: + self.logger.warning("Unable to send DISCONNECT packet: {}".format(e)) + if self.logger is not None: self.logger.debug("Closing socket") - self._free_sockets() + self._sock.close() self._is_connected = False - self._subscribed_topics = None + self._subscribed_topics = [] if self.on_disconnect is not None: - self.on_disconnect(self, self._user_data, 0) + self.on_disconnect(self, self.user_data, 0) def ping(self): """Pings the MQTT Broker to confirm if the broker is alive or if there is an active network connection. + Returns response codes of any messages received while waiting for PINGRESP. """ self.is_connected() - buf = self._rx_buffer if self.logger: self.logger.debug("Sending PINGREQ") self._sock.send(MQTT_PINGREQ) - if self.logger: - self.logger.debug("Checking PINGRESP") - while True: - op = self._wait_for_msg() - if op == 208: - self._recv_into(buf, 2) - if buf[0] != 0x00: - raise MMQTTException("PINGRESP not returned from broker.") - return + ping_timeout = self.keep_alive + stamp = time.monotonic() + rc, rcs = None, [] + while rc != MQTT_PINGRESP: + rc = self._wait_for_msg() + if rc: + rcs.append(rc) + if time.monotonic() - stamp > ping_timeout: + raise MMQTTException("PINGRESP not returned from broker.") + return rcs # pylint: disable=too-many-branches, too-many-statements def publish(self, topic, msg, retain=False, qos=0): @@ -794,12 +798,12 @@ def unsubscribe(self, topic): while True: op = self._wait_for_msg() if op == 176: - self._recv_into(buf, 3) - assert buf[0] == 0x02 + rc = self._sock_exact_recv(3) + assert rc[0] == 0x02 # [MQTT-3.32] assert ( - buf[1] == packet_id_bytes[0] - and buf[2] == packet_id_bytes[1] + rc[1] == packet_id_bytes[0] + and rc[2] == packet_id_bytes[1] ) for t in topics: if self.on_unsubscribe is not None: @@ -828,33 +832,36 @@ def reconnect(self, resub_topics=True): feed = subscribed_topics.pop() self.subscribe(feed) - def loop(self, timeout=0.01): + def loop(self, timeout=1): """Non-blocking message loop. Use this method to check incoming subscription messages. - :param float timeout: Set timeout in seconds for - polling the message queue. + Returns response codes of any messages received. + :param int timeout: Socket timeout, in seconds. + """ if self._timestamp == 0: self._timestamp = time.monotonic() current_time = time.monotonic() if current_time - self._timestamp >= self.keep_alive: # Handle KeepAlive by expecting a PINGREQ/PINGRESP from the server - if self.logger: + if self.logger is not None: self.logger.debug( "KeepAlive period elapsed - \ requesting a PINGRESP from the server..." ) - self.ping() + rcs = self.ping() self._timestamp = 0 - return self._wait_for_msg(timeout) + return rcs + self._sock.settimeout(timeout) + rc = self._wait_for_msg() + return [rc] if rc else None + def _wait_for_msg(self, timeout=0.1): """Reads and processes network events.""" buf = self._rx_buffer res = bytearray(1) - # Attempt to read - self._sock.settimeout(1) try: self._sock.recv_into(res, 1) except OSError as error: From ea966339379c8081dba94c95bd1f286dfe4dcca1 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 11:29:47 -0500 Subject: [PATCH 46/77] update recv_len to master, bound pid to 0 so it can never be unbound --- adafruit_minimqtt/adafruit_minimqtt.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index b2a043c8..e81ec48f 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -890,6 +890,7 @@ def _wait_for_msg(self, timeout=0.1): self._recv_into(topic, topic_len) topic = str(topic, "utf-8") sz -= topic_len + 2 + pid = 0 if res[0] & 0x06: pid = bytearray(2) self._recv_into(pid, 2) @@ -918,13 +919,14 @@ def _wait_for_msg(self, timeout=0.1): def _recv_len(self): + """Unpack MQTT message length.""" n = 0 sh = 0 b = bytearray(1) while True: - self._recv_into(b, 1) - n |= (b[0] & 0x7F) << sh - if not b[0] & 0x80: + b = self._sock_exact_recv(1)[0] + n |= (b & 0x7F) << sh + if not b & 0x80: return n sh += 7 From fc31e2d566ea9b6e171270a6d59bd2d08abd03b5 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 11:40:29 -0500 Subject: [PATCH 47/77] remove buf, decode on_message within wait_for_msg --- adafruit_minimqtt/adafruit_minimqtt.py | 42 ++++++-------------------- 1 file changed, 10 insertions(+), 32 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index e81ec48f..0770989f 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -395,15 +395,15 @@ def on_message(self): def on_message(self, method): self._on_message = method - def _handle_on_message(self, client, topic): + def _handle_on_message(self, client, topic, message): matched = False if topic is not None: for callback in self._on_message_filtered.iter_match(topic): - callback(client, topic, self._rx_buffer.decode()) # on_msg with callback + callback(client, topic, message) # on_msg with callback matched = True if not matched and self.on_message: # regular on_message - self.on_message(client, topic, self._rx_buffer.decode()) + self.on_message(client, topic, message) def username_pw_set(self, username, password=None): """Set client's username and an optional password. @@ -428,7 +428,6 @@ def connect(self, clean_session=True, host=None, port=None, keep_alive=None): with the broker, in seconds """ - buf = self._rx_buffer if host: self.broker = host if port: @@ -603,7 +602,6 @@ def publish(self, topic, msg, retain=False, qos=0): 0 <= qos <= 1 ), "Quality of Service Level 2 is unsupported by this library." - buf = self._rx_buffer # fixed header. [3.3.1.2], [3.3.1.3] pub_hdr_fixed = bytearray([0x30 | retain | qos << 1]) @@ -614,7 +612,6 @@ def publish(self, topic, msg, retain=False, qos=0): remaining_length = 2 + len(msg) + len(topic) if qos > 0: # packet identifier where QoS level is 1 or 2. [3.3.2.2] - pid = self._pid remaining_length += 2 pub_hdr_var.append(0x00) pub_hdr_var.append(self._pid) @@ -711,8 +708,6 @@ def subscribe(self, topic, qos=0): self._check_qos(q) self._check_topic(t) topics.append((t, q)) - # Rx buffer - buf = self._rx_buffer # Assemble packet packet_length = 2 + (2 * len(topics)) + (1 * len(topics)) packet_length += sum(len(topic) for topic, qos in topics) @@ -777,8 +772,6 @@ def unsubscribe(self, topic): raise MMQTTException( "Topic must be subscribed to before attempting unsubscribe." ) - # Rx buffer - buf = self._rx_buffer # Assemble packet packet_length = 2 + (2 * len(topics)) packet_length += sum(len(topic) for topic in topics) @@ -859,9 +852,7 @@ def loop(self, timeout=1): def _wait_for_msg(self, timeout=0.1): """Reads and processes network events.""" - buf = self._rx_buffer res = bytearray(1) - try: self._sock.recv_into(res, 1) except OSError as error: @@ -872,7 +863,7 @@ def _wait_for_msg(self, timeout=0.1): raise MMQTTException(error) # Block while we parse the rest of the response - self._sock.settimeout(None) + self._sock.settimeout(timeout) if res in [None, b""]: return None if res == MQTT_PINGRESP: @@ -883,32 +874,19 @@ def _wait_for_msg(self, timeout=0.1): return res[0] sz = self._recv_len() # topic length MSB & LSB - topic_len = bytearray(2) - self._recv_into(topic_len, 2) + topic_len = self._sock_exact_recv(2) topic_len = (topic_len[0] << 8) | topic_len[1] - topic = bytearray(topic_len) - self._recv_into(topic, topic_len) + topic = self._sock_exact_recv(topic_len) topic = str(topic, "utf-8") sz -= topic_len + 2 pid = 0 if res[0] & 0x06: - pid = bytearray(2) - self._recv_into(pid, 2) + pid = self._sock_exact_recv(2) pid = pid[0] << 0x08 | pid[1] sz -= 0x02 - - # if expected packet size is larger than buffer - if sz > len(buf): - # resize buffer to match expected size - new_size = sz + len(buf) - new_buf = bytearray(new_size) - new_buf[: len(buf)] = buf - buf = new_buf - self._rx_buffer = buf - # read incoming packet - self._recv_into(self._rx_buffer, sz) - - self._handle_on_message(self, topic) + # read message contents + msg = self._sock_exact_recv(sz) + self._handle_on_message(self, topic, str(msg, "utf-8")) if res[0] & 0x06 == 0x02: pkt = bytearray(b"\x40\x02\0\0") struct.pack_into("!H", pkt, 2, pid) From f34b80f9f225573e66aca0cb75421afeef4fe96b Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 11:45:20 -0500 Subject: [PATCH 48/77] _check -> _valid, remove unused set_interface method --- adafruit_minimqtt/adafruit_minimqtt.py | 48 ++++++++++---------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 0770989f..2091d10d 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -345,7 +345,7 @@ def will_set(self, topic=None, payload=None, qos=0, retain=False): """ if self.logger: self.logger.debug("Setting last will properties") - self._check_qos(qos) + self._valid_qos(qos) if self._is_connected: raise MMQTTException("Last Will should only be called before connect().") if payload is None: @@ -584,7 +584,7 @@ def publish(self, topic, msg, retain=False, qos=0): mqtt_client.publish('topics/piVal', 'threepointonefour') """ self.is_connected() - self._check_topic(topic) + self._valid_topic(topic) if "+" in topic or "#" in topic: raise MMQTTException("Publish topic can not contain wildcards.") # check msg/qos kwargs @@ -696,17 +696,17 @@ def subscribe(self, topic, qos=0): topics = None if isinstance(topic, tuple): topic, qos = topic - self._check_topic(topic) - self._check_qos(qos) + self._valid_topic(topic) + self._valid_qos(qos) if isinstance(topic, str): - self._check_topic(topic) - self._check_qos(qos) + self._valid_topic(topic) + self._valid_qos(qos) topics = [(topic, qos)] if isinstance(topic, list): topics = [] for t, q in topic: - self._check_qos(q) - self._check_topic(t) + self._valid_qos(q) + self._valid_topic(t) topics.append((t, q)) # Assemble packet packet_length = 2 + (2 * len(topics)) + (1 * len(topics)) @@ -760,12 +760,12 @@ def unsubscribe(self, topic): """ topics = None if isinstance(topic, str): - self._check_topic(topic) + self._valid_topic(topic) topics = [(topic)] if isinstance(topic, list): topics = [] for t in topic: - self._check_topic(t) + self._valid_topic(t) topics.append((t)) for t in topics: if t not in self._subscribed_topics: @@ -854,7 +854,7 @@ def _wait_for_msg(self, timeout=0.1): """Reads and processes network events.""" res = bytearray(1) try: - self._sock.recv_into(res, 1) + self._recv_into(res, 1) except OSError as error: if error.errno == errno.ETIMEDOUT: # raised by a socketpool @@ -955,9 +955,9 @@ def _sock_exact_recv(self, bufsize): return rc def _send_str(self, string): - """Packs and encodes a string to a socket. - + """Encodes a string and sends it to a socket. :param str string: String to write to the socket. + """ self._sock.send(struct.pack("!H", len(string))) if isinstance(string, str): @@ -966,10 +966,10 @@ def _send_str(self, string): self._sock.send(string) @staticmethod - def _check_topic(topic): - """Checks if topic provided is a valid mqtt topic. - + def _valid_topic(topic): + """Validates if topic provided is proper MQTT topic format. :param str topic: Topic identifier + """ if topic is None: raise MMQTTException("Topic may not be NoneType") @@ -981,10 +981,10 @@ def _check_topic(topic): raise MMQTTException("Topic length is too large.") @staticmethod - def _check_qos(qos_level): - """Validates the quality of service level. - + def _valid_qos(qos_level): + """Validates if the QoS level is supported by this library :param int qos_level: Desired QoS level. + """ if isinstance(qos_level, int): if qos_level < 0 or qos_level > 2: @@ -992,16 +992,6 @@ def _check_qos(qos_level): else: raise MMQTTException("QoS must be an integer.") - def _set_interface(self): - """Sets a desired network hardware interface. - The network hardware must be set in init - prior to calling this method. - """ - if self._wifi: - self._socket.set_interface(self._wifi.esp) - else: - raise TypeError("Network Manager Required.") - def is_connected(self): """Returns MQTT client session status as True if connected, raises a `MMQTTException` if `False`. From 6e56ddbc57bb99a96899ceee091d1dc6bae3861a Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 12:22:09 -0500 Subject: [PATCH 49/77] remove shared rx_buffer, simplify logging API! --- adafruit_minimqtt/adafruit_minimqtt.py | 76 +++++++------------------- 1 file changed, 19 insertions(+), 57 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 2091d10d..c239d880 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -228,8 +228,6 @@ def __init__( self.on_subscribe = None self.on_unsubscribe = None - self._rx_buffer = bytearray(32) - # Socket helpers def _free_socket(self, socket): """Frees a socket for re-use.""" @@ -329,6 +327,20 @@ def deinit(self): """De-initializes the MQTT client and disconnects from the mqtt broker.""" self.disconnect() + @property + def mqtt_msg(self): + """Returns maximum MQTT payload and topic size.""" + return self._msg_size_lim, MQTT_TOPIC_LENGTH_LIMIT + + @mqtt_msg.setter + def mqtt_msg(self, msg_size): + """Sets the maximum MQTT message payload size. + + :param int msg_size: Maximum MQTT payload size. + """ + if msg_size < MQTT_MSG_MAX_SZ: + self._msg_size_lim = msg_size + def will_set(self, topic=None, payload=None, qos=0, retain=False): """Sets the last will and testament properties. MUST be called before `connect()`. @@ -857,10 +869,9 @@ def _wait_for_msg(self, timeout=0.1): self._recv_into(res, 1) except OSError as error: if error.errno == errno.ETIMEDOUT: - # raised by a socketpool + # raised by a socket timeout in socketpool/cpython return None - else: - raise MMQTTException(error) + raise MMQTTException(error) # Block while we parse the rest of the response self._sock.settimeout(timeout) @@ -1000,22 +1011,7 @@ def is_connected(self): raise MMQTTException("MiniMQTT is not connected.") return self._is_connected - @property - def mqtt_msg(self): - """Returns maximum MQTT payload and topic size.""" - return self._msg_size_lim, MQTT_TOPIC_LENGTH_LIMIT - - @mqtt_msg.setter - def mqtt_msg(self, msg_size): - """Sets the maximum MQTT message payload size. - - :param int msg_size: Maximum MQTT payload size. - """ - if msg_size < MQTT_MSG_MAX_SZ: - self._msg_size_lim = msg_size - - ### Logging API ### - + # Logging def enable_logger(self, logger, log_level=20): """Enables library logging provided a `logger` object. @@ -1023,45 +1019,11 @@ def enable_logger(self, logger, log_level=20): :param log_level: Numeric value of a logging level, defaults to `logging.INFO`. """ - self.logger = logging.getLogger("log") + self.logger = logger.getLogger("log") self.logger.setLevel(log_level) def disable_logger(self): """Disables logging.""" if not self.logger: - raise ValueError("Can't disable logging - no logger enabled!") + raise MMQTTException("Can not disable logger, no logger found.") self.logger = None - - def set_logger_level(self, log_level): - """Sets the level of the logger, if defined during init. - NOTE: This method is deprecated and will be removed. Use - `enable_logger`'s `log_level` as an alternative to this method. - - :param str log_level: Level of logging to output to the REPL. - Acceptable options are ``DEBUG``, ``INFO``, ``WARNING``, or - ``ERROR``. - """ - print("This method ") - if self.logger is None: - raise MMQTTException( - "No logger attached - did you create it during initialization?" - ) - if log_level == "DEBUG": - self.logger.setLevel(logging.DEBUG) - elif log_level == "INFO": - self.logger.setLevel(logging.INFO) - elif log_level == "WARNING": - self.logger.setLevel(logging.WARNING) - elif log_level == "ERROR": - self.logger.setLevel(logging.CRITICIAL) - else: - raise MMQTTException("Incorrect logging level provided!") - - def attach_logger(self, logger_name="log"): - """Initializes and attaches a logger to the MQTTClient. - :param str logger_name: Name of the logger instance - NOTE: This method is deprecated and will be removed. Use - `enable_logger` as an alternative to this method. - - """ - self.enable_logger() \ No newline at end of file From 609ab86d8a8c14e516520686d85f4a39d6224ddb Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 12:54:09 -0500 Subject: [PATCH 50/77] cpython catch --- adafruit_minimqtt/adafruit_minimqtt.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index c239d880..a2d2b014 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -865,13 +865,22 @@ def loop(self, timeout=1): def _wait_for_msg(self, timeout=0.1): """Reads and processes network events.""" res = bytearray(1) - try: - self._recv_into(res, 1) - except OSError as error: - if error.errno == errno.ETIMEDOUT: - # raised by a socket timeout in socketpool/cpython + + # CPython socket module contains a timeout attribute + if (hasattr(self._socket_pool, "timeout")): + try: + self._recv_into(res, 1) + except self._socket_pool.timeout as error: + print("timed out", error) return None - raise MMQTTException(error) + else: # socketpool, esp32spi + try: + self._recv_into(res, 1) + except OSError as error: + if error.errno == errno.ETIMEDOUT: + # raised by a socket timeout in socketpool + return None + raise MMQTTException(error) # Block while we parse the rest of the response self._sock.settimeout(timeout) @@ -906,7 +915,6 @@ def _wait_for_msg(self, timeout=0.1): assert 0 return res[0] - def _recv_len(self): """Unpack MQTT message length.""" n = 0 From 6db42d23e5d10a9c572610efeb46b24a3c702a2d Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 14:24:10 -0500 Subject: [PATCH 51/77] private user_data attribute! --- adafruit_minimqtt/adafruit_minimqtt.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index a2d2b014..d2a69c5a 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -256,6 +256,7 @@ def _free_sockets(self): for sock in free_sockets: self._close_socket(sock) + # pylint: disable=too-many-branches def _get_socket(self, host, port, *, timeout=1): key = (host, port) if key in self._open_sockets: @@ -459,7 +460,6 @@ def connect(self, clean_session=True, host=None, port=None, keep_alive=None): # NOTE: Variable header is # MQTT_HDR_CONNECT = bytearray(b"\x04MQTT\x04\x02\0\0") # because final 4 bytes are 4, 2, 0, 0 - # Variable Header var_header = MQTT_HDR_CONNECT var_header[6] = clean_session << 1 @@ -542,7 +542,7 @@ def disconnect(self): self._is_connected = False self._subscribed_topics = [] if self.on_disconnect is not None: - self.on_disconnect(self, self.user_data, 0) + self.on_disconnect(self, self._user_data, 0) def ping(self): """Pings the MQTT Broker to confirm if the broker is alive or if @@ -864,18 +864,16 @@ def loop(self, timeout=1): def _wait_for_msg(self, timeout=0.1): """Reads and processes network events.""" - res = bytearray(1) - # CPython socket module contains a timeout attribute if (hasattr(self._socket_pool, "timeout")): try: - self._recv_into(res, 1) + res = self._sock_exact_recv(1) except self._socket_pool.timeout as error: print("timed out", error) return None else: # socketpool, esp32spi try: - self._recv_into(res, 1) + res = self._sock_exact_recv(1) except OSError as error: if error.errno == errno.ETIMEDOUT: # raised by a socket timeout in socketpool From 1f6bfd621fc15cb9fdaa6201788d23362d6cd644 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 14:27:55 -0500 Subject: [PATCH 52/77] examples suffixed with folder name --- ...ifi.py => minimqtt_adafruitio_esp32spi.py} | 0 ...te.py => minimqtt_certificate_esp32spi.py} | 0 ... => minimqtt_pub_sub_blocking_esp32spi.py} | 0 ..._sub_blocking_topic_callbacks_esp32spi.py} | 0 ... minimqtt_pub_sub_nonblocking_esp32spi.py} | 0 ... => minimqtt_pub_sub_pyportal_esp32spi.py} | 0 ...est.py => minimqtt_simpletest_esp32spi.py} | 0 examples/minimqtt_simpletest_esp32s2.py | 2 - .../minimqtt_adafruitio_native_networking.py | 116 ++++++++++++++++++ 9 files changed, 116 insertions(+), 2 deletions(-) rename examples/esp32spi/{minimqtt_adafruitio_wifi.py => minimqtt_adafruitio_esp32spi.py} (100%) rename examples/esp32spi/{minimqtt_certificate.py => minimqtt_certificate_esp32spi.py} (100%) rename examples/esp32spi/{minimqtt_pub_sub_blocking.py => minimqtt_pub_sub_blocking_esp32spi.py} (100%) rename examples/esp32spi/{minimqtt_pub_sub_blocking_topic_callbacks.py => minimqtt_pub_sub_blocking_topic_callbacks_esp32spi.py} (100%) rename examples/esp32spi/{minimqtt_pub_sub_nonblocking.py => minimqtt_pub_sub_nonblocking_esp32spi.py} (100%) rename examples/esp32spi/{minimqtt_pub_sub_pyportal.py => minimqtt_pub_sub_pyportal_esp32spi.py} (100%) rename examples/esp32spi/{minimqtt_simpletest.py => minimqtt_simpletest_esp32spi.py} (100%) create mode 100644 examples/native_networking/minimqtt_adafruitio_native_networking.py diff --git a/examples/esp32spi/minimqtt_adafruitio_wifi.py b/examples/esp32spi/minimqtt_adafruitio_esp32spi.py similarity index 100% rename from examples/esp32spi/minimqtt_adafruitio_wifi.py rename to examples/esp32spi/minimqtt_adafruitio_esp32spi.py diff --git a/examples/esp32spi/minimqtt_certificate.py b/examples/esp32spi/minimqtt_certificate_esp32spi.py similarity index 100% rename from examples/esp32spi/minimqtt_certificate.py rename to examples/esp32spi/minimqtt_certificate_esp32spi.py diff --git a/examples/esp32spi/minimqtt_pub_sub_blocking.py b/examples/esp32spi/minimqtt_pub_sub_blocking_esp32spi.py similarity index 100% rename from examples/esp32spi/minimqtt_pub_sub_blocking.py rename to examples/esp32spi/minimqtt_pub_sub_blocking_esp32spi.py diff --git a/examples/esp32spi/minimqtt_pub_sub_blocking_topic_callbacks.py b/examples/esp32spi/minimqtt_pub_sub_blocking_topic_callbacks_esp32spi.py similarity index 100% rename from examples/esp32spi/minimqtt_pub_sub_blocking_topic_callbacks.py rename to examples/esp32spi/minimqtt_pub_sub_blocking_topic_callbacks_esp32spi.py diff --git a/examples/esp32spi/minimqtt_pub_sub_nonblocking.py b/examples/esp32spi/minimqtt_pub_sub_nonblocking_esp32spi.py similarity index 100% rename from examples/esp32spi/minimqtt_pub_sub_nonblocking.py rename to examples/esp32spi/minimqtt_pub_sub_nonblocking_esp32spi.py diff --git a/examples/esp32spi/minimqtt_pub_sub_pyportal.py b/examples/esp32spi/minimqtt_pub_sub_pyportal_esp32spi.py similarity index 100% rename from examples/esp32spi/minimqtt_pub_sub_pyportal.py rename to examples/esp32spi/minimqtt_pub_sub_pyportal_esp32spi.py diff --git a/examples/esp32spi/minimqtt_simpletest.py b/examples/esp32spi/minimqtt_simpletest_esp32spi.py similarity index 100% rename from examples/esp32spi/minimqtt_simpletest.py rename to examples/esp32spi/minimqtt_simpletest_esp32spi.py diff --git a/examples/minimqtt_simpletest_esp32s2.py b/examples/minimqtt_simpletest_esp32s2.py index 5dc37416..430a671e 100644 --- a/examples/minimqtt_simpletest_esp32s2.py +++ b/examples/minimqtt_simpletest_esp32s2.py @@ -1,7 +1,5 @@ # adafruit_minimqtt usage with native wifi import ssl -from random import randint -import adafruit_requests import socketpool import wifi import adafruit_minimqtt.adafruit_minimqtt as MQTT diff --git a/examples/native_networking/minimqtt_adafruitio_native_networking.py b/examples/native_networking/minimqtt_adafruitio_native_networking.py new file mode 100644 index 00000000..9a22616c --- /dev/null +++ b/examples/native_networking/minimqtt_adafruitio_native_networking.py @@ -0,0 +1,116 @@ +# Adafruit MiniMQTT Pub/Sub Example +# Written by Tony DiCola for Adafruit Industries +# Modified by Brent Rubell for Adafruit Industries +import time +import board +import busio +from digitalio import DigitalInOut +import neopixel +from adafruit_esp32spi import adafruit_esp32spi +from adafruit_esp32spi import adafruit_esp32spi_wifimanager +import adafruit_esp32spi.adafruit_esp32spi_socket as socket + +import adafruit_minimqtt.adafruit_minimqtt as MQTT + +### WiFi ### + +# Get wifi details and more from a secrets.py file +try: + from secrets import secrets +except ImportError: + print("WiFi secrets are kept in secrets.py, please add them there!") + raise + +# If you are using a board with pre-defined ESP32 Pins: +esp32_cs = DigitalInOut(board.ESP_CS) +esp32_ready = DigitalInOut(board.ESP_BUSY) +esp32_reset = DigitalInOut(board.ESP_RESET) + +# If you have an externally connected ESP32: +# esp32_cs = DigitalInOut(board.D9) +# esp32_ready = DigitalInOut(board.D10) +# esp32_reset = DigitalInOut(board.D5) + +spi = busio.SPI(board.SCK, board.MOSI, board.MISO) +esp = adafruit_esp32spi.ESP_SPIcontrol(spi, esp32_cs, esp32_ready, esp32_reset) +"""Use below for Most Boards""" +status_light = neopixel.NeoPixel( + board.NEOPIXEL, 1, brightness=0.2 +) # Uncomment for Most Boards +"""Uncomment below for ItsyBitsy M4""" +# status_light = dotstar.DotStar(board.APA102_SCK, board.APA102_MOSI, 1, brightness=0.2) +# Uncomment below for an externally defined RGB LED +# import adafruit_rgbled +# from adafruit_esp32spi import PWMOut +# RED_LED = PWMOut.PWMOut(esp, 26) +# GREEN_LED = PWMOut.PWMOut(esp, 27) +# BLUE_LED = PWMOut.PWMOut(esp, 25) +# status_light = adafruit_rgbled.RGBLED(RED_LED, BLUE_LED, GREEN_LED) +wifi = adafruit_esp32spi_wifimanager.ESPSPI_WiFiManager(esp, secrets, status_light) + +### Feeds ### + +# Setup a feed named 'photocell' for publishing to a feed +photocell_feed = secrets["aio_username"] + "/feeds/photocell" + +# Setup a feed named 'onoff' for subscribing to changes +onoff_feed = secrets["aio_username"] + "/feeds/onoff" + +### Code ### + +# Define callback methods which are called when events occur +# pylint: disable=unused-argument, redefined-outer-name +def connected(client, userdata, flags, rc): + # This function will be called when the client is connected + # successfully to the broker. + print("Connected to Adafruit IO! Listening for topic changes on %s" % onoff_feed) + # Subscribe to all changes on the onoff_feed. + client.subscribe(onoff_feed) + + +def disconnected(client, userdata, rc): + # This method is called when the client is disconnected + print("Disconnected from Adafruit IO!") + + +def message(client, topic, message): + # This method is called when a topic the client is subscribed to + # has a new message. + print("New message on topic {0}: {1}".format(topic, message)) + + +# Connect to WiFi +print("Connecting to WiFi...") +wifi.connect() +print("Connected!") + +# Initialize MQTT interface with the esp interface +MQTT.set_socket(socket, esp) + +# Set up a MiniMQTT Client +mqtt_client = MQTT.MQTT( + broker="io.adafruit.com", + username=secrets["aio_username"], + password=secrets["aio_key"], +) + +# Setup the callback methods above +mqtt_client.on_connect = connected +mqtt_client.on_disconnect = disconnected +mqtt_client.on_message = message + +# Connect the client to the MQTT broker. +print("Connecting to Adafruit IO...") +mqtt_client.connect() + +photocell_val = 0 +while True: + # Poll the message queue + mqtt_client.loop() + + # Send a new message + print("Sending photocell value: %d..." % photocell_val) + mqtt_client.publish(photocell_feed, photocell_val) + print("Sent!") + photocell_val += 1 + time.sleep(5) From 32bea5477f8114b655e88e4bed38f7394b83b866 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 14:30:08 -0500 Subject: [PATCH 53/77] port AIO to ESP32S2 example --- examples/minimqtt_simpletest_esp32s2.py | 9 --- .../minimqtt_adafruitio_native_networking.py | 71 ++++++------------- 2 files changed, 23 insertions(+), 57 deletions(-) diff --git a/examples/minimqtt_simpletest_esp32s2.py b/examples/minimqtt_simpletest_esp32s2.py index 430a671e..25ca99ae 100644 --- a/examples/minimqtt_simpletest_esp32s2.py +++ b/examples/minimqtt_simpletest_esp32s2.py @@ -24,15 +24,6 @@ wifi.radio.connect(secrets["ssid"], secrets["password"]) print("Connected to %s!"%secrets["ssid"]) - -### Secrets File Setup ### - -try: - from secrets import secrets -except ImportError: - print("Connection secrets are kept in secrets.py, please add them there!") - raise - ### Topic Setup ### # MQTT Topic diff --git a/examples/native_networking/minimqtt_adafruitio_native_networking.py b/examples/native_networking/minimqtt_adafruitio_native_networking.py index 9a22616c..86951244 100644 --- a/examples/native_networking/minimqtt_adafruitio_native_networking.py +++ b/examples/native_networking/minimqtt_adafruitio_native_networking.py @@ -2,52 +2,30 @@ # Written by Tony DiCola for Adafruit Industries # Modified by Brent Rubell for Adafruit Industries import time -import board -import busio -from digitalio import DigitalInOut -import neopixel -from adafruit_esp32spi import adafruit_esp32spi -from adafruit_esp32spi import adafruit_esp32spi_wifimanager -import adafruit_esp32spi.adafruit_esp32spi_socket as socket - +import ssl +import socketpool +import wifi import adafruit_minimqtt.adafruit_minimqtt as MQTT -### WiFi ### - -# Get wifi details and more from a secrets.py file +# Add a secrets.py to your filesystem that has a dictionary called secrets with "ssid" and +# "password" keys with your WiFi credentials. DO NOT share that file or commit it into Git or other +# source control. +# pylint: disable=no-name-in-module,wrong-import-order try: from secrets import secrets except ImportError: print("WiFi secrets are kept in secrets.py, please add them there!") raise -# If you are using a board with pre-defined ESP32 Pins: -esp32_cs = DigitalInOut(board.ESP_CS) -esp32_ready = DigitalInOut(board.ESP_BUSY) -esp32_reset = DigitalInOut(board.ESP_RESET) - -# If you have an externally connected ESP32: -# esp32_cs = DigitalInOut(board.D9) -# esp32_ready = DigitalInOut(board.D10) -# esp32_reset = DigitalInOut(board.D5) - -spi = busio.SPI(board.SCK, board.MOSI, board.MISO) -esp = adafruit_esp32spi.ESP_SPIcontrol(spi, esp32_cs, esp32_ready, esp32_reset) -"""Use below for Most Boards""" -status_light = neopixel.NeoPixel( - board.NEOPIXEL, 1, brightness=0.2 -) # Uncomment for Most Boards -"""Uncomment below for ItsyBitsy M4""" -# status_light = dotstar.DotStar(board.APA102_SCK, board.APA102_MOSI, 1, brightness=0.2) -# Uncomment below for an externally defined RGB LED -# import adafruit_rgbled -# from adafruit_esp32spi import PWMOut -# RED_LED = PWMOut.PWMOut(esp, 26) -# GREEN_LED = PWMOut.PWMOut(esp, 27) -# BLUE_LED = PWMOut.PWMOut(esp, 25) -# status_light = adafruit_rgbled.RGBLED(RED_LED, BLUE_LED, GREEN_LED) -wifi = adafruit_esp32spi_wifimanager.ESPSPI_WiFiManager(esp, secrets, status_light) +# Set your Adafruit IO Username and Key in secrets.py +# (visit io.adafruit.com if you need to create an account, +# or if you need your Adafruit IO key.) +aio_username = secrets["aio_username"] +aio_key = secrets["aio_key"] +print("Connecting to %s"%secrets["ssid"]) +wifi.radio.connect(secrets["ssid"], secrets["password"]) +print("Connected to %s!"%secrets["ssid"]) ### Feeds ### # Setup a feed named 'photocell' for publishing to a feed @@ -78,20 +56,17 @@ def message(client, topic, message): # has a new message. print("New message on topic {0}: {1}".format(topic, message)) - -# Connect to WiFi -print("Connecting to WiFi...") -wifi.connect() -print("Connected!") - -# Initialize MQTT interface with the esp interface -MQTT.set_socket(socket, esp) +# Create a socket pool +pool = socketpool.SocketPool(wifi.radio) # Set up a MiniMQTT Client mqtt_client = MQTT.MQTT( - broker="io.adafruit.com", - username=secrets["aio_username"], - password=secrets["aio_key"], + broker=secrets['broker'], + port=secrets['port'], + username=secrets['aio_username'], + password=secrets['aio_key'], + socket_pool=pool, + ssl_context= ssl.create_default_context() ) # Setup the callback methods above From 625fef6c7309d6319fadd00895e6b856301a63fb Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 14:34:11 -0500 Subject: [PATCH 54/77] add blocking example for nativenetworking --- ...test_esp32s2.py => minimqtt_simpletest.py} | 2 +- ...mqtt_pub_sub_blocking_native_networking.py | 87 +++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) rename examples/{minimqtt_simpletest_esp32s2.py => minimqtt_simpletest.py} (98%) create mode 100644 examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py diff --git a/examples/minimqtt_simpletest_esp32s2.py b/examples/minimqtt_simpletest.py similarity index 98% rename from examples/minimqtt_simpletest_esp32s2.py rename to examples/minimqtt_simpletest.py index 25ca99ae..2c7792cb 100644 --- a/examples/minimqtt_simpletest_esp32s2.py +++ b/examples/minimqtt_simpletest.py @@ -1,4 +1,4 @@ -# adafruit_minimqtt usage with native wifi +# adafruit_minimqtt usage with native networking import ssl import socketpool import wifi diff --git a/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py b/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py new file mode 100644 index 00000000..24ce95f6 --- /dev/null +++ b/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py @@ -0,0 +1,87 @@ +import time +import ssl +import socketpool +import wifi +import adafruit_minimqtt.adafruit_minimqtt as MQTT + +# Add a secrets.py to your filesystem that has a dictionary called secrets with "ssid" and +# "password" keys with your WiFi credentials. DO NOT share that file or commit it into Git or other +# source control. +# pylint: disable=no-name-in-module,wrong-import-order +try: + from secrets import secrets +except ImportError: + print("WiFi secrets are kept in secrets.py, please add them there!") + raise + +# Set your Adafruit IO Username and Key in secrets.py +# (visit io.adafruit.com if you need to create an account, +# or if you need your Adafruit IO key.) +aio_username = secrets["aio_username"] +aio_key = secrets["aio_key"] + +print("Connecting to %s"%secrets["ssid"]) +wifi.radio.connect(secrets["ssid"], secrets["password"]) +print("Connected to %s!"%secrets["ssid"]) + +### Adafruit IO Setup ### + +# Setup a feed named `testfeed` for publishing. +default_topic = secrets["aio_username"] + "/feeds/testfeed" + +### Code ### +# Define callback methods which are called when events occur +# pylint: disable=unused-argument, redefined-outer-name +def connected(client, userdata, flags, rc): + # This function will be called when the client is connected + # successfully to the broker. + print("Connected to MQTT broker! Listening for topic changes on %s" % default_topic) + # Subscribe to all changes on the default_topic feed. + client.subscribe(default_topic) + +def disconnected(client, userdata, rc): + # This method is called when the client is disconnected + print("Disconnected from MQTT Broker!") + +def message(client, topic, message): + """Method callled when a client's subscribed feed has a new + value. + :param str topic: The topic of the feed with a new value. + :param str message: The new value + """ + print("New message on topic {0}: {1}".format(topic, message)) + +# Create a socket pool +pool = socketpool.SocketPool(wifi.radio) + +# Set up a MiniMQTT Client +mqtt_client = MQTT.MQTT( + broker=secrets['broker'], + port=secrets['port'], + username=secrets['aio_username'], + password=secrets['aio_key'], + socket_pool=pool, + ssl_context= ssl.create_default_context() +) + +# Setup the callback methods above +mqtt_client.on_connect = connected +mqtt_client.on_disconnect = disconnected +mqtt_client.on_message = message + +# Connect the client to the MQTT broker. +print("Connecting to MQTT broker...") +mqtt_client.connect() + +# Start a blocking message loop... +# NOTE: NO code below this loop will execute +# NOTE: Network reconnection is handled within this loop +while True: + try: + mqtt_client.loop() + except (ValueError, RuntimeError) as e: + print("Failed to get data, retrying\n", e) + wifi.reset() + mqtt_client.reconnect() + continue + time.sleep(1) From 07c417ff843d252604a0b1e75e68f81809374c2f Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 14:43:21 -0500 Subject: [PATCH 55/77] _callbacks should interface with Adafruit IO groups instead of traditional topic format for testing --- ...b_sub_blocking_topic_callbacks_esp32spi.py | 8 +- ...cking_topic_callbacks_native_networking.py | 102 ++++++++++++++++++ 2 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py diff --git a/examples/esp32spi/minimqtt_pub_sub_blocking_topic_callbacks_esp32spi.py b/examples/esp32spi/minimqtt_pub_sub_blocking_topic_callbacks_esp32spi.py index 8b2ebc3e..f7679a0d 100644 --- a/examples/esp32spi/minimqtt_pub_sub_blocking_topic_callbacks_esp32spi.py +++ b/examples/esp32spi/minimqtt_pub_sub_blocking_topic_callbacks_esp32spi.py @@ -74,7 +74,7 @@ def on_battery_msg(client, topic, message): # Method called when device/batteryLife has a new value print("Battery level: {}v".format(message)) - # client.remove_topic_callback("device/batteryLevel") + # client.remove_topic_callback(secrets["aio_username"] + "/feeds/device.batterylevel") def on_message(client, topic, message): @@ -98,14 +98,14 @@ def on_message(client, topic, message): client.on_subscribe = subscribe client.on_unsubscribe = unsubscribe client.on_message = on_message -client.add_topic_callback("device/batteryLevel", on_battery_msg) +client.add_topic_callback(secrets["aio_username"] + "/feeds/device.batterylevel", on_battery_msg) # Connect the client to the MQTT broker. print("Connecting to MQTT broker...") client.connect() -# Subscribe to all notifications on the device/ topic -client.subscribe("device/#", 1) +# Subscribe to all notifications on the device group +client.subscribe(secrets["aio_username"] + "/groups/device", 1) # Start a blocking message loop... # NOTE: NO code below this loop will execute diff --git a/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py b/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py new file mode 100644 index 00000000..24756aac --- /dev/null +++ b/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py @@ -0,0 +1,102 @@ +import time +import time +import ssl +import socketpool +import wifi +import adafruit_minimqtt.adafruit_minimqtt as MQTT + +# Add a secrets.py to your filesystem that has a dictionary called secrets with "ssid" and +# "password" keys with your WiFi credentials. DO NOT share that file or commit it into Git or other +# source control. +# pylint: disable=no-name-in-module,wrong-import-order +try: + from secrets import secrets +except ImportError: + print("WiFi secrets are kept in secrets.py, please add them there!") + raise + +# Set your Adafruit IO Username and Key in secrets.py +# (visit io.adafruit.com if you need to create an account, +# or if you need your Adafruit IO key.) +aio_username = secrets["aio_username"] +aio_key = secrets["aio_key"] + +print("Connecting to %s"%secrets["ssid"]) +wifi.radio.connect(secrets["ssid"], secrets["password"]) +print("Connected to %s!"%secrets["ssid"]) + +### Code ### + +# Define callback methods which are called when events occur +# pylint: disable=unused-argument, redefined-outer-name +def connected(client, userdata, flags, rc): + # This function will be called when the client is connected + # successfully to the broker. + print("Connected to MQTT Broker!") + + +def disconnected(client, userdata, rc): + # This method is called when the client is disconnected + print("Disconnected from MQTT Broker!") + + +def subscribe(client, userdata, topic, granted_qos): + # This method is called when the client subscribes to a new feed. + print("Subscribed to {0} with QOS level {1}".format(topic, granted_qos)) + + +def unsubscribe(client, userdata, topic, pid): + # This method is called when the client unsubscribes from a feed. + print("Unsubscribed from {0} with PID {1}".format(topic, pid)) + + +def on_battery_msg(client, topic, message): + # Method called when device/batteryLife has a new value + print("Battery level: {}v".format(message)) + + # client.remove_topic_callback(secrets["aio_username"] + "/feeds/device.batterylevel") + + +def on_message(client, topic, message): + # Method callled when a client's subscribed feed has a new value. + print("New message on topic {0}: {1}".format(topic, message)) + +# Create a socket pool +pool = socketpool.SocketPool(wifi.radio) + +# Set up a MiniMQTT Client +client = MQTT.MQTT( + broker=secrets['broker'], + port=secrets['port'], + username=secrets['aio_username'], + password=secrets['aio_key'], + socket_pool=pool, + ssl_context= ssl.create_default_context() +) + +# Setup the callback methods above +client.on_connect = connected +client.on_disconnect = disconnected +client.on_subscribe = subscribe +client.on_unsubscribe = unsubscribe +client.on_message = on_message +client.add_topic_callback(secrets["aio_username"] + "/feeds/device.batterylevel", on_battery_msg) + +# Connect the client to the MQTT broker. +print("Connecting to MQTT broker...") +client.connect() + +# Subscribe to all notifications on the device group +client.subscribe(secrets["aio_username"] + "/groups/device", 1) + +# Start a blocking message loop... +# NOTE: NO code below this loop will execute +while True: + try: + client.loop() + except (ValueError, RuntimeError) as e: + print("Failed to get data, retrying\n", e) + wifi.reset() + client.reconnect() + continue + time.sleep(1) From cfb61c348e38d124a0fae2b62bee9d35521fa640 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 14:52:30 -0500 Subject: [PATCH 56/77] CPython examples non-ssl --- examples/cpython/minimqtt_adafruitio_cpython.py | 2 +- examples/cpython/minimqtt_simpletest_cpython.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cpython/minimqtt_adafruitio_cpython.py b/examples/cpython/minimqtt_adafruitio_cpython.py index b4a853b1..10a81fd1 100644 --- a/examples/cpython/minimqtt_adafruitio_cpython.py +++ b/examples/cpython/minimqtt_adafruitio_cpython.py @@ -47,7 +47,7 @@ def message(client, topic, message): # Set up a MiniMQTT Client mqtt_client = MQTT.MQTT( broker=secrets['broker'], - port=secrets['port'], + port=1883, username=secrets['aio_username'], password=secrets['aio_key'], socket_pool=socket diff --git a/examples/cpython/minimqtt_simpletest_cpython.py b/examples/cpython/minimqtt_simpletest_cpython.py index 64aeea66..5521ff10 100644 --- a/examples/cpython/minimqtt_simpletest_cpython.py +++ b/examples/cpython/minimqtt_simpletest_cpython.py @@ -67,7 +67,7 @@ def message(client, topic, message): # Set up a MiniMQTT Client mqtt_client = MQTT.MQTT( broker=secrets['broker'], - port=secrets['port'], + port=1883, username=secrets['aio_username'], password=secrets['aio_key'], socket_pool=socket From 0a81e84be1afcccb1b769d75030094d0f4f99045 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 15:30:14 -0500 Subject: [PATCH 57/77] add licensing info --- examples/cellular/minimqtt_adafruitio_cellular.py | 4 ++++ examples/cellular/minimqtt_simpletest_cellular.py | 3 +++ examples/cpython/minimqtt_adafruitio_cpython.py | 7 ++++--- examples/cpython/minimqtt_simpletest_cpython.py | 3 +++ examples/ethernet/minimqtt_adafruitio_eth.py | 7 ++++--- examples/ethernet/minimqtt_simpletest_eth.py | 3 +++ examples/minimqtt_simpletest.py | 3 +++ .../minimqtt_adafruitio_native_networking.py | 7 ++++--- .../minimqtt_pub_sub_blocking_native_networking.py | 3 +++ ...t_pub_sub_blocking_topic_callbacks_native_networking.py | 4 +++- 10 files changed, 34 insertions(+), 10 deletions(-) diff --git a/examples/cellular/minimqtt_adafruitio_cellular.py b/examples/cellular/minimqtt_adafruitio_cellular.py index 724f4f06..7995fdf4 100755 --- a/examples/cellular/minimqtt_adafruitio_cellular.py +++ b/examples/cellular/minimqtt_adafruitio_cellular.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Tony DiCola for Adafruit Industries +# SPDX-FileCopyrightText: 2020 Brent Rubell for Adafruit Industries +# +# SPDX-License-Identifier: Unlicense import time import board import busio diff --git a/examples/cellular/minimqtt_simpletest_cellular.py b/examples/cellular/minimqtt_simpletest_cellular.py index d85bb561..d0080d94 100644 --- a/examples/cellular/minimqtt_simpletest_cellular.py +++ b/examples/cellular/minimqtt_simpletest_cellular.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2020 Brent Rubell for Adafruit Industries +# +# SPDX-License-Identifier: Unlicense import time import board import busio diff --git a/examples/cpython/minimqtt_adafruitio_cpython.py b/examples/cpython/minimqtt_adafruitio_cpython.py index 10a81fd1..bd36c95f 100644 --- a/examples/cpython/minimqtt_adafruitio_cpython.py +++ b/examples/cpython/minimqtt_adafruitio_cpython.py @@ -1,6 +1,7 @@ -# Adafruit MiniMQTT Pub/Sub Example -# Written by Tony DiCola for Adafruit Industries -# Modified by Brent Rubell for Adafruit Industries +# SPDX-FileCopyrightText: Tony DiCola for Adafruit Industries +# SPDX-FileCopyrightText: 2020 Brent Rubell for Adafruit Industries +# +# SPDX-License-Identifier: Unlicense import time import socket import adafruit_minimqtt.adafruit_minimqtt as MQTT diff --git a/examples/cpython/minimqtt_simpletest_cpython.py b/examples/cpython/minimqtt_simpletest_cpython.py index 5521ff10..7c373e84 100644 --- a/examples/cpython/minimqtt_simpletest_cpython.py +++ b/examples/cpython/minimqtt_simpletest_cpython.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2020 Brent Rubell for Adafruit Industries +# +# SPDX-License-Identifier: Unlicense import socket import adafruit_minimqtt.adafruit_minimqtt as MQTT diff --git a/examples/ethernet/minimqtt_adafruitio_eth.py b/examples/ethernet/minimqtt_adafruitio_eth.py index 9db6cf88..3f0d85d6 100755 --- a/examples/ethernet/minimqtt_adafruitio_eth.py +++ b/examples/ethernet/minimqtt_adafruitio_eth.py @@ -1,6 +1,7 @@ -# Adafruit MiniMQTT Pub/Sub Example -# Written by Tony DiCola for Adafruit Industries -# Modified by Brent Rubell for Adafruit Industries +# SPDX-FileCopyrightText: Tony DiCola for Adafruit Industries +# SPDX-FileCopyrightText: 2020 Brent Rubell for Adafruit Industries +# +# SPDX-License-Identifier: Unlicense import time import board import busio diff --git a/examples/ethernet/minimqtt_simpletest_eth.py b/examples/ethernet/minimqtt_simpletest_eth.py index 55a649db..c2be279a 100644 --- a/examples/ethernet/minimqtt_simpletest_eth.py +++ b/examples/ethernet/minimqtt_simpletest_eth.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2020 Brent Rubell for Adafruit Industries +# +# SPDX-License-Identifier: Unlicense import board import busio from digitalio import DigitalInOut diff --git a/examples/minimqtt_simpletest.py b/examples/minimqtt_simpletest.py index 2c7792cb..0ac65f5a 100644 --- a/examples/minimqtt_simpletest.py +++ b/examples/minimqtt_simpletest.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2020 Brent Rubell, written for Adafruit Industries +# +# SPDX-License-Identifier: Unlicense # adafruit_minimqtt usage with native networking import ssl import socketpool diff --git a/examples/native_networking/minimqtt_adafruitio_native_networking.py b/examples/native_networking/minimqtt_adafruitio_native_networking.py index 86951244..f38340ec 100644 --- a/examples/native_networking/minimqtt_adafruitio_native_networking.py +++ b/examples/native_networking/minimqtt_adafruitio_native_networking.py @@ -1,6 +1,7 @@ -# Adafruit MiniMQTT Pub/Sub Example -# Written by Tony DiCola for Adafruit Industries -# Modified by Brent Rubell for Adafruit Industries +# SPDX-FileCopyrightText: Tony DiCola for Adafruit Industries +# SPDX-FileCopyrightText: 2020 Brent Rubell for Adafruit Industries +# +# SPDX-License-Identifier: Unlicense import time import ssl import socketpool diff --git a/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py b/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py index 24ce95f6..e861dc71 100644 --- a/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py +++ b/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2020 Brent Rubell, written for Adafruit Industries +# +# SPDX-License-Identifier: Unlicense import time import ssl import socketpool diff --git a/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py b/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py index 24756aac..af7af0a7 100644 --- a/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py +++ b/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py @@ -1,4 +1,6 @@ -import time +# SPDX-FileCopyrightText: 2020 Brent Rubell, written for Adafruit Industries +# +# SPDX-License-Identifier: Unlicense import time import ssl import socketpool From 2fa79ba1d48dc56c4286252ffcee2222790cc941 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 15:31:35 -0500 Subject: [PATCH 58/77] remove logging dep, yay --- README.rst | 1 - docs/conf.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 8ee45df5..ec635da8 100644 --- a/README.rst +++ b/README.rst @@ -20,7 +20,6 @@ Dependencies This driver depends on: * `Adafruit CircuitPython `_ -* `Adafruit Logging `_ Please ensure all dependencies are available on the CircuitPython filesystem. This is easily achieved by downloading diff --git a/docs/conf.py b/docs/conf.py index 8b5279f6..cd797086 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ # Uncomment the below if you use native CircuitPython modules such as # digitalio, micropython and busio. List the modules you use. Without it, the # autodoc module docs will fail to generate with a warning. -autodoc_mock_imports = ["micropython", "microcontroller", "random", "adafruit_logging"] +autodoc_mock_imports = ["micropython", "microcontroller", "random"] intersphinx_mapping = { From 629ecb0211cbbd2b470421b3c3046697f161c7c7 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 15:40:30 -0500 Subject: [PATCH 59/77] black/lint all --- adafruit_minimqtt/adafruit_minimqtt.py | 41 ++++++++++--------- adafruit_minimqtt/matcher.py | 3 +- examples/.vscode/settings.json | 3 ++ .../cpython/minimqtt_adafruitio_cpython.py | 8 ++-- .../cpython/minimqtt_simpletest_cpython.py | 24 ++++------- ...b_sub_blocking_topic_callbacks_esp32spi.py | 4 +- .../esp32spi/minimqtt_simpletest_esp32spi.py | 25 ++++------- examples/minimqtt_simpletest.py | 30 +++++--------- .../minimqtt_adafruitio_native_networking.py | 15 +++---- ...mqtt_pub_sub_blocking_native_networking.py | 17 ++++---- ...cking_topic_callbacks_native_networking.py | 19 +++++---- 11 files changed, 88 insertions(+), 101 deletions(-) create mode 100644 examples/.vscode/settings.json diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index d2a69c5a..1ea74464 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -81,8 +81,10 @@ _the_interface = None # pylint: disable=invalid-name _the_sock = None # pylint: disable=invalid-name + class MMQTTException(Exception): """MiniMQTT Exception class.""" + # pylint: disable=unnecessary-pass # pass @@ -101,6 +103,7 @@ def set_socket(sock, iface=None): _the_interface = iface _the_sock.set_interface(iface) + class _FakeSSLSocket: def __init__(self, socket, tls_mode): self._socket = socket @@ -117,6 +120,7 @@ def connect(self, address): except RuntimeError as error: raise OSError(errno.ENOMEM) from error + class _FakeSSLContext: def __init__(self, iface): self._iface = iface @@ -126,6 +130,7 @@ def wrap_socket(self, socket, server_hostname=None): # pylint: disable=unused-argument return _FakeSSLSocket(socket, self._iface.TLS_MODE) + class MQTT: """MQTT Client for CircuitPython @@ -141,6 +146,7 @@ class MQTT: :param ssl_context: SSL context for long-lived SSL connections. """ + # pylint: disable=too-many-arguments,too-many-instance-attributes, not-callable, invalid-name, no-member def __init__( self, @@ -152,7 +158,7 @@ def __init__( is_ssl=True, keep_alive=60, socket_pool=None, - ssl_context=None + ssl_context=None, ): self._socket_pool = socket_pool @@ -183,9 +189,8 @@ def __init__( self._username = username self._password = password if ( - self._password - and len(password.encode("utf-8")) > MQTT_TOPIC_LENGTH_LIMIT - ): # [MQTT-3.1.3.5] + self._password and len(password.encode("utf-8")) > MQTT_TOPIC_LENGTH_LIMIT + ): # [MQTT-3.1.3.5] raise MMQTTException("Password length is too large.") self.port = MQTT_TCP_PORT @@ -208,7 +213,6 @@ def __init__( if len(self.client_id) > 23 or not self.client_id: raise ValueError("MQTT Client ID must be between 1 and 23 bytes") - # LWT self._lw_topic = None self._lw_qos = 0 @@ -295,8 +299,7 @@ def _get_socket(self, host, port, *, timeout=1): connect_host = addr_info[-1][0] if port == 8883: - sock = self._ssl_context.wrap_socket(sock, - server_hostname=host) + sock = self._ssl_context.wrap_socket(sock, server_hostname=host) connect_host = host sock.settimeout(timeout) @@ -497,7 +500,9 @@ def connect(self, clean_session=True, host=None, port=None, keep_alive=None): if self.logger: self.logger.debug("Sending CONNECT to broker...") - self.logger.debug("Fixed Header: %x\nVariable Header: %x", fixed_header, var_header) + self.logger.debug( + "Fixed Header: %x\nVariable Header: %x", fixed_header, var_header + ) self._sock.send(fixed_header) self._sock.send(var_header) # [MQTT-3.1.3-4] @@ -644,7 +649,10 @@ def publish(self, topic, msg, retain=False, qos=0): self.logger.debug( "Sending PUBLISH\nTopic: %s\nMsg: %x\ \nQoS: %d\nRetain? %r", - topic, msg, qos, retain + topic, + msg, + qos, + retain, ) self._sock.send(pub_hdr_fixed) self._sock.send(pub_hdr_var) @@ -806,10 +814,7 @@ def unsubscribe(self, topic): rc = self._sock_exact_recv(3) assert rc[0] == 0x02 # [MQTT-3.32] - assert ( - rc[1] == packet_id_bytes[0] - and rc[2] == packet_id_bytes[1] - ) + assert rc[1] == packet_id_bytes[0] and rc[2] == packet_id_bytes[1] for t in topics: if self.on_unsubscribe is not None: self.on_unsubscribe(self, self._user_data, t, self._pid) @@ -861,17 +866,16 @@ def loop(self, timeout=1): rc = self._wait_for_msg() return [rc] if rc else None - def _wait_for_msg(self, timeout=0.1): """Reads and processes network events.""" # CPython socket module contains a timeout attribute - if (hasattr(self._socket_pool, "timeout")): + if hasattr(self._socket_pool, "timeout"): try: res = self._sock_exact_recv(1) except self._socket_pool.timeout as error: print("timed out", error) return None - else: # socketpool, esp32spi + else: # socketpool, esp32spi try: res = self._sock_exact_recv(1) except OSError as error: @@ -926,8 +930,7 @@ def _recv_len(self): sh += 7 def _recv_into(self, buf, size=0): - """Backwards-compatible _recv_into implementation. - """ + """Backwards-compatible _recv_into implementation.""" if self._backwards_compatible_sock: size = len(buf) if size == 0 else size b = self._sock.recv(size) @@ -952,7 +955,7 @@ def _sock_exact_recv(self, bufsize): # CPython/Socketpool Impl. rc = bytearray(bufsize) self._sock.recv_into(rc, bufsize) - else: # ESP32SPI Impl. + else: # ESP32SPI Impl. stamp = time.monotonic() read_timeout = self.keep_alive rc = self._sock.recv(bufsize) diff --git a/adafruit_minimqtt/matcher.py b/adafruit_minimqtt/matcher.py index 0b56faac..b2d43ba8 100755 --- a/adafruit_minimqtt/matcher.py +++ b/adafruit_minimqtt/matcher.py @@ -25,8 +25,7 @@ class MQTTMatcher: # pylint: disable=too-few-public-methods class Node: - """Individual node on the MQTT prefix tree. - """ + """Individual node on the MQTT prefix tree.""" __slots__ = "children", "content" diff --git a/examples/.vscode/settings.json b/examples/.vscode/settings.json new file mode 100644 index 00000000..3cce948f --- /dev/null +++ b/examples/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "restructuredtext.confPath": "" +} \ No newline at end of file diff --git a/examples/cpython/minimqtt_adafruitio_cpython.py b/examples/cpython/minimqtt_adafruitio_cpython.py index bd36c95f..9b2b276b 100644 --- a/examples/cpython/minimqtt_adafruitio_cpython.py +++ b/examples/cpython/minimqtt_adafruitio_cpython.py @@ -47,11 +47,11 @@ def message(client, topic, message): # Set up a MiniMQTT Client mqtt_client = MQTT.MQTT( - broker=secrets['broker'], + broker=secrets["broker"], port=1883, - username=secrets['aio_username'], - password=secrets['aio_key'], - socket_pool=socket + username=secrets["aio_username"], + password=secrets["aio_key"], + socket_pool=socket, ) # Setup the callback methods above diff --git a/examples/cpython/minimqtt_simpletest_cpython.py b/examples/cpython/minimqtt_simpletest_cpython.py index 7c373e84..799a2074 100644 --- a/examples/cpython/minimqtt_simpletest_cpython.py +++ b/examples/cpython/minimqtt_simpletest_cpython.py @@ -16,11 +16,11 @@ # MQTT Topic # Use this topic if you'd like to connect to a standard MQTT broker -#mqtt_topic = "test/topic" +# mqtt_topic = "test/topic" # Adafruit IO-style Topic # Use this topic if you'd like to connect to io.adafruit.com -mqtt_topic = secrets["aio_username"] + '/feeds/temperature' +mqtt_topic = secrets["aio_username"] + "/feeds/temperature" ### Code ### @@ -59,21 +59,16 @@ def publish(mqtt_client, userdata, topic, pid): def message(client, topic, message): # Method callled when a client's subscribed feed has a new value. - global disconnect_client print("New message on topic {0}: {1}".format(topic, message)) - - print("Unsubscribing from %s" % mqtt_topic) - mqtt_client.unsubscribe(mqtt_topic) - # Allow us to gracefully stop the `while True` loop - disconnect_client = True + # Set up a MiniMQTT Client mqtt_client = MQTT.MQTT( - broker=secrets['broker'], + broker=secrets["broker"], port=1883, - username=secrets['aio_username'], - password=secrets['aio_key'], - socket_pool=socket + username=secrets["aio_username"], + password=secrets["aio_key"], + socket_pool=socket, ) # Connect callback handlers to mqtt_client @@ -93,9 +88,8 @@ def message(client, topic, message): print("Publishing to %s" % mqtt_topic) mqtt_client.publish(mqtt_topic, "Hello Broker!") -# Pump the loop until we receive a message on `mqtt_topic` -while disconnect_client == False: - mqtt_client.loop() +print("Unsubscribing from %s" % mqtt_topic) +mqtt_client.unsubscribe(mqtt_topic) print("Disconnecting from %s" % mqtt_client.broker) mqtt_client.disconnect() diff --git a/examples/esp32spi/minimqtt_pub_sub_blocking_topic_callbacks_esp32spi.py b/examples/esp32spi/minimqtt_pub_sub_blocking_topic_callbacks_esp32spi.py index f7679a0d..4dc02709 100644 --- a/examples/esp32spi/minimqtt_pub_sub_blocking_topic_callbacks_esp32spi.py +++ b/examples/esp32spi/minimqtt_pub_sub_blocking_topic_callbacks_esp32spi.py @@ -98,7 +98,9 @@ def on_message(client, topic, message): client.on_subscribe = subscribe client.on_unsubscribe = unsubscribe client.on_message = on_message -client.add_topic_callback(secrets["aio_username"] + "/feeds/device.batterylevel", on_battery_msg) +client.add_topic_callback( + secrets["aio_username"] + "/feeds/device.batterylevel", on_battery_msg +) # Connect the client to the MQTT broker. print("Connecting to MQTT broker...") diff --git a/examples/esp32spi/minimqtt_simpletest_esp32spi.py b/examples/esp32spi/minimqtt_simpletest_esp32spi.py index 9292847e..f1c50d85 100644 --- a/examples/esp32spi/minimqtt_simpletest_esp32spi.py +++ b/examples/esp32spi/minimqtt_simpletest_esp32spi.py @@ -2,7 +2,6 @@ import board import busio from digitalio import DigitalInOut -import neopixel from adafruit_esp32spi import adafruit_esp32spi import adafruit_esp32spi.adafruit_esp32spi_socket as socket import adafruit_minimqtt.adafruit_minimqtt as MQTT @@ -57,9 +56,6 @@ ### Code ### -# Keep track of client connection state -disconnect_client = False - # Define callback methods which are called when events occur # pylint: disable=unused-argument, redefined-outer-name def connect(mqtt_client, userdata, flags, rc): @@ -91,24 +87,18 @@ def publish(mqtt_client, userdata, topic, pid): def message(client, topic, message): - # Method callled when a client's subscribed feed has a new value. - global disconnect_client print("New message on topic {0}: {1}".format(topic, message)) - - print("Unsubscribing from %s" % mqtt_topic) - mqtt_client.unsubscribe(mqtt_topic) - # Allow us to gracefully stop the `while True` loop - disconnect_client = True + socket.set_interface(esp) MQTT.set_socket(socket, esp) # Set up a MiniMQTT Client mqtt_client = MQTT.MQTT( - broker=secrets['broker'], - port=secrets['port'], - username=secrets['username'], - password=secrets['password'] + broker=secrets["broker"], + port=secrets["port"], + username=secrets["username"], + password=secrets["password"], ) # Connect callback handlers to mqtt_client @@ -128,9 +118,8 @@ def message(client, topic, message): print("Publishing to %s" % mqtt_topic) mqtt_client.publish(mqtt_topic, "Hello Broker!") -# Pump the loop until we receive a message on `mqtt_topic` -while disconnect_client == False: - mqtt_client.loop() +print("Unsubscribing from %s" % mqtt_topic) +mqtt_client.unsubscribe(mqtt_topic) print("Disconnecting from %s" % mqtt_client.broker) mqtt_client.disconnect() diff --git a/examples/minimqtt_simpletest.py b/examples/minimqtt_simpletest.py index 0ac65f5a..307151af 100644 --- a/examples/minimqtt_simpletest.py +++ b/examples/minimqtt_simpletest.py @@ -23,9 +23,9 @@ aio_username = secrets["aio_username"] aio_key = secrets["aio_key"] -print("Connecting to %s"%secrets["ssid"]) +print("Connecting to %s" % secrets["ssid"]) wifi.radio.connect(secrets["ssid"], secrets["password"]) -print("Connected to %s!"%secrets["ssid"]) +print("Connected to %s!" % secrets["ssid"]) ### Topic Setup ### @@ -38,10 +38,6 @@ # mqtt_topic = secrets["aio_username"] + '/feeds/temperature' ### Code ### - -# Keep track of client connection state -disconnect_client = False - # Define callback methods which are called when events occur # pylint: disable=unused-argument, redefined-outer-name def connect(mqtt_client, userdata, flags, rc): @@ -74,25 +70,20 @@ def publish(mqtt_client, userdata, topic, pid): def message(client, topic, message): # Method callled when a client's subscribed feed has a new value. - global disconnect_client print("New message on topic {0}: {1}".format(topic, message)) - - print("Unsubscribing from %s" % mqtt_topic) - mqtt_client.unsubscribe(mqtt_topic) - # Allow us to gracefully stop the `while True` loop - disconnect_client = True + # Create a socket pool pool = socketpool.SocketPool(wifi.radio) # Set up a MiniMQTT Client mqtt_client = MQTT.MQTT( - broker=secrets['broker'], - port=secrets['port'], - username=secrets['aio_username'], - password=secrets['aio_key'], + broker=secrets["broker"], + port=secrets["port"], + username=secrets["aio_username"], + password=secrets["aio_key"], socket_pool=pool, - ssl_context= ssl.create_default_context() + ssl_context=ssl.create_default_context(), ) # Connect callback handlers to mqtt_client @@ -112,9 +103,8 @@ def message(client, topic, message): print("Publishing to %s" % mqtt_topic) mqtt_client.publish(mqtt_topic, "Hello Broker!") -# Pump the loop until we receive a message on `mqtt_topic` -while disconnect_client == False: - mqtt_client.loop() +print("Unsubscribing from %s" % mqtt_topic) +mqtt_client.unsubscribe(mqtt_topic) print("Disconnecting from %s" % mqtt_client.broker) mqtt_client.disconnect() diff --git a/examples/native_networking/minimqtt_adafruitio_native_networking.py b/examples/native_networking/minimqtt_adafruitio_native_networking.py index f38340ec..b6fedcd7 100644 --- a/examples/native_networking/minimqtt_adafruitio_native_networking.py +++ b/examples/native_networking/minimqtt_adafruitio_native_networking.py @@ -24,9 +24,9 @@ aio_username = secrets["aio_username"] aio_key = secrets["aio_key"] -print("Connecting to %s"%secrets["ssid"]) +print("Connecting to %s" % secrets["ssid"]) wifi.radio.connect(secrets["ssid"], secrets["password"]) -print("Connected to %s!"%secrets["ssid"]) +print("Connected to %s!" % secrets["ssid"]) ### Feeds ### # Setup a feed named 'photocell' for publishing to a feed @@ -57,17 +57,18 @@ def message(client, topic, message): # has a new message. print("New message on topic {0}: {1}".format(topic, message)) + # Create a socket pool pool = socketpool.SocketPool(wifi.radio) # Set up a MiniMQTT Client mqtt_client = MQTT.MQTT( - broker=secrets['broker'], - port=secrets['port'], - username=secrets['aio_username'], - password=secrets['aio_key'], + broker=secrets["broker"], + port=secrets["port"], + username=secrets["aio_username"], + password=secrets["aio_key"], socket_pool=pool, - ssl_context= ssl.create_default_context() + ssl_context=ssl.create_default_context(), ) # Setup the callback methods above diff --git a/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py b/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py index e861dc71..647da483 100644 --- a/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py +++ b/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py @@ -23,9 +23,9 @@ aio_username = secrets["aio_username"] aio_key = secrets["aio_key"] -print("Connecting to %s"%secrets["ssid"]) +print("Connecting to %s" % secrets["ssid"]) wifi.radio.connect(secrets["ssid"], secrets["password"]) -print("Connected to %s!"%secrets["ssid"]) +print("Connected to %s!" % secrets["ssid"]) ### Adafruit IO Setup ### @@ -42,10 +42,12 @@ def connected(client, userdata, flags, rc): # Subscribe to all changes on the default_topic feed. client.subscribe(default_topic) + def disconnected(client, userdata, rc): # This method is called when the client is disconnected print("Disconnected from MQTT Broker!") + def message(client, topic, message): """Method callled when a client's subscribed feed has a new value. @@ -54,17 +56,18 @@ def message(client, topic, message): """ print("New message on topic {0}: {1}".format(topic, message)) + # Create a socket pool pool = socketpool.SocketPool(wifi.radio) # Set up a MiniMQTT Client mqtt_client = MQTT.MQTT( - broker=secrets['broker'], - port=secrets['port'], - username=secrets['aio_username'], - password=secrets['aio_key'], + broker=secrets["broker"], + port=secrets["port"], + username=secrets["aio_username"], + password=secrets["aio_key"], socket_pool=pool, - ssl_context= ssl.create_default_context() + ssl_context=ssl.create_default_context(), ) # Setup the callback methods above diff --git a/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py b/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py index af7af0a7..e9cdaf6e 100644 --- a/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py +++ b/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py @@ -23,9 +23,9 @@ aio_username = secrets["aio_username"] aio_key = secrets["aio_key"] -print("Connecting to %s"%secrets["ssid"]) +print("Connecting to %s" % secrets["ssid"]) wifi.radio.connect(secrets["ssid"], secrets["password"]) -print("Connected to %s!"%secrets["ssid"]) +print("Connected to %s!" % secrets["ssid"]) ### Code ### @@ -63,17 +63,18 @@ def on_message(client, topic, message): # Method callled when a client's subscribed feed has a new value. print("New message on topic {0}: {1}".format(topic, message)) + # Create a socket pool pool = socketpool.SocketPool(wifi.radio) # Set up a MiniMQTT Client client = MQTT.MQTT( - broker=secrets['broker'], - port=secrets['port'], - username=secrets['aio_username'], - password=secrets['aio_key'], + broker=secrets["broker"], + port=secrets["port"], + username=secrets["aio_username"], + password=secrets["aio_key"], socket_pool=pool, - ssl_context= ssl.create_default_context() + ssl_context=ssl.create_default_context(), ) # Setup the callback methods above @@ -82,7 +83,9 @@ def on_message(client, topic, message): client.on_subscribe = subscribe client.on_unsubscribe = unsubscribe client.on_message = on_message -client.add_topic_callback(secrets["aio_username"] + "/feeds/device.batterylevel", on_battery_msg) +client.add_topic_callback( + secrets["aio_username"] + "/feeds/device.batterylevel", on_battery_msg +) # Connect the client to the MQTT broker. print("Connecting to MQTT broker...") From 93c55af4ad33270a8f8683a7ccb8c676d1d2120d Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 15:46:12 -0500 Subject: [PATCH 60/77] lint --- adafruit_minimqtt/adafruit_minimqtt.py | 48 +------------------------- 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 1ea74464..b1bd8ecf 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -81,14 +81,12 @@ _the_interface = None # pylint: disable=invalid-name _the_sock = None # pylint: disable=invalid-name - class MMQTTException(Exception): """MiniMQTT Exception class.""" # pylint: disable=unnecessary-pass # pass - # Legacy ESP32SPI Socket API def set_socket(sock, iface=None): """Legacy API for setting the socket and network interface, use a `Session` instead. @@ -572,7 +570,6 @@ def ping(self): # pylint: disable=too-many-branches, too-many-statements def publish(self, topic, msg, retain=False, qos=0): """Publishes a message to a topic provided. - :param str topic: Unique topic identifier. :param str,int,float msg: Data to send to the broker. :param bool retain: Whether the message is saved by the broker. @@ -580,25 +577,6 @@ def publish(self, topic, msg, retain=False, qos=0): zero. Conventional options are ``0`` (send at most once), ``1`` (send at least once), or ``2`` (send exactly once). - .. note:: Only options ``1`` or ``0`` are QoS levels supported by this library. - - Example of sending an integer, 3, to the broker on topic 'piVal'. - - .. code-block:: python - - mqtt_client.publish('topics/piVal', 3) - - Example of sending a float, 3.14, to the broker on topic 'piVal'. - - .. code-block:: python - - mqtt_client.publish('topics/piVal', 3.14) - - Example of sending a string, 'threepointonefour', to the broker on topic piVal. - - .. code-block:: python - - mqtt_client.publish('topics/piVal', 'threepointonefour') """ self.is_connected() self._valid_topic(topic) @@ -687,30 +665,6 @@ def subscribe(self, topic, qos=0): .. note:: Only options ``1`` or ``0`` are QoS levels supported by this library. - Example of subscribing a topic string. - - .. code-block:: python - - mqtt_client.subscribe('topics/ledState') - - Example of subscribing to a topic and setting the qos level to 1. - - .. code-block:: python - - mqtt_client.subscribe('topics/ledState', 1) - - Example of subscribing to topic string and setting qos level to 1, as a tuple. - - .. code-block:: python - - mqtt_client.subscribe(('topics/ledState', 1)) - - Example of subscribing to multiple topics with different qos levels. - - .. code-block:: python - - mqtt_client.subscribe([('topics/ledState', 1), ('topics/servoAngle', 0)]) - """ self.is_connected() topics = None @@ -882,7 +836,7 @@ def _wait_for_msg(self, timeout=0.1): if error.errno == errno.ETIMEDOUT: # raised by a socket timeout in socketpool return None - raise MMQTTException(error) + raise MMQTTException from error # Block while we parse the rest of the response self._sock.settimeout(timeout) From 96b681d1f603c3d57dc2af7f8a5bee872257de26 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 15:47:50 -0500 Subject: [PATCH 61/77] black --- adafruit_minimqtt/adafruit_minimqtt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index b1bd8ecf..4953a13e 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -81,12 +81,14 @@ _the_interface = None # pylint: disable=invalid-name _the_sock = None # pylint: disable=invalid-name + class MMQTTException(Exception): """MiniMQTT Exception class.""" # pylint: disable=unnecessary-pass # pass + # Legacy ESP32SPI Socket API def set_socket(sock, iface=None): """Legacy API for setting the socket and network interface, use a `Session` instead. From 3b9a2f5ec8fe4adfd627a08287e2136a2693bbc0 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 15:51:51 -0500 Subject: [PATCH 62/77] push fix --- adafruit_minimqtt/adafruit_minimqtt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 4953a13e..84a99023 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -578,7 +578,6 @@ def publish(self, topic, msg, retain=False, qos=0): :param int qos: Quality of Service level for the message, defaults to zero. Conventional options are ``0`` (send at most once), ``1`` (send at least once), or ``2`` (send exactly once). - """ self.is_connected() self._valid_topic(topic) From 81a241726ce6dbf04bf81985f9e312f965c606a4 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:01:43 -0500 Subject: [PATCH 63/77] fix copyrightlicense --- examples/cellular/minimqtt_adafruitio_cellular.py | 7 ------- examples/cpython/minimqtt_adafruitio_cpython.py | 7 +++---- examples/cpython/minimqtt_simpletest_cpython.py | 6 +++--- examples/esp32spi/minimqtt_adafruitio_esp32spi.py | 3 --- examples/esp32spi/minimqtt_certificate_esp32spi.py | 5 ----- examples/esp32spi/minimqtt_pub_sub_blocking_esp32spi.py | 2 -- examples/esp32spi/minimqtt_simpletest_esp32spi.py | 3 ++- examples/ethernet/minimqtt_adafruitio_eth.py | 3 --- examples/ethernet/minimqtt_simpletest_eth.py | 6 ------ examples/minimqtt_simpletest.py | 7 +++---- .../minimqtt_adafruitio_native_networking.py | 7 +++---- .../minimqtt_pub_sub_blocking_native_networking.py | 6 +++--- ...t_pub_sub_blocking_topic_callbacks_native_networking.py | 6 +++--- 13 files changed, 20 insertions(+), 48 deletions(-) diff --git a/examples/cellular/minimqtt_adafruitio_cellular.py b/examples/cellular/minimqtt_adafruitio_cellular.py index 3b576b53..d83b410b 100755 --- a/examples/cellular/minimqtt_adafruitio_cellular.py +++ b/examples/cellular/minimqtt_adafruitio_cellular.py @@ -1,13 +1,6 @@ -<<<<<<< HEAD:examples/cellular/minimqtt_adafruitio_cellular.py -# SPDX-FileCopyrightText: Tony DiCola for Adafruit Industries -# SPDX-FileCopyrightText: 2020 Brent Rubell for Adafruit Industries -# -# SPDX-License-Identifier: Unlicense -======= # SPDX-FileCopyrightText: 2021 ladyada for Adafruit Industries # SPDX-License-Identifier: MIT ->>>>>>> master:examples/minimqtt_adafruitio_cellular.py import time import board import busio diff --git a/examples/cpython/minimqtt_adafruitio_cpython.py b/examples/cpython/minimqtt_adafruitio_cpython.py index 9b2b276b..7eb4f5fb 100644 --- a/examples/cpython/minimqtt_adafruitio_cpython.py +++ b/examples/cpython/minimqtt_adafruitio_cpython.py @@ -1,7 +1,6 @@ -# SPDX-FileCopyrightText: Tony DiCola for Adafruit Industries -# SPDX-FileCopyrightText: 2020 Brent Rubell for Adafruit Industries -# -# SPDX-License-Identifier: Unlicense +# SPDX-FileCopyrightText: 2021 ladyada for Adafruit Industries +# SPDX-License-Identifier: MIT + import time import socket import adafruit_minimqtt.adafruit_minimqtt as MQTT diff --git a/examples/cpython/minimqtt_simpletest_cpython.py b/examples/cpython/minimqtt_simpletest_cpython.py index 799a2074..35fe18e9 100644 --- a/examples/cpython/minimqtt_simpletest_cpython.py +++ b/examples/cpython/minimqtt_simpletest_cpython.py @@ -1,6 +1,6 @@ -# SPDX-FileCopyrightText: 2020 Brent Rubell for Adafruit Industries -# -# SPDX-License-Identifier: Unlicense +# SPDX-FileCopyrightText: 2021 ladyada for Adafruit Industries +# SPDX-License-Identifier: MIT + import socket import adafruit_minimqtt.adafruit_minimqtt as MQTT diff --git a/examples/esp32spi/minimqtt_adafruitio_esp32spi.py b/examples/esp32spi/minimqtt_adafruitio_esp32spi.py index 38817455..7141e8bc 100644 --- a/examples/esp32spi/minimqtt_adafruitio_esp32spi.py +++ b/examples/esp32spi/minimqtt_adafruitio_esp32spi.py @@ -1,9 +1,6 @@ # SPDX-FileCopyrightText: 2021 ladyada for Adafruit Industries # SPDX-License-Identifier: MIT -# Adafruit MiniMQTT Pub/Sub Example -# Written by Tony DiCola for Adafruit Industries -# Modified by Brent Rubell for Adafruit Industries import time import board import busio diff --git a/examples/esp32spi/minimqtt_certificate_esp32spi.py b/examples/esp32spi/minimqtt_certificate_esp32spi.py index 50f002eb..ac9cb807 100644 --- a/examples/esp32spi/minimqtt_certificate_esp32spi.py +++ b/examples/esp32spi/minimqtt_certificate_esp32spi.py @@ -67,28 +67,23 @@ def connect(client, userdata, flags, rc): print("Connected to MQTT Broker!") print("Flags: {0}\n RC: {1}".format(flags, rc)) - def disconnect(client, userdata, rc): # This method is called when the client disconnects # from the broker. print("Disconnected from MQTT Broker!") - def subscribe(client, userdata, topic, granted_qos): # This method is called when the client subscribes to a new feed. print("Subscribed to {0} with QOS level {1}".format(topic, granted_qos)) - def unsubscribe(client, userdata, topic, pid): # This method is called when the client unsubscribes from a feed. print("Unsubscribed from {0} with PID {1}".format(topic, pid)) - def publish(client, userdata, topic, pid): # This method is called when the client publishes data to a feed. print("Published to {0} with PID {1}".format(topic, pid)) - # Get certificate and private key from a certificates.py file try: from certificates import DEVICE_CERT, DEVICE_KEY diff --git a/examples/esp32spi/minimqtt_pub_sub_blocking_esp32spi.py b/examples/esp32spi/minimqtt_pub_sub_blocking_esp32spi.py index 3c41545f..734816ea 100644 --- a/examples/esp32spi/minimqtt_pub_sub_blocking_esp32spi.py +++ b/examples/esp32spi/minimqtt_pub_sub_blocking_esp32spi.py @@ -1,8 +1,6 @@ # SPDX-FileCopyrightText: 2021 ladyada for Adafruit Industries # SPDX-License-Identifier: MIT -# CircuitPython MiniMQTT Library -# Adafruit IO SSL/TLS Example for WiFi import time import board import busio diff --git a/examples/esp32spi/minimqtt_simpletest_esp32spi.py b/examples/esp32spi/minimqtt_simpletest_esp32spi.py index f1c50d85..254253dc 100644 --- a/examples/esp32spi/minimqtt_simpletest_esp32spi.py +++ b/examples/esp32spi/minimqtt_simpletest_esp32spi.py @@ -1,4 +1,5 @@ -# adafruit_minimqtt usage with esp32spi wifi +# SPDX-FileCopyrightText: 2021 ladyada for Adafruit Industries +# SPDX-License-Identifier: MIT import board import busio from digitalio import DigitalInOut diff --git a/examples/ethernet/minimqtt_adafruitio_eth.py b/examples/ethernet/minimqtt_adafruitio_eth.py index 1d8cf9cc..753bf473 100755 --- a/examples/ethernet/minimqtt_adafruitio_eth.py +++ b/examples/ethernet/minimqtt_adafruitio_eth.py @@ -1,9 +1,6 @@ # SPDX-FileCopyrightText: 2021 ladyada for Adafruit Industries # SPDX-License-Identifier: MIT -# Adafruit MiniMQTT Pub/Sub Example -# Written by Tony DiCola for Adafruit Industries -# Modified by Brent Rubell for Adafruit Industries import time import board import busio diff --git a/examples/ethernet/minimqtt_simpletest_eth.py b/examples/ethernet/minimqtt_simpletest_eth.py index 02894f88..c585cf78 100644 --- a/examples/ethernet/minimqtt_simpletest_eth.py +++ b/examples/ethernet/minimqtt_simpletest_eth.py @@ -1,12 +1,6 @@ -<<<<<<< HEAD:examples/ethernet/minimqtt_simpletest_eth.py -# SPDX-FileCopyrightText: 2020 Brent Rubell for Adafruit Industries -# -# SPDX-License-Identifier: Unlicense -======= # SPDX-FileCopyrightText: 2021 ladyada for Adafruit Industries # SPDX-License-Identifier: MIT ->>>>>>> master:examples/minimqtt_simpletest_eth.py import board import busio from digitalio import DigitalInOut diff --git a/examples/minimqtt_simpletest.py b/examples/minimqtt_simpletest.py index 386365c1..0a4ca3fa 100644 --- a/examples/minimqtt_simpletest.py +++ b/examples/minimqtt_simpletest.py @@ -1,7 +1,6 @@ -# SPDX-FileCopyrightText: 2021 Brent Rubell, written for Adafruit Industries -# -# SPDX-License-Identifier: Unlicense -# adafruit_minimqtt usage with native networking +# SPDX-FileCopyrightText: 2021 ladyada for Adafruit Industries +# SPDX-License-Identifier: MIT + import ssl import socketpool import wifi diff --git a/examples/native_networking/minimqtt_adafruitio_native_networking.py b/examples/native_networking/minimqtt_adafruitio_native_networking.py index b6fedcd7..a4f5ecaa 100644 --- a/examples/native_networking/minimqtt_adafruitio_native_networking.py +++ b/examples/native_networking/minimqtt_adafruitio_native_networking.py @@ -1,7 +1,6 @@ -# SPDX-FileCopyrightText: Tony DiCola for Adafruit Industries -# SPDX-FileCopyrightText: 2020 Brent Rubell for Adafruit Industries -# -# SPDX-License-Identifier: Unlicense +# SPDX-FileCopyrightText: 2021 ladyada for Adafruit Industries +# SPDX-License-Identifier: MIT + import time import ssl import socketpool diff --git a/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py b/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py index 647da483..58dbc7f7 100644 --- a/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py +++ b/examples/native_networking/minimqtt_pub_sub_blocking_native_networking.py @@ -1,6 +1,6 @@ -# SPDX-FileCopyrightText: 2020 Brent Rubell, written for Adafruit Industries -# -# SPDX-License-Identifier: Unlicense +# SPDX-FileCopyrightText: 2021 ladyada for Adafruit Industries +# SPDX-License-Identifier: MIT + import time import ssl import socketpool diff --git a/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py b/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py index e9cdaf6e..2a2eddf3 100644 --- a/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py +++ b/examples/native_networking/minimqtt_pub_sub_blocking_topic_callbacks_native_networking.py @@ -1,6 +1,6 @@ -# SPDX-FileCopyrightText: 2020 Brent Rubell, written for Adafruit Industries -# -# SPDX-License-Identifier: Unlicense +# SPDX-FileCopyrightText: 2021 ladyada for Adafruit Industries +# SPDX-License-Identifier: MIT + import time import ssl import socketpool From e3d662f648251eeb6faedae4bf63d1598a384485 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:03:27 -0500 Subject: [PATCH 64/77] remove vscode from examples again --- examples/.vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 examples/.vscode/settings.json diff --git a/examples/.vscode/settings.json b/examples/.vscode/settings.json deleted file mode 100644 index 3cce948f..00000000 --- a/examples/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "restructuredtext.confPath": "" -} \ No newline at end of file From d77e588fd5a82cfcb88667c2465377cad10faacf Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:04:14 -0500 Subject: [PATCH 65/77] black precommit --- examples/esp32spi/minimqtt_certificate_esp32spi.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/esp32spi/minimqtt_certificate_esp32spi.py b/examples/esp32spi/minimqtt_certificate_esp32spi.py index ac9cb807..50f002eb 100644 --- a/examples/esp32spi/minimqtt_certificate_esp32spi.py +++ b/examples/esp32spi/minimqtt_certificate_esp32spi.py @@ -67,23 +67,28 @@ def connect(client, userdata, flags, rc): print("Connected to MQTT Broker!") print("Flags: {0}\n RC: {1}".format(flags, rc)) + def disconnect(client, userdata, rc): # This method is called when the client disconnects # from the broker. print("Disconnected from MQTT Broker!") + def subscribe(client, userdata, topic, granted_qos): # This method is called when the client subscribes to a new feed. print("Subscribed to {0} with QOS level {1}".format(topic, granted_qos)) + def unsubscribe(client, userdata, topic, pid): # This method is called when the client unsubscribes from a feed. print("Unsubscribed from {0} with PID {1}".format(topic, pid)) + def publish(client, userdata, topic, pid): # This method is called when the client publishes data to a feed. print("Published to {0} with PID {1}".format(topic, pid)) + # Get certificate and private key from a certificates.py file try: from certificates import DEVICE_CERT, DEVICE_KEY From 094c2f8c21fb5a61fec9e361e40d8f2475010256 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:07:55 -0500 Subject: [PATCH 66/77] fix line length --- adafruit_minimqtt/adafruit_minimqtt.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 6bcc083f..df0dd390 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -115,8 +115,7 @@ def wrap_socket(self, socket, server_hostname=None): class MQTT: - """MQTT Client for CircuitPython - + """MQTT Client for CircuitPython. :param str broker: MQTT Broker URL or IP Address. :param int port: Optional port definition, defaults to 8883. :param str username: Username for broker authentication. @@ -448,7 +447,6 @@ def username_pw_set(self, username, password=None): # pylint: disable=too-many-branches, too-many-statements, too-many-locals def connect(self, clean_session=True, host=None, port=None, keep_alive=None): """Initiates connection with the MQTT Broker. - :param bool clean_session: Establishes a persistent session. :param str host: Hostname or IP address of the remote broker. :param int port: Network port of the remote broker. @@ -727,23 +725,10 @@ def subscribe(self, topic, qos=0): def unsubscribe(self, topic): """Unsubscribes from a MQTT topic. - :param str,list topic: Unique MQTT topic identifier string or a list of tuples, where each tuple contains an MQTT topic identier string. - Example of unsubscribing from a topic string. - - .. code-block:: python - - mqtt_client.unsubscribe('topics/ledState') - - Example of unsubscribing from multiple topics. - - .. code-block:: python - - mqtt_client.unsubscribe([('topics/ledState'), ('topics/servoAngle')]) - """ topics = None if isinstance(topic, str): @@ -790,8 +775,8 @@ def unsubscribe(self, topic): def reconnect(self, resub_topics=True): """Attempts to reconnect to the MQTT broker. - :param bool resub_topics: Resubscribe to previously subscribed topics. + """ if self.logger: self.logger.debug("Attempting to reconnect with MQTT broker") @@ -996,7 +981,6 @@ def is_connected(self): # Logging def enable_logger(self, logger, log_level=20): """Enables library logging provided a `logger` object. - :param logging.Logger logger: A python logger pacakge. :param log_level: Numeric value of a logging level, defaults to `logging.INFO`. From f732a159251dfd9a1ea6b728dc534a0df64d16fb Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:17:19 -0500 Subject: [PATCH 67/77] fix indentation: --- adafruit_minimqtt/adafruit_minimqtt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index df0dd390..187a4408 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -450,8 +450,7 @@ def connect(self, clean_session=True, host=None, port=None, keep_alive=None): :param bool clean_session: Establishes a persistent session. :param str host: Hostname or IP address of the remote broker. :param int port: Network port of the remote broker. - :param int keep_alive: Maximum period allowed for communication - with the broker, in seconds + :param int keep_alive: Maximum period allowed for communication, in seconds. """ if host: From 10177a6e4ad82e351e58379dcd350b96a75a70e3 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:21:38 -0500 Subject: [PATCH 68/77] trim --- adafruit_minimqtt/adafruit_minimqtt.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 187a4408..a688a8d3 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -584,9 +584,8 @@ def publish(self, topic, msg, retain=False, qos=0): :param str topic: Unique topic identifier. :param str,int,float msg: Data to send to the broker. :param bool retain: Whether the message is saved by the broker. - :param int qos: Quality of Service level for the message, defaults to - zero. Conventional options are ``0`` (send at most once), ``1`` - (send at least once), or ``2`` (send exactly once). + :param int qos: Quality of Service level for the message, defaults to zero. + """ self.is_connected() self._valid_topic(topic) @@ -665,14 +664,11 @@ def subscribe(self, topic, qos=0): This method can subscribe to one topics or multiple topics. :param str,tuple,list topic: Unique MQTT topic identifier string. If - this is a `tuple`, then the tuple should contain topic identifier + this is a `tuple`, then the tuple should contain topic identifier string and qos level integer. If this is a `list`, then each list element should be a tuple containing a topic identifier string and qos level integer. - :param int qos: Quality of Service level for the topic, defaults to - zero. Conventional options are ``0`` (send at most once), ``1`` - (send at least once), or ``2`` (send exactly once). - + :param int qos: Quality of Service level for the topic, defaults to zero. .. note:: Only options ``1`` or ``0`` are QoS levels supported by this library. """ From 91faed719f47747902accc20f3ba00fe1848d5d4 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:25:04 -0500 Subject: [PATCH 69/77] more verbose qos docstring --- adafruit_minimqtt/adafruit_minimqtt.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index a688a8d3..3d0b9dba 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -665,11 +665,13 @@ def subscribe(self, topic, qos=0): :param str,tuple,list topic: Unique MQTT topic identifier string. If this is a `tuple`, then the tuple should contain topic identifier - string and qos level integer. If this is a `list`, then each list - element should be a tuple containing a topic identifier string and - qos level integer. - :param int qos: Quality of Service level for the topic, defaults to zero. - .. note:: Only options ``1`` or ``0`` are QoS levels supported by this library. + string and qos level integer. If this is a `list`, then each list + element should be a tuple containing a topic identifier string and + qos level integer. + :param int qos: Quality of Service level for the topic, defaults to + zero. Conventional options are ``0`` (send at most once), ``1`` + (send at least once), or ``2`` (send exactly once). + .. note:: Only options ``1`` or ``0`` are QoS levels supported by this library. """ self.is_connected() From 5dbfe90e294c0d3b522c5085f328e153344cf60c Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:29:21 -0500 Subject: [PATCH 70/77] docstring again --- adafruit_minimqtt/adafruit_minimqtt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 3d0b9dba..95066f43 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -723,8 +723,8 @@ def subscribe(self, topic, qos=0): def unsubscribe(self, topic): """Unsubscribes from a MQTT topic. :param str,list topic: Unique MQTT topic identifier string or a list - of tuples, where each tuple contains an MQTT topic identier - string. + of tuples, where each tuple contains an MQTT topic identier + string. """ topics = None From 49ca671c711429c52aa6845894a0e5c73e0d8a3d Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:33:19 -0500 Subject: [PATCH 71/77] docstring again2 --- adafruit_minimqtt/adafruit_minimqtt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 95066f43..ee39bd73 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -723,8 +723,7 @@ def subscribe(self, topic, qos=0): def unsubscribe(self, topic): """Unsubscribes from a MQTT topic. :param str,list topic: Unique MQTT topic identifier string or a list - of tuples, where each tuple contains an MQTT topic identier - string. + of tuples, where each tuple contains an MQTT topic identier string. """ topics = None From 885e7d4725ee7e4431b4be83ed934e481484f60d Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:37:48 -0500 Subject: [PATCH 72/77] unindent --- adafruit_minimqtt/adafruit_minimqtt.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index ee39bd73..6e83245b 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -664,14 +664,13 @@ def subscribe(self, topic, qos=0): This method can subscribe to one topics or multiple topics. :param str,tuple,list topic: Unique MQTT topic identifier string. If - this is a `tuple`, then the tuple should contain topic identifier - string and qos level integer. If this is a `list`, then each list - element should be a tuple containing a topic identifier string and - qos level integer. + this is a `tuple`, then the tuple should contain topic identifier + string and qos level integer. If this is a `list`, then each list + element should be a tuple containing a topic identifier string and + qos level integer. :param int qos: Quality of Service level for the topic, defaults to - zero. Conventional options are ``0`` (send at most once), ``1`` - (send at least once), or ``2`` (send exactly once). - .. note:: Only options ``1`` or ``0`` are QoS levels supported by this library. + zero. Conventional options are ``0`` (send at most once), ``1`` + (send at least once), or ``2`` (send exactly once). """ self.is_connected() @@ -723,7 +722,7 @@ def subscribe(self, topic, qos=0): def unsubscribe(self, topic): """Unsubscribes from a MQTT topic. :param str,list topic: Unique MQTT topic identifier string or a list - of tuples, where each tuple contains an MQTT topic identier string. + of tuples, where each tuple contains an MQTT topic identier string. """ topics = None From 53c8ea4a30af5506ba94826aad708da0886b912d Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:38:37 -0500 Subject: [PATCH 73/77] remove print() --- adafruit_minimqtt/adafruit_minimqtt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 6e83245b..9a0f11fb 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -820,7 +820,6 @@ def _wait_for_msg(self, timeout=0.1): try: res = self._sock_exact_recv(1) except self._socket_pool.timeout as error: - print("timed out", error) return None else: # socketpool, esp32spi try: From ecdf956fbbbb423c2ab5a46e5ff4c90dbd02c533 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:40:42 -0500 Subject: [PATCH 74/77] shorten line --- adafruit_minimqtt/adafruit_minimqtt.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index 9a0f11fb..d48be45a 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -664,10 +664,11 @@ def subscribe(self, topic, qos=0): This method can subscribe to one topics or multiple topics. :param str,tuple,list topic: Unique MQTT topic identifier string. If - this is a `tuple`, then the tuple should contain topic identifier - string and qos level integer. If this is a `list`, then each list - element should be a tuple containing a topic identifier string and - qos level integer. + this is a `tuple`, then the tuple should + contain topic identifier string and qos + level integer. If this is a `list`, then + each list element should be a tuple containing + a topic identifier string and qos level integer. :param int qos: Quality of Service level for the topic, defaults to zero. Conventional options are ``0`` (send at most once), ``1`` (send at least once), or ``2`` (send exactly once). From d0f32a3bd15c76dbd6531fab8cc79cc2d0a3dd75 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:44:25 -0500 Subject: [PATCH 75/77] unsub shorten --- adafruit_minimqtt/adafruit_minimqtt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index d48be45a..f2fa2e34 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -722,8 +722,7 @@ def subscribe(self, topic, qos=0): def unsubscribe(self, topic): """Unsubscribes from a MQTT topic. - :param str,list topic: Unique MQTT topic identifier string or a list - of tuples, where each tuple contains an MQTT topic identier string. + :param str,list topic: Unique MQTT topic identifier string or list. """ topics = None From 341859bb981836dba6804f32d6be6c7b7bbe57f0 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:50:59 -0500 Subject: [PATCH 76/77] fix inher. autodoc issues --- adafruit_minimqtt/adafruit_minimqtt.py | 8 ++++---- docs/examples.rst | 12 +----------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/adafruit_minimqtt/adafruit_minimqtt.py b/adafruit_minimqtt/adafruit_minimqtt.py index f2fa2e34..aecde993 100755 --- a/adafruit_minimqtt/adafruit_minimqtt.py +++ b/adafruit_minimqtt/adafruit_minimqtt.py @@ -74,7 +74,7 @@ class MMQTTException(Exception): # Legacy ESP32SPI Socket API def set_socket(sock, iface=None): - """Legacy API for setting the socket and network interface, use a `Session` instead. + """Legacy API for setting the socket and network interface, use a Session instead. :param sock: socket object. :param iface: internet interface object @@ -974,9 +974,9 @@ def is_connected(self): # Logging def enable_logger(self, logger, log_level=20): - """Enables library logging provided a `logger` object. - :param logging.Logger logger: A python logger pacakge. - :param log_level: Numeric value of a logging level, defaults to `logging.INFO`. + """Enables library logging provided a logger object. + :param logger: A python logger pacakge. + :param log_level: Numeric value of a logging level, defaults to INFO. """ self.logger = logger.getLogger("log") diff --git a/docs/examples.rst b/docs/examples.rst index c7fbaf6c..abe06c59 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -5,14 +5,4 @@ Ensure your device works with this simple test. .. literalinclude:: ../examples/minimqtt_simpletest.py :caption: examples/minimqtt_simpletest.py - :linenos: - -Basic forever loop ------------------- - -This example shows how to write a loop that runs forever -& can handle disconnect/re-connect events. - -.. literalinclude:: ../examples/minimqtt_pub_sub_blocking.py - :caption: examples/minimqtt_pub_sub_blocking.py - :linenos: + :linenos: \ No newline at end of file From 5038331714716eafef83246d0c565729a30fec18 Mon Sep 17 00:00:00 2001 From: brentru Date: Mon, 1 Feb 2021 16:52:59 -0500 Subject: [PATCH 77/77] trailing --- docs/examples.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples.rst b/docs/examples.rst index abe06c59..847e6bb1 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -5,4 +5,4 @@ Ensure your device works with this simple test. .. literalinclude:: ../examples/minimqtt_simpletest.py :caption: examples/minimqtt_simpletest.py - :linenos: \ No newline at end of file + :linenos: