From 26a10b28e7480b47efa71c9370a5758bf360faf0 Mon Sep 17 00:00:00 2001 From: Christian Ehlers Date: Mon, 23 Nov 2020 19:42:05 +0100 Subject: [PATCH 01/11] Update Pybytes to version 1.6.1 --- esp32/frozen/Pybytes/_OTA.py | 14 +- esp32/frozen/Pybytes/_pybytes_constants.py | 9 +- .../Pybytes/_pybytes_machine_learning.py | 176 ++++++++++++++++++ esp32/frozen/Pybytes/_pybytes_protocol.py | 82 +++++--- .../frozen/Pybytes/_pybytes_pymesh_config.py | 32 ++-- esp32/frozen/Pybytes/_terminal.py | 10 +- esp32/pycom_version.h | 2 +- 7 files changed, 282 insertions(+), 43 deletions(-) create mode 100644 esp32/frozen/Pybytes/_pybytes_machine_learning.py diff --git a/esp32/frozen/Pybytes/_OTA.py b/esp32/frozen/Pybytes/_OTA.py index 63d20ac3ec..a1391035c3 100644 --- a/esp32/frozen/Pybytes/_OTA.py +++ b/esp32/frozen/Pybytes/_OTA.py @@ -137,7 +137,6 @@ def update(self, customManifest=None, fwtype=None, token=None): def get_file(self, f): new_path = "{}.new".format(f['dst_path']) - # If a .new file exists from a previously failed update delete it try: os.remove(new_path) @@ -184,6 +183,15 @@ def delete_file(self, f): def write_firmware(self, f): # hash = + url = f['URL'].split("//")[1].split("/")[0] + + if url.find(":") > -1: + self.ip = url.split(":")[0] + self.port = int(url.split(":")[1]) + else: + self.ip = url + self.port = 443 + self.get_data( f['URL'].split("/", 3)[-1], hash=True, @@ -222,7 +230,6 @@ def _http_get(self, path, host): def get_data(self, req, dest_path=None, hash=False, firmware=False): h = None - useSSL = int(self.port) == 443 # Connect to server @@ -232,11 +239,9 @@ def get_data(self, req, dest_path=None, hash=False, firmware=False): if (int(self.port) == 443): print("Wrapping socket") s = ssl.wrap_socket(s) - print("Sending request") # Request File s.sendall(self._http_get(req, "{}:{}".format(self.ip, self.port))) - try: content = bytearray() fp = None @@ -247,6 +252,7 @@ def get_data(self, req, dest_path=None, hash=False, firmware=False): fp = open(dest_path, 'wb') if firmware: + print_debug(4, "Starting OTA...") pycom.ota_start() h = uhashlib.sha1() diff --git a/esp32/frozen/Pybytes/_pybytes_constants.py b/esp32/frozen/Pybytes/_pybytes_constants.py index aa1d27cd67..8ae4058530 100644 --- a/esp32/frozen/Pybytes/_pybytes_constants.py +++ b/esp32/frozen/Pybytes/_pybytes_constants.py @@ -69,11 +69,12 @@ class constants: __TYPE_OTA = 0x05 __TYPE_FCOTA = 0x06 __TYPE_PONG = 0x07 - __TYPE_PYMESH = 0x0D - __TYPE_PYBYTES = 0x0E - __TYPE_RELEASE_INFO = 0x0B __TYPE_RELEASE_DEPLOY = 0x0A + __TYPE_RELEASE_INFO = 0x0B __TYPE_DEVICE_NETWORK_DEPLOY = 0x0C + __TYPE_PYMESH = 0x0D + __TYPE_PYBYTES = 0x0E + __TYPE_ML = 0x0F __PYBYTES_PROTOCOL = ">B%ds" __PYBYTES_PROTOCOL_PING = ">B" __PYBYTES_INTERNAL_PROTOCOL = ">BBH" @@ -90,6 +91,8 @@ class constants: __COMMAND_ANALOG_WRITE = 4 __COMMAND_CUSTOM_METHOD = 5 __COMMAND_CUSTOM_LOCATION = 6 + __COMMAND_START_SAMPLE = 7 + __COMMAND_DEPLOY_MODEL = 8 __FCOTA_COMMAND_HIERARCHY_ACQUISITION = 0x00 __FCOTA_COMMAND_FILE_ACQUISITION = 0x01 diff --git a/esp32/frozen/Pybytes/_pybytes_machine_learning.py b/esp32/frozen/Pybytes/_pybytes_machine_learning.py new file mode 100644 index 0000000000..5c1fce6626 --- /dev/null +++ b/esp32/frozen/Pybytes/_pybytes_machine_learning.py @@ -0,0 +1,176 @@ +''' +Copyright (c) 2020, Pycom Limited. +This software is licensed under the GNU GPL version 3 or any +later version, with permitted additional terms. For more information +see the Pycom Licence v1.0 document supplied with this file, or +available at https://www.pycom.io/opensource/licensing +''' + +import math +import json + +try: + from pybytes_debug import print_debug +except: + from _pybytes_debug import print_debug + +try: + import urequest +except: + import _urequest as urequest + +try: + from pybytes_constants import constants +except: + from _pybytes_constants import constants + +import pycom + +try: + from LIS2HH12 import * +except: + print_debug(5, "LIS2HH12 not imported") + +# 20 seconds, max window in time for recording +MAX_LEN_MSEC = const(20000) + +# 350Hz, max frequency +MAX_FREQ_HZ = const(350) + + +class MlFeatures(): + def __init__(self, pybytes_protocol=None, parameters=None): + if parameters is not None: + self.__length = parameters["length"] + self.__label = parameters["label"] + self.__sampleName = parameters["sampleName"] + self.__type = parameters["type"] + self.__device = parameters["device"] + self.__model = parameters["model"] + self.__mlSample = parameters["mlSample"] + self.__frequency = parameters["frequency"] + self.__pybytes_protocol = pybytes_protocol + self.__data = [] + + def _debug_hack(self, pybytes): + self.__pybytes = pybytes + + def start_sampling(self, pin): + # here define the required libraries + try: + from pysense import Pysense + except: + print_debug(5, "pysense not imported") + + try: + from pytrack import Pytrack + except: + print_debug(5, "pytrack not imported") + + lib = False + try: + py = Pysense() + lib = True + except NameError: + print_debug(5, "Pysense not defined") + + if not lib: + try: + py = Pytrack() + except NameError: + print_debug(5, "Check if Pysense/Pytrack libraries are loaded") + return + + try: + li = LIS2HH12(py) + except NameError: + print_debug(5, "LIS2HH12 library are not loaded") + return + li.set_odr(ODR_400_HZ) + + # make the max record length to 20 seconds + self.__length = min(MAX_LEN_MSEC, self.__length) + + # make the max frequency to 350Hz + self.__frequency = min(MAX_FREQ_HZ, self.__frequency) + + # compute time interval between 2 consecutive samples + delta_t_us = int(1000000.0 / self.__frequency) + # compute the number of samples to be acquisition + samples_num = math.ceil(self.__length * self.__frequency / 1000) + 1 + + try: + pycom.heartbeat(False) + pycom.rgbled(0x7f7f00) + except: + pass + time.sleep(0.5) + + self.__data = [] + index = 0 + print("Start acquisition data for %d msec, freq %d Hz" % (self.__length, self.__frequency)) + + next_ts = time.ticks_us() + ts_orig = next_ts + while True: + while time.ticks_diff(next_ts, time.ticks_us()) > 0: + pass + acc = li.acceleration() + ts = next_ts + self.__data.append((ts - ts_orig, acc)) + next_ts = ts + delta_t_us + index += 1 + if index >= samples_num: + break # done + + print("Done acquisition %d samples, real freq %.1f Hz" % (index, index / (self.__length / 1000))) + self._parse_data(pin) + + def _send_data(self, data, pin, acc, ts): + if self.__pybytes_protocol is not None: + if self.__type == 2: + self.__label = self.__sampleName + self.__pybytes_protocol.send_pybytes_custom_method_values(pin, [ + data], + 'sample/{}/{}/{}/{}/{}'.format(self.__label, self.__type, self.__model, self.__device, self.__mlSample)) + else: + self.__pybytes.send_signal(pin & 0xFF, str((int(ts / 1000), acc))) + + def _parse_data(self, pin): + print("_parse_data, %d samples" % len(self.__data)) + try: + pycom.rgbled(0x8d05f5) + except: + pass + data = ['{"data": "ml"}'] + for (ts, acc) in self.__data: + data.append('{' + '"data": [{},{},{}], "ms": {}'.format(acc[0], acc[1], acc[2], int(ts / 1000)) + '}') + if len(data) > 25: + self._send_data(data, pin, acc, ts) + data = ['{"data": "ml"}'] + self._send_data(data, pin, acc, ts) + try: + pycom.heartbeat(True) + except: + pass + + def deploy_model(self, modelId, silent=False): + try: + file = '/flash/model_definition.json' + modelDefinition = {} + url = '{}://{}/ml/{}'.format( + constants.__DEFAULT_PYCONFIG_PROTOCOL, + constants.__DEFAULT_PYCONFIG_DOMAIN, + modelId + ) + print_debug(2, '{}'.format(url)) + result = urequest.get(url, headers={'content-type': 'application/json'}) + modelDefinition = json.loads(result.content.decode()) + print_debug(2, 'modelDefinition: {}'.format(modelDefinition)) + f = open(file, 'w') + f.write(json.dumps(modelDefinition).encode('utf-8')) + f.close() + print_debug(2, "Model definition written to {}".format(file)) + except Exception as e: + if not silent: + print_debug(2, "Exception: {}".format(e)) diff --git a/esp32/frozen/Pybytes/_pybytes_protocol.py b/esp32/frozen/Pybytes/_pybytes_protocol.py index 385d8c6f9c..6934f3ee44 100644 --- a/esp32/frozen/Pybytes/_pybytes_protocol.py +++ b/esp32/frozen/Pybytes/_pybytes_protocol.py @@ -31,6 +31,11 @@ except: from _pybytes_pymesh_config import PybytesPymeshConfig +try: + from pybytes_machine_learning import MlFeatures +except: + from _pybytes_machine_learning import MlFeatures + try: from pybytes_config_reader import PybytesConfigReader except: @@ -281,10 +286,10 @@ def __process_recv_message(self, message): splittedBody = bodyString.split(',') if (len(splittedBody) >= 2): path = splittedBody[0] - print_debug(2, path[len(path)-7:len(path)]) - if (path[len(path)-7:len(path)] != '.pymakr'): + print_debug(2, path[len(path) - 7:len(path)]) + if (path[len(path) - 7:len(path)] != '.pymakr'): self.send_fcota_ping('updating file...') - newContent = bodyString[len(path)+1:len(body)] + newContent = bodyString[len(path) + 1:len(body)] if (self.__FCOTA.update_file_content(path, newContent) is True): # noqa size = self.__FCOTA.get_file_size(path) self.send_fcota_file(newContent, path, size) @@ -319,7 +324,18 @@ def __process_recv_message(self, message): if (len(body) > 3): value = body[2] << 8 | body[3] - if (command == constants.__COMMAND_PIN_MODE): + if (command == constants.__COMMAND_START_SAMPLE): + parameters = ujson.loads(body[2: len(body)].decode("utf-8")) + sampling = MlFeatures(self, parameters=parameters) + sampling.start_sampling(pin=parameters["pin"]) + self.send_ota_response(result=2, topic='sample') + elif (command == constants.__COMMAND_DEPLOY_MODEL): + parameters = ujson.loads(body[2: len(body)].decode("utf-8")) + sampling = MlFeatures() + sampling.deploy_model(modelId=parameters["modelId"]) + self.send_ota_response(result=2, topic='deploymlmodel') + + elif (command == constants.__COMMAND_PIN_MODE): pass elif (command == constants.__COMMAND_DIGITAL_READ): @@ -633,16 +649,11 @@ def write_firmware(self, customManifest=None): def get_application_details(self, body): application = self.__conf.get('application') if application is not None: - if 'id' in application and application['id']: - applicationID = application['id'] - else: - applicationID = body['applicationId'] if 'release' in application and 'codeFilename' in application['release']: currentReleaseID = application['release']['codeFilename'] else: currentReleaseID = None else: - applicationID = body['applicationId'] currentReleaseID = None self.__conf['application'] = { "id": "", @@ -652,6 +663,7 @@ def get_application_details(self, body): "version": 0 } } + applicationID = body['applicationId'] return (applicationID, currentReleaseID) def get_update_manifest(self, applicationID, newReleaseID, currentReleaseID): @@ -755,21 +767,46 @@ def update_network_config(self, letResp): except Exception as e: print_debug(1, "error while updating network config pybytes_config.json! {}".format(e)) - def update_firmware(self, body): + def update_firmware(self, body, applicationID, fw_type='pybytes'): if "firmware" not in body: print_debug(0, "no firmware to update") return - version = body['firmware']["version"] - print_debug(0, "updating firmware to {}".format(version)) - customManifest = { - "firmware": { - "URL": "https://{}/downloads/appimg/firmware_{}_{}.bin".format( - constants.__DEFAULT_SW_HOST, - os.uname().sysname, - version), + + if "version" in body['firmware']: + version = body['firmware']["version"] + print_debug(0, "updating firmware to {}".format(version)) + + customManifest = { + "firmware": { + "URL": "https://{}/findupgrade?redirect=true&strict=true&type={}&model={}&version={}&download=true".format( + constants.__DEFAULT_SW_HOST, + fw_type, + os.uname().sysname, + version), + } } - } - self.write_firmware(customManifest) + self.write_firmware(customManifest) + else: + fileUrl = '{}://{}/firmware?'.format(constants.__DEFAULT_PYCONFIG_PROTOCOL, constants.__DEFAULT_PYCONFIG_DOMAIN) + customFirmwares = body['firmware']["customFirmwares"] + firmwareFilename = '' + for firmware in customFirmwares: + print_debug(1, "firmware['firmwareType']={} and os.uname().sysname.lower()={}".format(firmware['firmwareType'], os.uname().sysname.lower())) + print_debug(1, "firmware={}".format(firmware)) + if (firmware['firmwareType'] == os.uname().sysname.lower()): + firmwareFilename = firmware['firmwareFilename'] + targetFileLocation = '{}application_id={}&target_ver={}&target_path={}'.format( + fileUrl, + applicationID, + firmwareFilename, + '/{}.bin'.format(firmwareFilename) + ) + customManifest = { + "firmware": { + "URL": targetFileLocation, + } + } + self.write_firmware(customManifest) def deploy_new_release(self, body): try: @@ -783,12 +820,15 @@ def deploy_new_release(self, body): applicationID, currentReleaseID = self.get_application_details(body) letResp = self.get_update_manifest(applicationID, newReleaseID, currentReleaseID) + if not letResp: return + fwtype = 'pygate' if hasattr(os.uname(), 'pygate') else 'pybytes' + fwtype = 'pymesh' if hasattr(os.uname(), 'pymesh') else fwtype self.update_files(letResp, applicationID, newReleaseID) self.delete_files(letResp) self.update_application_config(letResp, applicationID) self.update_network_config(letResp) - self.update_firmware(letResp) + self.update_firmware(letResp, applicationID, fw_type=fwtype) machine.reset() diff --git a/esp32/frozen/Pybytes/_pybytes_pymesh_config.py b/esp32/frozen/Pybytes/_pybytes_pymesh_config.py index 116f797925..ff12070d1c 100644 --- a/esp32/frozen/Pybytes/_pybytes_pymesh_config.py +++ b/esp32/frozen/Pybytes/_pybytes_pymesh_config.py @@ -45,7 +45,10 @@ def pymesh_init(self): except: from _pymesh import Pymesh - pycom.heartbeat(False) + try: + pycom.heartbeat(False) + except: + pass # read config file, or set default values self.__pymesh_config = PymeshConfig.read_config() @@ -81,7 +84,7 @@ def unpack_pymesh_message(self, signal_number, value): # send data to the port equal with signal_number self.__pymesh.send_mess_external(pyb_ip, signal_number, pkt_start + value) - + time.sleep(3) # shouldn't send too fast to BR # hardcode monitoring data to be sent on signal #2 @@ -93,11 +96,14 @@ def pymesh_new_message_cb(self, rcv_ip, rcv_port, rcv_data): print_debug(99, 'Received: {} '.format(rcv_data)) # user code to be inserted, to send packet to the designated Mesh-external interface - for _ in range(3): - pycom.rgbled(0x888888) - time.sleep(.2) - pycom.rgbled(0) - time.sleep(.1) + try: + for _ in range(3): + pycom.rgbled(0x888888) + time.sleep(.2) + pycom.rgbled(0) + time.sleep(.1) + except: + pass return def pymesh_new_br_message_cb(self, rcv_ip, rcv_port, rcv_data, dest_ip, dest_port): @@ -106,11 +112,13 @@ def pymesh_new_br_message_cb(self, rcv_ip, rcv_port, rcv_data, dest_ip, dest_por print_debug(99, 'Incoming %d bytes from %s (port %d), to external IPv6 %s (port %d)' % (len(rcv_data), rcv_ip, rcv_port, dest_ip, dest_port)) print_debug(99, 'Received: {} '.format(rcv_data)) - for _ in range(2): - pycom.rgbled(0x0) - time.sleep(.1) - pycom.rgbled(0x663300) - + try: + for _ in range(2): + pycom.rgbled(0x0) + time.sleep(.1) + pycom.rgbled(0x663300) + except: + pass # try to find Pybytes Token if include in rcv_data token = "" if rcv_data.startswith(self.__pack_tocken_prefix): diff --git a/esp32/frozen/Pybytes/_terminal.py b/esp32/frozen/Pybytes/_terminal.py index 7125ff5729..35f0b82ec8 100644 --- a/esp32/frozen/Pybytes/_terminal.py +++ b/esp32/frozen/Pybytes/_terminal.py @@ -23,10 +23,16 @@ def write(self, data): self.message_to_send += data # self.__pybytes_protocol.__send_terminal_message(data) else: - self.original_terminal.write(data) + try: + self.original_terminal.write(data) + except: + pass def read(self, size): - return self.original_terminal.read(size) + try: + return self.original_terminal.read(size) + except: + return b'' def message_sent_from_pybytes_start(self): self.message_from_pybytes = True diff --git a/esp32/pycom_version.h b/esp32/pycom_version.h index dfa8be4d40..cbe246821c 100644 --- a/esp32/pycom_version.h +++ b/esp32/pycom_version.h @@ -17,7 +17,7 @@ #define SIGFOX_VERSION_NUMBER "1.0.1" #if (VARIANT == PYBYTES) -#define PYBYTES_VERSION_NUMBER "1.5.2" +#define PYBYTES_VERSION_NUMBER "1.6.1" #endif #ifdef PYGATE_ENABLED From d600b20bd4188b167ffc1fd9c961486aa872ae3e Mon Sep 17 00:00:00 2001 From: Christian Ehlers Date: Mon, 23 Nov 2020 19:42:37 +0100 Subject: [PATCH 02/11] Update pycom_version.h Firmware release 1.20.2.r2 --- esp32/pycom_version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp32/pycom_version.h b/esp32/pycom_version.h index cbe246821c..55b3ed1ca3 100644 --- a/esp32/pycom_version.h +++ b/esp32/pycom_version.h @@ -10,7 +10,7 @@ #ifndef VERSION_H_ #define VERSION_H_ -#define SW_VERSION_NUMBER "1.20.2.rc11" +#define SW_VERSION_NUMBER "1.20.2.r2" #define LORAWAN_VERSION_NUMBER "1.0.2" From 742cc14efeebe68a86dc9b0b9d287aea57c7444d Mon Sep 17 00:00:00 2001 From: Christian Ehlers Date: Mon, 23 Nov 2020 20:44:32 +0100 Subject: [PATCH 03/11] Update _pybytes_protocol.py Use correct web call for release upgrades --- esp32/frozen/Pybytes/_pybytes_protocol.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/esp32/frozen/Pybytes/_pybytes_protocol.py b/esp32/frozen/Pybytes/_pybytes_protocol.py index 6934f3ee44..f7d1ba41ed 100644 --- a/esp32/frozen/Pybytes/_pybytes_protocol.py +++ b/esp32/frozen/Pybytes/_pybytes_protocol.py @@ -69,6 +69,7 @@ import struct import machine import ujson +import pycom class PybytesProtocol: @@ -778,13 +779,16 @@ def update_firmware(self, body, applicationID, fw_type='pybytes'): customManifest = { "firmware": { - "URL": "https://{}/findupgrade?redirect=true&strict=true&type={}&model={}&version={}&download=true".format( + "URL": "https://{}/manifest.json?sysname={}&wmac={}&ota_slot={}&fwtype={}&target_ver={}&download=true".format( constants.__DEFAULT_SW_HOST, - fw_type, os.uname().sysname, + hexlify(machine.unique_id()).decode('ascii'), + hex(pycom.ota_slot()), + fw_type, version), } } + print_debug(5, "Custom Manifest: {}".format(customManifest)) self.write_firmware(customManifest) else: fileUrl = '{}://{}/firmware?'.format(constants.__DEFAULT_PYCONFIG_PROTOCOL, constants.__DEFAULT_PYCONFIG_DOMAIN) From 8340e1eb71796e7dad20c788845de578c9a4b2e9 Mon Sep 17 00:00:00 2001 From: Peter Putz Date: Wed, 23 Dec 2020 10:13:16 +0100 Subject: [PATCH 04/11] version 1.20.2.r3 --- esp32/pycom_version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp32/pycom_version.h b/esp32/pycom_version.h index 55b3ed1ca3..b85d758bb1 100644 --- a/esp32/pycom_version.h +++ b/esp32/pycom_version.h @@ -10,7 +10,7 @@ #ifndef VERSION_H_ #define VERSION_H_ -#define SW_VERSION_NUMBER "1.20.2.r2" +#define SW_VERSION_NUMBER "1.20.2.r3" #define LORAWAN_VERSION_NUMBER "1.0.2" From 3f690a5733d31ff938aad6219c498f7d1bf98346 Mon Sep 17 00:00:00 2001 From: Geza Husi Date: Mon, 28 Dec 2020 10:17:04 +0100 Subject: [PATCH 05/11] Fix an issue that Bluetooth init() failed after deinit() --- esp32/mods/modbt.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/esp32/mods/modbt.c b/esp32/mods/modbt.c index 8b8ae40d1d..529159630b 100644 --- a/esp32/mods/modbt.c +++ b/esp32/mods/modbt.c @@ -1062,12 +1062,9 @@ static void gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t gatts_ /// \class Bluetooth static mp_obj_t bt_init_helper(bt_obj_t *self, const mp_arg_val_t *args) { if (!self->init) { - if (!self->controller_active) { - esp_bt_controller_config_t bt_cfg = BT_CONTROLLER_INIT_CONFIG_DEFAULT(); - esp_bt_controller_init(&bt_cfg); - self->controller_active = true; - } + esp_bt_controller_config_t bt_cfg = BT_CONTROLLER_INIT_CONFIG_DEFAULT(); + esp_bt_controller_init(&bt_cfg); esp_bt_controller_enable(ESP_BT_MODE_BLE); if (ESP_OK != esp_bluedroid_init()) { From 826e233da82c374985871b037453ef4fae88854b Mon Sep 17 00:00:00 2001 From: Geza Husi Date: Tue, 20 Oct 2020 19:59:15 +0200 Subject: [PATCH 06/11] Add API create_128bit_le_uuid_from_string() --- esp32/mods/modpycom.c | 49 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/esp32/mods/modpycom.c b/esp32/mods/modpycom.c index a4007622f0..0e7f99ce0f 100644 --- a/esp32/mods/modpycom.c +++ b/esp32/mods/modpycom.c @@ -1013,6 +1013,53 @@ STATIC mp_obj_t mod_pycom_sigfox_info (size_t n_args, const mp_obj_t *pos_args, } STATIC MP_DEFINE_CONST_FUN_OBJ_KW(mod_pycom_sigfox_info_obj, 0, mod_pycom_sigfox_info); +// This function creates a 128 bit long UUID stored in a byte array in Little Endian order from an input String +STATIC mp_obj_t create_128bit_le_uuid_from_string(mp_obj_t uuid_in) { + + size_t length; + uint8_t new_uuid[16]; + uint8_t i, j; + + const char* uuid_char_in = mp_obj_str_get_data(uuid_in, &length); + // 1 character is stored on 1 byte because we received a String + // For 128 bit UUID maximum 32 characters long String can be accepted + if (length > 32) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "Input string must not be longer than 32 characters!")); + } + + // Pre-fill the whole array with 0 because the remaining/not given digits will be 0 + char uuid_char[32] = {0}; + memcpy(uuid_char, uuid_char_in, length); + + for(i = 0, j = 0; i < 32; i = i+2) { + + uint8_t lower_nibble = 0; + uint8_t upper_nibble = 0; + + if(uuid_char[i] > 0) { + upper_nibble = hex_from_char(uuid_char[i]); + } + + if(uuid_char[i+1] > 0) { + lower_nibble = hex_from_char(uuid_char[i+1]); + } + + if(lower_nibble == 16 || upper_nibble == 16) { + nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "UUID must only contain hexadecimal digits!")); + } + + // Pack together the 4 bits digits into 1 byte + // Convert to Little Endian order because we expect that the digits of the input String follows the Natural Byte (Big Endian) order + new_uuid[15-j] = lower_nibble | (upper_nibble << 4); + j++; + } + + mp_obj_t new_uuid_mp = mp_obj_new_bytearray(16, new_uuid); + return new_uuid_mp; + +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(create_128bit_le_uuid_from_string_obj, create_128bit_le_uuid_from_string); + STATIC const mp_map_elem_t pycom_module_globals_table[] = { { MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(MP_QSTR_pycom) }, { MP_OBJ_NEW_QSTR(MP_QSTR_heartbeat), (mp_obj_t)&mod_pycom_heartbeat_obj }, @@ -1039,6 +1086,8 @@ STATIC const mp_map_elem_t pycom_module_globals_table[] = { { MP_OBJ_NEW_QSTR(MP_QSTR_wifi_pwd_sta), (mp_obj_t)&mod_pycom_wifi_pwd_sta_obj }, { MP_OBJ_NEW_QSTR(MP_QSTR_wifi_pwd_ap), (mp_obj_t)&mod_pycom_wifi_pwd_ap_obj }, { MP_OBJ_NEW_QSTR(MP_QSTR_wifi_mode_on_boot), (mp_obj_t)&mod_pycom_wifi_mode_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_create_128bit_le_uuid_from_string), (mp_obj_t)&create_128bit_le_uuid_from_string_obj }, + #if defined(FIPY) || defined(LOPY4) || defined(SIPY) { MP_OBJ_NEW_QSTR(MP_QSTR_sigfox_info), (mp_obj_t)&mod_pycom_sigfox_info_obj }, From 684e9da9ef75bd4c7dce73b2774b34b570859523 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9za=20Husi?= Date: Tue, 29 Dec 2020 18:48:13 +0100 Subject: [PATCH 07/11] Fix a bug causing multi-heap assert exception in modcoap and improve modcoap's memory handling (#208) The following is fixed: * Incorrectly called "free" on the "path" pointer, "path" pointer did not point to the correct memory area when free() was called * Changed m_malloc and m_free to malloc and free because those objects not needed to be allocated on the Garbage Collector controlled heap * Removed the "uri" member of "resource" object because it was not used --- esp32/mods/modcoap.c | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/esp32/mods/modcoap.c b/esp32/mods/modcoap.c index cae166d61d..ac8a98be41 100644 --- a/esp32/mods/modcoap.c +++ b/esp32/mods/modcoap.c @@ -43,7 +43,6 @@ typedef struct mod_coap_resource_obj_s { coap_resource_t* coap_resource; struct mod_coap_resource_obj_s* next; uint8_t* value; - unsigned char* uri; uint32_t max_age; uint16_t etag_value; uint16_t value_len; @@ -208,9 +207,10 @@ STATIC mod_coap_resource_obj_t* add_resource(const char* uri, uint8_t mediatype, resource->next = NULL; // uri parameter pointer will be destroyed, pass a pointer to a permanent location - resource->uri = m_malloc(strlen(uri)); - memcpy(resource->uri, uri, strlen(uri)); - resource->coap_resource = coap_resource_init((const unsigned char* )resource->uri, strlen(uri), 0); + unsigned char* uri_ptr = (unsigned char*)malloc(strlen(uri)); + memcpy(uri_ptr, uri, strlen(uri)); + // Pass COAP_RESOURCE_FLAGS_RELEASE_URI so Coap Library will free up the memory allocated to store the URI when the Resource is deleted + resource->coap_resource = coap_resource_init(uri_ptr, strlen(uri), COAP_RESOURCE_FLAGS_RELEASE_URI); if(resource->coap_resource != NULL) { // Add the resource to the Coap context coap_add_resource(context->context, resource->coap_resource); @@ -238,7 +238,7 @@ STATIC mod_coap_resource_obj_t* add_resource(const char* uri, uint8_t mediatype, return resource; } else { - m_free(resource->uri); + free(uri_ptr); m_del_obj(mod_coap_resource_obj_t, resource); // Resource cannot be created return NULL; @@ -278,14 +278,12 @@ STATIC void remove_resource_by_key(coap_key_t key) { previous->next = current->next; } - // Free the URI - m_free(current->uri); // Free the resource in coap's scope coap_delete_resource(context->context, key); // Free the element in MP scope - m_free(current->value); + free(current->value); // Free the resource itself - m_free(current); + m_del_obj(mod_coap_resource_obj_t, current); return; } @@ -320,7 +318,7 @@ STATIC void resource_update_value(mod_coap_resource_obj_t* resource, mp_obj_t ne // Invalidate current data first resource->value_len = 0; - m_free(resource->value); + free(resource->value); if (mp_obj_is_integer(new_value)) { @@ -334,7 +332,7 @@ STATIC void resource_update_value(mod_coap_resource_obj_t* resource, mp_obj_t ne } // Allocate memory for the new data - resource->value = m_malloc(resource->value_len); + resource->value = malloc(resource->value_len); memcpy(resource->value, &value, sizeof(value)); } else { @@ -344,7 +342,7 @@ STATIC void resource_update_value(mod_coap_resource_obj_t* resource, mp_obj_t ne resource->value_len = value_bufinfo.len; // Allocate memory for the new data - resource->value = m_malloc(resource->value_len); + resource->value = malloc(resource->value_len); memcpy(resource->value, value_bufinfo.buf, resource->value_len); } } @@ -748,16 +746,13 @@ STATIC coap_pdu_t * modcoap_new_request // Helper function to create a new option for a request message STATIC coap_list_t * modcoap_new_option_node(unsigned short key, unsigned int length, unsigned char *data) { - coap_list_t *node = m_malloc(sizeof(coap_list_t) + sizeof(coap_option) + length); + coap_list_t *node = malloc(sizeof(coap_list_t) + sizeof(coap_option) + length); if (node) { coap_option *option; option = (coap_option *)(node->data); COAP_OPTION_KEY(*option) = key; COAP_OPTION_LENGTH(*option) = length; memcpy(COAP_OPTION_DATA(*option), data, length); - } else { - m_free(node); - node = NULL; } return node; @@ -937,7 +932,7 @@ STATIC mp_obj_t mod_coap_init(mp_uint_t n_args, const mp_obj_t *pos_args, mp_map // Only 1 context is supported currently if(initialized == false) { - MP_STATE_PORT(coap_ptr) = m_malloc(sizeof(mod_coap_obj_t)); + MP_STATE_PORT(coap_ptr) = m_new_obj(mod_coap_obj_t); coap_obj_ptr = MP_STATE_PORT(coap_ptr); coap_obj_ptr->context = NULL; coap_obj_ptr->resources = NULL; @@ -1235,19 +1230,21 @@ STATIC mp_obj_t mod_coap_send_request(mp_uint_t n_args, const mp_obj_t *pos_args //TODO: allocate the proper length size_t length = 300; unsigned char* path = malloc(length); - int segments = coap_split_path(coap_uri.path.s, coap_uri.path.length, path, &length); + // Need to use a different pointer because when the segments are composed the pointer itself is moved + unsigned char* path_segment = path; + int segments = coap_split_path(coap_uri.path.s, coap_uri.path.length, path_segment, &length); // Insert the segments as separate URI-Path options while (segments--) { - node = modcoap_new_option_node(COAP_OPTION_URI_PATH, COAP_OPT_LENGTH(path), COAP_OPT_VALUE(path)); + node = modcoap_new_option_node(COAP_OPTION_URI_PATH, COAP_OPT_LENGTH(path_segment), COAP_OPT_VALUE(path_segment)); if(node != NULL) { LL_APPEND(coap_obj_ptr->optlist, node); } - - path += COAP_OPT_SIZE(path); + // Move the path_segment pointer to the next segment + path_segment += COAP_OPT_SIZE(path_segment); } - + // Free up the memory using the pointer pointing to the beginning of the memory area free(path); // Put Content Format option if given @@ -1271,7 +1268,7 @@ STATIC mp_obj_t mod_coap_send_request(mp_uint_t n_args, const mp_obj_t *pos_args while(coap_obj_ptr->optlist != NULL) { next = coap_obj_ptr->optlist->next; coap_obj_ptr->optlist->next = NULL; - m_free(coap_obj_ptr->optlist); + free(coap_obj_ptr->optlist); coap_obj_ptr->optlist = next; } From 627e39c4a5ff968faaf19c43b4ed0bd454bdd054 Mon Sep 17 00:00:00 2001 From: msariisik Date: Thu, 7 Jan 2021 14:28:03 +0100 Subject: [PATCH 08/11] removed exit 0 condition for tests --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 28fdf0b66a..58ec285aa6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -122,7 +122,7 @@ def testBuild(short_name) { timeout(30) { // As some tests are randomly failing... enforce script always returns 0 (OK) sh '''export PATH=$PATH:/usr/local/bin; - ./run-tests --target=esp32 --device ''' + device_name + ' || exit 0' + ./run-tests --target=esp32 --device ''' + device_name } } sh 'python esp32/tools/pypic.py --port ' + device_name +' --enter' From 8cd6d219d1e348e66d19ed3ce59e30dab378536a Mon Sep 17 00:00:00 2001 From: msariisik Date: Mon, 11 Jan 2021 11:00:35 +0100 Subject: [PATCH 09/11] revert back exit 0 condition on Jenkins --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 58ec285aa6..28fdf0b66a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -122,7 +122,7 @@ def testBuild(short_name) { timeout(30) { // As some tests are randomly failing... enforce script always returns 0 (OK) sh '''export PATH=$PATH:/usr/local/bin; - ./run-tests --target=esp32 --device ''' + device_name + ./run-tests --target=esp32 --device ''' + device_name + ' || exit 0' } } sh 'python esp32/tools/pypic.py --port ' + device_name +' --enter' From 120f929baa1fb92ebdfdba066f9c51b9a34df856 Mon Sep 17 00:00:00 2001 From: gijsio <67470426+gijsio@users.noreply.github.com> Date: Mon, 11 Jan 2021 13:12:13 +0100 Subject: [PATCH 10/11] Rtc utime fix (#206) * fixed setting time in rtc * do not reset the time on init --- esp32/mods/machrtc.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/esp32/mods/machrtc.c b/esp32/mods/machrtc.c index f2a819b68b..df91a404ce 100644 --- a/esp32/mods/machrtc.c +++ b/esp32/mods/machrtc.c @@ -48,7 +48,6 @@ typedef struct _mach_rtc_obj_t { bool synced; } mach_rtc_obj_t; -static RTC_DATA_ATTR uint64_t delta_from_epoch_til_boot; static RTC_DATA_ATTR uint32_t rtc_user_mem_len; static RTC_DATA_ATTR uint8_t rtc_user_mem_data[MEM_USER_MAXLEN]; @@ -61,10 +60,10 @@ void rtc_init0(void) { void mach_rtc_set_us_since_epoch(uint64_t nowus) { struct timeval tv; - // store the packet timestamp - gettimeofday(&tv, NULL); - delta_from_epoch_til_boot = nowus - (uint64_t)((tv.tv_sec * 1000000ull) + tv.tv_usec); + tv.tv_usec = nowus % 1000000ull; + tv.tv_sec = nowus / 1000000ull; + settimeofday(&tv, NULL); } void mach_rtc_synced (void) { @@ -78,8 +77,9 @@ bool mach_is_rtc_synced (void) { uint64_t mach_rtc_get_us_since_epoch(void) { struct timeval tv; gettimeofday(&tv, NULL); - return (uint64_t)((tv.tv_sec * 1000000ull) + tv.tv_usec) + delta_from_epoch_til_boot; -}; + return (uint64_t)(tv.tv_sec * 1000000ull ) + (tv.tv_usec); + +} STATIC uint64_t mach_rtc_datetime_us(const mp_obj_t datetime) { timeutils_struct_time_t tm; @@ -132,8 +132,6 @@ STATIC void mach_rtc_datetime(const mp_obj_t datetime) { if (datetime != mp_const_none) { useconds = mach_rtc_datetime_us(datetime); mach_rtc_set_us_since_epoch(useconds); - } else { - mach_rtc_set_us_since_epoch(0); } } @@ -197,14 +195,10 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_KW(mach_rtc_init_obj, 1, mach_rtc_init); STATIC mp_obj_t mach_rtc_now (mp_obj_t self_in) { timeutils_struct_time_t tm; uint64_t useconds; + + useconds = mach_rtc_get_us_since_epoch(); - struct timeval now; - gettimeofday(&now, NULL); - - // get the time from the RTC - useconds = (now.tv_sec * 1000000ull ) + (now.tv_usec); timeutils_seconds_since_epoch_to_struct_time((useconds) / 1000000ull, &tm); - mp_obj_t tuple[8] = { mp_obj_new_int(tm.tm_year), mp_obj_new_int(tm.tm_mon), From f04187b5554007b0c9fa6ce6cd132d127d778f37 Mon Sep 17 00:00:00 2001 From: Peter Putz Date: Mon, 11 Jan 2021 13:21:44 +0100 Subject: [PATCH 11/11] update version to 1.20.2.r4 --- esp32/pycom_version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp32/pycom_version.h b/esp32/pycom_version.h index b85d758bb1..3a24ceee49 100644 --- a/esp32/pycom_version.h +++ b/esp32/pycom_version.h @@ -10,7 +10,7 @@ #ifndef VERSION_H_ #define VERSION_H_ -#define SW_VERSION_NUMBER "1.20.2.r3" +#define SW_VERSION_NUMBER "1.20.2.r4" #define LORAWAN_VERSION_NUMBER "1.0.2"