Pybytes 1.3.0 (#372)

.gitignore

@@ -1,14 +1,3 @@
-# Compiled Sources
-###################
-*.o
-*.a
-*.elf
-*.bin
-*.map
-*.hex
-*.dis
-*.exe
-
 # Packages 
 ############
 
@@ -20,10 +9,13 @@
 ######################
 *.swp
 
-# Build directory
+# Build directories
 ######################
-build/
-build-*/
+esp32/build/*
+esp32/build-PYBYTES/*
+mpy-cross/build/*
+# This map file is generated here instead of the build folder...
+mpy-cross/*.map
 
 # Test failure outputs
 ######################
@@ -50,8 +42,9 @@ secure_boot_signing_key.pem
 signature_verification_key.bin
 secure-bootloader-key.bin
 flash_encryption_key.bin
- 
+
 .DS_Store
 org.eclipse.cdt.ui.prefs
 org.eclipse.cdt.core.prefs
 language.settings.xml
+.idea

esp32/frozen/LTE/sqnsupgrade.py

@@ -129,6 +129,7 @@ def return_code(self, rsp, debug=False):
 
 
     def wait_for_modem(self, send=True, expected=b'OK', echo_char=None):
+        self.__serial.read()
         rsp = b''
         start = time.time()
         while True:
@@ -339,13 +340,14 @@ def __get_wait_msg(self, load_fff=True):
 
 
 
-    def __run(self, file_path=None, baudrate=921600, port=None, resume=False, load_ffh=False, mirror=False, switch_ffh=False, bootrom=False, rgbled=0x050505, debug=False, pkgdebug=False, atneg=True, max_try=10, direct=True, atneg_only=False, info_only=False, expected_smod=None, verbose=False, load_fff=False, mtools=False, fc=False):
+    def __run(self, file_path=None, baudrate=921600, port=None, resume=False, load_ffh=False, mirror=False, switch_ffh=False, bootrom=False, rgbled=0x050505, debug=False, pkgdebug=False, atneg=True, max_try=10, direct=True, atneg_only=False, info_only=False, expected_smod=None, verbose=False, load_fff=False, mtools=False, fc=False, force_fff=False):
         self.__wait_msg = False
         mirror = True if atneg_only else mirror
         recover = True if atneg_only else load_ffh
         resume = True if mirror or recover or atneg_only or info_only else resume
         verbose = True if debug else verbose
         load_fff = False if bootrom and switch_ffh else load_fff
+        load_fff = True if force_fff else load_fff
         target_baudrate = baudrate
         baudrate = self.__modem_speed if self.__speed_detected else baudrate
         if debug: print('mirror? {}  recover? {}  resume? {}  direct? {}  atneg_only? {} bootrom? {} load_fff? {}'.format(mirror, recover, resume, direct, atneg_only, bootrom, load_fff))
@@ -410,9 +412,9 @@ def __run(self, file_path=None, baudrate=921600, port=None, resume=False, load_f
                     reconnect_uart()
                     return False
                 if blobsize > 4194304:
-                    if load_fff:
+                    if load_fff and not force_fff:
                         print("Firmware file is too big to load via FFF method. Using ON_THE_FLY")
-                    load_fff = False
+                        load_fff = False
                 blob = open(file_path, "rb")
 
         if not load_ffh:
@@ -550,7 +552,8 @@ def __run(self, file_path=None, baudrate=921600, port=None, resume=False, load_f
                 raise OSError("Invalid answer '%s' from the device" % response)
                 try:
                     blob.close()
-                except:
+                except Exception as ex:
+                    if debug: print('Exception: {}'.format(ex))
                     pass
 
             self.__serial.read()
@@ -595,7 +598,8 @@ def __run(self, file_path=None, baudrate=921600, port=None, resume=False, load_f
             if start == True:
                 try:
                     blob.close()
-                except:
+                except Exception as ex:
+                    if debug: print('Exception: {}'.format(ex))
                     pass
                 self.__serial.read()
                 if switch_ffh:
@@ -612,15 +616,18 @@ def __run(self, file_path=None, baudrate=921600, port=None, resume=False, load_f
             else:
                 try:
                     blob.close()
-                except:
+                except Exception as ex:
+                    if debug: print('Exception: {}'.format(ex))
                     pass
                 print('Code download failed[1], aborting!')
                 return False
         except Exception as ex:
             try:
                 blob.close()
-            except:
+            except Exception as ex:
+                if debug: print('Exception: {}'.format(ex))
                 pass
+
             print('Exception: {}'.format(ex))
             print('Code download failed [2], aborting!')
             abort = True
@@ -870,7 +877,7 @@ def success_message(self, port=None, verbose=False, debug=False):
         print("Here is the current firmware version:\n")
         self.show_info(port=port, verbose=verbose, debug=debug)
 
-    def upgrade(self, ffile, mfile=None, baudrate=921600, retry=False, resume=False, debug=False, pkgdebug=False, verbose=False, load_fff=True, load_only=False, mtools=False):
+    def upgrade(self, ffile, mfile=None, baudrate=921600, retry=False, resume=False, debug=False, pkgdebug=False, verbose=False, load_fff=True, load_only=False, mtools=False, force_fff=False):
         success = True
         if not retry and mfile is not None:
             if resume or self.__check_br(br_only=True, verbose=verbose, debug=debug):
@@ -895,7 +902,7 @@ def upgrade(self, ffile, mfile=None, baudrate=921600, retry=False, resume=False,
             print('Unable to upgrade bootrom.')
         if debug: print('Success2? {}'.format(success))
         if success:
-            if self.__run(file_path=ffile, resume=True if mfile is not None else resume, baudrate=baudrate, direct=False, debug=debug, pkgdebug=pkgdebug, verbose=verbose, load_fff=False if mfile else load_fff, mtools=mtools):
+            if self.__run(file_path=ffile, resume=True if mfile is not None else resume, baudrate=baudrate, direct=False, debug=debug, pkgdebug=pkgdebug, verbose=verbose, load_fff=False if mfile else load_fff, mtools=mtools, force_fff=force_fff):
                 if self.__check_br(verbose=verbose, debug=debug):
                     success = self.__run(bootrom=True, debug=debug, direct=False, pkgdebug=pkgdebug, verbose=verbose, load_fff=True)
                 self.success_message(verbose=verbose, debug=debug)
@@ -942,13 +949,13 @@ def upgrade_uart(self, ffh_mode=False, mfile=None, retry=False, resume=False, co
     def show_info(self, port=None, debug=False, verbose=False, fc=False):
         self.__run(port=port, debug=debug, info_only=True, verbose=verbose, fc=fc)
 
-    def upgrade_ext(self, port, ffile, mfile, resume=False, debug=False, pkgdebug=False, verbose=False, load_fff=True, fc=False):
+    def upgrade_ext(self, port, ffile, mfile, resume=False, debug=False, pkgdebug=False, verbose=False, load_fff=True, fc=False, force_fff=False):
         success = True
         if mfile is not None:
             success = False
             success = self.__run(file_path=mfile, load_ffh=True, port=port, debug=debug, pkgdebug=pkgdebug, verbose=verbose, fc=fc)
         if success:
-            if self.__run(file_path=ffile, resume=True if mfile is not None else resume, direct=False, port=port, debug=debug, pkgdebug=pkgdebug, verbose=verbose, load_fff=load_fff, fc=fc):
+            if self.__run(file_path=ffile, resume=True if mfile is not None else resume, direct=False, port=port, debug=debug, pkgdebug=pkgdebug, verbose=verbose, load_fff=load_fff, fc=fc, force_fff=force_fff):
                 self.success_message(port=port, verbose=verbose, debug=debug)
         else:
             print('Unable to load updater from {}'.format(mfile))
@@ -984,7 +991,7 @@ def load(mfile, baudrate=921600, verbose=False, debug=False, hangup=False):
                 print('Modem must be in recovery mode!')
         reconnect_uart()
 
-    def run(ffile, mfile=None, baudrate=921600, verbose=False, debug=False, load_fff=True, hangup=True):
+    def run(ffile, mfile=None, baudrate=921600, verbose=False, debug=False, load_fff=True, hangup=True, force_fff=False):
         print_welcome()
         retry = False
         resume = False
@@ -1008,7 +1015,7 @@ def run(ffile, mfile=None, baudrate=921600, verbose=False, debug=False, load_fff
                 mtools = True
             elif state == -1:
                 detect_error()
-            success = sqnup.upgrade(ffile=ffile, mfile=mfile, baudrate=baudrate, retry=retry, resume=resume, debug=debug, pkgdebug=False, verbose=verbose, load_fff=load_fff, mtools=mtools)
+            success = sqnup.upgrade(ffile=ffile, mfile=mfile, baudrate=baudrate, retry=retry, resume=resume, debug=debug, pkgdebug=False, verbose=verbose, load_fff=load_fff, mtools=mtools, force_fff=force_fff)
         reconnect_uart()
         return success
 
@@ -1072,11 +1079,11 @@ def state(verbose=False, debug=False, retry=5, hangup=False):
         return sqnup.detect_modem_state(debug=debug, hangup=hangup, retry=retry)
 
 else:
-    def run(port, ffile, mfile=None, resume=False, debug=False, verbose=False, load_fff=True, fc=False):
+    def run(port, ffile, mfile=None, resume=False, debug=False, verbose=False, load_fff=True, fc=False, force_fff=False):
         print_welcome()
         sqnup = sqnsupgrade()
         if sqnup.check_files(ffile, mfile, debug):
-            sqnup.upgrade_ext(port=port, ffile=ffile, mfile=mfile, resume=resume, debug=debug, pkgdebug=False, verbose=verbose, load_fff=load_fff)
+            sqnup.upgrade_ext(port=port, ffile=ffile, mfile=mfile, resume=resume, debug=debug, pkgdebug=False, verbose=verbose, load_fff=load_fff, force_fff=force_fff)
 
     def version(port, verbose=False, debug=False, fc=False):
         sqnup = sqnsupgrade()

esp32/frozen/Pybytes/_OTA.py

@@ -5,6 +5,20 @@
 see the Pycom Licence v1.0 document supplied with this file, or
 available at https://www.pycom.io/opensource/licensing
 '''
+try:
+    from pybytes_debug import print_debug
+except:
+    from _pybytes_debug import print_debug
+
+try:
+    from pybytes_constants import constants
+except:
+    from _pybytes_constants import constants
+
+try:
+    import urequest
+except:
+    import _urequest as urequest
 
 import network
 import socket
@@ -18,11 +32,6 @@
 import os
 from binascii import hexlify
 
-try:
-    from pybytes_debug import print_debug
-except:
-    from _pybytes_debug import print_debug
-
 # Try to get version number
 # try:
 #     from OTA_VERSION import VERSION
@@ -40,6 +49,9 @@ def connect(self):
     def get_data(self, req, dest_path=None, hash=False):
         raise NotImplementedError()
 
+    def update_device_network_config(self, fcota, config):
+        raise NotImplementedError()
+
     # OTA methods
 
     def get_current_version(self):
@@ -56,9 +68,9 @@ def get_update_manifest(self):
         gc.collect()
         return manifest
 
-    def update(self):
+    def update(self, customManifest=None):
         try:
-            manifest = self.get_update_manifest()
+            manifest = self.get_update_manifest() if not customManifest else customManifest
         except Exception as e:
             print('Error reading the manifest, aborting: {}'.format(e))
             return 0
@@ -68,36 +80,38 @@ def update(self):
             return 1
 
         # Download new files and verify hashes
-        for f in manifest['new'] + manifest['update']:
-            # Upto 5 retries
-            for _ in range(5):
-                try:
-                    self.get_file(f)
-                    break
-                except Exception as e:
-                    print(e)
-                    msg = "Error downloading `{}` retrying..."
-                    print(msg.format(f['URL']))
-                    return 0
-            else:
-                raise Exception("Failed to download `{}`".format(f['URL']))
-
-        # Backup old files
-        # only once all files have been successfully downloaded
-        for f in manifest['update']:
-            self.backup_file(f)
-
-        # Rename new files to proper name
-        for f in manifest['new'] + manifest['update']:
-            new_path = "{}.new".format(f['dst_path'])
-            dest_path = "{}".format(f['dst_path'])
-
-            os.rename(new_path, dest_path)
-
-        # `Delete` files no longer required
-        # This actually makes a backup of the files incase we need to roll back
-        for f in manifest['delete']:
-            self.delete_file(f)
+        if "new" in manifest and "update" in manifest:
+            for f in manifest['new'] + manifest['update']:
+                # Upto 5 retries
+                for _ in range(5):
+                    try:
+                        self.get_file(f)
+                        break
+                    except Exception as e:
+                        print(e)
+                        msg = "Error downloading `{}` retrying..."
+                        print(msg.format(f['URL']))
+                        return 0
+                else:
+                    raise Exception("Failed to download `{}`".format(f['URL']))
+
+            # Backup old files
+            # only once all files have been successfully downloaded
+            for f in manifest['update']:
+                self.backup_file(f)
+
+            # Rename new files to proper name
+            for f in manifest['new'] + manifest['update']:
+                new_path = "{}.new".format(f['dst_path'])
+                dest_path = "{}".format(f['dst_path'])
+
+                os.rename(new_path, dest_path)
+
+        if "delete" in manifest:
+            # `Delete` files no longer required
+            # This actually makes a backup of the files incase we need to roll back
+            for f in manifest['delete']:
+                self.delete_file(f)
 
         # Flash firmware
         if "firmware" in manifest:
@@ -162,9 +176,12 @@ def delete_file(self, f):
         os.rename(dest_path, bak_path)
 
     def write_firmware(self, f):
-        hash = self.get_data(f['URL'].split("/", 3)[-1],
-                             hash=True,
-                             firmware=True)
+        # hash =
+        self.get_data(
+            f['URL'].split("/", 3)[-1],
+            hash=True,
+            firmware=True
+        )
         # TODO: Add verification when released in future firmware
 
 
@@ -258,6 +275,7 @@ def get_data(self, req, dest_path=None, hash=False, firmware=False):
             if fp is not None:
                 fp.close()
             if firmware:
+                print_debug(6, 'ota_finish')
                 pycom.ota_finish()
 
         except Exception as e:
@@ -277,3 +295,47 @@ def get_data(self, req, dest_path=None, hash=False, firmware=False):
                 return bytes(content)
         elif hash:
             return hash_val
+
+    def update_device_network_config(self, fcota, config):
+        targetURL = '{}://{}/device/networks/{}'.format(
+            constants.__DEFAULT_PYCONFIG_PROTOCOL, constants.__DEFAULT_PYCONFIG_DOMAIN, config['device_id']
+        )
+        print_debug(6, "request device update URL: {}".format(targetURL))
+        try:
+            pybytes_activation = urequest.get(targetURL, headers={'content-type': 'application/json'})
+            responseDetails = pybytes_activation.json()
+            pybytes_activation.close()
+            print_debug(6, "Response Details: {}".format(responseDetails))
+            self.update_network_config(responseDetails, fcota, config)
+            machine.reset()
+        except Exception as ex:
+            print_debug(1, "error while calling {}!: {}".format(targetURL, ex))
+
+    def update_network_config(self, letResp, fcota, config):
+        try:
+            if 'networkConfig' in letResp:
+                netConf = letResp['networkConfig']
+                config['network_preferences'] = netConf['networkPreferences']
+                if 'wifi' in netConf:
+                    config['wifi'] = netConf['wifi']
+                elif 'wifi' in config:
+                    del config['wifi']
+
+                if 'lte' in netConf:
+                    config['lte'] = netConf['lte']
+                elif 'lte' in config:
+                    del config['lte']
+
+                if 'lora' in netConf:
+                    config['lora'] = {
+                        'otaa': netConf['lora']['otaa'],
+                        'abp': netConf['lora']['abp']
+                    }
+                elif 'lora' in config:
+                    del config['lora']
+
+                json_string = ujson.dumps(config)
+                print_debug(1, "update_network_config : {}".format(json_string))
+                fcota.update_file_content('/flash/pybytes_config.json', json_string)
+        except Exception as e:
+            print_debug(1, "error while updating network config pybytes_config.json! {}".format(e))