[v1.20.0.rc13] Release Candidate

### Improvments
- LoRaWan refactoring and bug Fixing
- Updated Pybytes lib
- Updated sqnsupgrade
- Update Make Environment for IDF version check and checkout
- modpycom: Added API to get Internal and External Free RAM size
- PYFW-356 support for sending bytes in lte.send_at_cmd()

### Bug Fix
- Fix PYFW-359: When building SW with "BTYPE=debug" littleFS assertion fails with crash
- PYFW-348 updated rx buff size max size exception message
- PYFW-354: LittleFS causing random crash

Author: Islam Wahdan

.issuetracker

@@ -0,0 +1,7 @@
+# Integration with Issue Tracker
+#
+# (note that '\' need to be escaped).
+
+[issuetracker "PYFW jira rule"]
+  regex = "PYFW-(\\d+)"
+  url = "https://pycomiot.atlassian.net/browse/PYFW-$1"

esp32/Makefile

@@ -14,6 +14,8 @@ ifeq ($(wildcard boards/$(BOARD)/.),)
     $(error Invalid BOARD specified)
 endif
 
+IDF_VERSION=3.1
+
 TARGET ?= boot_app
 
 OEM ?= 0
@@ -70,9 +72,6 @@ ifndef PROJECT_PATH
 PROJECT_PATH := $(abspath $(dir $(firstword $(MAKEFILE_LIST))))
 endif
 
-#check Frozen files changes
-$(shell bash tools/mpy-build-check.sh $(BOARD) $(BTYPE) $(VARIANT))
-
 FROZEN_MPY_DIR = frozen
 
 include ../py/mkenv.mk
@@ -130,7 +129,7 @@ endif
 
 # Enable or Disable LTE_LOG_BUFF
 ifeq ($(BOARD), $(filter $(BOARD), GPY FIPY))
-ifeq ($(LTE_DEBUG_BUFF),1)
+ifeq ($(LTE_LOG_BUFF),1)
 	CFLAGS += -DLTE_DEBUG_BUFF
 endif #ifeq ($(LTE_LOG_BUFF),1)
 endif #ifeq ($(BOARD), $(filter $(BOARD), GPY FIPY))

esp32/application.mk

@@ -503,7 +503,7 @@ endif
 ifeq ($(TARGET), boot_app)
 all: $(BOOT_BIN) $(APP_BIN)
 endif
-.PHONY: all
+.PHONY: all CHECK_DEP
 
 $(info $(VARIANT) Variant) 
 ifeq ($(SECURE), on)
@@ -562,10 +562,6 @@ $(BUILD)/bootloader/bootloader.a: $(BOOT_OBJ) sdkconfig.h
 	$(Q) $(AR) cru $@ $^
 
 $(BUILD)/bootloader/bootloader.elf: $(BUILD)/bootloader/bootloader.a $(SECURE_BOOT_VERIFICATION_KEY)
-ifeq ($(COPY_IDF_LIB), 1)
-	$(ECHO) "COPY IDF LIBRARIES $@"
-	$(Q) $(PYTHON) get_idf_libs.py --idflibs $(IDF_PATH)/examples/wifi/scan/build
-endif
 ifeq ($(SECURE), on)
 # unpack libbootloader_support.a, and archive again using the right key for verifying signatures
 	$(ECHO) "Inserting verification key $(SECURE_BOOT_VERIFICATION_KEY) in $@"
@@ -627,10 +623,6 @@ $(BUILD)/application.a: $(OBJ)
 	$(Q) rm -f $@
 	$(Q) $(AR) cru $@ $^
 $(BUILD)/application.elf: $(BUILD)/application.a $(BUILD)/esp32_out.ld $(SECURE_BOOT_VERIFICATION_KEY)
-ifeq ($(COPY_IDF_LIB), 1)
-	$(ECHO) "COPY IDF LIBRARIES $@"
-	$(Q) $(PYTHON) get_idf_libs.py --idflibs $(IDF_PATH)/examples/wifi/scan/build
-endif
 ifeq ($(SECURE), on)
 # unpack libbootloader_support.a, and archive again using the right key for verifying signatures
 	$(ECHO) "Inserting verification key $(SECURE_BOOT_VERIFICATION_KEY) in $@"
@@ -750,13 +742,27 @@ GEN_PINS_SRC = $(BUILD)/pins.c
 GEN_PINS_HDR = $(HEADER_BUILD)/pins.h
 GEN_PINS_QSTR = $(BUILD)/pins_qstr.h
 
+.NOTPARALLEL: CHECK_DEP $(OBJ)
+.NOTPARALLEL: CHECK_DEP $(BOOT_OBJ)
+
+$(BOOT_OBJ) $(OBJ): | CHECK_DEP
+
 # Making OBJ use an order-only dependency on the generated pins.h file
 # has the side effect of making the pins.h file before we actually compile
 # any of the objects. The normal dependency generation will deal with the
 # case when pins.h is modified. But when it doesn't exist, we don't know
 # which source files might need it.
 $(OBJ): | $(GEN_PINS_HDR)
 
+# Check Dependencies (IDF version, Frozen code and IDF LIBS)
+CHECK_DEP:
+	$(Q) bash tools/idfVerCheck.sh $(IDF_PATH) "$(IDF_VERSION)"
+	$(Q) bash tools/mpy-build-check.sh $(BOARD) $(BTYPE) $(VARIANT)
+ifeq ($(COPY_IDF_LIB), 1)
+	$(ECHO) "COPY IDF LIBRARIES"
+	$(Q) $(PYTHON) get_idf_libs.py --idflibs $(IDF_PATH)/examples/wifi/scan/build
+endif
+
 # Call make-pins.py to generate both pins_gen.c and pins.h
 $(GEN_PINS_SRC) $(GEN_PINS_HDR) $(GEN_PINS_QSTR): $(BOARD_PINS) $(MAKE_PINS) $(AF_FILE) $(PREFIX_FILE) | $(HEADER_BUILD)
 	$(ECHO) "Create $@"

esp32/frozen/LTE/sqnsupgrade.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-VERSION = "1.2.5"
+VERSION = "1.2.6"
 
 # Copyright (c) 2019, Pycom Limited.
 #
@@ -339,7 +339,7 @@ def __get_wait_msg(self, load_fff=True):
 
 
 
-    def __run(self, file_path=None, baudrate=921600, port=None, resume=False, load_ffh=False, mirror=False, switch_ffh=False, bootrom=False, rgbled=0x050505, debug=False, pkgdebug=False, atneg=True, max_try=10, direct=True, atneg_only=False, info_only=False, expected_smod=None, verbose=False, load_fff=False, mtools=False):
+    def __run(self, file_path=None, baudrate=921600, port=None, resume=False, load_ffh=False, mirror=False, switch_ffh=False, bootrom=False, rgbled=0x050505, debug=False, pkgdebug=False, atneg=True, max_try=10, direct=True, atneg_only=False, info_only=False, expected_smod=None, verbose=False, load_fff=False, mtools=False, fc=False):
         self.__wait_msg = False
         mirror = True if atneg_only else mirror
         recover = True if atneg_only else load_ffh
@@ -365,7 +365,7 @@ def __run(self, file_path=None, baudrate=921600, port=None, resume=False, load_f
             external = True
             br = 115200 if recover and not direct else baudrate
             if debug: print('Setting baudrate to {}'.format(br))
-            self.__serial = serial.Serial(port, br, bytesize=serial.EIGHTBITS, timeout=1 if info_only else 0.1)
+            self.__serial = serial.Serial(port, br, bytesize=serial.EIGHTBITS, timeout=1 if info_only else 0.1, rtscts=fc)
             self.__serial.reset_input_buffer()
             self.__serial.reset_output_buffer()
 
@@ -447,7 +447,7 @@ def __run(self, file_path=None, baudrate=921600, port=None, resume=False, load_f
 
             if verbose: print('Sending AT+FSRDFILE="/fs/crashdump"')
             self.__serial.write(b'AT+FSRDFILE="/fs/crashdump"\r\n')
-            response = self.read_rsp(size=100)
+            response = self.read_rsp(size=1024)
             if verbose: print('AT+FSRDFILE="/fs/crashdump" returned {}'.format(response))
             self.__serial.read()
 
@@ -819,7 +819,7 @@ def at_negotiation(self, baudrate, port, max_try, mirror, atneg_only, debug, tar
                     self.__serial = UART(1, baudrate=target_baudrate, pins=self.__pins, timeout_chars=100)
                 else:
                     self.__serial = None
-                    self.__serial = serial.Serial(port, target_baudrate, bytesize=serial.EIGHTBITS, timeout=0.1)
+                    self.__serial = serial.Serial(port, target_baudrate, bytesize=serial.EIGHTBITS, timeout=0.1, rtscts=fc)
                     self.__serial.reset_input_buffer()
                     self.__serial.reset_output_buffer()
                     self.__serial.flush()
@@ -882,10 +882,11 @@ def upgrade(self, ffile, mfile=None, baudrate=921600, retry=False, resume=False,
         if success:
             if self.__run(file_path=ffile, resume=True if mfile is not None else resume, baudrate=baudrate, direct=False, debug=debug, pkgdebug=pkgdebug, verbose=verbose, load_fff=False if mfile else load_fff, mtools=mtools):
                 if self.__check_br(verbose=verbose, debug=debug):
-                    self.__run(bootrom=True, debug=debug, direct=False, pkgdebug=pkgdebug, verbose=verbose, load_fff=True)
+                    success = self.__run(bootrom=True, debug=debug, direct=False, pkgdebug=pkgdebug, verbose=verbose, load_fff=True)
                 self.success_message(verbose=verbose, debug=debug)
         else:
             print('Unable to load updater from {}'.format(mfile))
+        return success
 
     def upgrade_uart(self, ffh_mode=False, mfile=None, retry=False, resume=False, color=0x050505, debug=False, pkgdebug=False, verbose=False, load_fff=True):
         success = False
@@ -923,16 +924,16 @@ def upgrade_uart(self, ffh_mode=False, mfile=None, retry=False, resume=False, co
         else:
             print('Unable to upgrade bootrom.')
 
-    def show_info(self, port=None, debug=False, verbose=False):
-        self.__run(port=port, debug=debug, info_only=True, verbose=verbose)
+    def show_info(self, port=None, debug=False, verbose=False, fc=False):
+        self.__run(port=port, debug=debug, info_only=True, verbose=verbose, fc=fc)
 
-    def upgrade_ext(self, port, ffile, mfile, resume=False, debug=False, pkgdebug=False, verbose=False, load_fff=True):
+    def upgrade_ext(self, port, ffile, mfile, resume=False, debug=False, pkgdebug=False, verbose=False, load_fff=True, fc=False):
         success = True
         if mfile is not None:
             success = False
-            success = self.__run(file_path=mfile, load_ffh=True, port=port, debug=debug, pkgdebug=pkgdebug, verbose=verbose)
+            success = self.__run(file_path=mfile, load_ffh=True, port=port, debug=debug, pkgdebug=pkgdebug, verbose=verbose, fc=fc)
         if success:
-            if self.__run(file_path=ffile, resume=True if mfile is not None else resume, direct=False, port=port, debug=debug, pkgdebug=pkgdebug, verbose=verbose, load_fff=load_fff):
+            if self.__run(file_path=ffile, resume=True if mfile is not None else resume, direct=False, port=port, debug=debug, pkgdebug=pkgdebug, verbose=verbose, load_fff=load_fff, fc=fc):
                 self.success_message(port=port, verbose=verbose, debug=debug)
         else:
             print('Unable to load updater from {}'.format(mfile))
@@ -973,6 +974,7 @@ def run(ffile, mfile=None, baudrate=921600, verbose=False, debug=False, load_fff
         retry = False
         resume = False
         mtools = False
+        success = False
         sqnup = sqnsupgrade()
         if sqnup.check_files(ffile, mfile, debug):
             state = sqnup.detect_modem_state(debug=debug, hangup=hangup)
@@ -991,8 +993,9 @@ def run(ffile, mfile=None, baudrate=921600, verbose=False, debug=False, load_fff
                 mtools = True
             elif state == -1:
                 detect_error()
-            sqnup.upgrade(ffile=ffile, mfile=mfile, baudrate=baudrate, retry=retry, resume=resume, debug=debug, pkgdebug=False, verbose=verbose, load_fff=load_fff, mtools=mtools)
+            success = sqnup.upgrade(ffile=ffile, mfile=mfile, baudrate=baudrate, retry=retry, resume=resume, debug=debug, pkgdebug=False, verbose=verbose, load_fff=load_fff, mtools=mtools)
         reconnect_uart()
+        return success
 
     def uart(ffh_mode=False, mfile=None, color=0x050505, verbose=False, debug=False, hangup=True):
         print_welcome()
@@ -1054,12 +1057,12 @@ def state(verbose=False, debug=False, retry=5, hangup=False):
         return sqnup.detect_modem_state(debug=debug, hangup=hangup, retry=retry)
 
 else:
-    def run(port, ffile, mfile=None, resume=False, debug=False, verbose=False, load_fff=True):
+    def run(port, ffile, mfile=None, resume=False, debug=False, verbose=False, load_fff=True, fc=False):
         print_welcome()
         sqnup = sqnsupgrade()
         if sqnup.check_files(ffile, mfile, debug):
             sqnup.upgrade_ext(port=port, ffile=ffile, mfile=mfile, resume=resume, debug=debug, pkgdebug=False, verbose=verbose, load_fff=load_fff)
 
-    def version(port, verbose=False, debug=False):
+    def version(port, verbose=False, debug=False, fc=False):
         sqnup = sqnsupgrade()
         sqnup.show_info(port=port, debug=debug, verbose=verbose)

esp32/frozen/Pybytes/_OTA.py

@@ -1,3 +1,11 @@
+'''
+Copyright (c) 2019, Pycom Limited.
+This software is licensed under the GNU GPL version 3 or any
+later version, with permitted additional terms. For more information
+see the Pycom Licence v1.0 document supplied with this file, or
+available at https://www.pycom.io/opensource/licensing
+'''
+
 import network
 import socket
 import ssl

esp32/frozen/Pybytes/_flash_control_OTA.py

@@ -1,9 +1,18 @@
+'''
+Copyright (c) 2019, Pycom Limited.
+This software is licensed under the GNU GPL version 3 or any
+later version, with permitted additional terms. For more information
+see the Pycom Licence v1.0 document supplied with this file, or
+available at https://www.pycom.io/opensource/licensing
+'''
+
 import os
 try:
     from pybytes_debug import print_debug
 except:
     from _pybytes_debug import print_debug
 
+
 class FCOTA:
     def __init__(self):
         pass