From a42c76254554f5189a3b017fabe97aac35679f13 Mon Sep 17 00:00:00 2001 From: tobozo Date: Mon, 26 Sep 2022 18:09:57 +0200 Subject: [PATCH 01/32] store manifest url locally, fixes #99 --- src/esp32FOTA.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/esp32FOTA.hpp b/src/esp32FOTA.hpp index b7e8aa7..18869cb 100644 --- a/src/esp32FOTA.hpp +++ b/src/esp32FOTA.hpp @@ -164,7 +164,7 @@ class esp32FOTA int getPayloadVersion(); void getPayloadVersion(char * version_string); - void setManifestURL( String manifest_url ) { _cfg.manifest_url = manifest_url.c_str(); } + void setManifestURL( String manifest_url ) { _manifestUrl = manifest_url; _cfg.manifest_url = _manifestUrl.c_str(); } void useDeviceId( bool use=true ) { _cfg.use_device_id = use; } bool validate_sig( const esp_partition_t* partition, unsigned char *signature, uint32_t firmware_size ); @@ -204,10 +204,15 @@ class esp32FOTA private: + HTTPClient _http; + WiFiClientSecure _client; + bool setupHTTP( String url ); + FOTAConfig_t _cfg; SemverClass _payload_sem = SemverClass(0,0,0); + String _manifestUrl; String _firmwareUrl; String _flashFileSystemUrl; From 156b6733a734f43ebe9322ff540aba0bf9c9d125 Mon Sep 17 00:00:00 2001 From: tobozo Date: Mon, 26 Sep 2022 18:11:26 +0200 Subject: [PATCH 02/32] refactoring http --- src/esp32FOTA.cpp | 172 +++++++++++++++++++++------------------------- 1 file changed, 78 insertions(+), 94 deletions(-) diff --git a/src/esp32FOTA.cpp b/src/esp32FOTA.cpp index ce8b9d1..84f81fb 100644 --- a/src/esp32FOTA.cpp +++ b/src/esp32FOTA.cpp @@ -276,6 +276,54 @@ bool esp32FOTA::validate_sig( const esp_partition_t* partition, unsigned char *s } +bool esp32FOTA::setupHTTP( String url ) +{ + const char* rootcastr = nullptr; + _http.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); + + log_i("Connecting to: %s\r\n", url.c_str() ); + if( url.substring( 0, 5 ) == "https" ) { + if (!_cfg.unsafe) { + if( !_cfg.root_ca ) { + Serial.println("A strict security context has been set but no RootCA was provided"); + return false; + } + rootcastr = _cfg.root_ca->get(); + if( _cfg.root_ca->size() == 0 ) { + Serial.println("A strict security context has been set but an empty RootCA was provided"); + Serial.println(rootcastr); + return false; + } + if( !rootcastr ) { + Serial.println("Unable to get RootCA, aborting"); + return false; + } + Serial.println("Loading root_ca.pem"); + _client.setCACert( rootcastr ); + } else { + // We're downloading from a secure URL, but we don't want to validate the root cert. + _client.setInsecure(); + } + _http.begin( _client, url ); + } else { + _http.begin( url ); + } + + if( extraHTTPHeaders.size() > 0 ) { + // add custom headers provided by user e.g. _http.addHeader("Authorization", "Basic " + auth) + for( const auto &header : extraHTTPHeaders ) { + _http.addHeader(header.first, header.second); + } + } + + // TODO: add more watched headers e.g. Authorization: Signature keyId="rsa-key-1",algorithm="rsa-sha256",signature="Base64(RSA-SHA256(signing string))" + const char* get_headers[] = { "Content-Length", "Content-type" }; + _http.collectHeaders( get_headers, 2 ); + + return true; +} + + // OTA Logic bool esp32FOTA::execOTA() { @@ -319,57 +367,23 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) size_t contentLength = 0; bool isValidContentType = false; - const char* rootcastr = nullptr; - - HTTPClient http; - WiFiClientSecure client; - //http.setConnectTimeout( 1000 ); - http.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); - log_i("Connecting to: %s\r\n", UpdateURL.c_str() ); - if( UpdateURL.substring( 0, 5 ) == "https" ) { - if (!_cfg.unsafe) { - if( !_cfg.root_ca ) { - Serial.println("A strict security context has been set for "+PartitionLabel+" partition but no RootCA was provided"); - return false; - } - rootcastr = _cfg.root_ca->get(); - if( _cfg.root_ca->size() == 0 ) { - Serial.println("A strict security context has been set for "+PartitionLabel+" partition but an empty RootCA was provided"); - Serial.println(rootcastr); - return false; - } - if( !rootcastr ) { - Serial.println("Unable to get RootCA for "+PartitionLabel+", aborting"); - return false; - } - Serial.println("Loading root_ca.pem"); - client.setCACert( rootcastr ); - } else { - // We're downloading from a secure URL, but we don't want to validate the root cert. - client.setInsecure(); - } - http.begin( client, UpdateURL ); - } else { - http.begin( UpdateURL ); + if(! setupHTTP( UpdateURL ) ) { + log_e("unable to setup http, aborting!"); + return false; } - if( extraHTTPHeaders.size() > 0 ) { - // add custom headers provided by user e.g. http.addHeader("Authorization", "Basic " + auth) - for( const auto &header : extraHTTPHeaders ) { - http.addHeader(header.first, header.second); - } - } - - // TODO: add more watched headers e.g. Authorization: Signature keyId="rsa-key-1",algorithm="rsa-sha256",signature="Base64(RSA-SHA256(signing string))" - const char* get_headers[] = { "Content-Length", "Content-type" }; - http.collectHeaders( get_headers, 2 ); + // Stream& stream; + // if( ! getHTTPPayload( &stream ) ) { + // log_e("HTTP Error"); + // return false; + // } - int httpCode = http.GET(); + int httpCode = _http.GET(); if( httpCode == HTTP_CODE_OK || httpCode == HTTP_CODE_MOVED_PERMANENTLY ) { - contentLength = http.header( "Content-Length" ).toInt(); - String contentType = http.header( "Content-type" ); + contentLength = _http.header( "Content-Length" ).toInt(); + String contentType = _http.header( "Content-type" ); if( contentType == "application/octet-stream" ) { isValidContentType = true; } else if( contentType == "application/gzip" ) { @@ -405,7 +419,7 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) break; } - http.end(); + _http.end(); return false; } @@ -415,7 +429,7 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) // check contentLength and content type if( !contentLength || !isValidContentType ) { Serial.printf("There was no content in the http response: (length: %i, valid: %s)\n", contentLength, isValidContentType?"true":"false"); - http.end(); + _http.end(); return false; } @@ -430,7 +444,7 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) if( !canBegin ) { Serial.println("Not enough space to begin OTA, partition size mismatch?"); - http.end(); + _http.end(); if( onUpdateBeginFail ) onUpdateBeginFail( partition ); return false; } @@ -444,7 +458,7 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) }); } - Stream& stream = http.getStream(); + Stream& stream = _http.getStream(); unsigned char signature[512]; if( _cfg.check_sig ) { @@ -483,7 +497,7 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) return false; } - http.end(); + _http.end(); if( onUpdateEnd ) onUpdateEnd( partition ); @@ -671,8 +685,6 @@ bool esp32FOTA::execHTTPcheck() useURL = checkURL; } - const char* rootcastr = nullptr; - // being deprecated, soon unsupported! if( useDeviceID ) { Serial.println("useDeviceID will soon be unsupported, use FOTAConfig_t::use_device_id instead!!"); @@ -693,45 +705,19 @@ bool esp32FOTA::execHTTPcheck() log_i("Getting HTTP: %s", useURL.c_str()); log_i("------"); - HTTPClient http; - WiFiClientSecure client; - http.setFollowRedirects( HTTPC_STRICT_FOLLOW_REDIRECTS ); - - if( useURL.substring( 0, 5 ) == "https" ) { - if( _cfg.unsafe ) { - // We're downloading from a secure port, but we don't want to validate the root cert. - client.setInsecure(); - } else { - // We're downloading from a secure port, and want to validate the root cert. - if( !_cfg.root_ca ) { - Serial.println("A strict security context has been set to fetch json manifest, but no RootCA was provided, aborting"); - return false; - } - rootcastr = _cfg.root_ca->get(); - if( _cfg.root_ca->size() == 0 ) { - Serial.println("A strict security context has been set to fetch json manifest, but an empty RootCA was provided, aborting"); - return false; - } - if( !rootcastr ) { - Serial.println("Unable to get RootCA to fetch json manifest, aborting"); - return false; - } - Serial.println("Loading root_ca.pem"); - client.setCACert( rootcastr ); - } - http.begin( client, useURL ); - } else { - http.begin( useURL ); + if(! setupHTTP( useURL ) ) { + log_e("unable to setup http, aborting!"); + return false; } - if( extraHTTPHeaders.size() > 0 ) { - // add custom headers provided by user e.g. http.addHeader("Authorization", "Basic " + auth) - for( const auto &header : extraHTTPHeaders ) { - http.addHeader(header.first, header.second); - } - } - int httpCode = http.GET(); //Make the request + // String payload; + // if( ! getHTTPPayload( &payload ) ) { + // log_e("HTTP Error"); + // return false; + // } + + int httpCode = _http.GET(); //Make the request // only handle 200/301, fail on everything else if( httpCode != HTTP_CODE_OK && httpCode != HTTP_CODE_MOVED_PERMANENTLY ) { @@ -742,20 +728,18 @@ bool esp32FOTA::execHTTPcheck() } else { log_d("Unknown HTTP response"); } - http.end(); + _http.end(); return false; } + String payload = _http.getString(); + _http.end(); // We're done with HTTP - free the resources + // TODO: use payload.length() to speculate on JSONResult buffer size #define JSON_FW_BUFF_SIZE 2048 DynamicJsonDocument JSONResult( JSON_FW_BUFF_SIZE ); - - String payload = http.getString(); - DeserializationError err = deserializeJson( JSONResult, payload.c_str() ); - http.end(); // We're done with HTTP - free the resources - if (err) { // Check for errors in parsing, or JSON length may exceed buffer size Serial.printf("JSON Parsing failed (%s, in=%d bytes, buff=%d bytes):\n%s\n", err.c_str(), payload.length(), JSON_FW_BUFF_SIZE, payload.c_str() ); return false; From f4fcf921fe10e62821d625a19a8da2f7d9777b54 Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 14:33:32 +0200 Subject: [PATCH 03/32] esp32-FlashZ tests --- .github/workflows/gen-test-suite.yml | 17 +- README.md | 5 + examples/anyFS/anyFS.ino | 54 +++- library.json | 2 +- library.properties | 2 +- src/esp32FOTA.cpp | 421 +++++++++++++++++---------- src/esp32FOTA.hpp | 120 ++++++-- 7 files changed, 446 insertions(+), 175 deletions(-) diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index b8ac852..d97318c 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -182,11 +182,18 @@ jobs: steps: - - name: Checkout + - name: Checkout Current uses: actions/checkout@v2 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Checkout optional dependency + uses: actions/checkout@v2 + with: + repository: vortigont/esp32-flashz + ref: master + path: CustomLibrary_flashZ # must contain string "Custom" + - name: Retrieve RootCA/PubKey uses: actions/download-artifact@v3 with: @@ -213,13 +220,13 @@ jobs: with: platform-url: https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_dev_index.json arduino-board-fqbn: ${{inputs.board_fqbn}}:PartitionScheme=${{inputs.partition_scheme}} - required-libraries: ArduinoJson + required-libraries: ArduinoJson,CustomLibrary_flashZ extra-arduino-lib-install-args: --no-deps extra-arduino-cli-args: "--warnings default " # see https://github.com/ArminJo/arduino-test-compile/issues/28 sketch-names: ${{ matrix.sketch }} set-build-path: true - - name: Sign and Save compiled binary + - name: Sign and Save binaries run: | mkdir -p ${{env.artifact_path}} full_ino_bin_path=`find ${{env.work_path}}/ | grep "build/${{ matrix.sketch }}.bin"` @@ -227,6 +234,7 @@ jobs: cat firmware.sign $full_ino_bin_path > $full_ino_bin_path.img cp $full_ino_bin_path ${{env.artifact_path}}/${{ matrix.sketch }}.bin cp $full_ino_bin_path.img ${{env.artifact_path}}/${{ matrix.sketch }}.signed.bin + gzip -c $full_ino_bin_path > ${{env.artifact_path}}/${{ matrix.sketch }}.bin.gz - name: Prepare data folder @@ -252,6 +260,9 @@ jobs: # Create the partition binaries ${{env.mkspiffs_esp32}} -c ${{env.data_path}} -p 256 -b 4096 -s $${{inputs.partition_scheme}}_size ${{env.spiffs_bin_path}} ${{env.mklittlefs_esp32}} -c ${{env.data_path}} -p 256 -b 4096 -s $${{inputs.partition_scheme}}_size ${{env.littlefs_bin_path}} + # Create gzipped versions + gzip -c ${{env.littlefs_bin_path}} > ${{env.littlefs_bin_path}}.gz + gzip -c ${{env.spiffs_bin_path}} > ${{env.spiffs_bin_path}}.gz # Sign partition binaries openssl dgst -sign ${{env.privkey_path}} -keyform PEM -sha256 -out firmware.sign -binary ${{env.spiffs_bin_path}} diff --git a/README.md b/README.md index d529cfa..b4359cb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ [![PlatformIO](https://github.com/chrisjoyce911/esp32FOTA/workflows/PlatformIO/badge.svg)](https://github.com/chrisjoyce911/esp32FOTA/actions/) +[![arduino-library-badge](https://www.ardu-badge.com/badge/esp32FOTA.svg?)](https://www.ardu-badge.com/esp32FOTA) +[![PlatformIO Registry](https://badges.registry.platformio.org/packages/chrisjoyce911/library/esp32FOTA.svg)](https://registry.platformio.org/libraries/chrisjoyce911/esp32FOTA) + + + # esp32FOTA library for Arduino ## Purpose diff --git a/examples/anyFS/anyFS.ino b/examples/anyFS/anyFS.ino index c38fee5..13dd138 100644 --- a/examples/anyFS/anyFS.ino +++ b/examples/anyFS/anyFS.ino @@ -13,9 +13,12 @@ #include //#include +#include // optional esp32-flashz for gzipped firmwares + #include // fota pulls WiFi library + // esp32fota settings const int firmware_version = 1; #if !defined FOTA_URL @@ -36,11 +39,50 @@ CryptoFileAsset *MyRSAKey = new CryptoFileAsset( "/rsa_key.pub", &LittleFS ); // CryptoMemAsset *MyRSAKey = new CryptoMemAsset("RSA Public Key", rsa_key_pub, strlen(rsa_key_pub)+1 ); -esp32FOTA FOTA; -//esp32FOTA esp32FOTA( String(firmware_name), firmware_version, check_signature, disable_security ); +esp32FOTA FOTA; // empty constructor + + +// bool available( stream*) // returns stream->available() or (stream->peek() == ZLIB_HEADER) +// size_t size() // returns size or UPDATE_SIZE_UNKNOWN +// bool canBegin( fwsize, partition ) // should call Update.begin( fwsize, partition ) +// void onBeginFailCb( partition ) // abort ! +// void onProgressCb( size_t progress, size_t total ) // delegate to Update.onProgress +// void writeCb( *stream, size ) // FlashZ::getInstance().writezStream(*stream, contentLength), FlashZ::getInstance().writeStream(*stream), Update, Update.writeStream + + + +int64_t myStreamGetter( esp32FOTA* fota, int partition ) +{ + const char* path = fota->getPath( partition ); + Serial.printf("Opening %s\n", path ); + + // retrieve fota stream pointer + Stream* fotaStream = fota->getFotaStreamPtr(); + //auto myStream = FlashZ::getInstance(); + + // overwrite pointer with custom stream object + //*fotaStream = new blahStream( my args ... ); + + int size = fotaStream->available(); + + if( size <= 0 ) { + fota->setFotaStream( nullptr ); + return -1; + } + + return size; +} +// FOTA.setStreamGetter( myStreamGetter ); + + + -//esp32FOTA esp32FOTA("esp32-fota-http", 1, false ); +bool WiFiConnected() +{ + return (WiFi.status() == WL_CONNECTED); +} + void setup_wifi() { @@ -50,7 +92,7 @@ void setup_wifi() WiFi.begin(); // no WiFi creds in this demo :-) - while (WiFi.status() != WL_CONNECTED) + while ( !WiFiConnected() ) { delay(500); Serial.print("."); @@ -86,6 +128,10 @@ void setup() FOTA.setConfig( cfg ); } + + // FOTA.setStatusChecker( WiFiConnected ); + + // /!\ FOTA.checkURL is deprecated, use setManifestURL( String ) instead //FOTA.setManifestURL( FOTA_URL ); //FOTA.setRootCA( MyRootCA ); diff --git a/library.json b/library.json index a50da1f..04bf8d0 100644 --- a/library.json +++ b/library.json @@ -1,6 +1,6 @@ { "name": "esp32FOTA", - "version": "0.2.2", + "version": "0.2.3", "keywords": "firmware, OTA, Over The Air Updates, ArduinoOTA", "description": "Allows for firmware to be updated from a webserver, the device can check for updates at any time. Uses a simple JSON file to outline if a new firmware is avaiable.", "examples": "examples/*/*.ino", diff --git a/library.properties b/library.properties index 3bb027b..dc1807c 100644 --- a/library.properties +++ b/library.properties @@ -1,5 +1,5 @@ name=esp32FOTA -version=0.2.2 +version=0.2.3 author=Chris Joyce maintainer=Chris Joyce sentence=A simple library for firmware OTA updates diff --git a/src/esp32FOTA.cpp b/src/esp32FOTA.cpp index 84f81fb..f4ea87a 100644 --- a/src/esp32FOTA.cpp +++ b/src/esp32FOTA.cpp @@ -41,6 +41,14 @@ #pragma GCC diagnostic ignored "-Wmissing-field-initializers" #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#define FW_SIGNATURE_LENGTH 512 + + +static int64_t getHTTPStream( esp32FOTA* fota, int partition ); +static int64_t getFileStream( esp32FOTA* fota, int partition ); +static int64_t getSerialStream( esp32FOTA* fota, int partition ); +static bool WiFiStatusCheck(); + SemverClass::SemverClass( const char* version ) { @@ -106,7 +114,9 @@ size_t CryptoFileAsset::size() -esp32FOTA::esp32FOTA() { } +esp32FOTA::esp32FOTA(){} +esp32FOTA::~esp32FOTA(){} + esp32FOTA::esp32FOTA( FOTAConfig_t cfg ) { @@ -114,9 +124,9 @@ esp32FOTA::esp32FOTA( FOTAConfig_t cfg ) } -esp32FOTA::esp32FOTA(String firmwareType, int firmwareVersion, bool validate, bool allow_insecure_https) +esp32FOTA::esp32FOTA(const char* firmwareType, int firmwareVersion, bool validate, bool allow_insecure_https) { - _cfg.name = firmwareType.c_str(); + _cfg.name = firmwareType; _cfg.sem = SemverClass( firmwareVersion ); _cfg.check_sig = validate; _cfg.unsafe = allow_insecure_https; @@ -126,21 +136,18 @@ esp32FOTA::esp32FOTA(String firmwareType, int firmwareVersion, bool validate, bo } -esp32FOTA::esp32FOTA(String firmwareType, String firmwareSemanticVersion, bool validate, bool allow_insecure_https) +esp32FOTA::esp32FOTA(const char* firmwareType, const char* firmwareSemanticVersion, bool validate, bool allow_insecure_https) { - _cfg.name = firmwareType.c_str(); + _cfg.name = firmwareType; _cfg.check_sig = validate; _cfg.unsafe = allow_insecure_https; - _cfg.sem = SemverClass( firmwareSemanticVersion.c_str() ); + _cfg.sem = SemverClass( firmwareSemanticVersion ); setupCryptoAssets(); debugSemVer("Current firmware version", _cfg.sem.ver() ); } -esp32FOTA::~esp32FOTA() -{ -} void esp32FOTA::setCertFileSystem( fs::FS *cert_filesystem ) @@ -258,7 +265,7 @@ bool esp32FOTA::validate_sig( const esp_partition_t* partition, unsigned char *s } mbedtls_md_finish( &rsa, hash ); - ret = mbedtls_pk_verify( &pk, MBEDTLS_MD_SHA256, hash, mdinfo->size, (unsigned char*)signature, 512 ); + ret = mbedtls_pk_verify( &pk, MBEDTLS_MD_SHA256, hash, mdinfo->size, (unsigned char*)signature, FW_SIGNATURE_LENGTH ); free( hash ); mbedtls_md_free( &rsa ); @@ -276,13 +283,13 @@ bool esp32FOTA::validate_sig( const esp_partition_t* partition, unsigned char *s } -bool esp32FOTA::setupHTTP( String url ) +bool esp32FOTA::setupHTTP( const char* url ) { const char* rootcastr = nullptr; _http.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); - log_i("Connecting to: %s\r\n", url.c_str() ); - if( url.substring( 0, 5 ) == "https" ) { + log_i("Connecting to: %s\r\n", url ); + if( String(url).startsWith("https") ) { if (!_cfg.unsafe) { if( !_cfg.root_ca ) { Serial.println("A strict security context has been set but no RootCA was provided"); @@ -324,9 +331,60 @@ bool esp32FOTA::setupHTTP( String url ) } + + +void esp32FOTA::setupStream() +{ + if(!getStream) { + switch( _stream_type ) { + case FOTA_FILE_STREAM: + setStreamGetter( getFileStream ); + break; + case FOTA_SERIAL_STREAM: + setStreamGetter( getSerialStream ); + break; + case FOTA_HTTP_STREAM: + default: + setStreamGetter( getHTTPStream ); + break; + } + } + + if( !isConnected ) { + setStatusChecker( WiFiStatusCheck ); + } +} + + +void esp32FOTA::stopStream() +{ + if( endStream ) { // user function provided via ::setStreamEnder( fn ) + endStream( this ); + return; + } + + // no user function provided, apply default behaviour + switch( _stream_type ) { + case FOTA_FILE_STREAM: + if( _file ) _file.close(); + break; + case FOTA_HTTP_STREAM: + _http.end(); + break; + case FOTA_SERIAL_STREAM: + default: + break; + } + +} + + + // OTA Logic bool esp32FOTA::execOTA() { + setupStream(); + if( _flashFileSystemUrl != "" ) { // a data partition was specified in the json manifest, handle the spiffs partition first if( _fs ) { // Possible risk of overwriting certs and signatures, cancel flashing! Serial.println("Cowardly refusing to overwrite U_SPIFFS with "+_flashFileSystemUrl+". Use setCertFileSystem(nullptr) along with setPubKey()/setCAPem() to enable this feature."); @@ -339,165 +397,98 @@ bool esp32FOTA::execOTA() log_i("This update is for U_FLASH only"); } // handle the application partition and restart on success - return execOTA( U_FLASH, true ); -} - + bool ret = execOTA( U_FLASH, true ); -bool esp32FOTA::execOTA( int partition, bool restart_after ) -{ - String UpdateURL = ""; - String PartitionLabel = ""; - - switch( partition ) { - case U_SPIFFS: // spiffs/littlefs/fatfs partition - PartitionLabel = "data"; - if( _flashFileSystemUrl == "" ) { - log_i("[SKIP] No spiffs/littlefs/fatfs partition was speficied"); - return true; - } - UpdateURL = _flashFileSystemUrl; - break; - case U_FLASH: // app partition (default) - default: - PartitionLabel = "app" + String( partition ); - partition = U_FLASH; - UpdateURL = _firmwareUrl; - break; - } + stopStream(); - size_t contentLength = 0; - bool isValidContentType = false; + return ret; +} - if(! setupHTTP( UpdateURL ) ) { - log_e("unable to setup http, aborting!"); - return false; - } - // Stream& stream; - // if( ! getHTTPPayload( &stream ) ) { - // log_e("HTTP Error"); - // return false; - // } - int httpCode = _http.GET(); - if( httpCode == HTTP_CODE_OK || httpCode == HTTP_CODE_MOVED_PERMANENTLY ) { - contentLength = _http.header( "Content-Length" ).toInt(); - String contentType = _http.header( "Content-type" ); - if( contentType == "application/octet-stream" ) { - isValidContentType = true; - } else if( contentType == "application/gzip" ) { - // was gzipped by the server, needs decompression - // TODO: use gzStreamUpdater - } else if( contentType == "application/tar+gz" ) { - // was packaged and compressed, may contain more than one file - // TODO: use tarGzStreamUpdater - } +bool esp32FOTA::execOTA( int partition, bool restart_after ) +{ + // health checks + if( partition == U_SPIFFS && _flashFileSystemUrl == "" ) { + log_i("[SKIP] No spiffs/littlefs/fatfs partition was specified"); + return true; // data partition is optional, so not an error + } else if ( partition == U_FLASH && _firmwareUrl == "" ) { + log_e("No app partition was specified"); + return false; // app partition is mandatory } else { - switch( httpCode ) { - // 1xx = Hold on - // 2xx = Here you go - // 3xx = Go away - // 4xx = You fucked up - // 5xx = I fucked up + log_e("Bad partition number: %i or empty URL", partition); + return false; + } - case 204: log_e("Status: 204 (No contents), "); break; - case 401: log_e("Status: 401 (Unauthorized), check setExtraHTTPHeader() values"); break; - case 403: log_e("Status: 403 (Forbidden), check path on webserver?"); break; - case 404: log_e("Status: 404 (Not Found), also a palindrom, check path in manifest?"); break; - case 418: log_e("Status: 418 (I'm a teapot), Brit alert!"); break; - case 429: log_e("Status: 429 (Too many requests), throttle things down?"); break; - case 500: log_e("Status: 500 (Internal Server Error), you broke the webs!"); break; - default: - // This error may be a false positive or a consequence of the network being disconnected. - // Since the network is controlled from outside this class, only significant error messages are reported. - if( httpCode > 0 ) { - Serial.printf("Server responded with HTTP Status '%i' when calling url '%s'. Please check your setup\n", httpCode, UpdateURL.c_str() ); - } else { - log_d("Unknown HTTP response"); - } - break; - } + int64_t updateSize = getStream( this, partition ); - _http.end(); + if( updateSize<=0 || _stream == nullptr ) { + log_e("HTTP Error"); return false; } - // TODO: Not all streams respond with a content length. - // TODO: Set contentLength to UPDATE_SIZE_UNKNOWN when content type is valid. + bool mode_z = F_UseZlib; + // If using compression, the size is implicitely unknown + updateSize = mode_z ? UPDATE_SIZE_UNKNOWN : updateSize; - // check contentLength and content type - if( !contentLength || !isValidContentType ) { - Serial.printf("There was no content in the http response: (length: %i, valid: %s)\n", contentLength, isValidContentType?"true":"false"); - _http.end(); + if( _cfg.check_sig ) { + if( mode_z ) { + Serial.println("[ERROR] Compressed && signed image is not (yet) supported"); return false; + } + if( updateSize != UPDATE_SIZE_UNKNOWN ) { + if( updateSize <= FW_SIGNATURE_LENGTH ) { + Serial.println("[ERROR] Malformed signature+fw combo"); + return false; + } + updateSize -= FW_SIGNATURE_LENGTH; + } } - log_d("contentLength : %i, isValidContentType : %s", contentLength, String(isValidContentType)); - - if( _cfg.check_sig && contentLength != UPDATE_SIZE_UNKNOWN ) { - // If firmware is signed, extract signature and decrease content-length by 512 bytes for signature - contentLength -= 512; - } - // Check if there is enough available space on the partition to perform the Update - bool canBegin = Update.begin( contentLength, partition ); + bool canBegin = F_canBegin(); if( !canBegin ) { Serial.println("Not enough space to begin OTA, partition size mismatch?"); - _http.end(); + F_abort(); if( onUpdateBeginFail ) onUpdateBeginFail( partition ); return false; } if( onOTAProgress ) { - Update.onProgress( onOTAProgress ); + F_Update.onProgress( onOTAProgress ); } else { - Update.onProgress( [](size_t progress, size_t size) { + F_Update.onProgress( [](size_t progress, size_t size) { if( progress >= size ) Serial.println(); else if( progress > 0) Serial.print("."); }); } - Stream& stream = _http.getStream(); - - unsigned char signature[512]; + unsigned char signature[FW_SIGNATURE_LENGTH]; if( _cfg.check_sig ) { - stream.readBytes( signature, 512 ); + _stream->readBytes( signature, FW_SIGNATURE_LENGTH ); } + Serial.printf("Begin %s OTA. This may take 2 - 5 mins to complete. Things might be quiet for a while.. Patience!\n", partition==U_FLASH?"Firmware":"Filesystem"); - // Some activity may appear in the Serial monitor during the update (depends on Update.onProgress). - // This may take 2 - 5mins to complete - size_t written = Update.writeStream( stream ); + // Some activity may appear in the Serial monitor during the update (depends on Update.onProgress) + size_t written = F_Update.writeStream( *_stream ); - if ( written == contentLength) { + if ( written == updateSize) { Serial.println("Written : " + String(written) + " successfully"); - } else if ( contentLength == UPDATE_SIZE_UNKNOWN ) { + } else if ( updateSize == UPDATE_SIZE_UNKNOWN ) { Serial.println("Written : " + String(written) + " successfully"); - contentLength = written; // populate value as it was unknown until now + updateSize = written; // populate value as it was unknown until now } else { - Serial.println("Written only : " + String(written) + "/" + String(contentLength) + ". Premature end of stream?"); - contentLength = written; // flatten value to prevent overflow when checking signature - } - - if (!Update.end()) { - Serial.println("An Update Error Occurred. Error #: " + String(Update.getError())); - // #define UPDATE_ERROR_OK (0) - // #define UPDATE_ERROR_WRITE (1) - // #define UPDATE_ERROR_ERASE (2) - // #define UPDATE_ERROR_READ (3) - // #define UPDATE_ERROR_SPACE (4) - // #define UPDATE_ERROR_SIZE (5) - // #define UPDATE_ERROR_STREAM (6) - // #define UPDATE_ERROR_MD5 (7) - // #define UPDATE_ERROR_MAGIC_BYTE (8) - // #define UPDATE_ERROR_ACTIVATE (9) - // #define UPDATE_ERROR_NO_PARTITION (10) - // #define UPDATE_ERROR_BAD_ARGUMENT (11) - // #define UPDATE_ERROR_ABORT (12) + Serial.println("Written only : " + String(written) + "/" + String(updateSize) + ". Premature end of stream?"); + F_abort(); return false; + //updateSize = written; // flatten value to prevent overflow when checking signature } - _http.end(); + if (!F_Update.end()) { + Serial.println("An Update Error Occurred. Error #: " + String(F_Update.getError())); + return false; + } if( onUpdateEnd ) onUpdateEnd( partition ); @@ -527,11 +518,11 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) // by temporarily reassigning the bootable flag to the running-partition instead // of the next-partition. esp_ota_set_boot_partition( running_partition ); - // By doing so the ESP will NOT boot any unvalidated partition should a crash occur - // during signature validation. + // By doing so the ESP will NOT boot any unvalidated partition should a reset occur + // during signature validation (crash, oom, power failure). } - if( !validate_sig( _target_partition, signature, contentLength ) ) { + if( !validate_sig( _target_partition, signature, updateSize ) ) { // erase partition esp_partition_erase_range( _target_partition, _target_partition->address, _target_partition->size ); @@ -552,7 +543,7 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) } } //Serial.println("OTA Update complete!"); - if (Update.isFinished()) { + if (F_Update.isFinished()) { if( onUpdateFinished ) onUpdateFinished( partition, restart_after ); @@ -675,6 +666,8 @@ bool esp32FOTA::checkJSONManifest(JsonVariant doc) } + + bool esp32FOTA::execHTTPcheck() { String useURL = String( _cfg.manifest_url ); @@ -697,26 +690,19 @@ bool esp32FOTA::execHTTPcheck() useURL += argseparator + "id=" + getDeviceID(); } - if ((WiFi.status() != WL_CONNECTED)) { //Check the current connection status - log_i("WiFi not connected - skipping HTTP check"); + if ( isConnected && !isConnected() ) { // Check the current connection status + log_i("Connection check requested but network not ready - skipping"); return false; // WiFi not connected } log_i("Getting HTTP: %s", useURL.c_str()); log_i("------"); - if(! setupHTTP( useURL ) ) { - log_e("unable to setup http, aborting!"); + if(! setupHTTP( useURL.c_str() ) ) { + log_e("Unable to setup http, aborting!"); return false; } - - // String payload; - // if( ! getHTTPPayload( &payload ) ) { - // log_e("HTTP Error"); - // return false; - // } - int httpCode = _http.GET(); //Make the request // only handle 200/301, fail on everything else @@ -830,4 +816,145 @@ void esp32FOTA::debugSemVer( const char* label, semver_t* version ) log_i("%s: %s", label, version_no ); } + + + + + +static int64_t getHTTPStream( esp32FOTA* fota, int partition ) +{ + + const char* url = partition==U_SPIFFS ? fota->getFlashFS_URL() : fota->getFirmwareURL(); + Serial.printf("Opening item %s\n", url ); + + HTTPClient *http = fota->getHTTPCLient(); + + if(! fota->setupHTTP( url ) ) { // certs + log_e("unable to setup http, aborting!"); + return -1; + } + + int64_t updateSize = 0; + bool isValidContentType = false; + int httpCode = http->GET(); + + fota->setFotaStream( nullptr ); + + if( httpCode == HTTP_CODE_OK || httpCode == HTTP_CODE_MOVED_PERMANENTLY ) { + updateSize = http->header( "Content-Length" ).toInt(); + String contentType = http->header( "Content-type" ); + if( contentType == "application/octet-stream" ) { + isValidContentType = true; + } else if( contentType == "application/gzip" ) { + // was gzipped by the server, needs decompression + // TODO: use gzStreamUpdater + } else if( contentType == "application/tar+gz" || contentType == "application/x-gtar" ) { + // was packaged and compressed, may contain more than one file + // TODO: use tarGzStreamUpdater + } + } else { + switch( httpCode ) { + // 1xx = Hold on + // 2xx = Here you go + // 3xx = Go away + // 4xx = You fucked up + // 5xx = I fucked up + + case 204: log_e("Status: 204 (No contents), "); break; + case 401: log_e("Status: 401 (Unauthorized), check setExtraHTTPHeader() values"); break; + case 403: log_e("Status: 403 (Forbidden), check path on webserver?"); break; + case 404: log_e("Status: 404 (Not Found), also a palindrom, check path in manifest?"); break; + case 418: log_e("Status: 418 (I'm a teapot), Brit alert!"); break; + case 429: log_e("Status: 429 (Too many requests), throttle things down?"); break; + case 500: log_e("Status: 500 (Internal Server Error), you broke the webs!"); break; + default: + // This error may be a false positive or a consequence of the network being disconnected. + // Since the network is controlled from outside this class, only significant error messages are reported. + if( httpCode > 0 ) { + Serial.printf("Server responded with HTTP Status '%i'. Please check your setup\n", httpCode ); + } else { + log_d("Unknown HTTP response"); + } + break; + } + + return -1; + } + + // TODO: Not all streams respond with a content length. + // TODO: Set updateSize to UPDATE_SIZE_UNKNOWN when content type is valid. + + // check updateSize and content type + if( !updateSize || !isValidContentType ) { + Serial.printf("There was no content in the http response: (length: %lli, valid: %s)\n", updateSize, isValidContentType?"true":"false"); + return -1; + } + + log_d("updateSize : %i, isValidContentType : %s", updateSize, String(isValidContentType)); + + fota->setFotaStream( http->getStreamPtr() ); + + return updateSize; +} + + + +static int64_t getFileStream( esp32FOTA* fota, int partition) +{ + fs::FS* fs = fota->getFotaFS(); + + if(!fs ) { + Serial.println("[ERROR] No filesystem defined, use ::setCertFileSystem( &SD ) "); + return -1; + } + + const char* path = partition==U_SPIFFS ? fota->getFlashFS_URL() : fota->getFirmwareURL(); + Serial.printf("Opening item %s\n", path ); + + fs::File* file = (fs::File*)fota->getFotaStreamPtr(); + *file = fs->open( path ); + + if(! file ) { + log_e("unable to access filesystem, aborting!"); + return -1; + } + + int64_t updateSize = file->size(); + + // check updateSize and content type + if( !updateSize ) { + Serial.println("[ERROR] Empty file"); + file->close(); + fota->setFotaStream( nullptr ); + return -1; + } + + log_d("updateSize : %i", updateSize); + + return updateSize; +} + + + +static int64_t getSerialStream( esp32FOTA* fota, int partition) +{ + return -1; +} + + + +static bool WiFiStatusCheck() +{ + return (WiFi.status() == WL_CONNECTED); +} + +/* +static bool EthernetStatusCheck() +{ + return eth_connected; +} +*/ + + + #pragma GCC diagnostic pop diff --git a/src/esp32FOTA.hpp b/src/esp32FOTA.hpp index 18869cb..6072a6d 100644 --- a/src/esp32FOTA.hpp +++ b/src/esp32FOTA.hpp @@ -40,7 +40,6 @@ extern "C" { #include #include #include -#include #include #include @@ -73,6 +72,23 @@ extern "C" { #endif +#if __has_include() + #include + #define F_Update FlashZ::getInstance() + #define F_UseZlib (_stream->peek() == ZLIB_HEADER) + #define F_canBegin() mode_z ? F_Update.beginz(updateSize, partition) : F_Update.begin(updateSize, partition) + #define F_abort() if (mode_z) F_Update.abortz() + #define F_writeStream() mode_z ? F_Update.writezStream(*stream, contentLength) : F_Update.writeStream(*stream) +#else + #include + #define F_Update Update + #define F_UseZlib false + #define F_canBegin() F_Update.begin( updateSize, partition ) + #define F_abort() { } + #define F_writeStream() F_Update.writeStream( *_stream ); +#endif + + struct SemverClass { @@ -140,36 +156,62 @@ struct FOTAConfig_t }; +enum FOTAStreamType_t +{ + FOTA_HTTP_STREAM, + FOTA_FILE_STREAM, + FOTA_SERIAL_STREAM +}; + // Main Class -class esp32FOTA +class esp32FOTA : public UpdateClass { public: esp32FOTA(); - esp32FOTA( FOTAConfig_t cfg ); - esp32FOTA(String firwmareType, int firwmareVersion, bool validate = false, bool allow_insecure_https = false ); - esp32FOTA(String firwmareType, String firmwareSemanticVersion, bool validate = false, bool allow_insecure_https = false ); ~esp32FOTA(); + esp32FOTA( FOTAConfig_t cfg ); + esp32FOTA(const char* firwmareType, int firwmareVersion, bool validate = false, bool allow_insecure_https = false ); + esp32FOTA(const String &firwmareType, int firwmareVersion, bool validate = false, bool allow_insecure_https = false ) + : esp32FOTA(firwmareType.c_str(), firwmareVersion, validate, allow_insecure_https){}; + esp32FOTA(const char* firwmareType, const char* firmwareSemanticVersion, bool validate = false, bool allow_insecure_https = false ); + esp32FOTA(const String &firwmareType, const String &firmwareSemanticVersion, bool validate = false, bool allow_insecure_https = false ) + : esp32FOTA(firwmareType.c_str(), firmwareSemanticVersion.c_str(), validate, allow_insecure_https){}; + + + template void setPubKey( T* asset ) { _cfg.pub_key = (CryptoAsset*)asset; _cfg.check_sig = true; } template void setRootCA( T* asset ) { _cfg.root_ca = (CryptoAsset*)asset; _cfg.unsafe = false; } + //template static U& getInstance() { static U updater; return updater; } + void forceUpdate(String firmwareHost, uint16_t firmwarePort, String firmwarePath, bool validate ); void forceUpdate(String firmwareURL, bool validate ); void forceUpdate(bool validate ); + bool execOTA(); bool execOTA( int partition, bool restart_after = true ); bool execHTTPcheck(); - int getPayloadVersion(); - void getPayloadVersion(char * version_string); - void setManifestURL( String manifest_url ) { _manifestUrl = manifest_url; _cfg.manifest_url = _manifestUrl.c_str(); } void useDeviceId( bool use=true ) { _cfg.use_device_id = use; } - bool validate_sig( const esp_partition_t* partition, unsigned char *signature, uint32_t firmware_size ); + + // config setter + void setConfig( FOTAConfig_t cfg ) { _cfg = cfg; } + + // Manually specify the manifest url, this is provided as a transition between legagy and new config system + void setManifestURL( const String &manifest_url ) { _cfg.manifest_url = manifest_url.c_str(); } + + // use this to set "Authorization: Basic" or other specific headers to be sent with the queries + void setExtraHTTPHeader( String name, String value ) { extraHTTPHeaders[name] = value; } + + // /!\ Only use this to change filesystem for **default** RootCA and PubKey paths. + // Otherwise use setPubKey() and setRootCA() + void setCertFileSystem( fs::FS *cert_filesystem = nullptr ); // this is passed to Update.onProgress() - typedef std::function ProgressCallback_cb; // size_t progress, size_t size + typedef std::function ProgressCallback_cb; // size_t progress, size_t size void setProgressCb(ProgressCallback_cb fn) { onOTAProgress = fn; } // callback setter // when Update.begin() returned false @@ -188,25 +230,60 @@ class esp32FOTA typedef std::function UpdateFinished_cb; // int partition (U_FLASH or U_SPIFFS), bool restart_after void setUpdateFinishedCb(UpdateFinished_cb fn) { onUpdateFinished = fn; } // callback setter - // use this to set "Authorization: Basic" or other specific headers to be sent with the queries - void setExtraHTTPHeader( String name, String value ) { extraHTTPHeaders[name] = value; } + // stream getter + typedef std::function getStream_cb; // esp32FOTA* this, int partition (U_FLASH or U_SPIFFS), returns stream size + void setStreamGetter( getStream_cb fn ) { getStream = fn; } // callback setter - // /!\ Only use this to change filesystem for **default** RootCA and PubKey paths. - // Otherwise use setPubKey() and setRootCA() - void setCertFileSystem( fs::FS *cert_filesystem = nullptr ); + // stream ender + typedef std::function endStream_cb; // esp32FOTA* this + void setStreamEnder( endStream_cb fn ) { endStream = fn; } // callback setter + + // connection check + typedef std::function isConnected_cb; // + void setStatusChecker( isConnected_cb fn ) { isConnected = fn; } // callback setter + + // updating from a File or from Serial? + void setStreamType( FOTAStreamType_t stream_type ) { _stream_type = stream_type; } + + const char* getManifestURL() { return _manifestUrl.c_str(); } + const char* getFirmwareURL() { return _firmwareUrl.c_str(); } + const char* getFlashFS_URL() { return _flashFileSystemUrl.c_str(); } + const char* getPath(int part) { return part==U_SPIFFS ? getFlashFS_URL() : getFirmwareURL(); } + + int getPayloadVersion(); + void getPayloadVersion(char * version_string); + + FOTAConfig_t getConfig() { return _cfg; }; + FOTAStreamType_t getStreamType() { return _stream_type; } + HTTPClient* getHTTPCLient() { return &_http; } + WiFiClientSecure* getWiFiClient() { return &_client; } + fs::File* getFotaFilePtr() { return &_file; } + Stream* getFotaStreamPtr() { return _stream; } + fs::FS* getFotaFS() { return _fs; } + + // internals but need to be exposed to the callbacks + bool setupHTTP( const char* url ); + void setFotaStream( Stream* stream ) { _stream = stream; } + + + //friend class FlashZ; - // config getters and setters - FOTAConfig_t getConfig() { return _cfg; }; - void setConfig( FOTAConfig_t cfg ) { _cfg = cfg; } [[deprecated("Use setManifestURL( String ) or cfg.manifest_url with setConfig( FOTAConfig_t )")]] String checkURL = ""; [[deprecated("Use cfg.use_device_id with setConfig( FOTAConfig_t )")]] bool useDeviceID = false; + private: HTTPClient _http; WiFiClientSecure _client; - bool setupHTTP( String url ); + Stream *_stream; + fs::File _file; + + FOTAStreamType_t _stream_type = FOTA_HTTP_STREAM; // defaults to HTTP + + void setupStream(); + void stopStream(); FOTAConfig_t _cfg; @@ -224,6 +301,9 @@ class esp32FOTA UpdateEnd_cb onUpdateEnd; // after Update.end() and before validate_sig() UpdateCheckFail_cb onUpdateCheckFail; // validate_sig() error handling, mixed situations UpdateFinished_cb onUpdateFinished; // update successful + getStream_cb getStream; // optional stream getter, defaults to http.getStreamPtr() + endStream_cb endStream; // optional stream closer, defaults to http.end() + isConnected_cb isConnected; // optional connection checker, defaults to WiFi.status()==WL_CONNECTED std::map extraHTTPHeaders; // this holds the extra http headers defined by the user @@ -232,6 +312,8 @@ class esp32FOTA void debugSemVer( const char* label, semver_t* version ); void getPartition( int update_partition ); + bool validate_sig( const esp_partition_t* partition, unsigned char *signature, uint32_t firmware_size ); + // temporary partition holder for signature check operations const esp_partition_t* _target_partition = nullptr; From 9dd7d876bf8fbb81044ce9a032802a71f1ed21d9 Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 14:49:12 +0200 Subject: [PATCH 04/32] CI Improvements --- .github/templates/firmware.test-suite.json | 9 +- .github/workflows/gen-test-suite.yml | 3 +- .gitignore | 1 + examples/anyFS/anyFS.ino | 36 ------ .../1.1.nosecurity.ino} | 2 +- .../1.2.nosecurity.gz/1.2.nosecurity.gz.ino | 103 ++++++++++++++++++ src/esp32FOTA.cpp | 2 +- 7 files changed, 116 insertions(+), 40 deletions(-) rename examples/anyFS/test/{1.nosecurity/1.nosecurity.ino => 1.1.nosecurity/1.1.nosecurity.ino} (98%) create mode 100644 examples/anyFS/test/1.2.nosecurity.gz/1.2.nosecurity.gz.ino diff --git a/.github/templates/firmware.test-suite.json b/.github/templates/firmware.test-suite.json index 35554c9..18bb68f 100644 --- a/.github/templates/firmware.test-suite.json +++ b/.github/templates/firmware.test-suite.json @@ -1,10 +1,17 @@ [{ "type": "FIRMWARE_TYPE", - "version": 1, + "version": "1.1", "host": "FIRMWARE_HOST", "port": FIRMWARE_PORT, "bin": "FIRMWARE_PATH/1.nosecurity.ino.bin?raw=true" }, +{ + "type": "FIRMWARE_TYPE", + "version": "1.2", + "host": "FIRMWARE_HOST", + "port": FIRMWARE_PORT, + "bin": "FIRMWARE_PATH/1.nosecurity.ino.bin.gz?raw=true" +}, { "type": "FIRMWARE_TYPE", "version": 2, diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index d97318c..3795468 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -172,7 +172,8 @@ jobs: matrix: sketch: - - 1.nosecurity.ino + - 1.1.nosecurity + - 1.2.nosecurity.gz - 2.cert-in-spiffs.ino - 3.cert-in-progmem.ino - 4.cert-in-littlefs.ino diff --git a/.gitignore b/.gitignore index 2cec19e..151434f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ _Notes .pio test/stage1 +.directory diff --git a/examples/anyFS/anyFS.ino b/examples/anyFS/anyFS.ino index 13dd138..7f7c176 100644 --- a/examples/anyFS/anyFS.ino +++ b/examples/anyFS/anyFS.ino @@ -42,42 +42,6 @@ CryptoFileAsset *MyRSAKey = new CryptoFileAsset( "/rsa_key.pub", &LittleFS ); esp32FOTA FOTA; // empty constructor -// bool available( stream*) // returns stream->available() or (stream->peek() == ZLIB_HEADER) -// size_t size() // returns size or UPDATE_SIZE_UNKNOWN -// bool canBegin( fwsize, partition ) // should call Update.begin( fwsize, partition ) -// void onBeginFailCb( partition ) // abort ! -// void onProgressCb( size_t progress, size_t total ) // delegate to Update.onProgress -// void writeCb( *stream, size ) // FlashZ::getInstance().writezStream(*stream, contentLength), FlashZ::getInstance().writeStream(*stream), Update, Update.writeStream - - - -int64_t myStreamGetter( esp32FOTA* fota, int partition ) -{ - const char* path = fota->getPath( partition ); - Serial.printf("Opening %s\n", path ); - - // retrieve fota stream pointer - Stream* fotaStream = fota->getFotaStreamPtr(); - //auto myStream = FlashZ::getInstance(); - - // overwrite pointer with custom stream object - //*fotaStream = new blahStream( my args ... ); - - int size = fotaStream->available(); - - if( size <= 0 ) { - fota->setFotaStream( nullptr ); - return -1; - } - - return size; -} -// FOTA.setStreamGetter( myStreamGetter ); - - - - - bool WiFiConnected() { return (WiFi.status() == WL_CONNECTED); diff --git a/examples/anyFS/test/1.nosecurity/1.nosecurity.ino b/examples/anyFS/test/1.1.nosecurity/1.1.nosecurity.ino similarity index 98% rename from examples/anyFS/test/1.nosecurity/1.nosecurity.ino rename to examples/anyFS/test/1.1.nosecurity/1.1.nosecurity.ino index 2a302dc..a358e52 100644 --- a/examples/anyFS/test/1.nosecurity/1.nosecurity.ino +++ b/examples/anyFS/test/1.1.nosecurity/1.1.nosecurity.ino @@ -16,7 +16,7 @@ // esp32fota settings int firmware_version_major = 1; -int firmware_version_minor = 0; +int firmware_version_minor = 1; int firmware_version_patch = 0; #if !defined FOTA_URL diff --git a/examples/anyFS/test/1.2.nosecurity.gz/1.2.nosecurity.gz.ino b/examples/anyFS/test/1.2.nosecurity.gz/1.2.nosecurity.gz.ino new file mode 100644 index 0000000..89fa69b --- /dev/null +++ b/examples/anyFS/test/1.2.nosecurity.gz/1.2.nosecurity.gz.ino @@ -0,0 +1,103 @@ +/** + esp32 firmware OTA + + Purpose: Perform an OTA update to both firmware and filesystem from binaries located + on a webserver (HTTPS) without checking for certificate validity + + Usage: If the ESP32 had a previous successful WiFi connection, then no need to set the ssid/password + to run this sketch, the credentials are still cached :-) + Sketch 1 will FOTA to Sketch 2, then Sketch 3, and so on until all versions in firmware.json are + exhausted. + + +*/ + +#include // optional esp32-flashz for gzipped firmwares +#include + +// esp32fota settings +int firmware_version_major = 1; +int firmware_version_minor = 2; +int firmware_version_patch = 0; + +#if !defined FOTA_URL + #define FOTA_URL "http://server/fota/fota.json" +#endif +const char* firmware_name = "esp32-fota-http"; +const bool check_signature = false; +const bool disable_security = true; +// for debug only +const char* description = "Basic *gzipped* example with no security and no filesystem"; + +const char* fota_debug_fmt = R"DBG_FMT( + +***************** STAGE %i ***************** + + Description : %s + Firmware type : %s + Firmware version : %i + Signature check : %s + TLS Cert check : %s + +******************************************** + +)DBG_FMT"; + + +// esp32fota esp32fota("", , , ); +// esp32FOTA esp32FOTA( String(firmware_name), firmware_version, check_signature, disable_security ); + + +esp32FOTA FOTA; + + +void setup_wifi() +{ + delay(10); + + Serial.print("MAC Address "); + Serial.println( WiFi.macAddress() ); + + WiFi.begin(); // no WiFi creds in this demo :-) + + while (WiFi.status() != WL_CONNECTED) + { + delay(500); + Serial.print("."); + } + + Serial.println(""); + Serial.println(WiFi.localIP()); +} + + +void setup() +{ + Serial.begin(115200); + Serial.printf( fota_debug_fmt, firmware_version_major, description, firmware_name, firmware_version_major, check_signature?"Enabled":"Disabled", disable_security?"Disabled":"Enabled" ); + + { + auto cfg = FOTA.getConfig(); + cfg.name = firmware_name; + cfg.manifest_url = FOTA_URL; + cfg.sem = SemverClass( firmware_version_major, firmware_version_minor, firmware_version_patch ); + cfg.check_sig = check_signature; + cfg.unsafe = disable_security; + //cfg.root_ca = MyRootCA; + //cfg.pub_key = MyRSAKey; + FOTA.setConfig( cfg ); + } + + setup_wifi(); +} + +void loop() +{ + bool updatedNeeded = FOTA.execHTTPcheck(); + if (updatedNeeded) { + FOTA.execOTA(); + } + + delay(20000); +} + diff --git a/src/esp32FOTA.cpp b/src/esp32FOTA.cpp index f4ea87a..8763d06 100644 --- a/src/esp32FOTA.cpp +++ b/src/esp32FOTA.cpp @@ -847,7 +847,7 @@ static int64_t getHTTPStream( esp32FOTA* fota, int partition ) isValidContentType = true; } else if( contentType == "application/gzip" ) { // was gzipped by the server, needs decompression - // TODO: use gzStreamUpdater + isValidContentType = F_UseZlib; } else if( contentType == "application/tar+gz" || contentType == "application/x-gtar" ) { // was packaged and compressed, may contain more than one file // TODO: use tarGzStreamUpdater From 8eba67700b4397bbfe33ac6ddbbdb8a5aeca5615 Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 14:56:13 +0200 Subject: [PATCH 05/32] CI Improvements --- .github/workflows/gen-test-suite.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index 3795468..987bb1d 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -188,13 +188,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - - name: Checkout optional dependency - uses: actions/checkout@v2 - with: - repository: vortigont/esp32-flashz - ref: master - path: CustomLibrary_flashZ # must contain string "Custom" - - name: Retrieve RootCA/PubKey uses: actions/download-artifact@v3 with: @@ -216,6 +209,13 @@ jobs: cp ${{env.cert_path}}/root_ca.h $root_ca_c_path cp ${{env.cert_path}}/pub_key.h $pub_key_c_path + - name: Checkout esp32-flashz + uses: actions/checkout@v2 + with: + repository: vortigont/esp32-flashz + ref: main + path: CustomLibrary_flashZ # must contain string "Custom" + - name: Compile ${{ matrix.sketch }} uses: ArminJo/arduino-test-compile@v3 with: From b66620cf36e0ea92ab9d7425315ebe7146fe7cf3 Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 15:01:05 +0200 Subject: [PATCH 06/32] CI Improvements --- .github/workflows/gen-test-suite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index 987bb1d..1b09112 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -172,8 +172,8 @@ jobs: matrix: sketch: - - 1.1.nosecurity - - 1.2.nosecurity.gz + - 1.1.nosecurity.ino + - 1.2.nosecurity.gz.ino - 2.cert-in-spiffs.ino - 3.cert-in-progmem.ino - 4.cert-in-littlefs.ino From 8ad1ab084303eb5032a8d0da25c823422c2548aa Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 15:11:39 +0200 Subject: [PATCH 07/32] CI Improvements --- .github/workflows/gen-test-suite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index 1b09112..e45dd5d 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -214,14 +214,14 @@ jobs: with: repository: vortigont/esp32-flashz ref: main - path: CustomLibrary_flashZ # must contain string "Custom" + path: CustomLibrary # must contain string "Custom" - name: Compile ${{ matrix.sketch }} uses: ArminJo/arduino-test-compile@v3 with: platform-url: https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_dev_index.json arduino-board-fqbn: ${{inputs.board_fqbn}}:PartitionScheme=${{inputs.partition_scheme}} - required-libraries: ArduinoJson,CustomLibrary_flashZ + required-libraries: ArduinoJson,flashZ extra-arduino-lib-install-args: --no-deps extra-arduino-cli-args: "--warnings default " # see https://github.com/ArminJo/arduino-test-compile/issues/28 sketch-names: ${{ matrix.sketch }} From a128d9b8e367def707ed56c7bb2bf9b763363cf3 Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 15:12:05 +0200 Subject: [PATCH 08/32] CI Improvements --- .github/workflows/gen-test-suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index e45dd5d..867c139 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -214,7 +214,7 @@ jobs: with: repository: vortigont/esp32-flashz ref: main - path: CustomLibrary # must contain string "Custom" + path: CustomLibrary_flashZ # must contain string "Custom" - name: Compile ${{ matrix.sketch }} uses: ArminJo/arduino-test-compile@v3 From 403ceaa790092b01afcdae20ab13bf9777fe7cf2 Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 15:14:57 +0200 Subject: [PATCH 09/32] CI Improvements --- .github/workflows/gen-test-suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index 867c139..d178598 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -214,7 +214,7 @@ jobs: with: repository: vortigont/esp32-flashz ref: main - path: CustomLibrary_flashZ # must contain string "Custom" + path: CustomflashZ # must contain string "Custom" - name: Compile ${{ matrix.sketch }} uses: ArminJo/arduino-test-compile@v3 From f9804d2b1b987de79c87a37d782a9ef1af91e616 Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 15:18:48 +0200 Subject: [PATCH 10/32] CI Improvements --- .github/workflows/gen-test-suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index d178598..9330016 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -221,7 +221,7 @@ jobs: with: platform-url: https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_dev_index.json arduino-board-fqbn: ${{inputs.board_fqbn}}:PartitionScheme=${{inputs.partition_scheme}} - required-libraries: ArduinoJson,flashZ + required-libraries: ArduinoJson extra-arduino-lib-install-args: --no-deps extra-arduino-cli-args: "--warnings default " # see https://github.com/ArminJo/arduino-test-compile/issues/28 sketch-names: ${{ matrix.sketch }} From f0311416be375871c2b9039397a4c44cba222dc2 Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 15:27:53 +0200 Subject: [PATCH 11/32] CI Improvements --- src/esp32FOTA.cpp | 4 ++-- src/esp32FOTA.hpp | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/esp32FOTA.cpp b/src/esp32FOTA.cpp index 8763d06..0270cb5 100644 --- a/src/esp32FOTA.cpp +++ b/src/esp32FOTA.cpp @@ -428,7 +428,7 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) return false; } - bool mode_z = F_UseZlib; + bool mode_z = F_isZlibStream(); // If using compression, the size is implicitely unknown updateSize = mode_z ? UPDATE_SIZE_UNKNOWN : updateSize; @@ -847,7 +847,7 @@ static int64_t getHTTPStream( esp32FOTA* fota, int partition ) isValidContentType = true; } else if( contentType == "application/gzip" ) { // was gzipped by the server, needs decompression - isValidContentType = F_UseZlib; + isValidContentType = F_hasZlib(); } else if( contentType == "application/tar+gz" || contentType == "application/x-gtar" ) { // was packaged and compressed, may contain more than one file // TODO: use tarGzStreamUpdater diff --git a/src/esp32FOTA.hpp b/src/esp32FOTA.hpp index 6072a6d..a89a095 100644 --- a/src/esp32FOTA.hpp +++ b/src/esp32FOTA.hpp @@ -75,14 +75,16 @@ extern "C" { #if __has_include() #include #define F_Update FlashZ::getInstance() - #define F_UseZlib (_stream->peek() == ZLIB_HEADER) + #define F_hasZlib() true + #define F_isZlibStream() (_stream->peek() == ZLIB_HEADER) #define F_canBegin() mode_z ? F_Update.beginz(updateSize, partition) : F_Update.begin(updateSize, partition) #define F_abort() if (mode_z) F_Update.abortz() #define F_writeStream() mode_z ? F_Update.writezStream(*stream, contentLength) : F_Update.writeStream(*stream) #else #include #define F_Update Update - #define F_UseZlib false + #define F_hasZlib() false + #define F_isZlibStream() false #define F_canBegin() F_Update.begin( updateSize, partition ) #define F_abort() { } #define F_writeStream() F_Update.writeStream( *_stream ); From 44bb942ad5ba4b9c9af84e279d06dbd85918073f Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 15:48:12 +0200 Subject: [PATCH 12/32] Fix crash --- src/esp32FOTA.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/esp32FOTA.cpp b/src/esp32FOTA.cpp index 0270cb5..eef9e55 100644 --- a/src/esp32FOTA.cpp +++ b/src/esp32FOTA.cpp @@ -416,7 +416,7 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) } else if ( partition == U_FLASH && _firmwareUrl == "" ) { log_e("No app partition was specified"); return false; // app partition is mandatory - } else { + } else if( partition != U_SPIFFS && partition != U_FLASH ) { log_e("Bad partition number: %i or empty URL", partition); return false; } @@ -825,6 +825,7 @@ static int64_t getHTTPStream( esp32FOTA* fota, int partition ) { const char* url = partition==U_SPIFFS ? fota->getFlashFS_URL() : fota->getFirmwareURL(); + Serial.printf("Opening item %s\n", url ); HTTPClient *http = fota->getHTTPCLient(); @@ -886,11 +887,11 @@ static int64_t getHTTPStream( esp32FOTA* fota, int partition ) // check updateSize and content type if( !updateSize || !isValidContentType ) { - Serial.printf("There was no content in the http response: (length: %lli, valid: %s)\n", updateSize, isValidContentType?"true":"false"); + Serial.printf("There was no content in the http response: (length: %"PRId64", valid: %s)\n", updateSize, isValidContentType?"true":"false"); return -1; } - log_d("updateSize : %i, isValidContentType : %s", updateSize, String(isValidContentType)); + log_d("updateSize : %"PRId64", isValidContentType : %s", updateSize, String(isValidContentType)); fota->setFotaStream( http->getStreamPtr() ); From 120e62234f866e1f42088278d9413272d9ef347c Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 15:54:01 +0200 Subject: [PATCH 13/32] CI Improvements --- .github/templates/firmware.test-suite.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/templates/firmware.test-suite.json b/.github/templates/firmware.test-suite.json index 18bb68f..11ded01 100644 --- a/.github/templates/firmware.test-suite.json +++ b/.github/templates/firmware.test-suite.json @@ -3,14 +3,14 @@ "version": "1.1", "host": "FIRMWARE_HOST", "port": FIRMWARE_PORT, - "bin": "FIRMWARE_PATH/1.nosecurity.ino.bin?raw=true" + "bin": "FIRMWARE_PATH/1.1.nosecurity.ino.bin?raw=true" }, { "type": "FIRMWARE_TYPE", "version": "1.2", "host": "FIRMWARE_HOST", "port": FIRMWARE_PORT, - "bin": "FIRMWARE_PATH/1.nosecurity.ino.bin.gz?raw=true" + "bin": "FIRMWARE_PATH/1.2.nosecurity.ino.bin.gz?raw=true" }, { "type": "FIRMWARE_TYPE", From 95e489a6bd3f08da9dfcd18e2da2181227a77593 Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 17:06:39 +0200 Subject: [PATCH 14/32] CI Improvements --- .github/workflows/gen-test-suite.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index 9330016..87b9174 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -236,6 +236,7 @@ jobs: cp $full_ino_bin_path ${{env.artifact_path}}/${{ matrix.sketch }}.bin cp $full_ino_bin_path.img ${{env.artifact_path}}/${{ matrix.sketch }}.signed.bin gzip -c $full_ino_bin_path > ${{env.artifact_path}}/${{ matrix.sketch }}.bin.gz + pigz -9kzc $full_ino_bin_path > ${{env.artifact_path}}/${{ matrix.sketch }}.bin.z - name: Prepare data folder @@ -264,6 +265,9 @@ jobs: # Create gzipped versions gzip -c ${{env.littlefs_bin_path}} > ${{env.littlefs_bin_path}}.gz gzip -c ${{env.spiffs_bin_path}} > ${{env.spiffs_bin_path}}.gz + # Create flashZ versions + pigz -9kzc ${{env.littlefs_bin_path}} > ${{env.littlefs_bin_path}}.z + pigz -9kzc ${{env.spiffs_bin_path}} > ${{env.spiffs_bin_path}}.z # Sign partition binaries openssl dgst -sign ${{env.privkey_path}} -keyform PEM -sha256 -out firmware.sign -binary ${{env.spiffs_bin_path}} From 3f0cf5b015174ee33e6beeedd539f2b509da0dbe Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 17:38:26 +0200 Subject: [PATCH 15/32] Working flashZ POC --- src/esp32FOTA.cpp | 39 ++++++++++++++++++++++----------------- src/esp32FOTA.hpp | 15 +++++++++++---- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/esp32FOTA.cpp b/src/esp32FOTA.cpp index eef9e55..dd04688 100644 --- a/src/esp32FOTA.cpp +++ b/src/esp32FOTA.cpp @@ -428,9 +428,12 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) return false; } - bool mode_z = F_isZlibStream(); + mode_z = F_isZlibStream(); + log_d("compression: %s", mode_z ? "enabled" : "disabled" ); // If using compression, the size is implicitely unknown - updateSize = mode_z ? UPDATE_SIZE_UNKNOWN : updateSize; + size_t fwsize = mode_z ? UPDATE_SIZE_UNKNOWN : updateSize; // fw_size is unknown if we have a compressed image + + //updateSize = mode_z ? UPDATE_SIZE_UNKNOWN : updateSize; if( _cfg.check_sig ) { if( mode_z ) { @@ -471,13 +474,13 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) Serial.printf("Begin %s OTA. This may take 2 - 5 mins to complete. Things might be quiet for a while.. Patience!\n", partition==U_FLASH?"Firmware":"Filesystem"); // Some activity may appear in the Serial monitor during the update (depends on Update.onProgress) - size_t written = F_Update.writeStream( *_stream ); + size_t written = F_writeStream(); - if ( written == updateSize) { - Serial.println("Written : " + String(written) + " successfully"); - } else if ( updateSize == UPDATE_SIZE_UNKNOWN ) { + if (fwsize == UPDATE_SIZE_UNKNOWN) // match compressed fw size to responce length + fwsize = updateSize; + + if ( written == fwsize ) { Serial.println("Written : " + String(written) + " successfully"); - updateSize = written; // populate value as it was unknown until now } else { Serial.println("Written only : " + String(written) + "/" + String(updateSize) + ". Premature end of stream?"); F_abort(); @@ -485,7 +488,7 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) //updateSize = written; // flatten value to prevent overflow when checking signature } - if (!F_Update.end()) { + if (!F_UpdateEnd()) { Serial.println("An Update Error Occurred. Error #: " + String(F_Update.getError())); return false; } @@ -828,7 +831,7 @@ static int64_t getHTTPStream( esp32FOTA* fota, int partition ) Serial.printf("Opening item %s\n", url ); - HTTPClient *http = fota->getHTTPCLient(); + //HTTPClient *http = fota->getHTTPCLient(); if(! fota->setupHTTP( url ) ) { // certs log_e("unable to setup http, aborting!"); @@ -837,18 +840,20 @@ static int64_t getHTTPStream( esp32FOTA* fota, int partition ) int64_t updateSize = 0; bool isValidContentType = false; - int httpCode = http->GET(); + int httpCode = fota->getHTTPCLient()->GET(); + String contentType; fota->setFotaStream( nullptr ); if( httpCode == HTTP_CODE_OK || httpCode == HTTP_CODE_MOVED_PERMANENTLY ) { - updateSize = http->header( "Content-Length" ).toInt(); - String contentType = http->header( "Content-type" ); + //updateSize = fota->getHTTPCLient()->header( "Content-Length" ).toInt(); + updateSize = fota->getHTTPCLient()->getSize(); + contentType = fota->getHTTPCLient()->header( "Content-type" ); if( contentType == "application/octet-stream" ) { isValidContentType = true; - } else if( contentType == "application/gzip" ) { + } else if( contentType == "application/gzip" || contentType == "application/x-gzip" || contentType == "application/x-compress" ) { // was gzipped by the server, needs decompression - isValidContentType = F_hasZlib(); + isValidContentType = fota->canUnzip(); } else if( contentType == "application/tar+gz" || contentType == "application/x-gtar" ) { // was packaged and compressed, may contain more than one file // TODO: use tarGzStreamUpdater @@ -887,13 +892,13 @@ static int64_t getHTTPStream( esp32FOTA* fota, int partition ) // check updateSize and content type if( !updateSize || !isValidContentType ) { - Serial.printf("There was no content in the http response: (length: %"PRId64", valid: %s)\n", updateSize, isValidContentType?"true":"false"); + Serial.printf("There was no content in the http response: (length: %" PRId64 ", valid: %s, contentType: %s)\n", updateSize, isValidContentType?"true":"false", contentType.c_str()); return -1; } - log_d("updateSize : %"PRId64", isValidContentType : %s", updateSize, String(isValidContentType)); + log_d("updateSize : %" PRId64 ", isValidContentType : %s, contentType: %s", updateSize, String(isValidContentType).c_str(), contentType.c_str()); - fota->setFotaStream( http->getStreamPtr() ); + fota->setFotaStream( fota->getHTTPCLient()->getStreamPtr() ); return updateSize; } diff --git a/src/esp32FOTA.hpp b/src/esp32FOTA.hpp index a89a095..bde73a3 100644 --- a/src/esp32FOTA.hpp +++ b/src/esp32FOTA.hpp @@ -73,21 +73,24 @@ extern "C" { #if __has_include() + #pragma message "Using FlashZ as Update agent" #include #define F_Update FlashZ::getInstance() #define F_hasZlib() true #define F_isZlibStream() (_stream->peek() == ZLIB_HEADER) - #define F_canBegin() mode_z ? F_Update.beginz(updateSize, partition) : F_Update.begin(updateSize, partition) + #define F_canBegin() mode_z ? F_Update.beginz(UPDATE_SIZE_UNKNOWN, partition) : F_Update.begin(fwsize, partition) + #define F_UpdateEnd() F_Update.endz() #define F_abort() if (mode_z) F_Update.abortz() - #define F_writeStream() mode_z ? F_Update.writezStream(*stream, contentLength) : F_Update.writeStream(*stream) + #define F_writeStream() (mode_z ? F_Update.writezStream(*_stream, updateSize) : F_Update.writeStream(*_stream)) #else #include #define F_Update Update #define F_hasZlib() false #define F_isZlibStream() false - #define F_canBegin() F_Update.begin( updateSize, partition ) + #define F_canBegin() F_Update.begin(fwsize, partition) + #define F_UpdateEnd() F_Update.end() #define F_abort() { } - #define F_writeStream() F_Update.writeStream( *_stream ); + #define F_writeStream() F_Update.writeStream(*_stream); #endif @@ -252,6 +255,8 @@ class esp32FOTA : public UpdateClass const char* getFlashFS_URL() { return _flashFileSystemUrl.c_str(); } const char* getPath(int part) { return part==U_SPIFFS ? getFlashFS_URL() : getFirmwareURL(); } + bool canUnzip() { return mode_z; } + int getPayloadVersion(); void getPayloadVersion(char * version_string); @@ -282,6 +287,8 @@ class esp32FOTA : public UpdateClass Stream *_stream; fs::File _file; + bool mode_z = F_hasZlib(); // F_isZlibStream(); + FOTAStreamType_t _stream_type = FOTA_HTTP_STREAM; // defaults to HTTP void setupStream(); From a3a85f825e9632b1644abaa88bf1c54524f0abbc Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 18:07:46 +0200 Subject: [PATCH 16/32] update macros for flashZ POC --- src/esp32FOTA.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/esp32FOTA.hpp b/src/esp32FOTA.hpp index bde73a3..04907c3 100644 --- a/src/esp32FOTA.hpp +++ b/src/esp32FOTA.hpp @@ -78,9 +78,9 @@ extern "C" { #define F_Update FlashZ::getInstance() #define F_hasZlib() true #define F_isZlibStream() (_stream->peek() == ZLIB_HEADER) - #define F_canBegin() mode_z ? F_Update.beginz(UPDATE_SIZE_UNKNOWN, partition) : F_Update.begin(fwsize, partition) - #define F_UpdateEnd() F_Update.endz() - #define F_abort() if (mode_z) F_Update.abortz() + #define F_canBegin() (mode_z ? F_Update.beginz(UPDATE_SIZE_UNKNOWN, partition) : F_Update.begin(fwsize, partition)) + #define F_UpdateEnd() (mode_z ? F_Update.endz() : F_Update.end()) + #define F_abort() if (mode_z) F_Update.abortz(); else F_Update.abort() #define F_writeStream() (mode_z ? F_Update.writezStream(*_stream, updateSize) : F_Update.writeStream(*_stream)) #else #include @@ -89,7 +89,7 @@ extern "C" { #define F_isZlibStream() false #define F_canBegin() F_Update.begin(fwsize, partition) #define F_UpdateEnd() F_Update.end() - #define F_abort() { } + #define F_abort() F_Update.abort() #define F_writeStream() F_Update.writeStream(*_stream); #endif From aa66960b6f814fd42287f5e0093028ddcb300f0a Mon Sep 17 00:00:00 2001 From: tobozo Date: Thu, 29 Sep 2022 18:47:16 +0200 Subject: [PATCH 17/32] Cherry picked @vortigont's #101 --- src/esp32FOTA.cpp | 68 +++++++++++++++++++++++++---------------------- src/esp32FOTA.hpp | 4 +-- 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/src/esp32FOTA.cpp b/src/esp32FOTA.cpp index dd04688..a4fcb15 100644 --- a/src/esp32FOTA.cpp +++ b/src/esp32FOTA.cpp @@ -385,7 +385,7 @@ bool esp32FOTA::execOTA() { setupStream(); - if( _flashFileSystemUrl != "" ) { // a data partition was specified in the json manifest, handle the spiffs partition first + if( !_flashFileSystemUrl.isEmpty() ) { // a data partition was specified in the json manifest, handle the spiffs partition first if( _fs ) { // Possible risk of overwriting certs and signatures, cancel flashing! Serial.println("Cowardly refusing to overwrite U_SPIFFS with "+_flashFileSystemUrl+". Use setCertFileSystem(nullptr) along with setPubKey()/setCAPem() to enable this feature."); return false; @@ -410,10 +410,10 @@ bool esp32FOTA::execOTA() bool esp32FOTA::execOTA( int partition, bool restart_after ) { // health checks - if( partition == U_SPIFFS && _flashFileSystemUrl == "" ) { + if( partition == U_SPIFFS && _flashFileSystemUrl.isEmpty() ) { log_i("[SKIP] No spiffs/littlefs/fatfs partition was specified"); return true; // data partition is optional, so not an error - } else if ( partition == U_FLASH && _firmwareUrl == "" ) { + } else if ( partition == U_FLASH && _firmwareUrl.isEmpty() ) { log_e("No app partition was specified"); return false; // app partition is mandatory } else if( partition != U_SPIFFS && partition != U_FLASH ) { @@ -481,11 +481,11 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) if ( written == fwsize ) { Serial.println("Written : " + String(written) + " successfully"); + updateSize = written; // flatten value to prevent overflow when checking signature } else { Serial.println("Written only : " + String(written) + "/" + String(updateSize) + ". Premature end of stream?"); F_abort(); return false; - //updateSize = written; // flatten value to prevent overflow when checking signature } if (!F_UpdateEnd()) { @@ -594,8 +594,8 @@ bool esp32FOTA::checkJSONManifest(JsonVariant doc) } log_i("Payload type in manifest %s matches current firmware %s", doc["type"].as(), _cfg.name ); - _flashFileSystemUrl = ""; - _firmwareUrl = ""; + _flashFileSystemUrl.clear(); + _firmwareUrl.clear(); if(doc["version"].is()) { uint16_t v = doc["version"].as(); @@ -614,26 +614,26 @@ bool esp32FOTA::checkJSONManifest(JsonVariant doc) debugSemVer("Payload firmware version", _payload_sem.ver() ); // Memoize some values to help with the decision tree - bool has_url = doc.containsKey("url") && doc["url"].is(); - bool has_firmware = doc.containsKey("bin") && doc["bin"].is(); - bool has_hostname = doc.containsKey("host") && doc["host"].is(); - bool has_port = doc.containsKey("port") && doc["port"].is(); - uint16_t portnum = has_port ? doc["port"].as() : 0; + bool has_url = doc["url"].is(); + bool has_firmware = doc["bin"].is(); + bool has_hostname = doc["host"].is(); + bool has_port = doc["port"].is(); + uint16_t portnum = doc["port"].as(); bool has_tls = has_port ? (portnum == 443 || portnum == 4433) : false; - bool has_spiffs = doc.containsKey("spiffs") && doc["spiffs"].is(); - bool has_littlefs = doc.containsKey("littlefs") && doc["littlefs"].is(); - bool has_fatfs = doc.containsKey("fatfs") && doc["fatfs"].is(); + bool has_spiffs = doc["spiffs"].is(); + bool has_littlefs = doc["littlefs"].is(); + bool has_fatfs = doc["fatfs"].is(); bool has_filesystem = has_littlefs || has_spiffs || has_fatfs; - String protocol = has_tls ? "https" : "http"; + String protocol(has_tls ? "https" : "http"); String flashFSPath = has_filesystem ? ( has_littlefs - ? doc["littlefs"].as() + ? doc["littlefs"].as() : has_spiffs - ? doc["spiffs"].as() - : doc["fatfs"].as() + ? doc["spiffs"].as() + : doc["fatfs"].as() ) : ""; @@ -646,14 +646,14 @@ bool esp32FOTA::checkJSONManifest(JsonVariant doc) ); if( has_url ) { // Basic scenario: a complete URL was provided in the JSON manifest, all other keys will be ignored - _firmwareUrl = doc["url"].as(); + _firmwareUrl = doc["url"].as(); if( has_hostname ) { // If the manifest provides both, warn the user log_w("Manifest provides both url and host - Using URL"); } } else if( has_firmware && has_hostname && has_port ) { // Precise scenario: Hostname, Port and Firmware Path were provided - _firmwareUrl = protocol + "://" + doc["host"].as() + ":" + String( portnum ) + doc["bin"].as(); + _firmwareUrl = protocol + "://" + doc["host"].as() + ":" + portnum + doc["bin"].as(); if( has_filesystem ) { // More complex scenario: the manifest also provides a [spiffs, littlefs or fatfs] partition - _flashFileSystemUrl = protocol + "://" + doc["host"].as() + ":" + String( portnum ) + flashFSPath; + _flashFileSystemUrl = protocol + "://" + doc["host"].as() + ":" + portnum + flashFSPath; } } else { // JSON was malformed - no firmware target was provided log_e("JSON manifest was missing one of the required keys :(" ); @@ -676,7 +676,7 @@ bool esp32FOTA::execHTTPcheck() String useURL = String( _cfg.manifest_url ); // being deprecated, soon unsupported! - if( useURL=="" && checkURL!="" ) { + if( useURL.isEmpty() && !checkURL.isEmpty() ) { Serial.println("checkURL will soon be unsupported, use FOTAConfig_t::manifest_url instead!!"); useURL = checkURL; } @@ -701,6 +701,7 @@ bool esp32FOTA::execHTTPcheck() log_i("Getting HTTP: %s", useURL.c_str()); log_i("------"); + //_http.useHTTP10(true); if(! setupHTTP( useURL.c_str() ) ) { log_e("Unable to setup http, aborting!"); return false; @@ -721,19 +722,18 @@ bool esp32FOTA::execHTTPcheck() return false; } - String payload = _http.getString(); - _http.end(); // We're done with HTTP - free the resources - // TODO: use payload.length() to speculate on JSONResult buffer size #define JSON_FW_BUFF_SIZE 2048 DynamicJsonDocument JSONResult( JSON_FW_BUFF_SIZE ); - DeserializationError err = deserializeJson( JSONResult, payload.c_str() ); + DeserializationError err = deserializeJson( JSONResult, _http.getStream() ); if (err) { // Check for errors in parsing, or JSON length may exceed buffer size - Serial.printf("JSON Parsing failed (%s, in=%d bytes, buff=%d bytes):\n%s\n", err.c_str(), payload.length(), JSON_FW_BUFF_SIZE, payload.c_str() ); + Serial.printf("JSON Parsing failed (%s, in=%d bytes, buff=%d bytes):\n", err.c_str(), _http.getSize(), JSON_FW_BUFF_SIZE ); return false; } + _http.end(); // We're done with HTTP - free the resources + if (JSONResult.is()) { // Although improbable given the size on JSONResult buffer, we already received an array of multiple firmware types and/or versions JsonArray arr = JSONResult.as(); @@ -764,7 +764,7 @@ String esp32FOTA::getDeviceID() // Force a firmware update regardless on current version -void esp32FOTA::forceUpdate(String firmwareURL, bool validate ) +void esp32FOTA::forceUpdate(const char* firmwareURL, bool validate ) { _firmwareUrl = firmwareURL; _cfg.check_sig = validate; @@ -772,11 +772,15 @@ void esp32FOTA::forceUpdate(String firmwareURL, bool validate ) } -void esp32FOTA::forceUpdate(String firmwareHost, uint16_t firmwarePort, String firmwarePath, bool validate ) +void esp32FOTA::forceUpdate(const char* firmwareHost, uint16_t firmwarePort, const char* firmwarePath, bool validate ) { - String firmwareURL = ( firmwarePort == 443 || firmwarePort == 4433 ) ? "https" : "http"; - firmwareURL += firmwareHost + ":" + String( firmwarePort ) + firmwarePath; - forceUpdate( firmwareURL, validate ); + String firmwareURL("http"); + if ( firmwarePort == 443 || firmwarePort == 4433 ) firmwareURL += "s"; + firmwareURL += String(firmwareHost); + firmwareURL += ":"; + firmwareURL += String(firmwarePort); + firmwareURL += firmwarePath; + forceUpdate( firmwareURL.c_str(), validate ); } diff --git a/src/esp32FOTA.hpp b/src/esp32FOTA.hpp index 04907c3..936fa6c 100644 --- a/src/esp32FOTA.hpp +++ b/src/esp32FOTA.hpp @@ -192,8 +192,8 @@ class esp32FOTA : public UpdateClass //template static U& getInstance() { static U updater; return updater; } - void forceUpdate(String firmwareHost, uint16_t firmwarePort, String firmwarePath, bool validate ); - void forceUpdate(String firmwareURL, bool validate ); + void forceUpdate(const char* firmwareHost, uint16_t firmwarePort, const char* firmwarePath, bool validate ); + void forceUpdate(const char* firmwareURL, bool validate ); void forceUpdate(bool validate ); bool execOTA(); From 51acddf22dca6ce161a1401f9af472c8ff0c43e8 Mon Sep 17 00:00:00 2001 From: tobozo Date: Fri, 30 Sep 2022 19:12:54 +0200 Subject: [PATCH 18/32] added gz support --- src/esp32FOTA.cpp | 45 ++++++++++++++++++++++++--------------------- src/esp32FOTA.hpp | 37 ++++++++++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 28 deletions(-) diff --git a/src/esp32FOTA.cpp b/src/esp32FOTA.cpp index a4fcb15..bed638f 100644 --- a/src/esp32FOTA.cpp +++ b/src/esp32FOTA.cpp @@ -168,6 +168,15 @@ void esp32FOTA::setupCryptoAssets() } + +void esp32FOTA::handle() +{ + if ( execHTTPcheck() ) { + execOTA(); + } +} + + // SHA-Verify the OTA partition after it's been written // https://techtutorialsx.com/2018/05/10/esp32-arduino-mbed-tls-using-the-sha-256-algorithm/ // https://github.com/ARMmbed/mbedtls/blob/development/programs/pkey/rsa_verify.c @@ -283,6 +292,8 @@ bool esp32FOTA::validate_sig( const esp_partition_t* partition, unsigned char *s } + + bool esp32FOTA::setupHTTP( const char* url ) { const char* rootcastr = nullptr; @@ -324,8 +335,8 @@ bool esp32FOTA::setupHTTP( const char* url ) } // TODO: add more watched headers e.g. Authorization: Signature keyId="rsa-key-1",algorithm="rsa-sha256",signature="Base64(RSA-SHA256(signing string))" - const char* get_headers[] = { "Content-Length", "Content-type" }; - _http.collectHeaders( get_headers, 2 ); + const char* get_headers[] = { "Content-Length", "Content-type", "Accept-Ranges" }; + _http.collectHeaders( get_headers, sizeof(get_headers)/sizeof(const char*) ); return true; } @@ -405,8 +416,6 @@ bool esp32FOTA::execOTA() } - - bool esp32FOTA::execOTA( int partition, bool restart_after ) { // health checks @@ -421,6 +430,7 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) return false; } + // call getHTTPStream int64_t updateSize = getStream( this, partition ); if( updateSize<=0 || _stream == nullptr ) { @@ -429,12 +439,11 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) } mode_z = F_isZlibStream(); + log_d("compression: %s", mode_z ? "enabled" : "disabled" ); // If using compression, the size is implicitely unknown size_t fwsize = mode_z ? UPDATE_SIZE_UNKNOWN : updateSize; // fw_size is unknown if we have a compressed image - //updateSize = mode_z ? UPDATE_SIZE_UNKNOWN : updateSize; - if( _cfg.check_sig ) { if( mode_z ) { Serial.println("[ERROR] Compressed && signed image is not (yet) supported"); @@ -617,6 +626,7 @@ bool esp32FOTA::checkJSONManifest(JsonVariant doc) bool has_url = doc["url"].is(); bool has_firmware = doc["bin"].is(); bool has_hostname = doc["host"].is(); + //bool has_signature = doc["sig"].is(); bool has_port = doc["port"].is(); uint16_t portnum = doc["port"].as(); bool has_tls = has_port ? (portnum == 443 || portnum == 4433) : false; @@ -835,32 +845,25 @@ static int64_t getHTTPStream( esp32FOTA* fota, int partition ) Serial.printf("Opening item %s\n", url ); - //HTTPClient *http = fota->getHTTPCLient(); - if(! fota->setupHTTP( url ) ) { // certs log_e("unable to setup http, aborting!"); return -1; } int64_t updateSize = 0; - bool isValidContentType = false; int httpCode = fota->getHTTPCLient()->GET(); String contentType; fota->setFotaStream( nullptr ); if( httpCode == HTTP_CODE_OK || httpCode == HTTP_CODE_MOVED_PERMANENTLY ) { - //updateSize = fota->getHTTPCLient()->header( "Content-Length" ).toInt(); updateSize = fota->getHTTPCLient()->getSize(); contentType = fota->getHTTPCLient()->header( "Content-type" ); - if( contentType == "application/octet-stream" ) { - isValidContentType = true; - } else if( contentType == "application/gzip" || contentType == "application/x-gzip" || contentType == "application/x-compress" ) { - // was gzipped by the server, needs decompression - isValidContentType = fota->canUnzip(); - } else if( contentType == "application/tar+gz" || contentType == "application/x-gtar" ) { - // was packaged and compressed, may contain more than one file - // TODO: use tarGzStreamUpdater + String acceptRange = fota->getHTTPCLient()->header( "Accept-Ranges" ); + if( !acceptRange.isEmpty() ) { + Serial.printf("This server supports resume! Accept-Ranges: %s\n", acceptRange.c_str() ); + } else { + Serial.println("This server does not support resume!" ); } } else { switch( httpCode ) { @@ -895,12 +898,12 @@ static int64_t getHTTPStream( esp32FOTA* fota, int partition ) // TODO: Set updateSize to UPDATE_SIZE_UNKNOWN when content type is valid. // check updateSize and content type - if( !updateSize || !isValidContentType ) { - Serial.printf("There was no content in the http response: (length: %" PRId64 ", valid: %s, contentType: %s)\n", updateSize, isValidContentType?"true":"false", contentType.c_str()); + if( updateSize<=0 ) { + Serial.printf("There was no content in the http response: (length: %" PRId64 ", contentType: %s)\n", updateSize, contentType.c_str()); return -1; } - log_d("updateSize : %" PRId64 ", isValidContentType : %s, contentType: %s", updateSize, String(isValidContentType).c_str(), contentType.c_str()); + log_d("updateSize : %" PRId64 ", contentType: %s", updateSize, contentType.c_str()); fota->setFotaStream( fota->getHTTPCLient()->getStreamPtr() ); diff --git a/src/esp32FOTA.hpp b/src/esp32FOTA.hpp index 936fa6c..8c7306a 100644 --- a/src/esp32FOTA.hpp +++ b/src/esp32FOTA.hpp @@ -72,16 +72,29 @@ extern "C" { #endif + + #if __has_include() #pragma message "Using FlashZ as Update agent" #include #define F_Update FlashZ::getInstance() #define F_hasZlib() true - #define F_isZlibStream() (_stream->peek() == ZLIB_HEADER) + #define F_isZlibStream() (_stream->peek() == ZLIB_HEADER && ((partition == U_SPIFFS && _flashFileSystemUrl.startsWith("zz")) || (partition == U_FLASH && _firmwareUrl.startsWith("zz")))) #define F_canBegin() (mode_z ? F_Update.beginz(UPDATE_SIZE_UNKNOWN, partition) : F_Update.begin(fwsize, partition)) #define F_UpdateEnd() (mode_z ? F_Update.endz() : F_Update.end()) #define F_abort() if (mode_z) F_Update.abortz(); else F_Update.abort() #define F_writeStream() (mode_z ? F_Update.writezStream(*_stream, updateSize) : F_Update.writeStream(*_stream)) + #define F_isGzStream() false +#elif __has_include("ESP32-targz.h") + #pragma message "Using GzUpdateClass as Update agent" + #include + #define F_Update GzUpdateClass::getInstance() + #define F_hasZlib() true + #define F_isZlibStream() (_stream->peek() == 0x1f && ((partition == U_SPIFFS && _flashFileSystemUrl.startsWith("gz")) || (partition == U_FLASH && _firmwareUrl.startsWith("gz"))) ) + #define F_canBegin() (mode_z ? F_Update.begingz(UPDATE_SIZE_UNKNOWN, partition) : F_Update.begin(fwsize, partition)) + #define F_UpdateEnd() (mode_z ? F_Update.endgz() : F_Update.end()) + #define F_abort() if (mode_z) F_Update.abortgz(); else F_Update.abort() + #define F_writeStream() (mode_z ? F_Update.writeGzStream(*_stream, updateSize) : F_Update.writeStream(*_stream)) #else #include #define F_Update Update @@ -147,12 +160,24 @@ class CryptoMemAsset : public CryptoAsset }; +// signature validation available methods +/* +enum SigType_t +{ + FOTA_SIG_PREPEND, // signature is contatenated with file (default) + FOTA_SIG_FILE, // signature is in a separate file + FOTA_SIG_JSON_PROPERTY, // signature is a JSON property + FOTA_SIG_HTTP_HEADER // signature is a HTTP header +}; +*/ + struct FOTAConfig_t { const char* name { nullptr }; const char* manifest_url { nullptr }; SemverClass sem {0}; bool check_sig { false }; + //SigType_t type_sig {FOTA_SIG_PREPEND}; bool unsafe { false }; bool use_device_id { false }; CryptoAsset* root_ca { nullptr }; @@ -196,6 +221,8 @@ class esp32FOTA : public UpdateClass void forceUpdate(const char* firmwareURL, bool validate ); void forceUpdate(bool validate ); + void handle(); + bool execOTA(); bool execOTA( int partition, bool restart_after = true ); bool execHTTPcheck(); @@ -255,7 +282,7 @@ class esp32FOTA : public UpdateClass const char* getFlashFS_URL() { return _flashFileSystemUrl.c_str(); } const char* getPath(int part) { return part==U_SPIFFS ? getFlashFS_URL() : getFirmwareURL(); } - bool canUnzip() { return mode_z; } + bool zlibSupported() { return mode_z; } int getPayloadVersion(); void getPayloadVersion(char * version_string); @@ -272,10 +299,6 @@ class esp32FOTA : public UpdateClass bool setupHTTP( const char* url ); void setFotaStream( Stream* stream ) { _stream = stream; } - - //friend class FlashZ; - - [[deprecated("Use setManifestURL( String ) or cfg.manifest_url with setConfig( FOTAConfig_t )")]] String checkURL = ""; [[deprecated("Use cfg.use_device_id with setConfig( FOTAConfig_t )")]] bool useDeviceID = false; @@ -287,7 +310,7 @@ class esp32FOTA : public UpdateClass Stream *_stream; fs::File _file; - bool mode_z = F_hasZlib(); // F_isZlibStream(); + bool mode_z = F_hasZlib(); FOTAStreamType_t _stream_type = FOTA_HTTP_STREAM; // defaults to HTTP From 8eab21dff9b70e1839a82c2f581d87401f27ca27 Mon Sep 17 00:00:00 2001 From: tobozo Date: Fri, 30 Sep 2022 19:13:20 +0200 Subject: [PATCH 19/32] updated examples --- .../test/1.1.nosecurity/1.1.nosecurity.ino | 24 ++-- .../1.2.nosecurity.gz/1.2.nosecurity.gz.ino | 16 ++- .../1.3.nosecurity.zz/1.3.nosecurity.zz.ino | 114 ++++++++++++++++++ platformio.ini | 2 + 4 files changed, 147 insertions(+), 9 deletions(-) create mode 100644 examples/anyFS/test/1.3.nosecurity.zz/1.3.nosecurity.zz.ino diff --git a/examples/anyFS/test/1.1.nosecurity/1.1.nosecurity.ino b/examples/anyFS/test/1.1.nosecurity/1.1.nosecurity.ino index a358e52..5fa268b 100644 --- a/examples/anyFS/test/1.1.nosecurity/1.1.nosecurity.ino +++ b/examples/anyFS/test/1.1.nosecurity/1.1.nosecurity.ino @@ -12,6 +12,8 @@ */ +//#include // optional esp32-flashz for zlib compressed firmwares +#include // optional ESP32-targz for gzip compressed firmwares #include // esp32fota settings @@ -34,9 +36,10 @@ const char* fota_debug_fmt = R"DBG_FMT( Description : %s Firmware type : %s - Firmware version : %i + Firmware version : %i.%i.%i Signature check : %s TLS Cert check : %s + Compression : %s ******************************************** @@ -73,7 +76,18 @@ void setup_wifi() void setup() { Serial.begin(115200); - Serial.printf( fota_debug_fmt, firmware_version_major, description, firmware_name, firmware_version_major, check_signature?"Enabled":"Disabled", disable_security?"Disabled":"Enabled" ); + + Serial.printf( fota_debug_fmt, + firmware_version_major, + description, + firmware_name, + firmware_version_major, + firmware_version_minor, + firmware_version_patch, + check_signature ?"Enabled":"Disabled", + disable_security ?"Disabled":"Enabled", + FOTA.zlibSupported() ?"Enabled":"Disabled" + ); { auto cfg = FOTA.getConfig(); @@ -92,10 +106,6 @@ void setup() void loop() { - bool updatedNeeded = FOTA.execHTTPcheck(); - if (updatedNeeded) { - FOTA.execOTA(); - } - + FOTA.handle(); delay(20000); } diff --git a/examples/anyFS/test/1.2.nosecurity.gz/1.2.nosecurity.gz.ino b/examples/anyFS/test/1.2.nosecurity.gz/1.2.nosecurity.gz.ino index 89fa69b..e3c3631 100644 --- a/examples/anyFS/test/1.2.nosecurity.gz/1.2.nosecurity.gz.ino +++ b/examples/anyFS/test/1.2.nosecurity.gz/1.2.nosecurity.gz.ino @@ -35,9 +35,10 @@ const char* fota_debug_fmt = R"DBG_FMT( Description : %s Firmware type : %s - Firmware version : %i + Firmware version : %i.%i.%i Signature check : %s TLS Cert check : %s + Compression : %s ******************************************** @@ -74,7 +75,18 @@ void setup_wifi() void setup() { Serial.begin(115200); - Serial.printf( fota_debug_fmt, firmware_version_major, description, firmware_name, firmware_version_major, check_signature?"Enabled":"Disabled", disable_security?"Disabled":"Enabled" ); + + Serial.printf( fota_debug_fmt, + firmware_version_major, + description, + firmware_name, + firmware_version_major, + firmware_version_minor, + firmware_version_patch, + check_signature ?"Enabled":"Disabled", + disable_security ?"Disabled":"Enabled", + FOTA.zlibSupported() ?"Enabled":"Disabled" + ); { auto cfg = FOTA.getConfig(); diff --git a/examples/anyFS/test/1.3.nosecurity.zz/1.3.nosecurity.zz.ino b/examples/anyFS/test/1.3.nosecurity.zz/1.3.nosecurity.zz.ino new file mode 100644 index 0000000..ecae4c8 --- /dev/null +++ b/examples/anyFS/test/1.3.nosecurity.zz/1.3.nosecurity.zz.ino @@ -0,0 +1,114 @@ +/** + esp32 firmware OTA + + Purpose: Perform an OTA update to both firmware and filesystem from binaries located + on a webserver (HTTPS) without checking for certificate validity + + Usage: If the ESP32 had a previous successful WiFi connection, then no need to set the ssid/password + to run this sketch, the credentials are still cached :-) + Sketch 1 will FOTA to Sketch 2, then Sketch 3, and so on until all versions in firmware.json are + exhausted. + + +*/ + +#include + +// esp32fota settings +int firmware_version_major = 1; +int firmware_version_minor = 3; +int firmware_version_patch = 0; + +#if !defined FOTA_URL + #define FOTA_URL "http://server/fota/fota.json" +#endif +const char* firmware_name = "esp32-fota-http"; +const bool check_signature = false; +const bool disable_security = true; +// for debug only +const char* description = "Basic *gzipped* example with no security and no filesystem"; + +const char* fota_debug_fmt = R"DBG_FMT( + +***************** STAGE %i ***************** + + Description : %s + Firmware type : %s + Firmware version : %i.%i.%i + Signature check : %s + TLS Cert check : %s + Compression : %s + +******************************************** + +)DBG_FMT"; + + +// esp32fota esp32fota("", , , ); +// esp32FOTA esp32FOTA( String(firmware_name), firmware_version, check_signature, disable_security ); + + +esp32FOTA FOTA; + + +void setup_wifi() +{ + delay(10); + + Serial.print("MAC Address "); + Serial.println( WiFi.macAddress() ); + + WiFi.begin(); // no WiFi creds in this demo :-) + + while (WiFi.status() != WL_CONNECTED) + { + delay(500); + Serial.print("."); + } + + Serial.println(""); + Serial.println(WiFi.localIP()); +} + + +void setup() +{ + Serial.begin(115200); + + Serial.printf( fota_debug_fmt, + firmware_version_major, + description, + firmware_name, + firmware_version_major, + firmware_version_minor, + firmware_version_patch, + check_signature ?"Enabled":"Disabled", + disable_security ?"Disabled":"Enabled", + FOTA.zlibSupported() ?"Enabled":"Disabled" + ); + + { + auto cfg = FOTA.getConfig(); + cfg.name = firmware_name; + cfg.manifest_url = FOTA_URL; + cfg.sem = SemverClass( firmware_version_major, firmware_version_minor, firmware_version_patch ); + cfg.check_sig = check_signature; + cfg.unsafe = disable_security; + //cfg.root_ca = MyRootCA; + //cfg.pub_key = MyRSAKey; + FOTA.setConfig( cfg ); + } + + setup_wifi(); +} + +void loop() +{ + bool updatedNeeded = FOTA.execHTTPcheck(); + if (updatedNeeded) { + FOTA.execOTA(); + } + + delay(20000); +} + diff --git a/platformio.ini b/platformio.ini index d8aa752..1d1380a 100644 --- a/platformio.ini +++ b/platformio.ini @@ -17,6 +17,8 @@ build_flags = lib_deps = bblanchon/ArduinoJson @ ^6 esp32FOTA +; vortigont/esp32-flashz +; tobozo/ESP32-targz [esp32_common] platform = espressif32 From 7f619d45137413d1f246382ce9daca5225419bc7 Mon Sep 17 00:00:00 2001 From: tobozo Date: Fri, 30 Sep 2022 19:13:29 +0200 Subject: [PATCH 20/32] updated CI --- .github/templates/firmware.test-suite.json | 7 +++++++ .github/workflows/gen-test-suite.yml | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/templates/firmware.test-suite.json b/.github/templates/firmware.test-suite.json index 11ded01..a2c202a 100644 --- a/.github/templates/firmware.test-suite.json +++ b/.github/templates/firmware.test-suite.json @@ -12,6 +12,13 @@ "port": FIRMWARE_PORT, "bin": "FIRMWARE_PATH/1.2.nosecurity.ino.bin.gz?raw=true" }, +{ + "type": "FIRMWARE_TYPE", + "version": "1.3", + "host": "FIRMWARE_HOST", + "port": FIRMWARE_PORT, + "bin": "FIRMWARE_PATH/1.2.nosecurity.ino.bin.zz?raw=true" +}, { "type": "FIRMWARE_TYPE", "version": 2, diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index 87b9174..599a4c5 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -174,6 +174,7 @@ jobs: sketch: - 1.1.nosecurity.ino - 1.2.nosecurity.gz.ino + - 1.3.nosecurity.zz.ino - 2.cert-in-spiffs.ino - 3.cert-in-progmem.ino - 4.cert-in-littlefs.ino @@ -216,6 +217,14 @@ jobs: ref: main path: CustomflashZ # must contain string "Custom" + - name: Checkout ESP32-targz + uses: actions/checkout@v2 + with: + repository: tobozo/ESP32-targz + ref: main + path: CustomESP32-targz # must contain string "Custom" + + - name: Compile ${{ matrix.sketch }} uses: ArminJo/arduino-test-compile@v3 with: @@ -236,7 +245,7 @@ jobs: cp $full_ino_bin_path ${{env.artifact_path}}/${{ matrix.sketch }}.bin cp $full_ino_bin_path.img ${{env.artifact_path}}/${{ matrix.sketch }}.signed.bin gzip -c $full_ino_bin_path > ${{env.artifact_path}}/${{ matrix.sketch }}.bin.gz - pigz -9kzc $full_ino_bin_path > ${{env.artifact_path}}/${{ matrix.sketch }}.bin.z + pigz -9kzc $full_ino_bin_path > ${{env.artifact_path}}/${{ matrix.sketch }}.bin.zz - name: Prepare data folder @@ -266,8 +275,8 @@ jobs: gzip -c ${{env.littlefs_bin_path}} > ${{env.littlefs_bin_path}}.gz gzip -c ${{env.spiffs_bin_path}} > ${{env.spiffs_bin_path}}.gz # Create flashZ versions - pigz -9kzc ${{env.littlefs_bin_path}} > ${{env.littlefs_bin_path}}.z - pigz -9kzc ${{env.spiffs_bin_path}} > ${{env.spiffs_bin_path}}.z + pigz -9kzc ${{env.littlefs_bin_path}} > ${{env.littlefs_bin_path}}.zz + pigz -9kzc ${{env.spiffs_bin_path}} > ${{env.spiffs_bin_path}}.zz # Sign partition binaries openssl dgst -sign ${{env.privkey_path}} -keyform PEM -sha256 -out firmware.sign -binary ${{env.spiffs_bin_path}} From 6c1de77bcb0bb3bdfcfeeb677f377eb515fc197e Mon Sep 17 00:00:00 2001 From: tobozo Date: Fri, 30 Sep 2022 19:20:52 +0200 Subject: [PATCH 21/32] updated CI --- .github/workflows/gen-test-suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index 599a4c5..5b71417 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -222,7 +222,7 @@ jobs: with: repository: tobozo/ESP32-targz ref: main - path: CustomESP32-targz # must contain string "Custom" + path: CustomESP32TarGz # must contain string "Custom" - name: Compile ${{ matrix.sketch }} From 52ad4c43009d8c641ffe6410b99b1b0697fb9280 Mon Sep 17 00:00:00 2001 From: tobozo Date: Fri, 30 Sep 2022 19:22:39 +0200 Subject: [PATCH 22/32] updated CI --- .github/workflows/gen-test-suite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index 5b71417..65818d0 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -221,8 +221,8 @@ jobs: uses: actions/checkout@v2 with: repository: tobozo/ESP32-targz - ref: main - path: CustomESP32TarGz # must contain string "Custom" + ref: master + path: CustomESP32-targz # must contain string "Custom" - name: Compile ${{ matrix.sketch }} From c34bc1f04dfe7f53bd666a1d7a99ae7208482299 Mon Sep 17 00:00:00 2001 From: tobozo Date: Fri, 30 Sep 2022 20:53:14 +0200 Subject: [PATCH 23/32] fix zlib macros --- .github/templates/firmware.test-suite.json | 4 ++-- src/esp32FOTA.hpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/templates/firmware.test-suite.json b/.github/templates/firmware.test-suite.json index a2c202a..41ea691 100644 --- a/.github/templates/firmware.test-suite.json +++ b/.github/templates/firmware.test-suite.json @@ -10,14 +10,14 @@ "version": "1.2", "host": "FIRMWARE_HOST", "port": FIRMWARE_PORT, - "bin": "FIRMWARE_PATH/1.2.nosecurity.ino.bin.gz?raw=true" + "bin": "FIRMWARE_PATH/1.2.nosecurity.gz.ino.bin.gz?raw=true" }, { "type": "FIRMWARE_TYPE", "version": "1.3", "host": "FIRMWARE_HOST", "port": FIRMWARE_PORT, - "bin": "FIRMWARE_PATH/1.2.nosecurity.ino.bin.zz?raw=true" + "bin": "FIRMWARE_PATH/1.3.nosecurity.zz.ino.bin.zz?raw=true" }, { "type": "FIRMWARE_TYPE", diff --git a/src/esp32FOTA.hpp b/src/esp32FOTA.hpp index 8c7306a..a82e8da 100644 --- a/src/esp32FOTA.hpp +++ b/src/esp32FOTA.hpp @@ -79,7 +79,7 @@ extern "C" { #include #define F_Update FlashZ::getInstance() #define F_hasZlib() true - #define F_isZlibStream() (_stream->peek() == ZLIB_HEADER && ((partition == U_SPIFFS && _flashFileSystemUrl.startsWith("zz")) || (partition == U_FLASH && _firmwareUrl.startsWith("zz")))) + #define F_isZlibStream() (_stream->peek() == ZLIB_HEADER && ((partition == U_SPIFFS && _flashFileSystemUrl.indexOf("zz")>-1) || (partition == U_FLASH && _firmwareUrl.indexOf("zz")>-1))) #define F_canBegin() (mode_z ? F_Update.beginz(UPDATE_SIZE_UNKNOWN, partition) : F_Update.begin(fwsize, partition)) #define F_UpdateEnd() (mode_z ? F_Update.endz() : F_Update.end()) #define F_abort() if (mode_z) F_Update.abortz(); else F_Update.abort() @@ -90,7 +90,7 @@ extern "C" { #include #define F_Update GzUpdateClass::getInstance() #define F_hasZlib() true - #define F_isZlibStream() (_stream->peek() == 0x1f && ((partition == U_SPIFFS && _flashFileSystemUrl.startsWith("gz")) || (partition == U_FLASH && _firmwareUrl.startsWith("gz"))) ) + #define F_isZlibStream() (_stream->peek() == 0x1f && ((partition == U_SPIFFS && _flashFileSystemUrl.indexOf("gz")>-1) || (partition == U_FLASH && _firmwareUrl.indexOf("gz")>-1)) ) #define F_canBegin() (mode_z ? F_Update.begingz(UPDATE_SIZE_UNKNOWN, partition) : F_Update.begin(fwsize, partition)) #define F_UpdateEnd() (mode_z ? F_Update.endgz() : F_Update.end()) #define F_abort() if (mode_z) F_Update.abortgz(); else F_Update.abort() From 6cc56c51cc7a6cff6b00b2a9617a8ba42f99ada9 Mon Sep 17 00:00:00 2001 From: tobozo Date: Fri, 30 Sep 2022 21:18:56 +0200 Subject: [PATCH 24/32] updated CI --- .github/workflows/gen-test-suite.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index 65818d0..77e6d37 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -70,6 +70,15 @@ on: - large_spiffs default: 'default' + debug_level: + description: 'Core Debug Level' + required: true + type: choice + - none + - error + - warning + - info + - verbose jobs: @@ -229,7 +238,7 @@ jobs: uses: ArminJo/arduino-test-compile@v3 with: platform-url: https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_dev_index.json - arduino-board-fqbn: ${{inputs.board_fqbn}}:PartitionScheme=${{inputs.partition_scheme}} + arduino-board-fqbn: ${{inputs.board_fqbn}}:PartitionScheme=${{inputs.partition_scheme}},DebugLevel=${{inputs.debug_level}} required-libraries: ArduinoJson extra-arduino-lib-install-args: --no-deps extra-arduino-cli-args: "--warnings default " # see https://github.com/ArminJo/arduino-test-compile/issues/28 From 07eaf2082ac04308240cd17812d6f4a6a3221683 Mon Sep 17 00:00:00 2001 From: tobozo Date: Fri, 30 Sep 2022 21:20:00 +0200 Subject: [PATCH 25/32] updated CI --- .github/workflows/gen-test-suite.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index 77e6d37..95f105d 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -79,6 +79,7 @@ on: - warning - info - verbose + default: 'none' jobs: From 4fb84680bee9186b804e1ae90834417fa2142c79 Mon Sep 17 00:00:00 2001 From: tobozo Date: Fri, 30 Sep 2022 21:21:22 +0200 Subject: [PATCH 26/32] updated CI --- .github/workflows/gen-test-suite.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index 95f105d..710a5cd 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -74,6 +74,7 @@ on: description: 'Core Debug Level' required: true type: choice + options: - none - error - warning From e085f8d95eecc0c1354f32ce6167b26fa517f69e Mon Sep 17 00:00:00 2001 From: tobozo Date: Fri, 30 Sep 2022 21:23:05 +0200 Subject: [PATCH 27/32] updated CI --- .github/workflows/gen-test-suite.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gen-test-suite.yml b/.github/workflows/gen-test-suite.yml index 710a5cd..8d3fad0 100644 --- a/.github/workflows/gen-test-suite.yml +++ b/.github/workflows/gen-test-suite.yml @@ -79,6 +79,7 @@ on: - error - warning - info + - debug - verbose default: 'none' From 31d376b84b89e450f86bea5f0b393a09fda22ec0 Mon Sep 17 00:00:00 2001 From: tobozo Date: Fri, 30 Sep 2022 21:56:55 +0200 Subject: [PATCH 28/32] fix signature check --- src/esp32FOTA.cpp | 11 +++++------ src/esp32FOTA.hpp | 6 +++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/esp32FOTA.cpp b/src/esp32FOTA.cpp index bed638f..b028bf8 100644 --- a/src/esp32FOTA.cpp +++ b/src/esp32FOTA.cpp @@ -441,23 +441,22 @@ bool esp32FOTA::execOTA( int partition, bool restart_after ) mode_z = F_isZlibStream(); log_d("compression: %s", mode_z ? "enabled" : "disabled" ); - // If using compression, the size is implicitely unknown - size_t fwsize = mode_z ? UPDATE_SIZE_UNKNOWN : updateSize; // fw_size is unknown if we have a compressed image if( _cfg.check_sig ) { if( mode_z ) { Serial.println("[ERROR] Compressed && signed image is not (yet) supported"); return false; } - if( updateSize != UPDATE_SIZE_UNKNOWN ) { - if( updateSize <= FW_SIGNATURE_LENGTH ) { + if( updateSize == UPDATE_SIZE_UNKNOWN || updateSize <= FW_SIGNATURE_LENGTH ) { Serial.println("[ERROR] Malformed signature+fw combo"); return false; - } - updateSize -= FW_SIGNATURE_LENGTH; } + updateSize -= FW_SIGNATURE_LENGTH; } + // If using compression, the size is implicitely unknown + size_t fwsize = mode_z ? UPDATE_SIZE_UNKNOWN : updateSize; // fw_size is unknown if we have a compressed image + bool canBegin = F_canBegin(); if( !canBegin ) { diff --git a/src/esp32FOTA.hpp b/src/esp32FOTA.hpp index a82e8da..64025a8 100644 --- a/src/esp32FOTA.hpp +++ b/src/esp32FOTA.hpp @@ -80,7 +80,7 @@ extern "C" { #define F_Update FlashZ::getInstance() #define F_hasZlib() true #define F_isZlibStream() (_stream->peek() == ZLIB_HEADER && ((partition == U_SPIFFS && _flashFileSystemUrl.indexOf("zz")>-1) || (partition == U_FLASH && _firmwareUrl.indexOf("zz")>-1))) - #define F_canBegin() (mode_z ? F_Update.beginz(UPDATE_SIZE_UNKNOWN, partition) : F_Update.begin(fwsize, partition)) + #define F_canBegin() (mode_z ? F_Update.beginz(UPDATE_SIZE_UNKNOWN, partition) : F_Update.begin(updateSize, partition)) #define F_UpdateEnd() (mode_z ? F_Update.endz() : F_Update.end()) #define F_abort() if (mode_z) F_Update.abortz(); else F_Update.abort() #define F_writeStream() (mode_z ? F_Update.writezStream(*_stream, updateSize) : F_Update.writeStream(*_stream)) @@ -91,7 +91,7 @@ extern "C" { #define F_Update GzUpdateClass::getInstance() #define F_hasZlib() true #define F_isZlibStream() (_stream->peek() == 0x1f && ((partition == U_SPIFFS && _flashFileSystemUrl.indexOf("gz")>-1) || (partition == U_FLASH && _firmwareUrl.indexOf("gz")>-1)) ) - #define F_canBegin() (mode_z ? F_Update.begingz(UPDATE_SIZE_UNKNOWN, partition) : F_Update.begin(fwsize, partition)) + #define F_canBegin() (mode_z ? F_Update.begingz(UPDATE_SIZE_UNKNOWN, partition) : F_Update.begin(updateSize, partition)) #define F_UpdateEnd() (mode_z ? F_Update.endgz() : F_Update.end()) #define F_abort() if (mode_z) F_Update.abortgz(); else F_Update.abort() #define F_writeStream() (mode_z ? F_Update.writeGzStream(*_stream, updateSize) : F_Update.writeStream(*_stream)) @@ -100,7 +100,7 @@ extern "C" { #define F_Update Update #define F_hasZlib() false #define F_isZlibStream() false - #define F_canBegin() F_Update.begin(fwsize, partition) + #define F_canBegin() F_Update.begin(updateSize, partition) #define F_UpdateEnd() F_Update.end() #define F_abort() F_Update.abort() #define F_writeStream() F_Update.writeStream(*_stream); From 7dfae128d91bd31ba5d2fb67dfb28eb26a45f61c Mon Sep 17 00:00:00 2001 From: tobozo Date: Fri, 30 Sep 2022 22:17:23 +0200 Subject: [PATCH 29/32] code cleanup --- src/esp32FOTA.cpp | 2 -- src/esp32FOTA.hpp | 17 ----------------- 2 files changed, 19 deletions(-) diff --git a/src/esp32FOTA.cpp b/src/esp32FOTA.cpp index b028bf8..1856540 100644 --- a/src/esp32FOTA.cpp +++ b/src/esp32FOTA.cpp @@ -618,7 +618,6 @@ bool esp32FOTA::checkJSONManifest(JsonVariant doc) _payload_sem = SemverClass(0); } - //debugSemVer("Payload firmware version", &_payloadVersion ); debugSemVer("Payload firmware version", _payload_sem.ver() ); // Memoize some values to help with the decision tree @@ -710,7 +709,6 @@ bool esp32FOTA::execHTTPcheck() log_i("Getting HTTP: %s", useURL.c_str()); log_i("------"); - //_http.useHTTP10(true); if(! setupHTTP( useURL.c_str() ) ) { log_e("Unable to setup http, aborting!"); return false; diff --git a/src/esp32FOTA.hpp b/src/esp32FOTA.hpp index 64025a8..40f0c7b 100644 --- a/src/esp32FOTA.hpp +++ b/src/esp32FOTA.hpp @@ -84,7 +84,6 @@ extern "C" { #define F_UpdateEnd() (mode_z ? F_Update.endz() : F_Update.end()) #define F_abort() if (mode_z) F_Update.abortz(); else F_Update.abort() #define F_writeStream() (mode_z ? F_Update.writezStream(*_stream, updateSize) : F_Update.writeStream(*_stream)) - #define F_isGzStream() false #elif __has_include("ESP32-targz.h") #pragma message "Using GzUpdateClass as Update agent" #include @@ -160,24 +159,12 @@ class CryptoMemAsset : public CryptoAsset }; -// signature validation available methods -/* -enum SigType_t -{ - FOTA_SIG_PREPEND, // signature is contatenated with file (default) - FOTA_SIG_FILE, // signature is in a separate file - FOTA_SIG_JSON_PROPERTY, // signature is a JSON property - FOTA_SIG_HTTP_HEADER // signature is a HTTP header -}; -*/ - struct FOTAConfig_t { const char* name { nullptr }; const char* manifest_url { nullptr }; SemverClass sem {0}; bool check_sig { false }; - //SigType_t type_sig {FOTA_SIG_PREPEND}; bool unsafe { false }; bool use_device_id { false }; CryptoAsset* root_ca { nullptr }; @@ -210,13 +197,9 @@ class esp32FOTA : public UpdateClass esp32FOTA(const String &firwmareType, const String &firmwareSemanticVersion, bool validate = false, bool allow_insecure_https = false ) : esp32FOTA(firwmareType.c_str(), firmwareSemanticVersion.c_str(), validate, allow_insecure_https){}; - - template void setPubKey( T* asset ) { _cfg.pub_key = (CryptoAsset*)asset; _cfg.check_sig = true; } template void setRootCA( T* asset ) { _cfg.root_ca = (CryptoAsset*)asset; _cfg.unsafe = false; } - //template static U& getInstance() { static U updater; return updater; } - void forceUpdate(const char* firmwareHost, uint16_t firmwarePort, const char* firmwarePath, bool validate ); void forceUpdate(const char* firmwareURL, bool validate ); void forceUpdate(bool validate ); From e38b722b36e09aa7ec945f81e1a8d1f3a761a735 Mon Sep 17 00:00:00 2001 From: tobozo Date: Fri, 30 Sep 2022 22:20:08 +0200 Subject: [PATCH 30/32] remove unnecessary inheritence --- src/esp32FOTA.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/esp32FOTA.hpp b/src/esp32FOTA.hpp index 40f0c7b..358873f 100644 --- a/src/esp32FOTA.hpp +++ b/src/esp32FOTA.hpp @@ -182,7 +182,7 @@ enum FOTAStreamType_t // Main Class -class esp32FOTA : public UpdateClass +class esp32FOTA { public: From 316f3ebf81501359d2361168c8feb309752d92c9 Mon Sep 17 00:00:00 2001 From: tobozo Date: Fri, 30 Sep 2022 22:36:22 +0200 Subject: [PATCH 31/32] Updated Readme --- README.md | 82 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b4359cb..5bb3bad 100644 --- a/README.md +++ b/README.md @@ -95,12 +95,15 @@ A single JSON file can also contain several versions of a single firmware type: { "type":"esp32-fota-http", "version":"0.0.3", - "url":"http://192.168.0.100/fota/esp32-fota-0.0.3.bin" + "url":"http://192.168.0.100/fota/esp32-fota-0.0.3.bin", + "spiffs":"http://192.168.0.100/fota/esp32-fota-0.0.3.spiffs.bin" } ] ``` + + #### Filesystem image (spiffs/littlefs) Adding `spiffs` key to the JSON entry will end up with the filesystem being updated first, then the firmware. @@ -173,10 +176,12 @@ void setup_wifi() void loop() { - bool updatedNeeded = esp32FOTA.execHTTPcheck(); - if (updatedNeeded) { - esp32FOTA.execOTA(); - } + esp32FOTA.handle(); + // or ... + // bool updatedNeeded = esp32FOTA.execHTTPcheck(); + // if (updatedNeeded) { + // esp32FOTA.execOTA(); + // } delay(2000); } ``` @@ -214,16 +219,67 @@ void setup() void loop() { - bool updatedNeeded = esp32FOTA.execHTTPcheck(); - if (updatedNeeded) { - esp32FOTA.execOTA(); - } + esp32FOTA.handle(); + // or ... + // bool updatedNeeded = esp32FOTA.execHTTPcheck(); + // if (updatedNeeded) { + // esp32FOTA.execOTA(); + // } delay(2000); } ``` +### Zlib/gzip support + +⚠️ This feature cannot be used with signature check. + + +For firmwares compressed with `pigz` utility (see , file extension must be `.zz`: + +```cpp +#include // http://github.com/vortigont/esp32-flashz +#include +``` + +```bash +$ pigz -9kzc esp32-fota-http-2.bin > esp32-fota-http-2.bin.zz +``` + +```json +{ + "type": "esp32-fota-http", + "version": "2.5.1", + "url": "http://192.168.0.100/fota/esp32-fota-http-2.bin.zz" +} +``` + + +For firmwares compressed with `gzip` utility, file extension must be `.gz` + +```cpp +#include // http://github.com/tobozo/ESP32-targz +#include +``` + +```bash +$ gzip -c esp32-fota-http-2.bin > esp32-fota-http-2.bin.gz +``` + +```json +{ + "type": "esp32-fota-http", + "version": "2.5.1", + "url": "http://192.168.0.100/fota/esp32-fota-http-2.bin.gz" +} +``` + + + + + + ### Root Certificates Certificates and signatures can be stored in different places: any fs::FS filesystem or progmem as const char*. @@ -433,11 +489,14 @@ On the next update-check the ESP32 will download the `firmware.img` extract the [#92]: https://github.com/chrisjoyce911/esp32FOTA/pull/92 +### Libraries +This library relies on [semver.c by h2non](https://github.com/h2non/semver.c) for semantic versioning support. semver.c is licensed under [MIT](https://github.com/h2non/semver.c/blob/master/LICENSE). -### Libraries +Optional dependencies (zlib/gzip support): +* [esp32-flashz](https://github.com/vortigont/esp32-flashz) +* [esp32-targz](https://github.com/tobozo/ESP32-targz) -This relies on [semver.c by h2non](https://github.com/h2non/semver.c) for semantic versioning support. semver.c is licensed under [MIT](https://github.com/h2non/semver.c/blob/master/LICENSE). ### Thanks to @@ -446,4 +505,5 @@ This relies on [semver.c by h2non](https://github.com/h2non/semver.c) for semant * @tuan-karma * @hpsaturn * @tobozo +* @vortigont From b45fe64b61178f29bef2ec19833eae48fe47fb66 Mon Sep 17 00:00:00 2001 From: tobozo Date: Sat, 1 Oct 2022 00:19:14 +0200 Subject: [PATCH 32/32] Updated examples / test-suite --- examples/anyFS/anyFS.ino | 4 ++-- .../test/1.1.nosecurity/1.1.nosecurity.ino | 5 ++--- .../1.2.nosecurity.gz/1.2.nosecurity.gz.ino | 4 ++-- .../1.3.nosecurity.zz/1.3.nosecurity.zz.ino | 4 ++-- .../test/2.cert-in-spiffs/2.cert-in-spiffs.ino | 18 +++++++++++++++--- .../3.cert-in-progmem/3.cert-in-progmem.ino | 18 +++++++++++++++--- .../4.cert-in-littlefs/4.cert-in-littlefs.ino | 18 +++++++++++++++--- .../test/5.sig-in-progmem/5.sig-in-progmem.ino | 18 +++++++++++++++--- 8 files changed, 68 insertions(+), 21 deletions(-) diff --git a/examples/anyFS/anyFS.ino b/examples/anyFS/anyFS.ino index 7f7c176..ef0c051 100644 --- a/examples/anyFS/anyFS.ino +++ b/examples/anyFS/anyFS.ino @@ -13,8 +13,8 @@ #include //#include -#include // optional esp32-flashz for gzipped firmwares - +//#include // optional esp32-flashz for zlib compressed firmwares +//#include // optional ESP32-targz for gzip compressed firmwares #include // fota pulls WiFi library diff --git a/examples/anyFS/test/1.1.nosecurity/1.1.nosecurity.ino b/examples/anyFS/test/1.1.nosecurity/1.1.nosecurity.ino index 5fa268b..d5d095f 100644 --- a/examples/anyFS/test/1.1.nosecurity/1.1.nosecurity.ino +++ b/examples/anyFS/test/1.1.nosecurity/1.1.nosecurity.ino @@ -12,7 +12,6 @@ */ -//#include // optional esp32-flashz for zlib compressed firmwares #include // optional ESP32-targz for gzip compressed firmwares #include @@ -32,7 +31,7 @@ const char* description = "Basic example with no security and no filesystem" const char* fota_debug_fmt = R"DBG_FMT( -***************** STAGE %i ***************** +***************** STAGE %s ***************** Description : %s Firmware type : %s @@ -78,7 +77,7 @@ void setup() Serial.begin(115200); Serial.printf( fota_debug_fmt, - firmware_version_major, + "1.1", description, firmware_name, firmware_version_major, diff --git a/examples/anyFS/test/1.2.nosecurity.gz/1.2.nosecurity.gz.ino b/examples/anyFS/test/1.2.nosecurity.gz/1.2.nosecurity.gz.ino index e3c3631..97e02e4 100644 --- a/examples/anyFS/test/1.2.nosecurity.gz/1.2.nosecurity.gz.ino +++ b/examples/anyFS/test/1.2.nosecurity.gz/1.2.nosecurity.gz.ino @@ -31,7 +31,7 @@ const char* description = "Basic *gzipped* example with no security and no f const char* fota_debug_fmt = R"DBG_FMT( -***************** STAGE %i ***************** +***************** STAGE %s ***************** Description : %s Firmware type : %s @@ -77,7 +77,7 @@ void setup() Serial.begin(115200); Serial.printf( fota_debug_fmt, - firmware_version_major, + "1.2", description, firmware_name, firmware_version_major, diff --git a/examples/anyFS/test/1.3.nosecurity.zz/1.3.nosecurity.zz.ino b/examples/anyFS/test/1.3.nosecurity.zz/1.3.nosecurity.zz.ino index ecae4c8..994722f 100644 --- a/examples/anyFS/test/1.3.nosecurity.zz/1.3.nosecurity.zz.ino +++ b/examples/anyFS/test/1.3.nosecurity.zz/1.3.nosecurity.zz.ino @@ -30,7 +30,7 @@ const char* description = "Basic *gzipped* example with no security and no f const char* fota_debug_fmt = R"DBG_FMT( -***************** STAGE %i ***************** +***************** STAGE %s ***************** Description : %s Firmware type : %s @@ -76,7 +76,7 @@ void setup() Serial.begin(115200); Serial.printf( fota_debug_fmt, - firmware_version_major, + "1.3", description, firmware_name, firmware_version_major, diff --git a/examples/anyFS/test/2.cert-in-spiffs/2.cert-in-spiffs.ino b/examples/anyFS/test/2.cert-in-spiffs/2.cert-in-spiffs.ino index b8e8c4e..370268e 100644 --- a/examples/anyFS/test/2.cert-in-spiffs/2.cert-in-spiffs.ino +++ b/examples/anyFS/test/2.cert-in-spiffs/2.cert-in-spiffs.ino @@ -25,13 +25,14 @@ const char* description = "SPIFFS example with security"; const char* fota_debug_fmt = R"DBG_FMT( -***************** STAGE %i ***************** +***************** STAGE %s ***************** Description : %s Firmware type : %s - Firmware version : %i + Firmware version : %i.%i.%i Signature check : %s TLS Cert check : %s + Compression : %s ******************************************** @@ -74,7 +75,18 @@ void setup_wifi() void setup() { Serial.begin(115200); - Serial.printf( fota_debug_fmt, firmware_version_major, description, firmware_name, firmware_version_major, check_signature?"Enabled":"Disabled", disable_security?"Disabled":"Enabled" ); + + Serial.printf( fota_debug_fmt, + "2", + description, + firmware_name, + firmware_version_major, + firmware_version_minor, + firmware_version_patch, + check_signature ?"Enabled":"Disabled", + disable_security ?"Disabled":"Enabled", + FOTA.zlibSupported() ?"Enabled":"Disabled" + ); // Provide filesystem with root_ca.pem to validate server certificate if( ! SPIFFS.begin( false ) ) { diff --git a/examples/anyFS/test/3.cert-in-progmem/3.cert-in-progmem.ino b/examples/anyFS/test/3.cert-in-progmem/3.cert-in-progmem.ino index a07f35c..3c94353 100644 --- a/examples/anyFS/test/3.cert-in-progmem/3.cert-in-progmem.ino +++ b/examples/anyFS/test/3.cert-in-progmem/3.cert-in-progmem.ino @@ -27,13 +27,14 @@ const char* description = "PROGMEM example with security"; const char* fota_debug_fmt = R"DBG_FMT( -***************** STAGE %i ***************** +***************** STAGE %s ***************** Description : %s Firmware type : %s - Firmware version : %i + Firmware version : %i.%i.%i Signature check : %s TLS Cert check : %s + Compression : %s ******************************************** @@ -78,7 +79,18 @@ void setup_wifi() void setup() { Serial.begin(115200); - Serial.printf( fota_debug_fmt, firmware_version_major, description, firmware_name, firmware_version_major, check_signature?"Enabled":"Disabled", disable_security?"Disabled":"Enabled" ); + + Serial.printf( fota_debug_fmt, + "3", + description, + firmware_name, + firmware_version_major, + firmware_version_minor, + firmware_version_patch, + check_signature ?"Enabled":"Disabled", + disable_security ?"Disabled":"Enabled", + FOTA.zlibSupported() ?"Enabled":"Disabled" + ); { auto cfg = FOTA.getConfig(); diff --git a/examples/anyFS/test/4.cert-in-littlefs/4.cert-in-littlefs.ino b/examples/anyFS/test/4.cert-in-littlefs/4.cert-in-littlefs.ino index 52f26a2..a75cac0 100644 --- a/examples/anyFS/test/4.cert-in-littlefs/4.cert-in-littlefs.ino +++ b/examples/anyFS/test/4.cert-in-littlefs/4.cert-in-littlefs.ino @@ -25,13 +25,14 @@ const char* description = "LittleFS example with enforced security"; const char* fota_debug_fmt = R"DBG_FMT( -***************** STAGE %i ***************** +***************** STAGE %s ***************** Description : %s Firmware type : %s - Firmware version : %i + Firmware version : %i.%i.%i Signature check : %s TLS Cert check : %s + Compression : %s ******************************************** @@ -75,7 +76,18 @@ void setup_wifi() void setup() { Serial.begin(115200); - Serial.printf( fota_debug_fmt, firmware_version_major, description, firmware_name, firmware_version_major, check_signature?"Enabled":"Disabled", disable_security?"Disabled":"Enabled" ); + + Serial.printf( fota_debug_fmt, + "4", + description, + firmware_name, + firmware_version_major, + firmware_version_minor, + firmware_version_patch, + check_signature ?"Enabled":"Disabled", + disable_security ?"Disabled":"Enabled", + FOTA.zlibSupported() ?"Enabled":"Disabled" + ); // Provide filesystem with root_ca.pem to validate server certificate if( ! LittleFS.begin( false ) ) { Serial.println("LittleFS Mounting failed, aborting!"); diff --git a/examples/anyFS/test/5.sig-in-progmem/5.sig-in-progmem.ino b/examples/anyFS/test/5.sig-in-progmem/5.sig-in-progmem.ino index 379e171..a6f2c33 100644 --- a/examples/anyFS/test/5.sig-in-progmem/5.sig-in-progmem.ino +++ b/examples/anyFS/test/5.sig-in-progmem/5.sig-in-progmem.ino @@ -28,13 +28,14 @@ const char* description = "PROGMEM example with enforced security"; const char* fota_debug_fmt = R"DBG_FMT( -***************** STAGE %i ***************** +***************** STAGE %s ***************** Description : %s Firmware type : %s - Firmware version : %i + Firmware version : %i.%i.%i Signature check : %s TLS Cert check : %s + Compression : %s ******************************************** @@ -76,7 +77,18 @@ void setup_wifi() void setup() { Serial.begin(115200); - Serial.printf( fota_debug_fmt, firmware_version_major, description, firmware_name, firmware_version_major, check_signature?"Enabled":"Disabled", disable_security?"Disabled":"Enabled" ); + + Serial.printf( fota_debug_fmt, + "5", + description, + firmware_name, + firmware_version_major, + firmware_version_minor, + firmware_version_patch, + check_signature ?"Enabled":"Disabled", + disable_security ?"Disabled":"Enabled", + FOTA.zlibSupported() ?"Enabled":"Disabled" + ); { auto cfg = FOTA.getConfig();