Merge branch 'rolling' of https://github.com/caco3/AI-on-the-edge-device into rolling

# Conflicts:
#	.github/workflows/build.yaml

.github/workflows/build.yaml

@@ -53,8 +53,8 @@ jobs:
 #########################################################################################
   pack-for-OTA-v1:
     # Old OTA concept
-    # firmware__*.zip needs to be unpacked before attaching to the release (the artifact uploader always zipps the files)!
-    # The bin filename is allowed to contain versioning information.
+    # firmware__*.zip needs to be unpacked before attaching to the release!
+    # The bin filename can contain versioning.
     runs-on: ubuntu-latest
     needs: build
 
@@ -84,6 +84,7 @@ jobs:
     - name: Upload Firmware artifact (old OTA concept)
       uses: actions/upload-artifact@v3
       with:
+#        name: "firmware__${{ github.ref_name }}__(${{ steps.vars.outputs.sha_short }})__(extract_before_upload__only_needed_for_migration_from_11.3.1)"
         name: "firmware__(extract_before_upload)__${{ github.ref_name }}__(${{ steps.vars.outputs.sha_short }})"
         path: ./dist_old_ota/*
 
@@ -99,7 +100,7 @@ jobs:
 ## Pack for new OTA (v2)
 #########################################################################################
   pack-for-OTA-v2:
-    # New OTA concept (> v11.3.1)
+    # New OTA concept
     # update__version.zip file with following content:
     #  - /firmware.bin
     #  - (optional) /html/*
@@ -129,9 +130,21 @@ jobs:
       run: |
         mkdir -p ./dist
         cp "./code/.pio/build/esp32cam/firmware.bin" "dist/firmware.bin"
+
+#    - name: Upload update.zip Artifact (Firmware only)
+#      uses: actions/upload-artifact@v3
+#      with:
+#        name: "update_firmware_only__${{ github.ref_name }}_(${{ steps.vars.outputs.sha_short }})"
+#        path: ./dist/*
         
     - name: Add Web UI to dist
       run: cp -r ./sd-card/html ./dist/
+
+#    - name: Upload update.zip artifact (Firmware + Web UI)
+#      uses: actions/upload-artifact@v3
+#      with:
+#        name: "update_firmware+webinterface__${{ github.ref_name }}_(${{ steps.vars.outputs.sha_short }})"
+#        path: ./dist/*
         
     - name: Add CNN to dist
       run: |
@@ -149,17 +162,14 @@ jobs:
       uses: actions/cache@v3
       with:
         path: |
-          ./code/.pio/build/esp32cam/firmware.bin
-          ./code/.pio/build/esp32cam/partitions.bin
-          ./code/.pio/build/esp32cam/bootloader.bin
-          ./sd-card/html/version.txt
           ./dist
         key: ${{ github.run_number }}-pack-for-OTA-v2
 
 
 
+
 #########################################################################################
-## Pack for a fresh install (USB flashing)
+## Pack for a fresh install (USB flashing) (initial_esp32_setup)
 #########################################################################################
   pack-for-fresh-install:
     # creates old style binaries for fresh installation (backward compatible to wiki)
@@ -179,6 +189,32 @@ jobs:
           ./sd-card/html/version.txt
         key: ${{ github.run_number }}
 
+    - name: Prepare artifacts for release
+      run: |
+        mkdir -p firmware
+        rm -rf firmware/*.zip
+        mkdir -p release
+        # copy builds to firmware folder
+        cp -f "./code/.pio/build/esp32cam/firmware.bin" "firmware/firmware.bin"
+        cp -f "./code/.pio/build/esp32cam/bootloader.bin" "firmware/bootloader.bin"
+        cp -f "./code/.pio/build/esp32cam/partitions.bin" "firmware/partitions.bin"
+        zip -r ./firmware/sd-card.zip sd-card
+        cd ./firmware
+  
+    - name: Upload initial_esp32_setup.zip artifact (Firmware + Bootloader + Partitions + Web UI)
+      uses: actions/upload-artifact@v3
+      with:
+        name: "initial_esp32_setup__${{ github.ref_name }}_(${{ steps.vars.outputs.sha_short }})"
+        path: ./firmware
+
+    - name: Store generated files in cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ./firmware
+        key: ${{ github.run_number }}-pack-for-fresh-install
+
+
 
 
 #########################################################################################
@@ -186,93 +222,94 @@ jobs:
 #########################################################################################
   release: 
     runs-on: ubuntu-latest
-    needs: pack-for-OTA-v2
-
+    needs: [pack-for-OTA-v2, pack-for-fresh-install]
+    if: startsWith(github.ref, 'refs/tags/') 
+      
     steps:
-      - uses: actions/checkout@v2
-
-      - name: Get generated files from cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ./code/.pio/build/esp32cam/firmware.bin
-            ./code/.pio/build/esp32cam/partitions.bin
-            ./code/.pio/build/esp32cam/bootloader.bin
-            ./sd-card/html/version.txt
+    - uses: actions/checkout@v2
+
+    - name: Get generated files from cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ./code/.pio/build/esp32cam/firmware.bin
+          ./code/.pio/build/esp32cam/partitions.bin
+          ./code/.pio/build/esp32cam/bootloader.bin
+          ./sd-card/html/version.txt
+        key: ${{ github.run_number }}
+
+    # import the changes from 
+    - name: Get generated files from cache
+      uses: actions/cache@v3
+      with:
+        path: |
             ./dist
-          key: ${{ github.run_number }}-pack-for-OTA-v2
-
-      - name: Set Variables
-        id: vars
-        run: |
-          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
-
-
-      - name: Prepare artifacts for release
-        run: |
-          ls -la ./dist
-          mkdir -p firmware
-          rm -rf firmware/*
-          mkdir -p release
-          # copy builds to firmware folder
-          cp -f "./code/.pio/build/esp32cam/firmware.bin" "firmware/firmware.bin"
-          cp -f "./code/.pio/build/esp32cam/bootloader.bin" "firmware/bootloader.bin"
-          cp -f "./code/.pio/build/esp32cam/partitions.bin" "firmware/partitions.bin"
-          zip -r ./firmware/sd-card.zip sd-card
-          # create a update.zip like "update__rolling"
-          cd ./dist
-          zip -r ../release/update.zip .
-          cd ../firmware
-          zip -r ../release/initial_esp32_setup.zip .
-          
+        key: ${{ github.run_number }}-pack-for-OTA-v2
 
-      - name: Upload initial_esp32_setup.zip artifact (Firmware + Bootloader + Partitions + Web UI)
-        uses: actions/upload-artifact@v3
-        with:
-          name: "initial_esp32_setup__${{ github.ref_name }}_(${{ steps.vars.outputs.sha_short }})"
-          path: ./firmware
+    # import cached artifacts from pack-for-fresh-install
+    - name: Get generated files from cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ./firmware
+        key: ${{ github.run_number }}-pack-for-fresh-install
+
+    - name: Set Variables
+      id: vars
+      run: |
+        echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
 
-          
-      # extract the version used in next step
-      - id: get_version
-        if: startsWith(github.ref, 'refs/tags/') 
-        uses: battila7/get-version-action@v2
-
-      # the changelog [unreleased] will now be changed to the release version
-      - name: Update changelog
-        uses: thomaseizinger/keep-a-changelog-new-release@v1
-        if: startsWith(github.ref, 'refs/tags/') 
-        with:
-          changelogPath: Changelog.md
-          version: ${{ steps.get_version.outputs.version-without-v }}
-              
-      # the release notes will be extracted from changelog 
-      - name: Extract release notes
-        id: extract-release-notes
-        if: startsWith(github.ref, 'refs/tags/') 
-        uses: ffurrer2/extract-release-notes@v1
-        with:
-            changelog_file: Changelog.md
-            
 
-      # Releases should only be created on master by tagging the last commit.
-      # all artifacts in firmware folder pushed to the release
-      - name: Release
-        uses: softprops/action-gh-release@v1
-        if: startsWith(github.ref, 'refs/tags/')
-        with:
-          name:  ${{ steps.get_version.outputs.version-without-v }}
-          body: ${{ steps.extract-release-notes.outputs.release_notes }}
-          files: |
-            release/*   
+    - name: Prepare artifacts for release
+      run: |
+        mkdir -p release
+        # create a update.zip like "update__rolling"
+        cd ./dist
+        zip -r ../release/update.zip .
+        cd ../firmware
+        zip -r ../release/initial_esp32_setup.zip .
+        
+        
+    # extract the version used in next step
+    - id: get_version
+      if: startsWith(github.ref, 'refs/tags/') 
+      uses: battila7/get-version-action@v2
+
+    # the changelog [unreleased] will now be changed to the release version
+    - name: Update changelog
+      uses: thomaseizinger/keep-a-changelog-new-release@v1
+      if: startsWith(github.ref, 'refs/tags/') 
+      with:
+        changelogPath: Changelog.md
+        version: ${{ steps.get_version.outputs.version-without-v }}
             
+    # the release notes will be extracted from changelog 
+    - name: Extract release notes
+      id: extract-release-notes
+      if: startsWith(github.ref, 'refs/tags/') 
+      uses: ffurrer2/extract-release-notes@v1
+      with:
+          changelog_file: Changelog.md
+          
 
-      # Commit&Push Changelog to master branch. Must be manually merged back to rolling
-      - name: Commit changes and push changes
-        if: startsWith(github.ref, 'refs/tags/') 
-        run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
-          git add Changelog.md
-          git commit Changelog.md -m "Update Changelog.md for ${{github.event.inputs.versionIncrement}} release"
-          git push origin HEAD:master
+    # Releases should only be created on master by tagging the last commit.
+    # all artifacts in firmware folder pushed to the release
+    - name: Release
+      uses: softprops/action-gh-release@v1
+      if: startsWith(github.ref, 'refs/tags/')
+      with:
+        name:  ${{ steps.get_version.outputs.version-without-v }}
+        body: ${{ steps.extract-release-notes.outputs.release_notes }}
+        files: |
+          release/*   
+          
+
+    # Commit&Push Changelog to master branch. Must be manually merged back to rolling
+    - name: Commit changes and push changes
+      if: startsWith(github.ref, 'refs/tags/') 
+      run: |
+        git config user.name github-actions
+        git config user.email github-actions@github.com
+        git add Changelog.md
+        git commit Changelog.md -m "Update Changelog.md for ${{github.event.inputs.versionIncrement}} release"
+        git push origin HEAD:master

Changelog.md

@@ -5,12 +5,87 @@
 ### Added
 
 -   automatic release creation
--   newest firmware of rolling branch now uploaded to <https://github.com/jomjol/AI-on-the-edge-device/actions>
--   \#1068 new safer and easier update mechanismn. Use only the update.zip of the release for firmware, html and new models. 
+-   newest firmware of rolling branch now uploaded to <https://github.com/jomjol/AI-on-the-edge-device/actions> (developers only)
+- \#1068 New update mechanism: 
+  - handling of all files (zip, tfl, tflite, bin) within in one updload
+  - using the update.zip from release page <https://github.com/jomjol/AI-on-the-edge-device/releases>
+  - status (upload, processing, ...) displayed
+  - auto suggestion for reboot (or not in case of web ui update only)
 
+### Changed
+- Integrated version info better into the html (main page, logfile)
+- Updated menue 
+- Update tflite, esp32-cam-master, esp-nn (as of today 20220924) 
 ### Fixed
 
--   \#1029 wrong change of checkDigitConsistency now working like releases before 11.3.1 
+- \#1029 wrong change of checkDigitConsistency now working like releases before 11.3.1 
+- Spelling corrections (**[cristianmitran](https://github.com/cristianmitran)**) 
+- Integrated version info better into the html (main page, logfile)
+
+
+### Removed
+- Remove `/firmware` from GitHub. 
+  - If you want to get the latest firmware and html files, please download from the automated (build action)[https://github.com/jomjol/AI-on-the-edge-device/actions] or (release page)[https://github.com/jomjol/AI-on-the-edge-device/releases]
+
+## [11.3.1] - (2022-09-17)
+Intermediate Digits
+
+- **ATTENTION**: 
+  - first update the 'firmware.bin' and ensure that the new version is running
+
+  - Only afterwards update the 'html.zip'
+  
+  - Otherwise the downwards compatibility of the new counter clockwise feature is not given and you end in a reboot loop, that needs manual flashing!
+  
+
+
+- **NEW v11.3.1**: corrected corrupted asset `firmware.bin`
+- Increased precision (more than 6-7 digits)
+- Implements Counter Clockwise Analog Pointers
+- Improved post processing algorithm
+- Debugging: intensive use of testcases
+- MQTT: improved handling, extended logging, automated reconnect
+- HTML: Backup Option for Configuration
+- HTML: Improved Reboot
+- HTML: Update WebUI (Reboot, Infos, CPU Temp, RSSI)
+- This version is largely also based on the work of **[caco3](https://github.com/caco3)**,  **[adellafave](https://github.com/adellafave)**,  **[haverland](https://github.com/haverland)**,  **[stefanbode](https://github.com/stefanbode)**, **[PLCHome](https://github.com/PLCHome)**
+
+## [11.2.0] -  (2022-08-28)
+Intermediate Digits
+
+- Updated Tensorflow / TFlite to newest tflite (version as of 2022-07-27)
+- Updated analog neural network file (`ana-cont_11.3.0_s2.tflite` - default, `ana-class100_0120_s1_q.tflite`)
+- Updated digital neural network file (`dig-cont_0570_s3.tflite` - default, `dig-class100_0120_s2_q.tflite`)
+
+- Added automated filtering of tflite-file in the graphical configuration (thanks to @**[caco3](https://github.com/caco3)**)
+- Updated consistency algorithm & test cases
+- HTML: added favicon and system name, Improved reboot dialog  (thanks to @**[caco3](https://github.com/caco3)**)
+
+## [11.1.1] -  (2022-08-22)
+Intermediate Digits
+
+- New and improved consistency check (especially with analog and digital counters mixed)
+- Bug Fix: digital counter algorithm
+
+## [11.0.1] - (2022-08-18)
+Intermediate Digits
+
+- **NEW v11.0.1**: Bug Fix InfluxDB configuration (only update of html.zip necessary)
+
+- Implementation of new CNN types to detect intermediate values of digits with rolling numbers
+
+  - By default the old algo (0, 1, ..., 9, "N") is active (due to the limited types of digits trained so far)
+  - Activation can be done by selection a tflite file with the new trained model in the 'config.ini'
+  - **Details can be found in the [wiki](https://github.com/jomjol/AI-on-the-edge-device/wiki/Neural-Network-Types)** (different types, trained image types, naming convention)
+
+- Updated  neural network files (and adaption to new naming convention)
+
+- Published a tool to download and combine log files - **Thanks to **
+
+  - Files see ['/tools/logfile-tool'](tbd), How-to see [wiki](https://github.com/jomjol/AI-on-the-edge-device/wiki/Gasmeter-Log-Downloader)
+
+- Bug Fix: InfluxDB enabling in grahic configuration
+
 
 ## [10.6.2] - (2022-07-24)
 

code/components/jomjol_fileserver_ota/server_file.cpp

@@ -54,7 +54,7 @@ extern "C" {
 
 
 /* Scratch buffer size */
-#define SCRATCH_BUFSIZE  8192 
+#define SCRATCH_BUFSIZE  4096 
 
 struct file_server_data {
     /* Base path of file storage */

code/components/jomjol_fileserver_ota/server_ota.cpp

@@ -303,7 +303,7 @@ esp_err_t handler_ota_update(httpd_req_t *req)
 
     LogFile.WriteToFile("handler_ota_update");    
     char _query[200];
-    char _filename[30];
+    char _filename[100];
     char _valuechar[30];    
     std::string fn = "/sdcard/firmware/";
     bool _file_del = false;
@@ -319,12 +319,12 @@ esp_err_t handler_ota_update(httpd_req_t *req)
             _task = std::string(_valuechar);
         }
 
-        if (httpd_query_key_value(_query, "file", _filename, 30) == ESP_OK)
+        if (httpd_query_key_value(_query, "file", _filename, 100) == ESP_OK)
         {
             fn.append(_filename);
             printf("File: "); printf(fn.c_str()); printf("\n");            
         }
-        if (httpd_query_key_value(_query, "delete", _filename, 30) == ESP_OK)
+        if (httpd_query_key_value(_query, "delete", _filename, 100) == ESP_OK)
         {
             fn.append(_filename);
             _file_del = true;

firmware/README.md

@@ -1,2 +1,6 @@
 # Firmware
 The firmware got moved to the [Release page](https://github.com/jomjol/AI-on-the-edge-device/releases).
+
+# Installation Guide
+
+You find the complete installation guide at <https://github.com/jomjol/AI-on-the-edge-device/wiki/Installation>