Calculate and validate MD5 on upload (#3590)

* added md5 library

* added MD5 calculation of uploaded file. And return JSON string instead of fileserver

* .

* .

* .

* .

* .

* .

* .

* .

* .

* Add fallback for older firmware

---------

Co-authored-by: CaCO3 <caco@ruinelli.ch>

code/components/jomjol_fileserver_ota/md5.cpp

@@ -0,0 +1,226 @@
+/* Src: https://github.com/Zunawe/md5-c, commit: f3529b6
+ * License: Unlicense */
+/*
+ * Derived from the RSA Data Security, Inc. MD5 Message-Digest Algorithm
+ * and modified slightly to be functionally identical but condensed into control structures.
+ */
+
+#include "md5.h"
+
+/*
+ * Constants defined by the MD5 algorithm
+ */
+#define A 0x67452301
+#define B 0xefcdab89
+#define C 0x98badcfe
+#define D 0x10325476
+
+static uint32_t S[] = {7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22,
+                       5,  9, 14, 20, 5,  9, 14, 20, 5,  9, 14, 20, 5,  9, 14, 20,
+                       4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23,
+                       6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21};
+
+static uint32_t K[] = {0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee,
+                       0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501,
+                       0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be,
+                       0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821,
+                       0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa,
+                       0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8,
+                       0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed,
+                       0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a,
+                       0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c,
+                       0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70,
+                       0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x04881d05,
+                       0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665,
+                       0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039,
+                       0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1,
+                       0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1,
+                       0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391};
+
+/*
+ * Padding used to make the size (in bits) of the input congruent to 448 mod 512
+ */
+static uint8_t PADDING[] = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+                            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+
+/*
+ * Bit-manipulation functions defined by the MD5 algorithm
+ */
+#define F(X, Y, Z) ((X & Y) | (~X & Z))
+#define G(X, Y, Z) ((X & Z) | (Y & ~Z))
+#define H(X, Y, Z) (X ^ Y ^ Z)
+#define I(X, Y, Z) (Y ^ (X | ~Z))
+
+/*
+ * Rotates a 32-bit word left by n bits
+ */
+uint32_t rotateLeft(uint32_t x, uint32_t n){
+    return (x << n) | (x >> (32 - n));
+}
+
+
+/*
+ * Initialize a context
+ */
+void md5Init(MD5Context *ctx){
+    ctx->size = (uint64_t)0;
+
+    ctx->buffer[0] = (uint32_t)A;
+    ctx->buffer[1] = (uint32_t)B;
+    ctx->buffer[2] = (uint32_t)C;
+    ctx->buffer[3] = (uint32_t)D;
+}
+
+/*
+ * Add some amount of input to the context
+ *
+ * If the input fills out a block of 512 bits, apply the algorithm (md5Step)
+ * and save the result in the buffer. Also updates the overall size.
+ */
+void md5Update(MD5Context *ctx, uint8_t *input_buffer, size_t input_len){
+    uint32_t input[16];
+    unsigned int offset = ctx->size % 64;
+    ctx->size += (uint64_t)input_len;
+
+    // Copy each byte in input_buffer into the next space in our context input
+    for(unsigned int i = 0; i < input_len; ++i){
+        ctx->input[offset++] = (uint8_t)*(input_buffer + i);
+
+        // If we've filled our context input, copy it into our local array input
+        // then reset the offset to 0 and fill in a new buffer.
+        // Every time we fill out a chunk, we run it through the algorithm
+        // to enable some back and forth between cpu and i/o
+        if(offset % 64 == 0){
+            for(unsigned int j = 0; j < 16; ++j){
+                // Convert to little-endian
+                // The local variable `input` our 512-bit chunk separated into 32-bit words
+                // we can use in calculations
+                input[j] = (uint32_t)(ctx->input[(j * 4) + 3]) << 24 |
+                           (uint32_t)(ctx->input[(j * 4) + 2]) << 16 |
+                           (uint32_t)(ctx->input[(j * 4) + 1]) <<  8 |
+                           (uint32_t)(ctx->input[(j * 4)]);
+            }
+            md5Step(ctx->buffer, input);
+            offset = 0;
+        }
+    }
+}
+
+/*
+ * Pad the current input to get to 448 bytes, append the size in bits to the very end,
+ * and save the result of the final iteration into digest.
+ */
+void md5Finalize(MD5Context *ctx){
+    uint32_t input[16];
+    unsigned int offset = ctx->size % 64;
+    unsigned int padding_length = offset < 56 ? 56 - offset : (56 + 64) - offset;
+
+    // Fill in the padding and undo the changes to size that resulted from the update
+    md5Update(ctx, PADDING, padding_length);
+    ctx->size -= (uint64_t)padding_length;
+
+    // Do a final update (internal to this function)
+    // Last two 32-bit words are the two halves of the size (converted from bytes to bits)
+    for(unsigned int j = 0; j < 14; ++j){
+        input[j] = (uint32_t)(ctx->input[(j * 4) + 3]) << 24 |
+                   (uint32_t)(ctx->input[(j * 4) + 2]) << 16 |
+                   (uint32_t)(ctx->input[(j * 4) + 1]) <<  8 |
+                   (uint32_t)(ctx->input[(j * 4)]);
+    }
+    input[14] = (uint32_t)(ctx->size * 8);
+    input[15] = (uint32_t)((ctx->size * 8) >> 32);
+
+    md5Step(ctx->buffer, input);
+
+    // Move the result into digest (convert from little-endian)
+    for(unsigned int i = 0; i < 4; ++i){
+        ctx->digest[(i * 4) + 0] = (uint8_t)((ctx->buffer[i] & 0x000000FF));
+        ctx->digest[(i * 4) + 1] = (uint8_t)((ctx->buffer[i] & 0x0000FF00) >>  8);
+        ctx->digest[(i * 4) + 2] = (uint8_t)((ctx->buffer[i] & 0x00FF0000) >> 16);
+        ctx->digest[(i * 4) + 3] = (uint8_t)((ctx->buffer[i] & 0xFF000000) >> 24);
+    }
+}
+
+/*
+ * Step on 512 bits of input with the main MD5 algorithm.
+ */
+void md5Step(uint32_t *buffer, uint32_t *input){
+    uint32_t AA = buffer[0];
+    uint32_t BB = buffer[1];
+    uint32_t CC = buffer[2];
+    uint32_t DD = buffer[3];
+
+    uint32_t E;
+
+    unsigned int j;
+
+    for(unsigned int i = 0; i < 64; ++i){
+        switch(i / 16){
+            case 0:
+                E = F(BB, CC, DD);
+                j = i;
+                break;
+            case 1:
+                E = G(BB, CC, DD);
+                j = ((i * 5) + 1) % 16;
+                break;
+            case 2:
+                E = H(BB, CC, DD);
+                j = ((i * 3) + 5) % 16;
+                break;
+            default:
+                E = I(BB, CC, DD);
+                j = (i * 7) % 16;
+                break;
+        }
+
+        uint32_t temp = DD;
+        DD = CC;
+        CC = BB;
+        BB = BB + rotateLeft(AA + E + K[i] + input[j], S[i]);
+        AA = temp;
+    }
+
+    buffer[0] += AA;
+    buffer[1] += BB;
+    buffer[2] += CC;
+    buffer[3] += DD;
+}
+
+/*
+ * Functions that run the algorithm on the provided input and put the digest into result.
+ * result should be able to store 16 bytes.
+ */
+void md5String(char *input, uint8_t *result){
+    MD5Context ctx;
+    md5Init(&ctx);
+    md5Update(&ctx, (uint8_t *)input, strlen(input));
+    md5Finalize(&ctx);
+
+    memcpy(result, ctx.digest, 16);
+}
+
+
+void md5File(FILE *file, uint8_t *result){
+    void *input_buffer = malloc(1024);
+    size_t input_size = 0;
+
+    MD5Context ctx;
+    md5Init(&ctx);
+
+    while((input_size = fread(input_buffer, 1, 1024, file)) > 0){
+        md5Update(&ctx, (uint8_t *)input_buffer, input_size);
+    }
+
+    md5Finalize(&ctx);
+
+    free(input_buffer);
+
+    memcpy(result, ctx.digest, 16);
+}

code/components/jomjol_fileserver_ota/md5.h

@@ -0,0 +1,28 @@
+/* Src: https://github.com/Zunawe/md5-c, commit: f3529b6
+ * License: Unlicense */
+#pragma once
+
+#ifndef MD5_H
+#define MD5_H
+
+#include <stdio.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+
+typedef struct{
+    uint64_t size;        // Size of input in bytes
+    uint32_t buffer[4];   // Current accumulation of hash
+    uint8_t input[64];    // Input to be used in the next step
+    uint8_t digest[16];   // Result of algorithm
+}MD5Context;
+
+void md5Init(MD5Context *ctx);
+void md5Update(MD5Context *ctx, uint8_t *input, size_t input_len);
+void md5Finalize(MD5Context *ctx);
+void md5Step(uint32_t *buffer, uint32_t *input);
+
+void md5String(char *input, uint8_t *result);
+void md5File(FILE *file, uint8_t *result);
+
+#endif // MD5_H

code/components/jomjol_fileserver_ota/server_file.cpp

@@ -36,6 +36,7 @@ extern "C" {
 #include "MainFlowControl.h"
 
 #include "server_help.h"
+#include "md5.h"
 #ifdef ENABLE_MQTT
     #include "interface_mqtt.h"
 #endif //ENABLE_MQTT
@@ -610,6 +611,8 @@ static esp_err_t upload_post_handler(httpd_req_t *req)
     FILE *fd = NULL;
     struct stat file_stat;
 
+    ESP_LOGI(TAG, "uri: %s", req->uri);
+
     httpd_resp_set_hdr(req, "Access-Control-Allow-Origin", "*");
 
     /* Skip leading "/upload" from URI to get filename */
@@ -711,43 +714,76 @@ static esp_err_t upload_post_handler(httpd_req_t *req)
     LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "File saved: " + string(filename));
     ESP_LOGI(TAG, "File reception completed");
 
-    std::string directory = std::string(filepath);
-	size_t zw = directory.find("/");
-	size_t found = zw;
-	while (zw != std::string::npos)
-	{
-		zw = directory.find("/", found+1);  
-		if (zw != std::string::npos)
-			found = zw;
-	}
-
-    int start_fn = strlen(((struct file_server_data *)req->user_ctx)->base_path);
-    ESP_LOGD(TAG, "Directory: %s, start_fn: %d, found: %d", directory.c_str(), start_fn, found);
-	directory = directory.substr(start_fn, found - start_fn + 1);
-    directory = "/fileserver" + directory;
-//    ESP_LOGD(TAG, "Directory danach 2: %s", directory.c_str());
-
-    /* Redirect onto root to see the updated file list */
-    if (strcmp(filename, "/config/config.ini") == 0 ||
-        strcmp(filename, "/config/ref0.jpg") == 0 ||
-        strcmp(filename, "/config/ref0_org.jpg") == 0 ||
-        strcmp(filename, "/config/ref1.jpg") == 0 ||
-        strcmp(filename, "/config/ref1_org.jpg") == 0 ||
-        strcmp(filename, "/config/reference.jpg") == 0 ||
-        strcmp(filename, "/img_tmp/ref0.jpg") == 0 ||
-        strcmp(filename, "/img_tmp/ref0_org.jpg") == 0 ||
-        strcmp(filename, "/img_tmp/ref1.jpg") == 0 ||
-        strcmp(filename, "/img_tmp/ref1_org.jpg") == 0 ||
-        strcmp(filename, "/img_tmp/reference.jpg") == 0 ) 
-    { 
-        httpd_resp_set_status(req, HTTPD_200); // Avoid reloading of folder content
-    }
-    else {
-        httpd_resp_set_status(req, "303 See Other"); // Reload folder content after upload
+    string s = req->uri;
+    if (isInString(s, "?md5")) {
+        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Calculate and return MD5 sum...");
+        
+        fd = fopen(filepath, "r");
+        if (!fd) {
+            LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to open file for reading: " + string(filepath));
+            /* Respond with 500 Internal Server Error */
+            httpd_resp_send_err(req, HTTPD_500_INTERNAL_SERVER_ERROR, "Failed to open file for reading");
+            return ESP_FAIL;
+        }
+
+        uint8_t result[16];
+        string md5hex = "";
+        string response = "{\"md5\":";
+        char hex[3];
+
+        md5File(fd, result);
+        fclose(fd);
+
+        for (int i = 0; i < sizeof(result); i++) {
+            snprintf(hex, sizeof(hex), "%02x", result[i]);
+            md5hex.append(hex);
+        }
+
+        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "MD5 of " + string(filepath) + ": " + md5hex);
+        response.append("\"" + md5hex + "\"");
+        response.append("}");
+
+        httpd_resp_sendstr(req, response.c_str());
     }
+    else {  // Return file server page
+        std::string directory = std::string(filepath);
+        size_t zw = directory.find("/");
+        size_t found = zw;
+        while (zw != std::string::npos)
+        {
+            zw = directory.find("/", found+1);  
+            if (zw != std::string::npos)
+                found = zw;
+        }
 
-    httpd_resp_set_hdr(req, "Location", directory.c_str());
-    httpd_resp_sendstr(req, "File uploaded successfully");
+        int start_fn = strlen(((struct file_server_data *)req->user_ctx)->base_path);
+        ESP_LOGD(TAG, "Directory: %s, start_fn: %d, found: %d", directory.c_str(), start_fn, found);
+        directory = directory.substr(start_fn, found - start_fn + 1);
+        directory = "/fileserver" + directory;
+    //    ESP_LOGD(TAG, "Directory danach 2: %s", directory.c_str());
+
+        /* Redirect onto root to see the updated file list */
+        if (strcmp(filename, "/config/config.ini") == 0 ||
+            strcmp(filename, "/config/ref0.jpg") == 0 ||
+            strcmp(filename, "/config/ref0_org.jpg") == 0 ||
+            strcmp(filename, "/config/ref1.jpg") == 0 ||
+            strcmp(filename, "/config/ref1_org.jpg") == 0 ||
+            strcmp(filename, "/config/reference.jpg") == 0 ||
+            strcmp(filename, "/img_tmp/ref0.jpg") == 0 ||
+            strcmp(filename, "/img_tmp/ref0_org.jpg") == 0 ||
+            strcmp(filename, "/img_tmp/ref1.jpg") == 0 ||
+            strcmp(filename, "/img_tmp/ref1_org.jpg") == 0 ||
+            strcmp(filename, "/img_tmp/reference.jpg") == 0 ) 
+        { 
+            httpd_resp_set_status(req, HTTPD_200); // Avoid reloading of folder content
+        }
+        else {
+            httpd_resp_set_status(req, "303 See Other"); // Reload folder content after upload
+        }
+
+        httpd_resp_set_hdr(req, "Location", directory.c_str());
+        httpd_resp_sendstr(req, "File uploaded successfully");
+    }
 
     return ESP_OK;
 }

sd-card/html/ota_page.html

@@ -25,6 +25,7 @@
     </style>
 
     <link href="firework.css?v=$COMMIT_HASH" rel="stylesheet">
+    <script src="md5.min.js?v=$COMMIT_HASH"></script>
     <script type="text/javascript" src="jquery-3.6.0.min.js?v=$COMMIT_HASH"></script>
     <script type="text/javascript" src="common.js?v=$COMMIT_HASH"></script>
     <script type="text/javascript" src="firework.js?v=$COMMIT_HASH"></script>
@@ -142,7 +143,7 @@
 
         function doRebootAfterUpdate() {
             var xhttp = new XMLHttpRequest();
-            xhttp.open("GET", "/reboot", true);
+            xhttp.open("GET", domainname + "/reboot", true);
             xhttp.send();
         }
 
@@ -169,15 +170,35 @@
                 }
             };
 
-            var _toDo = domainname + "/ota?task=emptyfirmwaredir";
-            xhttp.open("GET", _toDo, true);
+            var url = domainname + "/ota?task=emptyfirmwaredir";
+            xhttp.open("GET", url, true);
             xhttp.send();
         }
 
 
-        function extract() {
-            document.getElementById("status").innerText = "Status: Processing on device...";
+        function validateMd5(md5_on_device, callback) {
+            const reader = new FileReader();
+
+            reader.onload = (event) => {
+                const fileContent = event.target.result;
+                md5_on_webbrowser = md5(fileContent);
+                console.log("MD5 on device: " + md5_on_device + ", MD5 on web browser: " + md5_on_webbrowser);
+                if (md5_on_device == md5_on_webbrowser) {
+                    console.log("MD5 values are equal");
+                    callback(true);
+                }
+                else {
+                    console.log("MD5 values are NOT equal!");
+                    callback(false);
+                }
+            }
+
+            var fileInput = document.getElementById("file_selector").files;
+            reader.readAsArrayBuffer(fileInput[0]);
+        }
+
 
+        function extract() {
             var xhttp = new XMLHttpRequest();
             /* first delete the old firmware */	
             xhttp.onreadystatechange = function() {
@@ -186,9 +207,9 @@
                         document.cookie = "page=overview.html?v=$COMMIT_HASH" + "; path=/"; // Make sure after the reboot we go to the overview page
 
                         if (xhttp.responseText.startsWith("reboot")) { // Reboot required
-                            console.log("Upload completed, the device will now restart and install the update!");
+                            console.log("The device will now reboot and install the update!");
                             document.getElementById("status").innerText = "Status: Installing...";
-                            firework.launch('Upload completed, the device will now restart and install the update', 'success', 5000);
+                            firework.launch('Upload completed and validated. The device will now reboot and install the update', 'success', 5000);
                         
                             /* Tell it to reboot */
                             doRebootAfterUpdate();
@@ -228,8 +249,8 @@
 
             var file_name = document.getElementById("file_selector").value;
             filePath = file_name.split(/[\\\/]/).pop();
-            var _toDo = domainname + "/ota?task=update&file=" + filePath;
-            xhttp.open("GET", _toDo, true);
+            var url = domainname + "/ota?task=update&file=" + filePath;
+            xhttp.open("GET", url, true);
             xhttp.send();
         }
 
@@ -242,7 +263,7 @@
         function upload() {
             document.getElementById("status").innerText = "Status: Uploading...";
 
-            var upload_path = "/upload/firmware/" + filePath;
+            var url = domainname + "/upload/firmware/" + filePath + "?md5";
 
             var file = _("file_selector").files[0];
             var formdata = new FormData();
@@ -253,7 +274,7 @@
             ajax.addEventListener("error", errorHandler, false);
             ajax.addEventListener("abort", abortHandler, false);
 
-            ajax.open("POST", upload_path);
+            ajax.open("POST", url);
             ajax.send(file);
         }
 
@@ -263,16 +284,43 @@
                     " MB of " + (event.total / 1024/ 1024).toFixed(2) + " MB";
             var percent = (event.loaded / event.total) * 100;
             _("progressBar").value = Math.round(percent);
-            _("status").innerHTML = "Status: " + Math.round(percent) + "% uploaded. Please wait...";
+
+            if (Math.round(percent) == 100) {
+                _("progressBar").value = 0; //will clear progress bar after successful upload
+                _("loaded_n_total").innerHTML = "";
+                _("status").innerHTML = "Status: Upload completed. Validating file...";
+            }
+            else {
+                _("status").innerHTML = "Status: " + Math.round(percent) + "% uploaded...";
+            }
         }
 
 
         function completeHandler(event) {
-            _("status").innerHTML = "Status: " + event.target.responseText;
-            _("progressBar").value = 0; //will clear progress bar after successful upload
-            _("loaded_n_total").innerHTML = "";
-        
-            extract();
+            console.log("Upload completed");
+            console.log("Response: " + event.target.responseText);
+
+            try {
+                md5_on_device = JSON.parse(event.target.responseText).md5;
+                validateMd5(md5_on_device, (result) => {
+                    if (result == true) {
+                        _("status").innerHTML = "Status: The uploaded file is valid, installing it...";
+                        extract();
+                    }
+                    else {
+                        _("status").innerHTML = "Status: The file got corrupted! Please upload it again!";
+                        firework.launch('Upload failed, the file got corrupted! Please upload it again!', 'danger', 30000);
+                        document.getElementById("start_OTA_button").disabled = false;
+                    }
+                });
+            }
+            catch (e) {
+                // If the firmware is to old, it will return the file sever page instead of the JSON object with the MD5 sum.
+                // In juch case just proceed to keep legacy support.
+                console.log("It seems to be a legacy firmware, installing the update without validation!");
+                _("status").innerHTML = "Status: It seems to be a legacy firmware, installing the update without validation...";
+                extract();
+            }
         }