slow down reboot loops (#1396)

* slow down constant reboots caused by the flow. With this, after a restart due to exception/panic the first round gets delayed by 5 minutes

* retry InitCam() if it failed

* restart after 5 minutes if NTP init failed

* .

Co-authored-by: CaCO3 <caco@ruinelli.ch>

code/components/jomjol_tfliteclass/server_tflite.cpp

@@ -713,6 +713,13 @@ void task_autodoFlow(void *pvParameter)
 {
     int64_t fr_start, fr_delta_ms;
 
+    if (esp_reset_reason() == ESP_RST_PANIC) {
+        LogFile.WriteToFile(ESP_LOG_WARN, TAG, "Restarted due to an Exception/panic! Postponing first round start by 5 minutes to allow for an OTA or to fetch the log!"); 
+        LogFile.WriteToFile(ESP_LOG_WARN, TAG, "Setting logfile level to DEBUG until the next reboot!");
+        LogFile.setLogLevel(ESP_LOG_DEBUG);
+        vTaskDelay(60*5000 / portTICK_RATE_MS); // Wait 5 minutes to give time to do an OTA or fetch the log
+    }
+
     ESP_LOGD(TAG, "task_autodoFlow: start");
     doInit();
     gpio_handler_init();

code/components/jomjol_time_sntp/time_sntp.cpp

@@ -19,7 +19,7 @@ static const char *TAG = "SNTP";
 
 time_t bootTime;
 
-static void obtain_time(void);
+static bool obtain_time(void);
 static void initialize_sntp(void);
 static void logNtpStatus(sntp_sync_status_t status);
 
@@ -52,19 +52,23 @@ std::string gettimestring(const char * frm)
     return result;
 }
 
-void setup_time()
+bool setup_time()
 {
     time_t now;
     struct tm timeinfo;
     time(&now);
     localtime_r(&now, &timeinfo);
     char strftime_buf[64];
+    bool success = true;
 
     // Is time set? If not, tm_year will be (1970 - 1900).
     if (!getTimeIsSet()) {
         ESP_LOGI(TAG, "Time is not set yet. Getting time over NTP.");
         initialize_sntp();
-        obtain_time();
+        if (!obtain_time()) {
+            success = false;
+        }
+
         // update 'now' variable with current time
         time(&now);
 
@@ -79,6 +83,7 @@ void setup_time()
         strftime(strftime_buf, sizeof(strftime_buf), "%Y-%m-%d_%H:%M:%S", &timeinfo);
         ESP_LOGI(TAG, "Time is already set (%s)", strftime_buf);
     }
+    return success;
 }
 
 void setTimeZone(std::string _tzstring)
@@ -89,12 +94,14 @@ void setTimeZone(std::string _tzstring)
     LogFile.WriteToFile(ESP_LOG_INFO, TAG, _tzstring);
 }
 
-static void obtain_time(void)
+static bool obtain_time(void)
 {
     time_t now = 0;
     struct tm timeinfo = {};
     int retry = 0;
     const int retry_count = 10;
+    bool success = true;
+
     time(&now);
     localtime_r(&now, &timeinfo);    
 
@@ -104,6 +111,7 @@ static void obtain_time(void)
 
         if (retry == retry_count) {
             ESP_LOGW(TAG, "NTP time fetching seems to take longer, will check again on next round!"); // The NTP client will automatically retry periodically!
+            success = false;
             break;
         }
 
@@ -119,6 +127,7 @@ static void obtain_time(void)
     
     time(&now);
     localtime_r(&now, &timeinfo);
+    return success;
 }
 
 

code/components/jomjol_time_sntp/time_sntp.h

@@ -12,7 +12,7 @@
 // #include "nvs_flash.h"
 #include "esp_sntp.h"
 
-void setup_time(void);
+bool setup_time(void);
 
 std::string gettimestring(const char * frm);
 std::string ConvertTimeToString(time_t _time, const char * frm);

code/main/main.cpp

@@ -146,11 +146,12 @@ void task_NoSDBlink(void *pvParameter)
 extern "C" void app_main(void)
 {
     TickType_t xDelay;
+    bool initSucessful = true;
 
     ESP_LOGI(TAG, "\n\n\n\n\n"); // Add mark on log to see when it restarted
     
     PowerResetCamera();
-    esp_err_t cam = Camera.InitCam();
+    esp_err_t camStatus = Camera.InitCam();
     Camera.LightOnOff(false);
     xDelay = 2000 / portTICK_PERIOD_MS;
     ESP_LOGD(TAG, "After camera initialization: sleep for: %ldms", (long) xDelay);
@@ -218,7 +219,12 @@ extern "C" void app_main(void)
     xDelay = 2000 / portTICK_PERIOD_MS;
     ESP_LOGD(TAG, "main: sleep for: %ldms", (long) xDelay);
     vTaskDelay( xDelay );   
-    setup_time();
+
+    if (!setup_time()) {
+        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "NTP Initialization failed. Will restart in 5 minutes!");
+        initSucessful = false;
+    }
+
     setBootTime();
 
     LogFile.WriteToFile(ESP_LOG_INFO, TAG, "=============================================================================================");
@@ -238,22 +244,34 @@ extern "C" void app_main(void)
     if (_hsize < 4000000)
     {
         std::string _zws = "Not enough PSRAM available. Expected 4.194.304 MByte - available: " + std::to_string(_hsize);
-        _zws = _zws + "\nEither not initialzed, too small (2MByte only) or not present at all. Firmware cannot start!!";
+        _zws = _zws + "\nEither not initialized, too small (2MByte only) or not present at all. Firmware cannot start!!";
         LogFile.WriteToFile(ESP_LOG_ERROR, TAG, _zws);
-    } else {
-        if (cam != ESP_OK) {
-                LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to initialize camera module. "
-                        "Check that your camera module is working and connected properly.");
-        } else {
-// Test Camera            
+    } else { // Bad Camera Status, retry init   
+        if (camStatus != ESP_OK) {
+            LogFile.WriteToFile(ESP_LOG_WARN, TAG, "Failed to initialize camera module, retrying...");
+
+            PowerResetCamera();
+            esp_err_t camStatus = Camera.InitCam();
+            Camera.LightOnOff(false);
+            xDelay = 2000 / portTICK_PERIOD_MS;
+            ESP_LOGD(TAG, "After camera initialization: sleep for: %ldms", (long) xDelay);
+            vTaskDelay( xDelay ); 
+
+            if (camStatus != ESP_OK) {
+                LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Failed to initialize camera module. Will restart in 5 minutes!");
+                LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Check that your camera module is working and connected properly!");
+                initSucessful = false;
+            }
+        } else { // Test Camera            
             camera_fb_t * fb = esp_camera_fb_get();
             if (!fb) {
-                LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Camera cannot be initialzed. "
-                        "System will reboot.");
-                doReboot();
+                LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Camera cannot be initialzed. Will restart in 5 minutes!");
+                initSucessful = false;
+            }
+            else {
+                esp_camera_fb_return(fb);   
+                Camera.LightOnOff(false);
             }
-            esp_camera_fb_return(fb);   
-            Camera.LightOnOff(false);
         }
     }
 
@@ -263,7 +281,7 @@ extern "C" void app_main(void)
     ESP_LOGD(TAG, "main: sleep for: %ldms", (long) xDelay*10);
     vTaskDelay( xDelay ); 
 
-    ESP_LOGD(TAG, "starting server");
+    ESP_LOGD(TAG, "starting servers");
 
     server = start_webserver();   
     register_server_camera_uri(server); 
@@ -277,8 +295,17 @@ extern "C" void app_main(void)
     ESP_LOGD(TAG, "vor reg server main");
     register_server_main_uri(server, "/sdcard");
 
-    ESP_LOGD(TAG, "vor dotautostart");
-    TFliteDoAutoStart();
-
+    if (initSucessful) {
+        LogFile.WriteToFile(ESP_LOG_INFO, TAG, "Initialization completed successfully!");
+        ESP_LOGD(TAG, "vor do autostart");
+        TFliteDoAutoStart();
+    }
+    else { // Initialization failed
+        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Will restart in 5 minutes!");
+        vTaskDelay(60*4000 / portTICK_RATE_MS); // Wait 4 minutes to give time to do an OTA or fetch the log
+        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Will restart in 1 minute!");
+        vTaskDelay(60*1000 / portTICK_RATE_MS); // Wait 1 minute to give time to do an OTA or fetch the log
+        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Initialization failed. Will restart now!");
+        doReboot();
+    }
 }
-