Store preprocessed image with ROI to RAM (#1809)

* tflite model loading: error handling

* FlowAlignment: error handling

* CImageBasis+GetJPGStream : error handling

* store preprocessed ALG_ROI.jpg to memory

* Update

code/components/jomjol_controlcamera/ClassControllCamera.cpp

@@ -236,18 +236,18 @@ void CCamera::EnableAutoExposure(int flash_duration)
 
 esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
 {
-    string ftype;
-    int _size;
-
-    uint8_t *zwischenspeicher = NULL;
-
+	#ifdef DEBUG_DETAIL_ON
+	    LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - Start");
+	#endif
 
+    _Image->EmptyImage(); //Delete previous stored raw image -> black image
+    
+    #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG__SHOW_TAKE_IMAGE_PROCESS
+        tfliteflow.SetNewAlgROI(false);
+    #endif
+    
     LEDOnOff(true);
 
-#ifdef DEBUG_DETAIL_ON
-    LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - Start");
-#endif
-
     if (delay > 0) 
     {
         LightOnOff(true);
@@ -255,18 +255,18 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
         vTaskDelay( xDelay );
     }
 
-#ifdef DEBUG_DETAIL_ON
-    LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After LightOn");
-#endif
+	#ifdef DEBUG_DETAIL_ON
+	    LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After LightOn");
+	#endif
 
     camera_fb_t * fb = esp_camera_fb_get();
     esp_camera_fb_return(fb);        
     fb = esp_camera_fb_get();
     if (!fb) {
-        ESP_LOGE(TAG, "CaptureToBasisImage: Capture Failed");
         LEDOnOff(false);
         LightOnOff(false);
 
+        ESP_LOGE(TAG, "CaptureToBasisImage: Capture Failed");
         LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "is not working anymore (CCamera::CaptureToBasisImage) - most probably caused by a hardware problem (instablility, ...). "
                 "System will reboot.");
         doReboot();
@@ -279,20 +279,13 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
         loadNextDemoImage(fb);
     }
 
-    _size = fb->len;
-    zwischenspeicher = (uint8_t*) malloc(_size);
-    if (!zwischenspeicher)
-    {
-        ESP_LOGE(TAG, "Insufficient memory space for image in function CaptureToBasisImage()");
-        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Insufficient memory space for image in function CaptureToBasisImage()");
-    }
-    for (int i = 0; i < _size; ++i)
-        *(zwischenspeicher + i) = *(fb->buf + i);
+    CImageBasis* _zwImage = new CImageBasis();
+    _zwImage->LoadFromMemory(fb->buf, fb->len);
     esp_camera_fb_return(fb);        
 
-#ifdef DEBUG_DETAIL_ON
-    LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After fb_get");
-#endif
+    #ifdef DEBUG_DETAIL_ON
+        LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After fb_get");
+    #endif
 
     LEDOnOff(false);  
 
@@ -302,15 +295,9 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
 //    TickType_t xDelay = 1000 / portTICK_PERIOD_MS;     
 //    vTaskDelay( xDelay );  // wait for power to recover
     
-    uint8_t * buf = NULL;
-
-    CImageBasis _zwImage;
-    _zwImage.LoadFromMemory(zwischenspeicher, _size);
-    free(zwischenspeicher);
-
-#ifdef DEBUG_DETAIL_ON
-    LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After LoadFromMemory");
-#endif
+    #ifdef DEBUG_DETAIL_ON
+        LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After LoadFromMemory");
+    #endif
 
     stbi_uc* p_target;
     stbi_uc* p_source;    
@@ -318,31 +305,31 @@ esp_err_t CCamera::CaptureToBasisImage(CImageBasis *_Image, int delay)
     int width = image_width;
     int height = image_height;
 
-#ifdef DEBUG_DETAIL_ON
-    std::string _zw = "Targetimage: " + std::to_string((int) _Image->rgb_image) + " Size: " + std::to_string(_Image->width) + ", " + std::to_string(_Image->height);
-    _zw = _zw + " _zwImage: " + std::to_string((int) _zwImage.rgb_image)  + " Size: " + std::to_string(_zwImage.width) + ", " + std::to_string(_zwImage.height);
-    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, _zw);
-#endif
+    #ifdef DEBUG_DETAIL_ON
+        std::string _zw = "Targetimage: " + std::to_string((int) _Image->rgb_image) + " Size: " + std::to_string(_Image->width) + ", " + std::to_string(_Image->height);
+        _zw = _zw + " _zwImage: " + std::to_string((int) _zwImage.rgb_image)  + " Size: " + std::to_string(_zwImage.width) + ", " + std::to_string(_zwImage.height);
+        LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, _zw);
+    #endif
 
     for (int x = 0; x < width; ++x)
         for (int y = 0; y < height; ++y)
         {
             p_target = _Image->rgb_image + (channels * (y * width + x));
-            p_source = _zwImage.rgb_image + (channels * (y * width + x));
+            p_source = _zwImage->rgb_image + (channels * (y * width + x));
             p_target[0] = p_source[0];
             p_target[1] = p_source[1];
             p_target[2] = p_source[2];
         }
 
-#ifdef DEBUG_DETAIL_ON
-    LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After Copy To Target");
-#endif
+    #ifdef DEBUG_DETAIL_ON
+        LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - After Copy To Target");
+    #endif
 
-    free(buf);
+    delete _zwImage;
 
-#ifdef DEBUG_DETAIL_ON
-    LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - Done");
-#endif
+    #ifdef DEBUG_DETAIL_ON
+        LogFile.WriteHeapInfo("CCamera::CaptureToBasisImage - Done");
+    #endif
 
     return ESP_OK;    
 }
@@ -376,21 +363,21 @@ esp_err_t CCamera::CaptureToFile(std::string nm, int delay)
     }
     LEDOnOff(false);    
 
-#ifdef DEBUG_DETAIL_ON    
-    ESP_LOGD(TAG, "w %d, h %d, size %d", fb->width, fb->height, fb->len);
-#endif
+    #ifdef DEBUG_DETAIL_ON    
+        ESP_LOGD(TAG, "w %d, h %d, size %d", fb->width, fb->height, fb->len);
+    #endif
 
     nm = FormatFileName(nm);
 
-#ifdef DEBUG_DETAIL_ON
-    ESP_LOGD(TAG, "Save Camera to: %s", nm.c_str());
-#endif
+    #ifdef DEBUG_DETAIL_ON
+        ESP_LOGD(TAG, "Save Camera to: %s", nm.c_str());
+    #endif
 
     ftype = toUpper(getFileType(nm));
 
-#ifdef DEBUG_DETAIL_ON
-    ESP_LOGD(TAG, "Filetype: %s", ftype.c_str());
-#endif
+    #ifdef DEBUG_DETAIL_ON
+        ESP_LOGD(TAG, "Filetype: %s", ftype.c_str());
+    #endif
 
     uint8_t * buf = NULL;
     size_t buf_len = 0;   
@@ -604,7 +591,7 @@ void CCamera::GetCameraParameter(httpd_req_t *req, int &qual, framesize_t &resol
             if (qual < 0)
                 qual = 0;
         }
-    };
+    }
 }
 
 
@@ -674,6 +661,7 @@ bool CCamera::getCameraInitSuccessful()
     return CameraInitSuccessful;
 }
 
+
 std::vector<std::string> demoFiles;
 
 void CCamera::useDemoMode()

code/components/jomjol_flowcontroll/ClassFlowAlignment.cpp

@@ -1,6 +1,7 @@
 #include "ClassFlowAlignment.h"
 #include "ClassFlowMakeImage.h"
 #include "ClassFlow.h"
+#include "server_tflite.h"
 
 #include "CRotateImage.h"
 #include "esp_log.h"
@@ -29,6 +30,9 @@ void ClassFlowAlignment::SetInitialParameter(void)
     AlignAndCutImage = NULL;
     ImageBasis = NULL;
     ImageTMP = NULL;
+    #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG 
+    AlgROI = (ImageData*)heap_caps_malloc(sizeof(ImageData), MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM);
+    #endif
     previousElement = NULL;
     disabled = false;
     SAD_criteria = 0.05;
@@ -161,6 +165,25 @@ string ClassFlowAlignment::getHTMLSingleStep(string host)
 
 bool ClassFlowAlignment::doFlow(string time) 
 {
+    #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
+        if (!AlgROI)  // AlgROI needs to be allocated before ImageTMP to avoid heap fragmentation
+        {
+            AlgROI = (ImageData*)heap_caps_realloc(AlgROI, sizeof(ImageData), MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM);     
+            if (!AlgROI) 
+            {
+                LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't allocate AlgROI");
+                LogFile.WriteHeapInfo("ClassFlowAlignment-doFlow");
+                tfliteflow.SetNewAlgROI(false); // continue flow only with alg.jpg (no ROIs available)
+            }
+        }
+
+        if (AlgROI)
+        {
+            ImageBasis->writeToMemoryAsJPG((ImageData*)AlgROI, 90);
+            tfliteflow.SetNewAlgROI(true);
+        }
+    #endif
+
     if (!ImageTMP) 
     {
         ImageTMP = new CImageBasis(ImageBasis);
@@ -214,8 +237,15 @@ bool ClassFlowAlignment::doFlow(string time)
         SaveReferenceAlignmentValues();
     }
 
-    if (SaveAllFiles) AlignAndCutImage->SaveToFile(FormatFileName("/sdcard/img_tmp/alg.jpg"));
-
+    #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
+        if (AlgROI) {
+            DrawRef(ImageTMP);
+            tfliteflow.DigitalDrawROI(ImageTMP);
+            tfliteflow.AnalogDrawROI(ImageTMP);
+            ImageTMP->writeToMemoryAsJPG((ImageData*)AlgROI, 90);
+        }
+    #endif
+    
     if (SaveAllFiles)
     {
         if (initialflip)
@@ -224,7 +254,8 @@ bool ClassFlowAlignment::doFlow(string time)
             ImageTMP->width = ImageTMP->height;
             ImageTMP->height = _zw;
         }
-        DrawRef(ImageTMP);
+
+        AlignAndCutImage->SaveToFile(FormatFileName("/sdcard/img_tmp/alg.jpg"));
         ImageTMP->SaveToFile(FormatFileName("/sdcard/img_tmp/alg_roi.jpg"));
     }
 

code/components/jomjol_flowcontroll/ClassFlowAlignment.h

@@ -34,6 +34,9 @@ protected:
 
 public:
     CImageBasis *ImageBasis, *ImageTMP;
+    #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG 
+    ImageData *AlgROI;
+    #endif
     
     ClassFlowAlignment(std::vector<ClassFlow*>* lfc);
 

code/components/jomjol_flowcontroll/ClassFlowControll.cpp

@@ -141,6 +141,27 @@ t_CNNType ClassFlowControll::GetTypeAnalog()
 }
 
 
+#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
+void ClassFlowControll::DigitalDrawROI(CImageBasis *_zw)
+{
+    if (flowdigit)
+        flowdigit->DrawROI(_zw);
+}
+
+
+void ClassFlowControll::AnalogDrawROI(CImageBasis *_zw)
+{
+    if (flowanalog)
+        flowanalog->DrawROI(_zw);
+}
+
+
+void ClassFlowControll::SetNewAlgROI(bool _value)
+{
+    bNewAlgROI = _value;
+}
+#endif
+
 
 #ifdef ENABLE_MQTT
 string ClassFlowControll::GetMQTTMainTopic()
@@ -667,25 +688,47 @@ esp_err_t ClassFlowControll::GetJPGStream(std::string _fn, httpd_req_t *req)
         }
     }
     else if (_fn == "alg_roi.jpg") {
-        _send = new CImageBasis(flowalignment->ImageBasis);
-
-        if (_send->ImageOkay()) {
-            if (flowalignment) flowalignment->DrawRef(_send);
-            if (flowdigit) flowdigit->DrawROI(_send);
-            if (flowanalog) flowanalog->DrawROI(_send);
-            _sendDelete = true; // delete temporary _send element after sending
-        }
-        else {
-            LogFile.WriteToFile(ESP_LOG_WARN, TAG, "ClassFlowControll::GetJPGStream: Not enough memory to create alg_roi.jpg -> alg.jpg is going to be served!");
-
-            if (flowalignment && flowalignment->ImageBasis->ImageOkay()) {
-                _send = flowalignment->ImageBasis;  
+        #ifdef ALGROI_LOAD_FROM_MEM_AS_JPG      // no CImageBasis needed to create alg_roi.jpg (ca. 790kB less RAM)
+            if (bNewAlgROI) {
+                if (flowalignment && flowalignment->AlgROI) {
+                    httpd_resp_set_type(req, "image/jpeg");
+                    result = httpd_resp_send(req, (const char *)flowalignment->AlgROI->data, flowalignment->AlgROI->size);
+                }
+                else {
+                    LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "ClassFlowControll::GetJPGStream: alg_roi.jpg cannot be served");
+                    return ESP_FAIL;
+                }
+            }
+            else {
+                if (flowalignment && flowalignment->ImageBasis->ImageOkay()) {
+                    _send = flowalignment->ImageBasis;
+                }
+                else {
+                    httpd_resp_send(req, NULL, 0);
+                    return ESP_OK;
+                }
+            }
+        #else
+            _send = new CImageBasis(flowalignment->ImageBasis);
+			
+            if (_send->ImageOkay()) {
+                if (flowalignment) flowalignment->DrawRef(_send);
+                if (flowdigit) flowdigit->DrawROI(_send);
+                if (flowanalog) flowanalog->DrawROI(_send);
+                _sendDelete = true; // delete temporary _send element after sending
             }
             else {
-                httpd_resp_send(req, NULL, 0);
-                return ESP_OK;
+                LogFile.WriteToFile(ESP_LOG_WARN, TAG, "ClassFlowControll::GetJPGStream: Not enough memory to create alg_roi.jpg -> alg.jpg is going to be served!");
+                
+                if (flowalignment && flowalignment->ImageBasis->ImageOkay()) {
+                    _send = flowalignment->ImageBasis;
+                }
+                else {
+                    httpd_resp_send(req, NULL, 0);
+                    return ESP_OK;
+                }
             }
-        }
+        #endif
     }
     else {
         std::vector<HTMLInfo*> htmlinfo;

code/components/jomjol_flowcontroll/ClassFlowControll.h

@@ -38,6 +38,9 @@ protected:
 	void SetInitialParameter(void);	
 	std::string aktstatus;
 	int aktRunNr;
+	#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG 
+	bool bNewAlgROI = false;
+	#endif
 
 public:
 	void InitFlow(std::string config);
@@ -51,11 +54,20 @@ public:
 	bool ReadParameter(FILE* pfile, string& aktparamgraph);	
 	string getJSON();
 	string getNumbersName();
+	#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG 
+	void SetNewAlgROI(bool _value);
+	#endif
 
 	string TranslateAktstatus(std::string _input);
-#ifdef ENABLE_MQTT
+
+	#ifdef ENABLE_MQTT
 	string GetMQTTMainTopic();
-#endif //ENABLE_MQTT
+	#endif //ENABLE_MQTT
+
+	#ifdef ALGROI_LOAD_FROM_MEM_AS_JPG
+	void DigitalDrawROI(CImageBasis *_zw);
+	void AnalogDrawROI(CImageBasis *_zw);
+	#endif
 
 	esp_err_t GetJPGStream(std::string _fn, httpd_req_t *req);
 	esp_err_t SendRawJPG(httpd_req_t *req);
@@ -71,9 +83,10 @@ public:
 
 	t_CNNType GetTypeDigital();
 	t_CNNType GetTypeAnalog();
-#ifdef ENABLE_MQTT
+	
+	#ifdef ENABLE_MQTT
 	bool StartMQTTService();
-#endif //ENABLE_MQTT
+	#endif //ENABLE_MQTT
 
 	int CleanTempFolder();
 

code/components/jomjol_image_proc/CImageBasis.cpp

@@ -25,9 +25,9 @@ uint8_t * CImageBasis::RGBImageLock(int _waitmaxsec)
 {
     if (islocked)
     {
-#ifdef DEBUG_DETAIL_ON   
-        ESP_LOGD(TAG, "Image is locked: sleep for: %ds", _waitmaxsec);
-#endif
+        #ifdef DEBUG_DETAIL_ON   
+                ESP_LOGD(TAG, "Image is locked: sleep for: %ds", _waitmaxsec);
+        #endif
         TickType_t xDelay;
         xDelay = 1000 / portTICK_PERIOD_MS;
         for (int i = 0; i <= _waitmaxsec; ++i)
@@ -84,6 +84,19 @@ ImageData* CImageBasis::writeToMemoryAsJPG(const int quality)
 }
 
 
+void CImageBasis::writeToMemoryAsJPG(ImageData* i, const int quality)
+{
+    ImageData* ii = new ImageData;
+    
+    RGBImageLock();
+    stbi_write_jpg_to_func(writejpghelp, ii, width, height, channels, rgb_image, quality);
+    RGBImageRelease();
+
+    memCopy((uint8_t*) ii, (uint8_t*) i, sizeof(ImageData));
+    delete ii;
+}
+
+
 struct SendJPGHTTP
 {
     httpd_req_t *req;
@@ -380,6 +393,28 @@ void CImageBasis::CreateEmptyImage(int _width, int _height, int _channels)
 }
 
 
+void CImageBasis::EmptyImage()
+{
+    #ifdef DEBUG_DETAIL_ON 
+        LogFile.WriteHeapInfo("CImageBasis::EmptyImage");
+    #endif
+
+    stbi_uc* p_source;
+
+    RGBImageLock();
+
+    for (int x = 0; x < width; ++x)
+        for (int y = 0; y < height; ++y)
+        {
+            p_source = rgb_image + (channels * (y * width + x));
+            for (int _channels = 0; _channels < channels; ++_channels)
+                p_source[_channels] = (uint8_t) 0;
+        }
+
+    RGBImageRelease();
+}
+
+
 void CImageBasis::LoadFromMemory(stbi_uc *_buffer, int len)
 {
     RGBImageLock();
@@ -503,8 +538,8 @@ CImageBasis::CImageBasis(std::string _image)
 
     #ifdef DEBUG_DETAIL_ON 
         std::string zw = "CImageBasis after load " + _image;
-    ESP_LOGD(TAG, "%s", zw.c_str());
-    ESP_LOGD(TAG, "w %d, h %d, b %d, c %d", width, height, bpp, channels);
+        ESP_LOGD(TAG, "%s", zw.c_str());
+        ESP_LOGD(TAG, "w %d, h %d, b %d, c %d", width, height, bpp, channels);
     #endif
 
     #ifdef DEBUG_DETAIL_ON 

code/components/jomjol_image_proc/CImageBasis.h

@@ -61,6 +61,7 @@ class CImageBasis
         void SetIndepended(){externalImage = false;};
 
         void CreateEmptyImage(int _width, int _height, int _channels);
+        void EmptyImage();
 
 
         CImageBasis();
@@ -74,7 +75,8 @@ class CImageBasis
 
         void LoadFromMemory(stbi_uc *_buffer, int len);
 
-        ImageData* writeToMemoryAsJPG(const int quality = 90);   
+        ImageData* writeToMemoryAsJPG(const int quality = 90);
+        void writeToMemoryAsJPG(ImageData* ii, const int quality = 90);
 
         esp_err_t SendJPGtoHTTP(httpd_req_t *req, const int quality = 90);   
 

code/components/jomjol_tfliteclass/CTfLiteClass.cpp

@@ -170,9 +170,9 @@ bool CTfLiteClass::LoadInputImageBasis(CImageBasis *rs)
                 input_data_ptr++;
             }
 
-#ifdef DEBUG_DETAIL_ON          
+    #ifdef DEBUG_DETAIL_ON 
         LogFile.WriteHeapInfo("CTfLiteClass::LoadInputImageBasis - done");
-#endif
+    #endif
 
     return true;
 }
@@ -191,21 +191,21 @@ bool CTfLiteClass::MakeAllocate()
 
     if (this->interpreter) 
     {
-    TfLiteStatus allocate_status = this->interpreter->AllocateTensors();
-    if (allocate_status != kTfLiteOk) {
-        TF_LITE_REPORT_ERROR(error_reporter, "AllocateTensors() failed");
-        LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "AllocateTensors() failed");
+        TfLiteStatus allocate_status = this->interpreter->AllocateTensors();
+        if (allocate_status != kTfLiteOk) {
+            TF_LITE_REPORT_ERROR(error_reporter, "AllocateTensors() failed");
+            LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "AllocateTensors() failed");
 
-    this->GetInputDimension();   
+            this->GetInputDimension();   
             return false;
-  }
+        }
     }
     else 
     {
         LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "new tflite::MicroInterpreter failed");
         LogFile.WriteHeapInfo("CTfLiteClass::MakeAllocate-new tflite::MicroInterpreter failed");
         return false;
-}
+    }
 
 
     #ifdef DEBUG_DETAIL_ON 

code/include/defines.h

@@ -88,6 +88,10 @@
     #define READOUT_TYPE_RAWVALUE 2
     #define READOUT_TYPE_ERROR 3
 
+    //ClassFlowControll: Serve alg_roi.jpg from memory as JPG
+    #define ALGROI_LOAD_FROM_MEM_AS_JPG // Load ALG_ROI.JPG as rendered JPG from RAM
+    #define ALGROI_LOAD_FROM_MEM_AS_JPG__SHOW_TAKE_IMAGE_PROCESS // Show take image image processing on webinterface (overview.html)
+
     //ClassFlowMQTT
     #define LWT_TOPIC        "connection"
     #define LWT_CONNECTED    "connected"