Add rate threshold parameter (#3195)

* still needs to be tested

https://github.com/jomjol/AI-on-the-edge-device/issues/3143

* Update ClassFlowPostProcessing.cpp

code formatting

* Update ClassFlowDefineTypes.h

code formatting

* Update ClassFlowPostProcessing.h

code formatting

* Update edit_config_template.html

* fix

* Update config.ini

* Update edit_config_template.html

* Updated param doc

* Rename parameters

* Update edit_config_template.html

* Update NUMBER.ChangeRateThreshold.md

* Update NUMBER.ChangeRateThreshold.md

---------

Co-authored-by: CaCO3 <caco3@ruinelli.ch>

code/components/jomjol_flowcontroll/ClassFlowDefineTypes.h

@@ -42,6 +42,7 @@ struct NumberPost {
     bool useMaxRateValue;       // consistencyChecksEnabled; enables consistency checks; uses maxRate and maxRateType
     t_RateType RateType;        // maxRateType; affects how the value of maxRate is used for comparing the current and previous value
     bool ErrorMessage;          // FIXME: not used; can be removed
+    int ChangeRateThreshold;  // threshold parameter for negative rate detection
     bool PreValueOkay;          // previousValueValid; indicates that the reading of the previous round has no errors
     bool AllowNegativeRates;    // allowNegativeRate; defines if the consistency checks allow negative rates between consecutive meter readings.
     bool checkDigitIncreaseConsistency; // extendedConsistencyCheck; performs an additional consistency check to avoid wrong readings

code/components/jomjol_flowcontroll/ClassFlowPostProcessing.cpp

@@ -484,6 +484,32 @@ void ClassFlowPostProcessing::handleMaxRateValue(string _decsep, string _value)
     }
 }
 
+void ClassFlowPostProcessing::handleChangeRateThreshold(string _decsep, string _value) {
+    string _digit, _decpos;
+    int _pospunkt = _decsep.find_first_of(".");
+    // ESP_LOGD(TAG, "Name: %s, Pospunkt: %d", _decsep.c_str(), _pospunkt);
+
+    if (_pospunkt > -1) {
+        _digit = _decsep.substr(0, _pospunkt);
+    }
+    else {
+        _digit = "default";
+    }
+
+    for (int j = 0; j < NUMBERS.size(); ++j) {
+        int _zwdc = 2;
+
+        if (isStringNumeric(_value)) {
+            _zwdc = std::stof(_value);
+        }
+
+        // Set to default first (if nothing else is set)
+        if ((_digit == "default") || (NUMBERS[j]->name == _digit)) {
+            NUMBERS[j]->ChangeRateThreshold = _zwdc;
+        }
+    }
+}
+
 bool ClassFlowPostProcessing::ReadParameter(FILE* pfile, string& aktparamgraph) {
     std::vector<string> splitted;
     int _n;
@@ -530,6 +556,10 @@ bool ClassFlowPostProcessing::ReadParameter(FILE* pfile, string& aktparamgraph)
         if ((toUpper(_param) == "PREVALUEUSE") && (splitted.size() > 1)) {
             PreValueUse = alphanumericToBoolean(splitted[1]);
         }
+		
+        if ((toUpper(_param) == "CHANGERATETHRESHOLD") && (splitted.size() > 1)) {
+            handleChangeRateThreshold(splitted[0], splitted[1]);
+        }
 	    
         if ((toUpper(_param) == "CHECKDIGITINCREASECONSISTENCY") && (splitted.size() > 1)) {
             if (alphanumericToBoolean(splitted[1])) {
@@ -627,6 +657,7 @@ void ClassFlowPostProcessing::InitNUMBERS() {
         _number->DecimalShiftInitial = 0;
         _number->isExtendedResolution = false;
         _number->AnalogDigitalTransitionStart=9.2;
+        _number->ChangeRateThreshold = 2;
 
         _number->FlowRateAct = 0; // m3 / min
         _number->PreValue = 0; // last value read out well
@@ -833,25 +864,28 @@ bool ClassFlowPostProcessing::doFlow(string zwtime) {
             ESP_LOGD(TAG, "After checkDigitIncreaseConsistency: Value %f", NUMBERS[j]->Value);
         #endif
 
-        if (!NUMBERS[j]->AllowNegativeRates) {
-            LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "handleAllowNegativeRate for device: " + NUMBERS[j]->name);
+        if (PreValueUse && NUMBERS[j]->PreValueOkay) {
+            if (NUMBERS[j]->Nachkomma > 0) {
+                double _difference1 = (NUMBERS[j]->PreValue - (NUMBERS[j]->ChangeRateThreshold / pow(10, NUMBERS[j]->Nachkomma)));
+                double _difference2 = (NUMBERS[j]->PreValue + (NUMBERS[j]->ChangeRateThreshold / pow(10, NUMBERS[j]->Nachkomma)));
+
+                if ((NUMBERS[j]->Value >= _difference1) && (NUMBERS[j]->Value <= _difference2)) {
+                    NUMBERS[j]->Value = NUMBERS[j]->PreValue;
+                    NUMBERS[j]->ReturnValue = std::to_string(NUMBERS[j]->PreValue);
+                }
+            }
+
+            if ((!NUMBERS[j]->AllowNegativeRates) && (NUMBERS[j]->Value < NUMBERS[j]->PreValue)) {
+                LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "handleAllowNegativeRate for device: " + NUMBERS[j]->name);
 					
-            if ((NUMBERS[j]->Value < NUMBERS[j]->PreValue)) {
-                // more debug if extended resolution is on, see #2447
-                if (NUMBERS[j]->isExtendedResolution) {
-                    LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Neg: value=" + std::to_string(NUMBERS[j]->Value) 
+                if ((NUMBERS[j]->Value < NUMBERS[j]->PreValue)) {
+                    // more debug if extended resolution is on, see #2447
+                    if (NUMBERS[j]->isExtendedResolution) {
+                        LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Neg: value=" + std::to_string(NUMBERS[j]->Value) 
                                                     + ", preValue=" + std::to_string(NUMBERS[j]->PreValue) 
                                                     + ", preToll=" + std::to_string(NUMBERS[j]->PreValue-(2/pow(10, NUMBERS[j]->Nachkomma))));
-                } 
-
-                // Include inaccuracy of 0.2 for isExtendedResolution.
-                if ((NUMBERS[j]->Value >= (NUMBERS[j]->PreValue-(2/pow(10, NUMBERS[j]->Nachkomma))) && NUMBERS[j]->isExtendedResolution)
-                    // not extended resolution allows -1 on the lowest digit  
-                   || (NUMBERS[j]->Value >= (NUMBERS[j]->PreValue-(1/pow(10, NUMBERS[j]->Nachkomma))) && !NUMBERS[j]->isExtendedResolution)) {
-                        NUMBERS[j]->Value = NUMBERS[j]->PreValue;
-                        NUMBERS[j]->ReturnValue = to_string(NUMBERS[j]->PreValue);
-                } 
-                else {
+                    } 
+
                     NUMBERS[j]->ErrorMessageText = NUMBERS[j]->ErrorMessageText + "Neg. Rate - Read: " + zwvalue + " - Raw: " + NUMBERS[j]->ReturnRawValue + " - Pre: " + RundeOutput(NUMBERS[j]->PreValue, NUMBERS[j]->Nachkomma) + " "; 
                     NUMBERS[j]->Value = NUMBERS[j]->PreValue;
                     NUMBERS[j]->ReturnValue = "";
@@ -863,48 +897,48 @@ bool ClassFlowPostProcessing::doFlow(string zwtime) {
                     continue;
                 }
             }
-        }
 
-        #ifdef SERIAL_DEBUG
-            ESP_LOGD(TAG, "After AllowNegativeRates: Value %f", NUMBERS[j]->Value);
-        #endif
+            #ifdef SERIAL_DEBUG
+                ESP_LOGD(TAG, "After AllowNegativeRates: Value %f", NUMBERS[j]->Value);
+            #endif
 
-        // LastValueTimeDifference = LastValueTimeDifference / 60;       // in minutes
-        LastPreValueTimeDifference = LastPreValueTimeDifference / 60; // in minutes
-        NUMBERS[j]->FlowRateAct = (NUMBERS[j]->Value - NUMBERS[j]->PreValue) / LastPreValueTimeDifference;
-        NUMBERS[j]->ReturnRateValue =  to_string(NUMBERS[j]->FlowRateAct);
+            // LastValueTimeDifference = LastValueTimeDifference / 60;       // in minutes
+            LastPreValueTimeDifference = LastPreValueTimeDifference / 60; // in minutes
+            NUMBERS[j]->FlowRateAct = (NUMBERS[j]->Value - NUMBERS[j]->PreValue) / LastPreValueTimeDifference;
+            NUMBERS[j]->ReturnRateValue =  to_string(NUMBERS[j]->FlowRateAct);
 
-        if (NUMBERS[j]->useMaxRateValue && PreValueUse && NUMBERS[j]->PreValueOkay) {
-            double _ratedifference;
+            if ((NUMBERS[j]->useMaxRateValue) && (NUMBERS[j]->Value != NUMBERS[j]->PreValue)) {
+                double _ratedifference;
 					
-            if (NUMBERS[j]->RateType == RateChange) {
-                _ratedifference = NUMBERS[j]->FlowRateAct;
-            }
-            else {
-                // TODO:
-                // Since I don't know if this is desired, I'll comment it out first.
-                // int roundDifference = (int)(round(LastPreValueTimeDifference / LastValueTimeDifference)); // calculate how many rounds have passed since NUMBERS[j]->timeLastPreValue was set
-                // _ratedifference = ((NUMBERS[j]->Value - NUMBERS[j]->PreValue) / ((int)(round(LastPreValueTimeDifference / LastValueTimeDifference)))); // Difference per round, as a safeguard in case a reading error(Neg. Rate - Read: or Rate too high - Read:) occurs in the meantime
-                _ratedifference = (NUMBERS[j]->Value - NUMBERS[j]->PreValue);
-            }
+                if (NUMBERS[j]->RateType == RateChange) {
+                    _ratedifference = NUMBERS[j]->FlowRateAct;
+                }
+                else {
+                    // TODO:
+                    // Since I don't know if this is desired, I'll comment it out first.
+                    // int roundDifference = (int)(round(LastPreValueTimeDifference / LastValueTimeDifference)); // calculate how many rounds have passed since NUMBERS[j]->timeLastPreValue was set
+                    // _ratedifference = ((NUMBERS[j]->Value - NUMBERS[j]->PreValue) / ((int)(round(LastPreValueTimeDifference / LastValueTimeDifference)))); // Difference per round, as a safeguard in case a reading error(Neg. Rate - Read: or Rate too high - Read:) occurs in the meantime
+                    _ratedifference = (NUMBERS[j]->Value - NUMBERS[j]->PreValue);
+                }
 
-            if (abs(_ratedifference) > abs(NUMBERS[j]->MaxRateValue)) {
-                NUMBERS[j]->ErrorMessageText = NUMBERS[j]->ErrorMessageText + "Rate too high - Read: " + RundeOutput(NUMBERS[j]->Value, NUMBERS[j]->Nachkomma) + " - Pre: " + RundeOutput(NUMBERS[j]->PreValue, NUMBERS[j]->Nachkomma) + " - Rate: " + RundeOutput(_ratedifference, NUMBERS[j]->Nachkomma);
-                NUMBERS[j]->Value = NUMBERS[j]->PreValue;
-                NUMBERS[j]->ReturnValue = "";
-                NUMBERS[j]->ReturnRateValue = "";
-                NUMBERS[j]->timeStampLastValue = imagetime;
+                if (abs(_ratedifference) > abs(NUMBERS[j]->MaxRateValue)) {
+                    NUMBERS[j]->ErrorMessageText = NUMBERS[j]->ErrorMessageText + "Rate too high - Read: " + RundeOutput(NUMBERS[j]->Value, NUMBERS[j]->Nachkomma) + " - Pre: " + RundeOutput(NUMBERS[j]->PreValue, NUMBERS[j]->Nachkomma) + " - Rate: " + RundeOutput(_ratedifference, NUMBERS[j]->Nachkomma);
+                    NUMBERS[j]->Value = NUMBERS[j]->PreValue;
+                    NUMBERS[j]->ReturnValue = "";
+                    NUMBERS[j]->ReturnRateValue = "";
+                    NUMBERS[j]->timeStampLastValue = imagetime;
 
-                string _zw = NUMBERS[j]->name + ": Raw: " + NUMBERS[j]->ReturnRawValue + ", Value: " + NUMBERS[j]->ReturnValue + ", Status: " + NUMBERS[j]->ErrorMessageText;
-                LogFile.WriteToFile(ESP_LOG_ERROR, TAG, _zw);
-                WriteDataLog(j);
-                continue;
+                    string _zw = NUMBERS[j]->name + ": Raw: " + NUMBERS[j]->ReturnRawValue + ", Value: " + NUMBERS[j]->ReturnValue + ", Status: " + NUMBERS[j]->ErrorMessageText;
+                    LogFile.WriteToFile(ESP_LOG_ERROR, TAG, _zw);
+                    WriteDataLog(j);
+                    continue;
+                }
             }
-        }
 
         #ifdef SERIAL_DEBUG
            ESP_LOGD(TAG, "After MaxRateCheck: Value %f", NUMBERS[j]->Value);
         #endif
+        }
         
         NUMBERS[j]->ReturnChangeAbsolute = RundeOutput(NUMBERS[j]->Value - NUMBERS[j]->PreValue, NUMBERS[j]->Nachkomma);
         NUMBERS[j]->PreValue = NUMBERS[j]->Value;
@@ -949,11 +983,9 @@ void ClassFlowPostProcessing::WriteDataLog(int _index) {
         digital = flowDigit->getReadoutRawString(_index);
     }
 	
-    LogFile.WriteToData(timezw, NUMBERS[_index]->name, 
-                        NUMBERS[_index]->ReturnRawValue, NUMBERS[_index]->ReturnValue, NUMBERS[_index]->ReturnPreValue, 
-                        NUMBERS[_index]->ReturnRateValue, NUMBERS[_index]->ReturnChangeAbsolute,
-                        NUMBERS[_index]->ErrorMessageText, 
-                        digital, analog);
+    LogFile.WriteToData(timezw, NUMBERS[_index]->name, NUMBERS[_index]->ReturnRawValue, NUMBERS[_index]->ReturnValue, NUMBERS[_index]->ReturnPreValue, 
+        NUMBERS[_index]->ReturnRateValue, NUMBERS[_index]->ReturnChangeAbsolute, NUMBERS[_index]->ErrorMessageText, digital, analog);
+
     ESP_LOGD(TAG, "WriteDataLog: %s, %s, %s, %s, %s", NUMBERS[_index]->ReturnRawValue.c_str(), NUMBERS[_index]->ReturnValue.c_str(), NUMBERS[_index]->ErrorMessageText.c_str(), digital.c_str(), analog.c_str());
 }
 

code/components/jomjol_flowcontroll/ClassFlowPostProcessing.h

@@ -21,11 +21,9 @@ protected:
     bool ErrorMessage;
     bool IgnoreLeadingNaN;          // SPECIAL CASE for User Gustl ???
 
-
     ClassFlowCNNGeneral* flowAnalog;
     ClassFlowCNNGeneral* flowDigit;    
 
-
     string FilePreValue;
 
     ClassFlowTakeImage *flowTakeImage;
@@ -43,19 +41,16 @@ protected:
     void handleMaxRateType(string _decsep, string _value);
     void handleAnalogDigitalTransitionStart(string _decsep, string _value);
     void handleAllowNegativeRate(string _decsep, string _value);
+    void handleChangeRateThreshold(string _decsep, string _value);
     
     std::string GetStringReadouts(general);
 
     void WriteDataLog(int _index);
 
-
-
-
 public:
     bool PreValueUse;
     std::vector<NumberPost*> NUMBERS;
 
-
     ClassFlowPostProcessing(std::vector<ClassFlow*>* lfc, ClassFlowCNNGeneral *_analog, ClassFlowCNNGeneral *_digit);
     virtual ~ClassFlowPostProcessing(){};
     bool ReadParameter(FILE* pfile, string& aktparamgraph);

param-docs/parameter-pages/PostProcessing/NUMBER.ChangeRateThreshold.md

@@ -0,0 +1,25 @@
+# Parameter `<NUMBER>.ChangeRateThreshold`
+Default Value: `2`
+
+Range: `1` .. `9`.
+
+Threshold parameter for change rate detection.<br>
+This parameter is intended to compensate for small reading fluctuations that occur when the meter does not change its value for a long time (e.g. at night) or slightly turns backwards. This can eg. happen on watermeters.
+
+It is only applied to the last digit of the read value (See example below).
+If the read value is within PreValue +/- Threshold, no further calculation is carried out and the Value/Prevalue remains at the old value.
+
+Example:
+
+    Smallest ROI provides value for 0.000x
+    ChangeRateThreshold = 2
+  
+    Extended Resolution disabled:
+    PreValue: 123.456'7 >>> Threshold = +/- 0.000'2
+	Comparative value >>> max = 123.456'9 and min = 123.456'5
+	
+    Extended Resolution enabled:
+    PreValue: 123.456'78 >>> Threshold = +/- 0.000'02
+	Comparative value >>> max = 123.456'80 and min = 123.456'76
+
+![](img/ChangeRateThreshold.png)

sd-card/config/config.ini

@@ -66,6 +66,7 @@ main.ana4 155 328 92 92 false
 [PostProcessing]
 main.DecimalShift = 0
 main.AnalogDigitalTransitionStart = 9.2
+main.ChangeRateThreshold = 2
 PreValueUse = true
 PreValueAgeStartup = 720
 main.AllowNegativeRates = false

sd-card/html/edit_config_template.html

@@ -969,6 +969,19 @@
 			<td>$TOOLTIP_PostProcessing_NUMBER.MaxRateType</td>
 		</tr>
 
+		<tr>
+			<td class="indent2">
+				<input type="checkbox" id="PostProcessing_ChangeRateThreshold_enabled" value="1"  onclick = 'InvertEnableItem("PostProcessing", "ChangeRateThreshold")' unchecked >
+				<label for=PostProcessing_ChangeRateThreshold_enabled><class id="PostProcessing_ChangeRateThreshold_text" style="color:black;">Change Rate Threshold</class></label>
+			</td>
+			<td>
+				<input required type="number" id="PostProcessing_ChangeRateThreshold_value1" step="1" min="1" max="9" value="2"
+					oninput="(!validity.rangeUnderflow||(value=1)) && (!validity.rangeOverflow||(value=9)) && 
+						(!validity.stepMismatch||(value=parseInt(this.value)));">
+			</td>
+			<td>$TOOLTIP_PostProcessing_NUMBER.ChangeRateThreshold</td>
+		</tr>
+
 		<tr>
 			<td class="indent2">
 				<label><class id="PostProcessing_ExtendedResolution_text" style="color:black;">Extended Resolution</class></label>
@@ -2163,6 +2176,7 @@ function UpdateInputIndividual(sel) {
         // ReadParameter(param, "PostProcessing", "PreValueUse", false, NUNBERSAkt);
         ReadParameter(param, "PostProcessing", "DecimalShift", true, NUNBERSAkt);
         ReadParameter(param, "PostProcessing", "AnalogDigitalTransitionStart", true, NUNBERSAkt);
+        ReadParameter(param, "PostProcessing", "ChangeRateThreshold", true, NUNBERSAkt);
         ReadParameter(param, "PostProcessing", "MaxRateValue", true, NUNBERSAkt);
         ReadParameter(param, "PostProcessing", "MaxRateType", true, NUNBERSAkt);
         ReadParameter(param, "PostProcessing", "ExtendedResolution", false, NUNBERSAkt);
@@ -2180,6 +2194,7 @@ function UpdateInputIndividual(sel) {
     // WriteParameter(param, category, "PostProcessing", "PreValueUse", false, NUNBERSAkt);
     WriteParameter(param, category, "PostProcessing", "DecimalShift", true, NUNBERSAkt);
     WriteParameter(param, category, "PostProcessing", "AnalogDigitalTransitionStart", true, NUNBERSAkt);
+    WriteParameter(param, category, "PostProcessing", "ChangeRateThreshold", true, NUNBERSAkt);
     WriteParameter(param, category, "PostProcessing", "MaxRateValue", true, NUNBERSAkt);
     WriteParameter(param, category, "PostProcessing", "MaxRateType", true, NUNBERSAkt);
     WriteParameter(param, category, "PostProcessing", "ExtendedResolution", false, NUNBERSAkt);

sd-card/html/readconfigparam.js

@@ -173,12 +173,13 @@ function ParseConfig() {
     category[catname]["found"] = false;
     param[catname] = new Object();
     ParamAddValue(param, catname, "DecimalShift", 1, true);
-    ParamAddValue(param, catname, "AnalogDigitalTransitionStart", 1, true);
+    ParamAddValue(param, catname, "AnalogDigitalTransitionStart", 1, true, "9.2");
+    ParamAddValue(param, catname, "ChangeRateThreshold", 1, true, "2");
     // ParamAddValue(param, catname, "PreValueUse", 1, true, "true");
     ParamAddValue(param, catname, "PreValueUse");
     ParamAddValue(param, catname, "PreValueAgeStartup");
     ParamAddValue(param, catname, "AllowNegativeRates", 1, true, "false");
-    ParamAddValue(param, catname, "MaxRateValue", 1, true);
+    ParamAddValue(param, catname, "MaxRateValue", 1, true, "0.05");
     ParamAddValue(param, catname, "MaxRateType", 1, true);
     ParamAddValue(param, catname, "ExtendedResolution", 1, true, "false");
     ParamAddValue(param, catname, "IgnoreLeadingNaN", 1, true, "false");