ClassFlowCNNGeneral.cpp 51 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306
  1. #include "defines.h"
  2. #include <math.h>
  3. #include <iomanip>
  4. #include <sys/types.h>
  5. #include <sstream> // std::stringstream
  6. #include "esp_log.h"
  7. #include "ClassFlowCNNGeneral.h"
  8. #include "MainFlowControl.h"
  9. #include "ClassControllCamera.h"
  10. #include "CTfLiteClass.h"
  11. #include "ClassLogFile.h"
  12. static const char *TAG = "CNN";
  13. ClassFlowCNNGeneral::ClassFlowCNNGeneral(ClassFlowAlignment *_flowalign, t_CNNType _cnntype) : ClassFlowImage(NULL, TAG)
  14. {
  15. cnn_model_file = "";
  16. model_x_size = 1;
  17. model_y_size = 1;
  18. CNNGoodThreshold = 0.0;
  19. ListFlowControll = NULL;
  20. previousElement = NULL;
  21. isLogImageSelect = false;
  22. CNNType = _cnntype;
  23. flowpostalignment = _flowalign;
  24. imagesRetention = 5;
  25. disabled = false;
  26. }
  27. std::vector<double> ClassFlowCNNGeneral::getMeterValues(int _number = 0)
  28. {
  29. std::vector<double> meterValues;
  30. if (GENERAL[_number]->ROI.size() == 0)
  31. {
  32. return meterValues;
  33. }
  34. for (int i = 0; i < GENERAL[_number]->ROI.size(); ++i)
  35. {
  36. if (CNNType == Digit)
  37. {
  38. meterValues.push_back(GENERAL[_number]->ROI[i]->result_klasse);
  39. }
  40. else
  41. {
  42. meterValues.push_back(GENERAL[_number]->ROI[i]->result_float);
  43. }
  44. }
  45. return meterValues;
  46. }
  47. std::string ClassFlowCNNGeneral::getReadout(int _number = 0, bool _extendedResolution, int prev, float _before_narrow_Analog, float AnalogToDigitTransitionStart)
  48. {
  49. std::string result = "";
  50. if (GENERAL[_number]->ROI.size() == 0)
  51. {
  52. return result;
  53. }
  54. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "getReadout _number=" + std::to_string(_number) + ", _extendedResolution=" + std::to_string(_extendedResolution) + ", prev=" + std::to_string(prev));
  55. if (CNNType == Analogue || CNNType == Analogue100)
  56. {
  57. float number = GENERAL[_number]->ROI[GENERAL[_number]->ROI.size() - 1]->result_float;
  58. int result_after_decimal_point = ((int)floor(number * 10.0f) + 10) % 10;
  59. prev = PointerEvalAnalog(GENERAL[_number]->ROI[GENERAL[_number]->ROI.size() - 1]->result_float, prev);
  60. result = std::to_string(prev);
  61. if (_extendedResolution)
  62. {
  63. result = result + std::to_string(result_after_decimal_point);
  64. }
  65. for (int i = GENERAL[_number]->ROI.size() - 2; i >= 0; --i)
  66. {
  67. prev = PointerEvalAnalog(GENERAL[_number]->ROI[i]->result_float, prev);
  68. result = std::to_string(prev) + result;
  69. }
  70. return result;
  71. }
  72. if (CNNType == Digit)
  73. {
  74. for (int i = 0; i < GENERAL[_number]->ROI.size(); ++i)
  75. {
  76. if ((GENERAL[_number]->ROI[i]->result_klasse >= 0) && (GENERAL[_number]->ROI[i]->result_klasse < 10))
  77. {
  78. result = result + std::to_string(GENERAL[_number]->ROI[i]->result_klasse);
  79. }
  80. else
  81. {
  82. result = result + "N";
  83. }
  84. }
  85. return result;
  86. }
  87. if ((CNNType == DoubleHyprid10) || (CNNType == Digit100))
  88. {
  89. float number = GENERAL[_number]->ROI[GENERAL[_number]->ROI.size() - 1]->result_float;
  90. // NaN?
  91. if ((number >= 0.0f) && (number < 10.0f))
  92. {
  93. // is only set if it is the first digit (no analogue before!)
  94. if (_extendedResolution)
  95. {
  96. int result_after_decimal_point = ((int)floor(number * 10.0f)) % 10;
  97. int result_before_decimal_point = ((int)floor(number)) % 10;
  98. result = std::to_string(result_before_decimal_point) + std::to_string(result_after_decimal_point);
  99. prev = result_before_decimal_point;
  100. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "getReadout(dig100-ext) result_before_decimal_point=" + std::to_string(result_before_decimal_point) + ", result_after_decimal_point=" + std::to_string(result_after_decimal_point) + ", prev=" + std::to_string(prev));
  101. }
  102. else
  103. {
  104. if (_before_narrow_Analog >= 0.0f)
  105. {
  106. prev = PointerEvalHybrid(GENERAL[_number]->ROI[GENERAL[_number]->ROI.size() - 1]->result_float, _before_narrow_Analog, prev, true, AnalogToDigitTransitionStart);
  107. }
  108. else
  109. {
  110. prev = PointerEvalHybrid(GENERAL[_number]->ROI[GENERAL[_number]->ROI.size() - 1]->result_float, prev, prev);
  111. }
  112. // is necessary because a number greater than 9.994999 returns a 10! (for further details see check in PointerEvalHybrid)
  113. if ((prev >= 0) && (prev < 10))
  114. {
  115. result = std::to_string(prev);
  116. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "getReadout(dig100) prev=" + std::to_string(prev));
  117. }
  118. else
  119. {
  120. result = "N";
  121. }
  122. }
  123. }
  124. else
  125. {
  126. result = "N";
  127. if (_extendedResolution && (CNNType != Digit))
  128. {
  129. result = "NN";
  130. }
  131. }
  132. for (int i = GENERAL[_number]->ROI.size() - 2; i >= 0; --i)
  133. {
  134. if ((GENERAL[_number]->ROI[i]->result_float >= 0.0f) && (GENERAL[_number]->ROI[i]->result_float < 10.0f))
  135. {
  136. prev = PointerEvalHybrid(GENERAL[_number]->ROI[i]->result_float, GENERAL[_number]->ROI[i + 1]->result_float, prev);
  137. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "getReadout(DoubleHyprid10) - roi_" + std::to_string(i) + "prev= " + std::to_string(prev));
  138. result = std::to_string(prev) + result;
  139. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "getReadout(DoubleHyprid10) - roi_" + std::to_string(i) + "result= " + result);
  140. }
  141. else
  142. {
  143. prev = -1;
  144. result = "N" + result;
  145. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "getReadout(result_float < 0 /'N') - roi_" + std::to_string(i) + "result_float=" + std::to_string(GENERAL[_number]->ROI[i]->result_float));
  146. }
  147. }
  148. return result;
  149. }
  150. return result;
  151. }
  152. /**
  153. * @brief Determines the number of an ROI in connection with previous ROI results
  154. *
  155. * @param number: is the current ROI as float value from recognition
  156. * @param number_of_predecessors: is the last (lower) ROI as float from recognition
  157. * @param eval_predecessors: is the evaluated number. Sometimes a much lower value can change higer values
  158. * example: 9.8, 9.9, 0.1
  159. * 0.1 => 0 (eval_predecessors)
  160. * The 0 makes a 9.9 to 0 (eval_predecessors)
  161. * The 0 makes a 9.8 to 0
  162. * @param Analog_Predecessors false/true if the last ROI is an analog or digit ROI (default=false)
  163. * runs in special handling because analog is much less precise
  164. * @param digitAnalogTransitionStart start of the transitionlogic begins on number_of_predecessor (default=9.2)
  165. *
  166. * @return int the determined number of the current ROI
  167. */
  168. int ClassFlowCNNGeneral::PointerEvalHybrid(float number, float number_of_predecessors, int eval_predecessors, bool Analog_Predecessors, float digitAnalogTransitionStart)
  169. {
  170. int result = -1;
  171. int result_after_decimal_point = ((int)floor(number * 10.0f)) % 10;
  172. int result_before_decimal_point = ((int)floor(number) + 10) % 10;
  173. if (eval_predecessors < 0)
  174. {
  175. // on first digit is no spezial logic for transition needed
  176. // we use the recognition as given. The result is the int value of the recognition
  177. // add precisition of 2 digits and round before trunc
  178. // a number greater than 9.994999 is returned as 10, this leads to an error during the decimal shift because the NUMBERS[j]->ReturnRawValue is one digit longer.
  179. // To avoid this, an additional test must be carried out, see "if ((CNNType == DoubleHyprid10) || (CNNType == Digit100))" check in getReadout()
  180. result = (int) (trunc(round((float)((int)(number + 10.0f) % 10) * 100.0f)) / 100.0f);
  181. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "PointerEvalHybrid - No predecessor - Result = " + std::to_string(result) + " number: " + std::to_string(number) + " number_of_predecessors = " + std::to_string(number_of_predecessors) + " eval_predecessors = " + std::to_string(eval_predecessors) + " Digit_Uncertainty = " + std::to_string(Digit_Uncertainty));
  182. return result;
  183. }
  184. if (Analog_Predecessors)
  185. {
  186. result = PointerEvalAnalogToDigit(number, number_of_predecessors, eval_predecessors, digitAnalogTransitionStart);
  187. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "PointerEvalHybrid - Analog predecessor, evaluation over PointerEvalAnalog = " + std::to_string(result) + " number: " + std::to_string(number) + " number_of_predecessors = " + std::to_string(number_of_predecessors) + " eval_predecessors = " + std::to_string(eval_predecessors) + " Digit_Uncertainty = " + std::to_string(Digit_Uncertainty));
  188. return result;
  189. }
  190. if ((number_of_predecessors >= Digit_Transition_Area_Predecessor) && (number_of_predecessors <= (10.0 - Digit_Transition_Area_Predecessor)))
  191. {
  192. // no digit change, because predecessor is far enough away (0+/-DigitTransitionRangePredecessor) --> number is rounded
  193. // Band around the digit --> Round off, as digit reaches inaccuracy in the frame
  194. if ((result_after_decimal_point <= DigitBand) || (result_after_decimal_point >= (10 - DigitBand)))
  195. {
  196. result = ((int)round(number) + 10) % 10;
  197. }
  198. else
  199. {
  200. result = ((int)trunc(number) + 10) % 10;
  201. }
  202. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "PointerEvalHybrid - NO analogue predecessor, no change of digits, as pre-decimal point far enough away = " + std::to_string(result) + " number: " + std::to_string(number) + " number_of_predecessors = " + std::to_string(number_of_predecessors) + " eval_predecessors = " + std::to_string(eval_predecessors) + " Digit_Uncertainty = " + std::to_string(Digit_Uncertainty));
  203. return result;
  204. }
  205. // Zero crossing at the predecessor has taken place (! evaluation via Prev_value and not number!) --> round up here (2.8 --> 3, but also 3.1 --> 3)
  206. if (eval_predecessors <= 1)
  207. {
  208. // We simply assume that the current digit after the zero crossing of the predecessor
  209. // has passed through at least half (x.5)
  210. if (result_after_decimal_point > 5)
  211. {
  212. // The current digit does not yet have a zero crossing, but the predecessor does..
  213. result = (result_before_decimal_point + 1) % 10;
  214. }
  215. else
  216. {
  217. // Act. digit and predecessor have zero crossing
  218. result = result_before_decimal_point % 10;
  219. }
  220. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "PointerEvalHybrid - NO analogue predecessor, zero crossing has taken placen = " + std::to_string(result) + " number: " + std::to_string(number) + " number_of_predecessors = " + std::to_string(number_of_predecessors) + " eval_predecessors = " + std::to_string(eval_predecessors) + " Digit_Uncertainty = " + std::to_string(Digit_Uncertainty));
  221. return result;
  222. }
  223. // remains only >= 9.x --> no zero crossing yet --> 2.8 --> 2,
  224. // and from 9.7(DigitTransitionRangeLead) 3.1 --> 2
  225. // everything >=x.4 can be considered as current number in transition. With 9.x predecessor the current
  226. // number can still be x.6 - x.7.
  227. // Preceding (else - branch) does not already happen from 9.
  228. if (Digit_Transition_Area_Forward >= number_of_predecessors || result_after_decimal_point >= 4)
  229. {
  230. // The current digit, like the previous digit, does not yet have a zero crossing.
  231. result = result_before_decimal_point % 10;
  232. }
  233. else
  234. {
  235. // current digit precedes the smaller digit (9.x). So already >=x.0 while the previous digit has not yet
  236. // has no zero crossing. Therefore, it is reduced by 1.
  237. result = (result_before_decimal_point + 9) % 10;
  238. }
  239. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "PointerEvalHybrid - O analogue predecessor, >= 9.5 --> no zero crossing yet = " + std::to_string(result) + " number: " + std::to_string(number) + " number_of_predecessors = " + std::to_string(number_of_predecessors) + " eval_predecessors = " + std::to_string(eval_predecessors) + " Digit_Uncertainty = " + std::to_string(Digit_Uncertainty) + " result_after_decimal_point = " + std::to_string(result_after_decimal_point));
  240. return result;
  241. }
  242. int ClassFlowCNNGeneral::PointerEvalAnalogToDigit(float number, float numeral_preceder, int eval_predecessors, float AnalogToDigitTransitionStart)
  243. {
  244. int result = -1;
  245. int result_after_decimal_point = ((int)floor(number * 10.0f)) % 10;
  246. int result_before_decimal_point = ((int)floor(number) + 10) % 10;
  247. bool roundedUp = false;
  248. // Within the digit inequalities
  249. // Band around the digit --> Round off, as digit reaches inaccuracy in the frame
  250. if ((result_after_decimal_point >= (10 - (int)(Digit_Uncertainty * 10.0f))) || (eval_predecessors <= 4 && result_after_decimal_point >= 6))
  251. {
  252. // or digit runs after (analogue =0..4, digit >=6)
  253. result = (int)(round(number) + 10) % 10;
  254. roundedUp = true;
  255. // before/ after decimal point, because we adjust the number based on the uncertainty.
  256. result_after_decimal_point = ((int)floor(result * 10)) % 10;
  257. result_before_decimal_point = ((int)floor(result) + 10) % 10;
  258. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "PointerEvalAnalogToDigit - Digit Uncertainty - Result = " + std::to_string(result) + " number: " + std::to_string(number) + " numeral_preceder: " + std::to_string(numeral_preceder) + " erg before comma: " + std::to_string(result_before_decimal_point) + " erg after comma: " + std::to_string(result_after_decimal_point));
  259. }
  260. else
  261. {
  262. result = (int)((int)trunc(number) + 10) % 10;
  263. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "PointerEvalAnalogToDigit - NO digit Uncertainty - Result = " + std::to_string(result) + " number: " + std::to_string(number) + " numeral_preceder = " + std::to_string(numeral_preceder));
  264. }
  265. // No zero crossing has taken place.
  266. // Only eval_predecessors used because numeral_preceder could be wrong here.
  267. // numeral_preceder<=0.1 & eval_predecessors=9 corresponds to analogue was reset because of previous analogue that are not yet at 0.
  268. if ((eval_predecessors >= 6 && (numeral_preceder > AnalogToDigitTransitionStart || numeral_preceder <= 0.2) && roundedUp))
  269. {
  270. result = ((result_before_decimal_point + 10) - 1) % 10;
  271. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "PointerEvalAnalogToDigit - Nulldurchgang noch nicht stattgefunden = " + std::to_string(result) + " number: " + std::to_string(number) + " numeral_preceder = " + std::to_string(numeral_preceder) + " eerg after comma = " + std::to_string(result_after_decimal_point));
  272. }
  273. return result;
  274. }
  275. int ClassFlowCNNGeneral::PointerEvalAnalog(float number, int numeral_preceder)
  276. {
  277. int result = -1;
  278. if (numeral_preceder == -1)
  279. {
  280. result = (int)floor(number);
  281. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "PointerEvalAnalog - No predecessor - Result = " + std::to_string(result) + " number: " + std::to_string(number) + " numeral_preceder = " + std::to_string(numeral_preceder) + " Analog_error = " + std::to_string(Analog_error));
  282. return result;
  283. }
  284. float number_min = number - (float)Analog_error / 10.0f;
  285. float number_max = number + (float)Analog_error / 10.0f;
  286. if ((int)floor(number_max) - (int)floor(number_min) != 0)
  287. {
  288. if (numeral_preceder <= Analog_error)
  289. {
  290. result = ((int)floor(number_max) + 10) % 10;
  291. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "PointerEvalAnalog - number ambiguous, correction upwards - result = " + std::to_string(result) + " number: " + std::to_string(number) + " numeral_preceder = " + std::to_string(numeral_preceder) + " Analog_error = " + std::to_string(Analog_error));
  292. return result;
  293. }
  294. if (numeral_preceder >= (10 - Analog_error))
  295. {
  296. result = ((int)floor(number_min) + 10) % 10;
  297. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "PointerEvalAnalog - number ambiguous, downward correction - result = " + std::to_string(result) + " number: " + std::to_string(number) + " numeral_preceder = " + std::to_string(numeral_preceder) + " Analog_error = " + std::to_string(Analog_error));
  298. return result;
  299. }
  300. }
  301. result = ((int)floor(number) + 10) % 10;
  302. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "PointerEvalAnalog - number unambiguous, no correction necessary - result = " + std::to_string(result) + " number: " + std::to_string(number) + " numeral_preceder = " + std::to_string(numeral_preceder) + " Analog_error = " + std::to_string(Analog_error));
  303. return result;
  304. }
  305. bool ClassFlowCNNGeneral::ReadParameter(FILE *pfile, std::string &aktparamgraph)
  306. {
  307. aktparamgraph = trim_string_left_right(aktparamgraph);
  308. if (aktparamgraph.size() == 0)
  309. {
  310. if (!GetNextParagraph(pfile, aktparamgraph))
  311. {
  312. return false;
  313. }
  314. }
  315. if ((to_upper(aktparamgraph).compare("[ANALOG]") != 0) && (to_upper(aktparamgraph).compare(";[ANALOG]") != 0) &&
  316. (to_upper(aktparamgraph).compare("[DIGIT]") != 0) && (to_upper(aktparamgraph).compare(";[DIGIT]") != 0) &&
  317. (to_upper(aktparamgraph).compare("[DIGITS]") != 0) && (to_upper(aktparamgraph).compare(";[DIGITS]") != 0))
  318. {
  319. // Paragraph passt nicht
  320. return false;
  321. }
  322. if (aktparamgraph[0] == ';')
  323. {
  324. disabled = true;
  325. while (getNextLine(pfile, &aktparamgraph) && !isNewParagraph(aktparamgraph))
  326. ;
  327. ESP_LOGD(TAG, "[Analog/Digit] is disabled!");
  328. return true;
  329. }
  330. std::vector<std::string> splitted;
  331. while (getNextLine(pfile, &aktparamgraph) && !isNewParagraph(aktparamgraph))
  332. {
  333. splitted = split_line(aktparamgraph);
  334. if (splitted.size() > 1)
  335. {
  336. std::string _param = to_upper(splitted[0]);
  337. if (_param == "ROIIMAGESLOCATION")
  338. {
  339. imagesLocation = "/sdcard" + splitted[1];
  340. isLogImage = true;
  341. }
  342. else if (_param == "LOGIMAGESELECT")
  343. {
  344. LogImageSelect = splitted[1];
  345. isLogImageSelect = true;
  346. }
  347. else if (_param == "ROIIMAGESRETENTION")
  348. {
  349. if (is_string_numeric(splitted[1]))
  350. {
  351. imagesRetention = std::stoi(splitted[1]);
  352. }
  353. }
  354. else if (_param == "MODEL")
  355. {
  356. cnn_model_file = splitted[1];
  357. }
  358. else if (_param == "CNNGOODTHRESHOLD")
  359. {
  360. if (is_string_numeric(splitted[1]))
  361. {
  362. CNNGoodThreshold = std::stof(splitted[1]);
  363. }
  364. }
  365. else if (splitted.size() >= 5)
  366. {
  367. general *_general = GetGENERAL(splitted[0], true);
  368. roi *new_roi = _general->ROI[_general->ROI.size() - 1];
  369. if (is_string_numeric(splitted[1]) && is_string_numeric(splitted[2]) && is_string_numeric(splitted[3]) && is_string_numeric(splitted[4]))
  370. {
  371. new_roi->pos_x = std::stoi(splitted[1]);
  372. new_roi->pos_y = std::stoi(splitted[2]);
  373. new_roi->delta_x = std::stoi(splitted[3]);
  374. new_roi->delta_y = std::stoi(splitted[4]);
  375. }
  376. else
  377. {
  378. new_roi->pos_x = 0;
  379. new_roi->pos_y = 0;
  380. new_roi->delta_x = 30;
  381. new_roi->delta_y = 30;
  382. }
  383. new_roi->ccw = false;
  384. if (splitted.size() >= 6)
  385. {
  386. new_roi->ccw = alphanumeric_to_boolean(splitted[5]);
  387. }
  388. new_roi->result_float = -1;
  389. new_roi->image = NULL;
  390. new_roi->image_org = NULL;
  391. }
  392. }
  393. }
  394. if (!getNetworkParameter())
  395. {
  396. LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "An error occured on setting up the Network -> Disabling it!");
  397. disabled = true; // An error occured, disable this CNN!
  398. return false;
  399. }
  400. for (int i = 0; i < GENERAL.size(); ++i)
  401. {
  402. for (int j = 0; j < GENERAL[i]->ROI.size(); ++j)
  403. {
  404. GENERAL[i]->ROI[j]->image = new CImageBasis("ROI " + GENERAL[i]->ROI[j]->name, model_x_size, model_y_size, model_channel);
  405. GENERAL[i]->ROI[j]->image_org = new CImageBasis("ROI " + GENERAL[i]->ROI[j]->name + " original", GENERAL[i]->ROI[j]->delta_x, GENERAL[i]->ROI[j]->delta_y, 3);
  406. }
  407. }
  408. return true;
  409. }
  410. general *ClassFlowCNNGeneral::FindGENERAL(std::string _name_number)
  411. {
  412. for (int i = 0; i < GENERAL.size(); ++i)
  413. {
  414. if (GENERAL[i]->name == _name_number)
  415. {
  416. return GENERAL[i];
  417. }
  418. }
  419. return NULL;
  420. }
  421. general *ClassFlowCNNGeneral::GetGENERAL(std::string _name, bool _create = true)
  422. {
  423. std::string _number_sequence, _roi_name;
  424. int _pospunkt = _name.find_first_of(".");
  425. if (_pospunkt > -1)
  426. {
  427. _number_sequence = _name.substr(0, _pospunkt);
  428. _roi_name = _name.substr(_pospunkt + 1, _name.length() - _pospunkt - 1);
  429. }
  430. else
  431. {
  432. _number_sequence = "default";
  433. _roi_name = _name;
  434. }
  435. general *_ret = NULL;
  436. for (int i = 0; i < GENERAL.size(); ++i)
  437. {
  438. if (GENERAL[i]->name == _number_sequence)
  439. {
  440. _ret = GENERAL[i];
  441. }
  442. }
  443. // not found and should not be created
  444. if (!_create)
  445. {
  446. return _ret;
  447. }
  448. if (_ret == NULL)
  449. {
  450. _ret = new general;
  451. _ret->name = _number_sequence;
  452. GENERAL.push_back(_ret);
  453. }
  454. roi *new_roi = new roi;
  455. new_roi->name = _roi_name;
  456. _ret->ROI.push_back(new_roi);
  457. ESP_LOGD(TAG, "GetGENERAL - GENERAL %s - roi %s - ccw: %d", _number_sequence.c_str(), _roi_name.c_str(), new_roi->ccw);
  458. return _ret;
  459. }
  460. std::string ClassFlowCNNGeneral::getHTMLSingleStep(std::string host)
  461. {
  462. std::vector<HTMLInfo *> html_info;
  463. std::string result = "<p>Found ROIs: </p> <p><img src=\"" + host + "/img_tmp/alg_roi.jpg\"></p>\n";
  464. result = result + "Analog Pointers: <p> ";
  465. html_info = GetHTMLInfo();
  466. for (int i = 0; i < html_info.size(); ++i)
  467. {
  468. std::stringstream stream;
  469. stream << std::fixed << std::setprecision(1) << html_info[i]->val;
  470. std::string temp_string = stream.str();
  471. result = result + "<img src=\"" + host + "/img_tmp/" + html_info[i]->filename + "\"> " + temp_string;
  472. delete html_info[i];
  473. }
  474. html_info.clear();
  475. return result;
  476. }
  477. bool ClassFlowCNNGeneral::doFlow(std::string time_value)
  478. {
  479. if (disabled)
  480. {
  481. return true;
  482. }
  483. if (!doAlignAndCut(time_value))
  484. {
  485. return false;
  486. }
  487. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "doFlow after alignment");
  488. doNeuralNetwork(time_value);
  489. RemoveOldLogs();
  490. return true;
  491. }
  492. bool ClassFlowCNNGeneral::doAlignAndCut(std::string time_value)
  493. {
  494. if (disabled)
  495. {
  496. return true;
  497. }
  498. CAlignAndCutImage *caic = flowpostalignment->GetAlignAndCutImage();
  499. for (int _number = 0; _number < GENERAL.size(); ++_number)
  500. {
  501. for (int _roi = 0; _roi < GENERAL[_number]->ROI.size(); ++_roi)
  502. {
  503. ESP_LOGD(TAG, "General %d - Align&Cut", _roi);
  504. caic->CutAndSave(GENERAL[_number]->ROI[_roi]->pos_x, GENERAL[_number]->ROI[_roi]->pos_y, GENERAL[_number]->ROI[_roi]->delta_x, GENERAL[_number]->ROI[_roi]->delta_y, GENERAL[_number]->ROI[_roi]->image_org);
  505. if (Camera.SaveAllFiles)
  506. {
  507. if (GENERAL[_number]->name == "default")
  508. {
  509. GENERAL[_number]->ROI[_roi]->image_org->SaveToFile(format_filename("/sdcard/img_tmp/" + GENERAL[_number]->ROI[_roi]->name + ".jpg"));
  510. }
  511. else
  512. {
  513. GENERAL[_number]->ROI[_roi]->image_org->SaveToFile(format_filename("/sdcard/img_tmp/" + GENERAL[_number]->name + "_" + GENERAL[_number]->ROI[_roi]->name + ".jpg"));
  514. }
  515. }
  516. GENERAL[_number]->ROI[_roi]->image_org->Resize(model_x_size, model_y_size, GENERAL[_number]->ROI[_roi]->image);
  517. if (Camera.SaveAllFiles)
  518. {
  519. if (GENERAL[_number]->name == "default")
  520. {
  521. GENERAL[_number]->ROI[_roi]->image->SaveToFile(format_filename("/sdcard/img_tmp/" + GENERAL[_number]->ROI[_roi]->name + ".jpg"));
  522. }
  523. else
  524. {
  525. GENERAL[_number]->ROI[_roi]->image->SaveToFile(format_filename("/sdcard/img_tmp/" + GENERAL[_number]->name + "_" + GENERAL[_number]->ROI[_roi]->name + ".jpg"));
  526. }
  527. }
  528. }
  529. }
  530. return true;
  531. }
  532. void ClassFlowCNNGeneral::DrawROI(CImageBasis *Image)
  533. {
  534. if (Image->ImageOkay())
  535. {
  536. if (CNNType == Analogue || CNNType == Analogue100)
  537. {
  538. int r = 0;
  539. int g = 255;
  540. int b = 0;
  541. for (int _ana = 0; _ana < GENERAL.size(); ++_ana)
  542. {
  543. for (int _roi = 0; _roi < GENERAL[_ana]->ROI.size(); ++_roi)
  544. {
  545. Image->drawRect(GENERAL[_ana]->ROI[_roi]->pos_x, GENERAL[_ana]->ROI[_roi]->pos_y, GENERAL[_ana]->ROI[_roi]->delta_x, GENERAL[_ana]->ROI[_roi]->delta_y, r, g, b, 1);
  546. Image->drawEllipse((int)(GENERAL[_ana]->ROI[_roi]->pos_x + GENERAL[_ana]->ROI[_roi]->delta_x / 2), (int)(GENERAL[_ana]->ROI[_roi]->pos_y + GENERAL[_ana]->ROI[_roi]->delta_y / 2), (int)(GENERAL[_ana]->ROI[_roi]->delta_x / 2), (int)(GENERAL[_ana]->ROI[_roi]->delta_y / 2), r, g, b, 2);
  547. Image->drawLine((int)(GENERAL[_ana]->ROI[_roi]->pos_x + GENERAL[_ana]->ROI[_roi]->delta_x / 2), (int)GENERAL[_ana]->ROI[_roi]->pos_y, (int)(GENERAL[_ana]->ROI[_roi]->pos_x + GENERAL[_ana]->ROI[_roi]->delta_x / 2), (int)(GENERAL[_ana]->ROI[_roi]->pos_y + GENERAL[_ana]->ROI[_roi]->delta_y), r, g, b, 2);
  548. Image->drawLine((int)GENERAL[_ana]->ROI[_roi]->pos_x, (int)(GENERAL[_ana]->ROI[_roi]->pos_y + GENERAL[_ana]->ROI[_roi]->delta_y / 2), (int)GENERAL[_ana]->ROI[_roi]->pos_x + GENERAL[_ana]->ROI[_roi]->delta_x, (int)(GENERAL[_ana]->ROI[_roi]->pos_y + GENERAL[_ana]->ROI[_roi]->delta_y / 2), r, g, b, 2);
  549. }
  550. }
  551. }
  552. else
  553. {
  554. for (int _dig = 0; _dig < GENERAL.size(); ++_dig)
  555. {
  556. for (int _roi = 0; _roi < GENERAL[_dig]->ROI.size(); ++_roi)
  557. {
  558. Image->drawRect(GENERAL[_dig]->ROI[_roi]->pos_x, GENERAL[_dig]->ROI[_roi]->pos_y, GENERAL[_dig]->ROI[_roi]->delta_x, GENERAL[_dig]->ROI[_roi]->delta_y, 0, 0, (255 - _dig * 100), 2);
  559. }
  560. }
  561. }
  562. }
  563. }
  564. bool ClassFlowCNNGeneral::getNetworkParameter(void)
  565. {
  566. if (disabled)
  567. {
  568. return true;
  569. }
  570. std::string temp_cnn = "/sdcard" + cnn_model_file;
  571. temp_cnn = format_filename(temp_cnn);
  572. ESP_LOGD(TAG, "%s", temp_cnn.c_str());
  573. CTfLiteClass *tflite = new CTfLiteClass;
  574. if (!tflite->LoadModel(temp_cnn))
  575. {
  576. LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't load tflite model " + cnn_model_file + " -> Init aborted!");
  577. LogFile.WriteHeapInfo("getNetworkParameter-LoadModel");
  578. delete tflite;
  579. return false;
  580. }
  581. if (!tflite->MakeAllocate())
  582. {
  583. LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't allocate tflite model -> Init aborted!");
  584. LogFile.WriteHeapInfo("getNetworkParameter-MakeAllocate");
  585. delete tflite;
  586. return false;
  587. }
  588. if (CNNType == AutoDetect)
  589. {
  590. tflite->GetInputDimension(false);
  591. model_x_size = tflite->ReadInputDimenstion(0);
  592. model_y_size = tflite->ReadInputDimenstion(1);
  593. model_channel = tflite->ReadInputDimenstion(2);
  594. int anz_output_dimensions = tflite->GetAnzOutPut();
  595. switch (anz_output_dimensions)
  596. {
  597. case 2:
  598. CNNType = Analogue;
  599. ESP_LOGD(TAG, "TFlite-Type set to Analogue");
  600. break;
  601. case 10:
  602. CNNType = DoubleHyprid10;
  603. ESP_LOGD(TAG, "TFlite-Type set to DoubleHyprid10");
  604. break;
  605. case 11:
  606. CNNType = Digit;
  607. ESP_LOGD(TAG, "TFlite-Type set to Digit");
  608. break;
  609. // case 20:
  610. // CNNType = DigitHyprid10;
  611. // ESP_LOGD(TAG, "TFlite-Type set to DigitHyprid10");
  612. // break;
  613. // case 22:
  614. // CNNType = DigitHyprid;
  615. // ESP_LOGD(TAG, "TFlite-Type set to DigitHyprid");
  616. // break;
  617. case 100:
  618. if (model_x_size == 32 && model_y_size == 32)
  619. {
  620. CNNType = Analogue100;
  621. ESP_LOGD(TAG, "TFlite-Type set to Analogue100");
  622. }
  623. else
  624. {
  625. CNNType = Digit100;
  626. ESP_LOGD(TAG, "TFlite-Type set to Digit");
  627. }
  628. break;
  629. default:
  630. LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "tflite does not fit the firmware (outout_dimension=" + std::to_string(anz_output_dimensions) + ")");
  631. }
  632. }
  633. delete tflite;
  634. return true;
  635. }
  636. // wird von "bool ClassFlowCNNGeneral::doFlow(std::string time_value)" aufgerufen
  637. bool ClassFlowCNNGeneral::doNeuralNetwork(std::string time_value)
  638. {
  639. if (disabled)
  640. {
  641. return false;
  642. }
  643. std::string cnn_model = "/sdcard" + cnn_model_file;
  644. cnn_model = format_filename(cnn_model);
  645. ESP_LOGD(TAG, "%s", cnn_model.c_str());
  646. CTfLiteClass *tflite = new CTfLiteClass;
  647. if (!tflite->LoadModel(cnn_model))
  648. {
  649. LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't load tflite model " + cnn_model_file + " -> Exec aborted this round!");
  650. LogFile.WriteHeapInfo("doNeuralNetwork-LoadModel");
  651. delete tflite;
  652. return false;
  653. }
  654. if (!tflite->MakeAllocate())
  655. {
  656. LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "Can't allocate tfilte model -> Exec aborted this round!");
  657. LogFile.WriteHeapInfo("doNeuralNetwork-MakeAllocate");
  658. delete tflite;
  659. return false;
  660. }
  661. std::string logPath = CreateLogFolder(time_value);
  662. std::vector<NumberPost *> numbers = flowctrl.getNumbers();
  663. time_t _imagetime; // in seconds
  664. time(&_imagetime);
  665. localtime(&_imagetime);
  666. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "doNeuralNetwork, _imagetime: " + std::to_string(_imagetime));
  667. // For each NUMBER
  668. for (int j = 0; j < GENERAL.size(); ++j)
  669. {
  670. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Processing Number '" + GENERAL[j]->name + "'");
  671. int start_roi = 0;
  672. if ((numbers[j]->useMaxFlowRate) && (numbers[j]->PreValueValid) && (numbers[j]->timeStampLastValue == numbers[j]->timeStampLastPreValue))
  673. {
  674. int _AnzahlDigit = numbers[j]->AnzahlDigit;
  675. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "doNeuralNetwork, _AnzahlDigit: " + std::to_string(_AnzahlDigit));
  676. int _AnzahlAnalog = numbers[j]->AnzahlAnalog;
  677. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "doNeuralNetwork, _AnzahlAnalog: " + std::to_string(_AnzahlAnalog));
  678. float _MaxFlowRate = (numbers[j]->MaxFlowRate / 60); // in unit/minutes
  679. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "doNeuralNetwork, _MaxFlowRate: " + std::to_string(_MaxFlowRate));
  680. float _LastPreValueTimeDifference = (float)((difftime(_imagetime, numbers[j]->timeStampLastPreValue)) / 60); // in minutes
  681. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "doNeuralNetwork, _LastPreValueTimeDifference: " + std::to_string(_LastPreValueTimeDifference) + " minutes");
  682. std::string _PreValue_old = std::to_string((float)numbers[j]->PreValue);
  683. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "doNeuralNetwork, _PreValue_old: " + _PreValue_old);
  684. ////////////////////////////////////////////////////
  685. std::string _PreValue_new1 = std::to_string((float)numbers[j]->PreValue + (_MaxFlowRate * _LastPreValueTimeDifference));
  686. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "doNeuralNetwork, _PreValue_new1: " + _PreValue_new1);
  687. std::string _PreValue_new2 = std::to_string((float)numbers[j]->PreValue - (_MaxFlowRate * _LastPreValueTimeDifference));
  688. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "doNeuralNetwork, _PreValue_new2: " + _PreValue_new2);
  689. ////////////////////////////////////////////////////
  690. // is necessary because there are always 6 numbers after the DecimalPoint due to float
  691. int _CorrectionValue = 0;
  692. int _pospunkt = _PreValue_old.find_first_of(".");
  693. if (_pospunkt > -1)
  694. {
  695. _CorrectionValue = _PreValue_old.length() - _pospunkt;
  696. _CorrectionValue = _CorrectionValue - numbers[j]->Nachkomma;
  697. }
  698. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "doNeuralNetwork, _CorrectionValue: " + std::to_string(_CorrectionValue));
  699. int _PreValue_len = ((int)_PreValue_old.length() - _CorrectionValue);
  700. if (numbers[j]->isExtendedResolution)
  701. {
  702. _PreValue_len = _PreValue_len - 1;
  703. }
  704. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "doNeuralNetwork, _PreValue_len(without DecimalPoint and ExtendedResolution): " + std::to_string(_PreValue_len));
  705. ////////////////////////////////////////////////////
  706. // (+) Find out which Numbers should not change
  707. int _DecimalPoint1 = 0;
  708. int _NumbersNotChanged1 = 0;
  709. while ((_PreValue_old.length() > _NumbersNotChanged1) && (_PreValue_old[_NumbersNotChanged1] == _PreValue_new1[_NumbersNotChanged1]))
  710. {
  711. if (_PreValue_old[_NumbersNotChanged1] == '.')
  712. {
  713. _DecimalPoint1 = 1;
  714. }
  715. _NumbersNotChanged1++;
  716. }
  717. _NumbersNotChanged1 = _NumbersNotChanged1 - _DecimalPoint1;
  718. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Number of ROIs that should not change: " + std::to_string(_NumbersNotChanged1));
  719. if ((_AnzahlDigit + _AnzahlAnalog) > _PreValue_len)
  720. {
  721. _NumbersNotChanged1 = _NumbersNotChanged1 + ((_AnzahlDigit + _AnzahlAnalog) - _PreValue_len);
  722. }
  723. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Number of ROIs that should not change(corrected): " + std::to_string(_NumbersNotChanged1));
  724. ////////////////////////////////////////////////////
  725. // (-) Find out which Numbers should not change
  726. int _NumbersNotChanged2 = _NumbersNotChanged1;
  727. if (numbers[j]->AllowNegativeRates)
  728. {
  729. int _DecimalPoint2 = 0;
  730. while ((_PreValue_old.length() > _NumbersNotChanged2) && (_PreValue_old[_NumbersNotChanged2] == _PreValue_new2[_NumbersNotChanged2]))
  731. {
  732. if (_PreValue_old[_NumbersNotChanged2] == '.')
  733. {
  734. _DecimalPoint2 = 1;
  735. }
  736. _NumbersNotChanged2++;
  737. }
  738. _NumbersNotChanged2 = _NumbersNotChanged2 - _DecimalPoint2;
  739. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Number of ROIs that should not change: " + std::to_string(_NumbersNotChanged2));
  740. if ((_AnzahlDigit + _AnzahlAnalog) > _PreValue_len)
  741. {
  742. _NumbersNotChanged2 = _NumbersNotChanged2 + ((_AnzahlDigit + _AnzahlAnalog) - _PreValue_len);
  743. }
  744. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Number of ROIs that should not change(corrected): " + std::to_string(_NumbersNotChanged2));
  745. }
  746. ////////////////////////////////////////////////////
  747. int start_digit_new = 0;
  748. int start_analog_new = 0;
  749. int _NumbersNotChanged = min(_NumbersNotChanged1, _NumbersNotChanged2);
  750. if (_NumbersNotChanged <= _AnzahlDigit)
  751. {
  752. // The change already takes place at the digit ROIs
  753. start_digit_new = (_AnzahlDigit - _NumbersNotChanged);
  754. start_digit_new = (_AnzahlDigit - start_digit_new);
  755. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "From the " + std::to_string(start_digit_new) + " th digit ROI is evaluated");
  756. }
  757. else
  758. {
  759. // The change only takes place at the analog ROIs
  760. start_digit_new = _AnzahlDigit;
  761. start_analog_new = (_AnzahlAnalog - (_NumbersNotChanged - _AnzahlDigit));
  762. start_analog_new = (_AnzahlAnalog - start_analog_new);
  763. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "From the " + std::to_string(start_analog_new) + " th analog ROI is evaluated");
  764. }
  765. ////////////////////////////////////////////////////
  766. if (CNNType == Digit || CNNType == Digit100 || CNNType == DoubleHyprid10)
  767. {
  768. start_roi = start_digit_new;
  769. }
  770. else if (CNNType == Analogue || CNNType == Analogue100)
  771. {
  772. start_roi = start_analog_new;
  773. }
  774. }
  775. // For each ROI
  776. for (int i = 0; i < GENERAL[j]->ROI.size(); ++i)
  777. {
  778. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "ROI #" + std::to_string(i) + " - TfLite");
  779. switch (CNNType)
  780. {
  781. case Analogue:
  782. {
  783. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "CNN Type: Analogue");
  784. tflite->LoadInputImageBasis(GENERAL[j]->ROI[i]->image);
  785. tflite->Invoke();
  786. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Analogue - After Invoke");
  787. float _value1 = tflite->GetOutputValue(0);
  788. float _value2 = tflite->GetOutputValue(1);
  789. float _result = fmod((atan2(_value1, _value2) / (M_PI * 2.0f) + 2.0f), 1.0f);
  790. if (GENERAL[j]->ROI[i]->ccw)
  791. {
  792. _result = 10.0f - (_result * 10.0f);
  793. }
  794. else
  795. {
  796. _result = _result * 10.0f;
  797. }
  798. if (i >= start_roi)
  799. {
  800. GENERAL[j]->ROI[i]->result_float = _result;
  801. }
  802. GENERAL[j]->ROI[i]->raw_result_float = _result;
  803. if ((_result < 0.0f) || (_result >= 10.0f))
  804. {
  805. GENERAL[j]->ROI[i]->isReject = true;
  806. }
  807. else
  808. {
  809. GENERAL[j]->ROI[i]->isReject = false;
  810. }
  811. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "General result (Analog) - roi_" + std::to_string(i) + ": " + std::to_string(GENERAL[j]->ROI[i]->raw_result_float));
  812. ESP_LOGD(TAG, "General result (Analog) - roi_%i - ccw: %d - %f", i, GENERAL[j]->ROI[i]->ccw, GENERAL[j]->ROI[i]->raw_result_float);
  813. if (isLogImage)
  814. {
  815. std::string _image_name = GENERAL[j]->name + "_" + GENERAL[j]->ROI[i]->name;
  816. if (isLogImageSelect)
  817. {
  818. if (LogImageSelect.find(GENERAL[j]->ROI[i]->name) != std::string::npos)
  819. {
  820. LogImage(logPath, _image_name, &GENERAL[j]->ROI[i]->raw_result_float, NULL, time_value, GENERAL[j]->ROI[i]->image_org);
  821. }
  822. }
  823. }
  824. }
  825. break;
  826. case Digit:
  827. {
  828. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "CNN Type: Digit");
  829. int _result = tflite->GetClassFromImageBasis(GENERAL[j]->ROI[i]->image);
  830. if (i >= start_roi)
  831. {
  832. GENERAL[j]->ROI[i]->result_klasse = _result;
  833. }
  834. GENERAL[j]->ROI[i]->raw_result_klasse = _result;
  835. if ((_result < 0) || (_result >= 10))
  836. {
  837. GENERAL[j]->ROI[i]->isReject = true;
  838. }
  839. else
  840. {
  841. GENERAL[j]->ROI[i]->isReject = false;
  842. }
  843. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "General result (Digit) - roi_" + std::to_string(i) + ": " + std::to_string(GENERAL[j]->ROI[i]->raw_result_klasse));
  844. ESP_LOGD(TAG, "General result (Digit) - roi_%i: %d", i, GENERAL[j]->ROI[i]->raw_result_klasse);
  845. if (isLogImage)
  846. {
  847. std::string _image_name = GENERAL[j]->name + "_" + GENERAL[j]->ROI[i]->name;
  848. if (isLogImageSelect)
  849. {
  850. if (LogImageSelect.find(GENERAL[j]->ROI[i]->name) != std::string::npos)
  851. {
  852. LogImage(logPath, _image_name, NULL, &GENERAL[j]->ROI[i]->raw_result_klasse, time_value, GENERAL[j]->ROI[i]->image_org);
  853. }
  854. }
  855. }
  856. }
  857. break;
  858. case DoubleHyprid10:
  859. {
  860. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "CNN Type: DoubleHyprid10");
  861. tflite->LoadInputImageBasis(GENERAL[j]->ROI[i]->image);
  862. tflite->Invoke();
  863. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "DoubleHyprid10 - After Invoke");
  864. int _num = tflite->GetOutClassification(0, 9);
  865. int _numplus = (_num + 1) % 10;
  866. int _numminus = (_num - 1 + 10) % 10;
  867. float _value = tflite->GetOutputValue(_num);
  868. float _valueplus = tflite->GetOutputValue(_numplus);
  869. float _valueminus = tflite->GetOutputValue(_numminus);
  870. float _result = (float)_num - _valueminus / (_value + _valueminus);
  871. float _fit = _value + _valueminus;
  872. if (_valueplus > _valueminus)
  873. {
  874. _result = (float)_num + _valueplus / (_valueplus + _value);
  875. _fit = _value + _valueplus;
  876. }
  877. std::string temp_bufer = "DoubleHyprid10 - _num (plus, minus): " + std::to_string(_num) + " (" + std::to_string(_numplus) + ", " + std::to_string(_numminus) + ")";
  878. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, temp_bufer);
  879. temp_bufer = "DoubleHyprid10 - _val (plus, minus): " + std::to_string(_value) + " (" + std::to_string(_valueplus) + ", " + std::to_string(_valueminus) + ")";
  880. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, temp_bufer);
  881. temp_bufer = "DoubleHyprid10 - _result: " + std::to_string(_result) + ", _fit: " + std::to_string(_fit);
  882. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, temp_bufer);
  883. float _result_save_file = _result;
  884. if (_fit < CNNGoodThreshold)
  885. {
  886. GENERAL[j]->ROI[i]->isReject = true;
  887. _result = -1;
  888. temp_bufer = "DoubleHyprid10 - Value Rejected due to Threshold (Fit: " + std::to_string(_fit) + ", Threshold: " + std::to_string(CNNGoodThreshold) + ")";
  889. LogFile.WriteToFile(ESP_LOG_WARN, TAG, temp_bufer);
  890. }
  891. else
  892. {
  893. GENERAL[j]->ROI[i]->isReject = false;
  894. }
  895. if (GENERAL[j]->ROI[i]->ccw)
  896. {
  897. }
  898. if (i >= start_roi)
  899. {
  900. GENERAL[j]->ROI[i]->result_float = _result;
  901. }
  902. GENERAL[j]->ROI[i]->raw_result_float = _result;
  903. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Result General(DoubleHyprid10) - roi_" + std::to_string(i) + ": " + std::to_string(GENERAL[j]->ROI[i]->raw_result_float));
  904. ESP_LOGD(TAG, "Result General(DoubleHyprid10) - roi_%i: %f", i, GENERAL[j]->ROI[i]->raw_result_float);
  905. if (isLogImage)
  906. {
  907. std::string _image_name = GENERAL[j]->name + "_" + GENERAL[j]->ROI[i]->name;
  908. if (isLogImageSelect)
  909. {
  910. if (LogImageSelect.find(GENERAL[j]->ROI[i]->name) != std::string::npos)
  911. {
  912. LogImage(logPath, _image_name, &_result_save_file, NULL, time_value, GENERAL[j]->ROI[i]->image_org);
  913. }
  914. }
  915. }
  916. }
  917. break;
  918. case Digit100:
  919. case Analogue100:
  920. {
  921. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "CNN Type: Digit100 or Analogue100");
  922. tflite->LoadInputImageBasis(GENERAL[j]->ROI[i]->image);
  923. tflite->Invoke();
  924. int _num = tflite->GetOutClassification();
  925. float _result = 0.0f;
  926. if (GENERAL[j]->ROI[i]->ccw)
  927. {
  928. _result = 10.0f - ((float)_num / 10.0f);
  929. }
  930. else
  931. {
  932. _result = (float)_num / 10.0f;
  933. }
  934. if (i >= start_roi)
  935. {
  936. GENERAL[j]->ROI[i]->result_float = _result;
  937. }
  938. GENERAL[j]->ROI[i]->raw_result_float = _result;
  939. if ((_result < 0.0f) || (_result >= 10.0f))
  940. {
  941. GENERAL[j]->ROI[i]->isReject = true;
  942. }
  943. else
  944. {
  945. GENERAL[j]->ROI[i]->isReject = false;
  946. }
  947. LogFile.WriteToFile(ESP_LOG_DEBUG, TAG, "Result General(Digit100 or Analogue100) - roi_" + std::to_string(i) + ": " + std::to_string(GENERAL[j]->ROI[i]->raw_result_float));
  948. ESP_LOGD(TAG, "Result General(Digit100 or Analogue100) - roi_%i - ccw: %d - %f", i, GENERAL[j]->ROI[i]->ccw, GENERAL[j]->ROI[i]->raw_result_float);
  949. if (isLogImage)
  950. {
  951. std::string _image_name = GENERAL[j]->name + "_" + GENERAL[j]->ROI[i]->name;
  952. if (isLogImageSelect)
  953. {
  954. if (LogImageSelect.find(GENERAL[j]->ROI[i]->name) != std::string::npos)
  955. {
  956. LogImage(logPath, _image_name, &GENERAL[j]->ROI[i]->raw_result_float, NULL, time_value, GENERAL[j]->ROI[i]->image_org);
  957. }
  958. }
  959. }
  960. }
  961. break;
  962. default:
  963. {
  964. LogFile.WriteToFile(ESP_LOG_ERROR, TAG, "CNN Type: unknown");
  965. }
  966. break;
  967. }
  968. }
  969. }
  970. delete tflite;
  971. return true;
  972. }
  973. bool ClassFlowCNNGeneral::isExtendedResolution(int _number)
  974. {
  975. if (CNNType == Digit)
  976. {
  977. return false;
  978. }
  979. return true;
  980. }
  981. std::vector<HTMLInfo *> ClassFlowCNNGeneral::GetHTMLInfo(void)
  982. {
  983. std::vector<HTMLInfo *> result;
  984. for (int _number = 0; _number < GENERAL.size(); ++_number)
  985. {
  986. for (int _roi = 0; _roi < GENERAL[_number]->ROI.size(); ++_roi)
  987. {
  988. ESP_LOGD(TAG, "Image: %d", (int)GENERAL[_number]->ROI[_roi]->image);
  989. if (GENERAL[_number]->ROI[_roi]->image)
  990. {
  991. if (GENERAL[_number]->name == "default")
  992. {
  993. GENERAL[_number]->ROI[_roi]->image->SaveToFile(format_filename("/sdcard/img_tmp/" + GENERAL[_number]->ROI[_roi]->name + ".jpg"));
  994. }
  995. else
  996. {
  997. GENERAL[_number]->ROI[_roi]->image->SaveToFile(format_filename("/sdcard/img_tmp/" + GENERAL[_number]->name + "_" + GENERAL[_number]->ROI[_roi]->name + ".jpg"));
  998. }
  999. }
  1000. HTMLInfo *temp_info = new HTMLInfo;
  1001. if (GENERAL[_number]->name == "default")
  1002. {
  1003. temp_info->filename = GENERAL[_number]->ROI[_roi]->name + ".jpg";
  1004. temp_info->filename_org = GENERAL[_number]->ROI[_roi]->name + ".jpg";
  1005. }
  1006. else
  1007. {
  1008. temp_info->filename = GENERAL[_number]->name + "_" + GENERAL[_number]->ROI[_roi]->name + ".jpg";
  1009. temp_info->filename_org = GENERAL[_number]->name + "_" + GENERAL[_number]->ROI[_roi]->name + ".jpg";
  1010. }
  1011. if (CNNType == Digit)
  1012. {
  1013. temp_info->val = (float)GENERAL[_number]->ROI[_roi]->raw_result_klasse;
  1014. }
  1015. else
  1016. {
  1017. temp_info->val = GENERAL[_number]->ROI[_roi]->raw_result_float;
  1018. }
  1019. temp_info->image = GENERAL[_number]->ROI[_roi]->image;
  1020. temp_info->image_org = GENERAL[_number]->ROI[_roi]->image_org;
  1021. result.push_back(temp_info);
  1022. }
  1023. }
  1024. return result;
  1025. }
  1026. int ClassFlowCNNGeneral::getNumberGENERAL(void)
  1027. {
  1028. return GENERAL.size();
  1029. }
  1030. std::string ClassFlowCNNGeneral::getNameGENERAL(int _number)
  1031. {
  1032. if (_number < GENERAL.size())
  1033. {
  1034. return GENERAL[_number]->name;
  1035. }
  1036. return "GENERAL DOES NOT EXIST";
  1037. }
  1038. general *ClassFlowCNNGeneral::GetGENERAL(int _number)
  1039. {
  1040. if (_number < GENERAL.size())
  1041. {
  1042. return GENERAL[_number];
  1043. }
  1044. return NULL;
  1045. }
  1046. void ClassFlowCNNGeneral::UpdateNameNumbers(std::vector<std::string> *_name_numbers)
  1047. {
  1048. for (int _number = 0; _number < GENERAL.size(); ++_number)
  1049. {
  1050. std::string _name = GENERAL[_number]->name;
  1051. bool found = false;
  1052. for (int _roi = 0; _roi < (*_name_numbers).size(); ++_roi)
  1053. {
  1054. if ((*_name_numbers)[_roi] == _name)
  1055. {
  1056. found = true;
  1057. }
  1058. }
  1059. if (!found)
  1060. {
  1061. (*_name_numbers).push_back(_name);
  1062. }
  1063. }
  1064. }
  1065. std::string ClassFlowCNNGeneral::getReadoutRawString(int _number)
  1066. {
  1067. std::string temp_string = "";
  1068. if (_number >= GENERAL.size() || GENERAL[_number] == NULL || GENERAL[_number]->ROI.size() == 0)
  1069. {
  1070. return temp_string;
  1071. }
  1072. for (int _roi = 0; _roi < GENERAL[_number]->ROI.size(); ++_roi)
  1073. {
  1074. if (CNNType == Analogue || CNNType == Analogue100)
  1075. {
  1076. if ((GENERAL[_number]->ROI[_roi]->raw_result_float < 0.0f) || (GENERAL[_number]->ROI[_roi]->raw_result_float >= 10.0f))
  1077. {
  1078. temp_string = temp_string + ",N";
  1079. }
  1080. else
  1081. {
  1082. temp_string = temp_string + "," + round_output(GENERAL[_number]->ROI[_roi]->raw_result_float, 1);
  1083. }
  1084. }
  1085. else if (CNNType == Digit)
  1086. {
  1087. if ((GENERAL[_number]->ROI[_roi]->raw_result_klasse < 0) || (GENERAL[_number]->ROI[_roi]->raw_result_klasse >= 10))
  1088. {
  1089. temp_string = temp_string + ",N";
  1090. }
  1091. else
  1092. {
  1093. temp_string = temp_string + "," + round_output(GENERAL[_number]->ROI[_roi]->raw_result_klasse, 0);
  1094. }
  1095. }
  1096. else if ((CNNType == DoubleHyprid10) || (CNNType == Digit100))
  1097. {
  1098. if ((GENERAL[_number]->ROI[_roi]->raw_result_float < 0.0f) || (GENERAL[_number]->ROI[_roi]->raw_result_float >= 10.0f))
  1099. {
  1100. temp_string = temp_string + ",N";
  1101. }
  1102. else
  1103. {
  1104. temp_string = temp_string + "," + round_output(GENERAL[_number]->ROI[_roi]->raw_result_float, 1);
  1105. }
  1106. }
  1107. }
  1108. return temp_string;
  1109. }