add Prometheus/OpenMetrics exporter (#3081)

* add prometheus endpoint

* refine metrics implementation

* move metrics generator to ClassFlowControll

* add more metrics
align prefix

* add more metrics
clean up

* refine documentation

* revert dependencies change

* sanitize labels

* create separate module for openmetrics

* move openmetrics to separate folder

* clean up

* add basic unit-tests

* work with const numbers
add replaceAll for string replacement
avoid opening std namespace
adapt unit-tests

* Update code/main/server_main.cpp

---------

Co-authored-by: CaCO3 <caco3@ruinelli.ch>

code/components/jomjol_flowcontroll/CMakeLists.txt

@@ -2,6 +2,6 @@ FILE(GLOB_RECURSE app_sources ${CMAKE_CURRENT_SOURCE_DIR}/*.*)
 
 idf_component_register(SRCS ${app_sources}
                     INCLUDE_DIRS "."
-                    REQUIRES esp_timer esp_wifi jomjol_tfliteclass jomjol_helper jomjol_controlcamera jomjol_mqtt jomjol_influxdb jomjol_fileserver_ota jomjol_image_proc jomjol_wlan)
+                    REQUIRES esp_timer esp_wifi jomjol_tfliteclass jomjol_helper jomjol_controlcamera jomjol_mqtt jomjol_influxdb jomjol_fileserver_ota jomjol_image_proc jomjol_wlan openmetrics)
 
 

code/components/jomjol_flowcontroll/ClassFlowControll.cpp

@@ -927,3 +927,11 @@ string ClassFlowControll::getJSON()
 {
     return flowpostprocessing->GetJSON();
 }
+
+/** 
+ * @returns a vector of all current sequences
+ **/
+const std::vector<NumberPost*> &ClassFlowControll::getNumbers()
+{
+    return *flowpostprocessing->GetNumbers();
+}

code/components/jomjol_flowcontroll/ClassFlowControll.h

@@ -52,6 +52,7 @@ public:
 	string GetPrevalue(std::string _number = "");	
 	bool ReadParameter(FILE* pfile, string& aktparamgraph);	
 	string getJSON();
+	const std::vector<NumberPost*> &getNumbers();
 	string getNumbersName();
 
 	string TranslateAktstatus(std::string _input);

code/components/jomjol_flowcontroll/MainFlowControl.cpp

@@ -474,6 +474,71 @@ esp_err_t handler_json(httpd_req_t *req)
     return ESP_OK;
 }
 
+/**
+ * Generates a http response containing the OpenMetrics (https://openmetrics.io/) text wire format 
+ * according to https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#text-format.
+ * 
+ * A MetricFamily with a Metric for each Sequence is provided. If no valid value is available, the metric is not provided.
+ * MetricPoints are provided without a timestamp. Additional metrics with some device information is also provided.
+ * 
+ * The metric name prefix is 'ai_on_the_edge_device_'.
+ * 
+ * example configuration for Prometheus (`prometheus.yml`):
+ * 
+ *    - job_name: watermeter
+ *      static_configs:
+ *        - targets: ['watermeter.fritz.box']
+ * 
+*/
+esp_err_t handler_openmetrics(httpd_req_t *req)
+{
+#ifdef DEBUG_DETAIL_ON
+    LogFile.WriteHeapInfo("handler_openmetrics - Start");
+#endif
+
+    ESP_LOGD(TAG, "handler_openmetrics uri: %s", req->uri);
+
+    if (bTaskAutoFlowCreated)
+    {
+        httpd_resp_set_hdr(req, "Access-Control-Allow-Origin", "*");
+        httpd_resp_set_type(req, "text/plain"); // application/openmetrics-text is not yet supported by prometheus so we use text/plain for now
+
+        const string metricNamePrefix = "ai_on_the_edge_device";
+
+        // get current measurement (flow)
+        string response = createSequenceMetrics(metricNamePrefix, flowctrl.getNumbers());
+
+        // CPU Temperature
+        response += createMetric(metricNamePrefix + "_cpu_temperature_celsius", "current cpu temperature in celsius", "gauge", std::to_string((int)temperatureRead())); 
+
+        // WiFi signal strength
+        response += createMetric(metricNamePrefix + "_rssi_dbm", "current WiFi signal strength in dBm", "gauge", std::to_string(get_WIFI_RSSI())); 
+
+        // memory info
+        response += createMetric(metricNamePrefix + "_memory_heap_free_bytes", "available heap memory", "gauge", std::to_string(getESPHeapSize())); 
+
+        // device uptime
+        response += createMetric(metricNamePrefix + "_uptime_seconds", "device uptime in seconds", "gauge", std::to_string((long)getUpTime())); 
+
+        // data aquisition round
+        response += createMetric(metricNamePrefix + "_rounds_total", "data aquisition rounds since device startup", "counter", std::to_string(countRounds));
+
+        // the response always contains at least the metadata (HELP, TYPE) for the MetricFamily so no length check is needed
+        httpd_resp_send(req, response.c_str(), response.length());
+    }
+    else
+    {
+        httpd_resp_send_err(req, HTTPD_403_FORBIDDEN, "Flow not (yet) started: REST API /metrics not yet available!");
+        return ESP_ERR_NOT_FOUND;
+    }
+
+#ifdef DEBUG_DETAIL_ON
+    LogFile.WriteHeapInfo("handler_openmetrics - Done");
+#endif
+
+    return ESP_OK;
+}
+
 esp_err_t handler_wasserzaehler(httpd_req_t *req)
 {
 #ifdef DEBUG_DETAIL_ON
@@ -1650,4 +1715,12 @@ void register_server_main_flow_task_uri(httpd_handle_t server)
     camuri.handler = handler_stream;
     camuri.user_ctx = (void *)"stream";
     httpd_register_uri_handler(server, &camuri);
+
+    /** will handle metrics requests */
+    camuri.uri = "/metrics";
+    camuri.handler = handler_openmetrics;
+    camuri.user_ctx = (void *)"metrics";
+    httpd_register_uri_handler(server, &camuri);
+
+    /** when adding a new handler, make sure to increment the value for config.max_uri_handlers in `main/server_main.cpp` */
 }

code/components/jomjol_flowcontroll/MainFlowControl.h

@@ -9,6 +9,7 @@
 #include <esp_http_server.h>
 #include "CImageBasis.h"
 #include "ClassFlowControll.h"
+#include "openmetrics.h"
 
 typedef struct
 {

code/components/jomjol_helper/Helper.cpp

@@ -1206,3 +1206,13 @@ bool isInString(std::string &s, std::string const &toFind)
 
 	return true;
 }
+
+// from https://stackoverflow.com/a/14678800
+void replaceAll(std::string& s, const std::string& toReplace, const std::string& replaceWith)
+{
+    size_t pos = 0;
+    while ((pos = s.find(toReplace, pos)) != std::string::npos) {
+         s.replace(pos, toReplace.length(), replaceWith);
+         pos += replaceWith.length();
+    }
+}

code/components/jomjol_helper/Helper.h

@@ -98,6 +98,7 @@ const char* get404(void);
 
 std::string UrlDecode(const std::string& value);
 
+void replaceAll(std::string& s, const std::string& toReplace, const std::string& replaceWith);
 bool replaceString(std::string& s, std::string const& toReplace, std::string const& replaceWith);
 bool replaceString(std::string& s, std::string const& toReplace, std::string const& replaceWith, bool logIt);
 bool isInString(std::string& s, std::string const& toFind);

code/components/openmetrics/CMakeLists.txt

@@ -0,0 +1,7 @@
+FILE(GLOB_RECURSE app_sources ${CMAKE_CURRENT_SOURCE_DIR}/*.*)
+
+idf_component_register(SRCS ${app_sources}
+                    INCLUDE_DIRS "."
+                    REQUIRES jomjol_image_proc)
+
+

code/components/openmetrics/openmetrics.cpp

@@ -0,0 +1,43 @@
+#include "openmetrics.h"
+
+/**
+ * create a singe metric from the given input
+ **/
+std::string createMetric(const std::string &metricName, const std::string &help, const std::string &type, const std::string &value)
+{
+    return "# HELP " + metricName + " " + help + "\n" +
+           "# TYPE " + metricName + " " + type + "\n" +
+           metricName + " " + value + "\n";
+}
+
+/**
+ * Generate the MetricFamily from all available sequences
+ * @returns the string containing the text wire format of the MetricFamily
+ **/
+std::string createSequenceMetrics(std::string prefix, const std::vector<NumberPost *> &numbers)
+{
+    std::string res;
+
+    for (const auto &number : numbers)
+    {
+        // only valid data is reported (https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#missing-data)
+        if (number->ReturnValue.length() > 0)
+        {
+            auto label = number->name;
+
+            // except newline, double quote, and backslash (https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#abnf)
+            // to keep it simple, these characters are just removed from the label
+            replaceAll(label, "\\", "");
+            replaceAll(label, "\"", "");
+            replaceAll(label, "\n", "");
+            res += prefix + "_flow_value{sequence=\"" + label + "\"} " + number->ReturnValue + "\n";
+        }
+    }
+
+    // prepend metadata if a valid metric was created
+    if (res.length() > 0)
+    {
+        res = "# HELP " + prefix + "_flow_value current value of meter readout\n# TYPE " + prefix + "_flow_value gauge\n" + res;
+    }
+    return res;
+}

code/components/openmetrics/openmetrics.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#ifndef OPENMETRICS_H
+#define OPENMETRICS_H
+
+#include <string>
+#include <fstream>
+#include <vector>
+
+#include "ClassFlowDefineTypes.h"
+
+std::string createMetric(const std::string &metricName, const std::string &help, const std::string &type, const std::string &value);
+std::string createSequenceMetrics(std::string prefix, const std::vector<NumberPost *> &numbers);
+
+#endif // OPENMETRICS_H

code/main/server_main.cpp

@@ -459,7 +459,7 @@ httpd_handle_t start_webserver(void)
     config.server_port = 80;
     config.ctrl_port = 32768;
     config.max_open_sockets = 5; //20210921 --> previously 7   
-    config.max_uri_handlers = 39; // previously 24, 20220511: 35, 20221220: 37, 2023-01-02:38             
+    config.max_uri_handlers = 40; // Make sure this fits all URI handlers. Memory usage in bytes: 6*max_uri_handlers
     config.max_resp_headers = 8;                        
     config.backlog_conn = 5;                        
     config.lru_purge_enable = true; // this cuts old connections if new ones are needed.               

code/test/components/openmetrics/test_openmetrics.cpp

@@ -0,0 +1,65 @@
+#include <unity.h>
+#include <openmetrics.h>
+
+void test_createMetric()
+{
+    // simple happy path
+    const char *expected = "# HELP metric_name short description\n# TYPE metric_name gauge\nmetric_name 123.456\n";
+    std::string result = createMetric("metric_name", "short description", "gauge", "123.456");
+    TEST_ASSERT_EQUAL_STRING(expected, result.c_str());
+}
+
+/**
+ * test the replaceString function as it's a dependency to sanitize sequence names
+ */
+void test_replaceString()
+{
+    std::string sample = "hello\\world\\";
+    replaceAll(sample, "\\", "");
+    TEST_ASSERT_EQUAL_STRING("helloworld", sample.c_str());
+
+    sample = "hello\"world\"";
+    replaceAll(sample, "\"", "");
+    TEST_ASSERT_EQUAL_STRING("helloworld", sample.c_str());
+
+    sample = "hello\nworld\n";
+    replaceAll(sample, "\n", "");
+    TEST_ASSERT_EQUAL_STRING("helloworld", sample.c_str());
+
+    sample = "\\\\\\\\\\\\\\\\\\hello\\world\\\\\\\\\\\\\\\\\\\\";
+    replaceAll(sample, "\\", "");
+    TEST_ASSERT_EQUAL_STRING("helloworld", sample.c_str());
+}
+
+void test_createSequenceMetrics()
+{
+    std::vector<NumberPost *> NUMBERS;
+    NumberPost *number_1 = new NumberPost;
+    number_1->name = "main";
+    number_1->ReturnValue = "123.456";
+    NUMBERS.push_back(number_1);
+
+    const std::string metricNamePrefix = "ai_on_the_edge_device";
+    const std::string metricName = metricNamePrefix + "_flow_value";
+
+    std::string expected1 = "# HELP " + metricName + " current value of meter readout\n# TYPE " + metricName + " gauge\n" +
+                             metricName + "{sequence=\"" + number_1->name + "\"} " + number_1->ReturnValue + "\n";
+    TEST_ASSERT_EQUAL_STRING(expected1.c_str(), createSequenceMetrics(metricNamePrefix, NUMBERS).c_str());
+
+    NumberPost *number_2 = new NumberPost;
+    number_2->name = "secondary";
+    number_2->ReturnValue = "1.0";
+    NUMBERS.push_back(number_2);
+
+    std::string expected2 = "# HELP " + metricName + " current value of meter readout\n# TYPE " + metricName + " gauge\n" +
+                             metricName + "{sequence=\"" + number_1->name + "\"} " + number_1->ReturnValue + "\n" +
+                             metricName + "{sequence=\"" + number_2->name + "\"} " + number_2->ReturnValue + "\n";
+    TEST_ASSERT_EQUAL_STRING(expected2.c_str(), createSequenceMetrics(metricNamePrefix, NUMBERS).c_str());
+}
+
+void test_openmetrics()
+{
+    test_createMetric();
+    test_replaceString();
+    test_createSequenceMetrics();
+}

code/test/test_suite_flowcontroll.cpp

@@ -20,6 +20,7 @@
 #include "components/jomjol-flowcontroll/test_PointerEvalAnalogToDigitNew.cpp"
 #include "components/jomjol-flowcontroll/test_getReadoutRawString.cpp"
 #include "components/jomjol-flowcontroll/test_cnnflowcontroll.cpp"
+#include "components/openmetrics/test_openmetrics.cpp"
 #include "components/jomjol_mqtt/test_server_mqtt.cpp"
 
 bool Init_NVS_SDCard()
@@ -167,6 +168,7 @@ extern "C" void app_main()
 
     // getReadoutRawString test
     RUN_TEST(test_getReadoutRawString);
+    RUN_TEST(test_openmetrics);
     RUN_TEST(test_mqtt);
   
   UNITY_END();