Add prom metrics for Central controller specific things

controller/ConnectionPool.hpp

@@ -19,6 +19,8 @@
 	#define _DEBUG(x)
 #endif
 
+#include <prometheus/simpleapi.h>
+
 #include <deque>
 #include <set>
 #include <memory>
@@ -61,6 +63,7 @@ public:
     {
         while(m_pool.size() < m_minPoolSize){
             m_pool.push_back(m_factory->create());
+            _pool_avail++;
         }
     };
 
@@ -91,6 +94,7 @@ public:
         while((m_pool.size() + m_borrowed.size()) < m_minPoolSize) {
             std::shared_ptr<Connection> conn = m_factory->create();
             m_pool.push_back(conn);
+            _pool_avail++;
         }
 
         if(m_pool.size()==0){
@@ -99,6 +103,7 @@ public:
                 try {
                     std::shared_ptr<Connection> conn = m_factory->create();
                     m_borrowed.insert(conn);
+                    _pool_in_use++;
                     return std::static_pointer_cast<T>(conn);
                 } catch (std::exception &e) {
                     throw ConnectionUnavailable();
@@ -128,8 +133,10 @@ public:
         // Take one off the front
         std::shared_ptr<Connection> conn = m_pool.front();
         m_pool.pop_front();
+        _pool_avail--;
         // Add it to the borrowed list
         m_borrowed.insert(conn);
+        _pool_in_use++;
         return std::static_pointer_cast<T>(conn);
     };
 
@@ -143,7 +150,9 @@ public:
         // Lock
         std::unique_lock<std::mutex> lock(m_poolMutex);
         m_borrowed.erase(conn);
+        _pool_in_use--;
         if ((m_pool.size() + m_borrowed.size()) < m_maxPoolSize) {
+            _pool_avail++;
             m_pool.push_back(conn);
         }
     };
@@ -154,8 +163,14 @@ protected:
     std::deque<std::shared_ptr<Connection> > m_pool;
     std::set<std::shared_ptr<Connection> > m_borrowed;
     std::mutex m_poolMutex;
+
+    prometheus::simpleapi::counter_metric_t _max_pool_size { "controller_pgsql_max_conn_pool_size", "max connection pool size for postgres"};
+    prometheus::simpleapi::counter_metric_t _min_pool_size { "controller_pgsql_min_conn_pool_size", "minimum connection pool size for postgres" };
+    prometheus::simpleapi::gauge_metric_t _pool_avail { "controller_pgsql_available_conns", "number of available postgres connections" };
+    prometheus::simpleapi::gauge_metric_t _pool_in_use { "controller_pgsql_in_use_conns", "number of postgres database connections in use" };
+    prometheus::simpleapi::counter_metric_t _pool_errors { "controller_pgsql_connection_errors", "number of connection errors the connection pool has seen" };
 };
 
 }
 
-#endif
+#endif

controller/PostgreSQL.cpp

@@ -119,6 +119,7 @@ MemberNotificationReceiver::MemberNotificationReceiver(PostgreSQL *p, pqxx::conn
 
 void MemberNotificationReceiver::operator() (const std::string &payload, int packend_pid) {
 	fprintf(stderr, "Member Notification received: %s\n", payload.c_str());
+	_mem_notifications++;
 	json tmp(json::parse(payload));
 	json &ov = tmp["old_val"];
 	json &nv = tmp["new_val"];
@@ -141,6 +142,7 @@ NetworkNotificationReceiver::NetworkNotificationReceiver(PostgreSQL *p, pqxx::co
 
 void NetworkNotificationReceiver::operator() (const std::string &payload, int packend_pid) {
 	fprintf(stderr, "Network Notification received: %s\n", payload.c_str());
+	_net_notifications++;
 	json tmp(json::parse(payload));
 	json &ov = tmp["old_val"];
 	json &nv = tmp["new_val"];
@@ -1233,6 +1235,7 @@ void PostgreSQL::_networksWatcher_Redis() {
 						}
 						lastID = id;
 					}
+					_redis_net_notif++;
 				}
 			}
 		} catch (sw::redis::Error &e) {
@@ -1791,6 +1794,7 @@ uint64_t PostgreSQL::_doRedisUpdate(sw::redis::Transaction &tx, std::string &con
 			.sadd("network-nodes-all:{"+controllerId+"}:"+networkId, memberId)
 			.hmset("member:{"+controllerId+"}:"+networkId+":"+memberId, record.begin(), record.end());
 		++count;
+		_redis_mem_notif++;
 	}
 
 	// expire records from all-nodes and network-nodes member list

controller/PostgreSQL.hpp

@@ -26,6 +26,8 @@
 #include <memory>
 #include <redis++/redis++.h>
 
+#include <prometheus/simpleapi.h>
+
 extern "C" {
 typedef struct pg_conn PGconn;
 }
@@ -53,12 +55,14 @@ public:
 	}
 
 	virtual std::shared_ptr<Connection> create() {
+		_conn_counter++;
 		auto c = std::shared_ptr<PostgresConnection>(new PostgresConnection());
 		c->c = std::make_shared<pqxx::connection>(m_connString);
 		return std::static_pointer_cast<Connection>(c);
 	}
 private:
 	std::string m_connString;
+	prometheus::simpleapi::counter_metric_t _conn_counter { "controller_pgsql_connections_created", "number of pgsql connections created" };
 };
 
 class PostgreSQL;
@@ -73,6 +77,7 @@ public:
 	virtual void operator() (const std::string &payload, int backendPid);
 private:
 	PostgreSQL *_psql;
+	prometheus::simpleapi::counter_metric_t _mem_notifications { "controller_pgsql_member_notifications_received", "number of member change notifications received via pgsql NOTIFY" };
 };
 
 class NetworkNotificationReceiver : public pqxx::notification_receiver {
@@ -85,6 +90,7 @@ public:
 	virtual void operator() (const std::string &payload, int packend_pid);
 private:
 	PostgreSQL *_psql;
+	prometheus::simpleapi::counter_metric_t _net_notifications { "controller_pgsql_network_notifications_received", "number of network change notifications received via pgsql NOTIFY" };
 };
 
 /**
@@ -175,6 +181,10 @@ private:
 	std::shared_ptr<sw::redis::Redis> _redis;
 	std::shared_ptr<sw::redis::RedisCluster> _cluster;
     bool _redisMemberStatus;
+
+	prometheus::simpleapi::counter_metric_t _redis_mem_notif { "controller_redis_member_notifications_received", "number of member change notifications received via redis" };
+	prometheus::simpleapi::counter_metric_t _redis_net_notif { "controller_redis_network_notifications_received", "number of network change notifications received via redis" };
+
 };
 
 } // namespace ZeroTier