From 02947fb8b0bff71495d6541b11fd3f617aa069f7 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Thu, 20 Apr 2023 11:39:08 -0700 Subject: [PATCH] Consolidate metrics definitions Put all metric definitions into node/Metrics.hpp. Accessed as needed from there. --- controller/ConnectionPool.hpp | 28 ++++++++------------ controller/DB.cpp | 28 ++++++++------------ controller/DB.hpp | 7 ----- controller/FileDB.cpp | 10 +++++-- controller/PostgreSQL.cpp | 16 +++++------ controller/PostgreSQL.hpp | 11 ++------ node/Metrics.hpp | 50 +++++++++++++++++++++++++++++++++++ 7 files changed, 88 insertions(+), 62 deletions(-) create mode 100644 node/Metrics.hpp diff --git a/controller/ConnectionPool.hpp b/controller/ConnectionPool.hpp index b8e076087..97a20639b 100644 --- a/controller/ConnectionPool.hpp +++ b/controller/ConnectionPool.hpp @@ -19,7 +19,7 @@ #define _DEBUG(x) #endif -#include +#include "../node/Metrics.hpp" #include #include @@ -63,7 +63,7 @@ public: { while(m_pool.size() < m_minPoolSize){ m_pool.push_back(m_factory->create()); - _pool_avail++; + Metrics::pool_avail++; } }; @@ -94,7 +94,7 @@ public: while((m_pool.size() + m_borrowed.size()) < m_minPoolSize) { std::shared_ptr conn = m_factory->create(); m_pool.push_back(conn); - _pool_avail++; + Metrics::pool_avail++; } if(m_pool.size()==0){ @@ -103,10 +103,10 @@ public: try { std::shared_ptr conn = m_factory->create(); m_borrowed.insert(conn); - _pool_in_use++; + Metrics::pool_in_use++; return std::static_pointer_cast(conn); } catch (std::exception &e) { - _pool_errors++; + Metrics::pool_errors++; throw ConnectionUnavailable(); } } else { @@ -122,13 +122,13 @@ public: return std::static_pointer_cast(conn); } catch(std::exception& e) { // Error creating a replacement connection - _pool_errors++; + Metrics::pool_errors++; throw ConnectionUnavailable(); } } } // Nothing available - _pool_errors++; + Metrics::pool_errors++; throw ConnectionUnavailable(); } } @@ -136,10 +136,10 @@ public: // Take one off the front std::shared_ptr conn = m_pool.front(); m_pool.pop_front(); - _pool_avail--; + Metrics::pool_avail--; // Add it to the borrowed list m_borrowed.insert(conn); - _pool_in_use++; + Metrics::pool_in_use++; return std::static_pointer_cast(conn); }; @@ -153,9 +153,9 @@ public: // Lock std::unique_lock lock(m_poolMutex); m_borrowed.erase(conn); - _pool_in_use--; + Metrics::pool_in_use--; if ((m_pool.size() + m_borrowed.size()) < m_maxPoolSize) { - _pool_avail++; + Metrics::pool_avail++; m_pool.push_back(conn); } }; @@ -166,12 +166,6 @@ protected: std::deque > m_pool; std::set > m_borrowed; std::mutex m_poolMutex; - - prometheus::simpleapi::counter_metric_t _max_pool_size { "controller_pgsql_max_conn_pool_size", "max connection pool size for postgres"}; - prometheus::simpleapi::counter_metric_t _min_pool_size { "controller_pgsql_min_conn_pool_size", "minimum connection pool size for postgres" }; - prometheus::simpleapi::gauge_metric_t _pool_avail { "controller_pgsql_available_conns", "number of available postgres connections" }; - prometheus::simpleapi::gauge_metric_t _pool_in_use { "controller_pgsql_in_use_conns", "number of postgres database connections in use" }; - prometheus::simpleapi::counter_metric_t _pool_errors { "controller_pgsql_connection_errors", "number of connection errors the connection pool has seen" }; }; } diff --git a/controller/DB.cpp b/controller/DB.cpp index 9e0cef6e2..1de2fbe8b 100644 --- a/controller/DB.cpp +++ b/controller/DB.cpp @@ -13,6 +13,7 @@ #include "DB.hpp" #include "EmbeddedNetworkController.hpp" +#include "../node/Metrics.hpp" #include #include @@ -101,14 +102,7 @@ void DB::cleanMember(nlohmann::json &member) member.erase("authenticationClientID"); // computed } -DB::DB() - : _network_count{"controller_network_count", "number of networks the controller is serving"} - , _member_count{"controller_member_count", "number of network members the controller is serving"} - , _network_changes{"controller_network_change_count", "number of times a network configuration is changed"} - , _member_changes{"controller_member_change_count", "number of times a network member configuration is changed"} - , _member_auths{"controller_member_auth_count", "number of network member auths"} - , _member_deauths{"controller_member_deauth_count", "number of network member deauths"} -{} +DB::DB() {} DB::~DB() {} bool DB::get(const uint64_t networkId,nlohmann::json &network) @@ -270,7 +264,7 @@ void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool no } isAuth = OSUtils::jsonBool(memberConfig["authorized"],false); if (isAuth) { - _member_auths++; + Metrics::member_auths++; nw->authorizedMembers.insert(memberId); } json &ips = memberConfig["ipAssignments"]; @@ -319,18 +313,18 @@ void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool no if (notifyListeners) { if(networkId != 0 && memberId != 0 && old.is_object() && !memberConfig.is_object()) { // member delete - _member_count--; + Metrics::member_count--; } else if (networkId != 0 && memberId != 0 && !old.is_object() && memberConfig.is_object()) { // new member - _member_count++; + Metrics::member_count++; } if (!wasAuth && isAuth) { - _member_auths++; + Metrics::member_auths++; } else if (wasAuth && !isAuth) { - _member_deauths++; + Metrics::member_deauths++; } else { - _member_changes++; + Metrics::member_changes++; } } @@ -346,11 +340,11 @@ void DB::_networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool { if (notifyListeners) { if (old.is_object() && old.contains("id") && networkConfig.is_object() && networkConfig.contains("id")) { - _network_changes++; + Metrics::network_changes++; } else if (!old.is_object() && networkConfig.is_object() && networkConfig.contains("id")) { - _network_count++; + Metrics::network_count++; } else if (old.is_object() && old.contains("id") && !networkConfig.is_object()) { - _network_count--; + Metrics::network_count--; } } diff --git a/controller/DB.hpp b/controller/DB.hpp index 2f4319c8b..f70d66e03 100644 --- a/controller/DB.hpp +++ b/controller/DB.hpp @@ -190,13 +190,6 @@ protected: std::unordered_multimap< uint64_t,uint64_t > _networkByMember; mutable std::mutex _changeListeners_l; mutable std::mutex _networks_l; - - prometheus::simpleapi::gauge_metric_t _network_count; - prometheus::simpleapi::gauge_metric_t _member_count; - prometheus::simpleapi::counter_metric_t _network_changes; - prometheus::simpleapi::counter_metric_t _member_changes; - prometheus::simpleapi::counter_metric_t _member_auths; - prometheus::simpleapi::counter_metric_t _member_deauths; }; } // namespace ZeroTier diff --git a/controller/FileDB.cpp b/controller/FileDB.cpp index d454e93e1..0ced1226a 100644 --- a/controller/FileDB.cpp +++ b/controller/FileDB.cpp @@ -13,6 +13,8 @@ #include "FileDB.hpp" +#include "../node/Metrics.hpp" + namespace ZeroTier { @@ -39,6 +41,7 @@ FileDB::FileDB(const char *path) : if (nwids.length() == 16) { nlohmann::json nullJson; _networkChanged(nullJson,network,false); + Metrics::network_count++; std::string membersPath(_networksPath + ZT_PATH_SEPARATOR_S + nwids + ZT_PATH_SEPARATOR_S "member"); std::vector members(OSUtils::listDirectory(membersPath.c_str(),false)); for(auto m=members.begin();m!=members.end();++m) { @@ -50,6 +53,7 @@ FileDB::FileDB(const char *path) : if (addrs.length() == 10) { nlohmann::json nullJson2; _memberChanged(nullJson2,member,false); + Metrics::member_count++; } } catch ( ... ) {} } @@ -88,8 +92,9 @@ bool FileDB::save(nlohmann::json &record,bool notifyListeners) if ((!old.is_object())||(!_compareRecords(old,record))) { record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1ULL; OSUtils::ztsnprintf(p1,sizeof(p1),"%s" ZT_PATH_SEPARATOR_S "%.16llx.json",_networksPath.c_str(),nwid); - if (!OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1))) + if (!OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1))) { fprintf(stderr,"WARNING: controller unable to write to path: %s" ZT_EOL_S,p1); + } _networkChanged(old,record,notifyListeners); modified = true; } @@ -110,8 +115,9 @@ bool FileDB::save(nlohmann::json &record,bool notifyListeners) OSUtils::ztsnprintf(p2,sizeof(p2),"%s" ZT_PATH_SEPARATOR_S "%.16llx",_networksPath.c_str(),(unsigned long long)nwid); OSUtils::mkdir(p2); OSUtils::mkdir(pb); - if (!OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1))) + if (!OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1))) { fprintf(stderr,"WARNING: controller unable to write to path: %s" ZT_EOL_S,p1); + } } _memberChanged(old,record,notifyListeners); modified = true; diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index 29250ab78..ae705bb33 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -112,7 +112,6 @@ using namespace ZeroTier; MemberNotificationReceiver::MemberNotificationReceiver(PostgreSQL *p, pqxx::connection &c, const std::string &channel) : pqxx::notification_receiver(c, channel) , _psql(p) - , _mem_notifications{"controller_pgsql_member_notifications_received", "number of member change notifications received via pgsql NOTIFY"} { fprintf(stderr, "initialize MemberNotificationReceiver\n"); } @@ -120,7 +119,7 @@ MemberNotificationReceiver::MemberNotificationReceiver(PostgreSQL *p, pqxx::conn void MemberNotificationReceiver::operator() (const std::string &payload, int packend_pid) { fprintf(stderr, "Member Notification received: %s\n", payload.c_str()); - _mem_notifications++; + Metrics::pgsql_mem_notification++; json tmp(json::parse(payload)); json &ov = tmp["old_val"]; json &nv = tmp["new_val"]; @@ -137,14 +136,13 @@ void MemberNotificationReceiver::operator() (const std::string &payload, int pac NetworkNotificationReceiver::NetworkNotificationReceiver(PostgreSQL *p, pqxx::connection &c, const std::string &channel) : pqxx::notification_receiver(c, channel) , _psql(p) - , _net_notifications{"controller_pgsql_network_notifications_received", "number of network change notifications received via pgsql NOTIFY"} { fprintf(stderr, "initialize NetworkNotificationReceiver\n"); } void NetworkNotificationReceiver::operator() (const std::string &payload, int packend_pid) { fprintf(stderr, "Network Notification received: %s\n", payload.c_str()); - _net_notifications++; + Metrics::pgsql_net_notification++; json tmp(json::parse(payload)); json &ov = tmp["old_val"]; json &nv = tmp["new_val"]; @@ -175,8 +173,6 @@ PostgreSQL::PostgreSQL(const Identity &myId, const char *path, int listenPort, R , _redis(NULL) , _cluster(NULL) , _redisMemberStatus(false) - , _redis_mem_notif{"controller_redis_member_notifications_received", "number of member change notifications received via redis"} - , _redis_net_notif{"controller_redis_network_notifications_received", "number of network change notifications received via redis"} { char myAddress[64]; _myAddressStr = myId.address().toString(myAddress); @@ -711,7 +707,7 @@ void PostgreSQL::initializeNetworks() } } - _network_count++; + Metrics::network_count++; _networkChanged(empty, config, false); @@ -933,7 +929,7 @@ void PostgreSQL::initializeMembers() } } - _member_count++; + Metrics::member_count++; _memberChanged(empty, config, false); @@ -1239,7 +1235,7 @@ void PostgreSQL::_networksWatcher_Redis() { } lastID = id; } - _redis_net_notif++; + Metrics::redis_net_notification++; } } } catch (sw::redis::Error &e) { @@ -1798,7 +1794,7 @@ uint64_t PostgreSQL::_doRedisUpdate(sw::redis::Transaction &tx, std::string &con .sadd("network-nodes-all:{"+controllerId+"}:"+networkId, memberId) .hmset("member:{"+controllerId+"}:"+networkId+":"+memberId, record.begin(), record.end()); ++count; - _redis_mem_notif++; + Metrics::redis_mem_notification++; } // expire records from all-nodes and network-nodes member list diff --git a/controller/PostgreSQL.hpp b/controller/PostgreSQL.hpp index 3ff2857dd..8eea3608a 100644 --- a/controller/PostgreSQL.hpp +++ b/controller/PostgreSQL.hpp @@ -26,7 +26,7 @@ #include #include -#include +#include "../node/Metrics.hpp" extern "C" { typedef struct pg_conn PGconn; @@ -51,19 +51,17 @@ class PostgresConnFactory : public ConnectionFactory { public: PostgresConnFactory(std::string &connString) : m_connString(connString) - , _conn_counter{ "controller_pgsql_connections_created", "number of pgsql connections created"} { } virtual std::shared_ptr create() { - _conn_counter++; + Metrics::conn_counter++; auto c = std::shared_ptr(new PostgresConnection()); c->c = std::make_shared(m_connString); return std::static_pointer_cast(c); } private: std::string m_connString; - prometheus::simpleapi::counter_metric_t _conn_counter; }; class PostgreSQL; @@ -78,7 +76,6 @@ public: virtual void operator() (const std::string &payload, int backendPid); private: PostgreSQL *_psql; - prometheus::simpleapi::counter_metric_t _mem_notifications; }; class NetworkNotificationReceiver : public pqxx::notification_receiver { @@ -91,7 +88,6 @@ public: virtual void operator() (const std::string &payload, int packend_pid); private: PostgreSQL *_psql; - prometheus::simpleapi::counter_metric_t _net_notifications; }; /** @@ -182,9 +178,6 @@ private: std::shared_ptr _redis; std::shared_ptr _cluster; bool _redisMemberStatus; - - prometheus::simpleapi::counter_metric_t _redis_mem_notif; - prometheus::simpleapi::counter_metric_t _redis_net_notif; }; } // namespace ZeroTier diff --git a/node/Metrics.hpp b/node/Metrics.hpp new file mode 100644 index 000000000..9c7af45a6 --- /dev/null +++ b/node/Metrics.hpp @@ -0,0 +1,50 @@ +#ifndef METRICS_H_ +#define METRICS_H_ + +#include + +namespace ZeroTier { + namespace Metrics { + // General Controller Metrics + static prometheus::simpleapi::gauge_metric_t network_count + {"controller_network_count", "number of networks the controller is serving"}; + static prometheus::simpleapi::gauge_metric_t member_count + {"controller_member_count", "number of network members the controller is serving"}; + static prometheus::simpleapi::counter_metric_t network_changes + {"controller_network_change_count", "number of times a network configuration is changed"}; + static prometheus::simpleapi::counter_metric_t member_changes + {"controller_member_change_count", "number of times a network member configuration is changed"}; + static prometheus::simpleapi::counter_metric_t member_auths + {"controller_member_auth_count", "number of network member auths"}; + static prometheus::simpleapi::counter_metric_t member_deauths + {"controller_member_deauth_count", "number of network member deauths"}; + +#ifdef ZT_CONTROLLER_USE_LIBPQ + // Central Controller Metrics + static prometheus::simpleapi::counter_metric_t pgsql_mem_notification + { "controller_pgsql_member_notifications_received", "number of member change notifications received via pgsql NOTIFY" }; + static prometheus::simpleapi::counter_metric_t pgsql_net_notification + { "controller_pgsql_network_notifications_received", "number of network change notifications received via pgsql NOTIFY" }; + static prometheus::simpleapi::counter_metric_t redis_mem_notification + { "controller_redis_member_notifications_received", "number of member change notifications received via redis" }; + static prometheus::simpleapi::counter_metric_t redis_net_notification + { "controller_redis_network_notifications_received", "number of network change notifications received via redis" }; + + // Central DB Pool Metrics + static prometheus::simpleapi::counter_metric_t conn_counter + { "controller_pgsql_connections_created", "number of pgsql connections created"}; + static prometheus::simpleapi::counter_metric_t max_pool_size + { "controller_pgsql_max_conn_pool_size", "max connection pool size for postgres"}; + static prometheus::simpleapi::counter_metric_t min_pool_size + { "controller_pgsql_min_conn_pool_size", "minimum connection pool size for postgres" }; + static prometheus::simpleapi::gauge_metric_t pool_avail + { "controller_pgsql_available_conns", "number of available postgres connections" }; + static prometheus::simpleapi::gauge_metric_t pool_in_use + { "controller_pgsql_in_use_conns", "number of postgres database connections in use" }; + static prometheus::simpleapi::counter_metric_t pool_errors + { "controller_pgsql_connection_errors", "number of connection errors the connection pool has seen" }; +#endif + } // namespace Metrics +}// namespace ZeroTier + +#endif // METRICS_H_