From 925599cab0600ec99dd56dabb08c4dd889f2cfd5 Mon Sep 17 00:00:00 2001
From: Grant Limberg <glimberg@users.noreply.github.com>
Date: Wed, 3 May 2023 13:43:45 -0700
Subject: [PATCH] Network-metrics (#1994)
* Add a couple quick functions for converting a uint64_t network ID/node ID into std::string
* Network metrics
---
node/Metrics.cpp | 10 ++++++++++
node/Metrics.hpp | 6 ++++++
node/Network.cpp | 20 ++++++++++++++++++--
node/Network.hpp | 8 ++++++++
osdep/OSUtils.cpp | 13 +++++++++++++
osdep/OSUtils.hpp | 16 ++++++++++++++++
6 files changed, 71 insertions(+), 2 deletions(-)
diff --git a/node/Metrics.cpp b/node/Metrics.cpp
index c90331982..7c10540e5 100644
--- a/node/Metrics.cpp
+++ b/node/Metrics.cpp
@@ -166,6 +166,16 @@ namespace ZeroTier {
prometheus::simpleapi::counter_metric_t tcp_recv
{ "zt_tcp_data_recv", "number of bytes ZeroTier has received via TCP" };
+ // Network Metrics
+ prometheus::simpleapi::gauge_metric_t network_num_joined
+ { "zt_num_networks", "number of networks this instance is joined to" };
+ prometheus::simpleapi::gauge_family_t network_num_multicast_groups
+ { "zt_network_multcast_groups_subscribed", "number of multicast groups networks are subscribed to" };
+ prometheus::simpleapi::counter_family_t network_incoming_packets
+ { "zt_network_incoming_packets", "number of incoming packets per network" };
+ prometheus::simpleapi::counter_family_t network_outgoing_packets
+ { "zt_network_outgoing_packets", "number of outgoing packets per network" };
+
// General Controller Metrics
prometheus::simpleapi::gauge_metric_t network_count
{"controller_network_count", "number of networks the controller is serving"};
diff --git a/node/Metrics.hpp b/node/Metrics.hpp
index 7530e48c0..a3efcc284 100644
--- a/node/Metrics.hpp
+++ b/node/Metrics.hpp
@@ -101,6 +101,12 @@ namespace ZeroTier {
extern prometheus::simpleapi::counter_metric_t tcp_send;
extern prometheus::simpleapi::counter_metric_t tcp_recv;
+ // Network Metrics
+ extern prometheus::simpleapi::gauge_metric_t network_num_joined;
+ extern prometheus::simpleapi::gauge_family_t network_num_multicast_groups;
+ extern prometheus::simpleapi::counter_family_t network_incoming_packets;
+ extern prometheus::simpleapi::counter_family_t network_outgoing_packets;
+
// General Controller Metrics
extern prometheus::simpleapi::gauge_metric_t network_count;
extern prometheus::simpleapi::gauge_metric_t member_count;
diff --git a/node/Network.cpp b/node/Network.cpp
index 80858126a..10436aedb 100644
--- a/node/Network.cpp
+++ b/node/Network.cpp
@@ -32,6 +32,7 @@
#include "Node.hpp"
#include "Peer.hpp"
#include "Trace.hpp"
+#include "Metrics.hpp"
#include <set>
@@ -559,13 +560,19 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u
RR(renv),
_uPtr(uptr),
_id(nwid),
+ _nwidStr(OSUtils::networkIDStr(nwid)),
_lastAnnouncedMulticastGroupsUpstream(0),
_mac(renv->identity.address(),nwid),
_portInitialized(false),
_lastConfigUpdate(0),
_destroyed(false),
_netconfFailure(NETCONF_FAILURE_NONE),
- _portError(0)
+ _portError(0),
+ _num_multicast_groups{Metrics::network_num_multicast_groups.Add({{"network_id", _nwidStr}})},
+ _incoming_packets_accpeted{Metrics::network_incoming_packets.Add({{"network_id", _nwidStr},{"accepted","yes"}})},
+ _incoming_packets_dropped{Metrics::network_incoming_packets.Add({{"network_id", _nwidStr},{"accepted","no"}})},
+ _outgoing_packets_accepted{Metrics::network_outgoing_packets.Add({{"network_id", _nwidStr},{"accepted","yes"}})},
+ _outgoing_packets_dropped{Metrics::network_outgoing_packets.Add({{"network_id", _nwidStr},{"accepted","no"}})}
{
for(int i=0;i<ZT_NETWORK_MAX_INCOMING_UPDATES;++i) {
_incomingConfigChunks[i].ts = 0;
@@ -609,13 +616,15 @@ Network::Network(const RuntimeEnvironment *renv,void *tPtr,uint64_t nwid,void *u
_portError = RR->node->configureVirtualNetworkPort(tPtr,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp);
_portInitialized = true;
}
+
+ Metrics::network_num_joined++;
}
Network::~Network()
{
ZT_VirtualNetworkConfig ctmp;
_externalConfig(&ctmp);
-
+ Metrics::network_num_joined--;
if (_destroyed) {
// This is done in Node::leave() so we can pass tPtr properly
//RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp);
@@ -705,6 +714,7 @@ bool Network::filterOutgoingPacket(
}
if (accept) {
+ _outgoing_packets_accepted++;
if ((!noTee)&&(cc)) {
Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME);
outp.append(_id);
@@ -739,6 +749,7 @@ bool Network::filterOutgoingPacket(
return true;
}
} else {
+ _outgoing_packets_dropped++;
if (_config.remoteTraceTarget) {
RR->t->networkFilter(tPtr,*this,rrl,(localCapabilityIndex >= 0) ? &crrl : (Trace::RuleResultLog *)0,(localCapabilityIndex >= 0) ? &(_config.capabilities[localCapabilityIndex]) : (Capability *)0,ztSource,ztDest,macSource,macDest,frameData,frameLen,etherType,vlanId,noTee,false,0);
}
@@ -826,6 +837,7 @@ int Network::filterIncomingPacket(
}
if (accept) {
+ _incoming_packets_accpeted++;
if (cc) {
Packet outp(cc,RR->identity.address(),Packet::VERB_EXT_FRAME);
outp.append(_id);
@@ -854,6 +866,8 @@ int Network::filterIncomingPacket(
}
return 0; // DROP locally, since we redirected
}
+ } else {
+ _incoming_packets_dropped++;
}
if (_config.remoteTraceTarget) {
@@ -879,6 +893,7 @@ void Network::multicastSubscribe(void *tPtr,const MulticastGroup &mg)
if (!std::binary_search(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg)) {
_myMulticastGroups.insert(std::upper_bound(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg),mg);
_sendUpdatesToMembers(tPtr,&mg);
+ _num_multicast_groups++;
}
}
@@ -888,6 +903,7 @@ void Network::multicastUnsubscribe(const MulticastGroup &mg)
std::vector<MulticastGroup>::iterator i(std::lower_bound(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg));
if ( (i != _myMulticastGroups.end()) && (*i == mg) ) {
_myMulticastGroups.erase(i);
+ _num_multicast_groups--;
}
}
diff --git a/node/Network.hpp b/node/Network.hpp
index a86a5c48b..676e5556e 100644
--- a/node/Network.hpp
+++ b/node/Network.hpp
@@ -37,6 +37,7 @@
#include "Membership.hpp"
#include "NetworkConfig.hpp"
#include "CertificateOfMembership.hpp"
+#include "Metrics.hpp"
#define ZT_NETWORK_MAX_INCOMING_UPDATES 3
#define ZT_NETWORK_MAX_UPDATE_CHUNKS ((ZT_NETWORKCONFIG_DICT_CAPACITY / 1024) + 1)
@@ -439,6 +440,7 @@ private:
const RuntimeEnvironment *const RR;
void *_uPtr;
const uint64_t _id;
+ std::string _nwidStr;
uint64_t _lastAnnouncedMulticastGroupsUpstream;
MAC _mac; // local MAC address
bool _portInitialized;
@@ -479,6 +481,12 @@ private:
Mutex _lock;
AtomicCounter __refCount;
+
+ prometheus::simpleapi::gauge_metric_t _num_multicast_groups;
+ prometheus::simpleapi::counter_metric_t _incoming_packets_accpeted;
+ prometheus::simpleapi::counter_metric_t _incoming_packets_dropped;
+ prometheus::simpleapi::counter_metric_t _outgoing_packets_accepted;
+ prometheus::simpleapi::counter_metric_t _outgoing_packets_dropped;
};
} // namespace ZeroTier
diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp
index e237325c4..67536c56a 100644
--- a/osdep/OSUtils.cpp
+++ b/osdep/OSUtils.cpp
@@ -17,6 +17,7 @@
#include <stdarg.h>
#include <sys/stat.h>
#include <stdlib.h>
+#include <inttypes.h>
#include "../node/Constants.hpp"
#include "../node/Utils.hpp"
@@ -66,6 +67,18 @@ unsigned int OSUtils::ztsnprintf(char *buf,unsigned int len,const char *fmt,...)
return (unsigned int)n;
}
+std::string OSUtils::networkIDStr(const uint64_t nwid) {
+ char tmp[32] = {};
+ ztsnprintf(tmp, sizeof(tmp), "%.16" PRIx64, nwid);
+ return std::string(tmp);
+}
+
+std::string OSUtils::nodeIDStr(const uint64_t nid) {
+ char tmp[32] = {};
+ ztsnprintf(tmp, sizeof(tmp), "%.10" PRIx64, nid);
+ return std::string(tmp);
+}
+
#ifdef __UNIX_LIKE__
bool OSUtils::redirectUnixOutputs(const char *stdoutPath,const char *stderrPath)
throw()
diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp
index 021b3876f..43df98cb8 100644
--- a/osdep/OSUtils.hpp
+++ b/osdep/OSUtils.hpp
@@ -68,6 +68,22 @@ public:
*/
static unsigned int ztsnprintf(char *buf,unsigned int len,const char *fmt,...);
+ /**
+ * Converts a uint64_t network ID into a string
+ *
+ * @param nwid network ID
+ * @throws std::length_error buf[] too short (buf[] will still be left null-terminated)
+ */
+ static std::string networkIDStr(const uint64_t nwid);
+
+ /**
+ * Converts a uint64_t node ID into a string
+ *
+ * @param nid node ID
+ * @throws std::length_error buf[] too short (buf[] will still be left null-terminated)
+ */
+ static std::string nodeIDStr(const uint64_t nid);
+
#ifdef __UNIX_LIKE__
/**
* Close STDOUT_FILENO and STDERR_FILENO and replace them with output to given path