mirror of
https://github.com/zerotier/ZeroTierOne.git
synced 2025-08-02 23:12:51 +02:00
8285e0f45b
- Add comprehensive VL1 (ZeroTier protocol) fragmentation metrics: * Track fragmented packets, fragments, reassembly failures * Monitor fragment ordering issues and duplicates * Histogram for fragments per packet distribution - Add VL2 (TAP/Ethernet) fragmentation metrics for virtual ethernet interfaces: * Track oversized frames from TAP devices * Monitor frames that would fragment or drop * Histogram for frame size distribution with common MTU buckets - Integration across all TAP implementations (Linux, Mac, BSD, Windows) This allows monitoring of fragmentation patterns for nodes participating as members in ZeroTier networks, helping identify MTU mismatches and optimize virtual ethernet performance.
230 lines
20 KiB
C++
230 lines
20 KiB
C++
/*
|
|
* Copyright (c)2013-2023 ZeroTier, Inc.
|
|
*
|
|
* Use of this software is governed by the Business Source License included
|
|
* in the LICENSE.TXT file in the project's root directory.
|
|
*
|
|
* Change Date: 2026-01-01
|
|
*
|
|
* On the date above, in accordance with the Business Source License, use
|
|
* of this software will be governed by version 2.0 of the Apache License.
|
|
*/
|
|
|
|
// clang-format off
|
|
#include <prometheus/simpleapi.h>
|
|
#include <prometheus/histogram.h>
|
|
#include "Metrics.hpp"
|
|
// clang-format on
|
|
|
|
namespace prometheus {
|
|
namespace simpleapi {
|
|
std::shared_ptr<Registry> registry_ptr = std::make_shared<Registry>();
|
|
Registry& registry = *registry_ptr;
|
|
SaveToFile saver;
|
|
} // namespace simpleapi
|
|
} // namespace prometheus
|
|
|
|
namespace ZeroTier {
|
|
namespace Metrics {
|
|
// Packet Type Counts
|
|
prometheus::simpleapi::counter_family_t packets { "zt_packet", "ZeroTier packet type counts" };
|
|
|
|
// Incoming packets
|
|
prometheus::simpleapi::counter_metric_t pkt_nop_in { packets.Add({ { "packet_type", "nop" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_in { packets.Add({ { "packet_type", "error" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_ack_in { packets.Add({ { "packet_type", "ack" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_qos_in { packets.Add({ { "packet_type", "qos" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_hello_in { packets.Add({ { "packet_type", "hello" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_ok_in { packets.Add({ { "packet_type", "ok" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_whois_in { packets.Add({ { "packet_type", "whois" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_rendezvous_in { packets.Add({ { "packet_type", "rendezvous" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_frame_in { packets.Add({ { "packet_type", "frame" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_ext_frame_in { packets.Add({ { "packet_type", "ext_frame" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_echo_in { packets.Add({ { "packet_type", "echo" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_multicast_like_in { packets.Add({ { "packet_type", "multicast_like" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_network_credentials_in { packets.Add({ { "packet_type", "network_credentials" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_network_config_request_in { packets.Add({ { "packet_type", "network_config_request" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_network_config_in { packets.Add({ { "packet_type", "network_config" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_multicast_gather_in { packets.Add({ { "packet_type", "multicast_gather" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_multicast_frame_in { packets.Add({ { "packet_type", "multicast_frame" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_push_direct_paths_in { packets.Add({ { "packet_type", "push_direct_paths" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_user_message_in { packets.Add({ { "packet_type", "user_message" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_remote_trace_in { packets.Add({ { "packet_type", "remote_trace" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_path_negotiation_request_in { packets.Add({ { "packet_type", "path_negotiation_request" }, { "direction", "rx" } }) };
|
|
|
|
// Outgoing packets
|
|
prometheus::simpleapi::counter_metric_t pkt_nop_out { packets.Add({ { "packet_type", "nop" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_out { packets.Add({ { "packet_type", "error" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_ack_out { packets.Add({ { "packet_type", "ack" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_qos_out { packets.Add({ { "packet_type", "qos" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_hello_out { packets.Add({ { "packet_type", "hello" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_ok_out { packets.Add({ { "packet_type", "ok" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_whois_out { packets.Add({ { "packet_type", "whois" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_rendezvous_out { packets.Add({ { "packet_type", "rendezvous" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_frame_out { packets.Add({ { "packet_type", "frame" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_ext_frame_out { packets.Add({ { "packet_type", "ext_frame" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_echo_out { packets.Add({ { "packet_type", "echo" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_multicast_like_out { packets.Add({ { "packet_type", "multicast_like" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_network_credentials_out { packets.Add({ { "packet_type", "network_credentials" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_network_config_request_out { packets.Add({ { "packet_type", "network_config_request" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_network_config_out { packets.Add({ { "packet_type", "network_config" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_multicast_gather_out { packets.Add({ { "packet_type", "multicast_gather" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_multicast_frame_out { packets.Add({ { "packet_type", "multicast_frame" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_push_direct_paths_out { packets.Add({ { "packet_type", "push_direct_paths" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_user_message_out { packets.Add({ { "packet_type", "user_message" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_remote_trace_out { packets.Add({ { "packet_type", "remote_trace" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_path_negotiation_request_out { packets.Add({ { "packet_type", "path_negotiation_request" }, { "direction", "tx" } }) };
|
|
|
|
// Packet Error Counts
|
|
prometheus::simpleapi::counter_family_t packet_errors { "zt_packet_error", "ZeroTier packet errors" };
|
|
|
|
// Incoming Error Counts
|
|
prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found_in { packet_errors.Add({ { "error_type", "obj_not_found" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_unsupported_op_in { packet_errors.Add({ { "error_type", "unsupported_operation" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_identity_collision_in { packet_errors.Add({ { "error_type", "identity_collision" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_need_membership_cert_in { packet_errors.Add({ { "error_type", "need_membership_certificate" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied_in { packet_errors.Add({ { "error_type", "network_access_denied" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast_in { packet_errors.Add({ { "error_type", "unwanted_multicast" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_in { packet_errors.Add({ { "error_type", "authentication_required" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_in { packet_errors.Add({ { "error_type", "internal_server_error" }, { "direction", "rx" } }) };
|
|
|
|
// Outgoing Error Counts
|
|
prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found_out { packet_errors.Add({ { "error_type", "obj_not_found" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_unsupported_op_out { packet_errors.Add({ { "error_type", "unsupported_operation" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_identity_collision_out { packet_errors.Add({ { "error_type", "identity_collision" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_need_membership_cert_out { packet_errors.Add({ { "error_type", "need_membership_certificate" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied_out { packet_errors.Add({ { "error_type", "network_access_denied" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast_out { packet_errors.Add({ { "error_type", "unwanted_multicast" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_out { packet_errors.Add({ { "error_type", "authentication_required" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_out { packet_errors.Add({ { "error_type", "internal_server_error" }, { "direction", "tx" } }) };
|
|
|
|
// Data Sent/Received Metrics
|
|
prometheus::simpleapi::counter_family_t data { "zt_data", "number of bytes ZeroTier has transmitted or received" };
|
|
prometheus::simpleapi::counter_metric_t udp_recv { data.Add({ { "protocol", "udp" }, { "direction", "rx" } }) };
|
|
prometheus::simpleapi::counter_metric_t udp_send { data.Add({ { "protocol", "udp" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t tcp_send { data.Add({ { "protocol", "tcp" }, { "direction", "tx" } }) };
|
|
prometheus::simpleapi::counter_metric_t tcp_recv { data.Add({ { "protocol", "tcp" }, { "direction", "rx" } }) };
|
|
|
|
// Network Metrics
|
|
prometheus::simpleapi::gauge_metric_t network_num_joined { "zt_num_networks", "number of networks this instance is joined to" };
|
|
prometheus::simpleapi::gauge_family_t network_num_multicast_groups { "zt_network_multicast_groups_subscribed", "number of multicast groups networks are subscribed to" };
|
|
prometheus::simpleapi::counter_family_t network_packets { "zt_network_packets", "number of incoming/outgoing packets per network" };
|
|
|
|
#ifndef ZT_NO_PEER_METRICS
|
|
// PeerMetrics
|
|
prometheus::CustomFamily<prometheus::Histogram<uint64_t> >& peer_latency = prometheus::Builder<prometheus::Histogram<uint64_t> >().Name("zt_peer_latency").Help("peer latency (ms)").Register(prometheus::simpleapi::registry);
|
|
|
|
prometheus::simpleapi::gauge_family_t peer_path_count { "zt_peer_path_count", "number of paths to peer" };
|
|
prometheus::simpleapi::counter_family_t peer_packets { "zt_peer_packets", "number of packets to/from a peer" };
|
|
prometheus::simpleapi::counter_family_t peer_packet_errors { "zt_peer_packet_errors", "number of incoming packet errors from a peer" };
|
|
#endif
|
|
|
|
// General Controller Metrics
|
|
prometheus::simpleapi::gauge_metric_t network_count { "controller_network_count", "number of networks the controller is serving" };
|
|
prometheus::simpleapi::gauge_metric_t member_count { "controller_member_count", "number of network members the controller is serving" };
|
|
prometheus::simpleapi::counter_metric_t network_changes { "controller_network_change_count", "number of times a network configuration is changed" };
|
|
prometheus::simpleapi::counter_metric_t member_changes { "controller_member_change_count", "number of times a network member configuration is changed" };
|
|
prometheus::simpleapi::counter_metric_t member_auths { "controller_member_auth_count", "number of network member auths" };
|
|
prometheus::simpleapi::counter_metric_t member_deauths { "controller_member_deauth_count", "number of network member deauths" };
|
|
|
|
prometheus::simpleapi::gauge_metric_t network_config_request_queue_size { "controller_network_config_request_queue", "number of entries in the request queue for network configurations" };
|
|
|
|
prometheus::simpleapi::counter_metric_t sso_expiration_checks { "controller_sso_expiration_checks", "number of sso expiration checks done" };
|
|
|
|
prometheus::simpleapi::counter_metric_t sso_member_deauth { "controller_sso_timeouts", "number of sso timeouts" };
|
|
|
|
prometheus::simpleapi::counter_metric_t network_config_request { "controller_network_config_request", "count of config requests handled" };
|
|
prometheus::simpleapi::gauge_metric_t network_config_request_threads { "controller_network_config_request_threads", "number of active network config handling threads" };
|
|
prometheus::simpleapi::counter_metric_t db_get_network { "controller_db_get_network", "counter" };
|
|
prometheus::simpleapi::counter_metric_t db_get_network_and_member { "controller_db_get_network_and_member", "counter" };
|
|
prometheus::simpleapi::counter_metric_t db_get_network_and_member_and_summary { "controller_db_get_networK_and_member_summary", "counter" };
|
|
prometheus::simpleapi::counter_metric_t db_get_member_list { "controller_db_get_member_list", "counter" };
|
|
prometheus::simpleapi::counter_metric_t db_get_network_list { "controller_db_get_network_list", "counter" };
|
|
prometheus::simpleapi::counter_metric_t db_member_change { "controller_db_member_change", "counter" };
|
|
prometheus::simpleapi::counter_metric_t db_network_change { "controller_db_network_change", "counter" };
|
|
|
|
#ifdef ZT_CONTROLLER_USE_LIBPQ
|
|
// Central Controller Metrics
|
|
prometheus::simpleapi::counter_metric_t pgsql_mem_notification { "controller_pgsql_member_notifications_received", "number of member change notifications received via pgsql NOTIFY" };
|
|
prometheus::simpleapi::counter_metric_t pgsql_net_notification { "controller_pgsql_network_notifications_received", "number of network change notifications received via pgsql NOTIFY" };
|
|
prometheus::simpleapi::counter_metric_t pgsql_node_checkin { "controller_pgsql_node_checkin_count", "number of node check-ins (pgsql)" };
|
|
prometheus::simpleapi::counter_metric_t pgsql_commit_ticks { "controller_pgsql_commit_ticks", "number of commit ticks run (pgsql)" };
|
|
prometheus::simpleapi::counter_metric_t db_get_sso_info { "controller_db_get_sso_info", "counter" };
|
|
|
|
prometheus::simpleapi::counter_metric_t redis_mem_notification { "controller_redis_member_notifications_received", "number of member change notifications received via redis" };
|
|
prometheus::simpleapi::counter_metric_t redis_net_notification { "controller_redis_network_notifications_received", "number of network change notifications received via redis" };
|
|
prometheus::simpleapi::counter_metric_t redis_node_checkin { "controller_redis_node_checkin_count", "number of node check-ins (redis)" };
|
|
|
|
// Central DB Pool Metrics
|
|
prometheus::simpleapi::counter_metric_t conn_counter { "controller_pgsql_connections_created", "number of pgsql connections created" };
|
|
prometheus::simpleapi::counter_metric_t max_pool_size { "controller_pgsql_max_conn_pool_size", "max connection pool size for postgres" };
|
|
prometheus::simpleapi::counter_metric_t min_pool_size { "controller_pgsql_min_conn_pool_size", "minimum connection pool size for postgres" };
|
|
prometheus::simpleapi::gauge_metric_t pool_avail { "controller_pgsql_available_conns", "number of available postgres connections" };
|
|
prometheus::simpleapi::gauge_metric_t pool_in_use { "controller_pgsql_in_use_conns", "number of postgres database connections in use" };
|
|
prometheus::simpleapi::counter_metric_t pool_errors { "controller_pgsql_connection_errors", "number of connection errors the connection pool has seen" };
|
|
#endif
|
|
|
|
// Fragmentation Metrics
|
|
prometheus::simpleapi::counter_family_t packet_fragmentation
|
|
{ "zt_packet_fragmentation", "ZeroTier packet fragmentation events" };
|
|
|
|
// VL2 Fragmentation Metrics
|
|
prometheus::simpleapi::counter_metric_t vl2_oversized_frame_tx
|
|
{ packet_fragmentation.Add({{"layer", "VL2"}, {"direction", "tx"}, {"reason", "oversized_frame"}}) };
|
|
prometheus::simpleapi::counter_metric_t vl2_would_fragment_or_drop_rx
|
|
{ packet_fragmentation.Add({{"layer", "VL2"}, {"direction", "rx"}, {"reason", "would_fragment_or_drop"}}) };
|
|
|
|
// VL1 Fragmentation Metrics
|
|
prometheus::simpleapi::counter_metric_t vl1_fragmented_tx
|
|
{ packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "tx"}, {"reason", "mtu_exceeded"}}) };
|
|
prometheus::simpleapi::counter_metric_t vl1_fragment_rx
|
|
{ packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "rx"}, {"reason", "fragment"}}) };
|
|
prometheus::simpleapi::counter_metric_t vl1_reassembly_failed_rx
|
|
{ packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "rx"}, {"reason", "reassembly_failed"}}) };
|
|
prometheus::simpleapi::counter_metric_t vl1_fragment_without_head_rx
|
|
{ packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "rx"}, {"reason", "fragment_without_head"}}) };
|
|
prometheus::simpleapi::counter_metric_t vl1_fragment_before_head_rx
|
|
{ packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "rx"}, {"reason", "fragment_before_head"}}) };
|
|
prometheus::simpleapi::counter_metric_t vl1_duplicate_fragment_rx
|
|
{ packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "rx"}, {"reason", "duplicate_fragment"}}) };
|
|
prometheus::simpleapi::counter_metric_t vl1_duplicate_head_rx
|
|
{ packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "rx"}, {"reason", "duplicate_head"}}) };
|
|
|
|
// VL1 Fragmentation Histogram and Counters
|
|
prometheus::CustomFamily<prometheus::Histogram<uint64_t>> &vl1_fragments_per_packet_histogram =
|
|
prometheus::Builder<prometheus::Histogram<uint64_t>>()
|
|
.Name("zt_vl1_fragments_per_packet")
|
|
.Help("Histogram of fragments per packet at VL1")
|
|
.Register(prometheus::simpleapi::registry);
|
|
prometheus::simpleapi::counter_metric_t vl1_incomplete_reassembly_rx
|
|
{ packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "rx"}, {"reason", "incomplete_reassembly"}}) };
|
|
prometheus::simpleapi::counter_metric_t vl1_vl2_double_fragmentation_tx
|
|
{ packet_fragmentation.Add({{"layer", "VL1_VL2"}, {"direction", "tx"}, {"reason", "double_fragmentation"}}) };
|
|
|
|
prometheus::Histogram<uint64_t> &vl1_fragments_per_packet_hist =
|
|
vl1_fragments_per_packet_histogram.Add(
|
|
{},
|
|
std::vector<uint64_t>(std::begin(ZeroTier::Metrics::VL1_FRAGMENTS_PER_PACKET_BUCKETS), std::end(ZeroTier::Metrics::VL1_FRAGMENTS_PER_PACKET_BUCKETS))
|
|
);
|
|
|
|
// VL2 Frame Size Histogram
|
|
// Buckets: 512 (IoT/legacy), 576 (min IPv4), 1200 (QUIC/mobile), 1280 (min IPv6),
|
|
// 1332, 1380, 1400 (VPN/overlay), 1420 (cloud), 1460 (TCP MSS), 1472 (ICMP/MTU),
|
|
// 1480 (ICMP/MTU), 1492 (PPPoE), 1500 (Ethernet), 2800 (VL2 default), 9000 (jumbo)
|
|
prometheus::CustomFamily<prometheus::Histogram<uint64_t>> &vl2_frame_size_histogram =
|
|
prometheus::Builder<prometheus::Histogram<uint64_t>>()
|
|
.Name("zt_vl2_frame_size")
|
|
.Help("Histogram of frame sizes delivered to TAP (VL2)")
|
|
.Register(prometheus::simpleapi::registry);
|
|
prometheus::simpleapi::counter_metric_t vl2_incomplete_reassembly_rx
|
|
{ packet_fragmentation.Add({{"layer", "VL2"}, {"direction", "rx"}, {"reason", "incomplete_reassembly"}}) };
|
|
prometheus::simpleapi::counter_metric_t vl2_vl1_double_fragmentation_tx
|
|
{ packet_fragmentation.Add({{"layer", "VL2_VL1"}, {"direction", "tx"}, {"reason", "double_fragmentation"}}) };
|
|
|
|
prometheus::Histogram<uint64_t> &vl2_frame_size_hist =
|
|
vl2_frame_size_histogram.Add(
|
|
{},
|
|
std::vector<uint64_t>(std::begin(ZeroTier::Metrics::VL2_FRAME_SIZE_BUCKETS), std::end(ZeroTier::Metrics::VL2_FRAME_SIZE_BUCKETS))
|
|
);
|
|
}
|
|
}
|