A ton more work... almost there

include/ZeroTierCore.h

@@ -419,183 +419,44 @@ enum ZT_TraceCredentialRejectionReason
 	ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID = 4
 };
 
-#if 0
+// Fields used in trace output dictionaries. Which fields are present depends on
-
+// the trace event type. All trace dictionaries contain TYPE and CODE_LOCATION.
-/**
+#define ZT_TRACE_FIELD_TYPE "t"
- * Physical path address from a trace event
+#define ZT_TRACE_FIELD_CODE_LOCATION "@"
- *
+#define ZT_TRACE_FIELD_ENDPOINT "e"
- * This is a special packed address format that roughly mirrors Endpoint in the core
+#define ZT_TRACE_FIELD_OLD_ENDPOINT "oe"
- * and is designed to support both present and future address types.
+#define ZT_TRACE_FIELD_NEW_ENDPOINT "ne"
- */
+#define ZT_TRACE_FIELD_TRIGGER_FROM_ENDPOINT "te"
-ZT_PACKED_STRUCT(struct ZT_TraceEventPathAddress
+#define ZT_TRACE_FIELD_TRIGGER_FROM_PACKET_ID "ti"
-{
+#define ZT_TRACE_FIELD_TRIGGER_FROM_PACKET_VERB "tv"
-	uint8_t type;        /* ZT_TraceEventPathAddressType */
+#define ZT_TRACE_FIELD_TRIGGER_FROM_PEER_FINGERPRINT_HASH "tp"
-	uint8_t address[63]; /* Type-dependent address: 4-byte IPv4, 16-byte IPV6, etc. */
+#define ZT_TRACE_FIELD_MESSAGE "m"
-	uint16_t port;       /* UDP/TCP port for address types for which this is meaningful */
+#define ZT_TRACE_FIELD_RESET_ADDRESS_SCOPE "rs"
-});
+#define ZT_TRACE_FIELD_IDENTITY_FINGERPRINT_HASH "f"
-
+#define ZT_TRACE_FIELD_PACKET_ID "p"
-/**
+#define ZT_TRACE_FIELD_PACKET_VERB "v"
- * Header for all trace events
+#define ZT_TRACE_FIELD_PACKET_HOPS "h"
- *
+#define ZT_TRACE_FIELD_NETWORK_ID "n"
- * All packet types begin with these fields in this order.
+#define ZT_TRACE_FIELD_REASON "r"
- */
+#define ZT_TRACE_FIELD_SOURCE_MAC "sm"
-ZT_PACKED_STRUCT(struct ZT_TraceEvent
+#define ZT_TRACE_FIELD_DEST_MAC "dm"
-{
+#define ZT_TRACE_FIELD_ETHERTYPE "et"
-	uint16_t evSize;                                  /* sizeof(ZT_TraceEvent_XX structure) (inclusive) */
+#define ZT_TRACE_FIELD_VLAN_ID "vlid"
-	uint16_t evType;                                  /* ZT_TraceEventType */
+#define ZT_TRACE_FIELD_FRAME_LENGTH "fl"
-	uint32_t codeLocation;                            /* arbitrary identifier of location in source code */
+#define ZT_TRACE_FIELD_FRAME_DATA "fd"
-});
+#define ZT_TRACE_FIELD_FLAG_CREDENTIAL_REQUEST_SENT "crs"
-
+#define ZT_TRACE_FIELD_PRIMARY_RULE_SET_LOG "rL"
-/* Temporary macros to make it easier to declare all ZT_TraceEvent's sub-types */
+#define ZT_TRACE_FIELD_MATCHING_CAPABILITY_RULE_SET_LOG "caRL"
-#define _ZT_TRACE_EVENT_STRUCT_START(e) ZT_PACKED_STRUCT_START struct ZT_TraceEvent_##e { \
+#define ZT_TRACE_FIELD_MATCHING_CAPABILITY_ID "caID"
-	uint16_t evSize; \
+#define ZT_TRACE_FIELD_MATCHING_CAPABILITY_TIMESTAMP "caTS"
-	uint16_t evType; \
+#define ZT_TRACE_FIELD_SOURCE_ZT_ADDRESS "sz"
-	uint32_t codeLocation;
+#define ZT_TRACE_FIELD_DEST_ZT_ADDRESS "dz"
-#define _ZT_TRACE_EVENT_STRUCT_END() } ZT_PACKED_STRUCT_END;
+#define ZT_TRACE_FIELD_RULE_FLAG_NOTEE "rNT"
-
+#define ZT_TRACE_FIELD_RULE_FLAG_INBOUND "rIN"
-/**
+#define ZT_TRACE_FIELD_RULE_FLAG_ACCEPT "rACC"
- * An unexpected or internal error occurred
+#define ZT_TRACE_FIELD_CREDENTIAL_ID "crID"
- */
+#define ZT_TRACE_FIELD_CREDENTIAL_TYPE "crT"
-_ZT_TRACE_EVENT_STRUCT_START(UNEXPECTED_ERROR)
+#define ZT_TRACE_FIELD_CREDENTIAL_TIMESTAMP "crTS"
-	char message[256];                               /* arbitrary human-readable message */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * Node is resetting all paths in a given address scope
- *
- * This happens when the node detects and external surface IP addressing change
- * via a trusted (usually root) peer. It's used to renegotiate links when nodes
- * move around on the network.
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL1_RESETTING_PATHS_IN_SCOPE)
-	ZT_Fingerprint reporter;                         /* node that triggered the reset */
-	struct ZT_TraceEventPathAddress from;            /* physical origin of triggering packet */
-	struct ZT_TraceEventPathAddress oldExternal;     /* previous detected external address */
-	struct ZT_TraceEventPathAddress newExternal;     /* new detected external address */
-	uint8_t scope;                                   /* IP scope being reset */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * Node is trying a new path
- *
- * Paths are tried in response to PUSH_DIRECT_PATHS, RENDEZVOUS, and other places
- * we might hear of them. A node tries a path by sending a trial message to it.
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL1_TRYING_NEW_PATH)
-	ZT_Fingerprint peer;                             /* node we're trying to reach */
-	struct ZT_TraceEventPathAddress physicalAddress; /* physical address being tried */
-	struct ZT_TraceEventPathAddress triggerAddress;  /* physical origin of triggering packet */
-	uint64_t triggeringPacketId;                     /* packet ID of triggering packet */
-	uint8_t triggeringPacketVerb;                    /* packet verb of triggering packet */
-	ZT_Fingerprint triggeringPeer;                   /* peer that triggered attempt */
-	uint8_t reason;                                  /* ZT_TraceTryingNewPathReason */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * Node has learned a new path to another node
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL1_LEARNED_NEW_PATH)
-	uint64_t packetId;                               /* packet ID of confirming packet */
-	ZT_Fingerprint peer;                             /* peer on other side of new path */
-	struct ZT_TraceEventPathAddress physicalAddress; /* physical address learned */
-	struct ZT_TraceEventPathAddress replaced;        /* if non-empty, an older address that was replaced */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * An incoming packet was dropped at the VL1 level
- *
- * This indicates a packet that passed MAC check but was dropped for some other
- * reason such as rate limits, being malformed, etc.
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL1_INCOMING_PACKET_DROPPED)
-	uint64_t packetId;                               /* packet ID of failed packet */
-	uint64_t networkId;                              /* VL2 network ID or 0 if unrelated to a network or unknown */
-	ZT_Fingerprint peer;                             /* peer that sent packet */
-	struct ZT_TraceEventPathAddress physicalAddress; /* physical origin address of packet */
-	uint8_t hops;                                    /* hop count of packet */
-	uint8_t verb;                                    /* packet verb */
-	uint8_t reason;                                  /* ZT_TracePacketDropReason */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * Node declined to send a packet read from a local network port/tap
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL2_OUTGOING_FRAME_DROPPED)
-	uint64_t networkId;                              /* network ID */
-	uint64_t sourceMac;                              /* source MAC address */
-	uint64_t destMac;                                /* destination MAC address */
-	uint16_t etherType;                              /* Ethernet type of frame */
-	uint16_t frameLength;                            /* length of dropped frame */
-	uint8_t frameHead[64];                           /* first up to 64 bytes of dropped frame */
-	uint8_t reason;                                  /* ZT_TraceFrameDropReason */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * An incoming frame was dropped
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL2_INCOMING_FRAME_DROPPED)
-	uint64_t packetId;                               /* VL1 packet ID */
-	uint64_t networkId;                              /* VL2 network ID */
-	uint64_t sourceMac;                              /* 48-bit source MAC */
-	uint64_t destMac;                                /* 48-bit destination MAC */
-	ZT_Fingerprint sender;                           /* sending peer */
-	struct ZT_TraceEventPathAddress physicalAddress; /* physical source address of packet */
-	uint8_t hops;                                    /* hop count of packet */
-	uint16_t frameLength;                            /* length of frame in bytes */
-	uint8_t frameHead[64];                           /* first up to 64 bytes of dropped frame */
-	uint8_t verb;                                    /* packet verb indicating how frame was sent */
-	uint8_t credentialRequestSent;                   /* if non-zero a request for credentials was sent */
-	uint8_t reason;                                  /* ZT_TraceFrameDropReason */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * Node is requesting a new network config and certificate from a network controller
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL2_NETWORK_CONFIG_REQUESTED)
-	uint64_t networkId;                              /* VL2 network ID */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * Network filter trace results
- *
- * These are generated when filter tracing is enabled to allow filters to be debugged.
- * Format for rule set logs is documented elsewhere.
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL2_NETWORK_FILTER)
-	uint64_t networkId;                              /* VL2 network ID */
-	uint8_t primaryRuleSetLog[512];                  /* primary rule set log */
-	uint8_t matchingCapabilityRuleSetLog[512];       /* capability rule set log (if any) */
-	uint32_t matchingCapabilityId;                   /* capability ID or 0 if none */
-	int64_t matchingCapabilityTimestamp;             /* capability timestamp or 0 if none */
-	uint64_t source;                                 /* source ZeroTier address */
-	uint64_t dest;                                   /* destination ZeroTier address */
-	uint64_t sourceMac;                              /* packet source MAC */
-	uint64_t destMac;                                /* packet destination MAC */
-	uint16_t frameLength;                            /* length of filtered frame */
-	uint8_t frameHead[64];                           /* first up to 64 bytes of filtered frame */
-	uint16_t etherType;                              /* frame Ethernet type */
-	uint16_t vlanId;                                 /* frame VLAN ID (currently unused, always 0) */
-	uint8_t noTee;                                   /* if true noTee flag was set in filter */
-	uint8_t inbound;                                 /* direction: 1 for inbound, 0 for outbound */
-	int8_t accept;                                   /* 0: drop, 1: accept, 2: "super-accept" */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * An incoming credential from a peer was rejected
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL2_CREDENTIAL_REJECTED)
-	uint64_t networkId;                              /* VL2 network ID */
-	ZT_Fingerprint peer;                             /* sending peer */
-	uint32_t credentialId;                           /* credential ID */
-	int64_t credentialTimestamp;                     /* credential timestamp */
-	uint8_t credentialType;                          /* credential type */
-	uint8_t reason;                                  /* ZT_TraceCredentialRejectionReason */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-#undef _ZT_TRACE_EVENT_STRUCT_START
-#undef _ZT_TRACE_EVENT_STRUCT_END
-
-#endif
 
 /****************************************************************************/
 

node/AES.hpp

@@ -56,10 +56,7 @@ public:
 	/**
 	 * Create an un-initialized AES instance (must call init() before use)
 	 */
-	ZT_INLINE AES() noexcept
+	ZT_INLINE AES() noexcept {}
-	{
-		Utils::memoryLock(this,sizeof(AES));
-	}
 
 	/**
 	 * Create an AES instance with the given key
@@ -68,14 +65,12 @@ public:
 	 */
 	explicit ZT_INLINE AES(const void *const key) noexcept
 	{
-		Utils::memoryLock(this,sizeof(AES));
 		this->init(key);
 	}
 
 	ZT_INLINE ~AES()
 	{
 		Utils::burn(&_k,sizeof(_k));
-		Utils::memoryUnlock(this,sizeof(AES));
 	}
 
 	/**

node/Address.hpp

@@ -19,7 +19,9 @@
 #include "TriviallyCopyable.hpp"
 #include "Containers.hpp"
 
-#define ZT_ADDRESS_STRING_SIZE_MAX 11
+#define ZT_ADDRESS_STRING_SIZE_MAX (ZT_ADDRESS_LENGTH_HEX + 1)
+
+static_assert(ZT_ADDRESS_LENGTH == 5,"parts of Address will need modification for any change in ZT_ADDRESS_LENGTH");
 
 namespace ZeroTier {
 

node/Buf.hpp

@@ -86,6 +86,15 @@ public:
 	static void *operator new(std::size_t sz);
 	static void operator delete(void *ptr);
 
+	/**
+	 * Raw data held in buffer
+	 *
+	 * The additional eight bytes should not be used and should be considered undefined.
+	 * They exist to allow reads and writes of integer types to silently overflow if a
+	 * read or write is performed at the end of the buffer.
+	 */
+	uint8_t unsafeData[ZT_BUF_MEM_SIZE + 8];
+
 	/**
 	 * Free all instances of Buf in shared pool.
 	 *
@@ -129,47 +138,82 @@ public:
 	};
 
 	/**
-	 * Assemble all slices in a vector into a single slice starting at position 0
+	 * A vector of slices making up a packet that might span more than one buffer.
-	 *
-	 * The returned slice will start at 0 and contain the entire vector unless the
-	 * vector is too large to fit in a single buffer. If that or any other error
-	 * occurs the returned slice will be empty and contain a NULL Buf.
-	 *
-	 * The vector may be modified by this function and should be considered
-	 * undefined after it is called.
-	 *
-	 * @tparam FCVC Capacity of FCV (generally inferred automatically)
-	 * @param fcv FCV containing one or more slices
-	 * @return Single slice containing fully assembled buffer (empty on error)
 	 */
-	template<unsigned int FCVC>
+	class PacketVector : public ZeroTier::FCV<Slice,ZT_MAX_PACKET_FRAGMENTS>
-	static ZT_INLINE Buf::Slice assembleSliceVector(FCV<Buf::Slice,FCVC> &fcv) noexcept
 	{
-		Buf::Slice r;
+	public:
+		ZT_INLINE PacketVector() : ZeroTier::FCV<Slice,ZT_MAX_PACKET_FRAGMENTS>() {}
 
-		typename FCV<Buf::Slice,FCVC>::iterator s(fcv.begin());
+		ZT_INLINE unsigned int totalSize() const noexcept
-		unsigned int l = s->e - s->s;
+		{
-		if (l <= ZT_BUF_MEM_SIZE) {
+			unsigned int size = 0;
-			r.b.move(s->b);
+			for(PacketVector::const_iterator s(begin());s!=end();++s)
-			if (s->s > 0)
+				size += s->e - s->s;
-				memmove(r.b->unsafeData,r.b->unsafeData + s->s,l);
+			return size;
-			r.e = l;
-
-			while (++s != fcv.end()) {
-				l = s->e - s->s;
-				if (l > (ZT_BUF_MEM_SIZE - r.e)) {
-					r.b.zero();
-					r.e = 0;
-					break;
-				}
-				Utils::copy(r.b->unsafeData + r.e,s->b->unsafeData + s->s,l);
-				s->b.zero(); // let go of buffer in vector as soon as possible
-				r.e += l;
-			}
 		}
 
-		return r;
+		/**
-	}
+		 * Merge this packet vector into a single destination buffer
+		 * 
+		 * @param b Destination buffer
+		 * @return Size of data in destination or -1 on error
+		 */
+		ZT_INLINE int mergeCopy(Buf &b) const noexcept
+		{
+			unsigned int size = 0;
+			for(PacketVector::const_iterator s(begin());s!=end();++s) {
+				const unsigned int start = s->s;
+				const unsigned int rem = s->e - start;
+				if (likely((size + rem) <= ZT_BUF_MEM_SIZE)) {
+					Utils::copy(b.unsafeData + size,s->b->unsafeData + start,rem);
+					size += rem;
+				} else {
+					return -1;
+				}
+			}
+			return (int)size;
+		}
+
+		/**
+		 * Merge this packet vector into a single destination buffer with an arbitrary copy function
+		 * 
+		 * This can be used to e.g. simultaneously merge and decrypt a packet.
+		 * 
+		 * @param b Destination buffer
+		 * @param simpleCopyBefore Don't start using copyFunction until this index (0 to always use)
+		 * @param copyFunction Function to invoke with memcpy-like arguments: (dest, source, size)
+		 * @tparam F Type of copyFunction (typically inferred)
+		 * @return Size of data in destination or -1 on error
+		 */
+		template<typename F>
+		ZT_INLINE int mergeMap(Buf &b,const unsigned int simpleCopyBefore,F copyFunction) const noexcept
+		{
+			unsigned int size = 0;
+			for(PacketVector::const_iterator s(begin());s!=end();++s) {
+				unsigned int start = s->s;
+				unsigned int rem = s->e - start;
+				if (likely((size + rem) <= ZT_BUF_MEM_SIZE)) {
+					if (size < simpleCopyBefore) {
+						unsigned int sc = simpleCopyBefore - size;
+						if (unlikely(sc > rem))
+							sc = rem;
+						Utils::copy(b.unsafeData + size,s->b->unsafeData + start,sc);
+						start += sc;
+						rem -= sc;
+					}
+
+					if (likely(rem > 0)) {
+						copyFunction(b.unsafeData + size,s->b->unsafeData + start,rem);
+						size += rem;
+					}
+				} else {
+					return -1;
+				}
+			}
+			return (int)size;
+		}
+	};
 
 	/**
 	 * Create a new uninitialized buffer with undefined contents (use clear() to zero if needed)
@@ -421,6 +465,84 @@ public:
 		return ((ii += (int)len) <= ZT_BUF_MEM_SIZE) ? b : nullptr;
 	}
 
+	/**
+	 * Load a value at an index that is compile time checked against the maximum buffer size
+	 * 
+	 * @tparam I Static index
+	 * @return Value
+	 */
+	template<unsigned int I>
+	ZT_INLINE uint8_t lI8() const noexcept
+	{
+		static_assert(I < ZT_BUF_MEM_SIZE,"overflow");
+		return unsafeData[I];
+	}
+
+	/**
+	 * Load a value at an index that is compile time checked against the maximum buffer size
+	 * 
+	 * @tparam I Static index
+	 * @return Value
+	 */
+	template<unsigned int I>
+	ZT_INLINE uint8_t lI16() const noexcept
+	{
+		static_assert((I + 1) < ZT_BUF_MEM_SIZE,"overflow");
+#ifdef ZT_NO_UNALIGNED_ACCESS
+		return (
+			((uint16_t)unsafeData[I] << 8U) |
+			(uint16_t)unsafeData[I + 1]);
+#else
+		return Utils::ntoh(*reinterpret_cast<const uint16_t *>(unsafeData + I));
+#endif
+	}
+
+	/**
+	 * Load a value at an index that is compile time checked against the maximum buffer size
+	 * 
+	 * @tparam I Static index
+	 * @return Value
+	 */
+	template<unsigned int I>
+	ZT_INLINE uint8_t lI32() const noexcept
+	{
+		static_assert((I + 3) < ZT_BUF_MEM_SIZE,"overflow");
+#ifdef ZT_NO_UNALIGNED_ACCESS
+		return (
+			((uint32_t)unsafeData[I] << 24U) |
+			((uint32_t)unsafeData[I + 1] << 16U) |
+			((uint32_t)unsafeData[I + 2] << 8U) |
+			(uint32_t)unsafeData[I + 3]);
+#else
+		return Utils::ntoh(*reinterpret_cast<const uint32_t *>(unsafeData + I));
+#endif
+	}
+
+	/**
+	 * Load a value at an index that is compile time checked against the maximum buffer size
+	 * 
+	 * @tparam I Static index
+	 * @return Value
+	 */
+	template<unsigned int I>
+	ZT_INLINE uint8_t lI64() const noexcept
+	{
+		static_assert((I + 7) < ZT_BUF_MEM_SIZE,"overflow");
+#ifdef ZT_NO_UNALIGNED_ACCESS
+		return (
+			((uint64_t)unsafeData[I] << 56U) |
+			((uint64_t)unsafeData[I + 1] << 48U) |
+			((uint64_t)unsafeData[I + 2] << 40U) |
+			((uint64_t)unsafeData[I + 3] << 32U) |
+			((uint64_t)unsafeData[I + 4] << 24U) |
+			((uint64_t)unsafeData[I + 5] << 16U) |
+			((uint64_t)unsafeData[I + 6] << 8U) |
+			(uint64_t)unsafeData[I + 7]);
+#else
+		return Utils::ntoh(*reinterpret_cast<const uint64_t *>(unsafeData + I));
+#endif
+	}
+
 	/**
 	 * Load a value at an index without advancing the index
 	 *
@@ -688,43 +810,6 @@ public:
 	 */
 	static constexpr unsigned int capacity() noexcept { return ZT_BUF_MEM_SIZE; }
 
-	/**
-	 * Cast data in 'b' to a (usually packed) structure type
-	 *
-	 * Warning: this does no bounds checking. It should only be used with packed
-	 * struct types designed for use in packet decoding such as those in
-	 * Protocol.hpp, and if 'i' is non-zero the caller must check bounds.
-	 *
-	 * @tparam T Structure type to cast 'b' to
-	 * @param i Index of start of structure (default: 0)
-	 * @return Reference to 'b' cast to type T
-	 */
-	template<typename T>
-	ZT_INLINE T &as(const unsigned int i = 0) noexcept { return *reinterpret_cast<T *>(unsafeData + i); }
-
-	/**
-	 * Cast data in 'b' to a (usually packed) structure type (const)
-	 *
-	 * Warning: this does no bounds checking. It should only be used with packed
-	 * struct types designed for use in packet decoding such as those in
-	 * Protocol.hpp, and if 'i' is non-zero the caller must check bounds.
-	 *
-	 * @tparam T Structure type to cast 'b' to
-	 * @param i Index of start of structure (default: 0)
-	 * @return Reference to 'b' cast to type T
-	 */
-	template<typename T>
-	ZT_INLINE const T &as(const unsigned int i = 0) const noexcept { return *reinterpret_cast<const T *>(unsafeData + i); }
-
-	/**
-	 * Raw data held in buffer
-	 *
-	 * The additional eight bytes should not be used and should be considered undefined.
-	 * They exist to allow reads and writes of integer types to silently overflow if a
-	 * read or write is performed at the end of the buffer.
-	 */
-	uint8_t unsafeData[ZT_BUF_MEM_SIZE + 8];
-
 private:
 	// Next item in free buffer pool linked list if Buf is placed in pool, undefined and unused otherwise
 	std::atomic<uintptr_t> __nextInPool;

node/CMakeLists.txt

@@ -74,7 +74,6 @@ set(core_src
 	Path.cpp
 	Peer.cpp
 	Poly1305.cpp
-	Protocol.cpp
 	Revocation.cpp
 	Salsa20.cpp
 	SelfAwareness.cpp

node/Constants.hpp

@@ -36,6 +36,11 @@
  */
 #define ZT_ADDRESS_LENGTH 5
 
+/**
+ * Length of a ZeroTier address in digits
+ */
+#define ZT_ADDRESS_LENGTH_HEX 10
+
 /**
  * Addresses beginning with this byte are reserved for the joy of in-band signaling
  */
@@ -72,9 +77,9 @@
 #define ZT_MAX_NETWORK_CONFIG_BYTES 131072
 
 /**
- * Length of symmetric keys (currently all symmetric crypto is 256 bit).
+ * Length of symmetric keys
  */
-#define ZT_SYMMETRIC_KEY_SIZE 32
+#define ZT_SYMMETRIC_KEY_SIZE 48
 
 /**
  * Time limit for ephemeral keys: 30 minutes.
@@ -202,6 +207,11 @@
  */
 #define ZT_PEER_GENERAL_RATE_LIMIT 500
 
+/**
+ * Rate limit for responses to short probes to prevent amplification attacks
+ */
+#define ZT_PEER_PROBE_RESPONSE_RATE_LIMIT 5000
+
 /**
  * Don't do expensive identity validation more often than this
  *

node/Dictionary.cpp

@@ -19,70 +19,46 @@ Dictionary::Dictionary()
 {
 }
 
-std::vector<uint8_t> &Dictionary::operator[](const char *k)
+Vector<uint8_t> &Dictionary::operator[](const char *k)
 {
 	return m_entries[s_toKey(k)];
 }
 
-const std::vector<uint8_t> &Dictionary::operator[](const char *k) const
+const Vector<uint8_t> &Dictionary::operator[](const char *k) const
 {
-	static const std::vector<uint8_t> emptyEntry;
+	static const Vector<uint8_t> s_emptyEntry;
-	Map< uint64_t,std::vector<uint8_t> >::const_iterator e(m_entries.find(s_toKey(k)));
+	Map< uint64_t,Vector<uint8_t> >::const_iterator e(m_entries.find(s_toKey(k)));
-	return (e == m_entries.end()) ? emptyEntry : e->second;
+	return (e == m_entries.end()) ? s_emptyEntry : e->second;
 }
 
 void Dictionary::add(const char *k,bool v)
 {
-	std::vector<uint8_t> &e = (*this)[k];
+	Vector<uint8_t> &e = (*this)[k];
 	e.resize(2);
 	e[0] = (uint8_t)(v ? '1' : '0');
 	e[1] = 0;
 }
 
-void Dictionary::add(const char *k,uint16_t v)
-{
-	std::vector<uint8_t> &e = (*this)[k];
-	e.resize(5);
-	Utils::hex(v,(char *)e.data());
-}
-
-void Dictionary::add(const char *k,uint32_t v)
-{
-	std::vector<uint8_t> &e = (*this)[k];
-	e.resize(9);
-	Utils::hex(v,(char *)e.data());
-}
-
-void Dictionary::add(const char *k,uint64_t v)
-{
-	std::vector<uint8_t> &e = (*this)[k];
-	e.resize(17);
-	Utils::hex(v,(char *)e.data());
-}
-
 void Dictionary::add(const char *k,const Address &v)
 {
-	std::vector<uint8_t> &e = (*this)[k];
+	Vector<uint8_t> &e = (*this)[k];
 	e.resize(ZT_ADDRESS_STRING_SIZE_MAX);
 	v.toString((char *)e.data());
 }
 
 void Dictionary::add(const char *k,const char *v)
 {
-	std::vector<uint8_t> &e = (*this)[k];
+	if ((v)&&(*v)) {
-	e.clear();
+		Vector<uint8_t> &e = (*this)[k];
-	if (v) {
+		e.clear();
-		for(;;) {
+		while (*v)
-			const uint8_t c = (uint8_t)*(v++);
+			e.push_back((uint8_t)*(v++));
-			e.push_back(c);
-			if (!c) break;
-		}
 	}
 }
 
 void Dictionary::add(const char *k,const void *data,unsigned int len)
 {
-	std::vector<uint8_t> &e = (*this)[k];
+	Vector<uint8_t> &e = (*this)[k];
 	if (len != 0) {
 		e.assign((const uint8_t *)data,(const uint8_t *)data + len);
 	} else {
@@ -92,7 +68,7 @@ void Dictionary::add(const char *k,const void *data,unsigned int len)
 
 bool Dictionary::getB(const char *k,bool dfl) const
 {
-	const std::vector<uint8_t> &e = (*this)[k];
+	const Vector<uint8_t> &e = (*this)[k];
 	if (!e.empty()) {
 		switch ((char)e[0]) {
 			case '1':
@@ -112,7 +88,7 @@ uint64_t Dictionary::getUI(const char *k,uint64_t dfl) const
 {
 	uint8_t tmp[18];
 	uint64_t v = dfl;
-	const std::vector<uint8_t> &e = (*this)[k];
+	const Vector<uint8_t> &e = (*this)[k];
 	if (!e.empty()) {
 		if (e.back() != 0) {
 			const unsigned long sl = e.size();
@@ -125,11 +101,11 @@ uint64_t Dictionary::getUI(const char *k,uint64_t dfl) const
 	return v;
 }
 
-void Dictionary::getS(const char *k,char *v,unsigned int cap) const
+void Dictionary::getS(const char *k,char *v,const unsigned int cap) const
 {
 	if (cap == 0) // sanity check
 		return;
-	const std::vector<uint8_t> &e = (*this)[k];
+	const Vector<uint8_t> &e = (*this)[k];
 	unsigned int i = 0;
 	const unsigned int last = cap - 1;
 	for(;;) {
@@ -148,52 +124,14 @@ void Dictionary::clear()
 
 void Dictionary::encode(Vector<uint8_t> &out) const
 {
-	uint64_t str[2] = { 0,0 }; // second entry causes all strings to be null-terminated even if 8 chars in length
+	uint64_t str[2] = { 0,0 }; // second uint64_t being 0 means all strings always 0-terminated
-
 	out.clear();
-
+	for(Map< uint64_t,Vector<uint8_t> >::const_iterator ti(m_entries.begin());ti != m_entries.end();++ti) {
-	for(Map< uint64_t,std::vector<uint8_t> >::const_iterator ti(m_entries.begin());ti != m_entries.end();++ti) {
 		str[0] = ti->first;
-		const char *k = (const char *)str;
+		s_appendKey(out,reinterpret_cast<const char *>(str));
-		for(;;) {
+		for(std::vector<uint8_t>::const_iterator i(ti->second.begin());i!=ti->second.end();++i)
-			char kc = *(k++);
+			s_appendValueByte(out,*i);
-			if (!kc) break;
+		out.push_back((uint8_t)'\n');
-			if ((kc >= 33)&&(kc <= 126)&&(kc != 61)&&(kc != 92)) // printable ASCII with no spaces, equals, or backslash
-				out.push_back((uint8_t)kc);
-		}
-
-		out.push_back(61); // =
-
-		for(std::vector<uint8_t>::const_iterator i(ti->second.begin());i!=ti->second.end();++i) {
-			uint8_t c = *i;
-			switch(c) {
-				case 0:
-					out.push_back(92);
-					out.push_back(48);
-					break;
-				case 10:
-					out.push_back(92);
-					out.push_back(110);
-					break;
-				case 13:
-					out.push_back(92);
-					out.push_back(114);
-					break;
-				case 61:
-					out.push_back(92);
-					out.push_back(101);
-					break;
-				case 92:
-					out.push_back(92);
-					out.push_back(92);
-					break;
-				default:
-					out.push_back(c);
-					break;
-			}
-		}
-
-		out.push_back(10);
 	}
 }
 
@@ -203,7 +141,7 @@ bool Dictionary::decode(const void *data,unsigned int len)
 
 	uint64_t k = 0;
 	unsigned int ki = 0;
-	std::vector<uint8_t> *v = nullptr;
+	Vector<uint8_t> *v = nullptr;
 	bool escape = false;
 	for(unsigned int di=0;di<len;++di) {
 		uint8_t c = reinterpret_cast<const uint8_t *>(data)[di];
@@ -229,11 +167,11 @@ bool Dictionary::decode(const void *data,unsigned int len)
 						break;
 				}
 			} else {
-				if (c == 10) {
+				if (c == (uint8_t)'\n') {
 					k = 0;
 					ki = 0;
 					v = nullptr;
-				} else if (c == 92) {
+				} else if (c == 92) { // backslash
 					escape = true;
 				} else {
 					v->push_back(c);
@@ -242,7 +180,7 @@ bool Dictionary::decode(const void *data,unsigned int len)
 		} else {
 			if ((c < 33)||(c > 126)||(c == 92)) {
 				return false;
-			} else if (c == 61) {
+			} else if (c == (uint8_t)'=') {
 				v = &m_entries[k];
 			} else {
 				reinterpret_cast<uint8_t *>(&k)[ki & 7U] ^= c;

node/Dictionary.hpp

@@ -35,6 +35,9 @@ namespace ZeroTier {
  * of simple or standardized binary encoding. Nevertheless it is efficient
  * and it works so there is no need to change it and break backward
  * compatibility.
+ * 
+ * Use of the append functions is faster than building and then encoding a
+ * dictionary.
  */
 class Dictionary
 {
@@ -47,7 +50,7 @@ public:
 	 * @param k Key to look up
 	 * @return Reference to value
 	 */
-	std::vector<uint8_t> &operator[](const char *k);
+	Vector<uint8_t> &operator[](const char *k);
 
 	/**
 	 * Get a const reference to a value
@@ -55,7 +58,7 @@ public:
 	 * @param k Key to look up
 	 * @return Reference to value or to empty vector if not found
 	 */
-	const std::vector<uint8_t> &operator[](const char *k) const;
+	const Vector<uint8_t> &operator[](const char *k) const;
 
 	/**
 	 * Add a boolean as '1' or '0'
@@ -65,21 +68,12 @@ public:
 	/**
 	 * Add an integer as a hexadecimal string value
 	 */
-	void add(const char *k,uint16_t v);
+	ZT_INLINE void add(const char *const k,const uint64_t v) { char buf[17]; add(k,Utils::hex(v,buf)); }
-
+	ZT_INLINE void add(const char *const k,const int64_t v) { char buf[17]; add(k,Utils::hex((uint64_t)v,buf)); }
-	/**
+	ZT_INLINE void add(const char *const k,const uint32_t v) { char buf[17]; add(k,Utils::hex((uint64_t)v,buf)); }
-	 * Add an integer as a hexadecimal string value
+	ZT_INLINE void add(const char *const k,const int32_t v) { char buf[17]; add(k,Utils::hex((uint64_t)v,buf)); }
-	 */
+	ZT_INLINE void add(const char *const k,const uint16_t v) { char buf[17]; add(k,Utils::hex((uint64_t)v,buf)); }
-	void add(const char *k,uint32_t v);
+	ZT_INLINE void add(const char *const k,const int16_t v) { char buf[17]; add(k,Utils::hex((uint64_t)v,buf)); }
-
-	/**
-	 * Add an integer as a hexadecimal string value
-	 */
-	void add(const char *k,uint64_t v);
-
-	ZT_INLINE void add(const char *k,int16_t v) { add(k,(uint16_t)v); }
-	ZT_INLINE void add(const char *k,int32_t v) { add(k,(uint32_t)v); }
-	ZT_INLINE void add(const char *k,int64_t v) { add(k,(uint64_t)v); }
 
 	/**
 	 * Add an address in 10-digit hex string format
@@ -126,6 +120,24 @@ public:
 	 */
 	void getS(const char *k,char *v,unsigned int cap) const;
 
+	/**
+	 * Get an object supporting the marshal/unmarshal interface pattern
+	 * 
+	 * @param k Key to look up
+	 * @param obj Object to unmarshal() into
+	 * @return True if unmarshal was successful
+	 */
+	template<typename T>
+	ZT_INLINE bool getO(const char *k,T &obj) const
+	{
+		const Vector<uint8_t> &d = (*this)[k];
+		if (d.empty())
+			return false;
+		if (obj.unmarshal(d.data(),(unsigned int)d.size()) <= 0)
+			return false;
+		return true;
+	}
+
 	/**
 	 * Erase all entries in dictionary
 	 */
@@ -163,7 +175,190 @@ public:
 	 */
 	bool decode(const void *data,unsigned int len);
 
+	/**
+	 * Append a key=value pair to a buffer (vector or FCV)
+	 * 
+	 * @param out Buffer
+	 * @param k Key (must be <= 8 characters)
+	 * @param v Value
+	 */
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const bool v)
+	{
+		s_appendKey(out,k);
+		out.push_back((uint8_t)(v ? '1' : '0'));
+		out.push_back((uint8_t)'\n');
+	}
+
+	/**
+	 * Append a key=value pair to a buffer (vector or FCV)
+	 * 
+	 * @param out Buffer
+	 * @param k Key (must be <= 8 characters)
+	 * @param v Value
+	 */
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const Address v)
+	{
+		s_appendKey(out,k);
+		const uint64_t a = v.toInt();
+		static_assert(ZT_ADDRESS_LENGTH_HEX == 10,"this must be rewritten for any change in address length");
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 36U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 32U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 28U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 24U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 20U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 16U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 12U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 8U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 4U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[a & 0xfU]);
+		out.push_back((uint8_t)'\n');
+	}
+
+	/**
+	 * Append a key=value pair to a buffer (vector or FCV)
+	 * 
+	 * @param out Buffer
+	 * @param k Key (must be <= 8 characters)
+	 * @param v Value
+	 */
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const uint64_t v)
+	{
+		char buf[17];
+		Utils::hex(v,buf);
+		unsigned int i = 0;
+		while (buf[i])
+			out.push_back((uint8_t)buf[i++]);
+		out.push_back((uint8_t)'\n');
+	}
+
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const int64_t v) { append(out,k,(uint64_t)v); }
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const uint32_t v) { append(out,k,(uint64_t)v); }
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const int32_t v) { append(out,k,(uint64_t)v); }
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const uint16_t v) { append(out,k,(uint64_t)v); }
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const int16_t v) { append(out,k,(uint64_t)v); }
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const uint8_t v) { append(out,k,(uint64_t)v); }
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const int8_t v) { append(out,k,(uint64_t)v); }
+
+	/**
+	 * Append a key=value pair to a buffer (vector or FCV)
+	 * 
+	 * @param out Buffer
+	 * @param k Key (must be <= 8 characters)
+	 * @param v Value
+	 */
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const char *v)
+	{
+		if ((v)&&(*v)) {
+			s_appendKey(out,k);
+			while (*v)
+				s_appendValueByte(out,(uint8_t)*(v++));
+			out.push_back((uint8_t)'\n');
+		}
+	}
+
+	/**
+	 * Append a key=value pair to a buffer (vector or FCV)
+	 * 
+	 * @param out Buffer
+	 * @param k Key (must be <= 8 characters)
+	 * @param v Value
+	 * @param vlen Value length in bytes
+	 */
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const void *const v,const unsigned int vlen)
+	{
+		s_appendKey(out,k);
+		for(unsigned int i=0;i<vlen;++i)
+			s_appendValueByte(out,reinterpret_cast<const uint8_t *>(v)[i]);
+		out.push_back((uint8_t)'\n');
+	}
+
+	/**
+	 * Append a packet ID as raw bytes in the provided byte order
+	 * 
+	 * @param out Buffer
+	 * @param k Key (must be <= 8 characters)
+	 * @param pid Packet ID
+	 */
+	template<typename V>
+	static ZT_INLINE void appendPacketId(V &out,const char *const k,const uint64_t pid)
+	{
+		append(out,k,&pid,8);
+	}
+
+	/**
+	 * Append key=value with any object implementing the correct marshal interface
+	 * 
+	 * @param out Buffer
+	 * @param k Key (must be <= 8 characters)
+	 * @param v Marshal-able object
+	 * @return Bytes appended or negative on error (return value of marshal())
+	 */
+	template<typename V,typename T>
+	static ZT_INLINE int appendObject(V &out,const char *const k,const T &v)
+	{
+		uint8_t tmp[4096]; // large enough for any current object
+		if (T::marshalSizeMax() > sizeof(tmp))
+			return -1;
+		const int mlen = v.marshal(tmp);
+		if (mlen > 0)
+			append(out,k,tmp,(unsigned int)mlen);
+		return mlen;
+	}
+
 private:
+	template<typename V>
+	ZT_INLINE static void s_appendValueByte(V &out,const uint8_t c)
+	{
+		switch(c) {
+			case 0:
+				out.push_back(92); // backslash
+				out.push_back(48);
+				break;
+			case 10:
+				out.push_back(92);
+				out.push_back(110);
+				break;
+			case 13:
+				out.push_back(92);
+				out.push_back(114);
+				break;
+			case 61:
+				out.push_back(92);
+				out.push_back(101);
+				break;
+			case 92:
+				out.push_back(92);
+				out.push_back(92);
+				break;
+			default:
+				out.push_back(c);
+				break;
+		}
+	}
+	template<typename V>
+	ZT_INLINE static void s_appendKey(V &out,const char *const k)
+	{
+		for(unsigned int i=0;i<8;++i) {
+			const char kc = k[i];
+			if (!kc) break;
+			if ((kc >= 33)&&(kc <= 126)&&(kc != 61)&&(kc != 92)) // printable ASCII with no spaces, equals, or backslash
+				out.push_back((uint8_t)kc);
+		}
+		out.push_back((uint8_t)'=');
+	}
+
 	// This just packs up to 8 character bytes into a 64-bit word. There is no need
 	// for this to be portable in terms of endian-ness. It's just for fast key lookup.
 	static ZT_INLINE uint64_t s_toKey(const char *k)

node/FCV.hpp

@@ -128,6 +128,8 @@ public:
 
 	ZT_INLINE unsigned int size() const noexcept { return _s; }
 	ZT_INLINE bool empty() const noexcept { return (_s == 0); }
+	ZT_INLINE T *data() noexcept { return reinterpret_cast<T *>(_m); }
+	ZT_INLINE const T *data() const noexcept { return reinterpret_cast<const T *>(_m); }
 	static constexpr unsigned int capacity() noexcept { return C; }
 
 	/**
@@ -144,7 +146,7 @@ public:
 	}
 
 	/**
-	 * Push a new value onto the vector and return it, or return last item if capacity is reached
+	 * Push new default value or return last in vector if full.
 	 *
 	 * @return Reference to new item
 	 */
@@ -158,7 +160,7 @@ public:
 	}
 
 	/**
-	 * Push a new value onto the vector and return it, or return last item if capacity is reached
+	 * Push new default value or replace and return last in vector if full.
 	 *
 	 * @return Reference to new item
 	 */
@@ -218,7 +220,7 @@ public:
 	 * @param i Index to obtain as a reference, resizing if needed
 	 * @return Reference to value at this index
 	 */
-	ZT_INLINE T &at(unsigned int i)
+	ZT_INLINE T &at(const unsigned int i)
 	{
 		if (i >= _s) {
 			if (unlikely(i >= C))

node/Fingerprint.hpp

@@ -84,7 +84,7 @@ public:
 	}
 
 	ZT_INLINE void zero() noexcept { memoryZero(this); }
-	ZT_INLINE unsigned long hashCode() const noexcept { return m_cfp.address; }
+	ZT_INLINE unsigned long hashCode() const noexcept { return (unsigned long)m_cfp.address; }
 
 	ZT_INLINE operator bool() const noexcept { return (m_cfp.address != 0); } // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
 

node/Identity.cpp

@@ -75,7 +75,7 @@ struct identityV0ProofOfWorkCriteria
 };
 
 // This is a simpler memory-intensive hash function for V1 identity generation.
-// It's not quite as intensive as the V0 frankenhash, is a little more orderly in
+// It's not quite as heavy as the V0 frankenhash, is a little more orderly in
 // its design, but remains relatively resistant to GPU acceleration due to memory
 // requirements for efficient computation.
 #define ZT_IDENTITY_V1_POW_MEMORY_SIZE 98304
@@ -180,17 +180,20 @@ bool Identity::generate(const Type t)
 	m_hasPrivate = true;
 
 	switch(t) {
+
 		case C25519: {
 			// Generate C25519/Ed25519 key pair whose hash satisfies a "hashcash" criterion and generate the
 			// address from the last 40 bits of this hash. This is different from the fingerprint hash for V0.
 			uint8_t digest[64];
 			char *const genmem = new char[ZT_V0_IDENTITY_GEN_MEMORY];
+			Address address;
 			do {
-				C25519::generateSatisfying(identityV0ProofOfWorkCriteria(digest,genmem), m_pub.c25519, m_priv.c25519);
+				C25519::generateSatisfying(identityV0ProofOfWorkCriteria(digest,genmem),m_pub,m_priv);
-				m_address.setTo(digest + 59);
+				address.setTo(digest + 59);
-			} while (m_address.isReserved());
+			} while (address.isReserved());
 			delete[] genmem;
-			_computeHash();
+			m_fp.m_cfp.address = address.toInt();
+			m_computeHash();
 		} break;
 
 		case P384: {
@@ -201,21 +204,20 @@ bool Identity::generate(const Type t)
 				// Loop until we pass the PoW criteria. The nonce is only 8 bits, so generate
 				// some new key material every time it wraps. The ECC384 generator is slightly
 				// faster so use that one.
-				m_pub.nonce = 0;
+				m_pub[0] = 0; // zero nonce
-				C25519::generateCombined(m_pub.c25519, m_priv.c25519);
+				C25519::generateCombined(m_pub + 1,m_priv + 1);
-				ECC384GenerateKey(m_pub.p384, m_priv.p384);
+				ECC384GenerateKey(m_pub + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,m_priv + ZT_C25519_COMBINED_PRIVATE_KEY_SIZE);
 				for(;;) {
-					if (identityV1ProofOfWorkCriteria(&m_pub, sizeof(m_pub), b))
+					if (identityV1ProofOfWorkCriteria(&m_pub,sizeof(m_pub),b))
 						break;
-					if (++m_pub.nonce == 0)
+					if (++m_pub[0] == 0)
-						ECC384GenerateKey(m_pub.p384, m_priv.p384);
+						ECC384GenerateKey(m_pub + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,m_priv + ZT_C25519_COMBINED_PRIVATE_KEY_SIZE);
 				}
 
 				// If we passed PoW then check that the address is valid, otherwise loop
 				// back around and run the whole process again.
-				_computeHash();
+				m_computeHash();
-				m_address.setTo(m_fp.hash());
+				if (!m_fp.address().isReserved())
-				if (!m_address.isReserved())
 					break;
 			}
 			free(b);
@@ -231,28 +233,27 @@ bool Identity::generate(const Type t)
 bool Identity::locallyValidate() const noexcept
 {
 	try {
-		if ((!m_address.isReserved()) && (m_address)) {
+		if ((m_fp)&&((!m_fp.address().isReserved()))) {
 			switch (m_type) {
-
 				case C25519: {
 					uint8_t digest[64];
-					char *genmem = new char[ZT_V0_IDENTITY_GEN_MEMORY];
+					char *const genmem = (char *)malloc(ZT_V0_IDENTITY_GEN_MEMORY);
-					identityV0ProofOfWorkFrankenhash(m_pub.c25519, ZT_C25519_COMBINED_PUBLIC_KEY_SIZE, digest, genmem);
+					if (!genmem)
-					delete[] genmem;
+						return false;
-					return ((m_address == Address(digest + 59)) && (digest[0] < 17));
+					identityV0ProofOfWorkFrankenhash(m_pub,ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,digest,genmem);
+					free(genmem);
+					return ((m_fp.address() == Address(digest + 59)) && (digest[0] < 17));
 				}
-
 				case P384: {
-					if (m_address != Address(m_fp.hash()))
+					if (m_fp.address() != Address(m_fp.hash()))
 						return false;
-					uint64_t *const b = (uint64_t *)malloc(ZT_IDENTITY_V1_POW_MEMORY_SIZE * 8); // NOLINT(hicpp-use-auto,modernize-use-auto)
+					uint64_t *const genmem = (uint64_t *)malloc(ZT_IDENTITY_V1_POW_MEMORY_SIZE * 8);
-					if (!b)
+					if (!genmem)
 						return false;
-					const bool ok = identityV1ProofOfWorkCriteria(&m_pub, sizeof(m_pub), b);
+					const bool ok = identityV1ProofOfWorkCriteria(m_pub,sizeof(m_pub),genmem);
-					free(b);
+					free(genmem);
 					return ok;
 				}
-
 			}
 		}
 	} catch ( ... ) {}
@@ -263,15 +264,12 @@ void Identity::hashWithPrivate(uint8_t h[ZT_FINGERPRINT_HASH_SIZE]) const
 {
 	if (m_hasPrivate) {
 		switch (m_type) {
-
 			case C25519:
-				SHA384(h, m_pub.c25519, ZT_C25519_COMBINED_PUBLIC_KEY_SIZE, m_priv.c25519, ZT_C25519_COMBINED_PRIVATE_KEY_SIZE);
+				SHA384(h,m_pub,ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,m_priv,ZT_C25519_COMBINED_PRIVATE_KEY_SIZE);
 				break;
-
 			case P384:
-				SHA384(h, &m_pub, sizeof(m_pub), &m_priv, sizeof(m_priv));
+				SHA384(h,m_pub,sizeof(m_pub),m_priv,sizeof(m_priv));
 				break;
-
 		}
 		return;
 	}
@@ -282,21 +280,19 @@ unsigned int Identity::sign(const void *data,unsigned int len,void *sig,unsigned
 {
 	if (m_hasPrivate) {
 		switch(m_type) {
-
 			case C25519:
 				if (siglen >= ZT_C25519_SIGNATURE_LEN) {
-					C25519::sign(m_priv.c25519, m_pub.c25519, data, len, sig);
+					C25519::sign(m_priv,m_pub,data,len,sig);
 					return ZT_C25519_SIGNATURE_LEN;
 				}
-
 			case P384:
 				if (siglen >= ZT_ECC384_SIGNATURE_SIZE) {
+					// SECURITY: signatures also include the public keys to further enforce their coupling.
 					uint8_t h[48];
-					SHA384(h, data, len, &m_pub, ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE); // include C25519 public key in hash
+					SHA384(h,data,len,m_pub,sizeof(m_pub));
-					ECC384ECDSASign(m_priv.p384, h, (uint8_t *)sig);
+					ECC384ECDSASign(m_priv + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,h,(uint8_t *)sig);
 					return ZT_ECC384_SIGNATURE_SIZE;
 				}
-
 		}
 	}
 	return 0;
@@ -305,18 +301,15 @@ unsigned int Identity::sign(const void *data,unsigned int len,void *sig,unsigned
 bool Identity::verify(const void *data,unsigned int len,const void *sig,unsigned int siglen) const
 {
 	switch(m_type) {
-
 		case C25519:
-			return C25519::verify(m_pub.c25519, data, len, sig, siglen);
+			return C25519::verify(m_pub,data,len,sig,siglen);
-
 		case P384:
 			if (siglen == ZT_ECC384_SIGNATURE_SIZE) {
 				uint8_t h[48];
-				SHA384(h, data, len, &m_pub, ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE);
+				SHA384(h,data,len,m_pub,sizeof(m_pub));
-				return ECC384ECDSAVerify(m_pub.p384, h, (const uint8_t *)sig);
+				return ECC384ECDSAVerify(m_pub + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,h,(const uint8_t *)sig);
 			}
 			break;
-
 	}
 	return false;
 }
@@ -327,37 +320,33 @@ bool Identity::agree(const Identity &id,uint8_t key[ZT_SYMMETRIC_KEY_SIZE]) cons
 	uint8_t h[64];
 	if (m_hasPrivate) {
 		if (m_type == C25519) {
-
 			if ((id.m_type == C25519) || (id.m_type == P384)) {
 				// If we are a C25519 key we can agree with another C25519 key or with only the
 				// C25519 portion of a type 1 P-384 key.
-				C25519::agree(m_priv.c25519, id.m_pub.c25519, rawkey);
+				C25519::agree(m_priv,id.m_pub,rawkey);
 				SHA512(h,rawkey,ZT_C25519_ECDH_SHARED_SECRET_SIZE);
 				Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(key,h);
 				return true;
 			}
-
 		} else if (m_type == P384) {
-
 			if (id.m_type == P384) {
 				// For another P384 identity we execute DH agreement with BOTH keys and then
 				// hash the results together. For those (cough FIPS cough) who only consider
 				// P384 to be kosher, the C25519 secret can be considered a "salt"
 				// or something. For those who don't trust P384 this means the privacy of
 				// your traffic is also protected by C25519.
-				C25519::agree(m_priv.c25519, id.m_pub.c25519, rawkey);
+				C25519::agree(m_priv,id.m_pub,rawkey);
-				ECC384ECDH(id.m_pub.p384, m_priv.p384, rawkey + ZT_C25519_ECDH_SHARED_SECRET_SIZE);
+				ECC384ECDH(id.m_pub + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,m_priv + ZT_C25519_COMBINED_PRIVATE_KEY_SIZE,rawkey + ZT_C25519_ECDH_SHARED_SECRET_SIZE);
 				SHA384(h,rawkey,ZT_C25519_ECDH_SHARED_SECRET_SIZE + ZT_ECC384_SHARED_SECRET_SIZE);
 				Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(key,h);
 				return true;
 			} else if (id.m_type == C25519) {
 				// If the other identity is a C25519 identity we can agree using only that type.
-				C25519::agree(m_priv.c25519, id.m_pub.c25519, rawkey);
+				C25519::agree(m_priv,id.m_pub,rawkey);
 				SHA512(h,rawkey,ZT_C25519_ECDH_SHARED_SECRET_SIZE);
 				Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(key,h);
 				return true;
 			}
-
 		}
 	}
 	return false;
@@ -366,42 +355,39 @@ bool Identity::agree(const Identity &id,uint8_t key[ZT_SYMMETRIC_KEY_SIZE]) cons
 char *Identity::toString(bool includePrivate,char buf[ZT_IDENTITY_STRING_BUFFER_LENGTH]) const
 {
 	char *p = buf;
-	m_address.toString(p);
+	m_fp.address().toString(p);
 	p += 10;
 	*(p++) = ':';
 
 	switch(m_type) {
-
 		case C25519: {
 			*(p++) = '0';
 			*(p++) = ':';
-			Utils::hex(m_pub.c25519, ZT_C25519_COMBINED_PUBLIC_KEY_SIZE, p);
+			Utils::hex(m_pub,ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,p);
 			p += ZT_C25519_COMBINED_PUBLIC_KEY_SIZE * 2;
-			if ((m_hasPrivate) && (includePrivate)) {
+			if ((m_hasPrivate)&&(includePrivate)) {
 				*(p++) = ':';
-				Utils::hex(m_priv.c25519, ZT_C25519_COMBINED_PRIVATE_KEY_SIZE, p);
+				Utils::hex(m_priv,ZT_C25519_COMBINED_PRIVATE_KEY_SIZE,p);
 				p += ZT_C25519_COMBINED_PRIVATE_KEY_SIZE * 2;
 			}
 			*p = (char)0;
 			return buf;
 		}
-
 		case P384: {
 			*(p++) = '1';
 			*(p++) = ':';
-			int el = Utils::b32e((const uint8_t *)(&m_pub), sizeof(m_pub), p, (int)(ZT_IDENTITY_STRING_BUFFER_LENGTH - (uintptr_t)(p - buf)));
+			int el = Utils::b32e(m_pub,sizeof(m_pub),p,(int)(ZT_IDENTITY_STRING_BUFFER_LENGTH - (uintptr_t)(p - buf)));
 			if (el <= 0) return nullptr;
 			p += el;
-			if ((m_hasPrivate) && (includePrivate)) {
+			if ((m_hasPrivate)&&(includePrivate)) {
 				*(p++) = ':';
-				el = Utils::b32e((const uint8_t *)(&m_priv), sizeof(m_priv), p, (int)(ZT_IDENTITY_STRING_BUFFER_LENGTH - (uintptr_t)(p - buf)));
+				el = Utils::b32e(m_priv,sizeof(m_priv),p,(int)(ZT_IDENTITY_STRING_BUFFER_LENGTH - (uintptr_t)(p - buf)));
 				if (el <= 0) return nullptr;
 				p += el;
 			}
 			*p = (char)0;
 			return buf;
 		}
-
 	}
 
 	return nullptr;
@@ -409,19 +395,10 @@ char *Identity::toString(bool includePrivate,char buf[ZT_IDENTITY_STRING_BUFFER_
 
 bool Identity::fromString(const char *str)
 {
-	m_fp.zero();
-	m_hasPrivate = false;
-
-	if (!str) {
-		m_address.zero();
-		return false;
-	}
-
 	char tmp[ZT_IDENTITY_STRING_BUFFER_LENGTH];
-	if (!Utils::scopy(tmp,sizeof(tmp),str)) {
+	memoryZero(this);
-		m_address.zero();
+	if ((!str)||(!Utils::scopy(tmp,sizeof(tmp),str)))
 		return false;
-	}
 
 	int fno = 0;
 	char *saveptr = nullptr;
@@ -429,9 +406,9 @@ bool Identity::fromString(const char *str)
 		switch(fno++) {
 
 			case 0:
-				m_address = Address(Utils::hexStrToU64(f));
+				m_fp.m_cfp.address = Utils::hexStrToU64(f) & ZT_ADDRESS_MASK;
-				if (m_address.isReserved()) {
+				if (m_fp.address().isReserved()) {
-					m_address.zero();
+					memoryZero(this);
 					return false;
 				}
 				break;
@@ -442,7 +419,7 @@ bool Identity::fromString(const char *str)
 				} else if ((f[0] == '1')&&(!f[1])) {
 					m_type = P384;
 				} else {
-					m_address.zero();
+					memoryZero(this);
 					return false;
 				}
 				break;
@@ -451,15 +428,15 @@ bool Identity::fromString(const char *str)
 				switch(m_type) {
 
 					case C25519:
-						if (Utils::unhex(f, strlen(f), m_pub.c25519, ZT_C25519_COMBINED_PUBLIC_KEY_SIZE) != ZT_C25519_COMBINED_PUBLIC_KEY_SIZE) {
+						if (Utils::unhex(f,strlen(f),m_pub,ZT_C25519_COMBINED_PUBLIC_KEY_SIZE) != ZT_C25519_COMBINED_PUBLIC_KEY_SIZE) {
-							m_address.zero();
+							memoryZero(this);
 							return false;
 						}
 						break;
 
 					case P384:
-						if (Utils::b32d(f, (uint8_t *)(&m_pub), sizeof(m_pub)) != sizeof(m_pub)) {
+						if (Utils::b32d(f,m_pub,sizeof(m_pub)) != sizeof(m_pub)) {
-							m_address.zero();
+							memoryZero(this);
 							return false;
 						}
 						break;
@@ -472,8 +449,8 @@ bool Identity::fromString(const char *str)
 					switch(m_type) {
 
 						case C25519:
-							if (Utils::unhex(f, strlen(f), m_priv.c25519, ZT_C25519_COMBINED_PRIVATE_KEY_SIZE) != ZT_C25519_COMBINED_PRIVATE_KEY_SIZE) {
+							if (Utils::unhex(f,strlen(f),m_priv,ZT_C25519_COMBINED_PRIVATE_KEY_SIZE) != ZT_C25519_COMBINED_PRIVATE_KEY_SIZE) {
-								m_address.zero();
+								memoryZero(this);
 								return false;
 							} else {
 								m_hasPrivate = true;
@@ -481,8 +458,8 @@ bool Identity::fromString(const char *str)
 							break;
 
 						case P384:
-							if (Utils::b32d(f, (uint8_t *)(&m_priv), sizeof(m_priv)) != sizeof(m_priv)) {
+							if (Utils::b32d(f,m_priv,sizeof(m_priv)) != sizeof(m_priv)) {
-								m_address.zero();
+								memoryZero(this);
 								return false;
 							} else {
 								m_hasPrivate = true;
@@ -497,13 +474,13 @@ bool Identity::fromString(const char *str)
 	}
 
 	if (fno < 3) {
-		m_address.zero();
+		memoryZero(this);
 		return false;
 	}
 
-	_computeHash();
+	m_computeHash();
-	if ((m_type == P384) && (m_address != Address(m_fp.hash()))) {
+	if ((m_type == P384)&&(m_fp.address() != Address(m_fp.hash()))) {
-		m_address.zero();
+		memoryZero(this);
 		return false;
 	}
 
@@ -512,14 +489,15 @@ bool Identity::fromString(const char *str)
 
 int Identity::marshal(uint8_t data[ZT_IDENTITY_MARSHAL_SIZE_MAX],const bool includePrivate) const noexcept
 {
-	m_address.copyTo(data);
+	m_fp.address().copyTo(data);
 	switch(m_type) {
+
 		case C25519:
 			data[ZT_ADDRESS_LENGTH] = (uint8_t)C25519;
-			Utils::copy<ZT_C25519_COMBINED_PUBLIC_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1, m_pub.c25519);
+			Utils::copy<ZT_C25519_COMBINED_PUBLIC_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1,m_pub);
 			if ((includePrivate)&&(m_hasPrivate)) {
 				data[ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE] = ZT_C25519_COMBINED_PRIVATE_KEY_SIZE;
-				Utils::copy<ZT_C25519_COMBINED_PRIVATE_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1, m_priv.c25519);
+				Utils::copy<ZT_C25519_COMBINED_PRIVATE_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1,m_priv);
 				return ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1 + ZT_C25519_COMBINED_PRIVATE_KEY_SIZE;
 			} else {
 				data[ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE] = 0;
@@ -528,10 +506,10 @@ int Identity::marshal(uint8_t data[ZT_IDENTITY_MARSHAL_SIZE_MAX],const bool incl
 
 		case P384:
 			data[ZT_ADDRESS_LENGTH] = (uint8_t)P384;
-			Utils::copy<ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1,&m_pub);
+			Utils::copy<ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1,m_pub);
 			if ((includePrivate)&&(m_hasPrivate)) {
 				data[ZT_ADDRESS_LENGTH + 1 + ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE] = ZT_IDENTITY_P384_COMPOUND_PRIVATE_KEY_SIZE;
-				Utils::copy<ZT_IDENTITY_P384_COMPOUND_PRIVATE_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1 + ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE + 1,&m_priv);
+				Utils::copy<ZT_IDENTITY_P384_COMPOUND_PRIVATE_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1 + ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE + 1,m_priv);
 				return ZT_ADDRESS_LENGTH + 1 + ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE + 1 + ZT_IDENTITY_P384_COMPOUND_PRIVATE_KEY_SIZE;
 			} else {
 				data[ZT_ADDRESS_LENGTH + 1 + ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE] = 0;
@@ -544,12 +522,11 @@ int Identity::marshal(uint8_t data[ZT_IDENTITY_MARSHAL_SIZE_MAX],const bool incl
 
 int Identity::unmarshal(const uint8_t *data,const int len) noexcept
 {
-	m_fp.zero();
+	memoryZero(this);
-	m_hasPrivate = false;
 
 	if (len < (1 + ZT_ADDRESS_LENGTH))
 		return -1;
-	m_address.setTo(data);
+	m_fp.m_cfp.address = Address(data).toInt();
 
 	unsigned int privlen;
 	switch((m_type = (Type)data[ZT_ADDRESS_LENGTH])) {
@@ -558,15 +535,15 @@ int Identity::unmarshal(const uint8_t *data,const int len) noexcept
 			if (len < (ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1))
 				return -1;
 
-			Utils::copy<ZT_C25519_COMBINED_PUBLIC_KEY_SIZE>(m_pub.c25519, data + ZT_ADDRESS_LENGTH + 1);
+			Utils::copy<ZT_C25519_COMBINED_PUBLIC_KEY_SIZE>(m_pub,data + ZT_ADDRESS_LENGTH + 1);
-			_computeHash();
+			m_computeHash();
 
 			privlen = data[ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE];
 			if (privlen == ZT_C25519_COMBINED_PRIVATE_KEY_SIZE) {
 				if (len < (ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1 + ZT_C25519_COMBINED_PRIVATE_KEY_SIZE))
 					return -1;
 				m_hasPrivate = true;
-				Utils::copy<ZT_C25519_COMBINED_PRIVATE_KEY_SIZE>(m_priv.c25519, data + ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1);
+				Utils::copy<ZT_C25519_COMBINED_PRIVATE_KEY_SIZE>(m_priv,data + ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1);
 				return ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1 + ZT_C25519_COMBINED_PRIVATE_KEY_SIZE;
 			} else if (privlen == 0) {
 				m_hasPrivate = false;
@@ -578,9 +555,9 @@ int Identity::unmarshal(const uint8_t *data,const int len) noexcept
 			if (len < (ZT_ADDRESS_LENGTH + 1 + ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE + 1))
 				return -1;
 
-			Utils::copy<ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE>(&m_pub, data + ZT_ADDRESS_LENGTH + 1);
+			Utils::copy<ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE>(m_pub,data + ZT_ADDRESS_LENGTH + 1);
-			_computeHash(); // this sets the address for P384
+			m_computeHash(); // this sets the address for P384
-			if (m_address != Address(m_fp.hash())) // this sanity check is possible with V1 identities
+			if (m_fp.address() != Address(m_fp.hash())) // this sanity check is possible with V1 identities
 				return -1;
 
 			privlen = data[ZT_ADDRESS_LENGTH + 1 + ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE];
@@ -601,21 +578,18 @@ int Identity::unmarshal(const uint8_t *data,const int len) noexcept
 	return -1;
 }
 
-void Identity::_computeHash()
+void Identity::m_computeHash()
 {
 	switch(m_type) {
 		default:
 			m_fp.zero();
 			break;
-
 		case C25519:
-			m_fp.m_cfp.address = m_address.toInt();
+			SHA384(m_fp.m_cfp.hash,m_pub,ZT_C25519_COMBINED_PUBLIC_KEY_SIZE);
-			SHA384(m_fp.m_cfp.hash, m_pub.c25519, ZT_C25519_COMBINED_PUBLIC_KEY_SIZE);
 			break;
-
 		case P384:
-			SHA384(m_fp.m_cfp.hash, &m_pub, sizeof(m_pub));
+			SHA384(m_fp.m_cfp.hash,m_pub,sizeof(m_pub));
-			m_fp.m_cfp.address = m_address.toInt();
+			m_fp.m_cfp.address = Address(m_fp.m_cfp.hash).toInt();
 			break;
 	}
 }

node/Identity.hpp

@@ -24,10 +24,6 @@
 #include "Fingerprint.hpp"
 #include "Containers.hpp"
 
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-
 #define ZT_IDENTITY_STRING_BUFFER_LENGTH 1024
 #define ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE (1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + ZT_ECC384_PUBLIC_KEY_SIZE)
 #define ZT_IDENTITY_P384_COMPOUND_PRIVATE_KEY_SIZE (ZT_C25519_COMBINED_PRIVATE_KEY_SIZE + ZT_ECC384_PRIVATE_KEY_SIZE)
@@ -127,16 +123,12 @@ public:
 	ZT_INLINE bool hasPrivate() const noexcept { return m_hasPrivate; }
 
 	/**
-	 * Get a 384-bit hash of this identity's public key(s)
+	 * @return This identity's address
-	 *
+	 */
-	 * The hash returned by this function differs by identity type. For C25519 (type 0)
+	ZT_INLINE Address address() const noexcept { return Address(m_fp.m_cfp.address); }
-	 * identities this returns a simple SHA384 of the public key, which is NOT the same
+
-	 * as the hash used to generate the address. For type 1 C25519+P384 identities this
+	/**
-	 * returns the same compoound SHA384 hash that is used for purposes of hashcash
+	 * @return Full fingerprint of this identity (address plus SHA384 of keys)
-	 * and address computation. This difference is because the v0 hash is expensive while
-	 * the v1 hash is fast.
-	 *
-	 * @return Hash of public key(s)
 	 */
 	ZT_INLINE const Fingerprint &fingerprint() const noexcept { return m_fp; }
 
@@ -185,11 +177,6 @@ public:
 	 */
 	bool agree(const Identity &id,uint8_t key[ZT_SYMMETRIC_KEY_SIZE]) const;
 
-	/**
-	 * @return This identity's address
-	 */
-	ZT_INLINE Address address() const noexcept { return m_address; }
-
 	/**
 	 * Serialize to a more human-friendly string
 	 *
@@ -214,7 +201,7 @@ public:
 	/**
 	 * @return True if this identity contains something
 	 */
-	explicit ZT_INLINE operator bool() const noexcept { return (m_address); }
+	explicit ZT_INLINE operator bool() const noexcept { return (m_fp); }
 
 	ZT_INLINE unsigned long hashCode() const noexcept { return m_fp.hashCode(); }
 
@@ -230,19 +217,11 @@ public:
 	int unmarshal(const uint8_t *data,int len) noexcept;
 
 private:
-	void _computeHash();
+	void m_computeHash();
 
-	Address m_address;
 	Fingerprint m_fp;
-	ZT_PACKED_STRUCT(struct { // do not re-order these fields
+	uint8_t m_priv[ZT_IDENTITY_P384_COMPOUND_PRIVATE_KEY_SIZE];
-		uint8_t c25519[ZT_C25519_COMBINED_PRIVATE_KEY_SIZE];
+	uint8_t m_pub[ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE];
-		uint8_t p384[ZT_ECC384_PRIVATE_KEY_SIZE];
-	}) m_priv;
-	ZT_PACKED_STRUCT(struct { // do not re-order these fields
-		uint8_t nonce;                            // nonce for PoW generate/verify
-		uint8_t c25519[ZT_C25519_COMBINED_PUBLIC_KEY_SIZE]; // Curve25519 and Ed25519 public keys
-		uint8_t p384[ZT_ECC384_PUBLIC_KEY_SIZE];  // NIST P-384 public key
-	}) m_pub;
 	Type m_type; // _type determines which fields in _priv and _pub are used
 	bool m_hasPrivate;
 };

node/Membership.cpp

@@ -124,13 +124,13 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme
 {
 	const int64_t newts = com.timestamp();
 	if (newts <= m_comRevocationThreshold) {
-		RR->t->credentialRejected(tPtr,0xd9992121,com.networkId(),sourcePeerIdentity.address(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_REVOKED);
+		RR->t->credentialRejected(tPtr,0xd9992121,com.networkId(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_REVOKED);
 		return ADD_REJECTED;
 	}
 
 	const int64_t oldts = m_com.timestamp();
 	if (newts < oldts) {
-		RR->t->credentialRejected(tPtr,0xd9928192,com.networkId(),sourcePeerIdentity.address(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_OLDER_THAN_LATEST);
+		RR->t->credentialRejected(tPtr,0xd9928192,com.networkId(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_OLDER_THAN_LATEST);
 		return ADD_REJECTED;
 	}
 	if ((newts == oldts)&&(m_com == com))
@@ -138,13 +138,13 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme
 
 	switch(com.verify(RR,tPtr)) {
 		default:
-			RR->t->credentialRejected(tPtr,0x0f198241,com.networkId(),sourcePeerIdentity.address(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
+			RR->t->credentialRejected(tPtr,0x0f198241,com.networkId(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
 			return Membership::ADD_REJECTED;
 		case Credential::VERIFY_OK:
 			m_com = com;
 			return ADD_ACCEPTED_NEW;
 		case Credential::VERIFY_BAD_SIGNATURE:
-			RR->t->credentialRejected(tPtr,0xbaf0aaaa,com.networkId(),sourcePeerIdentity.address(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_SIGNATURE_VERIFICATION_FAILED);
+			RR->t->credentialRejected(tPtr,0xbaf0aaaa,com.networkId(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_SIGNATURE_VERIFICATION_FAILED);
 			return ADD_REJECTED;
 		case Credential::VERIFY_NEED_IDENTITY:
 			return ADD_DEFERRED_FOR_WHOIS;
@@ -165,7 +165,7 @@ static ZT_INLINE Membership::AddCredentialResult _addCredImpl(
 	C *rc = remoteCreds.get(cred.id());
 	if (rc) {
 		if (rc->timestamp() > cred.timestamp()) {
-			RR->t->credentialRejected(tPtr,0x40000001,nconf.networkId,sourcePeerIdentity.address(),sourcePeerIdentity,cred.id(),cred.timestamp(),C::credentialType(),ZT_TRACE_CREDENTIAL_REJECTION_REASON_OLDER_THAN_LATEST);
+			RR->t->credentialRejected(tPtr,0x40000001,nconf.networkId,sourcePeerIdentity,cred.id(),cred.timestamp(),C::credentialType(),ZT_TRACE_CREDENTIAL_REJECTION_REASON_OLDER_THAN_LATEST);
 			return Membership::ADD_REJECTED;
 		}
 		if (*rc == cred)
@@ -174,13 +174,13 @@ static ZT_INLINE Membership::AddCredentialResult _addCredImpl(
 
 	const int64_t *const rt = revocations.get(Membership::credentialKey(C::credentialType(),cred.id()));
 	if ((rt)&&(*rt >= cred.timestamp())) {
-		RR->t->credentialRejected(tPtr,0x24248124,nconf.networkId,sourcePeerIdentity.address(),sourcePeerIdentity,cred.id(),cred.timestamp(),C::credentialType(),ZT_TRACE_CREDENTIAL_REJECTION_REASON_REVOKED);
+		RR->t->credentialRejected(tPtr,0x24248124,nconf.networkId,sourcePeerIdentity,cred.id(),cred.timestamp(),C::credentialType(),ZT_TRACE_CREDENTIAL_REJECTION_REASON_REVOKED);
 		return Membership::ADD_REJECTED;
 	}
 
 	switch(cred.verify(RR,tPtr)) {
 		default:
-			RR->t->credentialRejected(tPtr,0x01feba012,nconf.networkId,sourcePeerIdentity.address(),sourcePeerIdentity,cred.id(),cred.timestamp(),C::credentialType(),ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
+			RR->t->credentialRejected(tPtr,0x01feba012,nconf.networkId,sourcePeerIdentity,cred.id(),cred.timestamp(),C::credentialType(),ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
 			return Membership::ADD_REJECTED;
 		case 0:
 			if (!rc)
@@ -200,7 +200,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme
 	int64_t *rt;
 	switch(rev.verify(RR,tPtr)) {
 		default:
-			RR->t->credentialRejected(tPtr,0x938fffff,nconf.networkId,sourcePeerIdentity.address(),sourcePeerIdentity,rev.id(),0,ZT_CREDENTIAL_TYPE_REVOCATION,ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
+			RR->t->credentialRejected(tPtr,0x938fffff,nconf.networkId,sourcePeerIdentity,rev.id(),0,ZT_CREDENTIAL_TYPE_REVOCATION,ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
 			return ADD_REJECTED;
 		case 0: {
 			const ZT_CredentialType ct = rev.typeBeingRevoked();
@@ -222,7 +222,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme
 					}
 					return ADD_ACCEPTED_REDUNDANT;
 				default:
-					RR->t->credentialRejected(tPtr,0x0bbbb1a4,nconf.networkId,sourcePeerIdentity.address(),sourcePeerIdentity,rev.id(),0,ZT_CREDENTIAL_TYPE_REVOCATION,ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
+					RR->t->credentialRejected(tPtr,0x0bbbb1a4,nconf.networkId,sourcePeerIdentity,rev.id(),0,ZT_CREDENTIAL_TYPE_REVOCATION,ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
 					return ADD_REJECTED;
 			}
 		}

node/Network.hpp

@@ -27,12 +27,6 @@
 #include "CertificateOfMembership.hpp"
 #include "Containers.hpp"
 
-#include <cstdint>
-#include <string>
-#include <map>
-#include <vector>
-#include <algorithm>
-
 #define ZT_NETWORK_MAX_INCOMING_UPDATES 3
 
 namespace ZeroTier {

node/NetworkConfig.cpp

@@ -21,7 +21,7 @@
 
 namespace ZeroTier {
 
-bool NetworkConfig::toDictionary(Dictionary &d,bool includeLegacy) const
+bool NetworkConfig::toDictionary(Dictionary &d) const
 {
 	uint8_t tmp[ZT_BUF_MEM_SIZE];
 	try {

node/NetworkConfig.hpp

@@ -165,10 +165,9 @@ struct NetworkConfig : TriviallyCopyable
 	 * Write this network config to a dictionary for transport
 	 *
 	 * @param d Dictionary
-	 * @param includeLegacy If true, include legacy fields for old node versions
 	 * @return True if dictionary was successfully created, false if e.g. overflow
 	 */
-	bool toDictionary(Dictionary &d,bool includeLegacy) const;
+	bool toDictionary(Dictionary &d) const;
 
 	/**
 	 * Read this network config from a dictionary

node/NetworkController.hpp

@@ -77,8 +77,8 @@ public:
 		virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode) = 0;
 	};
 
-	NetworkController() {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
+	NetworkController() {}
-	virtual ~NetworkController() {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
+	virtual ~NetworkController() {}
 
 	/**
 	 * Called when this is added to a Node to initialize and supply info

node/OS.hpp

@@ -17,10 +17,10 @@
 #ifndef ZT_OS_HPP
 #define ZT_OS_HPP
 
-#include <stdint.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
+#include <stdint.h>
-#include <stdlib.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
+#include <stdlib.h>
-#include <string.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
+#include <string.h>
-#include <stdio.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
+#include <stdio.h>
 
 #if defined(_WIN32) || defined(_WIN64)
 #ifdef _MSC_VER

node/Peer.cpp

@@ -21,10 +21,11 @@
 #include "InetAddress.hpp"
 #include "Protocol.hpp"
 #include "Endpoint.hpp"
+#include "Expect.hpp"
 
 namespace ZeroTier {
 
-Peer::Peer(const RuntimeEnvironment *renv) : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+Peer::Peer(const RuntimeEnvironment *renv) :
 	RR(renv),
 	m_lastReceive(0),
 	m_lastSend(0),
@@ -32,6 +33,7 @@ Peer::Peer(const RuntimeEnvironment *renv) : // NOLINT(cppcoreguidelines-pro-typ
 	m_lastWhoisRequestReceived(0),
 	m_lastEchoRequestReceived(0),
 	m_lastPrioritizedPaths(0),
+	m_lastProbeReceived(0),
 	m_alivePathCount(0),
 	m_tryQueue(),
 	m_tryQueuePtr(m_tryQueue.end()),
@@ -43,8 +45,9 @@ Peer::Peer(const RuntimeEnvironment *renv) : // NOLINT(cppcoreguidelines-pro-typ
 {
 }
 
-Peer::~Peer() // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
+Peer::~Peer()
 {
+	Utils::burn(m_helloMacKey,sizeof(m_helloMacKey));
 }
 
 bool Peer::init(const Identity &peerIdentity)
@@ -55,11 +58,13 @@ bool Peer::init(const Identity &peerIdentity)
 		return false;
 	m_id = peerIdentity;
 
-	uint8_t ktmp[ZT_SYMMETRIC_KEY_SIZE];
+	uint8_t k[ZT_SYMMETRIC_KEY_SIZE];
-	if (!RR->identity.agree(peerIdentity,ktmp))
+	if (!RR->identity.agree(peerIdentity,k))
 		return false;
-	m_identityKey.init(RR->node->now(), ktmp);
+	m_identityKey.set(new SymmetricKey(RR->node->now(),k,true));
-	Utils::burn(ktmp,sizeof(ktmp));
+	Utils::burn(k,sizeof(k));
+
+	m_deriveSecondaryIdentityKeys();
 
 	return true;
 }
@@ -76,7 +81,7 @@ void Peer::received(
 	const int64_t now = RR->node->now();
 
 	m_lastReceive = now;
-	m_inMeter.log(now, payloadLength);
+	m_inMeter.log(now,payloadLength);
 
 	if (hops == 0) {
 		RWMutex::RMaybeWLock l(m_lock);
@@ -98,7 +103,7 @@ void Peer::received(
 
 				// If the path list is full, replace the least recently active path. Otherwise append new path.
 				unsigned int newPathIdx = 0;
-				if (m_alivePathCount >= ZT_MAX_PEER_NETWORK_PATHS) {
+				if (m_alivePathCount == ZT_MAX_PEER_NETWORK_PATHS) {
 					int64_t lastReceiveTimeMax = 0;
 					for (unsigned int i=0;i < m_alivePathCount;++i) {
 						if ((m_paths[i]->address().family() == path->address().family()) &&
@@ -132,18 +137,12 @@ void Peer::received(
 				RR->t->learnedNewPath(tPtr, 0x582fabdd, packetId, m_id, path->address(), old);
 			} else {
 				path->sent(now,hello(tPtr,path->localSocket(),path->address(),now));
-				RR->t->tryingNewPath(tPtr, 0xb7747ddd, m_id, path->address(), path->address(), packetId, (uint8_t)verb, m_id, ZT_TRACE_TRYING_NEW_PATH_REASON_PACKET_RECEIVED_FROM_UNKNOWN_PATH);
+				RR->t->tryingNewPath(tPtr, 0xb7747ddd, m_id, path->address(), path->address(), packetId, (uint8_t)verb, m_id);
 			}
 		}
 	}
 }
 
-void Peer::send(void *const tPtr,const int64_t now,const void *const data,const unsigned int len,const SharedPtr<Path> &via) noexcept
-{
-	via->send(RR,tPtr,data,len,now);
-	sent(now,len);
-}
-
 void Peer::send(void *const tPtr,const int64_t now,const void *const data,const unsigned int len) noexcept
 {
 	SharedPtr<Path> via(this->path(now));
@@ -190,25 +189,6 @@ unsigned int Peer::hello(void *tPtr,int64_t localSocket,const InetAddress &atAdd
 #endif
 }
 
-unsigned int Peer::probe(void *tPtr,int64_t localSocket,const InetAddress &atAddress,int64_t now)
-{
-	if (m_vProto < 11) {
-		Buf outp;
-		Protocol::Header &ph = outp.as<Protocol::Header>(); // NOLINT(hicpp-use-auto,modernize-use-auto)
-		//ph.packetId = Protocol::getPacketId();
-		m_id.address().copyTo(ph.destination);
-		RR->identity.address().copyTo(ph.source);
-		ph.flags = 0;
-		ph.verb = Protocol::VERB_NOP;
-		Protocol::armor(outp, sizeof(Protocol::Header), m_identityKey.key(), this->cipher());
-		RR->node->putPacket(tPtr,localSocket,atAddress,outp.unsafeData,sizeof(Protocol::Header));
-		return sizeof(Protocol::Header);
-	} else {
-		RR->node->putPacket(tPtr, -1, atAddress, &m_probe, 4);
-		return 4;
-	}
-}
-
 void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
 {
 	RWMutex::Lock l(m_lock);
@@ -219,90 +199,104 @@ void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
 		needHello = true;
 	}
 
-	m_prioritizePaths(now);
+	// If we have no active paths and none queued to try, attempt any
-
+	// old paths we have cached in m_bootstrap or that external code
-	if (m_alivePathCount == 0) {
+	// supplies to the core via the optional API callback.
-		// If there are no direct paths, attempt to make one. If there are queued addresses
+	if (m_tryQueue.empty()&&(m_alivePathCount == 0)) {
-		// to try, attempt one of those. Otherwise try a path we can fetch via API callbacks
+		InetAddress addr;
-		// and/or a remembered bootstrap path.
+		if (RR->node->externalPathLookup(tPtr, m_id, -1, addr)) {
-		if (m_tryQueue.empty()) {
+			if ((addr)&&(RR->node->shouldUsePathForZeroTierTraffic(tPtr, m_id, -1, addr))) {
-			InetAddress addr;
+				RR->t->tryingNewPath(tPtr, 0x84a10000, m_id, addr, InetAddress::NIL, 0, 0, Identity::NIL);
-			if (RR->node->externalPathLookup(tPtr, m_id, -1, addr)) {
+				sent(now,m_sendProbe(tPtr,-1,addr,now));
-				if ((addr)&&(RR->node->shouldUsePathForZeroTierTraffic(tPtr, m_id, -1, addr))) {
-					RR->t->tryingNewPath(tPtr, 0x84a10000, m_id, addr, InetAddress::NIL, 0, 0, Identity::NIL, ZT_TRACE_TRYING_NEW_PATH_REASON_EXPLICITLY_SUGGESTED_ADDRESS);
-					sent(now,probe(tPtr,-1,addr,now));
-				}
-			}
-			if (!m_bootstrap.empty()) {
-				unsigned int tryAtIndex = (unsigned int)Utils::random() % (unsigned int)m_bootstrap.size();
-				for(SortedMap< Endpoint::Type,Endpoint >::const_iterator i(m_bootstrap.begin());i != m_bootstrap.end();++i) {
-					if (tryAtIndex > 0) {
-						--tryAtIndex;
-					} else {
-						if ((i->second.isInetAddr())&&(!i->second.inetAddr().ipsEqual(addr))) {
-							RR->t->tryingNewPath(tPtr, 0x0a009444, m_id, i->second.inetAddr(), InetAddress::NIL, 0, 0, Identity::NIL, ZT_TRACE_TRYING_NEW_PATH_REASON_BOOTSTRAP_ADDRESS);
-							sent(now,probe(tPtr,-1,i->second.inetAddr(),now));
-							break;
-						}
-					}
-				}
-			}
-		} else {
-			for(int k=0;(k<ZT_NAT_T_MAX_QUEUED_ATTEMPTS_PER_PULSE)&&(!m_tryQueue.empty());++k) {
-				if (m_tryQueuePtr == m_tryQueue.end())
-					m_tryQueuePtr = m_tryQueue.begin();
-
-				if ((now - m_tryQueuePtr->ts) > ZT_PATH_ALIVE_TIMEOUT) {
-					m_tryQueue.erase(m_tryQueuePtr++);
-					continue;
-				}
-
-				if (m_tryQueuePtr->target.isInetAddr()) {
-					if ((m_tryQueuePtr->breakSymmetricBFG1024) && (RR->node->natMustDie())) {
-						// Attempt aggressive NAT traversal if both requested and enabled.
-						uint16_t ports[1023];
-						for (unsigned int i=0;i<1023;++i)
-							ports[i] = (uint64_t)(i + 1);
-						for (unsigned int i=0;i<512;++i) {
-							const uint64_t rn = Utils::random();
-							const unsigned int a = (unsigned int)rn % 1023;
-							const unsigned int b = (unsigned int)(rn >> 32U) % 1023;
-							if (a != b) {
-								uint16_t tmp = ports[a];
-								ports[a] = ports[b];
-								ports[b] = tmp;
-							}
-						}
-						InetAddress addr(m_tryQueuePtr->target.inetAddr());
-						for (unsigned int i = 0;i < ZT_NAT_T_BFG1024_PORTS_PER_ATTEMPT;++i) {
-							addr.setPort(ports[i]);
-							sent(now,probe(tPtr,-1,addr,now));
-						}
-					} else {
-						// Otherwise send a normal probe.
-						sent(now,probe(tPtr, -1, m_tryQueuePtr->target.inetAddr(), now));
-					}
-				}
-
-				++m_tryQueuePtr;
 			}
 		}
-	} else {
+
-		// Keep direct paths alive, sending a HELLO if we need one or else just a simple byte.
+		if (!m_bootstrap.empty()) {
-		for(unsigned int i=0;i < m_alivePathCount;++i) {
+			unsigned int tryAtIndex = (unsigned int)Utils::random() % (unsigned int)m_bootstrap.size();
-			if (needHello) {
+			for(SortedMap< Endpoint::Type,Endpoint >::const_iterator i(m_bootstrap.begin());i != m_bootstrap.end();++i) {
-				needHello = false;
+				if (tryAtIndex > 0) {
-				const unsigned int bytes = hello(tPtr, m_paths[i]->localSocket(), m_paths[i]->address(), now);
+					--tryAtIndex;
-				m_paths[i]->sent(now, bytes);
+				} else {
-				sent(now,bytes);
+					if ((i->second.isInetAddr())&&(!i->second.inetAddr().ipsEqual(addr))) {
-			} else if ((now - m_paths[i]->lastOut()) >= ZT_PATH_KEEPALIVE_PERIOD) {
+						RR->t->tryingNewPath(tPtr, 0x0a009444, m_id, i->second.inetAddr(), InetAddress::NIL, 0, 0, Identity::NIL);
-				m_paths[i]->send(RR, tPtr, &now, 1, now);
+						sent(now,m_sendProbe(tPtr,-1,i->second.inetAddr(),now));
-				sent(now,1);
+						break;
+					}
+				}
 			}
 		}
 	}
 
-	// If we could not reliably send a HELLO via a direct path, send it by way of a root.
+	m_prioritizePaths(now);
+
+	// Attempt queued paths to try.
+	for(int k=0;(k<ZT_NAT_T_MAX_QUEUED_ATTEMPTS_PER_PULSE)&&(!m_tryQueue.empty());++k) {
+		if (m_tryQueuePtr == m_tryQueue.end())
+			m_tryQueuePtr = m_tryQueue.begin();
+
+		if ((now - m_tryQueuePtr->ts) > ZT_PATH_ALIVE_TIMEOUT) {
+			m_tryQueue.erase(m_tryQueuePtr++);
+			continue;
+		}
+
+		if (m_tryQueuePtr->target.isInetAddr()) {
+			// Make sure target does not overlap with any existing path.
+			bool duplicate = false;
+			for(unsigned int i=0;i<m_alivePathCount;++i) {
+				if (m_paths[i]->address() == m_tryQueuePtr->target.inetAddr()) {
+					duplicate = true;
+					break;
+				}
+			}
+			if (duplicate) {
+				m_tryQueue.erase(m_tryQueuePtr++);
+				continue;
+			}
+
+			if (m_tryQueuePtr->breakSymmetricBFG1024 && RR->node->natMustDie()) {
+				// Attempt aggressive NAT traversal if both requested and enabled.
+				uint16_t ports[1023];
+				for (unsigned int i=0;i<1023;++i)
+					ports[i] = (uint64_t)(i + 1);
+				for (unsigned int i=0;i<512;++i) {
+					const uint64_t rn = Utils::random();
+					const unsigned int a = (unsigned int)rn % 1023;
+					const unsigned int b = (unsigned int)(rn >> 32U) % 1023;
+					if (a != b) {
+						uint16_t tmp = ports[a];
+						ports[a] = ports[b];
+						ports[b] = tmp;
+					}
+				}
+				InetAddress addr(m_tryQueuePtr->target.inetAddr());
+				for (unsigned int i=0;i<ZT_NAT_T_BFG1024_PORTS_PER_ATTEMPT;++i) {
+					addr.setPort(ports[i]);
+					sent(now,m_sendProbe(tPtr,-1,addr,now));
+				}
+			} else {
+				// Otherwise send a normal probe.
+				sent(now,m_sendProbe(tPtr, -1, m_tryQueuePtr->target.inetAddr(), now));
+			}
+		}
+
+		++m_tryQueuePtr;
+	}
+
+	// Do keepalive on all currently active paths.
+	for(unsigned int i=0;i<m_alivePathCount;++i) {
+		if (needHello) {
+			needHello = false;
+			const unsigned int bytes = hello(tPtr, m_paths[i]->localSocket(), m_paths[i]->address(), now);
+			m_paths[i]->sent(now, bytes);
+			sent(now,bytes);
+		} else if ((now - m_paths[i]->lastOut()) >= ZT_PATH_KEEPALIVE_PERIOD) {
+			m_paths[i]->send(RR, tPtr, &now, 1, now);
+			sent(now,1);
+		}
+	}
+
+	// If we need a HELLO and were not able to send one via any other path,
+	// send one indirectly.
 	if (needHello) {
 		const SharedPtr<Peer> root(RR->topology->root());
 		if (root) {
@@ -321,7 +315,7 @@ void Peer::tryDirectPath(const int64_t now,const Endpoint &ep,const bool breakSy
 {
 	RWMutex::Lock l(m_lock);
 
-	for(List<p_TryQueueItem>::iterator i(m_tryQueue.begin());i != m_tryQueue.end();++i) { // NOLINT(modernize-loop-convert,hicpp-use-auto,modernize-use-auto)
+	for(List<p_TryQueueItem>::iterator i(m_tryQueue.begin());i != m_tryQueue.end();++i) {
 		if (i->target == ep) {
 			i->ts = now;
 			i->breakSymmetricBFG1024 = breakSymmetricBFG1024;
@@ -338,14 +332,20 @@ void Peer::tryDirectPath(const int64_t now,const Endpoint &ep,const bool breakSy
 
 void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,int64_t now)
 {
-	RWMutex::RLock l(m_lock);
+	RWMutex::Lock l(m_lock);
-	for(unsigned int i=0;i < m_alivePathCount;++i) {
+	unsigned int pc = 0;
+	for(unsigned int i=0;i<m_alivePathCount;++i) {
 		if ((m_paths[i]) && ((m_paths[i]->address().family() == inetAddressFamily) && (m_paths[i]->address().ipScope() == scope))) {
-			const unsigned int bytes = probe(tPtr, m_paths[i]->localSocket(), m_paths[i]->address(), now);
+			const unsigned int bytes = m_sendProbe(tPtr, m_paths[i]->localSocket(), m_paths[i]->address(), now);
 			m_paths[i]->sent(now, bytes);
 			sent(now,bytes);
+		} else if (pc != i) {
+			m_paths[pc++] = m_paths[i];
 		}
 	}
+	m_alivePathCount = pc;
+	while (pc < ZT_MAX_PEER_NETWORK_PATHS)
+		m_paths[pc].zero();
 }
 
 bool Peer::directlyConnected(int64_t now)
@@ -360,10 +360,11 @@ bool Peer::directlyConnected(int64_t now)
 	}
 }
 
-void Peer::getAllPaths(std::vector< SharedPtr<Path> > &paths)
+void Peer::getAllPaths(Vector< SharedPtr<Path> > &paths)
 {
 	RWMutex::RLock l(m_lock);
 	paths.clear();
+	paths.reserve(m_alivePathCount);
 	paths.assign(m_paths, m_paths + m_alivePathCount);
 }
 
@@ -385,16 +386,27 @@ void Peer::save(void *tPtr) const
 
 int Peer::marshal(uint8_t data[ZT_PEER_MARSHAL_SIZE_MAX]) const noexcept
 {
-	data[0] = 0; // serialized peer version
-
 	RWMutex::RLock l(m_lock);
 
-	int s = m_identityKey.marshal(RR->localCacheSymmetric, data + 1);
+	if (!m_identityKey)
-	if (s < 0)
 		return -1;
-	int p = 1 + s;
 
-	s = m_id.marshal(data + p, false);
+	data[0] = 0; // serialized peer version
+
+	// Include our identity's address to detect if this changes and require
+	// recomputation of m_identityKey.
+	RR->identity.address().copyTo(data + 1);
+
+	// SECURITY: encryption in place is only to protect secrets if they are
+	// cached to local storage. It's not used over the wire. Dumb ECB is fine
+	// because secret keys are random and have no structure to reveal.
+	RR->localCacheSymmetric.encrypt(m_identityKey->secret,data + 6);
+	RR->localCacheSymmetric.encrypt(m_identityKey->secret + 22,data + 17);
+	RR->localCacheSymmetric.encrypt(m_identityKey->secret + 38,data + 33);
+
+	int p = 54;
+
+	int s = m_id.marshal(data + p, false);
 	if (s < 0)
 		return -1;
 	p += s;
@@ -431,35 +443,38 @@ int Peer::unmarshal(const uint8_t *restrict data,const int len) noexcept
 {
 	RWMutex::Lock l(m_lock);
 
-	if ((len <= 1) || (data[0] != 0))
+	if ((len <= 54) || (data[0] != 0))
 		return -1;
 
-	int s = m_identityKey.unmarshal(RR->localCacheSymmetric, data + 1, len);
+	m_identityKey.zero();
-	if (s < 0)
+	m_ephemeralKeys[0].zero();
-		return -1;
+	m_ephemeralKeys[1].zero();
-	int p = 1 + s;
 
-	// If the identity key did not pass verification, it may mean that our local
+	if (Address(data + 1) == RR->identity.address()) {
-	// identity has changed. In this case we do not have to forget everything about
+		uint8_t k[ZT_SYMMETRIC_KEY_SIZE];
-	// the peer but we must generate a new identity key by key agreement with our
+		static_assert(ZT_SYMMETRIC_KEY_SIZE == 48,"marshal() and unmarshal() must be revisited if ZT_SYMMETRIC_KEY_SIZE is changed");
-	// new identity.
+		RR->localCacheSymmetric.decrypt(data + 1,k);
-	if (!m_identityKey) {
+		RR->localCacheSymmetric.decrypt(data + 17,k + 16);
-		uint8_t tmp[ZT_SYMMETRIC_KEY_SIZE];
+		RR->localCacheSymmetric.decrypt(data + 33,k + 32);
-		if (!RR->identity.agree(m_id, tmp))
+		m_identityKey.set(new SymmetricKey(RR->node->now(),k,true));
-			return -1;
+		Utils::burn(k,sizeof(k));
-		m_identityKey.init(RR->node->now(), tmp);
-		Utils::burn(tmp,sizeof(tmp));
 	}
 
-	// These are ephemeral and start out as NIL after unmarshal.
+	int p = 49;
-	m_ephemeralKeys[0].clear();
-	m_ephemeralKeys[1].clear();
 
-	s = m_id.unmarshal(data + 38, len - 38);
+	int s = m_id.unmarshal(data + 38, len - 38);
 	if (s < 0)
 		return s;
 	p += s;
 
+	if (!m_identityKey) {
+		uint8_t k[ZT_SYMMETRIC_KEY_SIZE];
+		if (!RR->identity.agree(m_id,k))
+			return -1;
+		m_identityKey.set(new SymmetricKey(RR->node->now(),k,true));
+		Utils::burn(k,sizeof(k));
+	}
+
 	s = m_locator.unmarshal(data + p, len - p);
 	if (s < 0)
 		return s;
@@ -523,4 +538,40 @@ void Peer::m_prioritizePaths(int64_t now)
 	}
 }
 
+unsigned int Peer::m_sendProbe(void *tPtr,int64_t localSocket,const InetAddress &atAddress,int64_t now)
+{
+	// Assumes m_lock is locked
+	if ((m_vProto < 11)||(m_probe == 0)) {
+		const SharedPtr<SymmetricKey> k(m_key());
+		const uint64_t packetId = k->nextMessage(RR->identity.address(),m_id.address());
+
+		uint8_t p[ZT_PROTO_MIN_PACKET_LENGTH + 1];
+		Utils::storeAsIsEndian<uint64_t>(p + ZT_PROTO_PACKET_ID_INDEX,packetId);
+		m_id.address().copyTo(p + ZT_PROTO_PACKET_DESTINATION_INDEX);
+		RR->identity.address().copyTo(p + ZT_PROTO_PACKET_SOURCE_INDEX);
+		p[ZT_PROTO_PACKET_FLAGS_INDEX] = 0;
+		p[ZT_PROTO_PACKET_VERB_INDEX] = Protocol::VERB_ECHO;
+		p[ZT_PROTO_PACKET_VERB_INDEX + 1] = (uint8_t)now; // arbitrary byte
+
+		Protocol::armor(p,ZT_PROTO_MIN_PACKET_LENGTH,k,cipher());
+
+		RR->expect->sending(packetId,now);
+		RR->node->putPacket(tPtr,-1,atAddress,p,ZT_PROTO_MIN_PACKET_LENGTH);
+
+		return ZT_PROTO_MIN_PACKET_LENGTH;
+	} else {
+		RR->node->putPacket(tPtr,-1,atAddress,&m_probe,4);
+		return 4;
+	}
+}
+
+void Peer::m_deriveSecondaryIdentityKeys() noexcept
+{
+	uint8_t hk[ZT_SYMMETRIC_KEY_SIZE];
+	KBKDFHMACSHA384(m_identityKey->secret,ZT_KBKDF_LABEL_HELLO_DICTIONARY_ENCRYPT,0,0,hk);
+	m_helloCipher.init(hk);
+	Utils::burn(hk,sizeof(hk));
+	KBKDFHMACSHA384(m_identityKey->secret,ZT_KBKDF_LABEL_PACKET_HMAC,0,0,m_helloMacKey);
+}
+
 } // namespace ZeroTier

node/Peer.hpp

@@ -32,7 +32,7 @@
 #include "Containers.hpp"
 
 // version, identity, locator, bootstrap, version info, length of any additional fields
-#define ZT_PEER_MARSHAL_SIZE_MAX (1 + ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX + ZT_IDENTITY_MARSHAL_SIZE_MAX + ZT_LOCATOR_MARSHAL_SIZE_MAX + 1 + (ZT_MAX_PEER_NETWORK_PATHS * ZT_ENDPOINT_MARSHAL_SIZE_MAX) + (2*4) + 2)
+#define ZT_PEER_MARSHAL_SIZE_MAX (1 + ZT_ADDRESS_LENGTH + ZT_SYMMETRIC_KEY_SIZE + ZT_IDENTITY_MARSHAL_SIZE_MAX + ZT_LOCATOR_MARSHAL_SIZE_MAX + 1 + (ZT_MAX_PEER_NETWORK_PATHS * ZT_ENDPOINT_MARSHAL_SIZE_MAX) + (2*4) + 2)
 
 namespace ZeroTier {
 
@@ -50,8 +50,8 @@ public:
 	/**
 	 * Create an uninitialized peer
 	 *
-	 * The peer will need to be initialized with init() or unmarshal() before
+	 * New peers must be initialized via either init() or unmarshal() prior to
-	 * it can be used.
+	 * use or null pointer dereference may occur.
 	 *
 	 * @param renv Runtime environment
 	 */
@@ -95,7 +95,7 @@ public:
 	 * @param t New probe token
 	 * @return Old probe token
 	 */
-	ZT_INLINE uint32_t setProbeToken(const uint32_t t) const noexcept
+	ZT_INLINE uint32_t setProbeToken(const uint32_t t) noexcept
 	{
 		RWMutex::Lock l(m_lock);
 		const uint32_t pt = m_probe;
@@ -186,7 +186,11 @@ public:
 	 * @param len Length in bytes
 	 * @param via Path over which to send data (may or may not be an already-learned path for this peer)
 	 */
-	void send(void *tPtr,int64_t now,const void *data,unsigned int len,const SharedPtr<Path> &via) noexcept;
+	ZT_INLINE void send(void *tPtr,int64_t now,const void *data,unsigned int len,const SharedPtr<Path> &via) noexcept
+	{
+		via->send(RR,tPtr,data,len,now);
+		sent(now,len);
+	}
 
 	/**
 	 * Send data to this peer over the best available path
@@ -212,17 +216,6 @@ public:
 	 */
 	unsigned int hello(void *tPtr,int64_t localSocket,const InetAddress &atAddress,int64_t now);
 
-	/**
-	 * Send a NOP message to e.g. probe a new link
-	 *
-	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
-	 * @param localSocket Local source socket
-	 * @param atAddress Destination address
-	 * @param now Current time
-	 * @return Number of bytes sent
-	 */
-	unsigned int probe(void *tPtr,int64_t localSocket,const InetAddress &atAddress,int64_t now);
-
 	/**
 	 * Ping this peer if needed and/or perform other periodic tasks.
 	 *
@@ -303,13 +296,71 @@ public:
 	}
 
 	/**
-	 * @return Preferred cipher suite for normal encrypted P2P communication
+	 * @return Cipher suite that should be used to communicate with this peer
 	 */
 	ZT_INLINE uint8_t cipher() const noexcept
 	{
+		//if (m_vProto >= 11)
+		//	return ZT_PROTO_CIPHER_SUITE__AES_GMAC_SIV;
 		return ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012;
 	}
 
+	/**
+	 * @return The permanent shared key for this peer computed by simple identity agreement
+	 */
+	ZT_INLINE SharedPtr<SymmetricKey> identityKey() noexcept
+	{
+		return m_identityKey;
+	}
+
+	/**
+	 * @return AES instance for HELLO dictionary / encrypted section encryption/decryption
+	 */
+	ZT_INLINE const AES &identityHelloDictionaryEncryptionCipher() noexcept
+	{
+		return m_helloCipher;
+	}
+
+	/**
+	 * @return Key for HMAC on HELLOs
+	 */
+	ZT_INLINE const uint8_t *identityHelloHmacKey() noexcept
+	{
+		return m_helloMacKey;
+	}
+
+	/**
+	 * @return Raw identity key bytes
+	 */
+	ZT_INLINE const uint8_t *rawIdentityKey() noexcept
+	{
+		RWMutex::RLock l(m_lock);
+		return m_identityKey->secret;
+	}
+
+	/**
+	 * @return Current best key: either the latest ephemeral or the identity key
+	 */
+	ZT_INLINE SharedPtr<SymmetricKey> key() noexcept
+	{
+		RWMutex::RLock l(m_lock);
+		return m_key();
+	}
+
+	/**
+	 * Check whether a key is ephemeral
+	 * 
+	 * This is used to check whether a packet is received with forward secrecy enabled
+	 * or not.
+	 * 
+	 * @param k Key to check
+	 * @return True if this key is ephemeral, false if it's the long-lived identity key
+	 */
+	ZT_INLINE bool isEphemeral(const SharedPtr<SymmetricKey> &k) const noexcept
+	{
+		return (m_identityKey != k);
+	}
+
 	/**
 	 * Set the currently known remote version of this peer's client
 	 *
@@ -342,7 +393,7 @@ public:
 	 *
 	 * @param paths Vector of paths with the first path being the current preferred path
 	 */
-	void getAllPaths(std::vector< SharedPtr<Path> > &paths);
+	void getAllPaths(Vector< SharedPtr<Path> > &paths);
 
 	/**
 	 * Save the latest version of this peer to the data store
@@ -379,19 +430,45 @@ public:
 		return false;
 	}
 
+	/**
+	 * Rate limit gate for inbound probes
+	 */
+	ZT_INLINE bool rateGateProbeRequest(const int64_t now) noexcept
+	{
+		if((now - m_lastProbeReceived) > ZT_PEER_PROBE_RESPONSE_RATE_LIMIT) {
+			m_lastProbeReceived = now;
+			return true;
+		}
+		return false;
+	}
+
 private:
 	void m_prioritizePaths(int64_t now);
+	unsigned int m_sendProbe(void *tPtr,int64_t localSocket,const InetAddress &atAddress,int64_t now);
+	void m_deriveSecondaryIdentityKeys() noexcept;
+
+	ZT_INLINE SharedPtr<SymmetricKey> m_key() noexcept
+	{
+		// assumes m_lock is locked (for read at least)
+		return (m_ephemeralKeys[0]) ? m_ephemeralKeys[0] : m_identityKey;
+	}
 
 	const RuntimeEnvironment *RR;
 
 	// Read/write mutex for non-atomic non-const fields.
 	RWMutex m_lock;
 
-	// The permanent identity key resulting from agreement between our identity and this peer's identity.
+	// Static identity key
-	SymmetricKey< AES,0,0 > m_identityKey;
+	SharedPtr<SymmetricKey> m_identityKey;
 
-	// Most recently successful (for decrypt) ephemeral key and one previous key.
+	// Cipher for encrypting or decrypting the encrypted section of HELLO packets.
-	SymmetricKey< AES,ZT_SYMMETRIC_KEY_TTL,ZT_SYMMETRIC_KEY_TTL_MESSAGES > m_ephemeralKeys[2];
+	AES m_helloCipher;
+
+	// Key for HELLO HMAC-SHA384
+	uint8_t m_helloMacKey[ZT_SYMMETRIC_KEY_SIZE];
+
+	// Current and previous ephemeral key
+	SharedPtr<SymmetricKey> m_ephemeralKeys[2];
 
 	Identity m_id;
 	Locator m_locator;
@@ -412,6 +489,9 @@ private:
 	// The last time we sorted paths in order of preference. (This happens pretty often.)
 	std::atomic<int64_t> m_lastPrioritizedPaths;
 
+	// The last time we got a probe from this peer.
+	std::atomic<int64_t> m_lastProbeReceived;
+
 	// Meters measuring actual bandwidth in, out, and relayed via this peer (mostly if this is a root).
 	Meter<> m_inMeter;
 	Meter<> m_outMeter;

node/Poly1305.cpp

@@ -425,12 +425,20 @@ ZT_INLINE void poly1305_update(poly1305_context *ctx,const unsigned char *m,size
 
 } // anonymous namespace
 
-void poly1305(void *auth,const void *data,unsigned int len,const void *key) noexcept
+void Poly1305::init(const void *key) noexcept
 {
-  poly1305_context ctx;
+  static_assert(sizeof(ctx) >= sizeof(poly1305_context),"buffer in class smaller than required structure size");
-  poly1305_init(&ctx,reinterpret_cast<const unsigned char *>(key));
+  poly1305_init(reinterpret_cast<poly1305_context *>(&ctx),reinterpret_cast<const unsigned char *>(key));
-  poly1305_update(&ctx,reinterpret_cast<const unsigned char *>(data),(size_t)len);
+}
-  poly1305_finish(&ctx,reinterpret_cast<unsigned char *>(auth));
+
+void Poly1305::update(const void *data,unsigned int len) noexcept
+{
+  poly1305_update(reinterpret_cast<poly1305_context *>(&ctx),reinterpret_cast<const unsigned char *>(data),(size_t)len);
+}
+
+void Poly1305::finish(void *auth) noexcept
+{
+  poly1305_finish(reinterpret_cast<poly1305_context *>(&ctx),reinterpret_cast<unsigned char *>(auth));
 }
 
 } // namespace ZeroTier

node/Poly1305.hpp

@@ -20,14 +20,24 @@ namespace ZeroTier {
 #define ZT_POLY1305_MAC_SIZE 16
 
 /**
- * Compute a one-time authentication code
+ * Poly1305 one-time MAC calculator
- *
- * @param auth Buffer to receive code -- MUST be 16 bytes in length
- * @param data Data to authenticate
- * @param len Length of data to authenticate in bytes
- * @param key 32-byte one-time use key to authenticate data (must not be reused)
  */
-void poly1305(void *auth,const void *data,unsigned int len,const void *key) noexcept;
+class Poly1305
+{
+public:
+	ZT_INLINE Poly1305() {}
+	ZT_INLINE Poly1305(const void *key) { this->init(key); }
+
+	void init(const void *key) noexcept;
+	void update(const void *data,unsigned int len) noexcept;
+	void finish(void *auth) noexcept;
+
+private:
+	struct {
+	  size_t aligner;
+  	unsigned char opaque[136];
+	} ctx;
+};
 
 } // namespace ZeroTier
 

node/Protocol.cpp

@@ -1,93 +0,0 @@
-/*
- * Copyright (c)2013-2020 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2024-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-#include "Protocol.hpp"
-#include "Buf.hpp"
-#include "Utils.hpp"
-
-#include <cstdlib>
-#include <ctime>
-
-#ifdef __WINDOWS__
-#include <process.h>
-#else
-#include <unistd.h>
-#endif
-
-namespace ZeroTier {
-namespace Protocol {
-
-void armor(Buf &pkt,int packetSize,const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],uint8_t cipherSuite) noexcept
-{
-	Protocol::Header &ph = pkt.as<Protocol::Header>(); // NOLINT(hicpp-use-auto,modernize-use-auto)
-	ph.flags = (ph.flags & 0xc7U) | ((cipherSuite << 3U) & 0x38U); // flags: FFCCCHHH where CCC is cipher
-
-	switch(cipherSuite) {
-		case ZT_PROTO_CIPHER_SUITE__POLY1305_NONE: {
-			uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
-			salsa2012DeriveKey(key,perPacketKey,pkt,packetSize);
-			Salsa20 s20(perPacketKey,&ph.packetId);
-
-			uint8_t macKey[ZT_POLY1305_KEY_SIZE];
-			s20.crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
-
-			// only difference here is that we don't encrypt the payload
-
-			uint64_t mac[2];
-			poly1305(mac,pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,macKey);
-			ph.mac = mac[0];
-		} break;
-
-		case ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012: {
-			uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
-			salsa2012DeriveKey(key,perPacketKey,pkt,packetSize);
-			Salsa20 s20(perPacketKey,&ph.packetId);
-
-			uint8_t macKey[ZT_POLY1305_KEY_SIZE];
-			s20.crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
-
-			const unsigned int encLen = packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START;
-			s20.crypt12(pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,encLen);
-
-			uint64_t mac[2];
-			poly1305(mac,pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,encLen,macKey);
-			ph.mac = mac[0];
-		} break;
-
-		case ZT_PROTO_CIPHER_SUITE__AES_GMAC_SIV: {
-		} break;
-	}
-}
-
-int compress(SharedPtr<Buf> &pkt,int packetSize) noexcept
-{
-	if (packetSize <= 128)
-		return packetSize;
-
-	SharedPtr<Buf> pkt2(new Buf());
-	if (!pkt2) return packetSize;
-
-	const int uncompressedLen = packetSize - ZT_PROTO_PACKET_PAYLOAD_START;
-	const int compressedLen = LZ4_compress_fast(reinterpret_cast<const char *>(pkt->unsafeData + ZT_PROTO_PACKET_PAYLOAD_START),reinterpret_cast<char *>(pkt2->unsafeData + ZT_PROTO_PACKET_PAYLOAD_START),uncompressedLen,ZT_BUF_MEM_SIZE - ZT_PROTO_PACKET_PAYLOAD_START);
-	if ((compressedLen > 0)&&(compressedLen < uncompressedLen)) {
-		Utils::copy<ZT_PROTO_PACKET_PAYLOAD_START>(pkt2->unsafeData,pkt->unsafeData);
-		pkt.swap(pkt2);
-		pkt->as<Protocol::Header>().verb |= ZT_PROTO_VERB_FLAG_COMPRESSED;
-		return compressedLen + ZT_PROTO_PACKET_PAYLOAD_START;
-	}
-
-	return packetSize;
-}
-
-} // namespace Protocol
-} // namespace ZeroTier

node/Protocol.hpp

@@ -22,6 +22,7 @@
 #include "Buf.hpp"
 #include "Address.hpp"
 #include "Identity.hpp"
+#include "SymmetricKey.hpp"
 
 /*
  * Packet format:
@@ -188,16 +189,6 @@
  */
 #define ZT_PROTO_PACKET_FRAGMENT_INDICATOR 0xff
 
-/**
- * Index at which fragment indicator is found in fragments
- */
-#define ZT_PROTO_PACKET_FRAGMENT_INDICATOR_INDEX 13
-
-/**
- * Index of flags field in regular packet headers
- */
-#define ZT_PROTO_PACKET_FLAGS_INDEX 18
-
 /**
  * Length of a probe packet
  */
@@ -248,6 +239,16 @@
  */
 #define ZT_KBKDF_LABEL_PACKET_HMAC 'M'
 
+#define ZT_PROTO_PACKET_FRAGMENT_INDICATOR_INDEX 13
+#define ZT_PROTO_PACKET_FRAGMENT_COUNTS          14
+
+#define ZT_PROTO_PACKET_ID_INDEX          0
+#define ZT_PROTO_PACKET_DESTINATION_INDEX 8
+#define ZT_PROTO_PACKET_SOURCE_INDEX      13
+#define ZT_PROTO_PACKET_FLAGS_INDEX       18
+#define ZT_PROTO_PACKET_MAC_INDEX         19
+#define ZT_PROTO_PACKET_VERB_INDEX        27
+
 #define ZT_PROTO_HELLO_NODE_META_INSTANCE_ID      "i"
 #define ZT_PROTO_HELLO_NODE_META_LOCATOR          "l"
 #define ZT_PROTO_HELLO_NODE_META_PROBE_TOKEN      "p"
@@ -288,8 +289,7 @@ enum Verb
 	 *   <[8] timestamp>
 	 *   <[...] binary serialized full sender identity>
 	 *   <[...] physical destination address of packet (LEGACY)>
-	 *   <[2] 16-bit reserved "encrypted zero" field (LEGACY)>
+	 *   <[12] 96-bit CTR IV>
-	 *   <[4] 32 additional random nonce bits>
 	 *   [... start of encrypted section ...]
 	 *   <[2] 16-bit length of encrypted dictionary>
 	 *   <[...] encrypted dictionary>
@@ -306,9 +306,8 @@ enum Verb
 	 * good to proactively limit exposed information.
 	 *
 	 * Inner encryption is AES-CTR with a key derived using KBKDF and a
-	 * label indicating this specific usage. The 96-bit CTR nonce is the
+	 * label indicating this specific usage. A 96-bit CTR IV precedes this
-	 * packet ID followed by the additional 32 random bits provided before
+	 * encrypted section.
-	 * the encrypted section.
 	 *
 	 * Authentication and encryption in HELLO and OK(HELLO) are always done
 	 * with the long-lived identity key, not ephemeral shared keys. This
@@ -319,7 +318,7 @@ enum Verb
 	 * HELLO and OK(HELLO) include an extra HMAC at the end of the packet.
 	 * This authenticates them to a level of certainty beyond that afforded
 	 * by regular AEAD. HMAC is computed over the whole packet prior to
-	 * encryption/MAC and with the 3-bit hop count field masked as it is
+	 * packet MAC and with the 3-bit hop count field masked as it is
 	 * with regular packet AEAD, and it is then included in the regular
 	 * packet MAC.
 	 *
@@ -811,15 +810,79 @@ static ZT_INLINE void salsa2012DeriveKey(const uint8_t *const in,uint8_t *const
 #endif
 }
 
+/**
+ * Fill out packet header fields (except for mac, which is filled out by armor())
+ * 
+ * @param pkt Start of packet buffer
+ * @param packetId Packet IV / cryptographic MAC
+ * @param destination Destination ZT address
+ * @param source Source (sending) ZT address
+ * @param verb Protocol verb
+ * @return Index of packet start
+ */
+static ZT_INLINE int newPacket(uint8_t pkt[28],const uint64_t packetId,const Address destination,const Address source,const Verb verb) noexcept
+{
+	Utils::storeAsIsEndian<uint64_t>(pkt + ZT_PROTO_PACKET_ID_INDEX,packetId);
+	destination.copyTo(pkt + ZT_PROTO_PACKET_DESTINATION_INDEX);
+	source.copyTo(pkt + ZT_PROTO_PACKET_SOURCE_INDEX);
+	pkt[ZT_PROTO_PACKET_FLAGS_INDEX] = 0;
+	// mac is left undefined as it's filled out by armor()
+	pkt[ZT_PROTO_PACKET_VERB_INDEX] = (uint8_t)verb;
+	return ZT_PROTO_PACKET_VERB_INDEX + 1;
+}
+static ZT_INLINE int newPacket(Buf &pkt,const uint64_t packetId,const Address destination,const Address source,const Verb verb) noexcept { return newPacket(pkt.unsafeData,packetId,destination,source,verb); }
+
 /**
  * Encrypt and compute packet MAC
  *
  * @param pkt Packet data to encrypt (in place)
  * @param packetSize Packet size, must be at least ZT_PROTO_MIN_PACKET_LENGTH or crash will occur
- * @param key Key to use for encryption (not per-packet key)
+ * @param key Key to use for encryption
  * @param cipherSuite Cipher suite to use for AEAD encryption or just MAC
  */
-void armor(Buf &pkt,int packetSize,const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],uint8_t cipherSuite) noexcept;
+static ZT_INLINE void armor(uint8_t *const pkt,const int packetSize,const SharedPtr<SymmetricKey> &key,const uint8_t cipherSuite) noexcept
+{
+#if 0
+	Protocol::Header &ph = pkt.as<Protocol::Header>(); // NOLINT(hicpp-use-auto,modernize-use-auto)
+	ph.flags = (ph.flags & 0xc7U) | ((cipherSuite << 3U) & 0x38U); // flags: FFCCCHHH where CCC is cipher
+
+	switch(cipherSuite) {
+		case ZT_PROTO_CIPHER_SUITE__POLY1305_NONE: {
+			uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
+			salsa2012DeriveKey(key,perPacketKey,pkt,packetSize);
+			Salsa20 s20(perPacketKey,&ph.packetId);
+
+			uint8_t macKey[ZT_POLY1305_KEY_SIZE];
+			s20.crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
+
+			// only difference here is that we don't encrypt the payload
+
+			uint64_t mac[2];
+			poly1305(mac,pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,macKey);
+			ph.mac = mac[0];
+		} break;
+
+		case ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012: {
+			uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
+			salsa2012DeriveKey(key,perPacketKey,pkt,packetSize);
+			Salsa20 s20(perPacketKey,&ph.packetId);
+
+			uint8_t macKey[ZT_POLY1305_KEY_SIZE];
+			s20.crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
+
+			const unsigned int encLen = packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START;
+			s20.crypt12(pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,encLen);
+
+			uint64_t mac[2];
+			poly1305(mac,pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,encLen,macKey);
+			ph.mac = mac[0];
+		} break;
+
+		case ZT_PROTO_CIPHER_SUITE__AES_GMAC_SIV: {
+		} break;
+	}
+#endif
+}
 
 /**
  * Attempt to compress packet payload
@@ -833,7 +896,11 @@ void armor(Buf &pkt,int packetSize,const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],uint
  * @param packetSize Total size of packet in bytes (including headers)
  * @return New size of packet after compression or original size of compression wasn't helpful
  */
-int compress(SharedPtr<Buf> &pkt,int packetSize) noexcept;
+static ZT_INLINE int compress(SharedPtr<Buf> &pkt,int packetSize) noexcept
+{
+	// TODO
+	return packetSize;
+}
 
 } // namespace Protocol
 } // namespace ZeroTier

node/RuntimeEnvironment.hpp

@@ -40,7 +40,7 @@ class Expect;
 class RuntimeEnvironment
 {
 public:
-	ZT_INLINE RuntimeEnvironment(Node *n) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init,google-explicit-constructor,hicpp-explicit-conversions)
+	ZT_INLINE RuntimeEnvironment(Node *n) noexcept :
 		node(n),
 		localNetworkController(nullptr),
 		rtmem(nullptr),
@@ -51,8 +51,8 @@ public:
 		topology(nullptr),
 		sa(nullptr)
 	{
-		publicIdentityStr[0] = (char)0;
+		publicIdentityStr[0] = nullptr;
-		secretIdentityStr[0] = (char)0;
+		secretIdentityStr[0] = nullptr;
 	}
 
 	ZT_INLINE ~RuntimeEnvironment() noexcept

node/SHA512.cpp

@@ -199,7 +199,7 @@ void SHA384(void *digest,const void *data0,unsigned int len0,const void *data1,u
 
 #endif // !ZT_HAVE_NATIVE_SHA512
 
-void HMACSHA384(const uint8_t key[32],const void *msg,const unsigned int msglen,uint8_t mac[48])
+void HMACSHA384(const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],const void *msg,const unsigned int msglen,uint8_t mac[48])
 {
 	uint64_t kInPadded[16]; // input padded key
 	uint64_t outer[22]; // output padded key | H(input padded key | msg)
@@ -208,14 +208,16 @@ void HMACSHA384(const uint8_t key[32],const void *msg,const unsigned int msglen,
 	const uint64_t k1 = Utils::loadAsIsEndian<uint64_t>(key + 8);
 	const uint64_t k2 = Utils::loadAsIsEndian<uint64_t>(key + 16);
 	const uint64_t k3 = Utils::loadAsIsEndian<uint64_t>(key + 24);
+	const uint64_t k4 = Utils::loadAsIsEndian<uint64_t>(key + 32);
+	const uint64_t k5 = Utils::loadAsIsEndian<uint64_t>(key + 40);
 
 	const uint64_t ipad = 0x3636363636363636ULL;
 	kInPadded[0] = k0 ^ ipad;
 	kInPadded[1] = k1 ^ ipad;
 	kInPadded[2] = k2 ^ ipad;
 	kInPadded[3] = k3 ^ ipad;
-	kInPadded[4] = ipad;
+	kInPadded[4] = k4 ^ ipad;
-	kInPadded[5] = ipad;
+	kInPadded[5] = k5 ^ ipad;
 	kInPadded[6] = ipad;
 	kInPadded[7] = ipad;
 	kInPadded[8] = ipad;
@@ -232,8 +234,8 @@ void HMACSHA384(const uint8_t key[32],const void *msg,const unsigned int msglen,
 	outer[1] = k1 ^ opad;
 	outer[2] = k2 ^ opad;
 	outer[3] = k3 ^ opad;
-	outer[4] = opad;
+	outer[4] = k4 ^ opad;
-	outer[5] = opad;
+	outer[5] = k5 ^ opad;
 	outer[6] = opad;
 	outer[7] = opad;
 	outer[8] = opad;
@@ -245,12 +247,12 @@ void HMACSHA384(const uint8_t key[32],const void *msg,const unsigned int msglen,
 	outer[14] = opad;
 	outer[15] = opad;
 
-	SHA384(reinterpret_cast<uint8_t *>(outer) + 128,kInPadded,128,msg,msglen); // H(input padded key | msg)
+	// H(output padded key | H(input padded key | msg))
-
+	SHA384(reinterpret_cast<uint8_t *>(outer) + 128,kInPadded,128,msg,msglen);
-	SHA384(mac,outer,176); // H(output padded key | H(input padded key | msg))
+	SHA384(mac,outer,176);
 }
 
-void KBKDFHMACSHA384(const uint8_t key[32],const char label,const char context,const uint32_t iter,uint8_t out[32])
+void KBKDFHMACSHA384(const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],const char label,const char context,const uint32_t iter,uint8_t out[ZT_SYMMETRIC_KEY_SIZE])
 {
 	uint8_t kbkdfMsg[13];
 	uint8_t kbuf[48];
@@ -265,7 +267,7 @@ void KBKDFHMACSHA384(const uint8_t key[32],const char label,const char context,c
 	kbkdfMsg[11] = 1;
 	kbkdfMsg[12] = 0; // key length: 256 bits as big-endian 32-bit value
 	HMACSHA384(key,&kbkdfMsg,sizeof(kbkdfMsg),kbuf);
-	Utils::copy<32>(out,kbuf);
+	Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(out,kbuf);
 }
 
 } // namespace ZeroTier

node/SHA512.hpp

@@ -70,7 +70,7 @@ void SHA384(void *digest,const void *data0,unsigned int len0,const void *data1,u
  * @param msglen Length of message
  * @param mac Buffer to fill with result
  */
-void HMACSHA384(const uint8_t key[32],const void *msg,unsigned int msglen,uint8_t mac[48]);
+void HMACSHA384(const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],const void *msg,unsigned int msglen,uint8_t mac[48]);
 
 /**
  * Compute KBKDF (key-based key derivation function) using HMAC-SHA-384 as a PRF
@@ -81,7 +81,7 @@ void HMACSHA384(const uint8_t key[32],const void *msg,unsigned int msglen,uint8_
  * @param iter Key iteration for generation of multiple keys for the same label/context
  * @param out Output to receive derived key
  */
-void KBKDFHMACSHA384(const uint8_t key[32],char label,char context,uint32_t iter,uint8_t out[32]);
+void KBKDFHMACSHA384(const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],char label,char context,uint32_t iter,uint8_t out[ZT_SYMMETRIC_KEY_SIZE]);
 
 } // namespace ZeroTier
 

node/Salsa20.hpp

@@ -29,6 +29,8 @@
 #define ZT_SALSA20_SSE 1
 #endif
 
+#define ZT_SALSA20_KEY_SIZE 32
+
 namespace ZeroTier {
 
 /**
@@ -43,14 +45,14 @@ public:
 	static constexpr bool accelerated() noexcept { return false; }
 #endif
 
-	ZT_INLINE Salsa20() noexcept {} // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init,hicpp-use-equals-default,modernize-use-equals-default)
+	ZT_INLINE Salsa20() noexcept {}
 	ZT_INLINE ~Salsa20() { Utils::burn(&_state,sizeof(_state)); }
 
 	/**
 	 * @param key 256-bit (32 byte) key
 	 * @param iv 64-bit initialization vector
 	 */
-	ZT_INLINE Salsa20(const void *key,const void *iv) noexcept { init(key,iv); } // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	ZT_INLINE Salsa20(const void *key,const void *iv) noexcept { init(key,iv); }
 
 	/**
 	 * Initialize cipher

node/ScopedPtr.hpp

@@ -49,7 +49,7 @@ public:
 	ZT_INLINE bool operator!=(T *const p) const noexcept { return (m_ptr != p); }
 
 private:
-	ZT_INLINE ScopedPtr() noexcept {} // NOLINT(hicpp-use-equals-default,hicpp-use-equals-delete,modernize-use-equals-default)
+	ZT_INLINE ScopedPtr() noexcept {}
 	ZT_INLINE ScopedPtr(const ScopedPtr &p) noexcept : m_ptr(nullptr) {}
 	ZT_INLINE ScopedPtr &operator=(const ScopedPtr &p) noexcept { return *this; }
 

node/SelfAwareness.hpp

@@ -70,7 +70,7 @@ private:
 		InetAddress reporterPhysicalAddress;
 		InetAddress::IpScope scope;
 
-		ZT_INLINE p_PhySurfaceKey() noexcept {} // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init,hicpp-use-equals-default,modernize-use-equals-default)
+		ZT_INLINE p_PhySurfaceKey() noexcept {}
 		ZT_INLINE p_PhySurfaceKey(const Address &r, const int64_t rol, const InetAddress &ra, InetAddress::IpScope s) noexcept : reporter(r), receivedOnLocalSocket(rol), reporterPhysicalAddress(ra), scope(s) {}
 
 		ZT_INLINE unsigned long hashCode() const noexcept { return ((unsigned long)reporter.toInt() + (unsigned long)receivedOnLocalSocket + (unsigned long)scope); }

node/SharedPtr.hpp

@@ -109,7 +109,7 @@ public:
 		from.m_ptr = nullptr;
 	}
 
-	ZT_INLINE operator bool() const noexcept { return (m_ptr != nullptr); } // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
+	ZT_INLINE operator bool() const noexcept { return (m_ptr != nullptr); }
 
 	ZT_INLINE T &operator*() const noexcept { return *m_ptr; }
 	ZT_INLINE T *operator->() const noexcept { return m_ptr; }

node/Speck128.hpp

@@ -39,14 +39,14 @@ public:
 	/**
 	 * Create an uninitialized instance, init() must be called to set up.
 	 */
-	ZT_INLINE Speck128() noexcept {} // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init,hicpp-use-equals-default,modernize-use-equals-default)
+	ZT_INLINE Speck128() noexcept {}
 
 	/**
 	 * Initialize Speck from a 128-bit key
 	 *
 	 * @param k 128-bit / 16 byte key
 	 */
-	ZT_INLINE Speck128(const void *k) noexcept { this->init(k); } // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init,google-explicit-constructor,hicpp-explicit-conversions)
+	ZT_INLINE Speck128(const void *k) noexcept { this->init(k); }
 
 	ZT_INLINE ~Speck128() { Utils::burn(m_expandedKey, sizeof(m_expandedKey)); }
 
@@ -162,8 +162,8 @@ public:
 	 */
 	ZT_INLINE void encrypt(const void *const in,void *const out) const noexcept
 	{
-		uint64_t x = Utils::loadLittleEndian<uint64_t>(in); // NOLINT(hicpp-use-auto,modernize-use-auto)
+		uint64_t x = Utils::loadLittleEndian<uint64_t>(in);
-		uint64_t y = Utils::loadLittleEndian<uint64_t>(reinterpret_cast<const uint8_t *>(in) + 8); // NOLINT(hicpp-use-auto,modernize-use-auto)
+		uint64_t y = Utils::loadLittleEndian<uint64_t>(reinterpret_cast<const uint8_t *>(in) + 8);
 		encryptXY(x,y);
 		Utils::storeLittleEndian<uint64_t>(out,x);
 		Utils::storeLittleEndian<uint64_t>(reinterpret_cast<uint8_t *>(out) + 8,y);
@@ -177,8 +177,8 @@ public:
 	 */
 	ZT_INLINE void decrypt(const void *const in,void *const out) const noexcept
 	{
-		uint64_t x = Utils::loadLittleEndian<uint64_t>(in); // NOLINT(hicpp-use-auto,modernize-use-auto)
+		uint64_t x = Utils::loadLittleEndian<uint64_t>(in);
-		uint64_t y = Utils::loadLittleEndian<uint64_t>(reinterpret_cast<const uint8_t *>(in) + 8); // NOLINT(hicpp-use-auto,modernize-use-auto)
+		uint64_t y = Utils::loadLittleEndian<uint64_t>(reinterpret_cast<const uint8_t *>(in) + 8);
 		decryptXY(x,y);
 		Utils::storeLittleEndian<uint64_t>(out,x);
 		Utils::storeLittleEndian<uint64_t>(reinterpret_cast<uint8_t *>(out) + 8,y);

node/SymmetricKey.hpp

@@ -17,119 +17,70 @@
 #include "Constants.hpp"
 #include "Utils.hpp"
 #include "InetAddress.hpp"
+#include "AES.hpp"
+#include "SharedPtr.hpp"
+#include "Address.hpp"
 
 namespace ZeroTier {
 
-#define ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX 52
-
 /**
  * Container for symmetric keys and ciphers initialized with them
  *
  * This container is responsible for tracking key TTL to maintain it
  * below our security bounds and tell us when it's time to re-key.
- *
- * Set TTL and TTLM to 0 for permanent keys. These still track uses
- * but do not signal expiration.
- *
- * @tparam C Cipher to embed (must accept key in constructor and/or init() method)
- * @tparam TTL Maximum time to live in milliseconds or 0 for a permanent key with unlimited TTL
- * @tparam TTLM Maximum time to live in messages or 0 for a permanent key with unlimited TTL
  */
-template<typename C,int64_t TTL,uint64_t TTLM>
 class SymmetricKey
 {
+	friend class SharedPtr<SymmetricKey>;
+
 public:
+	/**
+	 * Secret key
+	 */
+	const uint8_t secret[ZT_SYMMETRIC_KEY_SIZE];
+
 	/**
 	 * Symmetric cipher keyed with this key
 	 */
-	const C cipher;
+	const AES cipher;
-
-	/**
-	 * Construct an uninitialized symmetric key container
-	 */
-	ZT_INLINE SymmetricKey() noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init,hicpp-use-equals-default,modernize-use-equals-default)
-		cipher(),
-		m_ts(0),
-		m_nonceBase(0),
-		m_odometer(0)
-	{
-		Utils::memoryLock(m_secret, sizeof(m_secret));
-	}
 
 	/**
 	 * Construct a new symmetric key
 	 *
-	 * @param ts Current time (must still be given for permanent keys even though there is no expiry checking)
+	 * SECURITY: we use a best effort method to construct the nonce's starting point so as
-	 * @param key 32-byte / 256-bit key
+	 * to avoid nonce duplication across invocations. The most significant bits are the
+	 * number of seconds since epoch but with the most significant bit masked to zero.
+	 * The least significant bits are random. Key life time is limited to 2^31 messages
+	 * per key as per the AES-GMAC-SIV spec, and this is a SIV mode anyway so nonce repetition
+	 * is non-catastrophic.
+	 * 
+	 * The masking of the most significant bit is because we bisect the nonce space by
+	 * which direction the message is going. If the sender's ZeroTier address is
+	 * numerically greater than the receiver, this bit is flipped. This means that
+	 * two sides of a conversation that have created their key instances at the same
+	 * time are much less likely to duplicate nonces when sending pacekts from either
+	 * end.
+	 * 
+	 * @param ts Current time
+	 * @param key 48-bit / 384-byte key
+	 * @param perm If true this is a permanent key
 	 */
-	explicit ZT_INLINE SymmetricKey(const int64_t ts,const void *const key) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	explicit ZT_INLINE SymmetricKey(const int64_t ts,const void *const key,const bool perm) noexcept :
-		cipher(key),
+		secret(),
+		cipher(key), // uses first 256 bits of 384-bit key
 		m_ts(ts),
-		m_nonceBase((uint64_t)ts << 22U), // << 22 to shift approximately the seconds since epoch into the most significant 32 bits
+		m_nonceBase(((((uint64_t)ts / 1000ULL) << 32U) & 0x7fffffff00000000ULL) | (Utils::random() & 0x00000000ffffffffULL)),
-		m_odometer(0)
+		m_odometer(0),
+		m_permanent(perm)
 	{
-		Utils::memoryLock(m_secret, sizeof(m_secret));
+		Utils::memoryLock(this,sizeof(SymmetricKey));
-		Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(m_secret, key);
+		Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(const_cast<uint8_t *>(secret), key);
-	}
-
-	ZT_INLINE SymmetricKey(const SymmetricKey &k) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-		cipher(k.m_secret),
-		m_ts(k.ts),
-		m_nonceBase(k.m_nonceBase),
-		m_odometer(k.m_odometer)
-	{
-		Utils::memoryLock(m_secret, sizeof(m_secret));
-		Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(m_secret, k.m_secret);
 	}
 
 	ZT_INLINE ~SymmetricKey() noexcept
 	{
-		Utils::burn(m_secret, sizeof(m_secret));
+		Utils::burn(const_cast<uint8_t *>(secret),ZT_SYMMETRIC_KEY_SIZE);
-		Utils::memoryUnlock(m_secret, sizeof(m_secret));
+		Utils::memoryUnlock(this,sizeof(SymmetricKey));
-	}
-
-	ZT_INLINE SymmetricKey &operator=(const SymmetricKey &k) noexcept
-	{
-		if (&k != this) {
-			cipher.init(k.m_secret);
-			m_ts = k.m_ts;
-			m_nonceBase = k.m_nonceBase;
-			m_odometer = k.m_odometer;
-			Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(m_secret, k.m_secret);
-		}
-		return *this;
-	}
-
-	/**
-	 * Initialize or change the key wrapped by this SymmetricKey object
-	 *
-	 * If the supplied key is identical to the current key, no change occurs and false is returned.
-	 *
-	 * @param ts Current time
-	 * @param key 32-byte / 256-bit key
-	 * @return True if the symmetric key was changed
-	 */
-	ZT_INLINE bool init(const int64_t ts,const void *const key) noexcept
-	{
-		if ((m_ts > 0) && (memcmp(m_secret, key, ZT_SYMMETRIC_KEY_SIZE) == 0))
-			return false;
-		cipher.init(key);
-		m_ts = ts;
-		m_nonceBase = (uint64_t)ts << 22U; // << 22 to shift approximately the seconds since epoch into the most significant 32 bits;
-		m_odometer = 0;
-		Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(m_secret, key);
-		return true;
-	}
-
-	/**
-	 * Clear key and set to NIL value (boolean evaluates to false)
-	 */
-	ZT_INLINE void clear() noexcept
-	{
-		m_ts = 0;
-		m_nonceBase = 0;
-		m_odometer = 0;
-		Utils::zero<ZT_SYMMETRIC_KEY_SIZE>(m_secret);
 	}
 
 	/**
@@ -140,7 +91,7 @@ public:
 	 */
 	ZT_INLINE bool expiringSoon(const int64_t now) const noexcept
 	{
-		return (TTL > 0) && (((now - m_ts) >= (TTL / 2)) || (m_odometer >= (TTLM / 2)) );
+		return (!m_permanent) && (((now - m_ts) >= (ZT_SYMMETRIC_KEY_TTL / 2)) || (m_odometer >= (ZT_SYMMETRIC_KEY_TTL_MESSAGES / 2)) );
 	}
 
 	/**
@@ -151,116 +102,27 @@ public:
 	 */
 	ZT_INLINE bool expired(const int64_t now) const noexcept
 	{
-		return (TTL > 0) && (((now - m_ts) >= TTL) || (m_odometer >= TTLM) );
+		return (!m_permanent) && (((now - m_ts) >= ZT_SYMMETRIC_KEY_TTL) || (m_odometer >= ZT_SYMMETRIC_KEY_TTL_MESSAGES) );
 	}
 
 	/**
-	 * @return True if this is a never-expiring key, such as the identity key created by identity key agreement
+	 * Advance usage counter by one and return the next IV / packet ID.
-	 */
-	constexpr bool permanent() const noexcept
-	{
-		return TTL == 0;
-	}
-
-	/**
-	 * Get the raw key that was used to initialize the cipher.
-	 *
-	 * @return 32-byte / 256-bit symmetric key
-	 */
-	ZT_INLINE const uint8_t *key() const noexcept
-	{
-		return m_secret;
-	}
-
-	/**
-	 * Advance usage counter by one and return the next unique initialization vector for a new message.
 	 *
+	 * @param sender Sending ZeroTier address
+	 * @param receiver Receiving ZeroTier address
 	 * @return Next unique IV for next message
 	 */
-	ZT_INLINE uint64_t nextMessageIv() noexcept
+	ZT_INLINE uint64_t nextMessage(const Address sender,const Address receiver) noexcept
 	{
-		return m_nonceBase + m_odometer++;
+		return (m_nonceBase + m_odometer++) ^ (((uint64_t)(sender > receiver)) << 63U);
-	}
-
-	/**
-	 * @return True if this object is not NIL
-	 */
-	ZT_INLINE operator bool() const noexcept { return (m_ts > 0); } // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
-
-	static constexpr int marshalSizeMax() noexcept { return ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX; }
-
-	/**
-	 * Marshal with encryption at rest
-	 *
-	 * @tparam MC Cipher type (AES in our code) to use for encryption at rest
-	 * @param keyEncCipher Initialized cipher
-	 * @param data Destination for marshaled key
-	 * @return Bytes written or -1 on error
-	 */
-	template<typename MC>
-	ZT_INLINE int marshal(const MC &keyEncCipher,uint8_t data[ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX]) const noexcept
-	{
-		Utils::storeBigEndian<uint64_t>(data,(uint64_t)m_ts);
-		Utils::storeBigEndian<uint64_t>(data + 8, m_odometer.load());
-		Utils::storeBigEndian<uint32_t>(data + 16,Utils::fnv1a32(m_secret, sizeof(m_secret)));
-
-		// Key encryption at rest is CBC using the last 32 bits of the timestamp, the odometer,
-		// and the FNV1a checksum as a 128-bit IV. A duplicate IV wouldn't matter much anyway since
-		// keys should be unique. Simple ECB would be fine as they also have no structure, but this
-		// looks better.
-		uint8_t tmp[16];
-		for(int i=0;i<16;++i)
-			tmp[i] = data[i + 4] ^ m_secret[i];
-		keyEncCipher.encrypt(tmp,data + 20);
-		for(int i=0;i<16;++i)
-			tmp[i] = data[i + 20] ^ m_secret[i + 16];
-		keyEncCipher.encrypt(tmp,data + 36);
-
-		return ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX;
-	}
-
-	/**
-	 * Unmarshal, decrypt, and verify key checksum
-	 *
-	 * Key checksum verification failure results in the SymmetricKey being zeroed out to its
-	 * nil value, but the bytes read are still returned. The caller must check this if it
-	 * requires the key to be present and verified.
-	 *
-	 * @tparam MC Cipher type (AES in our code) to use for encryption at rest
-	 * @param keyDecCipher Initialized cipher for decryption
-	 * @param data Source to read
-	 * @param len Bytes remaining at source
-	 * @return Bytes read from source
-	 */
-	template<typename MC>
-	ZT_INLINE int unmarshal(const MC &keyDecCipher,const uint8_t *restrict data,int len) noexcept
-	{
-		if (len < ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX)
-			return -1;
-
-		m_ts = (int64_t)Utils::loadBigEndian<uint64_t>(data);
-		m_odometer = (uint64_t)Utils::loadBigEndian<uint64_t>(data + 8);
-		const uint32_t fnv = Utils::loadBigEndian<uint32_t>(data + 16); // NOLINT(hicpp-use-auto,modernize-use-auto)
-
-		uint8_t tmp[16];
-		keyDecCipher.decrypt(data + 20,tmp);
-		for(int i=0;i<16;++i)
-			m_secret[i] = data[i + 4] ^ tmp[i];
-		keyDecCipher.decrypt(data + 36,tmp);
-		for(int i=0;i<16;++i)
-			m_secret[i + 16] = data[i + 20] ^ tmp[i];
-
-		if (Utils::fnv1a32(m_secret, sizeof(m_secret)) != fnv)
-			clear();
-
-		return ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX;
 	}
 
 private:
-	int64_t m_ts;
+	const int64_t m_ts;
-	uint64_t m_nonceBase;
+	const uint64_t m_nonceBase;
 	std::atomic<uint64_t> m_odometer;
-	uint8_t m_secret[ZT_SYMMETRIC_KEY_SIZE];
+	std::atomic<int> __refCount;
+	const bool m_permanent;
 };
 
 } // namespace ZeroTier

node/Tag.hpp

@@ -55,7 +55,7 @@ class Tag : public Credential
 public:
 	static constexpr ZT_CredentialType credentialType() noexcept { return ZT_CREDENTIAL_TYPE_TAG; }
 
-	ZT_INLINE Tag() noexcept { memoryZero(this); } // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	ZT_INLINE Tag() noexcept { memoryZero(this); }
 
 	/**
 	 * @param nwid Network ID
@@ -64,7 +64,7 @@ public:
 	 * @param id Tag ID
 	 * @param value Tag value
 	 */
-	ZT_INLINE Tag(const uint64_t nwid,const int64_t ts,const Address &issuedTo,const uint32_t id,const uint32_t value) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	ZT_INLINE Tag(const uint64_t nwid,const int64_t ts,const Address &issuedTo,const uint32_t id,const uint32_t value) noexcept :
 		m_id(id),
 		m_value(value),
 		m_networkId(nwid),

node/Tests.cpp

@@ -48,7 +48,7 @@
 
 #ifdef __UNIX_LIKE__
 #include <unistd.h>
-#include <sys/time.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
+#include <sys/time.h>
 #include <sys/types.h>
 #endif
 

node/Tests.h

@@ -43,8 +43,8 @@
 
 #ifdef ZT_ENABLE_TESTS
 
-#include <stdint.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
+#include <stdint.h>
-#include <stdio.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
+#include <stdio.h>
 
 #ifndef ZT_T_PRINTF
 #define ZT_T_PRINTF(fmt,...) printf((fmt),##__VA_ARGS__),fflush(stdout)

node/Topology.hpp

@@ -160,7 +160,7 @@ public:
 	ZT_INLINE void eachPeer(F f) const
 	{
 		RWMutex::RLock l(m_peers_l);
-		for(Map< Address,SharedPtr<Peer> >::const_iterator i(m_peers.begin());i != m_peers.end();++i) // NOLINT(modernize-loop-convert,hicpp-use-auto,modernize-use-auto)
+		for(Map< Address,SharedPtr<Peer> >::const_iterator i(m_peers.begin());i != m_peers.end();++i)
 			f(i->second);
 	}
 
@@ -178,14 +178,14 @@ public:
 	{
 		RWMutex::RLock l(m_peers_l);
 
-		std::vector<uintptr_t> rootPeerPtrs;
+		Vector<uintptr_t> rootPeerPtrs;
 		rootPeerPtrs.reserve(m_rootPeers.size());
-		for(std::vector< SharedPtr<Peer> >::const_iterator rp(m_rootPeers.begin());rp != m_rootPeers.end();++rp) // NOLINT(modernize-loop-convert,hicpp-use-auto,modernize-use-auto)
+		for(Vector< SharedPtr<Peer> >::const_iterator rp(m_rootPeers.begin());rp != m_rootPeers.end();++rp)
 			rootPeerPtrs.push_back((uintptr_t)rp->ptr());
 		std::sort(rootPeerPtrs.begin(),rootPeerPtrs.end());
 
 		try {
-			for(Map< Address,SharedPtr<Peer> >::const_iterator i(m_peers.begin());i != m_peers.end();++i) // NOLINT(modernize-loop-convert,hicpp-use-auto,modernize-use-auto)
+			for(Map< Address,SharedPtr<Peer> >::const_iterator i(m_peers.begin());i != m_peers.end();++i)
 				f(i->second,std::binary_search(rootPeerPtrs.begin(),rootPeerPtrs.end(),(uintptr_t)i->second.ptr()));
 		} catch ( ... ) {} // should not throw
 	}

node/Trace.cpp

@@ -17,6 +17,7 @@
 #include "Peer.hpp"
 #include "Path.hpp"
 #include "InetAddress.hpp"
+#include "FCV.hpp"
 
 // NOTE: packet IDs are always handled in network byte order, so no need to convert them.
 
@@ -34,16 +35,12 @@ void Trace::unexpectedError(
 	const char *message,
 	...)
 {
-	va_list ap;
+	FCV<uint8_t,4096> buf;
-	ZT_TraceEvent_UNEXPECTED_ERROR ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_UNEXPECTED_ERROR);
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_UNEXPECTED_ERROR);
+	Dictionary::append(buf,ZT_TRACE_FIELD_MESSAGE,message);
-	ev.codeLocation = codeLocation;
+	buf.push_back(0);
-	Utils::zero<sizeof(ev.message)>(ev.message);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
-	va_start(ap,message);
-	vsnprintf(ev.message,sizeof(ev.message),message,ap);
-	va_end(ap);
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
 }
 
 void Trace::_resettingPathsInScope(
@@ -55,15 +52,18 @@ void Trace::_resettingPathsInScope(
 	const InetAddress &newExternal,
 	const InetAddress::IpScope scope)
 {
-	ZT_TraceEvent_VL1_RESETTING_PATHS_IN_SCOPE ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	FCV<uint8_t,4096> buf;
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL1_RESETTING_PATHS_IN_SCOPE);
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL1_RESETTING_PATHS_IN_SCOPE);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
-	ev.codeLocation = Utils::hton(codeLocation);
+	if (from)
-	from.forTrace(ev.from);
+		Dictionary::appendObject(buf,ZT_TRACE_FIELD_ENDPOINT,Endpoint(from));
-	oldExternal.forTrace(ev.oldExternal);
+	if (oldExternal)
-	newExternal.forTrace(ev.newExternal);
+		Dictionary::appendObject(buf,ZT_TRACE_FIELD_OLD_ENDPOINT,Endpoint(oldExternal));
-	ev.scope = (uint8_t)scope;
+	if (newExternal)
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+		Dictionary::appendObject(buf,ZT_TRACE_FIELD_NEW_ENDPOINT,Endpoint(newExternal));
+	Dictionary::append(buf,ZT_TRACE_FIELD_RESET_ADDRESS_SCOPE,scope);
+	buf.push_back(0);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 void Trace::_tryingNewPath(
@@ -74,21 +74,20 @@ void Trace::_tryingNewPath(
 	const InetAddress &triggerAddress,
 	const uint64_t triggeringPacketId,
 	const uint8_t triggeringPacketVerb,
-	const Identity &triggeringPeer,
+	const Identity &triggeringPeer)
-	const ZT_TraceTryingNewPathReason reason)
 {
-	ZT_TraceEvent_VL1_TRYING_NEW_PATH ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	FCV<uint8_t,4096> buf;
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL1_TRYING_NEW_PATH);
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL1_TRYING_NEW_PATH);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
-	ev.codeLocation = Utils::hton(codeLocation);
+	Dictionary::append(buf,ZT_TRACE_FIELD_IDENTITY_FINGERPRINT_HASH,trying.fingerprint().hash(),ZT_FINGERPRINT_HASH_SIZE);
-	Utils::copy<sizeof(ev.peer)>(&ev.peer,trying.fingerprint().apiFingerprint());
+	if (triggerAddress)
-	physicalAddress.forTrace(ev.physicalAddress);
+		Dictionary::appendObject(buf,ZT_TRACE_FIELD_TRIGGER_FROM_ENDPOINT,Endpoint(triggerAddress));
-	triggerAddress.forTrace(ev.triggerAddress);
+	Dictionary::appendPacketId(buf,ZT_TRACE_FIELD_TRIGGER_FROM_PACKET_ID,triggeringPacketId);
-	ev.triggeringPacketId = triggeringPacketId;
+	Dictionary::append(buf,ZT_TRACE_FIELD_TRIGGER_FROM_PACKET_VERB,triggeringPacketVerb);
-	ev.triggeringPacketVerb = triggeringPacketVerb;
+	if (triggeringPeer)
-	Utils::copy<sizeof(ev.triggeringPeer)>(&ev.triggeringPeer,triggeringPeer.fingerprint().apiFingerprint());
+		Dictionary::append(buf,ZT_TRACE_FIELD_TRIGGER_FROM_PEER_FINGERPRINT_HASH,triggeringPeer.fingerprint().hash(),ZT_FINGERPRINT_HASH_SIZE);
-	ev.reason = (uint8_t)reason;
+	buf.push_back(0);
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 void Trace::_learnedNewPath(
@@ -99,16 +98,17 @@ void Trace::_learnedNewPath(
 	const InetAddress &physicalAddress,
 	const InetAddress &replaced)
 {
-	ZT_TraceEvent_VL1_LEARNED_NEW_PATH ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	FCV<uint8_t,4096> buf;
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL1_LEARNED_NEW_PATH);
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL1_LEARNED_NEW_PATH);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
-	ev.codeLocation = Utils::hton(codeLocation);
+	Dictionary::appendPacketId(buf,ZT_TRACE_FIELD_PACKET_ID,packetId);
-	ev.packetId = packetId; // packet IDs are kept in big-endian
+	Dictionary::append(buf,ZT_TRACE_FIELD_IDENTITY_FINGERPRINT_HASH,peerIdentity.fingerprint().hash(),ZT_FINGERPRINT_HASH_SIZE);
-	Utils::copy<sizeof(ev.peer)>(&ev.peer,peerIdentity.fingerprint().apiFingerprint());
+	if (physicalAddress)
-	physicalAddress.forTrace(ev.physicalAddress);
+		Dictionary::appendObject(buf,ZT_TRACE_FIELD_ENDPOINT,Endpoint(physicalAddress));
-	replaced.forTrace(ev.replaced);
+	if (replaced)
-
+		Dictionary::appendObject(buf,ZT_TRACE_FIELD_OLD_ENDPOINT,Endpoint(replaced));
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+	buf.push_back(0);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 void Trace::_incomingPacketDropped(
@@ -122,19 +122,19 @@ void Trace::_incomingPacketDropped(
 	const uint8_t verb,
 	const ZT_TracePacketDropReason reason)
 {
-	ZT_TraceEvent_VL1_INCOMING_PACKET_DROPPED ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	FCV<uint8_t,4096> buf;
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL1_INCOMING_PACKET_DROPPED);
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL1_INCOMING_PACKET_DROPPED);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
-	ev.codeLocation = Utils::hton(codeLocation);
+	Dictionary::appendPacketId(buf,ZT_TRACE_FIELD_PACKET_ID,packetId);
-	ev.packetId = packetId; // packet IDs are kept in big-endian
+	Dictionary::append(buf,ZT_TRACE_FIELD_NETWORK_ID,networkId);
-	ev.networkId = Utils::hton(networkId);
+	Dictionary::append(buf,ZT_TRACE_FIELD_IDENTITY_FINGERPRINT_HASH,peerIdentity.fingerprint().hash(),ZT_FINGERPRINT_HASH_SIZE);
-	Utils::copy<sizeof(ev.peer)>(&ev.peer,peerIdentity.fingerprint().apiFingerprint());
+	if (physicalAddress)
-	physicalAddress.forTrace(ev.physicalAddress);
+		Dictionary::append(buf,ZT_TRACE_FIELD_ENDPOINT,Endpoint(physicalAddress));
-	ev.hops = hops;
+	Dictionary::append(buf,ZT_TRACE_FIELD_PACKET_HOPS,hops);
-	ev.verb = verb;
+	Dictionary::append(buf,ZT_TRACE_FIELD_PACKET_VERB,verb);
-	ev.reason = (uint8_t)reason;
+	Dictionary::append(buf,ZT_TRACE_FIELD_REASON,reason);
-
+	buf.push_back(0);
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 void Trace::_outgoingNetworkFrameDropped(
@@ -148,25 +148,18 @@ void Trace::_outgoingNetworkFrameDropped(
 	const uint8_t *frameData,
 	const ZT_TraceFrameDropReason reason)
 {
-	ZT_TraceEvent_VL2_OUTGOING_FRAME_DROPPED ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	FCV<uint8_t,4096> buf;
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL1_INCOMING_PACKET_DROPPED);
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_OUTGOING_FRAME_DROPPED);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
-	ev.codeLocation = Utils::hton(codeLocation);
+	Dictionary::append(buf,ZT_TRACE_FIELD_SOURCE_MAC,sourceMac.toInt());
-	ev.networkId = Utils::hton(networkId);
+	Dictionary::append(buf,ZT_TRACE_FIELD_DEST_MAC,destMac.toInt());
-	ev.sourceMac = Utils::hton(sourceMac.toInt());
+	Dictionary::append(buf,ZT_TRACE_FIELD_ETHERTYPE,etherType);
-	ev.destMac = Utils::hton(destMac.toInt());
+	Dictionary::append(buf,ZT_TRACE_FIELD_FRAME_LENGTH,frameLength);
-	ev.etherType = Utils::hton(etherType);
+	if (frameData)
-	ev.frameLength = Utils::hton(frameLength);
+		Dictionary::append(buf,ZT_TRACE_FIELD_FRAME_DATA,frameData,std::min((unsigned int)64,(unsigned int)frameLength));
-	if (frameData) {
+	Dictionary::append(buf,ZT_TRACE_FIELD_REASON,reason);
-		unsigned int l = frameLength;
+	buf.push_back(0);
-		if (l > sizeof(ev.frameHead))
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
-			l = sizeof(ev.frameHead);
-		Utils::copy(ev.frameHead,frameData,l);
-		Utils::zero(ev.frameHead + l,sizeof(ev.frameHead) - l);
-	}
-	ev.reason = (uint8_t)reason;
-
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
 }
 
 void Trace::_incomingNetworkFrameDropped(
@@ -184,29 +177,23 @@ void Trace::_incomingNetworkFrameDropped(
 	const bool credentialRequestSent,
 	const ZT_TraceFrameDropReason reason)
 {
-	ZT_TraceEvent_VL2_INCOMING_FRAME_DROPPED ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	FCV<uint8_t,4096> buf;
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL2_INCOMING_FRAME_DROPPED);
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_INCOMING_FRAME_DROPPED);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
-	ev.codeLocation = Utils::hton(codeLocation);
+	Dictionary::append(buf,ZT_TRACE_FIELD_SOURCE_MAC,sourceMac.toInt());
-	ev.networkId = Utils::hton(networkId);
+	Dictionary::append(buf,ZT_TRACE_FIELD_DEST_MAC,destMac.toInt());
-	ev.sourceMac = Utils::hton(sourceMac.toInt());
+	Dictionary::append(buf,ZT_TRACE_FIELD_IDENTITY_FINGERPRINT_HASH,peerIdentity.fingerprint().hash(),ZT_FINGERPRINT_HASH_SIZE);
-	ev.destMac = Utils::hton(destMac.toInt());
+	if (physicalAddress)
-	Utils::copy<sizeof(ev.sender)>(&ev.sender,peerIdentity.fingerprint().apiFingerprint());
+		Dictionary::appendObject(buf,ZT_TRACE_FIELD_ENDPOINT,Endpoint(physicalAddress));
-	physicalAddress.forTrace(ev.physicalAddress);
+	Dictionary::append(buf,ZT_TRACE_FIELD_PACKET_HOPS,hops);
-	ev.hops = hops;
+	Dictionary::append(buf,ZT_TRACE_FIELD_PACKET_VERB,verb);
-	ev.frameLength = Utils::hton(frameLength);
+	Dictionary::append(buf,ZT_TRACE_FIELD_FRAME_LENGTH,frameLength);
-	if (frameData) {
+	if (frameData)
-		unsigned int l = frameLength;
+		Dictionary::append(buf,ZT_TRACE_FIELD_FRAME_DATA,frameData,std::min((unsigned int)64,(unsigned int)frameLength));
-		if (l > sizeof(ev.frameHead))
+	Dictionary::append(buf,ZT_TRACE_FIELD_FLAG_CREDENTIAL_REQUEST_SENT,credentialRequestSent);
-			l = sizeof(ev.frameHead);
+	Dictionary::append(buf,ZT_TRACE_FIELD_REASON,reason);
-		Utils::copy(ev.frameHead,frameData,l);
+	buf.push_back(0);
-		Utils::zero(ev.frameHead + l,sizeof(ev.frameHead) - l);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
-	}
-	ev.verb = verb;
-	ev.credentialRequestSent = (uint8_t)credentialRequestSent;
-	ev.reason = (uint8_t)reason;
-
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
 }
 
 void Trace::_networkConfigRequestSent(
@@ -214,12 +201,12 @@ void Trace::_networkConfigRequestSent(
 	const uint32_t codeLocation,
 	const uint64_t networkId)
 {
-	ZT_TraceEvent_VL2_NETWORK_CONFIG_REQUESTED ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	FCV<uint8_t,4096> buf;
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL2_NETWORK_CONFIG_REQUESTED);
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_NETWORK_CONFIG_REQUESTED);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
-	ev.codeLocation = Utils::hton(codeLocation);
+	Dictionary::append(buf,ZT_TRACE_FIELD_NETWORK_ID,networkId);
-	ev.networkId = Utils::hton(networkId);
+	buf.push_back(0);
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 void Trace::_networkFilter(
@@ -242,64 +229,53 @@ void Trace::_networkFilter(
 	const bool inbound,
 	const int accept)
 {
-	ZT_TraceEvent_VL2_NETWORK_FILTER ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	FCV<uint8_t,4096> buf;
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL2_NETWORK_FILTER);
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_NETWORK_FILTER);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
-	ev.codeLocation = Utils::hton(codeLocation);
+	Dictionary::append(buf,ZT_TRACE_FIELD_NETWORK_ID,networkId);
-	ev.networkId = Utils::hton(networkId);
+	if ((primaryRuleSetLog)&&(!Utils::allZero(primaryRuleSetLog,512)))
-	Utils::copy<sizeof(ev.primaryRuleSetLog)>(ev.primaryRuleSetLog,primaryRuleSetLog);
+		Dictionary::append(buf,ZT_TRACE_FIELD_PRIMARY_RULE_SET_LOG,primaryRuleSetLog,512);
-	if (matchingCapabilityRuleSetLog)
+	if ((matchingCapabilityRuleSetLog)&&(!Utils::allZero(matchingCapabilityRuleSetLog,512)))
-		Utils::copy<sizeof(ev.matchingCapabilityRuleSetLog)>(ev.matchingCapabilityRuleSetLog,matchingCapabilityRuleSetLog);
+		Dictionary::append(buf,ZT_TRACE_FIELD_MATCHING_CAPABILITY_RULE_SET_LOG,matchingCapabilityRuleSetLog,512);
-	else Utils::zero<sizeof(ev.matchingCapabilityRuleSetLog)>(ev.matchingCapabilityRuleSetLog);
+	Dictionary::append(buf,ZT_TRACE_FIELD_MATCHING_CAPABILITY_ID,matchingCapabilityId);
-	ev.matchingCapabilityId = Utils::hton(matchingCapabilityId);
+	Dictionary::append(buf,ZT_TRACE_FIELD_MATCHING_CAPABILITY_TIMESTAMP,matchingCapabilityTimestamp);
-	ev.matchingCapabilityTimestamp = Utils::hton(matchingCapabilityTimestamp);
+	Dictionary::append(buf,ZT_TRACE_FIELD_SOURCE_ZT_ADDRESS,source);
-	ev.source = Utils::hton(source.toInt());
+	Dictionary::append(buf,ZT_TRACE_FIELD_DEST_ZT_ADDRESS,dest);
-	ev.dest = Utils::hton(dest.toInt());
+	Dictionary::append(buf,ZT_TRACE_FIELD_SOURCE_MAC,sourceMac.toInt());
-	ev.sourceMac = Utils::hton(sourceMac.toInt());
+	Dictionary::append(buf,ZT_TRACE_FIELD_DEST_MAC,destMac.toInt());
-	ev.destMac = Utils::hton(destMac.toInt());
+	Dictionary::append(buf,ZT_TRACE_FIELD_FRAME_LENGTH,frameLength);
-	ev.frameLength = Utils::hton(frameLength);
+	if (frameData)
-	if (frameData) {
+		Dictionary::append(buf,ZT_TRACE_FIELD_FRAME_DATA,frameData,std::min((unsigned int)64,(unsigned int)frameLength));
-		unsigned int l = frameLength;
+	Dictionary::append(buf,ZT_TRACE_FIELD_ETHERTYPE,etherType);
-		if (l > sizeof(ev.frameHead))
+	Dictionary::append(buf,ZT_TRACE_FIELD_VLAN_ID,vlanId);
-			l = sizeof(ev.frameHead);
+	Dictionary::append(buf,ZT_TRACE_FIELD_RULE_FLAG_NOTEE,noTee);
-		Utils::copy(ev.frameHead,frameData,l);
+	Dictionary::append(buf,ZT_TRACE_FIELD_RULE_FLAG_INBOUND,inbound);
-		Utils::zero(ev.frameHead + l,sizeof(ev.frameHead) - l);
+	Dictionary::append(buf,ZT_TRACE_FIELD_RULE_FLAG_ACCEPT,(int32_t)accept);
-	}
+	buf.push_back(0);
-	ev.etherType = Utils::hton(etherType);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
-	ev.vlanId = Utils::hton(vlanId);
-	ev.noTee = (uint8_t)noTee;
-	ev.inbound = (uint8_t)inbound;
-	ev.accept = (int8_t)accept;
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
 }
 
 void Trace::_credentialRejected(
 	void *const tPtr,
 	const uint32_t codeLocation,
 	const uint64_t networkId,
-	const Address &address,
 	const Identity &identity,
 	const uint32_t credentialId,
 	const int64_t credentialTimestamp,
 	const uint8_t credentialType,
 	const ZT_TraceCredentialRejectionReason reason)
 {
-	ZT_TraceEvent_VL2_CREDENTIAL_REJECTED ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	FCV<uint8_t,4096> buf;
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL2_NETWORK_FILTER);
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_NETWORK_FILTER);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
-	ev.codeLocation = Utils::hton(codeLocation);
+	Dictionary::append(buf,ZT_TRACE_FIELD_NETWORK_ID,networkId);
-	ev.networkId = Utils::hton(networkId);
+	Dictionary::append(buf,ZT_TRACE_FIELD_IDENTITY_FINGERPRINT_HASH,identity.fingerprint().hash(),ZT_FINGERPRINT_HASH_SIZE);
-	if (identity) {
+	Dictionary::append(buf,ZT_TRACE_FIELD_CREDENTIAL_ID,credentialId);
-		Utils::copy<sizeof(ev.peer)>(&ev.peer,identity.fingerprint().apiFingerprint());
+	Dictionary::append(buf,ZT_TRACE_FIELD_CREDENTIAL_TIMESTAMP,credentialTimestamp);
-	} else {
+	Dictionary::append(buf,ZT_TRACE_FIELD_CREDENTIAL_TYPE,credentialType);
-		ev.peer.address = address.toInt();
+	Dictionary::append(buf,ZT_TRACE_FIELD_REASON,reason);
-		Utils::zero<sizeof(ev.peer.hash)>(ev.peer.hash);
+	buf.push_back(0);
-	}
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
-	ev.credentialId = Utils::hton(credentialId);
-	ev.credentialTimestamp = Utils::hton(credentialTimestamp);
-	ev.credentialType = credentialType;
-	ev.reason = (uint8_t)reason;
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
 }
 
 } // namespace ZeroTier

node/Trace.hpp

@@ -104,11 +104,10 @@ public:
 		const InetAddress &triggerAddress,
 		uint64_t triggeringPacketId,
 		uint8_t triggeringPacketVerb,
-		const Identity &triggeringPeer,
+		const Identity &triggeringPeer)
-		ZT_TraceTryingNewPathReason reason)
 	{
 		if ((_f & ZT_TRACE_F_VL1) != 0)
-			_tryingNewPath(tPtr,codeLocation,trying,physicalAddress,triggerAddress,triggeringPacketId,triggeringPacketVerb,triggeringPeer,reason);
+			_tryingNewPath(tPtr,codeLocation,trying,physicalAddress,triggerAddress,triggeringPacketId,triggeringPacketVerb,triggeringPeer);
 	}
 
 	ZT_INLINE void learnedNewPath(
@@ -228,7 +227,6 @@ public:
 		void *const tPtr,
 		const uint32_t codeLocation,
 		uint64_t networkId,
-		const Address &address,
 		const Identity &identity,
 		uint32_t credentialId,
 		int64_t credentialTimestamp,
@@ -236,7 +234,7 @@ public:
 		ZT_TraceCredentialRejectionReason reason)
 	{
 		if ((_f & ZT_TRACE_F_VL2) != 0)
-			_credentialRejected(tPtr,codeLocation,networkId,address,identity,credentialId,credentialTimestamp,credentialType,reason);
+			_credentialRejected(tPtr,codeLocation,networkId,identity,credentialId,credentialTimestamp,credentialType,reason);
 	}
 
 private:
@@ -256,8 +254,7 @@ private:
 		const InetAddress &triggerAddress,
 		uint64_t triggeringPacketId,
 		uint8_t triggeringPacketVerb,
-		const Identity &triggeringPeer,
+		const Identity &triggeringPeer);
-		ZT_TraceTryingNewPathReason reason);
 	void _learnedNewPath(
 		void *tPtr,
 		uint32_t codeLocation,
@@ -326,7 +323,6 @@ private:
 		void *tPtr,
 		uint32_t codeLocation,
 		uint64_t networkId,
-		const Address &address,
 		const Identity &identity,
 		uint32_t credentialId,
 		int64_t credentialTimestamp,

node/Utils.cpp

@@ -36,7 +36,7 @@ namespace ZeroTier {
 namespace Utils {
 
 #ifdef ZT_ARCH_X64
-CPUIDRegisters::CPUIDRegisters() noexcept // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+CPUIDRegisters::CPUIDRegisters() noexcept
 {
 #ifdef __WINDOWS__
 	int regs[4];
@@ -100,58 +100,21 @@ char *decimal(unsigned long n,char s[24]) noexcept
 	return s;
 }
 
-char *hex(uint8_t i,char s[3]) noexcept
+char *hex(uint64_t i,char buf[17]) noexcept
 {
-	s[0] = HEXCHARS[(i >> 4U) & 0xfU];
+	if (i) {
-	s[1] = HEXCHARS[i & 0xfU];
+		char *p = buf + 16;
-	s[2] = 0;
+		*p = 0;
-	return s;
+		while (i) {
-}
+			*(--p) = HEXCHARS[i & 0xfU];
-
+			i >>= 4;
-char *hex(uint16_t i,char s[5]) noexcept
+		}
-{
+		return p;
-	s[0] = HEXCHARS[(i >> 12U) & 0xfU];
+	} else {
-	s[1] = HEXCHARS[(i >> 8U) & 0xfU];
+		buf[0] = '0';
-	s[2] = HEXCHARS[(i >> 4U) & 0xfU];
+		buf[1] = 0;
-	s[3] = HEXCHARS[i & 0xfU];
+		return buf;
-	s[4] = 0;
+	}
-	return s;
-}
-
-char *hex(uint32_t i,char s[9]) noexcept
-{
-	s[0] = HEXCHARS[(i >> 28U) & 0xfU];
-	s[1] = HEXCHARS[(i >> 24U) & 0xfU];
-	s[2] = HEXCHARS[(i >> 20U) & 0xfU];
-	s[3] = HEXCHARS[(i >> 16U) & 0xfU];
-	s[4] = HEXCHARS[(i >> 12U) & 0xfU];
-	s[5] = HEXCHARS[(i >> 8U) & 0xfU];
-	s[6] = HEXCHARS[(i >> 4U) & 0xfU];
-	s[7] = HEXCHARS[i & 0xfU];
-	s[8] = 0;
-	return s;
-}
-
-char *hex(uint64_t i,char s[17]) noexcept
-{
-	s[0] = HEXCHARS[(i >> 60U) & 0xfU];
-	s[1] = HEXCHARS[(i >> 56U) & 0xfU];
-	s[2] = HEXCHARS[(i >> 52U) & 0xfU];
-	s[3] = HEXCHARS[(i >> 48U) & 0xfU];
-	s[4] = HEXCHARS[(i >> 44U) & 0xfU];
-	s[5] = HEXCHARS[(i >> 40U) & 0xfU];
-	s[6] = HEXCHARS[(i >> 36U) & 0xfU];
-	s[7] = HEXCHARS[(i >> 32U) & 0xfU];
-	s[8] = HEXCHARS[(i >> 28U) & 0xfU];
-	s[9] = HEXCHARS[(i >> 24U) & 0xfU];
-	s[10] = HEXCHARS[(i >> 20U) & 0xfU];
-	s[11] = HEXCHARS[(i >> 16U) & 0xfU];
-	s[12] = HEXCHARS[(i >> 12U) & 0xfU];
-	s[13] = HEXCHARS[(i >> 8U) & 0xfU];
-	s[14] = HEXCHARS[(i >> 4U) & 0xfU];
-	s[15] = HEXCHARS[i & 0xfU];
-	s[16] = 0;
-	return s;
 }
 
 uint64_t unhex(const char *s) noexcept
@@ -165,11 +128,11 @@ uint64_t unhex(const char *s) noexcept
 
 			uint8_t c = 0;
 			if ((hc >= 48)&&(hc <= 57))
-				c = hc - 48;
+				c = (uint8_t)hc - 48;
 			else if ((hc >= 97)&&(hc <= 102))
-				c = hc - 87;
+				c = (uint8_t)hc - 87;
 			else if ((hc >= 65)&&(hc <= 70))
-				c = hc - 55;
+				c = (uint8_t)hc - 55;
 
 			n <<= 4U;
 			n |= (uint64_t)c;
@@ -294,7 +257,7 @@ void getSecureRandom(void *const buf,const unsigned int bytes) noexcept
 #ifdef ZT_ARCH_X64
 				if (CPUID.rdrand) {
 					uint64_t tmp = 0;
-					for(int k=0;k<ZT_GETSECURERANDOM_STATE_SIZE;++k) { // NOLINT(modernize-loop-convert)
+					for(int k=0;k<ZT_GETSECURERANDOM_STATE_SIZE;++k) {
 						_rdrand64_step((unsigned long long *)&tmp);
 						randomState[k] ^= tmp;
 					}

node/Utils.hpp

@@ -137,38 +137,14 @@ char *decimal(unsigned long n,char s[24]) noexcept;
 /**
  * Convert an unsigned integer into hex
  *
+ * The returned pointer won't point to the start of 'buf', since
+ * hex writing is done in reverse order.
+ * 
  * @param i Any unsigned integer
  * @param s Buffer to receive hex, must be at least (2*sizeof(i))+1 in size or overflow will occur.
  * @return Pointer to s containing hex string with trailing zero byte
  */
-char *hex(uint8_t i,char s[3]) noexcept;
+char *hex(uint64_t i,char buf[17]) noexcept;
-
-/**
- * Convert an unsigned integer into hex
- *
- * @param i Any unsigned integer
- * @param s Buffer to receive hex, must be at least (2*sizeof(i))+1 in size or overflow will occur.
- * @return Pointer to s containing hex string with trailing zero byte
- */
-char *hex(uint16_t i,char s[5]) noexcept;
-
-/**
- * Convert an unsigned integer into hex
- *
- * @param i Any unsigned integer
- * @param s Buffer to receive hex, must be at least (2*sizeof(i))+1 in size or overflow will occur.
- * @return Pointer to s containing hex string with trailing zero byte
- */
-char *hex(uint32_t i,char s[9]) noexcept;
-
-/**
- * Convert an unsigned integer into hex
- *
- * @param i Any unsigned integer
- * @param s Buffer to receive hex, must be at least (2*sizeof(i))+1 in size or overflow will occur.
- * @return Pointer to s containing hex string with trailing zero byte
- */
-char *hex(uint64_t i,char s[17]) noexcept;
 
 /**
  * Decode an unsigned integer in hex format
@@ -625,8 +601,8 @@ template<unsigned int L>
 static ZT_INLINE void copy(void *const dest,const void *const src) noexcept
 {
 #ifdef ZT_ARCH_X64
-	uint8_t *volatile d = reinterpret_cast<uint8_t *>(dest); // NOLINT(hicpp-use-auto,modernize-use-auto)
+	uint8_t *volatile d = reinterpret_cast<uint8_t *>(dest);
-	const uint8_t *s = reinterpret_cast<const uint8_t *>(src); // NOLINT(hicpp-use-auto,modernize-use-auto)
+	const uint8_t *s = reinterpret_cast<const uint8_t *>(src);
 	for(unsigned int i=0;i<(L >> 6U);++i) {
 		__m128i x0 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(s));
 		__m128i x1 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(s + 16));
@@ -698,7 +674,7 @@ template<unsigned int L>
 static ZT_INLINE void zero(void *const dest) noexcept
 {
 #ifdef ZT_ARCH_X64
-	uint8_t *volatile d = reinterpret_cast<uint8_t *>(dest); // NOLINT(hicpp-use-auto,modernize-use-auto)
+	uint8_t *volatile d = reinterpret_cast<uint8_t *>(dest);
 	__m128i z = _mm_setzero_si128();
 	for(unsigned int i=0;i<(L >> 6U);++i) {
 		_mm_storeu_si128(reinterpret_cast<__m128i *>(d),z);
@@ -764,16 +740,16 @@ struct Mallocator
 	typedef T value_type;
 
 	template <class U> struct rebind { typedef Mallocator<U> other; };
-	ZT_INLINE Mallocator() noexcept {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
+	ZT_INLINE Mallocator() noexcept {}
-	ZT_INLINE Mallocator(const Mallocator&) noexcept {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
+	ZT_INLINE Mallocator(const Mallocator&) noexcept {}
-	template <class U> ZT_INLINE Mallocator(const Mallocator<U>&) noexcept {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default,google-explicit-constructor,hicpp-explicit-conversions)
+	template <class U> ZT_INLINE Mallocator(const Mallocator<U>&) noexcept {}
-	ZT_INLINE ~Mallocator() noexcept {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
+	ZT_INLINE ~Mallocator() noexcept {}
 
 	ZT_INLINE pointer allocate(size_type s,void const * = nullptr)
 	{
 		if (0 == s)
 			return nullptr;
-		pointer temp = (pointer)malloc(s * sizeof(T)); // NOLINT(hicpp-use-auto,modernize-use-auto)
+		pointer temp = (pointer)malloc(s * sizeof(T));
 		if (temp == nullptr)
 			throw std::bad_alloc();
 		return temp;

node/VL1.hpp

@@ -23,6 +23,8 @@
 #include "FCV.hpp"
 #include "Containers.hpp"
 
+#define ZT_VL1_MAX_WHOIS_WAITING_PACKETS 32
+
 namespace ZeroTier {
 
 class RuntimeEnvironment;
@@ -66,7 +68,7 @@ private:
 	void m_sendPendingWhois(void *tPtr, int64_t now);
 
 	// Handlers for VL1 verbs -- for clarity's sake VL2 verbs are in the VL2 class.
-	bool m_HELLO(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize, bool authenticated);
+	SharedPtr<Peer> m_HELLO(void *tPtr, const SharedPtr<Path> &path, Buf &pkt, int packetSize);
 	bool m_ERROR(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize, Protocol::Verb &inReVerb);
 	bool m_OK(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize, Protocol::Verb &inReVerb);
 	bool m_WHOIS(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
@@ -76,16 +78,17 @@ private:
 	bool m_USER_MESSAGE(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
 	bool m_ENCAP(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
 
-	struct p_WhoisQueueItem
-	{
-		ZT_INLINE p_WhoisQueueItem() : lastRetry(0), inboundPackets(), retries(0) {}
-		int64_t lastRetry;
-		FCV<Buf::Slice,32> inboundPackets; // capacity can be changed but this should be plenty
-		unsigned int retries;
-	};
-
 	Defragmenter<ZT_MAX_PACKET_FRAGMENTS> m_inputPacketAssembler;
 
+	struct p_WhoisQueueItem
+	{
+		ZT_INLINE p_WhoisQueueItem() : lastRetry(0),retries(0),waitingPacketCount(0) {}
+		int64_t lastRetry;
+		unsigned int retries;
+		unsigned int waitingPacketCount;
+		unsigned int waitingPacketSize[ZT_VL1_MAX_WHOIS_WAITING_PACKETS];
+		SharedPtr<Buf> waitingPacket[ZT_VL1_MAX_WHOIS_WAITING_PACKETS];
+	};
 	Map<Address,p_WhoisQueueItem> m_whoisQueue;
 	Mutex m_whoisQueue_l;
 };