Add symmetric key container, tons of cleanup.

include/ZeroTierCore.h

@@ -416,10 +416,8 @@ enum ZT_TraceEventPathAddressType
 {
 	ZT_TRACE_EVENT_PATH_TYPE_NIL =          0, /* none/empty */
 	ZT_TRACE_EVENT_PATH_TYPE_ZEROTIER =     1, /* 5-byte ZeroTier + 48-byte identity hash */
-	ZT_TRACE_EVENT_PATH_TYPE_DNSNAME =      2, /* C string */
+	ZT_TRACE_EVENT_PATH_TYPE_ETHERNET =     2, /* 6-byte Ethernet */
-	ZT_TRACE_EVENT_PATH_TYPE_URL =          3, /* C string */
 	ZT_TRACE_EVENT_PATH_TYPE_INETADDR_V4 =  4, /* 4-byte IPv4 */
-	ZT_TRACE_EVENT_PATH_TYPE_ETHERNET =     5, /* 6-byte Ethernet */
 	ZT_TRACE_EVENT_PATH_TYPE_INETADDR_V6 =  6  /* 16-byte IPv6 */
 };
 
@@ -1392,13 +1390,18 @@ typedef struct
 	int root;
 
 	/**
-	 * Bootstrap address
+	 * Number of bootstrap addresses
+	 */
+	unsigned int bootstrapAddressCount;
+
+	/**
+	 * Bootstrap addresses
 	 *
 	 * This is a memo-ized recently valid address that can be saved and used
 	 * to attempt rapid reconnection with this peer. If the ss_family field
 	 * is 0 this field is considered null/empty.
 	 */
-	struct sockaddr_storage bootstrap;
+	struct sockaddr_storage bootstrap[ZT_MAX_PEER_NETWORK_PATHS];
 
 	/**
 	 * Number of networks in which this peer is authenticated

node/AES.hpp

@@ -57,7 +57,7 @@ public:
 	/**
 	 * Create an un-initialized AES instance (must call init() before use)
 	 */
-	ZT_INLINE AES() noexcept
+	ZT_INLINE AES() noexcept // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
 	{
 		Utils::memoryLock(this,sizeof(AES));
 	}
@@ -67,7 +67,7 @@ public:
 	 *
 	 * @param key 256-bit key
 	 */
-	explicit ZT_INLINE AES(const void *const key) noexcept
+	explicit ZT_INLINE AES(const void *const key) noexcept // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
 	{
 		Utils::memoryLock(this,sizeof(AES));
 		this->init(key);
@@ -84,7 +84,7 @@ public:
 	 *
 	 * @param key 256-bit / 32-byte key
 	 */
-	ZT_INLINE void init(const void *key) noexcept
+	ZT_INLINE void init(const void *const key) noexcept
 	{
 #ifdef ZT_AES_AESNI
 		if (likely(Utils::CPUID.aes)) {
@@ -264,7 +264,7 @@ public:
 		 * @param k0 First of two AES instances keyed with K0
 		 * @param k1 Second of two AES instances keyed with K1
 		 */
-		ZT_INLINE GMACSIVEncryptor(const AES &k0,const AES &k1) noexcept :
+		ZT_INLINE GMACSIVEncryptor(const AES &k0,const AES &k1) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
 			_gmac(k0),
 			_ctr(k1) {}
 
@@ -383,7 +383,7 @@ public:
 	class GMACSIVDecryptor
 	{
 	public:
-		ZT_INLINE GMACSIVDecryptor(const AES &k0,const AES &k1) noexcept :
+		ZT_INLINE GMACSIVDecryptor(const AES &k0,const AES &k1) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
 			_ctr(k1),
 			_gmac(k0) {}
 

node/Address.hpp

@@ -104,6 +104,7 @@ public:
 	ZT_INLINE bool isReserved() const noexcept { return ((!_a) || ((_a >> 32U) == ZT_ADDRESS_RESERVED_PREFIX)); }
 
 	ZT_INLINE unsigned long hashCode() const noexcept { return (unsigned long)_a; }
+
 	ZT_INLINE operator bool() const noexcept { return (_a != 0); } // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
 
 	ZT_INLINE bool operator==(const Address &a) const noexcept { return _a == a._a; }

node/C25519.cpp

@@ -676,7 +676,8 @@ ZT_INLINE void crecip(limb *out,const limb *z) {
 	/* 2^255 - 21 */ fmul(out,t1,z11);
 }
 
-void crypto_scalarmult(u8 *mypublic, const u8 *secret, const u8 *basepoint) {
+void crypto_scalarmult(u8 *mypublic, const u8 *secret, const u8 *basepoint)
+{
 	limb bp[10], x[10], z[11], zmone[10];
 	uint8_t e[32];
 	int i;
@@ -2367,13 +2368,19 @@ ZT_INLINE void get_hram(unsigned char *hram,const unsigned char *sm,const unsign
 
 namespace ZeroTier {
 
-void C25519::generate(uint8_t pub[ZT_C25519_COMBINED_PUBLIC_KEY_SIZE],uint8_t priv[ZT_C25519_COMBINED_PRIVATE_KEY_SIZE])
+void C25519::generateCombined(uint8_t *pub,uint8_t *priv)
 {
 	Utils::getSecureRandom(priv,ZT_C25519_COMBINED_PRIVATE_KEY_SIZE);
 	_calcPubDH(pub,priv);
 	_calcPubED(pub,priv);
 }
 
+void C25519::generateC25519(uint8_t pub[ZT_C25519_ECDH_PUBLIC_KEY_SIZE],uint8_t priv[ZT_C25519_ECDH_PRIVATE_KEY_SIZE])
+{
+	Utils::getSecureRandom(priv,ZT_C25519_ECDH_PRIVATE_KEY_SIZE);
+	_calcPubDH(pub,priv);
+}
+
 void C25519::agree(const uint8_t mine[ZT_C25519_COMBINED_PRIVATE_KEY_SIZE],const uint8_t their[ZT_C25519_COMBINED_PUBLIC_KEY_SIZE],uint8_t rawkey[ZT_C25519_ECDH_SHARED_SECRET_SIZE])
 {
 	crypto_scalarmult(rawkey,mine,their);
@@ -2465,7 +2472,7 @@ bool C25519::verify(const uint8_t their[ZT_C25519_COMBINED_PUBLIC_KEY_SIZE],cons
 	return Utils::secureEq(sig,t2,32);
 }
 
-void C25519::_calcPubDH(uint8_t pub[ZT_C25519_COMBINED_PUBLIC_KEY_SIZE],const uint8_t priv[ZT_C25519_COMBINED_PRIVATE_KEY_SIZE])
+void C25519::_calcPubDH(uint8_t *const pub,const uint8_t *const priv)
 {
 	// First 32 bytes of pub and priv are the keys for ECDH key
 	// agreement. This generates the public portion from the private.

node/C25519.hpp

@@ -22,6 +22,8 @@
 
 namespace ZeroTier {
 
+#define ZT_C25519_ECDH_PUBLIC_KEY_SIZE 32
+#define ZT_C25519_ECDH_PRIVATE_KEY_SIZE 32
 #define ZT_C25519_COMBINED_PUBLIC_KEY_SIZE 64
 #define ZT_C25519_COMBINED_PRIVATE_KEY_SIZE 64
 #define ZT_C25519_SIGNATURE_LEN 96
@@ -34,9 +36,14 @@ class C25519
 {
 public:
 	/**
-	 * Generate a C25519 elliptic curve key pair
+	 * Generate a set of two 25519 keys: a C25519 ECDH key pair and an Ed25519 EDDSA key pair.
 	 */
-	static void generate(uint8_t pub[ZT_C25519_COMBINED_PUBLIC_KEY_SIZE],uint8_t priv[ZT_C25519_COMBINED_PRIVATE_KEY_SIZE]);
+	static void generateCombined(uint8_t pub[ZT_C25519_COMBINED_PUBLIC_KEY_SIZE],uint8_t priv[ZT_C25519_COMBINED_PRIVATE_KEY_SIZE]);
+
+	/**
+	 * Generate a C25519 ECDH key pair only.
+	 */
+	static void generateC25519(uint8_t pub[ZT_C25519_ECDH_PUBLIC_KEY_SIZE],uint8_t priv[ZT_C25519_ECDH_PRIVATE_KEY_SIZE]);
 
 	/**
 	 * Generate a key pair satisfying a condition
@@ -109,7 +116,7 @@ public:
 private:
 	// derive first 32 bytes of kp.pub from first 32 bytes of kp.priv
 	// this is the ECDH key
-	static void _calcPubDH(uint8_t pub[ZT_C25519_COMBINED_PUBLIC_KEY_SIZE],const uint8_t priv[ZT_C25519_COMBINED_PRIVATE_KEY_SIZE]);
+	static void _calcPubDH(uint8_t *pub,const uint8_t *priv);
 
 	// derive 2nd 32 bytes of kp.pub from 2nd 32 bytes of kp.priv
 	// this is the Ed25519 sign/verify key

node/CMakeLists.txt

@@ -10,6 +10,7 @@ set(core_headers
 	CertificateOfMembership.hpp
 	CertificateOfOwnership.hpp
 	Constants.hpp
+	Containers.hpp
 	Credential.hpp
 	Defragmenter.hpp
 	Dictionary.hpp
@@ -22,7 +23,6 @@ set(core_headers
 	Locator.hpp
 	LZ4.hpp
 	MAC.hpp
-	Map.hpp
 	Membership.hpp
 	MulticastGroup.hpp
 	Mutex.hpp
@@ -41,6 +41,7 @@ set(core_headers
 	SHA512.hpp
 	SharedPtr.hpp
 	Speck128.hpp
+	SymmetricKey.hpp
 	Tag.hpp
 	Topology.hpp
 	Trace.hpp

node/Capability.hpp

@@ -55,6 +55,8 @@ class Capability : public Credential
 	friend class Credential;
 
 public:
+	static constexpr ZT_CredentialType credentialType() noexcept { return ZT_CREDENTIAL_TYPE_CAPABILITY; }
+
 	ZT_INLINE Capability() noexcept { memoryZero(this); } // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
 
 	/**

node/CertificateOfMembership.hpp

@@ -106,6 +106,8 @@ class CertificateOfMembership : public Credential
 	friend class Credential;
 
 public:
+	static constexpr ZT_CredentialType credentialType() noexcept { return ZT_CREDENTIAL_TYPE_COM; }
+
 	/**
 	 * Create an empty certificate of membership
 	 */

node/CertificateOfOwnership.hpp

@@ -50,6 +50,8 @@ class CertificateOfOwnership : public Credential
 	friend class Credential;
 
 public:
+	static constexpr ZT_CredentialType credentialType() noexcept { return ZT_CREDENTIAL_TYPE_COO; }
+
 	enum Thing
 	{
 		THING_NULL = 0,

node/Constants.hpp

@@ -26,6 +26,11 @@
 #define ZEROTIER_VERSION_BUILD 255
 #endif
 
+/**
+ * Version bit packed into four 16-bit fields in a 64-bit unsigned integer.
+ */
+#define ZT_VERSION_PACKED ( ((uint64_t)ZEROTIER_VERSION_MAJOR << 48U) | ((uint64_t)ZEROTIER_VERSION_MINOR << 32U) | ((uint64_t)ZEROTIER_VERSION_REVISION << 16U) | (uint64_t)ZEROTIER_VERSION_BUILD )
+
 /**
  * Length of a ZeroTier address in bytes
  */
@@ -42,14 +47,9 @@
 #define ZT_ADDRESS_MASK 0xffffffffffULL
 
 /**
- * Maximum DNS or URL name size for an Endpoint (set so that max marshaled endpoint size is 64 bytes)
+ * Size of an identity fingerprint hash (SHA384) in bytes
  */
-#define ZT_ENDPOINT_MAX_NAME_SIZE 61
+#define ZT_FINGERPRINT_HASH_SIZE 48
-
-/**
- * Size of an identity hash (SHA384) in bytes
- */
-#define ZT_IDENTITY_HASH_SIZE 48
 
 /**
  * Default virtual network MTU (not physical)
@@ -64,7 +64,7 @@
 /**
  * Anti-DOS limit on the maximum incoming fragments per path
  */
-#define ZT_MAX_INCOMING_FRAGMENTS_PER_PATH 32
+#define ZT_MAX_INCOMING_FRAGMENTS_PER_PATH 16
 
 /**
  * Sanity limit on the maximum size of a network config object
@@ -72,9 +72,19 @@
 #define ZT_MAX_NETWORK_CONFIG_BYTES 131072
 
 /**
- * Length of peer shared secrets (256-bit, do not change)
+ * Length of symmetric keys (currently all symmetric crypto is 256 bit).
  */
-#define ZT_PEER_SECRET_KEY_LENGTH 32
+#define ZT_SYMMETRIC_KEY_SIZE 32
+
+/**
+ * Time limit for ephemeral keys: 30 minutes.
+ */
+#define ZT_SYMMETRIC_KEY_TTL 1800000
+
+/**
+ * Maximum number of messages over which a key should be considered usable.
+ */
+#define ZT_SYMMETRIC_KEY_TTL_MESSAGES 2147483648
 
 /**
  * Maximum delay between timer task checks

node/Containers.hpp

@@ -14,38 +14,33 @@
 #ifndef ZT_MAP_HPP
 #define ZT_MAP_HPP
 
-/*
+/* This defines a Map, SortedMap, Vector, etc. based on STL templates. */
- * This wraps std::unordered_map (or std::map if that is not available) and gives
- * it a few extra methods. It also uses the built-in hashCode methods in key objects
- * in ZeroTier instead of requiring hashers all over the place.
- */
 
 #include "Constants.hpp"
 #include "Utils.hpp"
 
 #ifdef __CPP11__
 #include <unordered_map>
-#else
-#include <map>
 #endif
+#include <map>
+#include <vector>
+#include <list>
+#include <set>
 
 namespace ZeroTier {
 
 #ifdef __CPP11__
-
 struct _MapHasher
 {
 	template<typename O>
-	std::size_t operator()(const O &obj) const noexcept { return (std::size_t)obj.hashCode() ^ (std::size_t)Utils::s_mapNonce; }
+	std::size_t operator()(const O &obj) const noexcept { return (std::size_t)obj.hashCode(); }
-
 	std::size_t operator()(const uint64_t i) const noexcept { return (std::size_t)Utils::hash64(i ^ Utils::s_mapNonce); }
 	std::size_t operator()(const int64_t i) const noexcept { return (std::size_t)Utils::hash64((uint64_t)i ^ Utils::s_mapNonce); }
 	std::size_t operator()(const uint32_t i) const noexcept { return (std::size_t)Utils::hash32(i ^ (uint32_t)Utils::s_mapNonce); }
 	std::size_t operator()(const int32_t i) const noexcept { return (std::size_t)Utils::hash32((uint32_t)i ^ (uint32_t)Utils::s_mapNonce); }
 };
-
 template<typename K,typename V>
-class Map : public std::unordered_map<K,V,_MapHasher>
+class Map : public std::unordered_map< K,V,_MapHasher,std::equal_to<K>,Utils::Mallocator< std::pair<const K,V> > >
 {
 public:
 	ZT_INLINE V *get(const K &key) noexcept
@@ -69,11 +64,9 @@ public:
 		this->emplace(key,value);
 	}
 };
-
 #else
-
 template<typename K,typename V>
-class Map : public std::map<K,V>
+class Map : public std::map< K,V,std::less<K>,Utils::Mallocator< std::pair<const K,V> > >
 {
 public:
 	ZT_INLINE V *get(const K &key) noexcept
@@ -97,9 +90,49 @@ public:
 		(*this)[key] = value;
 	}
 };
-
 #endif
 
+template<typename K,typename V>
+class SortedMap : public std::map< K,V,std::less<K>,Utils::Mallocator< std::pair<const K,V> > >
+{
+public:
+	ZT_INLINE V *get(const K &key) noexcept
+	{
+		typename SortedMap::iterator i(this->find(key));
+		if (i == this->end())
+			return nullptr;
+		return &(i->second);
+	}
+
+	ZT_INLINE const V *get(const K &key) const noexcept
+	{
+		typename SortedMap::const_iterator i(this->find(key));
+		if (i == this->end())
+			return nullptr;
+		return &(i->second);
+	}
+
+	ZT_INLINE void set(const K &key,const V &value)
+	{
+		(*this)[key] = value;
+	}
+};
+
+template<typename V>
+class Vector : public std::vector< V,Utils::Mallocator<V> >
+{
+};
+
+template<typename V>
+class List : public std::list< V,Utils::Mallocator<V> >
+{
+};
+
+template<typename V>
+class Set : public std::set< V,std::less<V>,Utils::Mallocator<V> >
+{
+};
+
 } // ZeroTier
 
 #endif

node/Defragmenter.hpp

@@ -20,7 +20,7 @@
 #include "Mutex.hpp"
 #include "Path.hpp"
 #include "FCV.hpp"
-#include "Map.hpp"
+#include "Containers.hpp"
 
 #include <cstring>
 #include <cstdlib>
@@ -39,11 +39,16 @@ namespace ZeroTier {
  *
  * This class is thread-safe and handles locking internally.
  *
- * @tparam MF Maximum number of fragments that each message can possess
+ * @tparam MF Maximum number of fragments that each message can possess (default: ZT_MAX_PACKET_FRAGMENTS)
- * @tparam GCS Garbage collection target size for the incoming message queue
+ * @tparam MFP Maximum number of incoming fragments per path (if paths are specified) (default: ZT_MAX_INCOMING_FRAGMENTS_PER_PATH)
- * @tparam GCT Garbage collection trigger threshold, usually 2X GCS
+ * @tparam GCS Garbage collection target size for the incoming message queue (default: ZT_MAX_PACKET_FRAGMENTS * 2)
+ * @tparam GCT Garbage collection trigger threshold, usually 2X GCS (default: ZT_MAX_PACKET_FRAGMENTS * 4)
  */
-template<unsigned int MF = 16,unsigned int GCS = 32,unsigned int GCT = 64>
+template<
+  unsigned int MF = ZT_MAX_PACKET_FRAGMENTS,
+  unsigned int MFP = ZT_MAX_INCOMING_FRAGMENTS_PER_PATH,
+  unsigned int GCS = (ZT_MAX_PACKET_FRAGMENTS * 2),
+  unsigned int GCT = (ZT_MAX_PACKET_FRAGMENTS * 4)>
 class Defragmenter
 {
 public:
@@ -89,14 +94,14 @@ public:
 		ERR_OUT_OF_MEMORY
 	};
 
-	ZT_INLINE Defragmenter() {}
+	ZT_INLINE Defragmenter() {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
 
 	/**
 	 * Process a fragment of a multi-part message
 	 *
 	 * The message ID is arbitrary but must be something that can uniquely
-	 * group fragments for a given final message. The total fragments expected
+	 * group fragments for a given final message. The total fragments
-	 * value is expectded to be the same for all fragments in a message. Results
+	 * value is expected to be the same for all fragments in a message. Results
 	 * are undefined and probably wrong if this value changes across a message.
 	 * Fragment numbers must be sequential starting with 0 and going up to
 	 * one minus total fragments expected (non-inclusive range).
@@ -130,7 +135,6 @@ public:
 	 * @param totalFragmentsExpected Total number of expected fragments in this message or 0 to use cached value
 	 * @param now Current time
 	 * @param via If non-NULL this is the path on which this message fragment was received
-	 * @param maxIncomingFragmentsPerPath If via is non-NULL this is a cutoff for maximum fragments in flight via this path
 	 * @return Result code
 	 */
 	ZT_INLINE ResultCode assemble(
@@ -142,8 +146,7 @@ public:
 		const unsigned int fragmentNo,
 		const unsigned int totalFragmentsExpected,
 		const int64_t now,
-		const SharedPtr<Path> &via,
+		const SharedPtr<Path> &via)
-		const unsigned int maxIncomingFragmentsPerPath)
 	{
 		// Sanity checks for malformed fragments or invalid input parameters.
 		if ((fragmentNo >= totalFragmentsExpected)||(totalFragmentsExpected > MF)||(totalFragmentsExpected == 0))
@@ -214,7 +217,7 @@ public:
 			bool tooManyPerPath = false;
 			via->_inboundFragmentedMessages_l.lock();
 			try {
-				if (via->_inboundFragmentedMessages.size() < maxIncomingFragmentsPerPath) {
+				if (via->_inboundFragmentedMessages.size() < MFP) {
 					via->_inboundFragmentedMessages.insert(messageId);
 				} else {
 					tooManyPerPath = true;
@@ -327,7 +330,7 @@ private:
 		}
 
 		uint64_t id;
-		volatile int64_t lastUsed;
+		int64_t lastUsed;
 		unsigned int totalFragmentsExpected;
 		unsigned int fragmentsReceived;
 		SharedPtr<Path> via;
@@ -335,7 +338,7 @@ private:
 		Mutex lock;
 	};
 
-	Map< uint64_t,_E > _messages;
+	Map< uint64_t,Defragmenter<MF,MFP,GCS,GCT>::_E > _messages;
 	RWMutex _messages_l;
 };
 

node/Dictionary.hpp

@@ -18,10 +18,9 @@
 #include "Utils.hpp"
 #include "Address.hpp"
 #include "Buf.hpp"
-#include "Map.hpp"
+#include "Containers.hpp"
 
 #include <cstdint>
-#include <vector>
 
 namespace ZeroTier {
 
@@ -180,7 +179,7 @@ private:
 		return key;
 	}
 
-	Map< uint64_t,std::vector<uint8_t> > _t;
+	Map< uint64_t,Vector<uint8_t> > _t;
 };
 
 } // namespace ZeroTier

node/Endpoint.cpp

@@ -12,10 +12,11 @@
 /****/
 
 #include "Endpoint.hpp"
+#include "Utils.hpp"
 
 namespace ZeroTier {
 
-Endpoint::Endpoint(const InetAddress &sa,const Protocol proto) noexcept
+Endpoint::Endpoint(const InetAddress &sa,const Protocol proto) noexcept // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
 {
 	switch (sa.family()) {
 		case AF_INET:
@@ -23,25 +24,28 @@ Endpoint::Endpoint(const InetAddress &sa,const Protocol proto) noexcept
 			break;
 		case AF_INET6:
 			_t = TYPE_INETADDR_V6;
+			break;
 		default:
 			_t = TYPE_NIL;
 			return;
 	}
-	asInetAddress(_v.in.sa) = sa;
+	_proto = proto;
-	_v.in.proto = (uint8_t)proto;
+	asInetAddress(_v.sa) = sa;
 }
 
 bool Endpoint::operator==(const Endpoint &ep) const noexcept
 {
-	if (_t == ep._t) {
+	if ((_t == ep._t)&&(_proto == ep._proto)) {
 		switch(_t) {
-			default:               return true;
+			default:
-			case TYPE_ZEROTIER:    return ((_v.zt.address == ep._v.zt.address)&&(memcmp(_v.zt.hash,ep._v.zt.hash,sizeof(_v.zt.hash)) == 0));
+				return true;
-			case TYPE_DNSNAME:     return ((_v.dns.port == ep._v.dns.port)&&(strcmp(_v.dns.name,ep._v.dns.name) == 0));
+			case TYPE_ZEROTIER:
-			case TYPE_URL:         return (strcmp(_v.url,ep._v.url) == 0);
+				return ((_v.zt.address == ep._v.zt.address) && (memcmp(_v.zt.hash,ep._v.zt.hash,sizeof(_v.zt.hash)) == 0));
-			case TYPE_ETHERNET:    return (_v.eth == ep._v.eth);
+			case TYPE_ETHERNET:
+				return memcmp(_v.eth,ep._v.eth,6) == 0;
 			case TYPE_INETADDR_V4:
-			case TYPE_INETADDR_V6: return ((asInetAddress(_v.in.sa) == asInetAddress(ep._v.in.sa))&&(_v.in.proto == ep._v.in.proto));
+			case TYPE_INETADDR_V6:
+				return asInetAddress(_v.sa) == asInetAddress(ep._v.sa);
 		}
 	}
 	return false;
@@ -52,17 +56,20 @@ bool Endpoint::operator<(const Endpoint &ep) const noexcept
 	if ((int)_t < (int)ep._t) {
 		return true;
 	} else if (_t == ep._t) {
-		int ncmp;
+		if ((int)_proto < (int)ep._proto) {
-		switch(_t) {
+			return true;
-			case TYPE_ZEROTIER:    return (_v.zt.address < ep._v.zt.address) ? true : ((_v.zt.address == ep._v.zt.address)&&(memcmp(_v.zt.hash,ep._v.zt.hash,sizeof(_v.zt.hash)) < 0));
+		} else {
-			case TYPE_DNSNAME:
+			switch (_t) {
-				ncmp = strcmp(_v.dns.name,ep._v.dns.name);
+				case TYPE_ZEROTIER:
-				                     return ((ncmp < 0) ? true : (ncmp == 0)&&(_v.dns.port < ep._v.dns.port));
+					return (_v.zt.address < ep._v.zt.address) ? true : ((_v.zt.address == ep._v.zt.address) && (memcmp(_v.zt.hash,ep._v.zt.hash,sizeof(_v.zt.hash)) < 0));
-			case TYPE_URL:         return (strcmp(_v.url,ep._v.url) < 0);
+				case TYPE_ETHERNET:
-			case TYPE_ETHERNET:    return (_v.eth < ep._v.eth);
+					return memcmp(_v.eth,ep._v.eth,6) < 0;
-			case TYPE_INETADDR_V4:
+				case TYPE_INETADDR_V4:
-			case TYPE_INETADDR_V6: return ((_v.in.proto < ep._v.in.proto)||((_v.in.proto == ep._v.in.proto)&&(asInetAddress(_v.in.sa) < asInetAddress(ep._v.in.sa))));
+				case TYPE_INETADDR_V6:
-			default:               return false;
+					return asInetAddress(_v.sa) < asInetAddress(ep._v.sa);
+				default:
+					return false;
+			}
 		}
 	}
 	return false;
@@ -70,138 +77,82 @@ bool Endpoint::operator<(const Endpoint &ep) const noexcept
 
 int Endpoint::marshal(uint8_t data[ZT_ENDPOINT_MARSHAL_SIZE_MAX]) const noexcept
 {
-	int p;
 	data[0] = (uint8_t)_t;
-	Utils::storeBigEndian(data + 1,(uint16_t)_l[0]);
+	Utils::storeBigEndian(data + 1,(uint16_t)_proto);
-	Utils::storeBigEndian(data + 3,(uint16_t)_l[1]);
+	Utils::storeBigEndian(data + 3,(uint16_t)_l[0]);
-	Utils::storeBigEndian(data + 5,(uint16_t)_l[2]);
+	Utils::storeBigEndian(data + 5,(uint16_t)_l[1]);
+	Utils::storeBigEndian(data + 7,(uint16_t)_l[2]);
+
+	int p;
 	switch(_t) {
 		case TYPE_ZEROTIER:
-			data[7] = (uint8_t)(_v.zt.address >> 32U);
+			data[9] = (uint8_t)(_v.zt.address >> 32U);
-			data[8] = (uint8_t)(_v.zt.address >> 24U);
+			data[10] = (uint8_t)(_v.zt.address >> 24U);
-			data[9] = (uint8_t)(_v.zt.address >> 16U);
+			data[11] = (uint8_t)(_v.zt.address >> 16U);
-			data[10] = (uint8_t)(_v.zt.address >> 8U);
+			data[12] = (uint8_t)(_v.zt.address >> 8U);
-			data[11] = (uint8_t)_v.zt.address;
+			data[13] = (uint8_t)_v.zt.address;
-			Utils::copy<ZT_IDENTITY_HASH_SIZE>(data + 12,_v.zt.hash);
+			Utils::copy<ZT_FINGERPRINT_HASH_SIZE>(data + 14,_v.zt.hash);
-			return ZT_IDENTITY_HASH_SIZE + 12;
+			return ZT_FINGERPRINT_HASH_SIZE + 14;
-		case TYPE_DNSNAME:
-			p = 7;
-			for (;;) {
-				if ((data[p] = (uint8_t)_v.dns.name[p-1]) == 0)
-					break;
-				++p;
-				if (p == (ZT_ENDPOINT_MAX_NAME_SIZE+1))
-					return -1;
-			}
-			data[p++] = (uint8_t)(_v.dns.port >> 8U);
-			data[p++] = (uint8_t)_v.dns.port;
-			return p;
-		case TYPE_URL:
-			p = 7;
-			for (;;) {
-				if ((data[p] = (uint8_t)_v.url[p-1]) == 0)
-					break;
-				++p;
-				if (p == (ZT_ENDPOINT_MAX_NAME_SIZE+1))
-					return -1;
-			}
-			return p;
 		case TYPE_ETHERNET:
-			data[7] = (uint8_t)(_v.eth >> 40U);
+			Utils::copy<6>(data + 9,_v.eth);
-			data[8] = (uint8_t)(_v.eth >> 32U);
+			return 15;
-			data[9] = (uint8_t)(_v.eth >> 24U);
-			data[10] = (uint8_t)(_v.eth >> 16U);
-			data[11] = (uint8_t)(_v.eth >> 8U);
-			data[12] = (uint8_t)_v.eth;
-			return 13;
 		case TYPE_INETADDR_V4:
 		case TYPE_INETADDR_V6:
-			p = 7 + asInetAddress(_v.in.sa).marshal(data + 7);
+			p = 9 + asInetAddress(_v.sa).marshal(data + 7);
-			if (p <= 7)
+			if (p <= 9)
 				return -1;
-			data[p++] = _v.in.proto;
 			return p;
 		default:
 			data[0] = (uint8_t)TYPE_NIL;
-			return 7;
+			return 1;
 	}
 }
 
 int Endpoint::unmarshal(const uint8_t *restrict data,const int len) noexcept
 {
-	if (len < 7)
+	if (len < 1)
 		return -1;
-	int p;
+
 	_t = (Type)data[0];
-	_l[0] = (int)Utils::loadBigEndian<uint16_t>(data + 1);
+	if (_t == TYPE_NIL)
-	_l[1] = (int)Utils::loadBigEndian<uint16_t>(data + 3);
+		return 1;
-	_l[2] = (int)Utils::loadBigEndian<uint16_t>(data + 5);
+
+	_proto = (Protocol)Utils::loadBigEndian<uint16_t>(data + 1);
+	_l[0] = (int)Utils::loadBigEndian<uint16_t>(data + 3);
+	_l[1] = (int)Utils::loadBigEndian<uint16_t>(data + 5);
+	_l[2] = (int)Utils::loadBigEndian<uint16_t>(data + 7);
+
+	int p;
   switch(_t) {
-		case TYPE_NIL:
-			return 7;
 	  case TYPE_ZEROTIER:
-		  if (len < (12 + ZT_IDENTITY_HASH_SIZE))
+		  if (len < (14 + ZT_FINGERPRINT_HASH_SIZE))
 			  return -1;
-		  _v.zt.address = ((uint64_t)data[7]) << 32U;
+		  _v.zt.address = ((uint64_t)data[9]) << 32U;
-		  _v.zt.address |= ((uint64_t)data[8]) << 24U;
+		  _v.zt.address |= ((uint64_t)data[10]) << 24U;
-		  _v.zt.address |= ((uint64_t)data[9]) << 16U;
+		  _v.zt.address |= ((uint64_t)data[11]) << 16U;
-		  _v.zt.address |= ((uint64_t)data[10]) << 8U;
+		  _v.zt.address |= ((uint64_t)data[12]) << 8U;
-		  _v.zt.address |= (uint64_t)data[11];
+		  _v.zt.address |= (uint64_t)data[13];
-		  Utils::copy<ZT_IDENTITY_HASH_SIZE>(_v.zt.hash,data + 12);
+		  Utils::copy<ZT_FINGERPRINT_HASH_SIZE>(_v.zt.hash,data + 14);
-		  return 60;
+		  return ZT_FINGERPRINT_HASH_SIZE + 14;
-	  case TYPE_DNSNAME:
-		  if (len < 10)
-			  return -1;
-		  p = 7;
-		  for (;;) {
-			  if ((_v.dns.name[p-1] = (char)data[p]) == 0) {
-				  ++p;
-				  break;
-			  }
-			  ++p;
-			  if ((p >= (ZT_ENDPOINT_MARSHAL_SIZE_MAX-2))||(p >= (len-2)))
-				  return -1;
-		  }
-		  _v.dns.port = (uint16_t)(((unsigned int)data[p++]) << 8U);
-		  _v.dns.port |= (uint16_t)data[p++];
-		  return p;
-	  case TYPE_URL:
-		  if (len < 8)
-			  return -1;
-		  p = 7;
-		  for (;;) {
-			  if ((_v.url[p-1] = (char)data[p]) == 0) {
-				  ++p;
-				  break;
-			  }
-			  ++p;
-			  if ((p >= (ZT_ENDPOINT_MAX_NAME_SIZE+1))||(p >= len))
-				  return -1;
-		  }
-		  return p;
 	  case TYPE_ETHERNET:
-		  if (len < 13)
+		  if (len < 15)
 			  return -1;
-		  _v.eth = ((uint64_t)data[7]) << 40U;
+		  Utils::copy<6>(_v.eth,data + 9);
-		  _v.eth |= ((uint64_t)data[8]) << 32U;
+		  return 15;
-		  _v.eth |= ((uint64_t)data[9]) << 24U;
-		  _v.eth |= ((uint64_t)data[10]) << 16U;
-		  _v.eth |= ((uint64_t)data[11]) << 8U;
-		  _v.eth |= (uint64_t)data[12];
-		  return 13;
 		case TYPE_INETADDR_V4:
 		case TYPE_INETADDR_V6:
-			p = 7 + asInetAddress(_v.in.sa).unmarshal(data + 7,len - 7);
+			if (len <= 9)
-			if ((p <= 7)||(p >= len))
+				return -1;
+			p = 9 + asInetAddress(_v.sa).unmarshal(data + 9,len - 9);
+			if ((p <= 9)||(p >= len))
 				return -1;
-			_v.in.proto = data[p++];
 			return p;
 		default:
 			// Unrecognized endpoint types not yet specified must start with a 16-bit
 			// length so that older versions of ZeroTier can skip them.
-			if (len < 9)
+			if (len < 11)
 				return -1;
-			p = 9 + (int)Utils::loadBigEndian<uint16_t>(data + 7);
+			p = 11 + (int)Utils::loadBigEndian<uint16_t>(data + 9);
 			return (p > len) ? -1 : p;
 	}
 }

node/Endpoint.hpp

@@ -26,8 +26,7 @@
 #include <cstdint>
 #include <cstring>
 
-// max name size + type byte + port (for DNS name/port) + 3x 16-bit coordinate for location
+#define ZT_ENDPOINT_MARSHAL_SIZE_MAX 64
-#define ZT_ENDPOINT_MARSHAL_SIZE_MAX (ZT_ENDPOINT_MAX_NAME_SIZE+1+2+2+2+2)
 
 namespace ZeroTier {
 
@@ -49,83 +48,44 @@ public:
 	{
 		TYPE_NIL =          ZT_TRACE_EVENT_PATH_TYPE_NIL,
 		TYPE_ZEROTIER =     ZT_TRACE_EVENT_PATH_TYPE_ZEROTIER,
-		TYPE_DNSNAME =      ZT_TRACE_EVENT_PATH_TYPE_DNSNAME,
-		TYPE_URL =          ZT_TRACE_EVENT_PATH_TYPE_URL,
-		TYPE_INETADDR_V4 =  ZT_TRACE_EVENT_PATH_TYPE_INETADDR_V4,
 		TYPE_ETHERNET =     ZT_TRACE_EVENT_PATH_TYPE_ETHERNET,
+		TYPE_INETADDR_V4 =  ZT_TRACE_EVENT_PATH_TYPE_INETADDR_V4,
 		TYPE_INETADDR_V6 =  ZT_TRACE_EVENT_PATH_TYPE_INETADDR_V6
 	};
 
 	/**
-	 * Protocol identifiers for INETADDR endpoint types
+	 * Protocol identifier bits.
 	 *
-	 * Most of these are reserved for future use.
+	 * Endpoint types can support more than one of these, though it depends on the type.
 	 */
 	enum Protocol
 	{
-		PROTO_UDP_ZT =      0,
+		PROTO_DGRAM =       0x0001,
-		PROTO_TCP_ZT =      1,
+		PROTO_TCP  =        0x0002,
-		PROTO_IP_ZT =       2
+		PROTO_HTTP =        0x0004,
+		PROTO_HTTPS =       0x0008,
+		PROTO_WS =          0x0010,
+		PROTO_WEBRTC =      0x0020
 	};
 
 	ZT_INLINE Endpoint() noexcept { memoryZero(this); } // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
 
-	explicit Endpoint(const InetAddress &sa,Protocol proto = PROTO_UDP_ZT) noexcept;
+	explicit Endpoint(const InetAddress &sa,Protocol proto = PROTO_DGRAM) noexcept;
-
-	explicit ZT_INLINE Endpoint(const Address &zt,const uint8_t identityHash[ZT_IDENTITY_HASH_SIZE]) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-		_t(TYPE_ZEROTIER)
-	{
-		_v.zt.address = zt.toInt();
-		Utils::copy<ZT_IDENTITY_HASH_SIZE>(_v.zt.hash,identityHash);
-	}
-
-	explicit ZT_INLINE Endpoint(const char *name,const int port) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-		_t(TYPE_DNSNAME)
-	{
-		_v.dns.port = port;
-		Utils::scopy(_v.dns.name,sizeof(_v.dns.name),name);
-	}
-
-	explicit ZT_INLINE Endpoint(const char *url) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-		_t(TYPE_URL)
-	{
-		Utils::scopy(_v.url,sizeof(_v.url),url);
-	}
 
 	/**
 	 * @return InetAddress or NIL if not of this type
 	 */
-	ZT_INLINE const InetAddress &inetAddr() const noexcept { return ((_t == TYPE_INETADDR_V4) || (_t == TYPE_INETADDR_V6)) ? asInetAddress(_v.in.sa) : InetAddress::NIL; }
+	ZT_INLINE const InetAddress &inetAddr() const noexcept { return ((_t == TYPE_INETADDR_V4) || (_t == TYPE_INETADDR_V6)) ? asInetAddress(_v.sa) : InetAddress::NIL; }
 
 	/**
-	 * @return Protocol for INETADDR types, undefined for other endpoint types
+	 * @return Protocol bit mask
 	 */
-	ZT_INLINE Protocol inetAddrProto() const noexcept { return (Protocol)_v.in.proto; }
+	ZT_INLINE Protocol protocol() const noexcept { return _proto; }
-
-	/**
-	 * @return DNS name or empty string if not of this type
-	 */
-	ZT_INLINE const char *dnsName() const noexcept { return (_t == TYPE_DNSNAME) ? _v.dns.name : ""; }
-
-	/**
-	 * @return Port associated with DNS name or -1 if not of this type
-	 */
-	ZT_INLINE int dnsPort() const noexcept { return (_t == TYPE_DNSNAME) ? _v.dns.port : -1; }
-
-	/**
-	 * @return ZeroTier address or NIL if not of this type
-	 */
-	ZT_INLINE Address ztAddress() const noexcept { return Address((_t == TYPE_ZEROTIER) ? _v.zt.address : (uint64_t)0); }
 
 	/**
 	 * @return 384-bit hash of identity keys or NULL if not of this type
 	 */
-	ZT_INLINE const Fingerprint &ztFingerprint() const noexcept { return *reinterpret_cast<const Fingerprint *>(&_v.zt); }
+	ZT_INLINE const Fingerprint &fingerprint() const noexcept { return *reinterpret_cast<const Fingerprint *>(&_v.zt); }
-
-	/**
-	 * @return URL or empty string if not of this type
-	 */
-	ZT_INLINE const char *url() const noexcept { return (_t == TYPE_URL) ? _v.url : ""; }
 
 	/**
 	 * @return Ethernet address or NIL if not of this type
@@ -152,19 +112,12 @@ public:
 
 private:
 	Type _t;
+	Protocol _proto;
 	int _l[3]; // X,Y,Z location in kilometers from the nearest gravitational center of mass
 	union {
-		struct {
+		sockaddr_storage sa;
-			sockaddr_storage sa;
-			uint8_t proto;
-		} in;
-		struct {
-			uint16_t port;
-			char name[ZT_ENDPOINT_MAX_NAME_SIZE];
-		} dns;
 		ZT_Fingerprint zt;
-		char url[ZT_ENDPOINT_MAX_NAME_SIZE];
+		uint8_t eth[6];
-		uint64_t eth;
 	} _v;
 };
 

node/FCV.hpp

@@ -54,6 +54,16 @@ public:
 	ZT_INLINE FCV() noexcept : _s(0) {} // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
 	ZT_INLINE FCV(const FCV &v) : _s(0) { *this = v; } // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
 
+	template<typename I>
+	ZT_INLINE FCV(I i,I end) : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+		_s(0)
+	{
+		while (i != end) {
+			push_back(*i);
+			++i;
+		}
+	}
+
 	ZT_INLINE ~FCV() { this->clear(); }
 
 	ZT_INLINE FCV &operator=(const FCV &v)

node/Fingerprint.hpp

@@ -41,7 +41,7 @@ public:
 	/**
 	 * Create an empty/nil fingerprint
 	 */
-	ZT_INLINE Fingerprint() noexcept { memoryZero(this); }
+	ZT_INLINE Fingerprint() noexcept { memoryZero(this); } // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
 
 	ZT_INLINE Address address() const noexcept { return Address(_fp.address); }
 	ZT_INLINE const uint8_t *hash() const noexcept { return _fp.hash; }
@@ -88,9 +88,9 @@ public:
 
 	ZT_INLINE operator bool() const noexcept { return (_fp.address != 0); } // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
 
-	ZT_INLINE bool operator==(const Fingerprint &h) const noexcept { return ((_fp.address == h._fp.address) && (memcmp(_fp.hash,h._fp.hash,ZT_IDENTITY_HASH_SIZE) == 0)); }
+	ZT_INLINE bool operator==(const Fingerprint &h) const noexcept { return ((_fp.address == h._fp.address) && (memcmp(_fp.hash,h._fp.hash,ZT_FINGERPRINT_HASH_SIZE) == 0)); }
 	ZT_INLINE bool operator!=(const Fingerprint &h) const noexcept { return !(*this == h); }
-	ZT_INLINE bool operator<(const Fingerprint &h) const noexcept { return ((_fp.address < h._fp.address) || ((_fp.address == h._fp.address) && (memcmp(_fp.hash,h._fp.hash,ZT_IDENTITY_HASH_SIZE) < 0))); }
+	ZT_INLINE bool operator<(const Fingerprint &h) const noexcept { return ((_fp.address < h._fp.address) || ((_fp.address == h._fp.address) && (memcmp(_fp.hash,h._fp.hash,ZT_FINGERPRINT_HASH_SIZE) < 0))); }
 	ZT_INLINE bool operator>(const Fingerprint &h) const noexcept { return (h < *this); }
 	ZT_INLINE bool operator<=(const Fingerprint &h) const noexcept { return !(h < *this); }
 	ZT_INLINE bool operator>=(const Fingerprint &h) const noexcept { return !(*this < h); }

node/Identity.cpp

@@ -202,7 +202,7 @@ bool Identity::generate(const Type t)
 				// some new key material every time it wraps. The ECC384 generator is slightly
 				// faster so use that one.
 				_pub.nonce = 0;
-				C25519::generate(_pub.c25519,_priv.c25519);
+				C25519::generateCombined(_pub.c25519,_priv.c25519);
 				ECC384GenerateKey(_pub.p384,_priv.p384);
 				for(;;) {
 					if (identityV1ProofOfWorkCriteria(&_pub,sizeof(_pub),b))
@@ -259,7 +259,7 @@ bool Identity::locallyValidate() const noexcept
 	return false;
 }
 
-void Identity::hashWithPrivate(uint8_t h[ZT_IDENTITY_HASH_SIZE]) const
+void Identity::hashWithPrivate(uint8_t h[ZT_FINGERPRINT_HASH_SIZE]) const
 {
 	if (_hasPrivate) {
 		switch (_type) {
@@ -321,7 +321,7 @@ bool Identity::verify(const void *data,unsigned int len,const void *sig,unsigned
 	return false;
 }
 
-bool Identity::agree(const Identity &id,uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]) const
+bool Identity::agree(const Identity &id,uint8_t key[ZT_SYMMETRIC_KEY_SIZE]) const
 {
 	uint8_t rawkey[128];
 	uint8_t h[64];
@@ -333,7 +333,7 @@ bool Identity::agree(const Identity &id,uint8_t key[ZT_PEER_SECRET_KEY_LENGTH])
 				// C25519 portion of a type 1 P-384 key.
 				C25519::agree(_priv.c25519,id._pub.c25519,rawkey);
 				SHA512(h,rawkey,ZT_C25519_ECDH_SHARED_SECRET_SIZE);
-				Utils::copy<ZT_PEER_SECRET_KEY_LENGTH>(key,h);
+				Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(key,h);
 				return true;
 			}
 
@@ -348,13 +348,13 @@ bool Identity::agree(const Identity &id,uint8_t key[ZT_PEER_SECRET_KEY_LENGTH])
 				C25519::agree(_priv.c25519,id._pub.c25519,rawkey);
 				ECC384ECDH(id._pub.p384,_priv.p384,rawkey + ZT_C25519_ECDH_SHARED_SECRET_SIZE);
 				SHA384(h,rawkey,ZT_C25519_ECDH_SHARED_SECRET_SIZE + ZT_ECC384_SHARED_SECRET_SIZE);
-				Utils::copy<ZT_PEER_SECRET_KEY_LENGTH>(key,h);
+				Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(key,h);
 				return true;
 			} else if (id._type == C25519) {
 				// If the other identity is a C25519 identity we can agree using only that type.
 				C25519::agree(_priv.c25519,id._pub.c25519,rawkey);
 				SHA512(h,rawkey,ZT_C25519_ECDH_SHARED_SECRET_SIZE);
-				Utils::copy<ZT_PEER_SECRET_KEY_LENGTH>(key,h);
+				Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(key,h);
 				return true;
 			}
 

node/Identity.hpp

@@ -146,7 +146,7 @@ public:
 	 *
 	 * @param h Buffer to store SHA384 hash
 	 */
-	void hashWithPrivate(uint8_t h[ZT_IDENTITY_HASH_SIZE]) const;
+	void hashWithPrivate(uint8_t h[ZT_FINGERPRINT_HASH_SIZE]) const;
 
 	/**
 	 * Sign a message with this identity (private key required)
@@ -182,7 +182,7 @@ public:
 	 * @param key Result parameter to fill with key bytes
 	 * @return Was agreement successful?
 	 */
-	bool agree(const Identity &id,uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]) const;
+	bool agree(const Identity &id,uint8_t key[ZT_SYMMETRIC_KEY_SIZE]) const;
 
 	/**
 	 * @return This identity's address

node/InetAddress.cpp

@@ -60,12 +60,12 @@ InetAddress::IpScope InetAddress::ipScope() const noexcept
 					break;
 				case 0xff: return IP_SCOPE_NONE;                                   // 255.0.0.0/8 (broadcast, or unused/unusable)
 			}
-			switch(ip >> 28) {
+			switch(ip >> 28U) {
 				case 0xe: return IP_SCOPE_MULTICAST;                               // 224.0.0.0/4
 				case 0xf: return IP_SCOPE_PSEUDOPRIVATE;                           // 240.0.0.0/4 ("reserved," usually unusable)
 			}
 			return IP_SCOPE_GLOBAL;
-		}	break;
+		}
 
 		case AF_INET6: {
 			const unsigned char *ip = reinterpret_cast<const unsigned char *>(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr); // NOLINT(hicpp-use-auto,modernize-use-auto)
@@ -87,10 +87,9 @@ InetAddress::IpScope InetAddress::ipScope() const noexcept
 				if (ip[15] == 0x00) return IP_SCOPE_NONE;                          // ::/128
 			}
 			return IP_SCOPE_GLOBAL;
-		}	break;
+		}
 
 	}
-
 	return IP_SCOPE_NONE;
 }
 
@@ -300,25 +299,6 @@ bool InetAddress::containsAddress(const InetAddress &addr) const noexcept
 	return false;
 }
 
-unsigned long InetAddress::hashCode() const noexcept
-{
-	if (_data.ss_family == AF_INET) {
-		return ((unsigned long)reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr + (unsigned long)reinterpret_cast<const struct sockaddr_in *>(this)->sin_port);
-	} else if (_data.ss_family == AF_INET6) {
-		unsigned long tmp = reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_port;
-		const uint8_t *a = reinterpret_cast<const uint8_t *>(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr); // NOLINT(hicpp-use-auto,modernize-use-auto)
-		for(long i=0;i<16;++i)
-			reinterpret_cast<uint8_t *>(&tmp)[i % sizeof(tmp)] ^= a[i];
-		return tmp;
-	} else {
-		unsigned long tmp = reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_port;
-		const uint8_t *a = reinterpret_cast<const uint8_t *>(this); // NOLINT(hicpp-use-auto,modernize-use-auto)
-		for(long i=0;i<(long)sizeof(InetAddress);++i)
-			reinterpret_cast<uint8_t *>(&tmp)[i % sizeof(tmp)] ^= a[i];
-		return tmp;
-	}
-}
-
 void InetAddress::forTrace(ZT_TraceEventPathAddress &ta) const noexcept
 {
 	uint32_t tmp;

node/InetAddress.hpp

@@ -390,7 +390,19 @@ public:
 		return false;
 	}
 
-	unsigned long hashCode() const noexcept;
+	ZT_INLINE unsigned long hashCode() const noexcept
+	{
+		if (_data.ss_family == AF_INET) {
+			return (unsigned long)Utils::hash32(((uint32_t)reinterpret_cast<const struct sockaddr_in *>(&_data)->sin_addr.s_addr + (uint32_t)reinterpret_cast<const struct sockaddr_in *>(&_data)->sin_port) ^ (uint32_t)Utils::s_mapNonce);
+		} else if (_data.ss_family == AF_INET6) {
+			return (unsigned long)Utils::hash64(
+				(Utils::loadAsIsEndian<uint64_t>(reinterpret_cast<const struct sockaddr_in6 *>(&_data)->sin6_addr.s6_addr) +
+				Utils::loadAsIsEndian<uint64_t>(reinterpret_cast<const struct sockaddr_in6 *>(&_data)->sin6_addr.s6_addr + 8) +
+				(uint64_t)reinterpret_cast<const struct sockaddr_in6 *>(&_data)->sin6_port) ^ Utils::s_mapNonce
+			);
+		}
+		return Utils::fnv1a32(&_data,sizeof(_data));
+	}
 
 	/**
 	 * Fill out a ZT_TraceEventPathAddress from this InetAddress

node/Membership.hpp

@@ -18,7 +18,7 @@
 
 #include "Constants.hpp"
 #include "Credential.hpp"
-#include "Map.hpp"
+#include "Containers.hpp"
 #include "CertificateOfMembership.hpp"
 #include "Capability.hpp"
 #include "Tag.hpp"
@@ -218,7 +218,7 @@ public:
 		ZT_INLINE Capability *next() noexcept
 		{
 			while (_hti != _m._remoteCaps.end()) {
-				Map< uint32_t,Capability >::iterator i(_hti++);
+				Map< uint32_t,Capability >::iterator i(_hti++); // NOLINT(hicpp-use-auto,modernize-use-auto)
 				if (_m._isCredentialTimestampValid(_nconf,i->second))
 					return &(i->second);
 			}

node/MulticastGroup.hpp

@@ -94,7 +94,7 @@ public:
 	ZT_INLINE bool operator<=(const MulticastGroup &g) const noexcept { return !(g < *this); }
 	ZT_INLINE bool operator>=(const MulticastGroup &g) const noexcept { return !(*this < g); }
 
-	ZT_INLINE unsigned long hashCode() const noexcept { return (_mac.hashCode() ^ (unsigned long)_adi); }
+	ZT_INLINE unsigned long hashCode() const noexcept { return (_mac.hashCode() + (unsigned long)_adi); }
 
 private:
 	MAC _mac;

node/Network.cpp

@@ -1022,7 +1022,7 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD
 	try {
 		if ((nconf.issuedTo != RR->identity.address())||(nconf.networkId != _id))
 			return 0; // invalid config that is not for us or not for this network
-		if ((!Utils::allZero(nconf.issuedToFingerprintHash,ZT_IDENTITY_HASH_SIZE))&&(memcmp(nconf.issuedToFingerprintHash,RR->identity.fingerprint().hash(),ZT_IDENTITY_HASH_SIZE) != 0))
+		if ((!Utils::allZero(nconf.issuedToFingerprintHash,ZT_FINGERPRINT_HASH_SIZE)) && (memcmp(nconf.issuedToFingerprintHash,RR->identity.fingerprint().hash(),ZT_FINGERPRINT_HASH_SIZE) != 0))
 			return 0; // full identity hash is present and does not match
 
 		if (_config == nconf)

node/Network.hpp

@@ -25,7 +25,7 @@
 #include "Membership.hpp"
 #include "NetworkConfig.hpp"
 #include "CertificateOfMembership.hpp"
-#include "Map.hpp"
+#include "Containers.hpp"
 
 #include <cstdint>
 #include <string>

node/NetworkConfig.cpp

@@ -32,7 +32,7 @@ bool NetworkConfig::toDictionary(Dictionary &d,bool includeLegacy) const
 		d.add(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TIME_MAX_DELTA,this->credentialTimeMaxDelta);
 		d.add(ZT_NETWORKCONFIG_DICT_KEY_REVISION,this->revision);
 		d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,this->issuedTo.toString((char *)tmp));
-		d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO_IDENTITY_HASH,this->issuedToFingerprintHash,ZT_IDENTITY_HASH_SIZE);
+		d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO_IDENTITY_HASH,this->issuedToFingerprintHash,ZT_FINGERPRINT_HASH_SIZE);
 		d.add(ZT_NETWORKCONFIG_DICT_KEY_FLAGS,this->flags);
 		d.add(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,(uint64_t)this->multicastLimit);
 		d.add(ZT_NETWORKCONFIG_DICT_KEY_TYPE,(uint16_t)this->type);
@@ -122,10 +122,10 @@ bool NetworkConfig::fromDictionary(const Dictionary &d)
 		this->revision = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_REVISION,0);
 		this->issuedTo = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,0);
 		const std::vector<uint8_t> *blob = &(d[ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO_IDENTITY_HASH]);
-		if (blob->size() == ZT_IDENTITY_HASH_SIZE) {
+		if (blob->size() == ZT_FINGERPRINT_HASH_SIZE) {
-			Utils::copy<ZT_IDENTITY_HASH_SIZE>(this->issuedToFingerprintHash,blob->data());
+			Utils::copy<ZT_FINGERPRINT_HASH_SIZE>(this->issuedToFingerprintHash,blob->data());
 		} else {
-			Utils::zero<ZT_IDENTITY_HASH_SIZE>(this->issuedToFingerprintHash);
+			Utils::zero<ZT_FINGERPRINT_HASH_SIZE>(this->issuedToFingerprintHash);
 		}
 		if (!this->issuedTo)
 			return false;

node/NetworkConfig.hpp

@@ -276,7 +276,7 @@ struct NetworkConfig : TriviallyCopyable
 	 * If this field is all zero it is treated as undefined since old controllers
 	 * do not set it.
 	 */
-	uint8_t issuedToFingerprintHash[ZT_IDENTITY_HASH_SIZE];
+	uint8_t issuedToFingerprintHash[ZT_FINGERPRINT_HASH_SIZE];
 
 	/**
 	 * Flags (64-bit)

node/NetworkController.hpp

@@ -77,8 +77,8 @@ public:
 		virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode) = 0;
 	};
 
-	NetworkController() {}
+	NetworkController() {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
-	virtual ~NetworkController() {}
+	virtual ~NetworkController() {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
 
 	/**
 	 * Called when this is added to a Node to initialize and supply info

node/Node.cpp

@@ -26,7 +26,6 @@
 #include "Network.hpp"
 #include "Trace.hpp"
 #include "Locator.hpp"
-#include "Protocol.hpp"
 #include "Expect.hpp"
 #include "VL1.hpp"
 #include "VL2.hpp"
@@ -117,7 +116,7 @@ Node::Node(void *uPtr,void *tPtr,const struct ZT_Node_Callbacks *callbacks,int64
 			stateObjectPut(tPtr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr,(unsigned int)strlen(RR->publicIdentityStr));
 	}
 
-	uint8_t tmph[ZT_IDENTITY_HASH_SIZE];
+	uint8_t tmph[ZT_FINGERPRINT_HASH_SIZE];
 	RR->identity.hashWithPrivate(tmph);
 	RR->localCacheSymmetric.init(tmph);
 	Utils::burn(tmph,sizeof(tmph));
@@ -285,7 +284,7 @@ ZT_ResultCode Node::processBackgroundTasks(void *tPtr,int64_t now,volatile int64
 		std::vector<Fingerprint> bzzt;
 		{
 			Mutex::Lock l(_peerAlarms_l);
-			for(std::map<Fingerprint,int64_t>::iterator a(_peerAlarms.begin());a!=_peerAlarms.end();) { // NOLINT(hicpp-use-auto,modernize-use-auto)
+			for(std::map< Fingerprint,int64_t,std::less<Fingerprint>,Utils::Mallocator< std::pair<const Fingerprint,int64_t> > >::iterator a(_peerAlarms.begin());a!=_peerAlarms.end();) { // NOLINT(hicpp-use-auto,modernize-use-auto)
 				if (now >= a->second) {
 					bzzt.push_back(a->first);
 					_peerAlarms.erase(a++);
@@ -423,13 +422,13 @@ ZT_PeerList *Node::peers() const
 	const int64_t now = _now;
 	pl->peerCount = 0;
 	for(std::vector< SharedPtr<Peer> >::iterator pi(peers.begin());pi!=peers.end();++pi) { // NOLINT(modernize-use-auto,modernize-loop-convert,hicpp-use-auto)
-		ZT_Peer *p = &(pl->peers[pl->peerCount]);
+		ZT_Peer *const p = &(pl->peers[pl->peerCount]);
 
 		p->address = (*pi)->address().toInt();
 		identities[pl->peerCount] = (*pi)->identity(); // need to make a copy in case peer gets deleted
 		p->identity = &identities[pl->peerCount];
 		p->fingerprint.address = p->address;
-		Utils::copy<ZT_IDENTITY_HASH_SIZE>(p->fingerprint.hash,(*pi)->identity().fingerprint().hash());
+		Utils::copy<ZT_FINGERPRINT_HASH_SIZE>(p->fingerprint.hash,(*pi)->identity().fingerprint().hash());
 		if ((*pi)->remoteVersionKnown()) {
 			p->versionMajor = (int)(*pi)->remoteVersionMajor();
 			p->versionMinor = (int)(*pi)->remoteVersionMinor();
@@ -439,11 +438,15 @@ ZT_PeerList *Node::peers() const
 			p->versionMinor = -1;
 			p->versionRev = -1;
 		}
-		p->latency = (int)(*pi)->latency();
+		p->latency = (*pi)->latency();
-		if (p->latency >= 0xffff)
-			p->latency = -1;
 		p->root = RR->topology->isRoot((*pi)->identity()) ? 1 : 0;
-		Utils::copy<sizeof(sockaddr_storage)>(&p->bootstrap,&((*pi)->bootstrap()));
+
+		{
+			FCV<Endpoint,ZT_MAX_PEER_NETWORK_PATHS> bs((*pi)->bootstrap());
+			p->bootstrapAddressCount = 0;
+			for (FCV<Endpoint,ZT_MAX_PEER_NETWORK_PATHS>::const_iterator i(bs.begin());i!=bs.end();++i) // NOLINT(modernize-loop-convert)
+				Utils::copy<sizeof(sockaddr_storage)>(&(p->bootstrap[p->bootstrapAddressCount++]),&(*i));
+		}
 
 		std::vector< SharedPtr<Path> > paths;
 		(*pi)->getAllPaths(paths);

node/Node.hpp

@@ -24,7 +24,7 @@
 #include "Salsa20.hpp"
 #include "NetworkController.hpp"
 #include "Buf.hpp"
-#include "Map.hpp"
+#include "Containers.hpp"
 
 #include <cstdio>
 #include <cstdlib>
@@ -181,7 +181,7 @@ public:
 	/**
 	 * @return Known local interface addresses for this node
 	 */
-	ZT_INLINE std::vector<ZT_InterfaceAddress> localInterfaceAddresses() const
+	ZT_INLINE Vector<ZT_InterfaceAddress> localInterfaceAddresses() const
 	{
 		Mutex::Lock _l(_localInterfaceAddresses_m);
 		return _localInterfaceAddresses;
@@ -350,7 +350,7 @@ private:
 	// is harmless. This just exists as an optimization to prevent having to iterate through all peers
 	// on every processBackgroundTasks call. A simple map<> is used here because there are usually only
 	// a few of these, if any.
-	std::map<Fingerprint,int64_t> _peerAlarms;
+	std::map< Fingerprint,int64_t,std::less<Fingerprint>,Utils::Mallocator< std::pair<const Fingerprint,int64_t> > > _peerAlarms;
 	Mutex _peerAlarms_l;
 
 	// Cache that remembers whether or not the locally running network controller (if any) has authorized
@@ -375,7 +375,7 @@ private:
 
 	// These are local interface addresses that have been configured via the API
 	// and can be pushed to other nodes.
-	std::vector< ZT_InterfaceAddress > _localInterfaceAddresses;
+	Vector< ZT_InterfaceAddress > _localInterfaceAddresses;
 	Mutex _localInterfaceAddresses_m;
 
 	// This is locked while running processBackgroundTasks().

node/Path.cpp

@@ -17,7 +17,7 @@
 
 namespace ZeroTier {
 
-bool Path::send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,int64_t now) noexcept
+bool Path::send(const RuntimeEnvironment *const RR,void *const tPtr,const void *const data,const unsigned int len,const int64_t now) noexcept
 {
 	if (RR->node->putPacket(tPtr,_localSocket,_addr,data,len)) {
 		_lastOut = now;

node/Path.hpp

@@ -32,7 +32,7 @@ namespace ZeroTier {
 
 class RuntimeEnvironment;
 
-template<unsigned int MF,unsigned int GCT,unsigned int GCS>
+template<unsigned int MF,unsigned int MFP,unsigned int GCT,unsigned int GCS>
 class Defragmenter;
 
 /**
@@ -42,8 +42,8 @@ class Path
 {
 	friend class SharedPtr<Path>;
 
-	// Allow defragmenter to access fragment in flight info stored in Path for performance reasons.
+	// Allow defragmenter to access fragment-in-flight info stored in Path for performance reasons.
-	template<unsigned int MF,unsigned int GCT,unsigned int GCS>
+	template<unsigned int MF,unsigned int MFP,unsigned int GCT,unsigned int GCS>
 	friend class Defragmenter;
 
 public:
@@ -51,6 +51,7 @@ public:
 		_localSocket(l),
 		_lastIn(0),
 		_lastOut(0),
+		_latency(-1),
 		_addr(r)
 	{
 	}
@@ -91,6 +92,26 @@ public:
 		_inMeter.log(now,bytes);
 	}
 
+	/**
+	 * Update latency with a new measurement
+	 *
+	 * @param newMeasurement New latency measurement in milliseconds
+	 */
+	ZT_INLINE void updateLatency(const unsigned int newMeasurement) noexcept
+	{
+		int lat = _latency;
+		if (lat > 0) {
+			_latency = (lat + newMeasurement) / 2;
+		} else {
+			_latency = newMeasurement;
+		}
+	}
+
+	/**
+	 * @return Latency in milliseconds or -1 if unknown
+	 */
+	ZT_INLINE int latency() const noexcept { return _latency; }
+
 	/**
 	 * Check path aliveness
 	 *
@@ -122,6 +143,7 @@ private:
 	const int64_t _localSocket;
 	std::atomic<int64_t> _lastIn;
 	std::atomic<int64_t> _lastOut;
+	std::atomic<int> _latency;
 	const InetAddress _addr;
 	Meter<> _inMeter;
 	Meter<> _outMeter;

node/Peer.cpp

@@ -31,36 +31,37 @@ Peer::Peer(const RuntimeEnvironment *renv) : // NOLINT(cppcoreguidelines-pro-typ
 	_lastSentHello(),
 	_lastWhoisRequestReceived(0),
 	_lastEchoRequestReceived(0),
-	_lastPushDirectPathsReceived(0),
 	_lastProbeReceived(0),
 	_lastAttemptedP2PInit(0),
 	_lastPrioritizedPaths(0),
 	_lastAttemptedAggressiveNATTraversal(0),
-	_latency(-1),
 	_alivePathCount(0),
+	_probe(0),
 	_vProto(0),
 	_vMajor(0),
 	_vMinor(0),
 	_vRevision(0)
 {
-	Utils::memoryLock(_identityKey,sizeof(_identityKey));
 }
 
-Peer::~Peer()
+Peer::~Peer() // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
 {
-	Utils::memoryUnlock(_identityKey,sizeof(_identityKey));
-	Utils::burn(_identityKey,sizeof(_identityKey));
 }
 
 bool Peer::init(const Identity &peerIdentity)
 {
 	RWMutex::Lock l(_lock);
+
 	if (_id == peerIdentity)
 		return true;
 	_id = peerIdentity;
-	if (!RR->identity.agree(peerIdentity,_identityKey))
+
+	uint8_t ktmp[ZT_SYMMETRIC_KEY_SIZE];
+	if (!RR->identity.agree(peerIdentity,ktmp))
 		return false;
-	_incomingProbe = Protocol::createProbe(_id,RR->identity,_identityKey);
+	_identityKey.init(RR->node->now(),ktmp);
+	Utils::burn(ktmp,sizeof(ktmp));
+
 	return true;
 }
 
@@ -94,7 +95,12 @@ void Peer::received(
 			if (verb == Protocol::VERB_OK) {
 				l.writing();
 
-				// If the path list is full, replace the least recently active path.
+				// SECURITY: in the future we may not accept anything but OK(HELLO) to learn paths,
+				// but right now we accept any OK for backward compatibility. Note that OK will
+				// have been checked against expected packet IDs (see Expect.hpp) before we get here,
+				// and this guards against replay attacks.
+
+				// If the path list is full, replace the least recently active path. Otherwise append new path.
 				unsigned int newPathIdx = 0;
 				if (_alivePathCount >= ZT_MAX_PEER_NETWORK_PATHS) {
 					int64_t lastReceiveTimeMax = 0;
@@ -119,8 +125,11 @@ void Peer::received(
 				if (_paths[newPathIdx])
 					old = _paths[newPathIdx]->address();
 				_paths[newPathIdx] = path;
+
+				// Re-prioritize paths to include the new one.
 				_prioritizePaths(now);
 
+				// Remember most recently learned paths for future bootstrap attempts on restart.
 				Endpoint pathEndpoint(path->address());
 				_bootstrap[pathEndpoint.type()] = pathEndpoint;
 
@@ -130,29 +139,6 @@ void Peer::received(
 				RR->t->tryingNewPath(tPtr,0xb7747ddd,_id,path->address(),path->address(),packetId,(uint8_t)verb,_id,ZT_TRACE_TRYING_NEW_PATH_REASON_PACKET_RECEIVED_FROM_UNKNOWN_PATH);
 			}
 		}
-	} else if ((now - _lastAttemptedP2PInit) >= ZT_DIRECT_CONNECT_ATTEMPT_INTERVAL) {
-		_lastAttemptedP2PInit = now;
-		std::set<InetAddress> addrs;
-
-		// Addresses assigned to local system interfaces (as configured via the API).
-		std::vector<ZT_InterfaceAddress> localInterfaceAddresses(RR->node->localInterfaceAddresses());
-		for(std::vector<ZT_InterfaceAddress>::const_iterator i(localInterfaceAddresses.begin());i!=localInterfaceAddresses.end();++i)
-			addrs.insert(asInetAddress(i->address));
-
-		// We also advertise IPs reported to us by our peers in OK(HELLO) replies.
-		std::multimap<unsigned long,InetAddress> detectedAddresses(RR->sa->externalAddresses(now));
-		for(std::multimap<unsigned long,InetAddress>::const_reverse_iterator i(detectedAddresses.rbegin());i!=detectedAddresses.rend();++i) {
-			if (addrs.count(i->second) == 0) {
-				addrs.insert(i->second);
-				break;
-			}
-			if (i->first <= 1)
-				break;
-		}
-
-		if (!addrs.empty()) {
-			// TODO
-		}
 	}
 }
 
@@ -217,7 +203,7 @@ unsigned int Peer::sendNOP(void *const tPtr,const int64_t localSocket,const Inet
 	RR->identity.address().copyTo(ph.source);
 	ph.flags = 0;
 	ph.verb = Protocol::VERB_NOP;
-	Protocol::armor(outp,sizeof(Protocol::Header),_identityKey,this->cipher());
+	Protocol::armor(outp,sizeof(Protocol::Header),_identityKey.key(),this->cipher());
 	RR->node->putPacket(tPtr,localSocket,atAddress,outp.unsafeData,sizeof(Protocol::Header));
 	return sizeof(Protocol::Header);
 }
@@ -381,9 +367,11 @@ void Peer::tryToContactAt(void *const tPtr,const Endpoint &ep,const int64_t now,
 				// Chunk ports into chunks of 128 to try in few hundred millisecond intervals,
 				// abandoning attempts once there is at least one direct path.
 				{
+					static_assert((896 % ZT_PEER_BFG1024_PORT_SCAN_CHUNK_SIZE) == 0,"port scan chunk size doesn't evenly divide port list");
+					static_assert((1022 - 896) <= ZT_PEER_BFG1024_PORT_SCAN_CHUNK_SIZE,"port scan chunk size needs to be adjusted");
 					RWMutex::Lock l(_lock);
-					for (int i=0;i<896;i+=128)
+					for (int i=0;i<896;i+=ZT_PEER_BFG1024_PORT_SCAN_CHUNK_SIZE)
-						_contactQueue.push_back(_ContactQueueItem(ep.inetAddr(),ports + i,ports + i + 128,1)); // NOLINT(hicpp-use-emplace,modernize-use-emplace)
+						_contactQueue.push_back(_ContactQueueItem(ep.inetAddr(),ports + i,ports + i + ZT_PEER_BFG1024_PORT_SCAN_CHUNK_SIZE,1)); // NOLINT(hicpp-use-emplace,modernize-use-emplace)
 					_contactQueue.push_back(_ContactQueueItem(ep.inetAddr(),ports + 896,ports + 1022,1)); // NOLINT(hicpp-use-emplace,modernize-use-emplace)
 				}
 			} else {
@@ -424,11 +412,11 @@ void Peer::alarm(void *tPtr,const int64_t now)
 
 		_ContactQueueItem &qi2 = _contactQueue.front();
 		qi.address = qi2.address;
-		qi.ports.swap(qi2.ports);
+		qi.ports = qi2.ports;
 		qi.alivePathThreshold = qi2.alivePathThreshold;
 		_contactQueue.pop_front();
 
-		for(std::list<_ContactQueueItem>::iterator q(_contactQueue.begin());q!=_contactQueue.end();) { // NOLINT(hicpp-use-auto,modernize-use-auto)
+		for(std::list< _ContactQueueItem,Utils::Mallocator<_ContactQueueItem> >::iterator q(_contactQueue.begin());q!=_contactQueue.end();) { // NOLINT(hicpp-use-auto,modernize-use-auto)
 			if (_alivePathCount >= q->alivePathThreshold)
 				_contactQueue.erase(q++);
 			else ++q;
@@ -437,21 +425,20 @@ void Peer::alarm(void *tPtr,const int64_t now)
 		stillHaveContactQueueItems = !_contactQueue.empty();
 	}
 
-	if (_vProto >= 11) {
+	if ((_vProto >= 11) && (_probe != 0)) {
-		uint64_t outgoingProbe = Protocol::createProbe(RR->identity,_id,_identityKey);
 		if (qi.ports.empty()) {
-			RR->node->putPacket(tPtr,-1,qi.address,&outgoingProbe,ZT_PROTO_PROBE_LENGTH);
+			RR->node->putPacket(tPtr,-1,qi.address,&_probe,ZT_PROTO_PROBE_LENGTH);
 		} else {
-			for (std::vector<uint16_t>::iterator p(qi.ports.begin()); p != qi.ports.end(); ++p) { // NOLINT(hicpp-use-auto,modernize-use-auto)
+			for (FCV<uint16_t,ZT_PEER_BFG1024_PORT_SCAN_CHUNK_SIZE>::iterator p(qi.ports.begin()); p != qi.ports.end(); ++p) { // NOLINT(hicpp-use-auto,modernize-use-auto)
 				qi.address.setPort(*p);
-				RR->node->putPacket(tPtr,-1,qi.address,&outgoingProbe,ZT_PROTO_PROBE_LENGTH);
+				RR->node->putPacket(tPtr,-1,qi.address,&_probe,ZT_PROTO_PROBE_LENGTH);
 			}
 		}
 	} else {
 		if (qi.ports.empty()) {
 			this->sendNOP(tPtr,-1,qi.address,now);
 		} else {
-			for (std::vector<uint16_t>::iterator p(qi.ports.begin()); p != qi.ports.end(); ++p) { // NOLINT(hicpp-use-auto,modernize-use-auto)
+			for (FCV<uint16_t,ZT_PEER_BFG1024_PORT_SCAN_CHUNK_SIZE>::iterator p(qi.ports.begin()); p != qi.ports.end(); ++p) { // NOLINT(hicpp-use-auto,modernize-use-auto)
 				qi.address.setPort(*p);
 				this->sendNOP(tPtr,-1,qi.address,now);
 			}
@@ -466,21 +453,17 @@ int Peer::marshal(uint8_t data[ZT_PEER_MARSHAL_SIZE_MAX]) const noexcept
 {
 	data[0] = 0; // serialized peer version
 
-	// For faster unmarshaling on large nodes the long-term secret key is cached. It's
-	// encrypted with a symmetric key derived from a hash of the local node's identity
-	// secrets, so the local node's address is also included. That way the unmarshal
-	// code can check this address and not use this cached key if the local identity has
-	// changed. In that case agreement must be executed again.
-	RR->identity.address().copyTo(data + 1);
-	RR->localCacheSymmetric.encrypt(_identityKey,data + 6);
-	RR->localCacheSymmetric.encrypt(_identityKey + 16,data + 22);
-
 	RWMutex::RLock l(_lock);
 
-	int s = _id.marshal(data + 38,false);
+	int s = _identityKey.marshal(RR->localCacheSymmetric,data + 1);
-	if (s <= 0)
+	if (s < 0)
-		return s;
+		return -1;
-	int p = s + 38;
+	int p = 1 + s;
+
+	s = _id.marshal(data + p,false);
+	if (s < 0)
+		return -1;
+	p += s;
 
 	s = _locator.marshal(data + p);
 	if (s <= 0)
@@ -491,7 +474,7 @@ int Peer::marshal(uint8_t data[ZT_PEER_MARSHAL_SIZE_MAX]) const noexcept
 	for(std::map< Endpoint::Type,Endpoint >::const_iterator i(_bootstrap.begin());i!=_bootstrap.end();++i) { // NOLINT(modernize-loop-convert,hicpp-use-auto,modernize-use-auto)
 		s = i->second.marshal(data + p);
 		if (s <= 0)
-			return s;
+			return -1;
 		p += s;
 	}
 
@@ -512,68 +495,68 @@ int Peer::marshal(uint8_t data[ZT_PEER_MARSHAL_SIZE_MAX]) const noexcept
 
 int Peer::unmarshal(const uint8_t *restrict data,const int len) noexcept
 {
-	int p;
+	RWMutex::Lock l(_lock);
-	bool mustRecomputeSecret;
 
-	{
+	if ((len <= 1) || (data[0] != 0))
-		RWMutex::Lock l(_lock);
+		return -1;
 
-		if ((len <= 38) || (data[0] != 0))
+	int s = _identityKey.unmarshal(RR->localCacheSymmetric,data + 1,len);
+	if (s < 0)
+		return -1;
+	int p = 1 + s;
+
+	// If the identity key did not pass verification, it may mean that our local
+	// identity has changed. In this case we do not have to forget everything about
+	// the peer but we must generate a new identity key by key agreement with our
+	// new identity.
+	if (!_identityKey) {
+		uint8_t tmp[ZT_SYMMETRIC_KEY_SIZE];
+		if (!RR->identity.agree(_id,tmp))
 			return -1;
+		_identityKey.init(RR->node->now(),tmp);
+		Utils::burn(tmp,sizeof(tmp));
+	}
 
-		if (Address(data + 1) == RR->identity.address()) {
+	// These are ephemeral and start out as NIL after unmarshal.
-			RR->localCacheSymmetric.decrypt(data + 6,_identityKey);
+	_ephemeralKeys[0].clear();
-			RR->localCacheSymmetric.decrypt(data + 22,_identityKey + 16);
+	_ephemeralKeys[1].clear();
-			mustRecomputeSecret = false;
-		} else {
-			mustRecomputeSecret = true; // can't use cached key if local identity has changed
-		}
 
-		int s = _id.unmarshal(data + 38,len - 38);
+	s = _id.unmarshal(data + 38,len - 38);
-		if (s <= 0)
+	if (s < 0)
-			return s;
+		return s;
-		p = s + 38;
+	p += s;
-		s = _locator.unmarshal(data + p,len - p);
+
-		if (s <= 0)
+	s = _locator.unmarshal(data + p,len - p);
+	if (s < 0)
+		return s;
+	p += s;
+
+	if (p >= len)
+		return -1;
+	const unsigned int bootstrapCount = data[p++];
+	if (bootstrapCount > ZT_MAX_PEER_NETWORK_PATHS)
+		return -1;
+	_bootstrap.clear();
+	for(unsigned int i=0;i<bootstrapCount;++i) {
+		Endpoint tmp;
+		s = tmp.unmarshal(data + p,len - p);
+		if (s < 0)
 			return s;
 		p += s;
-
+		_bootstrap[tmp.type()] = tmp;
-		if (p >= len)
-			return -1;
-		const unsigned int bootstrapCount = data[p++];
-		if (bootstrapCount > ZT_MAX_PEER_NETWORK_PATHS)
-			return -1;
-		_bootstrap.clear();
-		for(unsigned int i=0;i<bootstrapCount;++i) {
-			Endpoint tmp;
-			s = tmp.unmarshal(data + p,len - p);
-			if (s <= 0)
-				return s;
-			p += s;
-			_bootstrap[tmp.type()] = tmp;
-		}
-
-		if ((p + 10) > len)
-			return -1;
-
-		_vProto = Utils::loadBigEndian<uint16_t>(data + p); p += 2;
-		_vMajor = Utils::loadBigEndian<uint16_t>(data + p); p += 2;
-		_vMinor = Utils::loadBigEndian<uint16_t>(data + p); p += 2;
-		_vRevision = Utils::loadBigEndian<uint16_t>(data + p); p += 2;
-		p += 2 + (int)Utils::loadBigEndian<uint16_t>(data + p);
-
-		if (p > len)
-			return -1;
 	}
 
-	if (mustRecomputeSecret) {
+	_probe = 0; // ephemeral token, reset on unmarshal
-		if (!RR->identity.agree(_id,_identityKey))
-			return -1;
-	}
 
-	_incomingProbe = Protocol::createProbe(_id,RR->identity,_identityKey);
+	if ((p + 10) > len)
+		return -1;
+	_vProto = Utils::loadBigEndian<uint16_t>(data + p); p += 2;
+	_vMajor = Utils::loadBigEndian<uint16_t>(data + p); p += 2;
+	_vMinor = Utils::loadBigEndian<uint16_t>(data + p); p += 2;
+	_vRevision = Utils::loadBigEndian<uint16_t>(data + p); p += 2;
+	p += 2 + (int)Utils::loadBigEndian<uint16_t>(data + p);
 
-	return p;
+	return (p > len) ? -1 : p;
 }
 
 struct _PathPriorityComparisonOperator

node/Peer.hpp

@@ -27,14 +27,14 @@
 #include "Endpoint.hpp"
 #include "Locator.hpp"
 #include "Protocol.hpp"
-
+#include "AES.hpp"
-#include <vector>
+#include "SymmetricKey.hpp"
-#include <list>
+#include "Containers.hpp"
-#include <set>
-#include <map>
 
 // version, identity, locator, bootstrap, version info, length of any additional fields
-#define ZT_PEER_MARSHAL_SIZE_MAX (1 + ZT_ADDRESS_LENGTH + ZT_PEER_SECRET_KEY_LENGTH + ZT_IDENTITY_MARSHAL_SIZE_MAX + ZT_LOCATOR_MARSHAL_SIZE_MAX + 1 + (ZT_MAX_PEER_NETWORK_PATHS * ZT_ENDPOINT_MARSHAL_SIZE_MAX) + (2*4) + 2)
+#define ZT_PEER_MARSHAL_SIZE_MAX (1 + ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX + ZT_IDENTITY_MARSHAL_SIZE_MAX + ZT_LOCATOR_MARSHAL_SIZE_MAX + 1 + (ZT_MAX_PEER_NETWORK_PATHS * ZT_ENDPOINT_MARSHAL_SIZE_MAX) + (2*4) + 2)
+
+#define ZT_PEER_BFG1024_PORT_SCAN_CHUNK_SIZE 128
 
 namespace ZeroTier {
 
@@ -224,18 +224,14 @@ public:
 	void resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,int64_t now);
 
 	/**
-	 * Method called to update peer latency with a new measurement.
+	 * @return All currently memorized bootstrap endpoints
-	 *
-	 * @param l New latency measurment (in milliseconds)
 	 */
-	ZT_INLINE void updateLatency(const unsigned int measurement) noexcept
+	ZT_INLINE FCV<Endpoint,ZT_MAX_PEER_NETWORK_PATHS> bootstrap() const noexcept
 	{
-		int l = _latency;
+		FCV<Endpoint,ZT_MAX_PEER_NETWORK_PATHS> r;
-		if (l > 0) {
+		for(std::map< Endpoint::Type,Endpoint,std::less<Endpoint::Type>,Utils::Mallocator< std::pair<const Endpoint::Type,Endpoint> > >::const_iterator i(_bootstrap.begin());i!=_bootstrap.end();++i) // NOLINT(hicpp-use-auto,modernize-use-auto,modernize-loop-convert)
-			_latency = (l + (int)measurement) / 2;
+			r.push_back(i->second);
-		} else {
+		return r;
-			_latency = (int)measurement;
-		}
 	}
 
 	/**
@@ -255,14 +251,22 @@ public:
 	ZT_INLINE int64_t lastReceive() const noexcept { return _lastReceive; }
 
 	/**
-	 * @return Latency in milliseconds of best/aggregate path or 0xffff if unknown
+	 * @return Average latency of all direct paths or -1 if no direct paths or unknown
 	 */
-	ZT_INLINE unsigned int latency() const noexcept { return _latency; }
+	ZT_INLINE int latency() const noexcept
-
+	{
-	/**
+		int ltot = 0;
-	 * @return 256-bit secret symmetric encryption key
+		int lcnt = 0;
-	 */
+		RWMutex::RLock l(_lock);
-	ZT_INLINE const unsigned char *key() const noexcept { return _identityKey; }
+		for(unsigned int i=0;i<_alivePathCount;++i) {
+			int lat = _paths[i]->latency();
+			if (lat > 0) {
+				ltot += lat;
+				++lcnt;
+			}
+		}
+		return (ltot > 0) ? (lcnt / ltot) : -1;
+	}
 
 	/**
 	 * @return Preferred cipher suite for normal encrypted P2P communication
@@ -272,11 +276,6 @@ public:
 		return ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012;
 	}
 
-	/**
-	 * @return Incoming probe packet (in big-endian byte order)
-	 */
-	ZT_INLINE uint64_t incomingProbe() const noexcept { return _incomingProbe; }
-
 	/**
 	 * Set the currently known remote version of this peer's client
 	 *
@@ -352,18 +351,6 @@ public:
 		return false;
 	}
 
-	/**
-	 * Rate limit gate for inbound PUSH_DIRECT_PATHS requests
-	 */
-	ZT_INLINE bool rateGateInboundPushDirectPaths(const int64_t now) noexcept
-	{
-		if ((now - _lastPushDirectPathsReceived) >= ZT_DIRECT_CONNECT_ATTEMPT_INTERVAL) {
-			_lastPushDirectPathsReceived = now;
-			return true;
-		}
-		return false;
-	}
-
 	/**
 	 * Rate limit attempts in response to incoming short probe packets
 	 */
@@ -391,24 +378,43 @@ public:
 private:
 	void _prioritizePaths(int64_t now);
 
-	// The long-lived identity key resulting from agreement between our identity and this peer's identity.
+	const RuntimeEnvironment *RR;
-	uint8_t _identityKey[ZT_PEER_SECRET_KEY_LENGTH];
 
 	// Read/write mutex for non-atomic non-const fields.
 	RWMutex _lock;
 
-	const RuntimeEnvironment *RR;
+	// The permanent identity key resulting from agreement between our identity and this peer's identity.
+	SymmetricKey< AES,0,0 > _identityKey;
 
-	// The last time various things happened, for rate limiting and periodic events.
+	// Most recently successful (for decrypt) ephemeral key and one previous key.
+	SymmetricKey< AES,ZT_SYMMETRIC_KEY_TTL,ZT_SYMMETRIC_KEY_TTL_MESSAGES > _ephemeralKeys[2];
+
+	Identity _id;
+	Locator _locator;
+
+	// the last time something was sent or received from this peer (direct or indirect).
 	std::atomic<int64_t> _lastReceive;
 	std::atomic<int64_t> _lastSend;
+
+	// The last time we sent a full HELLO to this peer.
 	int64_t _lastSentHello; // only checked while locked
+
+	// The last time a WHOIS request was received from this peer (anti-DOS / anti-flood).
 	std::atomic<int64_t> _lastWhoisRequestReceived;
+
+	// The last time an ECHO request was received from this peer (anti-DOS / anti-flood).
 	std::atomic<int64_t> _lastEchoRequestReceived;
-	std::atomic<int64_t> _lastPushDirectPathsReceived;
+
+	// The last time a probe was received from this peer (for anti-DOS / anti-flood use).
 	std::atomic<int64_t> _lastProbeReceived;
+
+	// The last time we tried to init P2P connectivity with this peer.
 	std::atomic<int64_t> _lastAttemptedP2PInit;
+
+	// The last time we sorted paths in order of preference. (This happens pretty often.)
 	std::atomic<int64_t> _lastPrioritizedPaths;
+
+	// The last time we opened a can of whupass against this peer's NAT (if enabled).
 	std::atomic<int64_t> _lastAttemptedAggressiveNATTraversal;
 
 	// Meters measuring actual bandwidth in, out, and relayed via this peer (mostly if this is a root).
@@ -419,9 +425,6 @@ private:
 	// For SharedPtr<>
 	std::atomic<int> __refCount;
 
-	// Milliseconds of latency over best path or -1 if unknown.
-	std::atomic<int> _latency;
-
 	// Direct paths sorted in descending order of preference.
 	SharedPtr<Path> _paths[ZT_MAX_PEER_NETWORK_PATHS];
 
@@ -441,17 +444,16 @@ private:
 			ports(),
 			alivePathThreshold(apt) {}
 		InetAddress address;
-		std::vector<uint16_t> ports; // if non-empty try these ports, otherwise use the one in address
+		FCV<uint16_t,ZT_PEER_BFG1024_PORT_SCAN_CHUNK_SIZE> ports; // if non-empty try these ports, otherwise use the one in address
 		unsigned int alivePathThreshold; // skip and forget if alive path count is >= this
 	};
-	std::list<_ContactQueueItem> _contactQueue;
+	List<_ContactQueueItem> _contactQueue;
 
 	// Remembered addresses by endpoint type (std::map is smaller for only a few keys).
-	std::map< Endpoint::Type,Endpoint > _bootstrap;
+	std::map< Endpoint::Type,Endpoint,std::less<Endpoint::Type>,Utils::Mallocator< std::pair<const Endpoint::Type,Endpoint> > > _bootstrap;
 
-	Identity _id;
+	// 32-bit probe or 0 if unknown.
-	uint64_t _incomingProbe;
+	uint32_t _probe;
-	Locator _locator;
 
 	uint16_t _vProto;
 	uint16_t _vMajor;

node/Protocol.cpp

@@ -27,27 +27,14 @@
 namespace ZeroTier {
 namespace Protocol {
 
-// The counter used to assign packet IDs / cryptographic nonces.
+void armor(Buf &pkt,int packetSize,const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],uint8_t cipherSuite) noexcept
-std::atomic<uint64_t> _s_packetIdCtr((uint64_t)time(nullptr) << 32U);
-
-uint64_t createProbe(const Identity &sender,const Identity &recipient,const uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]) noexcept
 {
-	uint8_t tmp[ZT_IDENTITY_HASH_SIZE + ZT_IDENTITY_HASH_SIZE];
+	Protocol::Header &ph = pkt.as<Protocol::Header>(); // NOLINT(hicpp-use-auto,modernize-use-auto)
-	Utils::copy<ZT_IDENTITY_HASH_SIZE>(tmp,sender.fingerprint().hash());
-	Utils::copy<ZT_IDENTITY_HASH_SIZE>(tmp + ZT_IDENTITY_HASH_SIZE,recipient.fingerprint().hash());
-	uint64_t hash[6];
-	SHA384(hash,tmp,sizeof(tmp),key,ZT_PEER_SECRET_KEY_LENGTH);
-	return hash[0];
-}
-
-void armor(Buf &pkt,int packetSize,const uint8_t key[ZT_PEER_SECRET_KEY_LENGTH],uint8_t cipherSuite) noexcept
-{
-	Protocol::Header &ph = pkt.as<Protocol::Header>();
 	ph.flags = (ph.flags & 0xc7U) | ((cipherSuite << 3U) & 0x38U); // flags: FFCCCHHH where CCC is cipher
 
 	switch(cipherSuite) {
 		case ZT_PROTO_CIPHER_SUITE__POLY1305_NONE: {
-			uint8_t perPacketKey[ZT_PEER_SECRET_KEY_LENGTH];
+			uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
 			salsa2012DeriveKey(key,perPacketKey,pkt,packetSize);
 			Salsa20 s20(perPacketKey,&ph.packetId);
 
@@ -62,7 +49,7 @@ void armor(Buf &pkt,int packetSize,const uint8_t key[ZT_PEER_SECRET_KEY_LENGTH],
 		} break;
 
 		case ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012: {
-			uint8_t perPacketKey[ZT_PEER_SECRET_KEY_LENGTH];
+			uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
 			salsa2012DeriveKey(key,perPacketKey,pkt,packetSize);
 			Salsa20 s20(perPacketKey,&ph.packetId);
 

node/Protocol.hpp

@@ -23,8 +23,6 @@
 #include "Address.hpp"
 #include "Identity.hpp"
 
-// TODO: mlock
-
 /*
  * Core ZeroTier protocol packet formats ------------------------------------------------------------------------------
  *
@@ -208,7 +206,7 @@
 /**
  * Length of a probe packet
  */
-#define ZT_PROTO_PROBE_LENGTH 8
+#define ZT_PROTO_PROBE_LENGTH 4
 
 /**
  * Index at which packet fragment payload starts
@@ -364,8 +362,8 @@ enum Verb
 	 *   INSTANCE_ID - a 64-bit unique value generated on each node start
 	 *   EPHEMERAL_C25519 - an ephemeral Curve25519 public key
 	 *   EPHEMERAL_P384 - an ephemeral NIST P-384 public key
-	 *   EPHEMERAL_REVISION - 64-bit monotonically increasing per-instance counter
 	 *   LOCATOR - signed record enumerating this node's trusted contact points
+	 *   PROBE_TOKEN - 32-bit token that can be used to try to contact this peer
 	 *
 	 * The following optional fields may also be present:
 	 *
@@ -375,9 +373,8 @@ enum Verb
 	 *   LOC_X, LOC_Y, LOC_Z - location relative to the nearest large center of mass
 	 *   PEER_LOC_X, PEER_LOC_Y, PEER_LOC_Z - where sender thinks peer is located
 	 *   SOFTWARE_VENDOR - short name or description of vendor, such as a URL
-	 *   SOFTWARE_VERSION - major, minor, revision, and build, and 16-bit integers
+	 *   SOFTWARE_VERSION - major, minor, revision, and build (packed 64-bit int)
 	 *   PHYSICAL_DEST - serialized Endpoint to which this message was sent
-	 *   VIRTUAL_DEST - ZeroTier address of first hop (if first hop wasn't destination)
 	 *   COMPLIANCE - bit mask containing bits for e.g. a FIPS-compliant node
 	 *
 	 * A valid and successfully authenticated HELLO will generate the following
@@ -1098,26 +1095,6 @@ static ZT_INLINE void salsa2012DeriveKey(const uint8_t *const in,uint8_t *const
 #endif
 }
 
-/**
- * Create a short probe packet for probing a recipient for e.g. NAT traversal and path setup
- *
- * @param sender Sender identity
- * @param recipient Recipient identity
- * @param key Long-term shared secret key resulting from sender and recipient agreement
- * @return Probe packed into 64-bit integer (in big-endian byte order)
- */
-uint64_t createProbe(const Identity &sender,const Identity &recipient,const uint8_t key[ZT_PEER_SECRET_KEY_LENGTH]) noexcept;
-
-// Do not use directly
-extern std::atomic<uint64_t> _s_packetIdCtr;
-
-/**
- * Get a packet ID (and nonce) for a new packet
- *
- * @return Next packet ID
- */
-static ZT_INLINE uint64_t getPacketId() noexcept { return ++_s_packetIdCtr; }
-
 /**
  * Encrypt and compute packet MAC
  *
@@ -1126,7 +1103,7 @@ static ZT_INLINE uint64_t getPacketId() noexcept { return ++_s_packetIdCtr; }
  * @param key Key to use for encryption (not per-packet key)
  * @param cipherSuite Cipher suite to use for AEAD encryption or just MAC
  */
-void armor(Buf &pkt,int packetSize,const uint8_t key[ZT_PEER_SECRET_KEY_LENGTH],uint8_t cipherSuite) noexcept;
+void armor(Buf &pkt,int packetSize,const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],uint8_t cipherSuite) noexcept;
 
 /**
  * Attempt to compress packet payload

node/Revocation.hpp

@@ -45,6 +45,8 @@ class Revocation : public Credential
 	friend class Credential;
 
 public:
+	static constexpr ZT_CredentialType credentialType() noexcept { return ZT_CREDENTIAL_TYPE_REVOCATION; }
+
 	ZT_INLINE Revocation() noexcept { memoryZero(this); } // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
 
 	/**

node/SelfAwareness.cpp

@@ -96,9 +96,9 @@ void SelfAwareness::clean(int64_t now)
 	}
 }
 
-std::multimap<unsigned long,InetAddress> SelfAwareness::externalAddresses(const int64_t now) const
+SelfAwareness::ExternalAddressList SelfAwareness::externalAddresses(const int64_t now) const
 {
-	std::multimap<unsigned long,InetAddress> r;
+	SelfAwareness::ExternalAddressList r;
 	Map<InetAddress,unsigned long> counts;
 
 	{

node/SelfAwareness.hpp

@@ -16,7 +16,7 @@
 
 #include "Constants.hpp"
 #include "InetAddress.hpp"
-#include "Map.hpp"
+#include "Containers.hpp"
 #include "Address.hpp"
 #include "Mutex.hpp"
 
@@ -35,6 +35,8 @@ class RuntimeEnvironment;
 class SelfAwareness
 {
 public:
+	typedef std::multimap< unsigned long,InetAddress,std::less<unsigned long>,Utils::Mallocator< std::pair<const unsigned long,InetAddress> > > ExternalAddressList;
+
 	explicit SelfAwareness(const RuntimeEnvironment *renv);
 
 	/**
@@ -62,7 +64,7 @@ public:
 	 * @param now Current time
 	 * @return Map of count to IP/port representing how many endpoints reported each address
 	 */
-	std::multimap<unsigned long,InetAddress> externalAddresses(int64_t now) const;
+	ExternalAddressList externalAddresses(int64_t now) const;
 
 private:
 	struct PhySurfaceKey

node/SymmetricKey.hpp

@@ -0,0 +1,268 @@
+/*
+ * Copyright (c)2013-2020 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2024-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#ifndef ZT_SYMMETRICKEY_HPP
+#define ZT_SYMMETRICKEY_HPP
+
+#include "Constants.hpp"
+#include "Utils.hpp"
+#include "InetAddress.hpp"
+
+namespace ZeroTier {
+
+#define ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX 52
+
+/**
+ * Container for symmetric keys and ciphers initialized with them
+ *
+ * This container is responsible for tracking key TTL to maintain it
+ * below our security bounds and tell us when it's time to re-key.
+ *
+ * Set TTL and TTLM to 0 for permanent keys. These still track uses
+ * but do not signal expiration.
+ *
+ * @tparam C Cipher to embed (must accept key in constructor and/or init() method)
+ * @tparam TTL Maximum time to live in milliseconds or 0 for a permanent key with unlimited TTL
+ * @tparam TTLM Maximum time to live in messages or 0 for a permanent key with unlimited TTL
+ */
+template<typename C,int64_t TTL,uint64_t TTLM>
+class SymmetricKey
+{
+public:
+	/**
+	 * Symmetric cipher keyed with this key
+	 */
+	const C cipher;
+
+	/**
+	 * Construct an uninitialized symmetric key container
+	 */
+	ZT_INLINE SymmetricKey() noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init,hicpp-use-equals-default,modernize-use-equals-default)
+		cipher(),
+		_ts(0),
+		_nonceBase(0),
+		_odometer(0)
+	{
+		Utils::memoryLock(_k,sizeof(_k));
+	}
+
+	/**
+	 * Construct a new symmetric key
+	 *
+	 * @param ts Current time (must still be given for permanent keys even though there is no expiry checking)
+	 * @param key 32-byte / 256-bit key
+	 */
+	explicit ZT_INLINE SymmetricKey(const int64_t ts,const void *const key) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+		cipher(key),
+		_ts(ts),
+		_nonceBase((uint64_t)ts << 22U), // << 22 to shift approximately the seconds since epoch into the most significant 32 bits
+		_odometer(0)
+	{
+		Utils::memoryLock(_k,sizeof(_k));
+		Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(_k,key);
+	}
+
+	ZT_INLINE SymmetricKey(const SymmetricKey &k) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+		cipher(k._k),
+		_ts(k.ts),
+		_nonceBase(k._nonceBase),
+		_odometer(k._odometer)
+	{
+		Utils::memoryLock(_k,sizeof(_k));
+		Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(_k,k._k);
+	}
+
+	ZT_INLINE ~SymmetricKey() noexcept
+	{
+		Utils::burn(_k,sizeof(_k));
+		Utils::memoryUnlock(_k,sizeof(_k));
+	}
+
+	ZT_INLINE SymmetricKey &operator=(const SymmetricKey &k) noexcept
+	{
+		if (&k != this) {
+			cipher.init(k._k);
+			_ts = k._ts;
+			_nonceBase = k._nonceBase;
+			_odometer = k._odometer;
+			Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(_k,k._k);
+		}
+		return *this;
+	}
+
+	/**
+	 * Initialize or change the key wrapped by this SymmetricKey object
+	 *
+	 * If the supplied key is identical to the current key, no change occurs and false is returned.
+	 *
+	 * @param ts Current time
+	 * @param key 32-byte / 256-bit key
+	 * @return True if the symmetric key was changed
+	 */
+	ZT_INLINE bool init(const int64_t ts,const void *const key) noexcept
+	{
+		if ((_ts > 0)&&(memcmp(_k,key,ZT_SYMMETRIC_KEY_SIZE) == 0))
+			return false;
+		cipher.init(key);
+		_ts = ts;
+		_nonceBase = (uint64_t)ts << 22U; // << 22 to shift approximately the seconds since epoch into the most significant 32 bits;
+		_odometer = 0;
+		Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(_k,key);
+		return true;
+	}
+
+	/**
+	 * Clear key and set to NIL value (boolean evaluates to false)
+	 */
+	ZT_INLINE void clear() noexcept
+	{
+		_ts = 0;
+		_nonceBase = 0;
+		_odometer = 0;
+		Utils::zero<ZT_SYMMETRIC_KEY_SIZE>(_k);
+	}
+
+	/**
+	 * Check whether this symmetric key may be expiring soon
+	 *
+	 * @param now Current time
+	 * @return True if re-keying should happen
+	 */
+	ZT_INLINE bool expiringSoon(const int64_t now) const noexcept
+	{
+		return (TTL > 0) && ( ((now - _ts) >= (TTL / 2)) || (_odometer >= (TTLM / 2)) );
+	}
+
+	/**
+	 * Check whether this symmetric key is expired due to too much time or too many messages
+	 *
+	 * @param now Current time
+	 * @return True if this symmetric key should no longer be used
+	 */
+	ZT_INLINE bool expired(const int64_t now) const noexcept
+	{
+		return (TTL > 0) && ( ((now - _ts) >= TTL) || (_odometer >= TTLM) );
+	}
+
+	/**
+	 * @return True if this is a never-expiring key, such as the identity key created by identity key agreement
+	 */
+	constexpr bool permanent() const noexcept
+	{
+		return TTL == 0;
+	}
+
+	/**
+	 * Get the raw key that was used to initialize the cipher.
+	 *
+	 * @return 32-byte / 256-bit symmetric key
+	 */
+	ZT_INLINE const uint8_t *key() const noexcept
+	{
+		return _k;
+	}
+
+	/**
+	 * Advance usage counter by one and return the next unique initialization vector for a new message.
+	 *
+	 * @return Next unique IV for next message
+	 */
+	ZT_INLINE uint64_t nextMessageIv() noexcept
+	{
+		return _nonceBase + _odometer++;
+	}
+
+	/**
+	 * @return True if this object is not NIL
+	 */
+	ZT_INLINE operator bool() const noexcept { return (_ts > 0); } // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
+
+	static constexpr int marshalSizeMax() noexcept { return ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX; }
+
+	/**
+	 * Marshal with encryption at rest
+	 *
+	 * @tparam MC Cipher type (AES in our code) to use for encryption at rest
+	 * @param keyEncCipher Initialized cipher
+	 * @param data Destination for marshaled key
+	 * @return Bytes written or -1 on error
+	 */
+	template<typename MC>
+	ZT_INLINE int marshal(const MC &keyEncCipher,uint8_t data[ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX]) const noexcept
+	{
+		Utils::storeBigEndian<uint64_t>(data,(uint64_t)_ts);
+		Utils::storeBigEndian<uint64_t>(data + 8,_odometer.load());
+		Utils::storeBigEndian<uint32_t>(data + 16,Utils::fnv1a32(_k,sizeof(_k)));
+
+		// Key encryption at rest is CBC using the last 32 bits of the timestamp, the odometer,
+		// and the FNV1a checksum as a 128-bit IV. A duplicate IV wouldn't matter much anyway since
+		// keys should be unique. Simple ECB would be fine as they also have no structure, but this
+		// looks better.
+		uint8_t tmp[16];
+		for(int i=0;i<16;++i)
+			tmp[i] = data[i + 4] ^ _k[i];
+		keyEncCipher.encrypt(tmp,data + 20);
+		for(int i=0;i<16;++i)
+			tmp[i] = data[i + 20] ^ _k[i + 16];
+		keyEncCipher.encrypt(tmp,data + 36);
+
+		return ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX;
+	}
+
+	/**
+	 * Unmarshal, decrypt, and verify key checksum
+	 *
+	 * Key checksum verification failure results in the SymmetricKey being zeroed out to its
+	 * nil value, but the bytes read are still returned. The caller must check this if it
+	 * requires the key to be present and verified.
+	 *
+	 * @tparam MC Cipher type (AES in our code) to use for encryption at rest
+	 * @param keyDecCipher Initialized cipher for decryption
+	 * @param data Source to read
+	 * @param len Bytes remaining at source
+	 * @return Bytes read from source
+	 */
+	template<typename MC>
+	ZT_INLINE int unmarshal(const MC &keyDecCipher,const uint8_t *restrict data,int len) noexcept
+	{
+		if (len < ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX)
+			return -1;
+
+		_ts = (int64_t)Utils::loadBigEndian<uint64_t>(data);
+		_odometer = (uint64_t)Utils::loadBigEndian<uint64_t>(data + 8);
+		const uint32_t fnv = Utils::loadBigEndian<uint32_t>(data + 16); // NOLINT(hicpp-use-auto,modernize-use-auto)
+
+		uint8_t tmp[16];
+		keyDecCipher.decrypt(data + 20,tmp);
+		for(int i=0;i<16;++i)
+			_k[i] = data[i + 4] ^ tmp[i];
+		keyDecCipher.decrypt(data + 36,tmp);
+		for(int i=0;i<16;++i)
+			_k[i + 16] = data[i + 20] ^ tmp[i];
+
+		if (Utils::fnv1a32(_k,sizeof(_k)) != fnv)
+			clear();
+
+		return ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX;
+	}
+
+private:
+	int64_t _ts;
+	uint64_t _nonceBase;
+	std::atomic<uint64_t> _odometer;
+	uint8_t _k[ZT_SYMMETRIC_KEY_SIZE];
+};
+
+} // namespace ZeroTier
+
+#endif

node/Tag.hpp

@@ -53,6 +53,8 @@ class Tag : public Credential
 	friend class Credential;
 
 public:
+	static constexpr ZT_CredentialType credentialType() noexcept { return ZT_CREDENTIAL_TYPE_TAG; }
+
 	ZT_INLINE Tag() noexcept { memoryZero(this); } // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
 
 	/**

node/Tests.cpp

@@ -38,7 +38,7 @@
 #include "SHA512.hpp"
 #include "Defragmenter.hpp"
 #include "Fingerprint.hpp"
-#include "Map.hpp"
+#include "Containers.hpp"
 
 #include <cstdint>
 #include <cstring>
@@ -534,8 +534,8 @@ extern "C" const char *ZTT_general()
 			int64_t ts = now();
 			for(int k=0;k<50000;++k) {
 				++messageId;
-				FCV<Buf::Slice,16> message;
+				FCV<Buf::Slice,ZT_MAX_PACKET_FRAGMENTS> message;
-				FCV<Buf::Slice,16> ref;
+				FCV<Buf::Slice,ZT_MAX_PACKET_FRAGMENTS> ref;
 
 				int frags = 1 + (int)(Utils::random() % 16);
 				int skip = ((k & 3) == 1) ? -1 : (int)(Utils::random() % frags);
@@ -559,7 +559,7 @@ extern "C" const char *ZTT_general()
 							ZT_T_PRINTF("FAILED (message prematurely complete)" ZT_EOL_S);
 							return "Defragmenter test failed: message prematurely complete";
 						}
-						switch (defrag.assemble(messageId,message,ref[f].b,ref[f].s,ref[f].e - ref[f].s,f,frags,ts++,nullvia,0)) {
+						switch (defrag.assemble(messageId,message,ref[f].b,ref[f].s,ref[f].e - ref[f].s,f,frags,ts++,nullvia)) {
 							case Defragmenter<>::OK:
 								break;
 							case Defragmenter<>::COMPLETE:

node/Tests.h

@@ -43,14 +43,8 @@
 
 #ifdef ZT_ENABLE_TESTS
 
-#ifdef __cplusplus
+#include <stdint.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
-#include <cstdint>
+#include <stdio.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
-#include <cstdio>
-extern "C" {
-#else
-#include <stdint.h>
-#include <stdio.h>
-#endif
 
 #ifndef ZT_T_PRINTF
 #define ZT_T_PRINTF(fmt,...) printf((fmt),##__VA_ARGS__),fflush(stdout)

node/Topology.hpp

@@ -30,7 +30,7 @@
 #include "SharedPtr.hpp"
 #include "ScopedPtr.hpp"
 #include "Fingerprint.hpp"
-#include "Map.hpp"
+#include "Containers.hpp"
 
 namespace ZeroTier {
 

node/Trace.hpp

@@ -56,21 +56,21 @@ struct NetworkConfig;
 class Trace
 {
 public:
-	struct RuleResultLog
+	struct RuleResultLog : public TriviallyCopyable
 	{
 		uint8_t l[ZT_MAX_NETWORK_RULES / 2]; // ZT_MAX_NETWORK_RULES 4-bit fields
 
-		ZT_INLINE void log(const unsigned int rn,const uint8_t thisRuleMatches,const uint8_t thisSetMatches)
+		ZT_INLINE void log(const unsigned int rn,const uint8_t thisRuleMatches,const uint8_t thisSetMatches) noexcept
 		{
 			l[rn >> 1U] |= ( ((thisRuleMatches + 1U) << 2U) | (thisSetMatches + 1U) ) << ((rn & 1U) << 2U);
 		}
-		ZT_INLINE void logSkipped(const unsigned int rn,const uint8_t thisSetMatches)
+		ZT_INLINE void logSkipped(const unsigned int rn,const uint8_t thisSetMatches) noexcept
 		{
 			l[rn >> 1U] |= (thisSetMatches + 1U) << ((rn & 1U) << 2U);
 		}
-		ZT_INLINE void clear()
+		ZT_INLINE void clear() noexcept
 		{
-			Utils::zero<sizeof(l)>(l);
+			memoryZero(this);
 		}
 	};
 

node/TriviallyCopyable.hpp

@@ -38,8 +38,8 @@ ZT_PACKED_STRUCT(struct TriviallyCopyable
 	template<typename T>
 	static ZT_INLINE void memoryZero(T *obj) noexcept
 	{
-		TriviallyCopyable *const tmp = obj;
+		static_assert(isTriviallyCopyable(obj),"parameter is not TriviallyCopyable");
-		Utils::zero<sizeof(T)>(tmp);
+		Utils::zero<sizeof(T)>(obj);
 	}
 
 	/**
@@ -51,8 +51,8 @@ ZT_PACKED_STRUCT(struct TriviallyCopyable
 	template<typename T>
 	static ZT_INLINE void memoryZero(T &obj) noexcept
 	{
-		TriviallyCopyable *const tmp = &obj;
+		static_assert(isTriviallyCopyable(obj),"parameter is not TriviallyCopyable");
-		Utils::zero<sizeof(T)>(tmp);
+		Utils::zero<sizeof(T)>(&obj);
 	}
 
 	/**
@@ -65,8 +65,8 @@ ZT_PACKED_STRUCT(struct TriviallyCopyable
 	template<typename T>
 	static ZT_INLINE void memoryCopyUnsafe(T *dest,const void *src) noexcept
 	{
-		TriviallyCopyable *const tmp = dest;
+		static_assert(isTriviallyCopyable(dest),"parameter is not TriviallyCopyable");
-		Utils::copy<sizeof(T)>(tmp,src);
+		Utils::copy<sizeof(T)>(dest,src);
 	}
 
 	/**
@@ -79,8 +79,8 @@ ZT_PACKED_STRUCT(struct TriviallyCopyable
 	template<typename T>
 	static ZT_INLINE void memoryCopyUnsafe(T &dest,const void *src) noexcept
 	{
-		TriviallyCopyable *const tmp = &dest;
+		static_assert(isTriviallyCopyable(dest),"parameter is not TriviallyCopyable");
-		Utils::copy<sizeof(T)>(tmp,src);
+		Utils::copy<sizeof(T)>(&dest,src);
 	}
 
 	/**
@@ -93,8 +93,8 @@ ZT_PACKED_STRUCT(struct TriviallyCopyable
 	template<typename T>
 	static ZT_INLINE void memoryCopy(T *dest,const T *src) noexcept
 	{
-		TriviallyCopyable *const tmp = dest;
+		static_assert(isTriviallyCopyable(dest),"parameter is not TriviallyCopyable");
-		Utils::copy<sizeof(T)>(tmp,src);
+		Utils::copy<sizeof(T)>(dest,src);
 	}
 
 	/**
@@ -107,8 +107,8 @@ ZT_PACKED_STRUCT(struct TriviallyCopyable
 	template<typename T>
 	static ZT_INLINE void memoryCopy(T *dest,const T &src) noexcept
 	{
-		TriviallyCopyable *const tmp = dest;
+		static_assert(isTriviallyCopyable(src),"parameter is not TriviallyCopyable");
-		Utils::copy<sizeof(T)>(tmp,&src);
+		Utils::copy<sizeof(T)>(dest,&src);
 	}
 
 	/**
@@ -121,8 +121,8 @@ ZT_PACKED_STRUCT(struct TriviallyCopyable
 	template<typename T>
 	static ZT_INLINE void memoryCopy(T &dest,const T *src) noexcept
 	{
-		TriviallyCopyable *const tmp = &dest;
+		static_assert(isTriviallyCopyable(dest),"parameter is not TriviallyCopyable");
-		Utils::copy<sizeof(T)>(tmp,src);
+		Utils::copy<sizeof(T)>(&dest,src);
 	}
 
 	/**
@@ -135,13 +135,13 @@ ZT_PACKED_STRUCT(struct TriviallyCopyable
 	template<typename T>
 	static ZT_INLINE void memoryCopy(T &dest,const T &src) noexcept
 	{
-		TriviallyCopyable *const tmp = &dest;
+		static_assert(isTriviallyCopyable(dest),"parameter is not TriviallyCopyable");
-		Utils::copy<sizeof(T)>(tmp,&src);
+		Utils::copy<sizeof(T)>(&dest,&src);
 	}
 });
 
-static constexpr bool isTriviallyCopyable(const TriviallyCopyable *const anything) noexcept { return true; }
+static constexpr bool isTriviallyCopyable(const TriviallyCopyable *) noexcept { return true; }
-static constexpr bool isTriviallyCopyable(const void *const anything) noexcept { return false; }
+static constexpr bool isTriviallyCopyable(const void *) noexcept { return false; }
 
 template<typename T>
 static constexpr bool isTriviallyCopyable(const T &anything) noexcept { return isTriviallyCopyable(&anything); }

node/Utils.hpp

@@ -22,6 +22,10 @@
 #include <immintrin.h>
 #endif
 
+#include <utility>
+#include <algorithm>
+#include <memory>
+
 namespace ZeroTier {
 
 namespace Utils {
@@ -621,8 +625,8 @@ template<unsigned int L>
 static ZT_INLINE void copy(void *const dest,const void *const src) noexcept
 {
 #ifdef ZT_ARCH_X64
-	uint8_t *volatile d = reinterpret_cast<uint8_t *>(dest);
+	uint8_t *volatile d = reinterpret_cast<uint8_t *>(dest); // NOLINT(hicpp-use-auto,modernize-use-auto)
-	const uint8_t *s = reinterpret_cast<const uint8_t *>(src);
+	const uint8_t *s = reinterpret_cast<const uint8_t *>(src); // NOLINT(hicpp-use-auto,modernize-use-auto)
 	for(unsigned int i=0;i<(L >> 6U);++i) {
 		__m128i x0 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(s));
 		__m128i x1 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(s + 16));
@@ -694,7 +698,7 @@ template<unsigned int L>
 static ZT_INLINE void zero(void *const dest) noexcept
 {
 #ifdef ZT_ARCH_X64
-	uint8_t *volatile d = reinterpret_cast<uint8_t *>(dest);
+	uint8_t *volatile d = reinterpret_cast<uint8_t *>(dest); // NOLINT(hicpp-use-auto,modernize-use-auto)
 	__m128i z = _mm_setzero_si128();
 	for(unsigned int i=0;i<(L >> 6U);++i) {
 		_mm_storeu_si128(reinterpret_cast<__m128i *>(d),z);
@@ -743,6 +747,46 @@ static ZT_INLINE void zero(void *const dest,const unsigned int len) noexcept
 	memset(dest,0,len);
 }
 
+/**
+ * Simple malloc/free based C++ STL allocator
+ *
+ * @tparam T Allocated type
+ */
+template<typename T>
+struct Mallocator
+{
+	typedef size_t size_type;
+	typedef ptrdiff_t difference_type;
+	typedef T * pointer;
+	typedef const T * const_pointer;
+	typedef T & reference;
+	typedef const T & const_reference;
+	typedef T value_type;
+
+	template <class U> struct rebind { typedef Mallocator<U> other; };
+	ZT_INLINE Mallocator() noexcept {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
+	ZT_INLINE Mallocator(const Mallocator&) noexcept {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
+	template <class U> ZT_INLINE Mallocator(const Mallocator<U>&) noexcept {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default,google-explicit-constructor,hicpp-explicit-conversions)
+	ZT_INLINE ~Mallocator() noexcept {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
+
+	ZT_INLINE pointer allocate(size_type s,void const * = nullptr)
+	{
+		if (0 == s)
+			return nullptr;
+		pointer temp = (pointer)malloc(s * sizeof(T)); // NOLINT(hicpp-use-auto,modernize-use-auto)
+		if (temp == nullptr)
+			throw std::bad_alloc();
+		return temp;
+	}
+
+	ZT_INLINE pointer address(reference x) const { return &x; }
+	ZT_INLINE const_pointer address(const_reference x) const { return &x; }
+	ZT_INLINE void deallocate(pointer p,size_type) { free(p); }
+	ZT_INLINE size_type max_size() const noexcept { return std::numeric_limits<size_t>::max() / sizeof(T); }
+	ZT_INLINE void construct(pointer p,const T& val) { new((void *)p) T(val); }
+	ZT_INLINE void destroy(pointer p) { p->~T(); }
+};
+
 } // namespace Utils
 
 } // namespace ZeroTier

node/VL1.cpp

@@ -218,7 +218,7 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 					ph = &(pkt.b->as<Protocol::Header>());
 
 					// Generate one-time-use MAC key using Salsa20.
-					uint8_t perPacketKey[ZT_PEER_SECRET_KEY_LENGTH];
+					uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
 					uint8_t macKey[ZT_POLY1305_KEY_SIZE];
 					Protocol::salsa2012DeriveKey(peer->key(),perPacketKey,*pktv[0].b,packetSize);
 					Salsa20(perPacketKey,&ph->packetId).crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
@@ -237,7 +237,7 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 			case ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012:
 				if (peer) {
 					// Derive per-packet key using symmetric key plus some data from the packet header.
-					uint8_t perPacketKey[ZT_PEER_SECRET_KEY_LENGTH];
+					uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
 					Protocol::salsa2012DeriveKey(peer->key(),perPacketKey,*pktv[0].b,packetSize);
 					Salsa20 s20(perPacketKey,&ph->packetId);
 
@@ -433,7 +433,7 @@ void VL1::_sendPendingWhois(void *const tPtr,const int64_t now)
 	std::vector<Address> toSend;
 	{
 		Mutex::Lock wl(_whoisQueue_l);
-		for(std::map<Address,_WhoisQueueItem>::iterator wi(_whoisQueue.begin());wi!=_whoisQueue.end();++wi) {
+		for(Map<Address,_WhoisQueueItem>::iterator wi(_whoisQueue.begin());wi!=_whoisQueue.end();++wi) {
 			if ((now - wi->second.lastRetry) >= ZT_WHOIS_RETRY_DELAY) {
 				wi->second.lastRetry = now;
 				++wi->second.retries;
@@ -496,9 +496,9 @@ bool VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 
 	// Packet is basically valid and identity unmarshaled successfully --------------------------------------------------
 
-	uint8_t key[ZT_PEER_SECRET_KEY_LENGTH];
+	uint8_t key[ZT_SYMMETRIC_KEY_SIZE];
 	if ((peer) && (id == peer->identity())) {
-		Utils::copy<ZT_PEER_SECRET_KEY_LENGTH>(key,peer->key());
+		Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(key,peer->key());
 	} else {
 		peer.zero();
 		if (!RR->identity.agree(id,key)) {
@@ -508,7 +508,7 @@ bool VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 	}
 
 	if ((!peer)||(!authenticated)) {
-		uint8_t perPacketKey[ZT_PEER_SECRET_KEY_LENGTH];
+		uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
 		uint8_t macKey[ZT_POLY1305_KEY_SIZE];
 		Protocol::salsa2012DeriveKey(peer->key(),perPacketKey,pkt,packetSize);
 		Salsa20(perPacketKey,&p.h.packetId).crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
@@ -522,7 +522,7 @@ bool VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 
 	// Packet has passed Poly1305 MAC authentication --------------------------------------------------------------------
 
-	uint8_t hmacKey[ZT_PEER_SECRET_KEY_LENGTH],hmac[ZT_HMACSHA384_LEN];
+	uint8_t hmacKey[ZT_SYMMETRIC_KEY_SIZE],hmac[ZT_HMACSHA384_LEN];
 	if (peer->remoteVersionProtocol() >= 11) {
 		if (packetSize <= ZT_HMACSHA384_LEN) { // sanity check, should be impossible
 			RR->t->incomingPacketDropped(tPtr,0x1000662a,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
@@ -883,12 +883,6 @@ bool VL1::_PUSH_DIRECT_PATHS(void *tPtr,const SharedPtr<Path> &path,const Shared
 	}
 	Protocol::PUSH_DIRECT_PATHS &pdp = pkt.as<Protocol::PUSH_DIRECT_PATHS>();
 
-	const uint64_t now = RR->node->now();
-	if (!peer->rateGateInboundPushDirectPaths(now)) {
-		RR->t->incomingPacketDropped(tPtr,0x35b1aaaa,pdp.h.packetId,0,peer->identity(),path->address(),Protocol::packetHops(pdp.h),Protocol::VERB_PUSH_DIRECT_PATHS,ZT_TRACE_PACKET_DROP_REASON_RATE_LIMIT_EXCEEDED);
-		return true;
-	}
-
 	int ptr = sizeof(Protocol::PUSH_DIRECT_PATHS);
 	const unsigned int numPaths = Utils::ntoh(pdp.numPaths);
 	InetAddress a;
@@ -965,6 +959,8 @@ bool VL1::_PUSH_DIRECT_PATHS(void *tPtr,const SharedPtr<Path> &path,const Shared
 		ptr += (int)addrRecordLen;
 	}
 
+	// TODO: add to a peer try-queue
+
 	return true;
 }
 

node/VL1.hpp

@@ -21,8 +21,8 @@
 #include "Protocol.hpp"
 #include "Mutex.hpp"
 #include "FCV.hpp"
+#include "Containers.hpp"
 
-#include <map>
 #include <vector>
 
 namespace ZeroTier {
@@ -89,7 +89,7 @@ private:
 
 	Defragmenter<ZT_MAX_PACKET_FRAGMENTS> _inputPacketAssembler;
 
-	std::map<Address,_WhoisQueueItem> _whoisQueue;
+	Map<Address,_WhoisQueueItem> _whoisQueue;
 	Mutex _whoisQueue_l;
 };
 

osdep/Arp.hpp

@@ -15,7 +15,7 @@
 #define ZT_ARP_HPP
 
 #include "../node/Constants.hpp"
-#include "../node/Map.hpp"
+#include "../node/Containers.hpp"
 #include "../node/MAC.hpp"
 
 #include <utility>

osdep/NeighborDiscovery.hpp

@@ -14,7 +14,7 @@
 #ifndef ZT_NEIGHBORDISCOVERY_HPP
 #define ZT_NEIGHBORDISCOVERY_HPP
 
-#include "../node/Map.hpp"
+#include "../node/Containers.hpp"
 #include "../node/MAC.hpp"
 #include "../node/InetAddress.hpp"