More endless work in progress...

include/ZeroTierCore.h

@@ -25,15 +25,15 @@
 #else
 #include <arpa/inet.h>
 #include <netinet/in.h>
+#include <net/if_dl.h>
 #include <sys/types.h>
 #include <sys/socket.h>
+#include <stdint.h>
+#include <stdlib.h>
 #endif
 
 #ifdef __cplusplus
-#include <cstdint>
 extern "C" {
-#else
-#include <stdint.h>
 #endif
 
 /* This symbol may be defined to anything we need to put in front of API function prototypes. */
@@ -41,16 +41,22 @@ extern "C" {
 #define ZT_SDK_API
 #endif
 
-/* ----------------------------------------------------------------------------------------------------------------- */
+/* ---------------------------------------------------------------------------------------------------------------- */
 
 /**
- * Default UDP port for devices running a ZeroTier endpoint
+ * Default primary UDP port for devices running a ZeroTier endpoint
- *
- * NOTE: as of V2 this has changed to 893 since many NATs (even symmetric)
- * treat privileged ports in a special way. The old default was 9993 and
- * this is likely to be seen in the wild quite a bit.
  */
-#define ZT_DEFAULT_PORT 793
+#define ZT_DEFAULT_PORT 9993
+
+/**
+ * IP protocol number for naked IP encapsulation (this is not currently used)
+ */
+#define ZT_DEFAULT_IP_PROTOCOL 193
+
+/**
+ * Ethernet type for naked Ethernet encapsulation (this is not currently used)
+ */
+#define ZT_DEFAULT_ETHERNET_PROTOCOL 0x9993
 
 /**
  * Size of a standard I/O buffer as returned by getBuffer().
@@ -170,8 +176,6 @@ extern "C" {
  */
 #define ZT_MAX_CERTIFICATES_OF_OWNERSHIP 4
 
-/* ----------------------------------------------------------------------------------------------------------------- */
-
 /**
  * Packet characteristics flag: packet direction, 1 if inbound 0 if outbound
  */
@@ -257,14 +261,13 @@ extern "C" {
  */
 #define ZT_RULE_PACKET_CHARACTERISTICS_TCP_FIN 0x0000000000000001ULL
 
-/* ----------------------------------------------------------------------------------------------------------------- */
+/* ---------------------------------------------------------------------------------------------------------------- */
 
 /**
- * Identity type codes
+ * Identity type codes (must be the same as Identity.hpp).
  */
 enum ZT_Identity_Type
 {
-	/* These values must be the same as in Identity.hpp in the core. */
 	ZT_IDENTITY_TYPE_C25519 = 0, /* C25519/Ed25519 */
 	ZT_IDENTITY_TYPE_P384 =   1  /* Combined C25519/NIST-P-384 key */
 };
@@ -303,6 +306,25 @@ enum ZT_CredentialType
 	ZT_CREDENTIAL_TYPE_REVOCATION =  6
 };
 
+/**
+ * Endpoint address and protocol types
+ *
+ * Most of these are not currently implemented and are just reserved
+ * for future use.
+ */
+enum ZT_EndpointType
+{
+	ZT_ENDPOINT_TYPE_NIL =           0,  // Nil/empty endpoint
+	ZT_ENDPOINT_TYPE_ZEROTIER =      1,  // ZeroTier relaying (address+fingerprint)
+	ZT_ENDPOINT_TYPE_ETHERNET =      2,  // Ethernet with ethertype 0x9993
+	ZT_ENDPOINT_TYPE_WIFI_DIRECT =   3,  // Ethernet using WiFi direct
+	ZT_ENDPOINT_TYPE_BLUETOOTH =     4,  // Bluetooth (same address type as Ethernet)
+	ZT_ENDPOINT_TYPE_IP =            5,  // Naked IP (protocol 193)
+	ZT_ENDPOINT_TYPE_IP_UDP =        6,  // IP/UDP
+	ZT_ENDPOINT_TYPE_IP_TCP =        7,  // IP/TCP
+	ZT_ENDPOINT_TYPE_IP_HTTP2 =      8   // IP/HTTP2 encapsulation
+};
+
 /**
  * Flag indicating that VL1 tracing should be generated
  */
@@ -330,16 +352,11 @@ enum ZT_CredentialType
  */
 enum ZT_TraceEventType
 {
-	/* An unexpected error is an internal assertion / sanity check failure, out of memory, etc. */
 	ZT_TRACE_UNEXPECTED_ERROR =             0,
-
-	/* VL1 events related to the peer-to-peer layer */
 	ZT_TRACE_VL1_RESETTING_PATHS_IN_SCOPE = 1,
 	ZT_TRACE_VL1_TRYING_NEW_PATH =          2,
 	ZT_TRACE_VL1_LEARNED_NEW_PATH =         3,
 	ZT_TRACE_VL1_INCOMING_PACKET_DROPPED =  4,
-
-	/* VL2 events relate to virtual networks, packet filtering, and authentication */
 	ZT_TRACE_VL2_OUTGOING_FRAME_DROPPED =   100,
 	ZT_TRACE_VL2_INCOMING_FRAME_DROPPED =   101,
 	ZT_TRACE_VL2_NETWORK_CONFIG_REQUESTED = 102,
@@ -378,36 +395,6 @@ enum ZT_TraceFrameDropReason
 	ZT_TRACE_FRAME_DROP_REASON_PERMISSION_DENIED =                    7
 };
 
-/**
- * Address types for ZT_TraceEventPathAddress
- *
- * These are currently the same as the types in Endpoint.hpp and should remain so
- * if possible for consistency. Not all of these are used (yet?) but they are defined
- * for possible future use and the structure is sized to support them.
- */
-enum ZT_EndpointType
-{
-	ZT_ENDPOINT_TYPE_NIL =          0, /* none/empty */
-	ZT_ENDPOINT_TYPE_ZEROTIER =     1, /* 5-byte ZeroTier + 48-byte identity hash */
-	ZT_ENDPOINT_TYPE_ETHERNET =     2, /* 6-byte Ethernet */
-	ZT_ENDPOINT_TYPE_INETADDR_V4 =  4, /* 4-byte IPv4 */
-	ZT_ENDPOINT_TYPE_INETADDR_V6 =  6  /* 16-byte IPv6 */
-};
-
-/**
- * Protocol bits allowed for endpoint addresses.
- */
-enum ZT_EndpointProtocol
-{
-	ZT_ENDPOINT_PROTO_DGRAM =       0x0001,
-	ZT_ENDPOINT_PROTO_STREAM  =     0x0002,
-	ZT_ENDPOINT_PROTO_HTTP2 =       0x0004,
-	ZT_ENDPOINT_PROTO_HTTPS2 =      0x0008,
-	ZT_ENDPOINT_PROTO_WS =          0x0010,
-	ZT_ENDPOINT_PROTO_WEBRTC =      0x0020,
-	ZT_ENDPOINT_PROTO_WIREGUARD =   0x0040
-};
-
 /**
  * Reasons for credential rejection
  */
@@ -419,8 +406,6 @@ enum ZT_TraceCredentialRejectionReason
 	ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID =                       4
 };
 
-// Fields used in trace output dictionaries. Which fields are present depends on
-// the trace event type. All trace dictionaries contain TYPE and CODE_LOCATION.
 #define ZT_TRACE_FIELD_TYPE                               "t"
 #define ZT_TRACE_FIELD_CODE_LOCATION                      "@"
 #define ZT_TRACE_FIELD_ENDPOINT                           "e"
@@ -458,8 +443,6 @@ enum ZT_TraceCredentialRejectionReason
 #define ZT_TRACE_FIELD_CREDENTIAL_TYPE                    "crT"
 #define ZT_TRACE_FIELD_CREDENTIAL_TIMESTAMP               "crTS"
 
-/****************************************************************************/
-
 /**
  * Function return code: OK (0) or error results
  *
@@ -1309,7 +1292,7 @@ enum ZT_StateObjectType
 	 * Peer and related state
 	 *
 	 * Object ID: peer address
-	 * Canonical path: <HOME>/peers.d/<ID> (10-digit address
+	 * Canonical path: <HOME>/peers.d/<ID> (10-digit address)
 	 * Persistence: optional, can be cleared at any time
 	 */
 	ZT_STATE_OBJECT_PEER = 5,
@@ -1338,7 +1321,7 @@ enum ZT_StateObjectType
  */
 typedef void ZT_Node;
 
-/****************************************************************************/
+/* ---------------------------------------------------------------------------------------------------------------- */
 
 /**
  * Callback called to update virtual network port configuration
@@ -1528,7 +1511,7 @@ typedef int (*ZT_PathLookupFunction)(
 	int,                              /* Desired ss_family or -1 for any */
 	struct sockaddr_storage *);       /* Result buffer */
 
-/****************************************************************************/
+/* ---------------------------------------------------------------------------------------------------------------- */
 
 /**
  * Structure for configuring ZeroTier core callback functions
@@ -1576,6 +1559,8 @@ struct ZT_Node_Callbacks
 	ZT_PathLookupFunction pathLookupFunction;
 };
 
+/* ---------------------------------------------------------------------------------------------------------------- */
+
 /**
  * Get a buffer for reading data to be passed back into the core via one of the processX() functions
  *
@@ -1595,6 +1580,8 @@ ZT_SDK_API void *ZT_getBuffer();
  */
 ZT_SDK_API void ZT_freeBuffer(void *b);
 
+/* ---------------------------------------------------------------------------------------------------------------- */
+
 /**
  * Create a new ZeroTier node
  *
@@ -1929,7 +1916,7 @@ ZT_SDK_API void ZT_Node_setController(ZT_Node *node,void *networkConfigMasterIns
  */
 ZT_SDK_API enum ZT_ResultCode ZT_Node_setPhysicalPathConfiguration(ZT_Node *node,const struct sockaddr_storage *pathNetwork,const ZT_PhysicalPathConfiguration *pathConfig);
 
-/****************************************************************************/
+/* ---------------------------------------------------------------------------------------------------------------- */
 
 /**
  * Generate a new identity
@@ -2039,7 +2026,7 @@ ZT_SDK_API const ZT_Fingerprint *ZT_Identity_fingerprint(const ZT_Identity *id);
  */
 ZT_SDK_API void ZT_Identity_delete(ZT_Identity *id);
 
-/****************************************************************************/
+/* ---------------------------------------------------------------------------------------------------------------- */
 
 /**
  * Get ZeroTier One version

node/C25519.cpp

@@ -2381,7 +2381,7 @@ void C25519::generateC25519(uint8_t pub[ZT_C25519_ECDH_PUBLIC_KEY_SIZE],uint8_t
 	s_calcPubDH(pub, priv);
 }
 
-void C25519::agree(const uint8_t mine[ZT_C25519_COMBINED_PRIVATE_KEY_SIZE],const uint8_t their[ZT_C25519_COMBINED_PUBLIC_KEY_SIZE],uint8_t rawkey[ZT_C25519_ECDH_SHARED_SECRET_SIZE])
+void C25519::agree(const uint8_t mine[ZT_C25519_ECDH_PRIVATE_KEY_SIZE],const uint8_t their[ZT_C25519_ECDH_PUBLIC_KEY_SIZE],uint8_t rawkey[ZT_C25519_ECDH_SHARED_SECRET_SIZE])
 {
 	crypto_scalarmult(rawkey,mine,their);
 }

node/C25519.hpp

@@ -80,7 +80,7 @@ public:
 	 * @param their Their public key
 	 * @param rawkey Buffer to receive raw (not hashed) agreed upon key
 	 */
-	static void agree(const uint8_t mine[ZT_C25519_COMBINED_PRIVATE_KEY_SIZE],const uint8_t their[ZT_C25519_COMBINED_PUBLIC_KEY_SIZE],uint8_t rawkey[ZT_C25519_ECDH_SHARED_SECRET_SIZE]);
+	static void agree(const uint8_t mine[ZT_C25519_ECDH_PRIVATE_KEY_SIZE],const uint8_t their[ZT_C25519_ECDH_PUBLIC_KEY_SIZE],uint8_t rawkey[ZT_C25519_ECDH_SHARED_SECRET_SIZE]);
 
 	/**
 	 * Sign a message with a sender's key pair

node/Endpoint.cpp

@@ -16,145 +16,99 @@
 
 namespace ZeroTier {
 
-Endpoint::Endpoint(const InetAddress &sa,const Protocol proto) noexcept // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-{
-	switch (sa.family()) {
-		case AF_INET:
-			_t = TYPE_INETADDR_V4;
-			break;
-		case AF_INET6:
-			_t = TYPE_INETADDR_V6;
-			break;
-		default:
-			_t = TYPE_NIL;
-			return;
-	}
-	_proto = proto;
-	asInetAddress(_v.sa) = sa;
-}
-
-bool Endpoint::operator==(const Endpoint &ep) const noexcept
-{
-	if ((_t == ep._t)&&(_proto == ep._proto)) {
-		switch(_t) {
-			default:
-				return true;
-			case TYPE_ZEROTIER:
-				return ((_v.zt.address == ep._v.zt.address) && (memcmp(_v.zt.hash,ep._v.zt.hash,sizeof(_v.zt.hash)) == 0));
-			case TYPE_ETHERNET:
-				return memcmp(_v.eth,ep._v.eth,6) == 0;
-			case TYPE_INETADDR_V4:
-			case TYPE_INETADDR_V6:
-				return asInetAddress(_v.sa) == asInetAddress(ep._v.sa);
-		}
-	}
-	return false;
-}
-
-bool Endpoint::operator<(const Endpoint &ep) const noexcept
-{
-	if ((int)_t < (int)ep._t) {
-		return true;
-	} else if (_t == ep._t) {
-		if ((int)_proto < (int)ep._proto) {
-			return true;
-		} else {
-			switch (_t) {
-				case TYPE_ZEROTIER:
-					return (_v.zt.address < ep._v.zt.address) ? true : ((_v.zt.address == ep._v.zt.address) && (memcmp(_v.zt.hash,ep._v.zt.hash,sizeof(_v.zt.hash)) < 0));
-				case TYPE_ETHERNET:
-					return memcmp(_v.eth,ep._v.eth,6) < 0;
-				case TYPE_INETADDR_V4:
-				case TYPE_INETADDR_V6:
-					return asInetAddress(_v.sa) < asInetAddress(ep._v.sa);
-				default:
-					return false;
-			}
-		}
-	}
-	return false;
-}
-
 int Endpoint::marshal(uint8_t data[ZT_ENDPOINT_MARSHAL_SIZE_MAX]) const noexcept
 {
-	data[0] = (uint8_t)_t;
+	switch(m_value[ZT_ENDPOINT_MARSHAL_SIZE_MAX-1]) {
-	Utils::storeBigEndian(data + 1,(uint16_t)_proto);
-	Utils::storeBigEndian(data + 3,(uint16_t)_l[0]);
-	Utils::storeBigEndian(data + 5,(uint16_t)_l[1]);
-	Utils::storeBigEndian(data + 7,(uint16_t)_l[2]);
-
-	int p;
-	switch(_t) {
-		case TYPE_ZEROTIER:
-			data[9] = (uint8_t)(_v.zt.address >> 32U);
-			data[10] = (uint8_t)(_v.zt.address >> 24U);
-			data[11] = (uint8_t)(_v.zt.address >> 16U);
-			data[12] = (uint8_t)(_v.zt.address >> 8U);
-			data[13] = (uint8_t)_v.zt.address;
-			Utils::copy<ZT_FINGERPRINT_HASH_SIZE>(data + 14,_v.zt.hash);
-			return ZT_FINGERPRINT_HASH_SIZE + 14;
-		case TYPE_ETHERNET:
-			Utils::copy<6>(data + 9,_v.eth);
-			return 15;
-		case TYPE_INETADDR_V4:
-		case TYPE_INETADDR_V6:
-			p = 9 + asInetAddress(_v.sa).marshal(data + 7);
-			if (p <= 9)
-				return -1;
-			return p;
 		default:
-			data[0] = (uint8_t)TYPE_NIL;
+		//case ZT_ENDPOINT_TYPE_NIL:
+			data[0] = 0;
 			return 1;
+
+		case ZT_ENDPOINT_TYPE_ZEROTIER:
+			data[0] = 16 + ZT_ENDPOINT_TYPE_ZEROTIER;
+			reinterpret_cast<const Fingerprint *>(m_value)->address().copyTo(data + 1);
+			Utils::copy<ZT_FINGERPRINT_HASH_SIZE>(data + 1 + ZT_ADDRESS_LENGTH,reinterpret_cast<const Fingerprint *>(m_value)->hash());
+			return 1 + ZT_ADDRESS_LENGTH + ZT_FINGERPRINT_HASH_SIZE;
+
+		case ZT_ENDPOINT_TYPE_ETHERNET:
+		case ZT_ENDPOINT_TYPE_WIFI_DIRECT:
+		case ZT_ENDPOINT_TYPE_BLUETOOTH:
+			data[0] = 16 + m_value[ZT_ENDPOINT_MARSHAL_SIZE_MAX-1];
+			reinterpret_cast<const MAC *>(m_value)->copyTo(data + 1);
+			return 7;
+
+		case ZT_ENDPOINT_TYPE_IP_UDP:
+			return reinterpret_cast<const InetAddress *>(m_value)->marshal(data);
+
+		case ZT_ENDPOINT_TYPE_IP:
+		case ZT_ENDPOINT_TYPE_IP_TCP:
+		case ZT_ENDPOINT_TYPE_IP_HTTP2:
+			data[0] = 16 + m_value[ZT_ENDPOINT_MARSHAL_SIZE_MAX-1];
+			return 1 + reinterpret_cast<const InetAddress *>(m_value)->marshal(data + 1);
 	}
 }
 
-int Endpoint::unmarshal(const uint8_t *restrict data,const int len) noexcept
+int Endpoint::unmarshal(const uint8_t *restrict data,int len) noexcept
 {
-	if (len < 1)
+	memoryZero(this);
+	if (unlikely(len <= 0))
 		return -1;
 
-	_t = (Type)data[0];
+	// Serialized endpoints with type bytes less than 16 are passed through
-	if (_t == TYPE_NIL)
+	// to the unmarshal method of InetAddress and considered UDP endpoints.
+	// This allows backward compatibility with old endpoint fields in the
+	// protocol that were serialized InetAddress instances.
+	if (data[0] < 16) {
+		switch(data[0]) {
+			case 0:
+				return 1;
+			case 4:
+			case 6:
+				m_value[ZT_ENDPOINT_MARSHAL_SIZE_MAX-1] = (uint8_t)ZT_ENDPOINT_TYPE_IP_UDP;
+				return reinterpret_cast<InetAddress *>(m_value)->unmarshal(data,len);
+		}
+		return -1;
+	}
+
+	switch((m_value[ZT_ENDPOINT_MARSHAL_SIZE_MAX-1] = (data[0] - 16))) {
+		case ZT_ENDPOINT_TYPE_NIL:
 			return 1;
 
-	_proto = (Protocol)Utils::loadBigEndian<uint16_t>(data + 1);
+		case ZT_ENDPOINT_TYPE_ZEROTIER:
-	_l[0] = (int)Utils::loadBigEndian<uint16_t>(data + 3);
+			if (len >= (1 + ZT_ADDRESS_LENGTH + ZT_FINGERPRINT_HASH_SIZE)) {
-	_l[1] = (int)Utils::loadBigEndian<uint16_t>(data + 5);
+				reinterpret_cast<Fingerprint *>(m_value)->apiFingerprint()->address = Address(data + 1).toInt();
-	_l[2] = (int)Utils::loadBigEndian<uint16_t>(data + 7);
+				Utils::copy<ZT_FINGERPRINT_HASH_SIZE>(reinterpret_cast<Fingerprint *>(m_value)->apiFingerprint()->hash,data + 1 + ZT_ADDRESS_LENGTH);
-
+				return 1 + ZT_ADDRESS_LENGTH + ZT_FINGERPRINT_HASH_SIZE;
-	int p;
-  switch(_t) {
-	  case TYPE_ZEROTIER:
-		  if (len < (14 + ZT_FINGERPRINT_HASH_SIZE))
-			  return -1;
-		  _v.zt.address = ((uint64_t)data[9]) << 32U;
-		  _v.zt.address |= ((uint64_t)data[10]) << 24U;
-		  _v.zt.address |= ((uint64_t)data[11]) << 16U;
-		  _v.zt.address |= ((uint64_t)data[12]) << 8U;
-		  _v.zt.address |= (uint64_t)data[13];
-		  Utils::copy<ZT_FINGERPRINT_HASH_SIZE>(_v.zt.hash,data + 14);
-		  return ZT_FINGERPRINT_HASH_SIZE + 14;
-	  case TYPE_ETHERNET:
-		  if (len < 15)
-			  return -1;
-		  Utils::copy<6>(_v.eth,data + 9);
-		  return 15;
-		case TYPE_INETADDR_V4:
-		case TYPE_INETADDR_V6:
-			if (len <= 9)
-				return -1;
-			p = 9 + asInetAddress(_v.sa).unmarshal(data + 9,len - 9);
-			if ((p <= 9)||(p >= len))
-				return -1;
-			return p;
-		default:
-			// Unrecognized endpoint types not yet specified must start with a 16-bit
-			// length so that older versions of ZeroTier can skip them.
-			if (len < 11)
-				return -1;
-			p = 11 + (int)Utils::loadBigEndian<uint16_t>(data + 9);
-			return (p > len) ? -1 : p;
 			}
+			return -1;
+
+		case ZT_ENDPOINT_TYPE_ETHERNET:
+		case ZT_ENDPOINT_TYPE_WIFI_DIRECT:
+		case ZT_ENDPOINT_TYPE_BLUETOOTH:
+			if (len >= 7) {
+				reinterpret_cast<MAC *>(m_value)->setTo(data + 1);
+				return 7;
+			}
+			return -1;
+
+		case ZT_ENDPOINT_TYPE_IP:
+		case ZT_ENDPOINT_TYPE_IP_UDP:
+		case ZT_ENDPOINT_TYPE_IP_TCP:
+		case ZT_ENDPOINT_TYPE_IP_HTTP2:
+			return reinterpret_cast<InetAddress *>(m_value)->unmarshal(data + 1,len - 1);
+
+		default:
+			break;
+	}
+
+	// Unrecognized types can still be passed over in a valid stream if they are
+	// prefixed by a 16-bit size. This allows forward compatibility with future
+	// endpoint types.
+	m_value[ZT_ENDPOINT_MARSHAL_SIZE_MAX-1] = (uint8_t)ZT_ENDPOINT_TYPE_NIL;
+	if (len < 3)
+		return -1;
+	const int unrecLen = 1 + (int)Utils::loadBigEndian<uint16_t>(data + 1);
+	return (unrecLen > len) ? -1 : unrecLen;
 }
 
 } // namespace ZeroTier

node/Endpoint.hpp

@@ -20,101 +20,137 @@
 #include "Utils.hpp"
 #include "TriviallyCopyable.hpp"
 #include "Fingerprint.hpp"
+#include "MAC.hpp"
 
-#define ZT_ENDPOINT_MARSHAL_SIZE_MAX 64
+#define ZT_ENDPOINT_MARSHAL_SIZE_MAX 128
+
+static_assert(ZT_ENDPOINT_MARSHAL_SIZE_MAX > (ZT_INETADDRESS_MARSHAL_SIZE_MAX + 1),"ZT_ENDPOINT_MARSHAL_SIZE_MAX not large enough");
+static_assert(ZT_ENDPOINT_MARSHAL_SIZE_MAX > (sizeof(ZT_Fingerprint) + 1),"ZT_ENDPOINT_MARSHAL_SIZE_MAX not large enough");
 
 namespace ZeroTier {
 
 /**
  * Endpoint variant specifying some form of network endpoint.
+ * 
+ * This is sort of a superset of InetAddress and for the standard UDP
+ * protocol marshals and unmarshals to a compatible format. This makes
+ * it backward compatible with older node versions' protocol fields
+ * where InetAddress was used as long as only the UDP type is exchanged
+ * with those nodes.
  */
 class Endpoint : public TriviallyCopyable
 {
 public:
 	/**
-	 * Endpoint type
+	 * Endpoint type (defined in the API)
+	 */
+	typedef ZT_EndpointType Type;
+
+	/**
+	 * Create a NIL/empty endpoint
+	 */
+	ZT_INLINE Endpoint() noexcept { memoryZero(this); }
+
+	/**
+	 * Create an endpoint for a type that uses an IP
 	 * 
-	 * These are set to be the same as the IDs used for trace events in ZeroTierCore.h.
+	 * @param a IP/port
+	 * @param et Endpoint type (default: IP_UDP)
 	 */
-	enum Type
+	ZT_INLINE Endpoint(const InetAddress &a,const Type et = ZT_ENDPOINT_TYPE_IP_UDP) noexcept
 	{
-		TYPE_NIL =          ZT_ENDPOINT_TYPE_NIL,
+		if (a) {
-		TYPE_ZEROTIER =     ZT_ENDPOINT_TYPE_ZEROTIER,
+			Utils::copy<sizeof(InetAddress)>(m_value,&a);
-		TYPE_ETHERNET =     ZT_ENDPOINT_TYPE_ETHERNET,
+			m_value[ZT_ENDPOINT_MARSHAL_SIZE_MAX-1] = (uint8_t)et;
-		TYPE_INETADDR_V4 =  ZT_ENDPOINT_TYPE_INETADDR_V4,
+		} else {
-		TYPE_INETADDR_V6 =  ZT_ENDPOINT_TYPE_INETADDR_V6
+			memoryZero(this);
-	};
+		}
+	}
 
 	/**
-	 * Protocol identifier bits.
+	 * Create an endpoint for ZeroTier relaying (ZEROTIER type)
+	 * 
+	 * @param zt_ ZeroTier identity fingerprint
 	 */
-	enum Protocol
+	ZT_INLINE Endpoint(const Fingerprint &zt_) noexcept
 	{
-		PROTO_DGRAM =       ZT_ENDPOINT_PROTO_DGRAM,
+		if (zt_) {
-		PROTO_STREAM  =     ZT_ENDPOINT_PROTO_STREAM,
+			Utils::copy<sizeof(Fingerprint)>(m_value,&zt_);
-		PROTO_HTTP2 =       ZT_ENDPOINT_PROTO_HTTP2,
+			m_value[ZT_ENDPOINT_MARSHAL_SIZE_MAX-1] = (uint8_t)ZT_ENDPOINT_TYPE_ZEROTIER;
-		PROTO_HTTPS2 =      ZT_ENDPOINT_PROTO_HTTPS2,
+		} else {
-		PROTO_WS =          ZT_ENDPOINT_PROTO_WS,
+			memoryZero(this);
-		PROTO_WEBRTC =      ZT_ENDPOINT_PROTO_WEBRTC,
+		}
-		PROTO_WIREGUARD =   ZT_ENDPOINT_PROTO_WIREGUARD
+	}
-	};
-
-	ZT_INLINE Endpoint() noexcept { memoryZero(this); } // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-
-	explicit Endpoint(const InetAddress &sa,Protocol proto = PROTO_DGRAM) noexcept;
 
 	/**
-	 * @return True if this is an IPv4 or IPv6 IP address
+	 * Create an endpoint for a type that uses a MAC address
+	 * 
+	 * @param eth_ Ethernet address
+	 * @param et Endpoint type (default: ETHERNET)
 	 */
-	ZT_INLINE bool isInetAddr() const noexcept { return ((_t == TYPE_INETADDR_V4)||(_t == TYPE_INETADDR_V6)); }
+	ZT_INLINE Endpoint(const MAC &eth_,const Type et = ZT_ENDPOINT_TYPE_ETHERNET) noexcept
+	{
+		if (eth_) {
+			Utils::copy<sizeof(MAC)>(m_value,&eth_);
+			m_value[ZT_ENDPOINT_MARSHAL_SIZE_MAX-1] = (uint8_t)et;
+		} else {
+			memoryZero(this);
+		}
+	}
 
 	/**
-	 * @return InetAddress or NIL if not of this type
+	 * @return Endpoint type
 	 */
-	ZT_INLINE const InetAddress &inetAddr() const noexcept { return ((_t == TYPE_INETADDR_V4) || (_t == TYPE_INETADDR_V6)) ? asInetAddress(_v.sa) : InetAddress::NIL; }
+	ZT_INLINE Type type() const noexcept { return (Type)m_value[ZT_ENDPOINT_MARSHAL_SIZE_MAX-1]; }
 
 	/**
-	 * @return Protocol bit mask
+	 * @return True if endpoint type isn't NIL
 	 */
-	ZT_INLINE Protocol protocol() const noexcept { return _proto; }
+	ZT_INLINE operator bool() const noexcept { return (m_value[ZT_ENDPOINT_MARSHAL_SIZE_MAX] != (uint8_t)ZT_ENDPOINT_TYPE_NIL); }
 
 	/**
-	 * @return 384-bit hash of identity keys or NULL if not of this type
+	 * @return True if this endpoint type has an InetAddress address type and thus ip() is valid
 	 */
-	ZT_INLINE const Fingerprint &fingerprint() const noexcept { return *reinterpret_cast<const Fingerprint *>(&_v.zt); }
+	ZT_INLINE bool isInetAddr() const noexcept
+	{
+		switch(this->type()) {
+			case ZT_ENDPOINT_TYPE_IP:
+			case ZT_ENDPOINT_TYPE_IP_UDP:
+			case ZT_ENDPOINT_TYPE_IP_TCP:
+			case ZT_ENDPOINT_TYPE_IP_HTTP2:
+				return true;
+			default:
+				return false;
+		}
+	}
 
 	/**
-	 * @return Ethernet address or NIL if not of this type
+	 * Get InetAddress if this type uses IPv4 or IPv6 addresses (undefined otherwise)
+	 * 
+	 * @return InetAddress instance
 	 */
-	ZT_INLINE MAC ethernet() const noexcept { return (_t == TYPE_ETHERNET) ? MAC(_v.eth) : MAC(); }
+	ZT_INLINE const InetAddress &ip() const noexcept { return *reinterpret_cast<const InetAddress *>(m_value); }
 
 	/**
-	 * @return Endpoint type or NIL if unset/empty
+	 * Get MAC if this is an Ethernet, WiFi direct, or Bluetooth type (undefined otherwise)
+	 *
+	 * @return Ethernet MAC
 	 */
-	ZT_INLINE Type type() const noexcept { return _t; }
+	ZT_INLINE const MAC &eth() const noexcept { return *reinterpret_cast<const MAC *>(m_value); }
 
-	ZT_INLINE operator bool() const noexcept { return _t != TYPE_NIL; } // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
+	/**
-
+	 * Get fingerprint if this is a ZeroTier endpoint type (undefined otherwise)
-	bool operator==(const Endpoint &ep) const noexcept;
+	 * 
-	ZT_INLINE bool operator!=(const Endpoint &ep) const noexcept { return (!(*this == ep)); }
+	 * @return ZeroTier fingerprint
-	bool operator<(const Endpoint &ep) const noexcept;
+	 */
-	ZT_INLINE bool operator>(const Endpoint &ep) const noexcept { return (ep < *this); }
+	ZT_INLINE const Fingerprint &zt() const noexcept { return *reinterpret_cast<const Fingerprint *>(m_value); }
-	ZT_INLINE bool operator<=(const Endpoint &ep) const noexcept { return !(ep < *this); }
-	ZT_INLINE bool operator>=(const Endpoint &ep) const noexcept { return !(*this < ep); }
 
 	static constexpr int marshalSizeMax() noexcept { return ZT_ENDPOINT_MARSHAL_SIZE_MAX; }
 	int marshal(uint8_t data[ZT_ENDPOINT_MARSHAL_SIZE_MAX]) const noexcept;
 	int unmarshal(const uint8_t *restrict data,int len) noexcept;
 
 private:
-	Type _t;
+	uint8_t m_value[ZT_ENDPOINT_MARSHAL_SIZE_MAX]; // the last byte in this buffer is the type
-	Protocol _proto;
-	int _l[3]; // X,Y,Z location in kilometers from the nearest gravitational center of mass (e.g. Earth)
-	union {
-		sockaddr_storage sa;
-		ZT_Fingerprint zt;
-		uint8_t eth[6];
-	} _v;
 };
 
 } // namespace ZeroTier

node/EphemeralKey.hpp

@@ -0,0 +1,135 @@
+/*
+ * Copyright (c)2013-2020 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2024-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#ifndef ZT_EPHEMERALKEY_HPP
+#define ZT_EPHEMERALKEY_HPP
+
+#include "Constants.hpp"
+#include "C25519.hpp"
+#include "ECC384.hpp"
+#include "SHA512.hpp"
+#include "Utils.hpp"
+
+#define ZT_EPHEMERALKEY_PUBLIC_SIZE (1 + ZT_C25519_ECDH_PUBLIC_KEY_SIZE + ZT_ECC384_PUBLIC_KEY_SIZE)
+
+namespace ZeroTier {
+
+/**
+ * Container for ephemeral key pair sets used in forward secrecy negotiation.
+ * 
+ * The ephemeral public key consists of public key(s) prefixed by a type byte.
+ * In the current version there are two keys: a Curve25519 ECDH public key and
+ * a NIST P-384 public key. Both are sent, and key agreement is performed by
+ * agreeing with both and then hashing the results together with the long-lived
+ * identity shared secret to produce a shared symmetric ephemeral key.
+ * 
+ * Unlike identities the private key never leaves this class. It dies when
+ * a new key pair is generated or when the node is shut down.
+ * 
+ * Each peer holds a copy of its current ephemeral key. This is re-generated
+ * after one half ZT_SYMMETRIC_KEY_TTL or after the the symmetric key has
+ * been used one half of ZT_SYMMETRIC_KEY_TTL_MESSAGES times. Half the total
+ * TTL is chosen to provide plenty of margin.
+ */
+class EphemeralKey
+{
+public:
+	enum Type
+	{
+		TYPE_NIL =         0,
+		TYPE_C25519_P384 = ZT_CRYPTO_ALG_P384
+	};
+
+	/**
+	 * The ephemeral public key(s)
+	 * 
+	 * This is sent with HELLO or OK(HELLO) and is re-written when
+	 * generate() is called. Its size is static.
+	 */
+	const uint8_t pub[ZT_EPHEMERALKEY_PUBLIC_SIZE];
+
+	/**
+	 * Create an uninitialized ephemeral key (must call generate())
+	 */
+	ZT_INLINE EphemeralKey() noexcept :
+		pub({0})
+	{
+		Utils::memoryLock(this,sizeof(EphemeralKey));
+	}
+
+	ZT_INLINE ~EphemeralKey() noexcept
+	{
+		Utils::burn(m_priv,sizeof(m_priv));
+		Utils::memoryUnlock(this,sizeof(EphemeralKey));
+	}
+
+	/**
+	 * @return True if this ephemeral key has been initialized with generate()
+	 */
+	ZT_INLINE operator bool() const noexcept { return pub[0] != (uint8_t)TYPE_NIL; }
+
+	/**
+	 * Generate or re-generate key pair.
+	 */
+	ZT_INLINE void generate() noexcept
+	{
+		uint8_t *const p = const_cast<uint8_t *>(pub);
+		p[0] = (uint8_t)TYPE_C25519_P384;
+		C25519::generateC25519(p + 1,m_priv);
+		ECC384GenerateKey(p + 1 + ZT_C25519_ECDH_PUBLIC_KEY_SIZE,m_priv + ZT_C25519_ECDH_PRIVATE_KEY_SIZE);
+	}
+
+	/**
+	 * Execute key agreement with another ephemeral public key set.
+	 * 
+	 * Final key is produced by hashing the two ECDH keys followed by
+	 * the identity secret key with SHA384.
+	 * 
+	 * @param identityKey Raw identity key shared between this node and peer
+	 * @param otherPub Other public key (prefixed by type)
+	 * @param key Key buffer to fill with symmetric key
+	 * @return True on success
+	 */
+	ZT_INLINE bool agree(const uint8_t identityKey[ZT_SYMMETRIC_KEY_SIZE],const uint8_t *otherPub,const unsigned int otherPubLength,uint8_t key[ZT_SYMMETRIC_KEY_SIZE]) const noexcept
+	{
+		if ((otherPubLength < ZT_EPHEMERALKEY_PUBLIC_SIZE)||(otherPub[0] != (uint8_t)TYPE_C25519_P384))
+			return false;
+		uint8_t tmp[ZT_C25519_ECDH_SHARED_SECRET_SIZE + ZT_ECC384_SHARED_SECRET_SIZE];
+		C25519::agree(m_priv,otherPub + 1,tmp);
+		if (!ECC384ECDH(otherPub + 1 + ZT_C25519_ECDH_PUBLIC_KEY_SIZE,m_priv + ZT_C25519_ECDH_PRIVATE_KEY_SIZE,tmp + ZT_C25519_ECDH_SHARED_SECRET_SIZE))
+			return false;
+		SHA384(key,tmp,ZT_C25519_ECDH_SHARED_SECRET_SIZE + ZT_ECC384_SHARED_SECRET_SIZE,identityKey,ZT_SYMMETRIC_KEY_SIZE);
+		Utils::burn(tmp,ZT_C25519_ECDH_SHARED_SECRET_SIZE + ZT_ECC384_SHARED_SECRET_SIZE);
+		return true;
+	}
+
+	/**
+	 * Check and see if an acknowledgement hash returned via OK(HELLO) matches our public key.
+	 * 
+	 * @param ackHash Hash provided in OK(HELLO)
+	 * @return True if this matches the hash of this ephemeral key
+	 */
+	ZT_INLINE bool acknowledged(const uint8_t ackHash[ZT_SHA384_DIGEST_SIZE]) const noexcept
+	{
+		uint8_t h[ZT_SHA384_DIGEST_SIZE];
+		SHA384(h,pub,ZT_EPHEMERALKEY_PUBLIC_SIZE);
+		return Utils::secureEq(ackHash,h,ZT_SHA384_DIGEST_SIZE);
+	}
+
+private:
+	uint8_t m_priv[ZT_C25519_ECDH_PRIVATE_KEY_SIZE + ZT_ECC384_PRIVATE_KEY_SIZE];
+};
+
+} // namespace ZeroTier
+
+#endif

node/InetAddress.cpp

@@ -346,21 +346,21 @@ int InetAddress::marshal(uint8_t data[ZT_INETADDRESS_MARSHAL_SIZE_MAX]) const no
 
 int InetAddress::unmarshal(const uint8_t *restrict data,const int len) noexcept
 {
-	if (len <= 0)
-		return -1;
 	memoryZero(this);
+	if (unlikely(len <= 0))
+		return -1;
 	switch(data[0]) {
 		case 0:
 			return 1;
 		case 4:
-			if (len < 7)
+			if (unlikely(len < 7))
 				return -1;
 			as.sa_in.sin_family = AF_INET;
 			as.sa_in.sin_port = Utils::loadAsIsEndian<uint16_t>(data + 5);
 			as.sa_in.sin_addr.s_addr = Utils::loadAsIsEndian<uint32_t>(data + 1);
 			return 7;
 		case 6:
-			if (len < 19)
+			if (unlikely(len < 19))
 				return -1;
 			as.sa_in6.sin6_family = AF_INET6;
 			as.sa_in6.sin6_port = Utils::loadAsIsEndian<uint16_t>(data + 17);

node/OS.hpp

@@ -195,6 +195,6 @@ typedef unsigned uint128_t __attribute__((mode(TI)));
 #endif
 
 // Macro to print very verbose tracing information to standard error.
-#define ZT_SPEW(f,...) fprintf(stderr,"%s(%d): " f ZT_EOL_S,__FILE__,__LINE__,__VA_ARGS__)
+#define ZT_SPEW(f,...) fprintf(stderr,"%s:%d(%s): " f ZT_EOL_S,__FILE__,__LINE__,__FUNCTION__,__VA_ARGS__)
 
 #endif

node/Peer.cpp

@@ -27,6 +27,7 @@ namespace ZeroTier {
 
 Peer::Peer(const RuntimeEnvironment *renv) :
 	RR(renv),
+	m_ephemeralPairTimestamp(0),
 	m_lastReceive(0),
 	m_lastSend(0),
 	m_lastSentHello(),
@@ -61,7 +62,7 @@ bool Peer::init(const Identity &peerIdentity)
 	uint8_t k[ZT_SYMMETRIC_KEY_SIZE];
 	if (!RR->identity.agree(peerIdentity,k))
 		return false;
-	m_identityKey.set(new SymmetricKey(RR->node->now(),k,true));
+	m_identityKey.set(new SymmetricKey(RR->node->now(),k));
 	Utils::burn(k,sizeof(k));
 
 	m_deriveSecondaryIdentityKeys();
@@ -113,7 +114,7 @@ void Peer::received(
 							// and other wacky stuff can change port number assignments.
 							m_paths[i] = path;
 							return;
-						} else if (m_paths[i]->lastIn() > lastReceiveTimeMax) {
+						} else if (m_paths[i]->lastIn() >= lastReceiveTimeMax) {
 							lastReceiveTimeMax = m_paths[i]->lastIn();
 							newPathIdx = i;
 						}
@@ -193,9 +194,13 @@ void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
 {
 	RWMutex::Lock l(m_lock);
 
+	// Determine if we need to send a full HELLO because we are refreshing ephemeral
+	// keys or it's simply been too long.
 	bool needHello = false;
-	if ((now - m_lastSentHello) >= ZT_PEER_HELLO_INTERVAL) {
+	if ( ((now - m_ephemeralPairTimestamp) >= (ZT_SYMMETRIC_KEY_TTL / 2)) || ((m_ephemeralKeys[0])&&(m_ephemeralKeys[0]->odometer() >= (ZT_SYMMETRIC_KEY_TTL_MESSAGES / 2))) ) {
-		m_lastSentHello = now;
+		m_ephemeralPair.generate();
+		needHello = true;
+	} else if ((now - m_lastSentHello) >= ZT_PEER_HELLO_INTERVAL) {
 		needHello = true;
 	}
 
@@ -217,9 +222,9 @@ void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
 				if (tryAtIndex > 0) {
 					--tryAtIndex;
 				} else {
-					if ((i->second.isInetAddr())&&(!i->second.inetAddr().ipsEqual(addr))) {
+					if ((i->second.isInetAddr())&&(!i->second.ip().ipsEqual(addr))) {
-						RR->t->tryingNewPath(tPtr, 0x0a009444, m_id, i->second.inetAddr(), InetAddress::NIL, 0, 0, Identity::NIL);
+						RR->t->tryingNewPath(tPtr, 0x0a009444, m_id, i->second.ip(), InetAddress::NIL, 0, 0, Identity::NIL);
-						sent(now,m_sendProbe(tPtr,-1,i->second.inetAddr(),now));
+						sent(now,m_sendProbe(tPtr,-1,i->second.ip(),now));
 						break;
 					}
 				}
@@ -231,19 +236,22 @@ void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
 
 	// Attempt queued paths to try.
 	for(int k=0;(k<ZT_NAT_T_MAX_QUEUED_ATTEMPTS_PER_PULSE)&&(!m_tryQueue.empty());++k) {
+		// This is a global circular pointer that iterates through the list of
+		// endpoints to attempt.
 		if (m_tryQueuePtr == m_tryQueue.end())
 			m_tryQueuePtr = m_tryQueue.begin();
 
+		// Delete timed out entries.
 		if ((now - m_tryQueuePtr->ts) > ZT_PATH_ALIVE_TIMEOUT) {
 			m_tryQueue.erase(m_tryQueuePtr++);
 			continue;
 		}
 
 		if (m_tryQueuePtr->target.isInetAddr()) {
-			// Make sure target does not overlap with any existing path.
+			// Delete entries that duplicate existing alive paths.
 			bool duplicate = false;
 			for(unsigned int i=0;i<m_alivePathCount;++i) {
-				if (m_paths[i]->address() == m_tryQueuePtr->target.inetAddr()) {
+				if (m_paths[i]->address() == m_tryQueuePtr->target.ip()) {
 					duplicate = true;
 					break;
 				}
@@ -268,27 +276,29 @@ void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
 						ports[b] = tmp;
 					}
 				}
-				InetAddress addr(m_tryQueuePtr->target.inetAddr());
+				InetAddress addr(m_tryQueuePtr->target.ip());
 				for (unsigned int i=0;i<ZT_NAT_T_BFG1024_PORTS_PER_ATTEMPT;++i) {
 					addr.setPort(ports[i]);
 					sent(now,m_sendProbe(tPtr,-1,addr,now));
 				}
 			} else {
 				// Otherwise send a normal probe.
-				sent(now,m_sendProbe(tPtr, -1, m_tryQueuePtr->target.inetAddr(), now));
+				sent(now,m_sendProbe(tPtr, -1, m_tryQueuePtr->target.ip(), now));
 			}
 		}
 
 		++m_tryQueuePtr;
 	}
 
-	// Do keepalive on all currently active paths.
+	// Do keepalive on all currently active paths, sending HELLO to the first
+	// if needHello is true and sending small keepalives to others.
 	for(unsigned int i=0;i<m_alivePathCount;++i) {
 		if (needHello) {
 			needHello = false;
 			const unsigned int bytes = hello(tPtr, m_paths[i]->localSocket(), m_paths[i]->address(), now);
 			m_paths[i]->sent(now, bytes);
 			sent(now,bytes);
+			m_lastSentHello = now;
 		} else if ((now - m_paths[i]->lastOut()) >= ZT_PATH_KEEPALIVE_PERIOD) {
 			m_paths[i]->send(RR, tPtr, &now, 1, now);
 			sent(now,1);
@@ -306,15 +316,26 @@ void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
 				via->sent(now,bytes);
 				root->relayed(now,bytes);
 				sent(now,bytes);
+				m_lastSentHello = now;
 			}
 		}
 	}
 }
 
-void Peer::tryDirectPath(const int64_t now,const Endpoint &ep,const bool breakSymmetricBFG1024)
+void Peer::contact(void *tPtr,const int64_t now,const Endpoint &ep,const bool breakSymmetricBFG1024)
 {
+	static uint8_t foo = 0;
 	RWMutex::Lock l(m_lock);
 
+	if (ep.isInetAddr()&&ep.ip().isV4()) {
+		// For IPv4 addresses we send a tiny packet with a low TTL, which helps to
+		// traverse some NAT types. It has no effect otherwise. It's important to
+		// send this right away in case this is a coordinated attempt via RENDEZVOUS.
+		RR->node->putPacket(tPtr,-1,ep.ip(),&foo,1,2);
+		++foo;
+	}
+
+	// Check to see if this endpoint overlaps an existing queue item. If so, just update it.
 	for(List<p_TryQueueItem>::iterator i(m_tryQueue.begin());i!=m_tryQueue.end();++i) {
 		if (i->target == ep) {
 			i->ts = now;
@@ -323,6 +344,7 @@ void Peer::tryDirectPath(const int64_t now,const Endpoint &ep,const bool breakSy
 		}
 	}
 
+	// Add endpoint to endpoint attempt queue.
 #ifdef __CPP11__
 	m_tryQueue.emplace_back(now, ep, breakSymmetricBFG1024);
 #else
@@ -456,7 +478,7 @@ int Peer::unmarshal(const uint8_t *restrict data,const int len) noexcept
 		RR->localCacheSymmetric.decrypt(data + 1,k);
 		RR->localCacheSymmetric.decrypt(data + 17,k + 16);
 		RR->localCacheSymmetric.decrypt(data + 33,k + 32);
-		m_identityKey.set(new SymmetricKey(RR->node->now(),k,true));
+		m_identityKey.set(new SymmetricKey(RR->node->now(),k));
 		Utils::burn(k,sizeof(k));
 	}
 
@@ -471,7 +493,7 @@ int Peer::unmarshal(const uint8_t *restrict data,const int len) noexcept
 		uint8_t k[ZT_SYMMETRIC_KEY_SIZE];
 		if (!RR->identity.agree(m_id,k))
 			return -1;
-		m_identityKey.set(new SymmetricKey(RR->node->now(),k,true));
+		m_identityKey.set(new SymmetricKey(RR->node->now(),k));
 		Utils::burn(k,sizeof(k));
 	}
 
@@ -505,6 +527,8 @@ int Peer::unmarshal(const uint8_t *restrict data,const int len) noexcept
 	m_vRevision = Utils::loadBigEndian<uint16_t>(data + p); p += 2;
 	p += 2 + (int)Utils::loadBigEndian<uint16_t>(data + p);
 
+	m_deriveSecondaryIdentityKeys();
+
 	return (p > len) ? -1 : p;
 }
 

node/Peer.hpp

@@ -28,6 +28,7 @@
 #include "Locator.hpp"
 #include "Protocol.hpp"
 #include "AES.hpp"
+#include "EphemeralKey.hpp"
 #include "SymmetricKey.hpp"
 #include "Containers.hpp"
 
@@ -226,13 +227,14 @@ public:
 	void pulse(void *tPtr,int64_t now,bool isRoot);
 
 	/**
-	 * Add a potential candidate direct path to the P2P "try" queue.
+	 * Attempt to contact this peer at a given endpoint.
 	 *
+	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
 	 * @param now Current time
 	 * @param ep Endpoint to attempt to contact
 	 * @param bfg1024 Use BFG1024 brute force symmetric NAT busting algorithm if applicable
 	 */
-	void tryDirectPath(int64_t now,const Endpoint &ep,bool breakSymmetricBFG1024);
+	void contact(void *tPtr,int64_t now,const Endpoint &ep,bool breakSymmetricBFG1024);
 
 	/**
 	 * Reset paths within a given IP scope and address family
@@ -467,6 +469,10 @@ private:
 	// Key for HELLO HMAC-SHA384
 	uint8_t m_helloMacKey[ZT_SYMMETRIC_KEY_SIZE];
 
+	// Currently active ephemeral public key pair
+	EphemeralKey m_ephemeralPair;
+	int64_t m_ephemeralPairTimestamp;
+
 	// Current and previous ephemeral key
 	SharedPtr<SymmetricKey> m_ephemeralKeys[2];
 

node/Protocol.hpp

@@ -256,9 +256,8 @@
 #define ZT_PROTO_HELLO_NODE_META_SOFTWARE_VERSION "v"
 #define ZT_PROTO_HELLO_NODE_META_PHYSICAL_DEST    "d"
 #define ZT_PROTO_HELLO_NODE_META_COMPLIANCE       "c"
-#define ZT_PROTO_HELLO_NODE_META_EPHEMERAL_C25519 "0"
+#define ZT_PROTO_HELLO_NODE_META_EPHEMERAL_PUBLIC "e"
-#define ZT_PROTO_HELLO_NODE_META_EPHEMERAL_P384   "1"
+#define ZT_PROTO_HELLO_NODE_META_EPHEMERAL_ACK    "E"
-#define ZT_PROTO_HELLO_NODE_META_EPHEMERAL_REMOTE "R"
 
 static_assert(ZT_PROTO_MAX_PACKET_LENGTH < ZT_BUF_MEM_SIZE,"maximum packet length won't fit in Buf");
 static_assert(ZT_PROTO_PACKET_ENCRYPTED_SECTION_START == (ZT_PROTO_MIN_PACKET_LENGTH-1),"encrypted packet section must start right before protocol verb at one less than minimum packet size");
@@ -324,7 +323,7 @@ enum Verb
 	 *
 	 * LEGACY: for legacy reasons the MAC field of HELLO is a poly1305
 	 * MAC initialized in the same manner as 1.x. Since HMAC provides
-	 * additional full 256-bit strength authentication this should not be
+	 * additional full 384-bit strength authentication this should not be
 	 * a problem for FIPS.
 	 *
 	 * Several legacy fields are present as well for the benefit of 1.x nodes.
@@ -339,32 +338,26 @@ enum Verb
 	 * ignore them due to the "encrypted zero" field indicating that the
 	 * packet contains no more information.
 	 *
-	 * Dictionary fields:
+	 * Dictionary fields (defines start with ZT_PROTO_HELLO_NODE_META_):
-	 *
-	 * The following fields are always present in HELLO:
 	 * 
 	 *   INSTANCE_ID - a 64-bit unique value generated on each node start
 	 *   LOCATOR - signed record enumerating this node's trusted contact points
 	 *   PROBE_TOKEN - 32-bit probe token
+	 *   EPHEMERAL_PUBLIC - Ephemeral public key(s)
 	 * 
-	 * The following fields are used to establish forward secrecy:
+	 * OK will contain EPHEMERAL_PUBLIC (of the sender) and:
 	 * 
-	 *   EPHEMERAL_C25519 - C25519 ephemeral public key (32 bytes)
+	 *   EPHEMERAL_ACK - SHA384 of EPHEMERAL_PUBLIC received
-	 *   EPHEMERAL_P384 - NIST P-384 ephemneral public key (49 bytes)
-	 *   EPHEMERAL_REMOTE - SHA-384 of keys we have for peer (absent if none)
 	 *
 	 * The following optional fields may also be present:
 	 *
-	 *   NAME - arbitrary short user-defined name for this node
+	 *   HOSTNAME - arbitrary short host name for this node
 	 *   CONTACT - arbitrary short contact information string for this node
 	 *   SOFTWARE_VENDOR - short name or description of vendor, such as a URL
 	 *   SOFTWARE_VERSION - major, minor, revision, and build (packed 64-bit int)
 	 *   PHYSICAL_DEST - serialized Endpoint to which this message was sent
 	 *   COMPLIANCE - bit mask containing bits for e.g. a FIPS-compliant node
 	 *
-	 * The actual keys for these fields are in corresponding #defines by these
-	 * names.
-	 *
 	 * The timestamp field in OK is echoed but the others represent the sender
 	 * of the OK and are not echoes from HELLO. The dictionary in OK typically
 	 * only contains the EPHEMERAL fields, allowing the receiver of the OK to
@@ -378,8 +371,14 @@ enum Verb
 	 *   <[...] dictionary>
 	 *   <[48] HMAC-SHA384 of plaintext packet>
 	 *
-	 * LEGACY: a legacy format OK will be sent to nodes with older protocol
+	 * Legacy OK payload (sent to pre-2.x nodes):
-	 * versions.
+	 *   <[8] timestamp echoed from original HELLO>
+	 *   <[1] protocol version of responding node>
+	 *   <[1] software major version>
+	 *   <[1] software minor version>
+	 *   <[2] software revision>
+	 *   <[...] physical destination address of packet>
+	 *   <[2] 16-bit zero length of additional fields>
 	 */
 	VERB_HELLO = 0x01,
 

node/ScopedPtr.hpp

@@ -33,7 +33,6 @@ public:
 
 	ZT_INLINE T *operator->() const noexcept { return m_ptr; }
 	ZT_INLINE T &operator*() const noexcept { return *m_ptr; }
-	explicit ZT_INLINE operator bool() const noexcept { return (m_ptr != (T *)0); }
 	ZT_INLINE T *ptr() const noexcept { return m_ptr; }
 
 	ZT_INLINE void swap(const ScopedPtr &p) noexcept
@@ -43,6 +42,8 @@ public:
 		p.m_ptr = tmp;
 	}
 
+	explicit ZT_INLINE operator bool() const noexcept { return (m_ptr != (T *)0); }
+
 	ZT_INLINE bool operator==(const ScopedPtr &p) const noexcept { return (m_ptr == p.m_ptr); }
 	ZT_INLINE bool operator!=(const ScopedPtr &p) const noexcept { return (m_ptr != p.m_ptr); }
 	ZT_INLINE bool operator==(T *const p) const noexcept { return (m_ptr == p); }

node/SymmetricKey.hpp

@@ -24,10 +24,7 @@
 namespace ZeroTier {
 
 /**
- * Container for symmetric keys and ciphers initialized with them
+ * Container for symmetric keys and ciphers initialized with them.
- *
- * This container is responsible for tracking key TTL to maintain it
- * below our security bounds and tell us when it's time to re-key.
  */
 class SymmetricKey
 {
@@ -63,15 +60,13 @@ public:
 	 * 
 	 * @param ts Current time
 	 * @param key 48-bit / 384-byte key
-	 * @param perm If true this is a permanent key
 	 */
-	explicit ZT_INLINE SymmetricKey(const int64_t ts,const void *const key,const bool perm) noexcept :
+	explicit ZT_INLINE SymmetricKey(const int64_t ts,const void *const key) noexcept :
 		secret(),
-		cipher(key), // uses first 256 bits of 384-bit key
+		cipher(key), // AES-256 uses first 256 bits of 384-bit key
-		m_ts(ts),
+		m_initialNonce(((((uint64_t)ts / 1000ULL) << 32U) & 0x7fffffff00000000ULL) | (Utils::random() & 0x00000000ffffffffULL)),
-		m_nonceBase(((((uint64_t)ts / 1000ULL) << 32U) & 0x7fffffff00000000ULL) | (Utils::random() & 0x00000000ffffffffULL)),
+		m_nonce(m_initialNonce),
-		m_odometer(0),
+		__refCount(0)
-		m_permanent(perm)
 	{
 		Utils::memoryLock(this,sizeof(SymmetricKey));
 		Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(const_cast<uint8_t *>(secret), key);
@@ -83,28 +78,6 @@ public:
 		Utils::memoryUnlock(this,sizeof(SymmetricKey));
 	}
 
-	/**
-	 * Check whether this symmetric key may be expiring soon
-	 *
-	 * @param now Current time
-	 * @return True if re-keying should happen
-	 */
-	ZT_INLINE bool expiringSoon(const int64_t now) const noexcept
-	{
-		return (!m_permanent) && (((now - m_ts) >= (ZT_SYMMETRIC_KEY_TTL / 2)) || (m_odometer >= (ZT_SYMMETRIC_KEY_TTL_MESSAGES / 2)) );
-	}
-
-	/**
-	 * Check whether this symmetric key is expired due to too much time or too many messages
-	 *
-	 * @param now Current time
-	 * @return True if this symmetric key should no longer be used
-	 */
-	ZT_INLINE bool expired(const int64_t now) const noexcept
-	{
-		return (!m_permanent) && (((now - m_ts) >= ZT_SYMMETRIC_KEY_TTL) || (m_odometer >= ZT_SYMMETRIC_KEY_TTL_MESSAGES) );
-	}
-
 	/**
 	 * Advance usage counter by one and return the next IV / packet ID.
 	 *
@@ -114,15 +87,21 @@ public:
 	 */
 	ZT_INLINE uint64_t nextMessage(const Address sender,const Address receiver) noexcept
 	{
-		return (m_nonceBase + m_odometer++) ^ (((uint64_t)(sender > receiver)) << 63U);
+		return m_nonce.fetch_add(1) ^ (((uint64_t)(sender > receiver)) << 63U);
+	}
+
+	/**
+	 * @return Number of times nextMessage() has been called since object creation
+	 */
+	ZT_INLINE uint64_t odometer() const noexcept
+	{
+		return m_nonce.load() - m_initialNonce;
 	}
 
 private:
-	const int64_t m_ts;
+	const uint64_t m_initialNonce;
-	const uint64_t m_nonceBase;
+	std::atomic<uint64_t> m_nonce;
-	std::atomic<uint64_t> m_odometer;
 	std::atomic<int> __refCount;
-	const bool m_permanent;
 };
 
 } // namespace ZeroTier

node/Trace.hpp

@@ -21,6 +21,7 @@
 #include "Address.hpp"
 #include "MAC.hpp"
 #include "Containers.hpp"
+#include "Utils.hpp"
 
 #define ZT_TRACE_F_VL1           0x01U
 #define ZT_TRACE_F_VL2           0x02U

node/Utils.hpp

@@ -659,10 +659,7 @@ static ZT_INLINE void copy(void *const dest,const void *const src) noexcept
  * @param src Source memory
  * @param len Bytes to copy
  */
-static ZT_INLINE void copy(void *const dest,const void *const src,unsigned int len) noexcept
+static ZT_INLINE void copy(void *const dest,const void *const src,unsigned int len) noexcept { memcpy(dest,src,len); }
-{
-	memcpy(dest,src,len);
-}
 
 /**
  * Zero memory block whose size is known at compile time
@@ -718,13 +715,14 @@ static ZT_INLINE void zero(void *const dest) noexcept
  * @param dest Memory to zero
  * @param len Size in bytes
  */
-static ZT_INLINE void zero(void *const dest,const unsigned int len) noexcept
+static ZT_INLINE void zero(void *const dest,const unsigned int len) noexcept { memset(dest,0,len); }
-{
-	memset(dest,0,len);
-}
 
 /**
- * Simple malloc/free based C++ STL allocator
+ * Simple malloc/free based C++ STL allocator.
+ * 
+ * This is used to make sure our containers don't use weird libc++
+ * allocators but instead use whatever malloc() is, which in turn
+ * can be overridden by things like jemaclloc or tcmalloc.
  *
  * @tparam T Allocated type
  */

node/VL1.cpp

@@ -83,27 +83,12 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 	const SharedPtr<Path> path(RR->topology->path(localSocket,fromAddr));
 	const int64_t now = RR->node->now();
 
-	// Update path's last receive time (this is updated when anything is received at all, even if invalid or a keepalive)
+	ZT_SPEW("%u bytes from %s (local socket %lld)",len,fromAddr.toString().c_str(),localSocket);
 	path->received(now,len);
 
-	try {
+	// NOTE: likely/unlikely are used here to highlight the most common code path
-		// Handle short probes, which are used as a low-bandwidth way to initiate a real handshake.
+	// for valid data packets. This may allow the compiler to generate very slightly
-		// These are subjected to a significant rate limit to prevent DOS or amplification attacks.
+	// faster code for that path.
-		// The probe itself is a token passed via HELLO, so these are only used with peers we've
-		// already started communicating with.
-		if (unlikely(len == ZT_PROTO_PROBE_LENGTH)) {
-			PeerList peers(RR->topology->peersByProbeToken(data->lI32(0)));
-			for(unsigned int pi=0;pi<peers.size();++pi) {
-				if (peers[pi]->rateGateProbeRequest(now))
-					peers[pi]->hello(tPtr,localSocket,fromAddr,now);
-			}
-			return;
-		}
-
-		// Any other "runt" packets are discarded, though they still count toward a path's
-		// last receive time as they may be keepalives.
-		if (unlikely(len < ZT_PROTO_MIN_FRAGMENT_LENGTH))
-			return;
 
 	/*
 	 * Packet format:
@@ -117,6 +102,24 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 	 *   [... verb-specific payload ...]
 	 */
 
+	try {
+		// If this is too short to be a packet or fragment, check if it's a probe and
+		// if not simply drop it.
+		if (unlikely(len < ZT_PROTO_MIN_FRAGMENT_LENGTH)) {
+			if (len == ZT_PROTO_PROBE_LENGTH) {
+				const uint32_t probeToken = data->lI32(0);
+				PeerList peers(RR->topology->peersByProbeToken(probeToken));
+				ZT_SPEW("probe %.8x matches %u peers",(unsigned long)probeToken,peers.size());
+				for(unsigned int pi=0;pi<peers.size();++pi) {
+					if (peers[pi]->rateGateProbeRequest(now)) {
+						ZT_SPEW("HELLO -> %s(%s)",peers[pi]->address().toString().c_str(),fromAddr.toString().c_str());
+						peers[pi]->hello(tPtr,localSocket,fromAddr,now);
+					}
+				}
+			}
+			return;
+		}
+
 		static_assert((ZT_PROTO_PACKET_DESTINATION_INDEX + ZT_ADDRESS_LENGTH) < ZT_PROTO_MIN_FRAGMENT_LENGTH,"overflow");
 		Address destination(data->unsafeData + ZT_PROTO_PACKET_DESTINATION_INDEX);
 		if (destination != RR->identity.address()) {
@@ -187,7 +190,7 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 				}
 			} else {
 				// This is a single whole packet with no fragments.
-				Buf::Slice &s = pktv.push();
+				Buf::Slice s = pktv.push();
 				s.b.swap(data);
 				s.s = 0;
 				s.e = len;
@@ -199,23 +202,21 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 		// ----------------------------------------------------------------------------------------------------------------
 
 		const uint8_t *const hdr = pktv[0].b->unsafeData + pktv[0].s;
-
 		static_assert((ZT_PROTO_PACKET_SOURCE_INDEX + ZT_ADDRESS_LENGTH) < ZT_PROTO_MIN_PACKET_LENGTH,"overflow");
 		const Address source(hdr + ZT_PROTO_PACKET_SOURCE_INDEX);
 		static_assert(ZT_PROTO_PACKET_FLAGS_INDEX < ZT_PROTO_MIN_PACKET_LENGTH,"overflow");
 		const uint8_t hops = hdr[ZT_PROTO_PACKET_FLAGS_INDEX] & ZT_PROTO_FLAG_FIELD_HOPS_MASK;
 		const uint8_t cipher = (hdr[ZT_PROTO_PACKET_FLAGS_INDEX] >> 3U) & 3U;
 
-		const SharedPtr<Buf> pkt(new Buf());
+		SharedPtr<Buf> pkt(new Buf());
 		int pktSize = 0;
-		bool authenticated = false;
 
 		static_assert(ZT_PROTO_PACKET_VERB_INDEX < ZT_PROTO_MIN_PACKET_LENGTH,"overflow");
-		if ( ((cipher == ZT_PROTO_CIPHER_SUITE__POLY1305_NONE)||(cipher == ZT_PROTO_CIPHER_SUITE__NONE)) && ((hdr[ZT_PROTO_PACKET_VERB_INDEX] & ZT_PROTO_VERB_MASK) == Protocol::VERB_HELLO) ) {
+		if (unlikely( ((cipher == ZT_PROTO_CIPHER_SUITE__POLY1305_NONE)||(cipher == ZT_PROTO_CIPHER_SUITE__NONE)) && ((hdr[ZT_PROTO_PACKET_VERB_INDEX] & ZT_PROTO_VERB_MASK) == Protocol::VERB_HELLO) )) {
 			// Handle unencrypted HELLO packets.
 			pktSize = pktv.mergeCopy(*pkt);
 			if (unlikely(pktSize < ZT_PROTO_MIN_PACKET_LENGTH)) {
-				ZT_SPEW("discarding packet %.16llx from %s: assembled packet size: %d",packetId,fromAddr.toString().c_str(),pktSize);
+				ZT_SPEW("discarding packet %.16llx from %s(%s): assembled packet size: %d",packetId,source.toString().c_str(),fromAddr.toString().c_str(),pktSize);
 				return;
 			}
 			const SharedPtr<Peer> peer(m_HELLO(tPtr, path, *pkt, pktSize));
@@ -228,6 +229,11 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 		// Making it this far means the packet is not a plaintext HELLO, so do normal AEAD decrypt and packet handling.
 		// ----------------------------------------------------------------------------------------------------------------
 
+		// This remains zero if authentication fails. Otherwise it gets set to a bit mask
+		// indicating authentication and other security flags like encryption and forward
+		// secrecy status.
+		unsigned int auth = 0;
+
 		SharedPtr<Peer> peer(RR->topology->peer(tPtr,source));
 		if (peer) {
 			switch(cipher) {
@@ -239,7 +245,7 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 
 					pktSize = pktv.mergeMap<p_PolyCopyFunction &>(*pkt,ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,s20cf);
 					if (unlikely(pktSize < ZT_PROTO_MIN_PACKET_LENGTH)) {
-						ZT_SPEW("discarding packet %.16llx from %s: assembled packet size: %d",packetId,fromAddr.toString().c_str(),pktSize);
+						ZT_SPEW("discarding packet %.16llx from %s(%s): assembled packet size: %d",packetId,source.toString().c_str(),fromAddr.toString().c_str(),pktSize);
 						return;
 					}
 
@@ -247,11 +253,12 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 					s20cf.poly1305.finish(mac);
 					static_assert((ZT_PROTO_PACKET_MAC_INDEX + 8) < ZT_PROTO_MIN_PACKET_LENGTH,"overflow");
 					if (unlikely(Utils::loadAsIsEndian<uint64_t>(hdr + ZT_PROTO_PACKET_MAC_INDEX) != mac[0])) {
+						ZT_SPEW("discarding packet %.16llx from %s(%s): packet MAC failed (none/poly1305)",packetId,source.toString().c_str(),fromAddr.toString().c_str());
 						RR->t->incomingPacketDropped(tPtr,0xcc89c812,packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 						return;
 					}
 
-					authenticated = true;
+					auth = ZT_VL1_AUTH_RESULT_FLAG_AUTHENTICATED;
 				}	break;
 
 				case ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012: {
@@ -261,7 +268,7 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 
 					pktSize = pktv.mergeMap<p_SalsaPolyCopyFunction &>(*pkt,ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,s20cf);
 					if (unlikely(pktSize < ZT_PROTO_MIN_PACKET_LENGTH)) {
-						ZT_SPEW("discarding packet %.16llx from %s: assembled packet size: %d",packetId,fromAddr.toString().c_str(),pktSize);
+						ZT_SPEW("discarding packet %.16llx from %s(%s): assembled packet size: %d",packetId,source.toString().c_str(),fromAddr.toString().c_str(),pktSize);
 						return;
 					}
 
@@ -269,11 +276,12 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 					s20cf.poly1305.finish(mac);
 					static_assert((ZT_PROTO_PACKET_MAC_INDEX + 8) < ZT_PROTO_MIN_PACKET_LENGTH,"overflow");
 					if (unlikely(Utils::loadAsIsEndian<uint64_t>(hdr + ZT_PROTO_PACKET_MAC_INDEX) != mac[0])) {
+						ZT_SPEW("discarding packet %.16llx from %s(%s): packet MAC failed (salsa/poly1305)",packetId,source.toString().c_str(),fromAddr.toString().c_str());
 						RR->t->incomingPacketDropped(tPtr,0xcc89c812,packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 						return;
 					}
 
-					authenticated = true;
+					auth = ZT_VL1_AUTH_RESULT_FLAG_AUTHENTICATED | ZT_VL1_AUTH_RESULT_FLAG_ENCRYPTED;
 				}	break;
 
 				case ZT_PROTO_CIPHER_SUITE__NONE: {
@@ -290,90 +298,80 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 			}
 		}
 
-		if (likely(authenticated)) {
+		if (likely(auth != 0)) {
 			// If authentication was successful go on and process the packet.
-#if 0
-			const Protocol::Verb verb = (Protocol::Verb)(ph->verb & ZT_PROTO_VERB_MASK);
 
-			// All verbs except HELLO require authentication before being handled. The HELLO
+			if (unlikely(pktSize < ZT_PROTO_MIN_PACKET_LENGTH)) {
-			// handler does its own authentication.
+				ZT_SPEW("discarding packet %.16llx from %s(%s): assembled packet size: %d",packetId,source.toString().c_str(),fromAddr.toString().c_str(),pktSize);
-			if (((!authenticated)||(!peer))&&(verb != Protocol::VERB_HELLO)) {
-				RR->t->incomingPacketDropped(tPtr,0x5b001099,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 				return;
 			}
+			static_assert(ZT_PROTO_PACKET_VERB_INDEX < ZT_PROTO_MIN_PACKET_LENGTH,"overflow");
+			const uint8_t verbFlags = pkt->unsafeData[ZT_PROTO_PACKET_VERB_INDEX];
+			const Protocol::Verb verb = (Protocol::Verb)(verbFlags & ZT_PROTO_VERB_MASK);
 
 			// Decompress packet payload if compressed. For additional safety decompression is
 			// only performed on packets whose MACs have already been validated. (Only HELLO is
 			// sent without this, and HELLO doesn't benefit from compression.)
-			if ((ph->verb & ZT_PROTO_VERB_FLAG_COMPRESSED) != 0) {
+			if (((verbFlags & ZT_PROTO_VERB_FLAG_COMPRESSED) != 0)&&(pktSize > ZT_PROTO_PACKET_PAYLOAD_START)) {
-				if (!authenticated) {
+				SharedPtr<Buf> dec(new Buf());
-					RR->t->incomingPacketDropped(tPtr,0x390bcd0a,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
+				Utils::copy<ZT_PROTO_PACKET_PAYLOAD_START>(dec->unsafeData,pkt->unsafeData);
-					return;
-				}
-
-				SharedPtr<Buf> nb(new Buf());
 				const int uncompressedLen = LZ4_decompress_safe(
-					reinterpret_cast<const char *>(pkt.b->unsafeData + ZT_PROTO_PACKET_PAYLOAD_START),
+					reinterpret_cast<const char *>(pkt->unsafeData + ZT_PROTO_PACKET_PAYLOAD_START),
-					reinterpret_cast<char *>(nb->unsafeData),
+					reinterpret_cast<char *>(dec->unsafeData + ZT_PROTO_PACKET_PAYLOAD_START),
-					(int)(packetSize - ZT_PROTO_PACKET_PAYLOAD_START),
+					pktSize - ZT_PROTO_PACKET_PAYLOAD_START,
 					ZT_BUF_MEM_SIZE - ZT_PROTO_PACKET_PAYLOAD_START);
 
-				if ((uncompressedLen > 0)&&(uncompressedLen <= (ZT_BUF_MEM_SIZE - ZT_PROTO_PACKET_PAYLOAD_START))) {
+				if (likely((uncompressedLen > 0)&&(uncompressedLen <= (ZT_BUF_MEM_SIZE - ZT_PROTO_PACKET_PAYLOAD_START)))) {
-					pkt.b.swap(nb);
+					pkt.swap(dec);
-					pkt.e = packetSize = (unsigned int)uncompressedLen;
+					ZT_SPEW("decompressed packet: %d -> %d",pktSize,ZT_PROTO_PACKET_PAYLOAD_START + uncompressedLen);
+					pktSize = ZT_PROTO_PACKET_PAYLOAD_START + uncompressedLen;
 				} else {
-					RR->t->incomingPacketDropped(tPtr,0xee9e4392,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_INVALID_COMPRESSED_DATA);
+					RR->t->incomingPacketDropped(tPtr,0xee9e4392,packetId,0,identityFromPeerPtr(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_INVALID_COMPRESSED_DATA);
 					return;
 				}
 			}
 
-			/*
+			// NOTE: HELLO is normally sent in the clear (in terms of our usual AEAD modes) and is handled
-			* Important notes:
+			// above. We will try to process it here, but if so it'll still get re-authenticated via HELLO's
-			*
+			// own internal authentication logic as usual. It would be abnormal to make it here with HELLO
-			* All verbs except HELLO assume that authenticated is true and peer is non-NULL.
+			// but not invalid.
-			* This is checked above. HELLO will accept either case and always performs its
-			* own secondary validation. The path argument is never NULL.
-			*
-			* VL1 and VL2 are conceptually separate layers of the ZeroTier protocol. In the
-			* code they are almost entirely logically separate. To make the code easier to
-			* understand the handlers for VL2 data paths have been moved to a VL2 class.
-			*/
 
-			bool ok = true; // set to false if a packet turns out to be invalid
+			bool ok = true;
-			Protocol::Verb inReVerb = Protocol::VERB_NOP; // set via result parameter to _ERROR and _OK
+			Protocol::Verb inReVerb = Protocol::VERB_NOP;
 			switch(verb) {
 				case Protocol::VERB_NOP:                        break;
-				case Protocol::VERB_HELLO:                      ok = (bool)(m_HELLO(tPtr, path, *pkt.b, (int) packetSize)); break;
+				case Protocol::VERB_HELLO:                      ok = (bool)(m_HELLO(tPtr, path, *pkt, pktSize)); break;
-				case Protocol::VERB_ERROR:                      ok = m_ERROR(tPtr, path, peer, *pkt.b, (int) packetSize, inReVerb); break;
+				case Protocol::VERB_ERROR:                      ok = m_ERROR(tPtr, auth, path, peer, *pkt, pktSize, inReVerb); break;
-				case Protocol::VERB_OK:                         ok = m_OK(tPtr, path, peer, *pkt.b, (int) packetSize, inReVerb); break;
+				case Protocol::VERB_OK:                         ok = m_OK(tPtr, auth, path, peer, *pkt, pktSize, inReVerb); break;
-				case Protocol::VERB_WHOIS:                      ok = m_WHOIS(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_WHOIS:                      ok = m_WHOIS(tPtr, auth, path, peer, *pkt, pktSize); break;
-				case Protocol::VERB_RENDEZVOUS:                 ok = m_RENDEZVOUS(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_RENDEZVOUS:                 ok = m_RENDEZVOUS(tPtr, auth, path, peer, *pkt, pktSize); break;
-				case Protocol::VERB_FRAME:                      ok = RR->vl2->m_FRAME(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_FRAME:                      ok = RR->vl2->m_FRAME(tPtr, auth, path, peer, *pkt, pktSize); break;
-				case Protocol::VERB_EXT_FRAME:                  ok = RR->vl2->m_EXT_FRAME(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_EXT_FRAME:                  ok = RR->vl2->m_EXT_FRAME(tPtr, auth, path, peer, *pkt, pktSize); break;
-				case Protocol::VERB_ECHO:                       ok = m_ECHO(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_ECHO:                       ok = m_ECHO(tPtr, auth, path, peer, *pkt, pktSize); break;
-				case Protocol::VERB_MULTICAST_LIKE:             ok = RR->vl2->m_MULTICAST_LIKE(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_MULTICAST_LIKE:             ok = RR->vl2->m_MULTICAST_LIKE(tPtr, auth, path, peer, *pkt, pktSize); break;
-				case Protocol::VERB_NETWORK_CREDENTIALS:        ok = RR->vl2->m_NETWORK_CREDENTIALS(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_NETWORK_CREDENTIALS:        ok = RR->vl2->m_NETWORK_CREDENTIALS(tPtr, auth, path, peer, *pkt, pktSize); break;
-				case Protocol::VERB_NETWORK_CONFIG_REQUEST:     ok = RR->vl2->m_NETWORK_CONFIG_REQUEST(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_NETWORK_CONFIG_REQUEST:     ok = RR->vl2->m_NETWORK_CONFIG_REQUEST(tPtr, auth, path, peer, *pkt, pktSize); break;
-				case Protocol::VERB_NETWORK_CONFIG:             ok = RR->vl2->m_NETWORK_CONFIG(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_NETWORK_CONFIG:             ok = RR->vl2->m_NETWORK_CONFIG(tPtr, auth, path, peer, *pkt, pktSize); break;
-				case Protocol::VERB_MULTICAST_GATHER:           ok = RR->vl2->m_MULTICAST_GATHER(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_MULTICAST_GATHER:           ok = RR->vl2->m_MULTICAST_GATHER(tPtr, auth, path, peer, *pkt, pktSize); break;
-				case Protocol::VERB_MULTICAST_FRAME_deprecated: ok = RR->vl2->m_MULTICAST_FRAME_deprecated(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_MULTICAST_FRAME_deprecated: ok = RR->vl2->m_MULTICAST_FRAME_deprecated(tPtr, auth, path, peer, *pkt, pktSize); break;
-				case Protocol::VERB_PUSH_DIRECT_PATHS:          ok = m_PUSH_DIRECT_PATHS(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_PUSH_DIRECT_PATHS:          ok = m_PUSH_DIRECT_PATHS(tPtr, auth, path, peer, *pkt, pktSize); break;
-				case Protocol::VERB_USER_MESSAGE:               ok = m_USER_MESSAGE(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_USER_MESSAGE:               ok = m_USER_MESSAGE(tPtr, auth, path, peer, *pkt, pktSize); break;
-				case Protocol::VERB_MULTICAST:                  ok = RR->vl2->m_MULTICAST(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_MULTICAST:                  ok = RR->vl2->m_MULTICAST(tPtr, auth, path, peer, *pkt, pktSize); break;
-				case Protocol::VERB_ENCAP:                      ok = m_ENCAP(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_ENCAP:                      ok = m_ENCAP(tPtr, auth, path, peer, *pkt, pktSize); break;
+
 				default:
-					RR->t->incomingPacketDropped(tPtr,0xeeeeeff0,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_UNRECOGNIZED_VERB);
+					RR->t->incomingPacketDropped(tPtr,0xeeeeeff0,packetId,0,identityFromPeerPtr(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_UNRECOGNIZED_VERB);
 					break;
 			}
-			if (ok)
+			if (likely(ok))
-				peer->received(tPtr,path,hops,ph->packetId,packetSize - ZT_PROTO_PACKET_PAYLOAD_START,verb,inReVerb);
+				peer->received(tPtr,path,hops,packetId,pktSize - ZT_PROTO_PACKET_PAYLOAD_START,verb,inReVerb);
-#endif
 		} else {
 			// If decryption and authentication were not successful, try to look up identities.
 			// This is rate limited by virtue of the retry rate limit timer.
 			if (pktSize <= 0)
 				pktSize = pktv.mergeCopy(*pkt);
 			if (pktSize >= ZT_PROTO_MIN_PACKET_LENGTH) {
+				ZT_SPEW("authentication failed or no peers match, queueing WHOIS for %s",source.toString().c_str());
 				bool sendPending;
 				{
 					Mutex::Lock wl(m_whoisQueue_l);
@@ -392,7 +390,7 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 	}
 }
 
-void VL1::m_relay(void *tPtr, const SharedPtr<Path> &path, const Address &destination, SharedPtr<Buf> &data, unsigned int len)
+void VL1::m_relay(void *tPtr, const SharedPtr<Path> &path, Address destination, SharedPtr<Buf> &pkt, int pktSize)
 {
 }
 
@@ -444,19 +442,16 @@ SharedPtr<Peer> VL1::m_HELLO(void *tPtr, const SharedPtr<Path> &path, Buf &pkt,
 
 	const uint64_t packetId = Utils::loadAsIsEndian<uint64_t>(pkt.unsafeData + ZT_PROTO_PACKET_ID_INDEX);
 	const uint64_t mac = Utils::loadAsIsEndian<uint64_t>(pkt.unsafeData + ZT_PROTO_PACKET_MAC_INDEX);
-
-	// Get hops field and then mask hops to zero for MAC checking.
 	const uint8_t hops = pkt.unsafeData[ZT_PROTO_PACKET_FLAGS_INDEX] & ZT_PROTO_FLAG_FIELD_HOPS_MASK;
-	pkt.unsafeData[ZT_PROTO_PACKET_FLAGS_INDEX] &= ~ZT_PROTO_FLAG_FIELD_HOPS_MASK;
 
-	const uint8_t protoVersion = pkt.lI8<ZT_PROTO_PACKET_PAYLOAD_START>;
+	const uint8_t protoVersion = pkt.lI8<ZT_PROTO_PACKET_PAYLOAD_START>();
 	unsigned int versionMajor = pkt.lI8<ZT_PROTO_PACKET_PAYLOAD_START + 1>(); // LEGACY
 	unsigned int versionMinor = pkt.lI8<ZT_PROTO_PACKET_PAYLOAD_START + 2>(); // LEGACY
 	unsigned int versionRev = pkt.lI16<ZT_PROTO_PACKET_PAYLOAD_START + 3>(); // LEGACY
 	const uint64_t timestamp = pkt.lI64<ZT_PROTO_PACKET_PAYLOAD_START + 5>();
-
 	int p = ZT_PROTO_PACKET_PAYLOAD_START + 13;
 
+	// Get identity and verify that it matches the sending address in the packet.
 	Identity id;
 	if (unlikely(pkt.rO(p,id) < 0)) {
 		RR->t->incomingPacketDropped(tPtr,0x707a9810,packetId,0,Identity::NIL,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
@@ -467,6 +462,7 @@ SharedPtr<Peer> VL1::m_HELLO(void *tPtr, const SharedPtr<Path> &path, Buf &pkt,
 		return SharedPtr<Peer>();
 	}
 
+	// Get the peer that matches this identity, or learn a new one if we don't know it.
 	SharedPtr<Peer> peer(RR->topology->peer(tPtr,id.address(),true));
 	if (peer) {
 		if (peer->identity() != id) {
@@ -492,14 +488,21 @@ SharedPtr<Peer> VL1::m_HELLO(void *tPtr, const SharedPtr<Path> &path, Buf &pkt,
 
 	if (protoVersion >= 11) {
 		// V2.x and newer use HMAC-SHA384 for HELLO, which offers a larger security margin
-		// to guard key exchange and connection setup than typical AEAD.
+		// to guard key exchange and connection setup than typical AEAD. The packet MAC
+		// field is ignored, and eventually it'll be undefined.
 		uint8_t hmac[ZT_HMACSHA384_LEN];
+		if (unlikely(packetSize < ZT_HMACSHA384_LEN)) {
+			RR->t->incomingPacketDropped(tPtr,0xab9c9891,packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+			return SharedPtr<Peer>();
+		}
+		packetSize -= ZT_HMACSHA384_LEN;
+		pkt.unsafeData[ZT_PROTO_PACKET_FLAGS_INDEX] &= ~ZT_PROTO_FLAG_FIELD_HOPS_MASK; // mask hops to 0
+		Utils::storeAsIsEndian<uint64_t>(pkt.unsafeData + ZT_PROTO_PACKET_MAC_INDEX,0); // set MAC field to 0
 		HMACSHA384(peer->identityHelloHmacKey(),pkt.unsafeData,packetSize,hmac);
-		if (unlikely((packetSize < ZT_HMACSHA384_LEN)||(!Utils::secureEq(hmac,(pkt.unsafeData + packetSize) - ZT_HMACSHA384_LEN,ZT_HMACSHA384_LEN)))) {
+		if (unlikely(!Utils::secureEq(hmac,pkt.unsafeData + packetSize,ZT_HMACSHA384_LEN))) {
 			RR->t->incomingPacketDropped(tPtr,0x707a9891,packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 			return SharedPtr<Peer>();
 		}
-		packetSize -= ZT_HMACSHA384_LEN; // trim this off the end since we're done with it
 	} else {
 		// Older versions use Poly1305 MAC (but no whole packet encryption) for HELLO.
 		if (likely(packetSize > ZT_PROTO_PACKET_ENCRYPTED_SECTION_START)) {
@@ -532,7 +535,11 @@ SharedPtr<Peer> VL1::m_HELLO(void *tPtr, const SharedPtr<Path> &path, Buf &pkt,
 		return SharedPtr<Peer>();
 	}
 
-	if ((protoVersion >= 11)&&((p + 12) < packetSize)) {
+	const SharedPtr<SymmetricKey> key(peer->identityKey());
+
+	if (protoVersion >= 11) {
+		// V2.x and newer supports an encrypted section and has a new OK format.
+		if ((p + 12) < packetSize) {
 			uint64_t ctrNonce[2];
 			ctrNonce[0] = Utils::loadAsIsEndian<uint64_t>(pkt.unsafeData + p);
 #if __BYTE_ORDER == __BIG_ENDIAN
@@ -541,10 +548,9 @@ SharedPtr<Peer> VL1::m_HELLO(void *tPtr, const SharedPtr<Path> &path, Buf &pkt,
 			ctrNonce[1] = Utils::loadAsIsEndian<uint32_t>(pkt.unsafeData + p + 8);
 #endif
 			p += 12;
-
 			AES::CTR ctr(peer->identityHelloDictionaryEncryptionCipher());
 			ctr.init(reinterpret_cast<uint8_t *>(ctrNonce),pkt.unsafeData + p);
-		ctr.crypt(pkt.unsafeData + p,(packetSize - p) - ZT_HMACSHA384_LEN);
+			ctr.crypt(pkt.unsafeData + p,packetSize - p);
 			ctr.finish();
 
 			const unsigned int dictSize = pkt.rI16(p);
@@ -574,11 +580,33 @@ SharedPtr<Peer> VL1::m_HELLO(void *tPtr, const SharedPtr<Path> &path, Buf &pkt,
 			}
 		}
 
+		Protocol::newPacket(pkt,key->nextMessage(RR->identity.address(),peer->address()),peer->address(),RR->identity.address(),Protocol::VERB_OK);
+		p = ZT_PROTO_PACKET_PAYLOAD_START;
+		pkt.wI8(p,Protocol::VERB_HELLO);
+		pkt.wI64(p,packetId);
+		pkt.wI64(p,timestamp);
+		pkt.wI8(p,(uint8_t)protoVersion);
+	} else {
+		// V1.x has nothing more for this version to parse, and has an older OK format.
+		Protocol::newPacket(pkt,key->nextMessage(RR->identity.address(),peer->address()),peer->address(),RR->identity.address(),Protocol::VERB_OK);
+		p = ZT_PROTO_PACKET_PAYLOAD_START;
+		pkt.wI8(p,Protocol::VERB_HELLO);
+		pkt.wI64(p,packetId);
+		pkt.wI64(p,timestamp);
+		pkt.wI8(p,(uint8_t)protoVersion);
+		pkt.wI8(p,(uint8_t)versionMajor);
+		pkt.wI8(p,(uint8_t)versionMinor);
+		pkt.wI16(p,(uint16_t)versionRev);
+		pkt.wO(p,path->address());
+		pkt.wI16(p,0);
+	}
+
 	peer->setRemoteVersion(protoVersion,versionMajor,versionMinor,versionRev);
 }
 
-bool VL1::m_ERROR(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize, Protocol::Verb &inReVerb)
+bool VL1::m_ERROR(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize, Protocol::Verb &inReVerb)
 {
+#if 0
 	if (packetSize < (int)sizeof(Protocol::ERROR::Header)) {
 		RR->t->incomingPacketDropped(tPtr,0x3beb1947,0,0,identityFromPeerPtr(peer),path->address(),0,Protocol::VERB_ERROR,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
 		return false;
@@ -622,7 +650,7 @@ bool VL1::m_ERROR(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer>
 	return true;
 }
 
-bool VL1::m_OK(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize, Protocol::Verb &inReVerb)
+bool VL1::m_OK(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize, Protocol::Verb &inReVerb)
 {
 	if (packetSize < (int)sizeof(Protocol::OK::Header)) {
 		RR->t->incomingPacketDropped(tPtr,0x4c1f1ff7,0,0,identityFromPeerPtr(peer),path->address(),0,Protocol::VERB_OK,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
@@ -653,10 +681,12 @@ bool VL1::m_OK(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &p
 
 	}
 	return true;
+#endif
 }
 
-bool VL1::m_WHOIS(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL1::m_WHOIS(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
+#if 0
 	if (packetSize < (int)sizeof(Protocol::OK::Header)) {
 		RR->t->incomingPacketDropped(tPtr,0x4c1f1ff7,0,0,identityFromPeerPtr(peer),path->address(),0,Protocol::VERB_OK,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
 		return false;
@@ -705,10 +735,12 @@ bool VL1::m_WHOIS(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer>
 	}
 
 	return true;
+#endif
 }
 
-bool VL1::m_RENDEZVOUS(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL1::m_RENDEZVOUS(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
+#if 0
 	if (RR->topology->isRoot(peer->identity())) {
 		if (packetSize < (int)sizeof(Protocol::RENDEZVOUS)) {
 			RR->t->incomingPacketDropped(tPtr,0x43e90ab3,Protocol::packetId(pkt,packetSize),0,peer->identity(),path->address(),Protocol::packetHops(pkt,packetSize),Protocol::VERB_RENDEZVOUS,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
@@ -751,10 +783,12 @@ bool VL1::m_RENDEZVOUS(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<
 		}
 	}
 	return true;
+#endif
 }
 
-bool VL1::m_ECHO(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL1::m_ECHO(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
+#if 0
 	const uint64_t packetId = Protocol::packetId(pkt,packetSize);
 	const uint64_t now = RR->node->now();
 	if (packetSize < (int)sizeof(Protocol::Header)) {
@@ -787,10 +821,12 @@ bool VL1::m_ECHO(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer>
 	}
 
 	return true;
+#endif
 }
 
-bool VL1::m_PUSH_DIRECT_PATHS(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL1::m_PUSH_DIRECT_PATHS(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
+#if 0
 	if (packetSize < (int)sizeof(Protocol::PUSH_DIRECT_PATHS)) {
 		RR->t->incomingPacketDropped(tPtr,0x1bb1bbb1,Protocol::packetId(pkt,packetSize),0,peer->identity(),path->address(),Protocol::packetHops(pkt,packetSize),Protocol::VERB_PUSH_DIRECT_PATHS,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
 		return false;
@@ -876,15 +912,16 @@ bool VL1::m_PUSH_DIRECT_PATHS(void *tPtr, const SharedPtr<Path> &path, const Sha
 	// TODO: add to a peer try-queue
 
 	return true;
+#endif
 }
 
-bool VL1::m_USER_MESSAGE(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL1::m_USER_MESSAGE(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
 	// TODO
 	return true;
 }
 
-bool VL1::m_ENCAP(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL1::m_ENCAP(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
 	// TODO: not implemented yet
 	return true;

node/VL1.hpp

@@ -25,6 +25,10 @@
 
 #define ZT_VL1_MAX_WHOIS_WAITING_PACKETS 32
 
+#define ZT_VL1_AUTH_RESULT_FLAG_AUTHENTICATED  0x01U
+#define ZT_VL1_AUTH_RESULT_FLAG_ENCRYPTED      0x02U
+#define ZT_VL1_AUTH_RESULT_FLAG_FORWARD_SECRET 0x04U
+
 namespace ZeroTier {
 
 class RuntimeEnvironment;
@@ -61,22 +65,19 @@ public:
 private:
 	const RuntimeEnvironment *RR;
 
-	// Code to handle relaying of packets to other nodes.
+	void m_relay(void *tPtr, const SharedPtr<Path> &path, Address destination, SharedPtr<Buf> &pkt, int pktSize);
-	void m_relay(void *tPtr, const SharedPtr<Path> &path, const Address &destination, SharedPtr<Buf> &data, unsigned int len);
-
-	// Send any pending WHOIS requests.
 	void m_sendPendingWhois(void *tPtr, int64_t now);
 
-	// Handlers for VL1 verbs -- for clarity's sake VL2 verbs are in the VL2 class.
 	SharedPtr<Peer> m_HELLO(void *tPtr, const SharedPtr<Path> &path, Buf &pkt, int packetSize);
-	bool m_ERROR(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize, Protocol::Verb &inReVerb);
+
-	bool m_OK(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize, Protocol::Verb &inReVerb);
+	bool m_ERROR(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize, Protocol::Verb &inReVerb);
-	bool m_WHOIS(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_OK(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize, Protocol::Verb &inReVerb);
-	bool m_RENDEZVOUS(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_WHOIS(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
-	bool m_ECHO(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_RENDEZVOUS(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
-	bool m_PUSH_DIRECT_PATHS(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_ECHO(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
-	bool m_USER_MESSAGE(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_PUSH_DIRECT_PATHS(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
-	bool m_ENCAP(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_USER_MESSAGE(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_ENCAP(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
 
 	Defragmenter<ZT_MAX_PACKET_FRAGMENTS> m_inputPacketAssembler;
 

node/VL2.cpp

@@ -27,43 +27,43 @@ VL2::VL2(const RuntimeEnvironment *renv)
 {
 }
 
-void VL2::onLocalEthernet(void *const tPtr,const SharedPtr<Network> &network,const MAC &from,const MAC &to,const unsigned int etherType,unsigned int vlanId,SharedPtr<Buf> &data,unsigned int len)
+void VL2::onLocalEthernet(void *const tPtr,const unsigned int auth,const SharedPtr<Network> &network,const MAC &from,const MAC &to,const unsigned int etherType,unsigned int vlanId,SharedPtr<Buf> &data,unsigned int len)
 {
 }
 
-bool VL2::m_FRAME(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL2::m_FRAME(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
 }
 
-bool VL2::m_EXT_FRAME(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL2::m_EXT_FRAME(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
 }
 
-bool VL2::m_MULTICAST_LIKE(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL2::m_MULTICAST_LIKE(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
 }
 
-bool VL2::m_NETWORK_CREDENTIALS(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL2::m_NETWORK_CREDENTIALS(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
 }
 
-bool VL2::m_NETWORK_CONFIG_REQUEST(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL2::m_NETWORK_CONFIG_REQUEST(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
 }
 
-bool VL2::m_NETWORK_CONFIG(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL2::m_NETWORK_CONFIG(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
 }
 
-bool VL2::m_MULTICAST_GATHER(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL2::m_MULTICAST_GATHER(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
 }
 
-bool VL2::m_MULTICAST_FRAME_deprecated(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL2::m_MULTICAST_FRAME_deprecated(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
 }
 
-bool VL2::m_MULTICAST(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
+bool VL2::m_MULTICAST(void *tPtr,const unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize)
 {
 }
 

node/VL2.hpp

@@ -53,15 +53,15 @@ public:
 	void onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,SharedPtr<Buf> &data,unsigned int len);
 
 protected:
-	bool m_FRAME(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_FRAME(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
-	bool m_EXT_FRAME(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_EXT_FRAME(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
-	bool m_MULTICAST_LIKE(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_MULTICAST_LIKE(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
-	bool m_NETWORK_CREDENTIALS(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_NETWORK_CREDENTIALS(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
-	bool m_NETWORK_CONFIG_REQUEST(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_NETWORK_CONFIG_REQUEST(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
-	bool m_NETWORK_CONFIG(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_NETWORK_CONFIG(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
-	bool m_MULTICAST_GATHER(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_MULTICAST_GATHER(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
-	bool m_MULTICAST_FRAME_deprecated(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_MULTICAST_FRAME_deprecated(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
-	bool m_MULTICAST(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
+	bool m_MULTICAST(void *tPtr, unsigned int auth, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
 
 private:
 };