Documentation changes, and move HMAC auth on HELLO to before object decoding since this is good cryptographic practice.

node/Identity.hpp

@@ -38,10 +38,8 @@ namespace ZeroTier {
  * Identities currently come in two types: type 0 identities based on just Curve25519
  * and Ed25519 and type 1 identities that include both a 25519 key pair and a NIST P-384
  * key pair. Type 1 identities use P-384 for signatures but use both key pairs at once
- * (hashing their results) for key agreement with other type 1 identities, and can agree
+ * (hashing both keys together) for key agreement with other type 1 identities, and can
- * with type 0 identities using only their Curve25519 keys. The ability of type 0 and 1
+ * agree with type 0 identities by only using the Curve25519 component.
- * identities to agree will allow type 0 identities to keep being used even after type
- * 1 becomes the default.
  *
  * Type 1 identities also use a simpler mechanism to rate limit identity generation (as
  * a defense in depth against intentional collision) that makes local identity validation

node/Membership.cpp

@@ -38,6 +38,8 @@ Membership::~Membership()
 
 void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const int64_t now,const Address &peerAddress,const NetworkConfig &nconf)
 {
+	Buf outp;
+#if 0
 	const Capability *sendCaps[ZT_MAX_NETWORK_CAPABILITIES];
 	unsigned int sendCapCount = 0;
 	for(unsigned int c=0;c<nconf.capabilityCount;++c)
@@ -99,6 +101,7 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const i
 		outp.compress();
 		RR->sw->send(tPtr,outp,true);
 	}
+#endif
 
 	_lastPushedCredentials = now;
 }

node/Protocol.hpp

@@ -23,6 +23,64 @@
 #include "Address.hpp"
 #include "Identity.hpp"
 
+/*
+ * Core ZeroTier protocol packet formats ------------------------------------------------------------------------------
+ *
+ * Packet format:
+ *   <[8] 64-bit packet ID / crypto IV>
+ *   <[5] destination ZT address>
+ *   <[5] source ZT address>
+ *   <[1] outer visible flags, cipher, and hop count (bits: FFCCHHH)>
+ *   <[8] 64-bit MAC (or trusted path ID in trusted path mode)>
+ *   [... -- begin encryption envelope -- ...]
+ *   <[1] inner envelope flags (MS 3 bits) and verb (LS 5 bits)>
+ *   [... verb-specific payload ...]
+ *
+ * Packets smaller than 28 bytes are invalid and silently discarded.
+ *
+ * The hop count field is masked during message authentication computation
+ * and is thus the only field that is mutable in transit. It's incremented
+ * when roots or other nodes forward packets and exists to prevent infinite
+ * forwarding loops and to detect direct paths.
+ *
+ * HELLO is normally sent in the clear with the POLY1305_NONE cipher suite
+ * and with Poly1305 computed on plain text (Salsa20/12 is still used to
+ * generate a one time use Poly1305 key). As of protocol version 11 HELLO
+ * also includes a terminating HMAC (last 48 bytes) that significantly
+ * hardens HELLO authentication beyond what a 64-bit MAC can guarantee.
+ *
+ * Fragmented packets begin with a packet header whose fragment bit (bit
+ * 0x40 in the flags field) is set. This constitutes fragment zero. The
+ * total number of expected fragments is contained in each subsequent
+ * fragment packet. Unfragmented packets must not have the fragment bit
+ * set or the receiver will expect at least one additional fragment.
+ *
+ * --
+ *
+ * Packet fragment format (fragments beyond 0):
+ *   <[8] packet ID of packet to which this fragment belongs>
+ *   <[5] destination ZT address>
+ *   <[1] 0xff here signals that this is a fragment>
+ *   <[1] total fragments (most significant 4 bits), fragment no (LS 4 bits)>
+ *   <[1] ZT hop count (least significant 3 bits; others are reserved)>
+ *   <[...] fragment data>
+ *
+ * The protocol supports a maximum of 16 fragments including fragment 0
+ * which contains the full packet header (with fragment bit set). Fragments
+ * thus always carry fragment numbers between 1 and 15. All fragments
+ * belonging to the same packet must carry the same total fragment count in
+ * the most significant 4 bits of the fragment numbering field.
+ *
+ * All fragments have the same packet ID and destination. The packet ID
+ * doubles as the grouping identifier for fragment reassembly.
+ *
+ * Fragments do not carry their own packet MAC. The entire packet is
+ * authenticated once it is assembled by the receiver. Incomplete packets
+ * are discarded after a receiver configured period of time.
+ *
+ * --------------------------------------------------------------------------------------------------------------------
+ */
+
 /**
  * Protocol version -- incremented only for major changes
  *
@@ -50,24 +108,25 @@
  *    + inline push of CertificateOfMembership deprecated
  * 9  - 1.2.0 ... 1.2.14
  * 10 - 1.4.0 ... 1.4.6
+ *    + Contained early pre-alpha versions of multipath, which are deprecated
  * 11 - 2.0.0 ... CURRENT
- *    + Peer-to-peer multicast replication
+ *    + New more WAN-efficient P2P-assisted multicast algorithm
- *    + HELLO and OK(HELLO) include an extra HMAC to further harden auth
+ *    + HELLO and OK(HELLO) include an extra HMAC to harden authentication
- *    + Old planet/moon stuff is DEAD!
+ *    + HELLO and OK(HELLO) can carry structured meta-data
- *    + AES encryption support
+ *    + Ephemeral keys for forward secrecy and limited key lifetime
- *    + NIST P-384 (type 1) identities
+ *    + Old planet/moon stuff is DEAD! Independent roots are easier.
- *    + Ephemeral keys
+ *    + AES encryption is now the default
+ *    + New combined Curve25519/NIST P-384 identity type (type 1)
  *    + Short probe packets to reduce probe bandwidth
+ *    + Aggressive NAT traversal techniques for IPv4 symmetric NATs
+ *    + Remote diagnostics including rewrite of remote tracing
  */
 #define ZT_PROTO_VERSION 11
 
 /**
  * Minimum supported protocol version
- *
- * As of v2 we don't "officially" support anything older than 1.2.14, but this
- * is the hard cutoff before which peers will be flat out rejected.
  */
-#define ZT_PROTO_VERSION_MIN 6
+#define ZT_PROTO_VERSION_MIN 8
 
 /**
  * Packet buffer size (can be changed)
@@ -93,8 +152,8 @@
  * Maximum hop count allowed by packet structure (3 bits, 0-7)
  *
  * This is a protocol constant. It's the maximum allowed by the length
- * of the hop counter -- three bits. See node/Constants.hpp for the
+ * of the hop counter -- three bits. A lower limit is specified as
- * pragmatic forwarding limit, which is typically lower.
+ * the actual maximum hop count.
  */
 #define ZT_PROTO_MAX_HOPS 7
 
@@ -121,7 +180,12 @@
 #define ZT_PROTO_CIPHER_SUITE__AES_GCM_NRH 3
 
 /**
- * Magic number indicating a fragment
+ * Minimum viable length for a fragment
+ */
+#define ZT_PROTO_MIN_FRAGMENT_LENGTH 16
+
+/**
+ * Magic number indicating a fragment if present at index 13
  */
 #define ZT_PROTO_PACKET_FRAGMENT_INDICATOR 0xff
 
@@ -136,12 +200,7 @@
 #define ZT_PROTO_PACKET_FLAGS_INDEX 18
 
 /**
- * Minimum viable length for a fragment
+ * Length of a probe packet
- */
-#define ZT_PROTO_MIN_FRAGMENT_LENGTH 16
-
-/**
- * Length of a probe
  */
 #define ZT_PROTO_PROBE_LENGTH 8
 
@@ -155,6 +214,11 @@
  */
 #define ZT_PROTO_FLAG_FRAGMENTED 0x40U
 
+/**
+ * Mask for obtaining hops from the combined flags, cipher, and hops field
+ */
+#define ZT_PROTO_FLAG_FIELD_HOPS_MASK 0x07U
+
 /**
  * Verb flag indicating payload is compressed with LZ4
  */
@@ -208,53 +272,6 @@
  */
 #define ZT_PROTO_HELLO_NODE_META_LOCATION_Z "gZ"
 
-/****************************************************************************/
-
-/*
- * Packet format:
- *   <[8] 64-bit packet ID / crypto IV>
- *   <[5] destination ZT address>
- *   <[5] source ZT address>
- *   <[1] flags/cipher/hops>
- *   <[8] 64-bit MAC (or trusted path ID in trusted path mode)>
- *   [... -- begin encryption envelope -- ...]
- *   <[1] encrypted flags (MS 3 bits) and verb (LS 5 bits)>
- *   [... verb-specific payload ...]
- *
- * Packets smaller than 28 bytes are invalid and silently discarded.
- *
- * The flags/cipher/hops bit field is: FFCCCHHH where C is a 3-bit cipher
- * selection allowing up to 7 cipher suites, F is outside-envelope flags,
- * and H is hop count.
- *
- * The three-bit hop count is the only part of a packet that is mutable in
- * transit without invalidating the MAC. All other bits in the packet are
- * immutable. This is because intermediate nodes can increment the hop
- * count up to 7 (protocol max).
- *
- * For unencrypted packets, MAC is computed on plaintext. Only HELLO is ever
- * sent in the clear, as it's the "here is my public key" message.
- *
- * The fragmented bit indicates that there is at least one fragment. Fragments
- * themselves contain the total, so the receiver must "learn" this from the
- * first fragment it receives.
- *
- * Fragments are sent with the following format:
- *   <[8] packet ID of packet to which this fragment belongs>
- *   <[5] destination ZT address>
- *   <[1] 0xff here signals that this is a fragment>
- *   <[1] total fragments (most significant 4 bits), fragment no (LS 4 bits)>
- *   <[1] ZT hop count (least significant 3 bits; others are reserved)>
- *   <[...] fragment data>
- *
- * The protocol supports a maximum of 16 fragments. If a fragment is received
- * before its main packet header, it should be cached for a brief period of
- * time to see if its parent arrives. Loss of any fragment constitutes packet
- * loss; there is no retransmission mechanism. The receiver must wait for full
- * receipt to authenticate and decrypt; there is no per-fragment MAC. (But if
- * fragments are corrupt, the MAC will fail for the whole assembled packet.)
- */
-
 namespace ZeroTier {
 namespace Protocol {
 
@@ -947,7 +964,7 @@ ZT_ALWAYS_INLINE uint64_t packetId(const Buf &pkt,const unsigned int packetSize)
  * @param packetSize Packet's actual size in bytes
  * @return 3-bit hops field embedded in packet flags field
  */
-ZT_ALWAYS_INLINE uint8_t packetHops(const Buf &pkt,const unsigned int packetSize) noexcept { return (packetSize >= ZT_PROTO_PACKET_FLAGS_INDEX) ? (pkt.b[ZT_PROTO_PACKET_FLAGS_INDEX] & 0x07U) : 0; }
+ZT_ALWAYS_INLINE uint8_t packetHops(const Buf &pkt,const unsigned int packetSize) noexcept { return (packetSize >= ZT_PROTO_PACKET_FLAGS_INDEX) ? (pkt.b[ZT_PROTO_PACKET_FLAGS_INDEX] & ZT_PROTO_FLAG_FIELD_HOPS_MASK) : 0; }
 
 /**
  * @param Packet to extract cipher ID from

node/VL1.cpp

@@ -484,6 +484,7 @@ bool VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 	}
 	Protocol::HELLO &p = pkt.as<Protocol::HELLO>();
 	const uint8_t hops = Protocol::packetHops(p.h);
+	p.h.flags &= (uint8_t)~ZT_PROTO_FLAG_FIELD_HOPS_MASK; // mask off hops for MAC calculation
 	int ptr = sizeof(Protocol::HELLO);
 
 	if (p.versionProtocol < ZT_PROTO_VERSION_MIN) {
@@ -529,12 +530,26 @@ bool VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 		}
 	}
 
-	// Packet has passed Poly1305 verification --------------------------------------------------------------------------
+	// Packet has passed Poly1305 MAC authentication --------------------------------------------------------------------
+
+	uint8_t hmacKey[ZT_PEER_SECRET_KEY_LENGTH],hmac[ZT_HMACSHA384_LEN];
+	if (peer->remoteVersionProtocol() >= 11) {
+		if (packetSize <= ZT_HMACSHA384_LEN) { // sanity check, should be impossible
+			RR->t->incomingPacketDropped(tPtr,0x1000662a,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+			return false;
+		}
+		KBKDFHMACSHA384(key,ZT_PROTO_KDF_KEY_LABEL_HELLO_HMAC,0,0,hmacKey); // iter == 0 for HELLO, 1 for OK(HELLO)
+		HMACSHA384(hmacKey,pkt.b,packetSize - ZT_HMACSHA384_LEN,hmac);
+		if (!Utils::secureEq(pkt.b + (packetSize - ZT_HMACSHA384_LEN),hmac,ZT_HMACSHA384_LEN)) {
+			RR->t->incomingPacketDropped(tPtr,0x1000662a,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+			return false;
+		}
+	}
+
+	// Packet has passed HMAC-SHA384 (if present and/or forced) ---------------------------------------------------------
 
 	InetAddress externalSurfaceAddress;
 	Dictionary nodeMetaData;
-	uint8_t hmacKey[ZT_PEER_SECRET_KEY_LENGTH],hmac[ZT_HMACSHA384_LEN];
-	bool hmacAuthenticated = false;
 
 	// Get external surface address if present.
 	if (ptr < packetSize) {
@@ -544,7 +559,7 @@ bool VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 		}
 	}
 
-	if (ptr < packetSize) {
+	if (((ptr + ZT_HMACSHA384_LEN) < packetSize)&&(peer->remoteVersionProtocol() >= 11)) {
 		// Everything after this point is encrypted with Salsa20/12. This is only a privacy measure
 		// since there's nothing truly secret in a HELLO packet. It also means that an observer
 		// can't even get ephemeral public keys without first knowing the long term secret key,
@@ -553,9 +568,9 @@ bool VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 		for (int i = 0; i < 8; ++i) iv[i] = pkt.b[i];
 		iv[7] &= 0xf8U;
 		Salsa20 s20(key,iv);
-		s20.crypt12(pkt.b + ptr,pkt.b + ptr,packetSize - ptr);
+		s20.crypt12(pkt.b + ptr,pkt.b + ptr,(packetSize - ZT_HMACSHA384_LEN) - ptr);
 
-		ptr += pkt.rI16(ptr); // this field is zero in v2.0+ but can indicate data between this point and dictionary
+		ptr += pkt.rI16(ptr); // skip length field which currently is always zero in v2.0+
 		if (ptr < packetSize) {
 			const unsigned int dictionarySize = pkt.rI16(ptr);
 			const void *const dictionaryBytes = pkt.b + ptr;
@@ -570,16 +585,6 @@ bool VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 				return false;
 			}
 
-			if ((ptr + ZT_SHA384_DIGEST_LEN) <= packetSize) {
-				KBKDFHMACSHA384(key,ZT_PROTO_KDF_KEY_LABEL_HELLO_HMAC,0,0,hmacKey); // iter == 0 for HELLO
-				HMACSHA384(hmacKey,pkt.b + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,hmac);
-				if (!Utils::secureEq(pkt.b + ptr,hmac,ZT_HMACSHA384_LEN)) {
-					RR->t->incomingPacketDropped(tPtr,0x1000662a,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
-					return false;
-				}
-				hmacAuthenticated = true;
-			}
-
 			if (dictionarySize) {
 				if (!nodeMetaData.decode(dictionaryBytes,dictionarySize)) {
 					RR->t->incomingPacketDropped(tPtr,0x67192344,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
@@ -589,9 +594,8 @@ bool VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 		}
 	}
 
-	// v2.x+ peers must include HMAC, older peers don't (we'll drop support for them when 1.x is dead)
+	if (Buf::readOverflow(ptr,packetSize)) { // sanity check, should be impossible
-	if ((!hmacAuthenticated) && (p.versionProtocol >= 11)) {
+		RR->t->incomingPacketDropped(tPtr,0x457f2347,0,p.h.packetId,id,path->address(),0,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
-		RR->t->incomingPacketDropped(tPtr,0x571feeea,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 		return false;
 	}