More refactoring to clean up code, and add a gate function to make sure we do not handle OK packets we did not expect. This hardens up a few potential edge cases around security, since such messages might be used to e.g. pollute a cache and DOS under certain conditions.

2025-06-05 20:13:44 +02:00 · 2016-09-09 08:43:58 -07:00 · 2016-09-09 08:43:58 -07:00 · 0d4109a9f1
commit 0d4109a9f1
parent 16df2c3363
13 changed files with 170 additions and 83 deletions
--- a/controller/EmbeddedNetworkController.cpp
+++ b/controller/EmbeddedNetworkController.cpp
@ -1585,7 +1585,6 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes
 		"\t\"upstream\": \"%.10llx\"," ZT_EOL_S
 		"\t\"current\": \"%.10llx\"," ZT_EOL_S
 		"\t\"receivedTimestamp\": %llu," ZT_EOL_S
-		"\t\"remoteTimestamp\": %llu," ZT_EOL_S
 		"\t\"sourcePacketId\": \"%.16llx\"," ZT_EOL_S
 		"\t\"flags\": %llu," ZT_EOL_S
 		"\t\"sourcePacketHopCount\": %u," ZT_EOL_S
@ -1606,7 +1605,6 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes
 		(unsigned long long)report->upstream,
 		(unsigned long long)report->current,
 		(unsigned long long)OSUtils::now(),
-		(unsigned long long)report->remoteTimestamp,
 		(unsigned long long)report->sourcePacketId,
 		(unsigned long long)report->flags,
 		report->sourcePacketHopCount,
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@ -154,6 +154,11 @@ extern "C" {
 */
 #define ZT_CIRCUIT_TEST_MAX_HOP_BREADTH 8

+/**
+ * Circuit test report flag: upstream peer authorized in path (e.g. by network COM)
+ */
+#define ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH 0x0000000000000001ULL
+
 /**
 * Maximum number of cluster members (and max member ID plus one)
 */
@ -1218,18 +1223,13 @@ typedef struct {
 	 */
 	uint64_t timestamp;

-	/**
-	 * Timestamp on remote device
-	 */
-	uint64_t remoteTimestamp;
-
 	/**
 	 * 64-bit packet ID of packet received by the reporting device
 	 */
 	uint64_t sourcePacketId;

 	/**
-	 * Flags (currently unused, will be zero)
+	 * Flags
 	 */
 	uint64_t flags;

--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@ -156,6 +156,17 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 					RR->node->postEvent(ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION);
 				break;

+			case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: {
+				SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
+				if ((network)&&(network->recentlyAllowedOnNetwork(peer))) {
+					Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS);
+					network->config().com.serialize(outp);
+					outp.append((uint8_t)0);
+					outp.armor(peer->key(),true);
+					_path->send(RR,outp.data(),outp.size(),RR->node->now());
+				}
+			}	break;
+
 			case Packet::ERROR_NETWORK_ACCESS_DENIED_: {
 				SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
 				if ((network)&&(network->controller() == peer->address()))
@ -163,10 +174,12 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 			}	break;

 			case Packet::ERROR_UNWANTED_MULTICAST: {
-				uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD);
+				SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
+				if ((network)&&(network->gate(peer,verb(),packetId()))) {
 					MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at<uint32_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14));
-				TRACE("%.16llx: peer %s unsubscrubed from multicast group %s",nwid,peer->address().toString().c_str(),mg.toString().c_str());
-				RR->mc->remove(nwid,mg,peer->address());
+					TRACE("%.16llx: peer %s unsubscrubed from multicast group %s",network->id(),peer->address().toString().c_str(),mg.toString().c_str());
+					RR->mc->remove(network->id(),mg,peer->address());
+				}
 			}	break;

 			default: break;
@ -352,7 +365,12 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 		const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB];
 		const uint64_t inRePacketId = at<uint64_t>(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID);

-		//TRACE("%s(%s): OK(%s)",source().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb));
+		if (!RR->node->expectingReplyTo(inRePacketId)) {
+			TRACE("%s(%s): OK(%s) DROPPED: not expecting reply to %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb),packetId());
+			return true;
+		}
+
+		//TRACE("%s(%s): OK(%s)",peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb));

 		switch(inReVerb) {

@ -424,10 +442,13 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p

 			case Packet::VERB_MULTICAST_GATHER: {
 				const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID);
+				SharedPtr<Network> network(RR->node->network(nwid));
+				if ((network)&&(network->gate(peer,verb(),packetId()))) {
 					const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI));
 					//TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size());
 					const unsigned int count = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4);
 					RR->mc->addMultiple(RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS));
+				}
 			}	break;

 			case Packet::VERB_MULTICAST_FRAME: {
@ -437,18 +458,18 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p

 				//TRACE("%s(%s): OK(MULTICAST_FRAME) %.16llx/%s flags %.2x",peer->address().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),flags);

+				SharedPtr<Network> network(RR->node->network(nwid));
+				if (network) {
 					unsigned int offset = 0;

 					if ((flags & 0x01) != 0) { // deprecated but still used by older peers
 						CertificateOfMembership com;
 						offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS);
-					if (com) {
-						SharedPtr<Network> network(RR->node->network(com.networkId()));
-						if (network)
+						if (com)
 							network->addCredential(com);
 					}
-				}

+					if (network->gate(peer,verb(),packetId())) {
 						if ((flags & 0x02) != 0) {
 							// OK(MULTICAST_FRAME) includes implicit gather results
 							offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS;
@ -456,6 +477,8 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 							unsigned int count = at<uint16_t>(offset); offset += 2;
 							RR->mc->addMultiple(RR->node->now(),nwid,mg,field(offset,count * 5),count,totalKnown);
 						}
+					}
+				}
 			}	break;

 			default: break;
@ -515,6 +538,9 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
 {
 	try {
+		if (!RR->topology->isUpstream(peer->identity())) {
+			TRACE("RENDEZVOUS from %s ignored since source is not upstream",peer->address().toString().c_str());
+		} else {
 			const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH);
 			const SharedPtr<Peer> rendezvousWith(RR->topology->getPeer(with));
 			if (rendezvousWith) {
@ -522,9 +548,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr<
 				const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN];
 				if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) {
 					const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port);
-				if (!RR->topology->isUpstream(peer->identity())) {
-					TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since peer is not upstream",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
-				} else if (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),atAddr)) {
+					if (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),atAddr)) {
 						RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls
 						rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now());
 						TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
@ -537,6 +561,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr<
 			} else {
 				TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_path->address().toString().c_str(),with.toString().c_str());
 			}
+		}
 		peer->received(_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false);
 	} catch ( ... ) {
 		TRACE("dropped RENDEZVOUS from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str());
@ -549,25 +574,25 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 	try {
 		const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID);
 		const SharedPtr<Network> network(RR->node->network(nwid));
-		bool approved = false;
+		bool trustEstablished = false;
 		if (network) {
-			if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
 			if (!network->gate(peer,verb(),packetId())) {
 				TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id());
 			} else {
+				trustEstablished = true;
+				if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
 					const unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE);
 					const MAC sourceMac(peer->address(),nwid);
 					const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
 					const uint8_t *const frameData = reinterpret_cast<const uint8_t *>(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
 					if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0)
 						RR->node->putFrame(nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen);
-					approved = true; // this means approved on the network in general, not this packet per se
 				}
 			}
 		} else {
 			TRACE("dropped FRAME from %s(%s): we are not a member of network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
 		}
-		peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,approved);
+		peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished);
 	} catch ( ... ) {
 		TRACE("dropped FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
@ -580,11 +605,10 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 		const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_EXT_FRAME_IDX_NETWORK_ID);
 		const SharedPtr<Network> network(RR->node->network(nwid));
 		if (network) {
-			if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) {
 			const unsigned int flags = (*this)[ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS];

 			unsigned int comLen = 0;
-				if ((flags & 0x01) != 0) { // deprecated but still used by old peers
+			if ((flags & 0x01) != 0) { // inline COM with EXT_FRAME is deprecated but still used with old peers
 				CertificateOfMembership com;
 				comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM);
 				if (com)
@ -597,6 +621,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 				return true;
 			}

+			if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) {
 				const unsigned int etherType = at<uint16_t>(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_ETHERTYPE);
 				const MAC to(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_TO,ZT_PROTO_VERB_EXT_FRAME_LEN_TO),ZT_PROTO_VERB_EXT_FRAME_LEN_TO);
 				const MAC from(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_FROM,ZT_PROTO_VERB_EXT_FRAME_LEN_FROM),ZT_PROTO_VERB_EXT_FRAME_LEN_FROM);
@ -604,7 +629,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 				const uint8_t *const frameData = (const uint8_t *)field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,frameLen);

 				if ((!from)||(from.isMulticast())||(from == network->mac())) {
-					TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str());
+					TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC %s",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),from.toString().c_str());
 					peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
 					return true;
 				}
@ -1139,6 +1164,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 		// Add length of second "additional fields" section.
 		vlf += at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 29 + vlf);

+		uint64_t reportFlags = 0;
+
 		// Check credentials (signature already verified)
 		if (originatorCredentialNetworkId) {
 			SharedPtr<Network> network(RR->node->network(originatorCredentialNetworkId));
@ -1147,6 +1174,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 				peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false);
 				return true;
 			}
+			if (network->gate(peer,verb(),packetId()))
+				reportFlags |= ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH;
 		} else {
 			TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str());
 			peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false);
@ -1188,7 +1217,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 			outp.append((uint16_t)ZT_PLATFORM_UNSPECIFIED);
 			outp.append((uint16_t)ZT_ARCHITECTURE_UNSPECIFIED);
 			outp.append((uint16_t)0); // error code, currently unused
-			outp.append((uint64_t)0); // flags, currently unused
+			outp.append((uint64_t)reportFlags);
 			outp.append((uint64_t)packetId());
 			peer->address().appendTo(outp);
 			outp.append((uint8_t)hops());
@ -1237,7 +1266,6 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S
 		report.upstream = Address(field(ZT_PACKET_IDX_PAYLOAD + 52,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH).toInt();
 		report.testId = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 8);
 		report.timestamp = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD);
-		report.remoteTimestamp = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 16);
 		report.sourcePacketId = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 44);
 		report.flags = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 36);
 		report.sourcePacketHopCount = (*this)[ZT_PACKET_IDX_PAYLOAD + 57]; // end of fixed length headers: 58
--- a/node/Membership.hpp
+++ b/node/Membership.hpp
@ -163,8 +163,10 @@ public:
 			return true;
 		if (_com) {
 			const uint64_t a = _com.timestamp().first;
-			const std::pair<uint64_t,uint64_t> b(nconf.com.timestamp());
-			return ((a <= b.first) ? ((b.first - a) <= ZT_PEER_ACTIVITY_TIMEOUT) : true);
+			if ((_blacklistBefore)&&(a <= _blacklistBefore))
+				return false;
+			const uint64_t b = nconf.com.timestamp().first;
+			return ((a <= b) ? ((b - a) <= ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA) : true);
 		}
 		return false;
 	}
--- a/node/Multicaster.cpp
+++ b/node/Multicaster.cpp
@ -253,6 +253,7 @@ void Multicaster::send(
 					outp.append((uint32_t)gatherLimit);
 					if (com)
 						com->serialize(outp);
+					RR->node->expectReplyTo(outp.packetId());
 					RR->sw->send(outp,true);
 				}
 			}
--- a/node/Network.cpp
+++ b/node/Network.cpp
@ -1054,6 +1054,7 @@ void Network::requestConfiguration()
 	} else {
 		outp.append((unsigned char)0,16);
 	}
+	RR->node->expectReplyTo(outp.packetId());
 	outp.compress();
 	RR->sw->send(outp,true);

@ -1092,6 +1093,15 @@ bool Network::gate(const SharedPtr<Peer> &peer,const Packet::Verb verb,const uin
 	return false;
 }

+bool Network::recentlyAllowedOnNetwork(const SharedPtr<Peer> &peer) const
+{
+	Mutex::Lock _l(_lock);
+	const Membership *m = _memberships.get(peer->address());
+	if (m)
+		return m->recentlyAllowedOnNetwork(_config);
+	return false;
+}
+
 void Network::clean()
 {
 	const uint64_t now = RR->node->now();
--- a/node/Network.hpp
+++ b/node/Network.hpp
@ -257,6 +257,12 @@ public:
 	 */
 	bool gate(const SharedPtr<Peer> &peer,const Packet::Verb verb,const uint64_t packetId);

+	/**
+	 * @param peer Peer to check
+	 * @return True if peer has recently been a valid member of this network
+	 */
+	bool recentlyAllowedOnNetwork(const SharedPtr<Peer> &peer) const;
+
 	/**
 	 * Perform cleanup and possibly save state
 	 */
--- a/node/Node.cpp
+++ b/node/Node.cpp
@ -75,6 +75,9 @@ Node::Node(
 {
 	_online = false;

+	memset(_expectingRepliesToBucketPtr,0,sizeof(_expectingRepliesToBucketPtr));
+	memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo));
+
 	// Use Salsa20 alone as a high-quality non-crypto PRNG
 	{
 		char foo[32];
--- a/node/Node.hpp
+++ b/node/Node.hpp
@ -44,6 +44,10 @@
 #define TRACE(f,...) {}
 #endif

+// Bit mask for "expecting reply" hash
+#define ZT_EXPECTING_REPLIES_BUCKET_MASK1 255
+#define ZT_EXPECTING_REPLIES_BUCKET_MASK2 31
+
 namespace ZeroTier {

 /**
@ -250,6 +254,33 @@ public:
 	void postCircuitTestReport(const ZT_CircuitTestReport *report);
 	void setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count);

+	/**
+	 * Register that we are expecting a reply to a packet ID
+	 *
+	 * @param packetId Packet ID to expect reply to
+	 */
+	inline void expectReplyTo(const uint64_t packetId)
+	{
+		const unsigned long bucket = (unsigned long)(packetId & ZT_EXPECTING_REPLIES_BUCKET_MASK1);
+		_expectingRepliesTo[bucket][_expectingRepliesToBucketPtr[bucket]++ & ZT_EXPECTING_REPLIES_BUCKET_MASK2] = packetId;
+	}
+
+	/**
+	 * Check whether a given packet ID is something we are expecting a reply to
+	 *
+	 * @param packetId Packet ID to check
+	 * @return True if we're expecting a reply
+	 */
+	inline bool expectingReplyTo(const uint64_t packetId) const
+	{
+		const unsigned long bucket = (unsigned long)(packetId & ZT_EXPECTING_REPLIES_BUCKET_MASK1);
+		for(unsigned long i=0;i<=ZT_EXPECTING_REPLIES_BUCKET_MASK2;++i) {
+			if (_expectingRepliesTo[bucket][i] == packetId)
+				return true;
+		}
+		return false;
+	}
+
 private:
 	inline SharedPtr<Network> _network(uint64_t nwid) const
 	{
@ -266,6 +297,9 @@ private:

 	void *_uPtr; // _uptr (lower case) is reserved in Visual Studio :P

+	uint8_t _expectingRepliesToBucketPtr[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1];
+	uint64_t _expectingRepliesTo[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1][ZT_EXPECTING_REPLIES_BUCKET_MASK2 + 1];
+
 	ZT_DataStoreGetFunction _dataStoreGetFunction;
 	ZT_DataStorePutFunction _dataStorePutFunction;
 	ZT_WirePacketSendFunction _wirePacketSendFunction;
--- a/node/OutboundMulticast.cpp
+++ b/node/OutboundMulticast.cpp
@ -91,6 +91,7 @@ void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toA
 		//TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str());
 		_packet.newInitializationVector();
 		_packet.setDestination(toAddr2);
+		RR->node->expectReplyTo(_packet.packetId());
 		RR->sw->send(_packet,true);
 	}
 }
--- a/node/Packet.hpp
+++ b/node/Packet.hpp
@ -965,7 +965,7 @@ public:
 		 *   <[2] 16-bit reporter OS/platform or 0 if not specified>
 		 *   <[2] 16-bit reporter architecture or 0 if not specified>
 		 *   <[2] 16-bit error code (set to 0, currently unused)>
-		 *   <[8] 64-bit report flags (set to 0, currently unused)>
+		 *   <[8] 64-bit report flags>
 		 *   <[8] 64-bit packet ID of received CIRCUIT_TEST packet>
 		 *   <[5] upstream ZeroTier address from which CIRCUIT_TEST was received>
 		 *   <[1] 8-bit packet hop count of received CIRCUIT_TEST>
@ -980,6 +980,9 @@ public:
 		 *   <[5] ZeroTier address of next hop>
 		 *   <[...] current best direct path address, if any, 0 if none>
 		 *
+		 * Report flags:
+		 *   0x1 - Upstream peer in circuit test path allowed in path (e.g. network COM valid)
+		 *
 		 * Circuit test reports can be sent by hops in a circuit test to report
 		 * back results. They should include information about the sender as well
 		 * as about the paths to which next hops are being sent.
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@ -266,6 +266,7 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u
 	atAddress.serialize(outp);
 	outp.append((uint64_t)RR->topology->worldId());
 	outp.append((uint64_t)RR->topology->worldTimestamp());
+	RR->node->expectReplyTo(outp.packetId());
 	outp.armor(_key,false); // HELLO is sent in the clear
 	RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size());
 }
@ -274,6 +275,7 @@ void Peer::attemptToContactAt(const InetAddress &localAddr,const InetAddress &at
 {
 	if ( (_vProto >= 5) && ( !((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0)) ) ) {
 		Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO);
+		RR->node->expectReplyTo(outp.packetId());
 		outp.armor(_key,true);
 		RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size());
 	} else {
--- a/node/Switch.cpp
+++ b/node/Switch.cpp
@ -734,13 +734,12 @@ unsigned long Switch::doTimerTasks(uint64_t now)

 Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted)
 {
-	SharedPtr<Peer> root(RR->topology->getBestRoot(peersAlreadyConsulted,numPeersAlreadyConsulted,false));
-	if (root) {
-		Packet outp(root->address(),RR->identity.address(),Packet::VERB_WHOIS);
+	SharedPtr<Peer> upstream(RR->topology->getBestRoot(peersAlreadyConsulted,numPeersAlreadyConsulted,false));
+	if (upstream) {
+		Packet outp(upstream->address(),RR->identity.address(),Packet::VERB_WHOIS);
 		addr.appendTo(outp);
-		outp.armor(root->key(),true);
-		if (root->sendDirect(outp.data(),outp.size(),RR->node->now(),true))
-			return root->address();
+		RR->node->expectReplyTo(outp.packetId());
+		send(outp,true);
 	}
 	return Address();
 }