So we need to keep track of external surface per reporter, since some NATs assign different external IPs for each external destination. Keeping just one known surface could create a race condition.

node/IncomingPacket.cpp

@@ -273,7 +273,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 			trusted = true;
 		}
 		if (destAddr)
-			RR->sa->iam(id.address(),_remoteAddress,destAddr,trusted);
+			RR->sa->iam(id.address(),_remoteAddress,destAddr,trusted,RR->node->now());
 
 		Packet outp(id.address(),RR->identity.address(),Packet::VERB_OK);
 
@@ -358,7 +358,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 					trusted = true;
 				}
 				if (destAddr)
-					RR->sa->iam(peer->address(),_remoteAddress,destAddr,trusted);
+					RR->sa->iam(peer->address(),_remoteAddress,destAddr,trusted,RR->node->now());
 			}	break;
 
 			case Packet::VERB_WHOIS: {

node/InetAddress.hpp

@@ -63,9 +63,6 @@ struct InetAddress : public sockaddr_storage
 
 	/**
 	 * IP address scope
-	 *
-	 * Do not change these numeric index values without taking a look
-	 * at SelfAwareness. Values 1-5 are mapped onto an array index.
 	 */
 	enum IpScope
 	{

node/Node.cpp

@@ -264,6 +264,7 @@ ZT1_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *next
 
 		try {
 			RR->topology->clean(now);
+			RR->sa->clean(now);
 		} catch ( ... ) {
 			return ZT1_RESULT_FATAL_ERROR_INTERNAL;
 		}

node/SelfAwareness.cpp

@@ -37,6 +37,9 @@
 #include "Packet.hpp"
 #include "Peer.hpp"
 
+// Entry timeout -- make it fairly long since this is just to prevent stale buildup
+#define ZT_SELFAWARENESS_ENTRY_TIMEOUT 3600000
+
 namespace ZeroTier {
 
 class _ResetWithinScope
@@ -64,53 +67,59 @@ private:
 SelfAwareness::SelfAwareness(const RuntimeEnvironment *renv) :
 	RR(renv)
 {
-	memset(_lastPhysicalAddress,0,sizeof(_lastPhysicalAddress));
 }
 
 SelfAwareness::~SelfAwareness()
 {
 }
 
-void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted)
+void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now)
 {
-	// This code depends on the numeric values assigned to scopes in InetAddress.hpp
+	const InetAddress::IpScope scope = myPhysicalAddress.ipScope();
-	const unsigned int scope = (unsigned int)myPhysicalAddress.ipScope();
+
-	if ((scope > 0)&&(scope < (unsigned int)InetAddress::IP_SCOPE_LOOPBACK)) {
+	switch(scope) {
-		if ( (!trusted) && ((scope == (unsigned int)InetAddress::IP_SCOPE_GLOBAL)||(scope != (unsigned int)reporterPhysicalAddress.ipScope())) ) {
+		case InetAddress::IP_SCOPE_NONE:
-			/* For now only trusted peers are permitted to inform us of changes to
+		case InetAddress::IP_SCOPE_LOOPBACK:
-			 * our global Internet IP or to changes of NATed IPs. We'll let peers on
+		case InetAddress::IP_SCOPE_MULTICAST:
-			 * private, shared, or link-local networks inform us of changes as long
-			 * as they too are at the same scope. This discrimination avoids a DoS
-			 * attack in which an attacker could force us to reset our connections. */
 			return;
-		} else {
+		case InetAddress::IP_SCOPE_GLOBAL:
-			Mutex::Lock _l(_lock);
+			if ((!trusted)||(scope != reporterPhysicalAddress.ipScope()))
-			InetAddress &lastPhy = _lastPhysicalAddress[scope - 1];
+				return;
-			if (!lastPhy) {
+			break;
-				TRACE("learned physical address %s for scope %u from reporter %s(%s) (replaced <null>)",myPhysicalAddress.toString().c_str(),scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str());
+		default:
-				lastPhy = myPhysicalAddress;
+			if (scope != reporterPhysicalAddress.ipScope())
-			} else if (lastPhy != myPhysicalAddress) {
+				return;
-				TRACE("learned physical address %s for scope %u from reporter %s(%s) (replaced %s, resetting within scope)",myPhysicalAddress.toString().c_str(),scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str(),lastPhy.toString().c_str());
+			break;
-				lastPhy = myPhysicalAddress;
+	}
-				uint64_t now = RR->node->now();
 
-				_ResetWithinScope rset(RR,now,(InetAddress::IpScope)scope);
+	Mutex::Lock _l(_phy_m);
-				RR->topology->eachPeer<_ResetWithinScope &>(rset);
 
-				// For all peers for whom we forgot an address, send a packet indirectly if
+	PhySurfaceEntry &entry = _phy[PhySurfaceKey(reporter,scope)];
-				// they are still considered alive so that we will re-establish direct links.
+
-				SharedPtr<Peer> sn(RR->topology->getBestSupernode());
+	if (!entry.ts) {
-				if (sn) {
+		entry.mySurface = myPhysicalAddress;
-					Path *snp = sn->getBestPath(now);
+		entry.ts = now;
-					if (snp) {
+		TRACE("learned physical address %s for scope %u as seen from %s(%s) (replaced <null>)",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str());
-						for(std::vector< SharedPtr<Peer> >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) {
+	} else if (entry.mySurface != myPhysicalAddress) {
-							if ((*p)->alive(now)) {
+		entry.mySurface = myPhysicalAddress;
-								TRACE("sending indirect NOP to %s via %s(%s) to re-establish link",(*p)->address().toString().c_str(),sn->address().toString().c_str(),snp->address().toString().c_str());
+		entry.ts = now;
-								Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP);
+		TRACE("learned physical address %s for scope %u as seen from %s(%s) (replaced %s, resetting all in scope)",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str(),entry.mySurface.toString().c_str());
-								outp.armor((*p)->key(),true);
+
-								snp->send(RR,outp.data(),outp.size(),now);
+		_ResetWithinScope rset(RR,now,(InetAddress::IpScope)scope);
-							}
+		RR->topology->eachPeer<_ResetWithinScope &>(rset);
-						}
+
+		// For all peers for whom we forgot an address, send a packet indirectly if
+		// they are still considered alive so that we will re-establish direct links.
+		SharedPtr<Peer> sn(RR->topology->getBestSupernode());
+		if (sn) {
+			Path *snp = sn->getBestPath(now);
+			if (snp) {
+				for(std::vector< SharedPtr<Peer> >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) {
+					if ((*p)->alive(now)) {
+						TRACE("sending indirect NOP to %s via %s(%s) to re-establish link",(*p)->address().toString().c_str(),sn->address().toString().c_str(),snp->address().toString().c_str());
+						Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP);
+						outp.armor((*p)->key(),true);
+						snp->send(RR,outp.data(),outp.size(),now);
 					}
 				}
 			}
@@ -118,4 +127,14 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi
 	}
 }
 
+void SelfAwareness::clean(uint64_t now)
+{
+	Mutex::Lock _l(_phy_m);
+	for(std::map< PhySurfaceKey,PhySurfaceEntry >::iterator p(_phy.begin());p!=_phy.end();) {
+		if ((now - p->second.ts) >= ZT_SELFAWARENESS_ENTRY_TIMEOUT)
+			_phy.erase(p++);
+		else ++p;
+	}
+}
+
 } // namespace ZeroTier

node/SelfAwareness.hpp

@@ -28,6 +28,8 @@
 #ifndef ZT_SELFAWARENESS_HPP
 #define ZT_SELFAWARENESS_HPP
 
+#include <map>
+
 #include "InetAddress.hpp"
 #include "Address.hpp"
 #include "Mutex.hpp"
@@ -52,13 +54,41 @@ public:
 	 * @param reporterPhysicalAddress Physical address that reporting peer seems to have
 	 * @param myPhysicalAddress Physical address that peer says we have
 	 * @param trusted True if this peer is trusted as an authority to inform us of external address changes
+	 * @param now Current time
 	 */
-	void iam(const Address &reporter,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted);
+	void iam(const Address &reporter,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,uint64_t now);
+
+	/**
+	 * Clean up database periodically
+	 *
+	 * @param now Current time
+	 */
+	void clean(uint64_t now);
 
 private:
+	struct PhySurfaceKey
+	{
+		Address reporter;
+		InetAddress::IpScope scope;
+
+		PhySurfaceKey() : reporter(),scope(InetAddress::IP_SCOPE_NONE) {}
+		PhySurfaceKey(const Address &r,InetAddress::IpScope s) : reporter(r),scope(s) {}
+		inline bool operator<(const PhySurfaceKey &k) const throw() { return ((reporter < k.reporter) ? true : ((reporter == k.reporter) ? ((int)scope < (int)k.scope) : false)); }
+		inline bool operator==(const PhySurfaceKey &k) const throw() { return ((reporter == k.reporter)&&(scope == k.scope)); }
+	};
+	struct PhySurfaceEntry
+	{
+		InetAddress mySurface;
+		uint64_t ts;
+
+		PhySurfaceEntry() : mySurface(),ts(0) {}
+		PhySurfaceEntry(const InetAddress &a,const uint64_t t) : mySurface(a),ts(t) {}
+	};
+
 	const RuntimeEnvironment *RR;
-	Mutex _lock;
+
-	InetAddress _lastPhysicalAddress[5]; // 5 == the number of address classes we care about, see InetAddress.hpp and SelfAwareness.cpp
+	std::map< PhySurfaceKey,PhySurfaceEntry > _phy;
+	Mutex _phy_m;
 };
 
 } // namespace ZeroTier