Wiring back up PUSH_DIRECT_PATHS and friends.

go/native/GoGlue.cpp

@@ -28,14 +28,10 @@
 
 #ifndef __WINDOWS__
 #include <unistd.h>
-#include <fcntl.h>
-#include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/un.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
-#include <netinet/ip6.h>
-#include <netinet/tcp.h>
 #ifdef __BSD__
 #include <net/if.h>
 #endif
@@ -104,7 +100,7 @@ const char *ZT_PLATFORM_DEFAULT_HOMEPATH = defaultHomePath.c_str();
 
 /* These functions are implemented in Go in pkg/ztnode/node-callbacks.go */
 extern "C" int goPathCheckFunc(void *,uint64_t,int,const void *,int);
-extern "C" int goPathLookupFunc(void *,uint64_t,int,int *,uint8_t [16],int *);
+extern "C" int goPathLookupFunc(void *,uint64_t,int,const ZT_Identity *,int *,uint8_t [16],int *);
 extern "C" void goStateObjectPutFunc(void *,int,const uint64_t [2],const void *,int);
 extern "C" int goStateObjectGetFunc(void *,int,const uint64_t [2],void *,unsigned int);
 extern "C" void goVirtualNetworkConfigFunc(void *,ZT_GoTap *,uint64_t,int,const ZT_VirtualNetworkConfig *);
@@ -282,6 +278,7 @@ static int ZT_GoNode_PathLookupFunction(
 		reinterpret_cast<ZT_GoNode *>(uptr)->goUserPtr,
 		ztAddress,
 		desiredAddressFamily,
+		id,
 		&family,
 		ip,
 		&port

go/pkg/zerotier/identity.go

@@ -46,6 +46,18 @@ type Identity struct {
 	privateKey []byte
 }
 
+func newIdentityFromCIdentity(cid unsafe.Pointer) (*Identity, error) {
+	if uintptr(cid) == 0 {
+		return nil, ErrInvalidParameter
+	}
+	var idStrBuf [4096]byte
+	idStr := C.ZT_Identity_toString(cid,(*C.char)(unsafe.Pointer(&idStrBuf[0])),4096,1)
+	if uintptr(unsafe.Pointer(idStr)) == 0 {
+		return nil, ErrInternal
+	}
+	return NewIdentityFromString(C.GoString(idStr))
+}
+
 // NewIdentity generates a new identity of the selected type
 func NewIdentity(identityType int) (*Identity, error) {
 	cid := C.ZT_Identity_new(C.enum_ZT_Identity_Type(identityType))
@@ -53,12 +65,7 @@ func NewIdentity(identityType int) (*Identity, error) {
 		return nil, ErrInternal
 	}
 	defer C.ZT_Identity_delete(cid)
-	var idStrBuf [4096]byte
+	return newIdentityFromCIdentity(cid)
-	idStr := C.ZT_Identity_toString(cid,(*C.char)(unsafe.Pointer(&idStrBuf[0])),4096,1)
-	if uintptr(unsafe.Pointer(idStr)) == 0 {
-		return nil, ErrInternal
-	}
-	return NewIdentityFromString(C.GoString(idStr))
 }
 
 // NewIdentityFromString generates a new identity from its string representation.

go/pkg/zerotier/node.go

@@ -660,10 +660,10 @@ func (n *Node) pathCheck(ztAddress Address, af int, ip net.IP, port int) bool {
 	return true
 }
 
-func (n *Node) pathLookup(ztAddress Address) (net.IP, int) {
+func (n *Node) pathLookup(id *Identity) (net.IP, int) {
 	n.localConfigLock.RLock()
 	defer n.localConfigLock.RUnlock()
-	virt := n.localConfig.Virtual[ztAddress]
+	virt := n.localConfig.Virtual[id.address]
 	if len(virt.Try) > 0 {
 		idx := rand.Int() % len(virt.Try)
 		return virt.Try[idx].IP, virt.Try[idx].Port
@@ -763,7 +763,7 @@ func goPathCheckFunc(gn unsafe.Pointer, ztAddress C.uint64_t, af C.int, ip unsaf
 }
 
 //export goPathLookupFunc
-func goPathLookupFunc(gn unsafe.Pointer, ztAddress C.uint64_t, _ int, familyP, ipP, portP unsafe.Pointer) C.int {
+func goPathLookupFunc(gn unsafe.Pointer, ztAddress C.uint64_t, desiredFamily int, identity, familyP, ipP, portP unsafe.Pointer) C.int {
 	nodesByUserPtrLock.RLock()
 	node := nodesByUserPtr[uintptr(gn)]
 	nodesByUserPtrLock.RUnlock()
@@ -771,7 +771,11 @@ func goPathLookupFunc(gn unsafe.Pointer, ztAddress C.uint64_t, _ int, familyP, i
 		return 0
 	}
 
-	ip, port := node.pathLookup(Address(ztAddress))
+	id, err := newIdentityFromCIdentity(identity)
+	if err != nil {
+		return 0
+	}
+	ip, port := node.pathLookup(id)
 	if len(ip) > 0 && port > 0 && port <= 65535 {
 		ip4 := ip.To4()
 		if len(ip4) == 4 {

go/pkg/zerotier/root.go

@@ -0,0 +1,8 @@
+package zerotier
+
+// Root is a root server with one or more permanent IPs.
+type Root struct {
+	Identity Identity
+	DNSName string
+	PhysicalAddresses []InetAddress
+}

node/Constants.hpp

@@ -141,6 +141,11 @@
  */
 #define ZT_PEER_ACTIVITY_TIMEOUT ((ZT_PEER_PING_PERIOD * 2) + 5000)
 
+/**
+ * Maximum interval between sort/prioritize of paths for a peer
+ */
+#define ZT_PEER_PRIORITIZE_PATHS_INTERVAL 5000
+
 /**
  * Delay between requests for updated network autoconf information
  *

node/IncomingPacket.cpp

@@ -348,6 +348,7 @@ ZT_ALWAYS_INLINE bool _doWHOIS(IncomingPacket &pkt,const RuntimeEnvironment *con
 ZT_ALWAYS_INLINE bool _doRENDEZVOUS(IncomingPacket &pkt,const RuntimeEnvironment *const RR,void *const tPtr,const SharedPtr<Peer> &peer,const SharedPtr<Path> &path)
 {
 	if (RR->topology->isRoot(peer->identity())) {
+		uint16_t junk = (uint16_t)Utils::random();
 		const Address with(pkt.field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH);
 		const SharedPtr<Peer> rendezvousWith(RR->topology->get(with));
 		if (rendezvousWith) {
@@ -355,9 +356,9 @@ ZT_ALWAYS_INLINE bool _doRENDEZVOUS(IncomingPacket &pkt,const RuntimeEnvironment
 			const unsigned int addrlen = pkt[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN];
 			if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) {
 				InetAddress atAddr(pkt.field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port);
-				if (RR->node->shouldUsePathForZeroTierTraffic(tPtr,with,path->localSocket(),atAddr)) {
+				if (rendezvousWith->shouldTryPath(tPtr,RR->node->now(),peer,atAddr)) {
-					const uint64_t junk = Utils::random();
+					if (atAddr.isV4())
-					RR->node->putPacket(tPtr,path->localSocket(),atAddr,&junk,4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls
+						RR->node->putPacket(tPtr,path->localSocket(),atAddr,&junk,2,2); // IPv4 "firewall opener"
 					rendezvousWith->sendHELLO(tPtr,path->localSocket(),atAddr,RR->node->now());
 				}
 			}
@@ -669,6 +670,7 @@ ZT_ALWAYS_INLINE bool _doPUSH_DIRECT_PATHS(IncomingPacket &pkt,const RuntimeEnvi
 
 		unsigned int count = pkt.at<uint16_t>(ZT_PACKET_IDX_PAYLOAD);
 		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 2;
+		uint16_t junk = (uint16_t)Utils::random();
 
 		while (count--) {
 			/* unsigned int flags = (*this)[ptr++]; */ ++ptr;
@@ -680,18 +682,17 @@ ZT_ALWAYS_INLINE bool _doPUSH_DIRECT_PATHS(IncomingPacket &pkt,const RuntimeEnvi
 			switch(addrType) {
 				case 4: {
 					const InetAddress a(pkt.field(ptr,4),4,pkt.at<uint16_t>(ptr + 4));
-					if ((!peer->hasActivePathTo(now,a)) && // not already known
+					if (peer->shouldTryPath(tPtr,now,peer,a)) {
-							(RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),-1,a)) ) // should use path
+						if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
-					{
+							RR->node->putPacket(tPtr,path->localSocket(),a,&junk,2,2); // IPv4 "firewall opener"
-						if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY)
+							++junk;
 							peer->sendHELLO(tPtr,-1,a,now);
+						}
 					}
 				}	break;
 				case 6: {
 					const InetAddress a(pkt.field(ptr,16),16,pkt.at<uint16_t>(ptr + 16));
-					if ((!peer->hasActivePathTo(now,a)) && // not already known
+					if (peer->shouldTryPath(tPtr,now,peer,a)) {
-							(RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),-1,a)) ) // should use path
-					{
 						if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY)
 							peer->sendHELLO(tPtr,-1,a,now);
 					}

node/Mutex.hpp

@@ -60,13 +60,14 @@ public:
 	ZT_ALWAYS_INLINE void lock() const { pthread_rwlock_wrlock(&((const_cast <RWMutex *> (this))->_mh)); }
 	ZT_ALWAYS_INLINE void rlock() const { pthread_rwlock_rdlock(&((const_cast <RWMutex *> (this))->_mh)); }
 	ZT_ALWAYS_INLINE void unlock() const { pthread_rwlock_unlock(&((const_cast <RWMutex *> (this))->_mh)); }
+	ZT_ALWAYS_INLINE void runlock() const { pthread_rwlock_unlock(&((const_cast <RWMutex *> (this))->_mh)); }
 
 	class RLock
 	{
 	public:
 		ZT_ALWAYS_INLINE RLock(RWMutex &m) : _m(&m) { m.rlock(); }
 		ZT_ALWAYS_INLINE RLock(const RWMutex &m) : _m(const_cast<RWMutex *>(&m)) { _m->rlock(); }
-		ZT_ALWAYS_INLINE ~RLock() { _m->unlock(); }
+		ZT_ALWAYS_INLINE ~RLock() { _m->runlock(); }
 	private:
 		RWMutex *const _m;
 	};

node/Node.cpp

@@ -174,10 +174,10 @@ struct _processBackgroundTasks_ping_eachPeer
 	ZT_ALWAYS_INLINE bool operator()(const SharedPtr<Peer> &peer,const bool isRoot)
 	{
 		unsigned int v4SendCount = 0,v6SendCount = 0;
-		peer->ping(tPtr,now,v4SendCount,v6SendCount);
+		peer->ping(tPtr,now,v4SendCount,v6SendCount,isRoot);
 
 		if (isRoot) {
-			if ((now - peer->lastReceive()) <= ZT_PEER_PING_PERIOD)
+			if ((now - peer->lastReceive()) <= (ZT_PEER_PING_PERIOD + 5000))
 				online = true;
 
 			if (v4SendCount == 0) {

node/Path.hpp

@@ -92,6 +92,13 @@ public:
 	 */
 	bool send(const RuntimeEnvironment *RR,void *tPtr,const void *data,unsigned int len,int64_t now);
 
+	/**
+	 * Explicitly update last sent time
+	 *
+	 * @param t Time of send
+	 */
+	ZT_ALWAYS_INLINE void sent(const uint64_t t) { _lastOut = t; }
+
 	/**
 	 * Called when a packet is received from this remote path, regardless of content
 	 *

node/Peer.cpp

@@ -23,15 +23,25 @@
 
 namespace ZeroTier {
 
+struct _PathPriorityComparisonOperator
+{
+	ZT_ALWAYS_INLINE bool operator()(const SharedPtr<Path> &a,const SharedPtr<Path> &b) const
+	{
+		return ( ((a)&&(a->lastIn() > 0)) && ((!b)||(b->lastIn() <= 0)||(a->lastIn() < b->lastIn())) );
+	}
+};
+
 Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) :
 	RR(renv),
 	_lastReceive(0),
 	_lastWhoisRequestReceived(0),
 	_lastEchoRequestReceived(0),
 	_lastPushDirectPathsReceived(0),
+	_lastPushDirectPathsSent(0),
 	_lastTriedStaticPath(0),
+	_lastPrioritizedPaths(0),
 	_latency(0xffff),
-	_pathCount(0),
+	_alivePathCount(0),
 	_id(peerIdentity),
 	_vProto(0),
 	_vMajor(0),
@@ -54,85 +64,64 @@ void Peer::received(
 	const uint64_t networkId)
 {
 	const int64_t now = RR->node->now();
-
 	_lastReceive = now;
 
-	/*
 	if (hops == 0) {
-		// If this is a direct packet (no hops), update existing paths or learn new ones
+		_paths_l.rlock();
-		bool havePath = false;
+		for(int i=0;i<(int)_alivePathCount; ++i) {
-		{
+			if (_paths[i] == path) {
-			Mutex::Lock _l(_paths_m);
+				_paths_l.runlock();
-			for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
+				goto path_check_done;
-				if (_paths[i]) {
-					if (_paths[i] == path) {
-						havePath = true;
-						break;
-					}
-				} else break;
 			}
 		}
+		_paths_l.runlock();
 
-		bool attemptToContact = false;
+		if (verb == Packet::VERB_OK) {
-		if ((!havePath)&&(RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localSocket(),path->address()))) {
+			RWMutex::Lock l(_paths_l);
-			Mutex::Lock _l(_paths_m);
 
-			// Paths are redundant if they duplicate an alive path to the same IP or
+			int64_t lastReceiveTimeMax = 0;
-			// with the same local socket and address family.
+			int lastReceiveTimeMaxAt = 0;
-			bool redundant = false;
+			for(int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
-			unsigned int replacePath = ZT_MAX_PEER_NETWORK_PATHS;
+				if ((_paths[i]->address().ss_family == path->address().ss_family) &&
-			for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
+				    (_paths[i]->localSocket() == path->localSocket()) && // TODO: should be localInterface when multipath is integrated
-				if (_paths[i]) {
+				    (_paths[i]->address().ipsEqual2(path->address()))) {
-					if ( (_paths[i]->alive(now)) && ( ((_paths[i]->localSocket() == path->localSocket())&&(_paths[i]->address().ss_family == path->address().ss_family)) || (_paths[i]->address().ipsEqual2(path->address())) ) ) {
+					// If this is another path to the same place, swap it out as the
-						redundant = true;
+					// one we just received from may replace an old one but don't
-						break;
+					// learn it as a new path.
-					}
+					_paths[i] = path;
-					// If the path is the same address and port, simply assume this is a replacement
+					goto path_check_done;
-					if ( (_paths[i]->address().ipsEqual2(path->address()))) {
+				} else {
-						replacePath = i;
-						break;
-					}
-				} else break;
-			}
-
-			// If the path isn't a duplicate of the same localSocket AND we haven't already determined a replacePath,
-			// then find the worst path and replace it.
-			if (!redundant && replacePath == ZT_MAX_PEER_NETWORK_PATHS) {
-				int replacePathQuality = 0;
-				for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
 					if (_paths[i]) {
-						const int q = _paths[i]->quality(now);
+						if (_paths[i]->lastIn() > lastReceiveTimeMax) {
-						if (q > replacePathQuality) {
+							lastReceiveTimeMax = _paths[i]->lastIn();
-							replacePathQuality = q;
+							lastReceiveTimeMaxAt = i;
-							replacePath = i;
 						}
 					} else {
-						replacePath = i;
+						lastReceiveTimeMax = 0x7fffffffffffffffLL;
-						break;
+						lastReceiveTimeMaxAt = i;
 					}
 				}
 			}
 
-			if (replacePath != ZT_MAX_PEER_NETWORK_PATHS) {
+			_lastPrioritizedPaths = now;
-				if (verb == Packet::VERB_OK) {
+			_paths[lastReceiveTimeMaxAt] = path;
-					RR->t->peerLearnedNewPath(tPtr,networkId,*this,path,packetId);
+			_prioritizePaths(now);
-					_paths[replacePath] = path;
+			RR->t->peerLearnedNewPath(tPtr,networkId,*this,path,packetId);
-				} else {
+		} else {
-					attemptToContact = true;
+			if (RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localSocket(),path->address())) {
-				}
+				sendHELLO(tPtr,path->localSocket(),path->address(),now);
+				path->sent(now);
+				RR->t->peerConfirmingUnknownPath(tPtr,networkId,*this,path,packetId,verb);
 			}
 		}
-
-		if (attemptToContact) {
-			sendHELLO(tPtr,path->localSocket(),path->address(),now);
-			path->sent(now);
-			RR->t->peerConfirmingUnknownPath(tPtr,networkId,*this,path,packetId,verb);
-		}
 	}
-	*/
 
-	// Periodically push direct paths to the peer, doing so more often if we do not
+path_check_done:
-	// currently have a direct path.
+	const int64_t sinceLastPush = now - _lastPushDirectPathsSent;
+	if (sinceLastPush >= ((hops == 0) ? ZT_DIRECT_PATH_PUSH_INTERVAL_HAVEPATH : ZT_DIRECT_PATH_PUSH_INTERVAL)) {
+		_lastPushDirectPathsReceived = now;
+	}
+
 	/*
 	const int64_t sinceLastPush = now - _lastDirectPathPushSent;
 	if (sinceLastPush >= ((hops == 0) ? ZT_DIRECT_PATH_PUSH_INTERVAL_HAVEPATH : ZT_DIRECT_PATH_PUSH_INTERVAL)) {
@@ -189,10 +178,23 @@ void Peer::received(
 	*/
 }
 
-bool Peer::hasActivePathTo(int64_t now,const InetAddress &addr) const
+bool Peer::shouldTryPath(void *tPtr,int64_t now,const SharedPtr<Peer> &suggestedBy,const InetAddress &addr) const
 {
-	// TODO
+	int maxHaveScope = -1;
-	return false;
+	{
+		RWMutex::RLock l(_paths_l);
+		for (unsigned int i = 0; i < _alivePathCount; ++i) {
+			if (_paths[i]) {
+				if (_paths[i]->address().ipsEqual2(addr))
+					return false;
+
+				int s = (int)_paths[i]->address().ipScope();
+				if (s > maxHaveScope)
+					maxHaveScope = s;
+			}
+		}
+	}
+	return ( ((int)addr.ipScope() > maxHaveScope) && RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),-1,addr) );
 }
 
 void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now)
@@ -217,54 +219,125 @@ void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atA
 	}
 }
 
-void Peer::ping(void *tPtr,int64_t now,unsigned int &v4SendCount,unsigned int &v6SendCount)
+void Peer::ping(void *tPtr,int64_t now,unsigned int &v4SendCount,unsigned int &v6SendCount,const bool pingAllAddressTypes)
 {
-	/*
+	RWMutex::RLock l(_paths_l);
-	Mutex::Lock _l(_paths_m);
 
-	unsigned int j = 0;
+	_lastPrioritizedPaths = now;
-	for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
+	_prioritizePaths(now);
-		if ((_paths[i])&&(_paths[i]->alive(now))) {
+
+	if (_alivePathCount > 0) {
+		for (unsigned int i = 0; i < _alivePathCount; ++i) {
 			sendHELLO(tPtr,_paths[i]->localSocket(),_paths[i]->address(),now);
-
 			_paths[i]->sent(now);
+
 			if (_paths[i]->address().isV4())
 				++v4SendCount;
 			else if (_paths[i]->address().isV6())
 				++v6SendCount;
 
-			if (i != j)
+			if (!pingAllAddressTypes)
-				_paths[j] = _paths[i];
+				break;
-			++j;
+		}
+	} else {
+		SharedPtr<Peer> r(RR->topology->root());
+		if (r) {
+			SharedPtr<Path> rp(r->path(now));
+			if (rp) {
+				sendHELLO(tPtr,rp->localSocket(),rp->address(),now);
+				rp->sent(now);
+			}
 		}
 	}
-	while(j < ZT_MAX_PEER_NETWORK_PATHS) {
-		_paths[j].zero();
-		++j;
-	}
-	*/
 }
 
 void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,int64_t now)
 {
-	/*
+	RWMutex::RLock l(_paths_l);
-	Mutex::Lock _l(_paths_m);
+	for(unsigned int i=0; i < _alivePathCount; ++i) {
-	for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
+		if ((_paths[i])&&((_paths[i]->address().ss_family == inetAddressFamily)&&(_paths[i]->address().ipScope() == scope))) {
-		if (_paths[i]) {
+			sendHELLO(tPtr,_paths[i]->localSocket(),_paths[i]->address(),now);
-			if ((_paths[i]->address().ss_family == inetAddressFamily)&&(_paths[i]->ipScope() == scope)) {
+			_paths[i]->sent(now);
-				sendHELLO(tPtr,_paths[i]->localSocket(),_paths[i]->address(),now);
+		}
-				_paths[i]->sent(now);
+	}
-			}
+}
-		} else break;
+
+void Peer::updateLatency(const unsigned int l)
+{
+	if ((l > 0)&&(l < 0xffff)) {
+		unsigned int lat = _latency;
+		if (lat < 0xffff) {
+			_latency = (l + l + lat) / 3;
+		} else {
+			_latency = l;
+		}
+	}
+}
+
+bool Peer::sendDirect(void *tPtr,const void *data,const unsigned int len,const int64_t now)
+{
+	if ((now - _lastPrioritizedPaths) > ZT_PEER_PRIORITIZE_PATHS_INTERVAL) {
+		_lastPrioritizedPaths = now;
+		_paths_l.lock();
+		_prioritizePaths(now);
+		if (_alivePathCount == 0) {
+			_paths_l.unlock();
+			return false;
+		}
+		const bool r = _paths[0]->send(RR,tPtr,data,len,now);
+		_paths_l.unlock();
+		return r;
+	} else {
+		_paths_l.rlock();
+		if (_alivePathCount == 0) {
+			_paths_l.runlock();
+			return false;
+		}
+		const bool r = _paths[0]->send(RR,tPtr,data,len,now);
+		_paths_l.runlock();
+		return r;
+	}
+}
+
+SharedPtr<Path> Peer::path(const int64_t now)
+{
+	if ((now - _lastPrioritizedPaths) > ZT_PEER_PRIORITIZE_PATHS_INTERVAL) {
+		_lastPrioritizedPaths = now;
+		RWMutex::Lock l(_paths_l);
+		_prioritizePaths(now);
+		if (_alivePathCount == 0)
+			return SharedPtr<Path>();
+		return _paths[0];
+	} else {
+		RWMutex::RLock l(_paths_l);
+		if (_alivePathCount == 0)
+			return SharedPtr<Path>();
+		return _paths[0];
 	}
-	*/
 }
 
 void Peer::getAllPaths(std::vector< SharedPtr<Path> > &paths)
 {
 	RWMutex::RLock l(_paths_l);
 	paths.clear();
-	paths.assign(_paths,_paths + _pathCount);
+	paths.assign(_paths,_paths + _alivePathCount);
+}
+
+void Peer::_prioritizePaths(const int64_t now)
+{
+	// assumes _paths_l is locked for writing
+	std::sort(_paths,_paths + ZT_MAX_PEER_NETWORK_PATHS,_PathPriorityComparisonOperator());
+
+	for(int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
+		if ((!_paths[i]) || (!_paths[i]->alive(now))) {
+			_alivePathCount = i;
+
+			for(;i<ZT_MAX_PEER_NETWORK_PATHS;++i)
+				_paths[i].zero();
+
+			return;
+		}
+	}
 }
 
 } // namespace ZeroTier

node/Peer.hpp

@@ -40,7 +40,7 @@ class Peer
 	friend class SharedPtr<Peer>;
 
 private:
-	inline Peer() {}
+	ZT_ALWAYS_INLINE Peer() {}
 
 public:
 	ZT_ALWAYS_INLINE ~Peer() { Utils::burn(_key,sizeof(_key)); }
@@ -92,13 +92,14 @@ public:
 		uint64_t networkId);
 
 	/**
-	 * Check whether we have an active path to this peer via the given address
+	 * Check whether a path to this peer should be tried if received via e.g. RENDEZVOUS OR PUSH_DIRECT_PATHS
 	 *
 	 * @param now Current time
+	 * @param suggestingPeer Peer suggesting path (may be this peer)
 	 * @param addr Remote address
 	 * @return True if we have an active path to this destination
 	 */
-	bool hasActivePathTo(int64_t now,const InetAddress &addr) const;
+	bool shouldTryPath(void *tPtr,int64_t now,const SharedPtr<Peer> &suggestedBy,const InetAddress &addr) const;
 
 	/**
 	 * Send a HELLO to this peer at a specified physical address
@@ -113,16 +114,15 @@ public:
 	void sendHELLO(void *tPtr,int64_t localSocket,const InetAddress &atAddress,int64_t now);
 
 	/**
-	 * Send pings to active paths
+	 * Send ping to this peer
-	 *
-	 * This also cleans up some internal data structures. It's called periodically from Node.
 	 *
 	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
 	 * @param now Current time
 	 * @param v4SendCount Number of IPv4 packets sent (result parameter)
 	 * @param v6SendCount Number of IPv6 packets sent (result parameter)
+	 * @param pingAllAddressTypes If true, try to keep a link up for each address type/family
 	 */
-	void ping(void *tPtr,int64_t now,unsigned int &v4SendCount,unsigned int &v6SendCount);
+	void ping(void *tPtr,int64_t now,unsigned int &v4SendCount,unsigned int &v6SendCount,bool pingAllAddressTypes);
 
 	/**
 	 * Reset paths within a given IP scope and address family
@@ -161,15 +161,7 @@ public:
 	 *
 	 * @param l New latency measurment (in milliseconds)
 	 */
-	ZT_ALWAYS_INLINE void updateLatency(const unsigned int l)
+	void updateLatency(const unsigned int l);
-	{
-		if ((l > 0)&&(l < 0xffff)) {
-			unsigned int lat = _latency;
-			if (lat < 0xffff)
-				_latency = (l + lat) / 2;
-			else _latency = l;
-		}
-	}
 
 	/**
 	 * @return 256-bit secret symmetric encryption key
@@ -255,28 +247,12 @@ public:
 	 * @param now Current time
 	 * @return True if packet appears to have been sent, false if no path or send failed
 	 */
-	ZT_ALWAYS_INLINE bool sendDirect(void *tPtr,const void *data,const unsigned int len,const int64_t now)
+	bool sendDirect(void *tPtr,const void *data,unsigned int len,const int64_t now);
-	{
-		_paths_l.rlock();
-		if (_pathCount == 0) {
-			_paths_l.unlock();
-			return false;
-		}
-		const bool r = _paths[0]->send(RR,tPtr,data,len,now);
-		_paths_l.unlock();
-		return r;
-	}
 
 	/**
 	 * @return Current best path
 	 */
-	ZT_ALWAYS_INLINE SharedPtr<Path> path()
+	SharedPtr<Path> path(int64_t now);
-	{
-		RWMutex::RLock l(_paths_l);
-		if (_pathCount == 0)
-			return SharedPtr<Path>();
-		return _paths[0];
-	}
 
 	/**
 	 * Get all paths
@@ -286,20 +262,24 @@ public:
 	void getAllPaths(std::vector< SharedPtr<Path> > &paths);
 
 private:
+	void _prioritizePaths(int64_t now);
+
 	uint8_t _key[ZT_PEER_SECRET_KEY_LENGTH];
 
 	const RuntimeEnvironment *RR;
 
-	int64_t _lastReceive;
+	volatile int64_t _lastReceive;
-	int64_t _lastWhoisRequestReceived;
+	volatile int64_t _lastWhoisRequestReceived;
-	int64_t _lastEchoRequestReceived;
+	volatile int64_t _lastEchoRequestReceived;
-	int64_t _lastPushDirectPathsReceived;
+	volatile int64_t _lastPushDirectPathsReceived;
-	int64_t _lastTriedStaticPath;
+	volatile int64_t _lastPushDirectPathsSent;
-	unsigned int _latency;
+	volatile int64_t _lastTriedStaticPath;
+	volatile int64_t _lastPrioritizedPaths;
+	volatile unsigned int _latency;
 
 	AtomicCounter __refCount;
 
-	unsigned int _pathCount;
+	unsigned int _alivePathCount;
 	SharedPtr<Path> _paths[ZT_MAX_PEER_NETWORK_PATHS];
 	RWMutex _paths_l;
 

node/Switch.cpp

@@ -59,7 +59,7 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre
 						fragment.incrementHops();
 						SharedPtr<Peer> relayTo = RR->topology->get(destination);
 						if ((!relayTo)||(!relayTo->sendDirect(tPtr,fragment.data(),fragment.size(),now))) {
-							relayTo = RR->topology->findRelayTo(now,destination);
+							relayTo = RR->topology->root();
 							if (relayTo)
 								relayTo->sendDirect(tPtr,fragment.data(),fragment.size(),now);
 						}
@@ -127,7 +127,7 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre
 						packet.incrementHops();
 						SharedPtr<Peer> relayTo = RR->topology->get(destination);
 						if ((!relayTo)||(!relayTo->sendDirect(tPtr,packet.data(),packet.size(),now))) {
-							relayTo = RR->topology->findRelayTo(now,destination);
+							relayTo = RR->topology->root();
 							if ((relayTo)&&(relayTo->address() != source))
 								relayTo->sendDirect(tPtr,packet.data(),packet.size(),now);
 						}
@@ -585,7 +585,7 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt)
 
 	const SharedPtr<Peer> peer(RR->topology->get(destination));
 	if (peer) {
-		viaPath = peer->path();
+		viaPath = peer->path(now);
 		if (!viaPath) {
 			if (peer->rateGateTryStaticPath(now)) {
 				InetAddress tryAddr;
@@ -599,9 +599,9 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt)
 				}
 			}
 
-			const SharedPtr<Peer> relay(RR->topology->findRelayTo(now,destination));
+			const SharedPtr<Peer> relay(RR->topology->root());
 			if (relay) {
-				viaPath = relay->path();
+				viaPath = relay->path(now);
 				if (!viaPath)
 					return false;
 			}

node/Topology.hpp

@@ -111,7 +111,7 @@ public:
 
 		_paths_l.rlock();
 		SharedPtr<Path> p(_paths[k]);
-		_paths_l.unlock();
+		_paths_l.runlock();
 		if (p)
 			return p;
 
@@ -204,21 +204,6 @@ public:
 		}
 	}
 
-	/**
-	 * Get the best relay to a given address, which may or may not be a root
-	 *
-	 * @param now Current time
-	 * @param toAddr Destination address
-	 * @return Best current relay or NULL if none
-	 */
-	ZT_ALWAYS_INLINE SharedPtr<Peer> findRelayTo(const int64_t now,const Address &toAddr)
-	{
-		RWMutex::RLock l(_peers_l);
-		if (_rootPeers.empty())
-			return SharedPtr<Peer>();
-		return _rootPeers[0];
-	}
-
 	/**
 	 * @param allPeers vector to fill with all current peers
 	 */