The concept of link desperation (escalating to less desirable transports) simplifies a ton of stuff. Loads of spaghetti logic can die since we no longer have to make these decisions down in the core.

include/ZeroTierOne.h

@@ -144,19 +144,17 @@ enum ZT1_ResultCode
 enum ZT1_NodeStatusCode
 {
 	/**
-	 * Node is online
+	 * Node is offline -- nothing is reachable
+	 */
+	ZT1_NODE_STATUS_OFFLINE = 0,
+
+	/**
+	 * Node is online -- at least one upstream is reachable
 	 */
 	ZT1_NODE_STATUS_ONLINE = 1,
 
 	/**
-	 * Node is offline -- nothing is reachable
+	 * Link desperation level has changed
-	 */
-	ZT1_NODE_STATUS_OFFLINE = 2,
-
-	/**
-	 * The desperation level has changed
-	 *
-	 * 'extra' will point to an int containing the new level.
 	 */
 	ZT1_NODE_STATUS_DESPERATION_CHANGE = 3
 };
@@ -407,12 +405,12 @@ typedef struct
 } ZT1_PeerPhysicalPath;
 
 /**
- * What trust hierarchy role does this device have?
+ * What trust hierarchy role does this peer have?
  */
 enum ZT1_PeerRole {
-	ZT1_PEER_ROLE_SUPERNODE = 0, // planetary supernode
+	ZT1_PEER_ROLE_NODE = 0       // ordinary node
 	ZT1_PEER_ROLE_HUB = 1,       // locally federated hub
-	ZT1_PEER_ROLE_NODE = 2       // ordinary node
+	ZT1_PEER_ROLE_SUPERNODE = 2, // planetary supernode
 };
 
 /**

node/Constants.hpp

@@ -287,9 +287,9 @@
 #define ZT_DESPERATION_INCREMENT (ZT_STARTUP_AGGRO * 2)
 
 /**
- * "Spam" packets to lower desperation links every Nth packet
+ * Interval between "spams" if desperation > 0
  */
-#define ZT_DESPERATION_SPAM_EVERY 10
+#define ZT_DESPERATION_SPAM_INTERVAL 60000
 
 /**
  * Maximum delay between runs of the main loop in Node.cpp

node/Node.cpp

@@ -61,10 +61,7 @@ Node::Node(
 	_statusCallback(statusCallback),
 	_networks(),
 	_networks_m(),
-	_now(now),
+	_now(now)
-	_timeOfLastPacketReceived(0),
-	_timeOfLastPrivilegedPacket(0),
-	_spamCounter(0)
 {
 	try {
 		RR->prng = new CMWC4096();

node/Node.hpp

@@ -104,51 +104,25 @@ public:
 	 */
 	inline uint64_t now() const throw() { return _now; }
 
-	/**
-	 * @return Current level of desperation
-	 */
-	inline int desperation() const throw() { return (int)((_now - _timeOfLastPrivilgedPacket) / ZT_DESPERATION_INCREMENT); }
-
-	/**
-	 * Called to update last packet receive time whenever a packet is received
-	 *
-	 * @param fromPrivilegedPeer If true, peer is a supernode or federated hub (a.k.a. an upstream link)
-	 */
-	inline void packetReceived(bool fromPrivilegedPeer)
-		throw()
-	{
-		const uint64_t n = _now;
-		_timeOfLastPacketReceived = n;
-		if (fromPrivilegedPeer)
-			_timeOfLastPrivilgedPacket = n;
-	}
-
-	/**
-	 * @return Most recent time of any packet receipt
-	 */
-	inline uint64_t timeOfLastPacketReceived() const throw() { return _timeOfLastPacketReceived; }
-
-	/**
-	 * @return Timestamp of last packet received from a supernode or hub (upstream link)
-	 */
-	inline uint64_t timeOfLastPrivilgedPacket() const throw() { return _timeOfLastPrivilgedPacket; }
-
 	/**
 	 * Enqueue a ZeroTier message to be sent
 	 *
 	 * @param addr Destination address
 	 * @param data Packet data
 	 * @param len Packet length
+	 * @param desperation Link desperation for reaching this address
+	 * @param spam If true, flag this packet to be spammed to lower-desperation links
+	 * @return True if packet appears to have been sent
 	 */
-	inline void putPacket(const InetAddress &addr,const void *data,unsigned int len)
+	inline bool putPacket(const InetAddress &addr,const void *data,unsigned int len,int desperation,bool spam)
 	{
-		_wirePacketSendFunction(
+		return (_wirePacketSendFunction(
 			reinterpret_cast<ZT1_Node *>(this),
 			reinterpret_cast<const struct sockaddr_storage *>(&addr),
-			this->desperation(),
+			desperation,
-			(int)((++_spamCounter % ZT_DESPERATION_SPAM_EVERY) == 0),
+			(int)spam,
 			data,
-			len);
+			len) == 0);
 	}
 
 	/**
@@ -216,9 +190,6 @@ private:
 	Mutex _networks_m;
 
 	volatile uint64_t _now; // time of last run()
-	volatile uint64_t _timeOfLastPacketReceived;
-	volatile _timeOfLastPrivilgedPacket;
-	volatile unsigned int _spamCounter; // used to "spam" every Nth packet
 };
 
 } // namespace ZeroTier

node/Path.hpp

@@ -33,6 +33,7 @@
 
 #include <stdexcept>
 #include <string>
+#include <algorithm>
 
 #include "Constants.hpp"
 #include "InetAddress.hpp"
@@ -49,42 +50,28 @@ namespace ZeroTier {
 class Path
 {
 public:
-	enum Type
-	{
-		PATH_TYPE_NULL = 0,
-		PATH_TYPE_UDP = 1,
-		PATH_TYPE_TCP_OUT = 2,
-		PATH_TYPE_TCP_IN = 3
-	};
-
 	Path() :
 		_lastSend(0),
 		_lastReceived(0),
-		_lastPing(0),
 		_addr(),
-		_type(PATH_TYPE_NULL),
+		_lastReceiveDesperation(0),
 		_fixed(false) {}
 
-	Path(const Path &p)
+	Path(const Path &p) throw() { memcpy(this,&p,sizeof(Path)); }
-	{
-		memcpy(this,&p,sizeof(Path));
-	}
 
-	Path(const InetAddress &addr,Type t,bool fixed = false) :
+	Path(const InetAddress &addr,bool fixed) :
 		_lastSend(0),
 		_lastReceived(0),
-		_lastPing(0),
 		_addr(addr),
-		_type(t),
+		_lastReceiveDesperation(0),
 		_fixed(fixed) {}
 
-	inline void init(const InetAddress &addr,Type t,bool fixed = false)
+	inline void init(const InetAddress &addr,bool fixed)
 	{
 		_lastSend = 0;
 		_lastReceived = 0;
-		_lastPing = 0;
 		_addr = addr;
-		_type = t;
+		_lastReceiveDesperation = 0;
 		_fixed = fixed;
 	}
 
@@ -97,19 +84,54 @@ public:
 
 	inline const InetAddress &address() const throw() { return _addr; }
 
-	inline Type type() const throw() { return _type; }
-	inline bool tcp() const throw() { return ((_type == PATH_TYPE_TCP_IN)||(_type == PATH_TYPE_TCP_OUT)); }
-
 	inline uint64_t lastSend() const throw() { return _lastSend; }
 	inline uint64_t lastReceived() const throw() { return _lastReceived; }
-	inline uint64_t lastPing() const throw() { return _lastPing; }
+	inline int lastReceiveDesperation() const throw() { return _lastReceiveDesperation; }
 
+	/**
+	 * Called when a packet is sent to this path
+	 *
+	 * @param t Time of send
+	 */
+	inline void sent(uint64_t t) throw() { _lastSend = t; }
+
+	/**
+	 * Called when a packet is received from this path
+	 *
+	 * @param t Time of receive
+	 * @param d Link desperation of receive
+	 */
+	inline void received(uint64_t t,int d) throw() { _lastReceived = t; _lastReceiveDesperation = d; }
+
+	/**
+	 * @return Is this a fixed path?
+	 */
 	inline bool fixed() const throw() { return _fixed; }
+
+	/**
+	 * @param f New value of fixed path flag
+	 */
 	inline void setFixed(bool f) throw() { _fixed = f; }
 
-	inline void sent(uint64_t t) throw() { _lastSend = t; }
+	/**
-	inline void received(uint64_t t) throw() { _lastReceived = t; }
+	 * Compute path desperation
-	inline void pinged(uint64_t t) throw() { _lastPing = t; }
+	 *
+	 * Path desperation affects escalation to less efficient fallback
+	 * transports such as TCP or HTTP relaying.
+	 *
+	 * Right now we only escalate desperation for fixed paths, which
+	 * are paths to supernodes. This causes our fallback tunneling
+	 * mechanisms to kick in.
+	 *
+	 * @param now Current time
+	 * @return Path desperation, starting at 0
+	 */
+	inline int desperation(uint64_t now) const
+	{
+		if ((_lastSend > _lastReceived)&&(_fixed))
+			return std::max(_lastReceiveDesperation,(int)((_lastSend - _lastReceived) / ZT_DESPERATION_INCREMENT));
+		return _lastReceiveDesperation;
+	}
 
 	/**
 	 * @param now Current time
@@ -118,53 +140,37 @@ public:
 	inline bool active(uint64_t now) const
 		throw()
 	{
-		return ((_addr)&&((_fixed)||((now - _lastReceived) < ZT_PEER_PATH_ACTIVITY_TIMEOUT)));
+		return ( (_fixed) || ((now - _lastReceived) < ZT_PEER_PATH_ACTIVITY_TIMEOUT) );
 	}
 
 	/**
-	 * @return Human-readable address and other information about this path, some computed as of current time
+	 * @param now Current time
+	 * @return Human-readable address and other information about this path
 	 */
-	inline std::string toString() const
+	inline std::string toString(uint64_t now) const
 	{
-		uint64_t now = Utils::now();
 		char tmp[1024];
-		const char *t = "";
+		Utils::snprintf(tmp,sizeof(tmp),"%s(%s)",
-		switch(_type) {
-			case PATH_TYPE_NULL: t = "null"; break;
-			case PATH_TYPE_UDP: t = "udp"; break;
-			case PATH_TYPE_TCP_OUT: t = "tcp_out"; break;
-			case PATH_TYPE_TCP_IN: t = "tcp_in"; break;
-		}
-		Utils::snprintf(tmp,sizeof(tmp),"%s;%s;%lld;%lld;%lld;%s",
-			t,
 			_addr.toString().c_str(),
-			(long long)((_lastSend != 0) ? ((now - _lastSend) / 1000LL) : -1),
-			(long long)((_lastReceived != 0) ? ((now - _lastReceived) / 1000LL) : -1),
-			(long long)((_lastPing != 0) ? ((now - _lastPing) / 1000LL) : -1),
 			((_fixed) ? "fixed" : (active(now) ? "active" : "inactive"))
 		);
 		return std::string(tmp);
 	}
 
-	inline bool operator==(const Path &p) const throw() { return ((_addr == p._addr)&&(_type == p._type)); }
+	inline operator bool() const throw() { return (_addr); }
-	inline bool operator!=(const Path &p) const throw() { return ((_addr != p._addr)||(_type != p._type)); }
+
-	inline bool operator<(const Path &p) const
+	inline bool operator==(const Path &p) const throw() { return (_addr == p._addr); }
-		throw()
+	inline bool operator!=(const Path &p) const throw() { return (_addr != p._addr); }
-	{
+	inline bool operator<(const Path &p) const throw() { return (_addr < p._addr); }
-		if (_addr == p._addr)
+	inline bool operator>(const Path &p) const throw() { return (_addr > p._addr); }
-			return ((int)_type < (int)p._type);
+	inline bool operator<=(const Path &p) const throw() { return (_addr <= p._addr); }
-		else return (_addr < p._addr);
+	inline bool operator>=(const Path &p) const throw() { return (_addr >= p._addr); }
-	}
-	inline bool operator>(const Path &p) const throw() { return (p < *this); }
-	inline bool operator<=(const Path &p) const throw() { return !(p < *this); }
-	inline bool operator>=(const Path &p) const throw() { return !(*this < p); }
 
 private:
-	volatile uint64_t _lastSend;
+	uint64_t _lastSend;
-	volatile uint64_t _lastReceived;
+	uint64_t _lastReceived;
-	volatile uint64_t _lastPing;
 	InetAddress _addr;
-	Type _type;
+	int _lastReceiveDesperation;
 	bool _fixed;
 };
 

node/Peer.cpp

@@ -27,10 +27,9 @@
 
 #include "Constants.hpp"
 #include "Peer.hpp"
+#include "Node.hpp"
 #include "Switch.hpp"
-#include "Packet.hpp"
 #include "Network.hpp"
-#include "NodeConfig.hpp"
 #include "AntiRecursion.hpp"
 
 #include <algorithm>
@@ -44,12 +43,13 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 	_lastUnicastFrame(0),
 	_lastMulticastFrame(0),
 	_lastAnnouncedTo(0),
+	_lastSpammed(0),
 	_vMajor(0),
 	_vMinor(0),
 	_vRevision(0),
+	_id(peerIdentity),
 	_numPaths(0),
-	_latency(0),
+	_latency(0)
-	_id(peerIdentity)
 {
 	if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH))
 		throw std::runtime_error("new peer identity key agreement failed");
@@ -57,53 +57,50 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 
 void Peer::received(
 	const RuntimeEnvironment *RR,
-	const SharedPtr<Socket> &fromSock,
 	const InetAddress &remoteAddr,
+	int linkDesperation
 	unsigned int hops,
 	uint64_t packetId,
 	Packet::Verb verb,
 	uint64_t inRePacketId,
-	Packet::Verb inReVerb,
+	Packet::Verb inReVerb)
-	uint64_t now)
 {
-	// Update system-wide last packet receive time
+	const uint64_t now = RR->node->now();
-	*((const_cast<uint64_t *>(&(RR->timeOfLastPacketReceived)))) = now;
-
-	// Global last receive time regardless of path
 	_lastReceive = now;
 
 	if (!hops) {
-		// Learn paths from direct packets (hops == 0)
+		/* Learn new paths from direct (hops == 0) packets */
 		{
+			unsigned int np = _numPaths;
+
 			bool havePath = false;
-			for(unsigned int p=0,np=_numPaths;p<np;++p) {
+			for(unsigned int p=0;p<np;++p) {
-				if ((_paths[p].address() == remoteAddr)&&(_paths[p].tcp() == fromSock->tcp())) {
+				if (_paths[p].address() == remoteAddr) {
-					_paths[p].received(now);
+					_paths[p].received(now,linkDesperation);
 					havePath = true;
 					break;
 				}
 			}
 
 			if (!havePath) {
-				unsigned int np = _numPaths;
+				Path *slot = (Path *)0;
-				if (np >= ZT_PEER_MAX_PATHS)
-					clean(now);
-				np = _numPaths;
 				if (np < ZT_PEER_MAX_PATHS) {
-					Path::Type pt = Path::PATH_TYPE_UDP;
+					// Add new path
-					switch(fromSock->type()) {
+					slot = &(_paths[np++]);
-						case Socket::ZT_SOCKET_TYPE_TCP_IN:
+				} else {
-							pt = Path::PATH_TYPE_TCP_IN;
+					// Replace oldest non-fixed path
-							break;
+					uint64_t slotLRmin = 0xffffffffffffffffULL;
-						case Socket::ZT_SOCKET_TYPE_TCP_OUT:
+					for(unsigned int p=0;p<ZT_PEER_MAX_PATHS;++p) {
-							pt = Path::PATH_TYPE_TCP_OUT;
+						if ((!_paths[p].fixed())&&(_paths[p].lastReceived() <= slotLRmin)) {
-							break;
+							slotLRmin = _paths[p].lastReceived();
-						default:
+							slot = &(_paths[p]);
-							break;
+						}
 					}
-					_paths[np].init(remoteAddr,pt,false);
+				}
-					_paths[np].received(now);
+				if (slot) {
-					_numPaths = ++np;
+					slot->init(remoteAddr,false);
+					slot->received(now,linkDesperation);
+					_numPaths = np;
 				}
 			}
 		}
@@ -126,7 +123,7 @@ void Peer::received(
 					for(std::set<MulticastGroup>::iterator mg(mgs.begin());mg!=mgs.end();++mg) {
 						if ((outp.size() + 18) > ZT_UDP_DEFAULT_PAYLOAD_MTU) {
 							outp.armor(_key,true);
-							fromSock->send(remoteAddr,outp.data(),outp.size());
+							RR->node->putPacket(remoteAddr,outp.data(),outp.size(),linkDesperation,false);
 							outp.reset(_id.address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
 						}
 
@@ -139,156 +136,70 @@ void Peer::received(
 			}
 			if (outp.size() > ZT_PROTO_MIN_PACKET_LENGTH) {
 				outp.armor(_key,true);
-				fromSock->send(remoteAddr,outp.data(),outp.size());
+				RR->node->putPacket(remoteAddr,outp.data(),outp.size(),linkDesperation,false);
 			}
 		}
 	}
 
 	if ((verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME))
 		_lastUnicastFrame = now;
-	else if ((verb == Packet::VERB_P5_MULTICAST_FRAME)||(verb == Packet::VERB_MULTICAST_FRAME))
+	else if (verb == Packet::VERB_MULTICAST_FRAME)
 		_lastMulticastFrame = now;
 }
 
-Path::Type Peer::send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
+bool Peer::send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
 {
-	/* For sending ordinary packets, paths are divided into two categories:
-	 * "normal" and "TCP out." Normal includes UDP and incoming TCP. We want
-	 * to treat outbound TCP differently since if we use it it may end up
-	 * overriding UDP and UDP performs much better. We only want to initiate
-	 * TCP if it looks like UDP isn't available. */
-	Path *bestNormalPath = (Path *)0;
-	Path *bestTcpOutPath = (Path *)0;
-	uint64_t bestNormalPathLastReceived = 0;
-	uint64_t bestTcpOutPathLastReceived = 0;
-	for(unsigned int p=0,np=_numPaths;p<np;++p) {
-		uint64_t lr = _paths[p].lastReceived();
-		if (_paths[p].type() == Path::PATH_TYPE_TCP_OUT) {
-			if (lr >= bestTcpOutPathLastReceived) {
-				bestTcpOutPathLastReceived = lr;
-				bestTcpOutPath = &(_paths[p]);
-			}
-		} else {
-			if (lr >= bestNormalPathLastReceived) {
-				bestNormalPathLastReceived = lr;
-				bestNormalPath = &(_paths[p]);
-			}
-		}
-	}
-
 	Path *bestPath = (Path *)0;
-	uint64_t normalPathAge = now - bestNormalPathLastReceived;
+	uint64_t lrMax = 0;
-	uint64_t tcpOutPathAge = now - bestTcpOutPathLastReceived;
-	if (normalPathAge < ZT_PEER_PATH_ACTIVITY_TIMEOUT) {
-		/* If we have a normal path that looks alive, only use TCP if it looks
-		 * even more alive, if the UDP path is not a very recent acquisition,
-		 * and if TCP tunneling is globally enabled. */
-		bestPath = ( (tcpOutPathAge < normalPathAge) && (normalPathAge > (ZT_PEER_DIRECT_PING_DELAY / 4)) && (RR->tcpTunnelingEnabled) ) ? bestTcpOutPath : bestNormalPath;
-	} else if ( (tcpOutPathAge < ZT_PEER_PATH_ACTIVITY_TIMEOUT) || ((RR->tcpTunnelingEnabled)&&(bestTcpOutPath)) ) {
-		/* Otherwise use a TCP path if we have an active one or if TCP
-		 * fallback has been globally triggered and we know of one at all. */
-		bestPath = bestTcpOutPath;
-	} else if ( (bestNormalPath) && (bestNormalPath->fixed()) ) {
-		/* Finally, use a normal path if we have a "fixed" one as these are
-		 * always considered basically alive. */
-		bestPath = bestNormalPath;
-	}
-
-	/* Old path choice logic -- would attempt to use inactive paths... deprecating and will probably kill.
-	Path *bestPath = (Path *)0;
-	if (bestTcpOutPath) { // we have a TCP out path
-		if (bestNormalPath) { // we have both paths, decide which to use
-			if (RR->tcpTunnelingEnabled) { // TCP tunneling is enabled, so use normal path only if it looks alive
-				if ((bestNormalPathLastReceived > RR->timeOfLastResynchronize)&&((now - bestNormalPathLastReceived) < ZT_PEER_PATH_ACTIVITY_TIMEOUT))
-					bestPath = bestNormalPath;
-				else bestPath = bestTcpOutPath;
-			} else { // TCP tunneling is disabled, use normal path
-				bestPath = bestNormalPath;
-			}
-		} else { // we only have a TCP_OUT path, so use it regardless
-			bestPath = bestTcpOutPath;
-		}
-	} else { // we only have a normal path (or none at all, that case is caught below)
-		bestPath = bestNormalPath;
-	}
-	*/
-
-	if (!bestPath)
-		return Path::PATH_TYPE_NULL;
-
-	RR->antiRec->logOutgoingZT(data,len);
-
-	if (RR->sm->send(bestPath->address(),bestPath->tcp(),bestPath->type() == Path::PATH_TYPE_TCP_OUT,data,len)) {
-		bestPath->sent(now);
-		return bestPath->type();
-	}
-
-	return Path::PATH_TYPE_NULL;
-}
-
-bool Peer::sendPing(const RuntimeEnvironment *RR,uint64_t now)
-{
-	bool sent = false;
-	SharedPtr<Peer> self(this);
-
-	/* Ping (and thus open) outbound TCP connections if we have no other options
-	 * or if the TCP tunneling master switch is enabled and pings have been
-	 * unanswered for ZT_TCP_TUNNEL_FAILOVER_TIMEOUT ms over normal channels. */
-	uint64_t lastNormalPingSent = 0;
-	uint64_t lastNormalReceive = 0;
-	bool haveNormal = false;
 	for(unsigned int p=0,np=_numPaths;p<np;++p) {
-		if (_paths[p].type() != Path::PATH_TYPE_TCP_OUT) {
+		if ((_paths[p].active(now)&&(_paths[p].lastReceived() >= lrMax)) {
-			lastNormalPingSent = std::max(lastNormalPingSent,_paths[p].lastPing());
+			lrMax = _paths[p].lastReceived();
-			lastNormalReceive = std::max(lastNormalReceive,_paths[p].lastReceived());
+			bestPath = &(_paths[p]);
-			haveNormal = true;
 		}
 	}
 
-	const bool useTcpOut = ( (!haveNormal) || ( (RR->tcpTunnelingEnabled) && (lastNormalPingSent > RR->timeOfLastResynchronize) && (lastNormalPingSent > lastNormalReceive) && ((lastNormalPingSent - lastNormalReceive) >= ZT_TCP_TUNNEL_FAILOVER_TIMEOUT) ) );
+	if (bestPath) {
-	TRACE("PING %s (useTcpOut==%d)",_id.address().toString().c_str(),(int)useTcpOut);
+		bool spam = ((now - _lastSpammed) >= ZT_DESPERATION_SPAM_INTERVAL);
-
+		if (RR->node->putPacket(bestPath->address(),data,len,bestPath->desperation(),spam)) {
-	for(unsigned int p=0,np=_numPaths;p<np;++p) {
+			bestPath->sent(now);
-		if ((useTcpOut)||(_paths[p].type() != Path::PATH_TYPE_TCP_OUT)) {
+			RR->antiRec->logOutgoingZT(data,len);
-			_paths[p].pinged(now); // attempts to ping are logged whether they look successful or not
+			if (spam)
-			if (RR->sw->sendHELLO(self,_paths[p])) {
+				_lastSpammed = now;
-				_paths[p].sent(now);
+			return true;
-				sent = true;
-			}
 		}
 	}
 
-	return sent;
+	return false;
-}
-
-void Peer::clean(uint64_t now)
-{
-	unsigned int np = _numPaths;
-	unsigned int x = 0;
-	unsigned int y = 0;
-	while (x < np) {
-		if (_paths[x].active(now))
-			_paths[y++] = _paths[x];
-		++x;
-	}
-	_numPaths = y;
 }
 
 void Peer::addPath(const Path &newp)
 {
 	unsigned int np = _numPaths;
+
 	for(unsigned int p=0;p<np;++p) {
-		if (_paths[p] == newp) {
+		if (_paths[p].address() == newp.address()) {
 			_paths[p].setFixed(newp.fixed());
 			return;
 		}
 	}
-	if (np >= ZT_PEER_MAX_PATHS)
+
-		clean(Utils::now());
+	Path *slot = (Path *)0;
-	np = _numPaths;
 	if (np < ZT_PEER_MAX_PATHS) {
-		_paths[np] = newp;
+		// Add new path
-		_numPaths = ++np;
+		slot = &(_paths[np++]);
+	} else {
+		// Replace oldest non-fixed path
+		uint64_t slotLRmin = 0xffffffffffffffffULL;
+		for(unsigned int p=0;p<ZT_PEER_MAX_PATHS;++p) {
+			if ((!_paths[p].fixed())&&(_paths[p].lastReceived() <= slotLRmin)) {
+				slotLRmin = _paths[p].lastReceived();
+				slot = &(_paths[p]);
+			}
+		}
+	}
+	if (slot) {
+		*slot = newp;
+		_numPaths = np;
 	}
 }
 
@@ -309,11 +220,11 @@ void Peer::clearPaths(bool fixedToo)
 	}
 }
 
-void Peer::getBestActiveUdpPathAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const
+void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const
 {
 	uint64_t bestV4 = 0,bestV6 = 0;
 	for(unsigned int p=0,np=_numPaths;p<np;++p) {
-		if ((_paths[p].type() == Path::PATH_TYPE_UDP)&&(_paths[p].active(now))) {
+		if (_paths[p].active(now)) {
 			uint64_t lr = _paths[p].lastReceived();
 			if (lr) {
 				if (_paths[p].address().isV4()) {

node/Peer.hpp

@@ -35,6 +35,8 @@
 #include <vector>
 #include <stdexcept>
 
+#include "../include/ZeroTierOne.h"
+
 #include "Constants.hpp"
 #include "RuntimeEnvironment.hpp"
 #include "Path.hpp"
@@ -51,7 +53,7 @@
 /**
  * Maximum number of paths a peer can have
  */
-#define ZT_PEER_MAX_PATHS 8
+#define ZT_PEER_MAX_PATHS 3
 
 namespace ZeroTier {
 
@@ -111,25 +113,23 @@ public:
 	 * and appears to be valid.
 	 * 
 	 * @param RR Runtime environment
-	 * @param fromSock Socket from which packet was received
 	 * @param remoteAddr Internet address of sender
+	 * @param linkDesperation Link desperation level
 	 * @param hops ZeroTier (not IP) hops
 	 * @param packetId Packet ID
 	 * @param verb Packet verb
-	 * @param inRePacketId Packet ID in reply to (for OK/ERROR, 0 otherwise)
+	 * @param inRePacketId Packet ID in reply to (default: none)
-	 * @param inReVerb Verb in reply to (for OK/ERROR, VERB_NOP otherwise)
+	 * @param inReVerb Verb in reply to (for OK/ERROR, default: VERB_NOP)
-	 * @param now Current time
 	 */
 	void received(
 		const RuntimeEnvironment *RR,
-		const SharedPtr<Socket> &fromSock,
 		const InetAddress &remoteAddr,
+		int linkDesperation,
 		unsigned int hops,
 		uint64_t packetId,
 		Packet::Verb verb,
-		uint64_t inRePacketId,
+		uint64_t inRePacketId = 0,
-		Packet::Verb inReVerb,
+		Packet::Verb inReVerb = Packet::VERB_NOP);
-		uint64_t now);
 
 	/**
 	 * Send a packet directly to this peer
@@ -141,26 +141,9 @@ public:
 	 * @param data Data to send
 	 * @param len Length of packet
 	 * @param now Current time
-	 * @return Type of path used or Path::PATH_TYPE_NULL on failure
+	 * @return True if packet appears to have been sent via some available path
 	 */
-	Path::Type send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now);
+	bool send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now);
-
-	/**
-	 * Send HELLO to a peer via all direct paths available
-	 *
-	 * This begins attempting to use TCP paths if no ping response has been
-	 * received from any UDP path in more than ZT_TCP_FALLBACK_AFTER.
-	 * 
-	 * @param RR Runtime environment
-	 * @param now Current time
-	 * @return True if send appears successful for at least one address type
-	 */
-	bool sendPing(const RuntimeEnvironment *RR,uint64_t now);
-
-	/**
-	 * Called periodically by Topology::clean() to remove stale paths and do other cleanup
-	 */
-	void clean(uint64_t now);
 
 	/**
 	 * @return All known direct paths to this peer
@@ -173,19 +156,6 @@ public:
 		return pp;
 	}
 
-	/**
-	 * @param addr IP:port
-	 * @return True if we have a UDP path to this address
-	 */
-	inline bool haveUdpPath(const InetAddress &addr) const
-	{
-		for(unsigned int p=0,np=_numPaths;p<np;++p) {
-			if ((_paths[p].type() == Path::PATH_TYPE_UDP)&&(_paths[p].address() == addr))
-				return true;
-		}
-		return false;
-	}
-
 	/**
 	 * @return Time of last direct packet receive for any path
 	 */
@@ -210,21 +180,6 @@ public:
 		return x;
 	}
 
-	/**
-	 * Get max timestamp of last ping and max timestamp of last receive in a single pass
-	 *
-	 * @param lp Last ping result parameter (init to 0 before calling)
-	 * @param lr Last receive result parameter (init to 0 before calling)
-	 */
-	inline void lastPingAndDirectReceive(uint64_t &lp,uint64_t &lr)
-		throw()
-	{
-		for(unsigned int p=0,np=_numPaths;p<np;++p) {
-			lp = std::max(lp,_paths[p].lastPing());
-			lr = std::max(lr,_paths[p].lastReceived());
-		}
-	}
-
 	/**
 	 * @return Time of last receive of anything, whether direct or relayed
 	 */
@@ -378,7 +333,7 @@ public:
 	 * @param v4 Result parameter to receive active IPv4 address, if any
 	 * @param v6 Result parameter to receive active IPv6 address, if any
 	 */
-	void getBestActiveUdpPathAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const;
+	void getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const;
 
 	/**
 	 * Find a common set of addresses by which two peers can link, if any
@@ -391,8 +346,8 @@ public:
 	static inline std::pair<InetAddress,InetAddress> findCommonGround(const Peer &a,const Peer &b,uint64_t now)
 	{
 		std::pair<InetAddress,InetAddress> v4,v6;
-		b.getBestActiveUdpPathAddresses(now,v4.first,v6.first);
+		b.getBestActiveAddresses(now,v4.first,v6.first);
-		a.getBestActiveUdpPathAddresses(now,v4.second,v6.second);
+		a.getBestActiveAddresses(now,v4.second,v6.second);
 		if ((v6.first)&&(v6.second)) // prefer IPv6 if both have it since NAT-t is (almost) unnecessary
 			return v6;
 		else if ((v4.first)&&(v4.second))
@@ -403,23 +358,26 @@ public:
 private:
 	void _announceMulticastGroups(const RuntimeEnvironment *RR,uint64_t now);
 
+	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
+
 	uint64_t _lastUsed;
 	uint64_t _lastReceive; // direct or indirect
 	uint64_t _lastUnicastFrame;
 	uint64_t _lastMulticastFrame;
 	uint64_t _lastAnnouncedTo;
+	uint64_t _lastSpammed;
 	uint16_t _vProto;
 	uint16_t _vMajor;
 	uint16_t _vMinor;
 	uint16_t _vRevision;
 
+	Identity _id;
+
 	Path _paths[ZT_PEER_MAX_PATHS];
 	unsigned int _numPaths;
 
 	unsigned int _latency;
-	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
+	unsigned int _spamCounter;
-	Identity _id;
-
 	AtomicCounter __refCount;
 };
 

node/Switch.cpp

@@ -270,6 +270,7 @@ void Switch::send(const Packet &packet,bool encrypt)
 	}
 }
 
+#if 0
 void Switch::sendHELLO(const Address &dest)
 {
 	Packet outp(dest,RR->identity.address(),Packet::VERB_HELLO);
@@ -281,36 +282,7 @@ void Switch::sendHELLO(const Address &dest)
 	RR->identity.serialize(outp,false);
 	send(outp,false);
 }
-
+#endif
-bool Switch::sendHELLO(const SharedPtr<Peer> &dest,const Path &path)
-{
-	uint64_t now = Utils::now();
-	Packet outp(dest->address(),RR->identity.address(),Packet::VERB_HELLO);
-	outp.append((unsigned char)ZT_PROTO_VERSION);
-	outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR);
-	outp.append((unsigned char)ZEROTIER_ONE_VERSION_MINOR);
-	outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
-	outp.append(now);
-	RR->identity.serialize(outp,false);
-	outp.armor(dest->key(),false);
-	RR->antiRec->logOutgoingZT(outp.data(),outp.size());
-	return RR->sm->send(path.address(),path.tcp(),path.type() == Path::PATH_TYPE_TCP_OUT,outp.data(),outp.size());
-}
-
-bool Switch::sendHELLO(const SharedPtr<Peer> &dest,const InetAddress &destUdp)
-{
-	uint64_t now = Utils::now();
-	Packet outp(dest->address(),RR->identity.address(),Packet::VERB_HELLO);
-	outp.append((unsigned char)ZT_PROTO_VERSION);
-	outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR);
-	outp.append((unsigned char)ZEROTIER_ONE_VERSION_MINOR);
-	outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
-	outp.append(now);
-	RR->identity.serialize(outp,false);
-	outp.armor(dest->key(),false);
-	RR->antiRec->logOutgoingZT(outp.data(),outp.size());
-	return RR->sm->send(destUdp,false,false,outp.data(),outp.size());
-}
 
 bool Switch::unite(const Address &p1,const Address &p2,bool force)
 {

node/Switch.hpp

@@ -116,31 +116,6 @@ public:
 	 */
 	void send(const Packet &packet,bool encrypt);
 
-	/**
-	 * Send a HELLO announcement
-	 *
-	 * @param dest Address of destination
-	 */
-	void sendHELLO(const Address &dest);
-
-	/**
-	 * Send a HELLO announcement immediately to the indicated address
-	 *
-	 * @param dest Destination peer
-	 * @param path Network path to peer
-	 * @return True if send appears successful
-	 */
-	bool sendHELLO(const SharedPtr<Peer> &dest,const Path &path);
-
-	/**
-	 * Send a HELLO announcement immediately to the indicated address via UDP
-	 *
-	 * @param dest Destination peer
-	 * @param destUdp UDP inet address
-	 * @return True if send appears successful
-	 */
-	bool sendHELLO(const SharedPtr<Peer> &dest,const InetAddress &destUdp);
-
 	/**
 	 * Send RENDEZVOUS to two peers to permit them to directly connect
 	 *

node/Topology.cpp

@@ -251,10 +251,7 @@ void Topology::clean(uint64_t now)
 	for(std::map< Address,SharedPtr<Peer> >::iterator p(_activePeers.begin());p!=_activePeers.end();) {
 		if (((now - p->second->lastUsed()) >= ZT_PEER_IN_MEMORY_EXPIRATION)&&(std::find(_supernodeAddresses.begin(),_supernodeAddresses.end(),p->first) == _supernodeAddresses.end())) {
 			_activePeers.erase(p++);
-		} else {
+		} else ++p;
-			p->second->clean(now);
-			++p;
-		}
 	}
 }
 

node/Topology.hpp

@@ -191,6 +191,7 @@ public:
 			f(*this,p->second);
 	}
 
+#if 0
 	/**
 	 * Apply a function or function object to all supernode peers
 	 *
@@ -274,7 +275,7 @@ public:
 			uint64_t lp = 0;
 			uint64_t lr = 0;
 			p->lastPingAndDirectReceive(lp,lr);
-			if ( (lr < RR->timeOfLastResynchronize) || ((lr < lp)&&((lp - lr) >= ZT_PING_UNANSWERED_AFTER)) || ((_now - lr) >= ZT_PEER_DIRECT_PING_DELAY) )
+			if ( ((lr < lp)&&((lp - lr) >= ZT_PING_UNANSWERED_AFTER)) || ((_now - lr) >= ZT_PEER_DIRECT_PING_DELAY) )
 				p->sendPing(RR,_now);
 		}
 
@@ -283,22 +284,9 @@ public:
 		const RuntimeEnvironment *RR;
 	};
 
-	/**
-	 * Computes most recent timestamp of direct packet receive over a list of peers
-	 */
-	class FindMostRecentDirectReceiveTimestamp
-	{
-	public:
-		FindMostRecentDirectReceiveTimestamp(uint64_t &ts) throw() : _ts(ts) {}
-		inline void operator()(Topology &t,const SharedPtr<Peer> &p) throw() { _ts = std::max(p->lastDirectReceive(),_ts); }
-	private:
-		uint64_t &_ts;
-	};
-
 	/**
 	 * Function object to forget direct links to active peers and then ping them indirectly
 	 */
-	/*
 	class ResetActivePeers
 	{
 	public:
@@ -333,28 +321,7 @@ public:
 		std::vector<Address> _supernodeAddresses;
 		const RuntimeEnvironment *RR;
 	};
-	*/
+#endif
-
-	/**
-	 * Function object to collect peers with any known direct path
-	 */
-	class CollectPeersWithActiveDirectPath
-	{
-	public:
-		CollectPeersWithActiveDirectPath(std::vector< SharedPtr<Peer> > &v,uint64_t now) throw() :
-			_now(now),
-			_v(v) {}
-
-		inline void operator()(Topology &t,const SharedPtr<Peer> &p)
-		{
-			if (p->hasActiveDirectPath(_now))
-				_v.push_back(p);
-		}
-
-	private:
-		uint64_t _now;
-		std::vector< SharedPtr<Peer> > &_v;
-	};
 
 	/**
 	 * Update our knowledge of exterior network addresses
@@ -396,7 +363,7 @@ private:
 	Mutex _lock;
 
 	// Set to true if my identity is in _supernodes
-	volatile bool _amSupernode;
+	bool _amSupernode;
 };
 
 } // namespace ZeroTier