Peer serialization and related changes.

include/ZeroTierOne.h

@@ -627,8 +627,8 @@ typedef struct
  */
 typedef enum {
 	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL = 0,
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_PRIVACY = 1,
+	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_PRIVACY = 10,
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_ULTIMATE = 2
+	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_ULTIMATE = 20
 } ZT_LocalInterfaceAddressTrust;
 
 /**

node/CertificateOfMembership.hpp

@@ -315,78 +315,6 @@ public:
 	 */
 	inline const Address &signedBy() const throw() { return _signedBy; }
 
-	/**
-	 * Serialize to std::string or compatible class
-	 *
-	 * @param b String or other class supporting push_back() and append() like std::string
-	 */
-	template<typename T>
-	inline void serialize2(T &b) const
-	{
-		uint64_t tmp[3];
-		char tmp2[ZT_ADDRESS_LENGTH];
-		b.push_back((char)COM_UINT64_ED25519);
-		b.push_back((char)((_qualifiers.size() >> 8) & 0xff));
-		b.push_back((char)(_qualifiers.size() & 0xff));
-		for(std::vector<_Qualifier>::const_iterator q(_qualifiers.begin());q!=_qualifiers.end();++q) {
-			tmp[0] = Utils::hton(q->id);
-			tmp[1] = Utils::hton(q->value);
-			tmp[2] = Utils::hton(q->maxDelta);
-			b.append(reinterpret_cast<const char *>(reinterpret_cast<void *>(tmp)),sizeof(tmp));
-		}
-		_signedBy.copyTo(tmp2,ZT_ADDRESS_LENGTH);
-		b.append(tmp2,ZT_ADDRESS_LENGTH);
-		if (_signedBy)
-			b.append((const char *)_signature.data,_signature.size());
-	}
-
-	/**
-	 * Deserialize from std::string::iterator or compatible iterator or char* pointer
-	 *
-	 * @param p Iterator
-	 * @param end End of buffer
-	 */
-	template<typename T>
-	inline void deserialize2(T &p,const T &end)
-	{
-		uint64_t tmp[3];
-		char tmp2[ZT_ADDRESS_LENGTH];
-		unsigned int qcount;
-
-		_qualifiers.clear();
-		_signedBy.zero();
-
-		if (p == end) throw std::out_of_range("incomplete certificate of membership");
-		if (*(p++) != (char)COM_UINT64_ED25519) throw std::invalid_argument("unknown certificate of membership type");
-
-		if (p == end) throw std::out_of_range("incomplete certificate of membership");
-		qcount = (unsigned int)*(p++) << 8;
-		if (p == end) throw std::out_of_range("incomplete certificate of membership");
-		qcount |= (unsigned int)*(p++);
-
-		for(unsigned int i=0;i<qcount;++i) {
-			char *p2 = reinterpret_cast<char *>(reinterpret_cast<void *>(tmp));
-			for(unsigned int j=0;j<sizeof(tmp);++j) {
-				if (p == end) throw std::out_of_range("incomplete certificate of membership");
-				*(p2++) = *(p++);
-			}
-			_qualifiers.push_back(_Qualifier(Utils::ntoh(tmp[0]),Utils::ntoh(tmp[1]),Utils::ntoh(tmp[2])));
-		}
-
-		for(unsigned int j=0;j<ZT_ADDRESS_LENGTH;++j) {
-			if (p == end) throw std::out_of_range("incomplete certificate of membership");
-			tmp2[j] = *(p++);
-		}
-		_signedBy.setTo(tmp2,ZT_ADDRESS_LENGTH);
-
-		if (_signedBy) {
-			for(unsigned int j=0;j<_signature.size();++j) {
-				if (p == end) throw std::out_of_range("incomplete certificate of membership");
-				_signature.data[j] = (unsigned char)*(p++);
-			}
-		}
-	}
-
 	template<unsigned int C>
 	inline void serialize(Buffer<C> &b) const
 	{

node/Identity.hpp

@@ -220,7 +220,6 @@ public:
 	 */
 	template<unsigned int C>
 	inline void serialize(Buffer<C> &b,bool includePrivate = false) const
-		throw(std::out_of_range)
 	{
 		_address.appendTo(b);
 		b.append((unsigned char)IDENTITY_TYPE_C25519);
@@ -245,7 +244,6 @@ public:
 	 */
 	template<unsigned int C>
 	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
-		throw(std::out_of_range,std::invalid_argument)
 	{
 		delete _privateKey;
 		_privateKey = (C25519::Private *)0;

node/InetAddress.hpp

@@ -38,6 +38,7 @@
 #include "../include/ZeroTierOne.h"
 #include "Utils.hpp"
 #include "MAC.hpp"
+#include "Buffer.hpp"
 
 namespace ZeroTier {
 
@@ -362,6 +363,51 @@ struct InetAddress : public sockaddr_storage
 	 */
 	inline operator bool() const throw() { return (ss_family != 0); }
 
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b) const
+	{
+		// Format is the same as in VERB_HELLO in Packet.hpp
+		switch(ss_family) {
+			case AF_INET:
+				b.append((uint8_t)0x04);
+				b.append(&(reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr),4);
+				b.append((uint16_t)port()); // just in case sin_port != uint16_t
+				return;
+			case AF_INET6:
+				b.append((uint8_t)0x06);
+				b.append(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,16);
+				b.append((uint16_t)port()); // just in case sin_port != uint16_t
+				return;
+			default:
+				b.append((uint8_t)0);
+				return;
+		}
+	}
+
+	template<unsigned int C>
+	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
+	{
+		unsigned int p = startAt;
+		memset(this,0,sizeof(InetAddress));
+		switch(b[p++]) {
+			case 0:
+				return 1;
+			case 0x04:
+				ss_family = AF_INET;
+				memcpy(&(reinterpret_cast<struct sockaddr_in *>(this)->sin_addr.s_addr),b.field(p,4),4); p += 4;
+				reinterpret_cast<struct sockaddr_in *>(this)->sin_port = Utils::hton(b.template at<uint16_t>(p)); p += 2;
+				break;
+			case 0x06:
+				ss_family = AF_INET6;
+				memcpy(reinterpret_cast<struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,b.field(p,16),16); p += 16;
+				reinterpret_cast<struct sockaddr_in *>(this)->sin_port = Utils::hton(b.template at<uint16_t>(p)); p += 2;
+				break;
+			default:
+				throw std::invalid_argument("invalid serialized InetAddress");
+		}
+		return (p - startAt);
+	}
+
 	bool operator==(const InetAddress &a) const throw();
 	bool operator<(const InetAddress &a) const throw();
 	inline bool operator!=(const InetAddress &a) const throw() { return !(*this == a); }

node/Path.hpp

@@ -59,11 +59,11 @@ public:
 	 *
 	 * These values MUST match ZT_LocalInterfaceAddressTrust in ZeroTierOne.h
 	 */
-	enum Trust
+	enum Trust // NOTE: max 255
 	{
 		TRUST_NORMAL = 0,
-		TRUST_PRIVACY = 1,
+		TRUST_PRIVACY = 10,
-		TRUST_ULTIMATE = 2
+		TRUST_ULTIMATE = 20
 	};
 
 	Path() :
@@ -155,7 +155,7 @@ public:
 		return false;
 	}
 
-private:
+protected:
 	InetAddress _addr;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
 	Trust _trust;

node/Peer.hpp

@@ -445,6 +445,127 @@ public:
 		else return std::pair<InetAddress,InetAddress>();
 	}
 
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b) const
+	{
+		Mutex::Lock _l(_lock);
+
+		const unsigned int lengthAt = b.size();
+		b.addSize(4); // space for uint32_t field length
+
+		b.append((uint32_t)1); // version of serialized Peer data
+
+		_id.serialize(b,false);
+
+		b.append((uint64_t)_lastUsed);
+		b.append((uint64_t)_lastReceive);
+		b.append((uint64_t)_lastUnicastFrame);
+		b.append((uint64_t)_lastMulticastFrame);
+		b.append((uint64_t)_lastAnnouncedTo);
+		b.append((uint64_t)_lastPathConfirmationSent);
+		b.append((uint64_t)_lastDirectPathPush);
+		b.append((uint64_t)_lastPathSort);
+		b.append((uint16_t)_vProto);
+		b.append((uint16_t)_vMajor);
+		b.append((uint16_t)_vMinor);
+		b.append((uint16_t)_vRevision);
+		b.append((uint32_t)_latency);
+
+		b.append((uint32_t)_numPaths);
+		for(unsigned int i=0;i<_numPaths;++i)
+			_paths[i].serialize(b);
+
+		b.append((uint32_t)_networkComs.size());
+		{
+			uint64_t *k = (uint64_t *)0;
+			_NetworkCom *v = (_NetworkCom *)0;
+			Hashtable<uint64_t,_NetworkCom>::Iterator i(const_cast<Peer *>(this)->_networkComs);
+			while (i.next(k,v)) {
+				b.append((uint64_t)*k);
+				b.append((uint64_t)v->ts);
+				v->com.serialize(b);
+			}
+		}
+
+		b.append((uint32_t)_lastPushedComs.size());
+		{
+			uint64_t *k = (uint64_t *)0;
+			uint64_t *v = (uint64_t *)0;
+			Hashtable<uint64_t,uint64_t>::Iterator i(const_cast<Peer *>(this)->_lastPushedComs);
+			while (i.next(k,v)) {
+				b.append((uint64_t)*k);
+				b.append((uint64_t)*v);
+			}
+		}
+
+		b.setAt(lengthAt,(uint32_t)((b.size() - 4) - lengthAt)); // set size, not including size field itself
+	}
+
+	/**
+	 * Create a new Peer from a serialized instance
+	 *
+	 * @param myIdentity This node's identity
+	 * @param b Buffer containing serialized Peer data
+	 * @param p Pointer to current position in buffer, will be updated in place as buffer is read (value/result)
+	 * @return New instance of Peer or NULL if serialized data was corrupt or otherwise invalid (may also throw an exception via Buffer)
+	 */
+	template<unsigned int C>
+	static inline SharedPtr<Peer> deserializeNew(const Identity &myIdentity,const Buffer<C> &b,unsigned int &p)
+	{
+		const uint32_t recSize = b.template at<uint32_t>(p); p += 4;
+		if ((p + recSize) > b.size())
+			return SharedPtr<Peer>(); // size invalid
+		if (b.template at<uint32_t>(p) != 1)
+			return SharedPtr<Peer>(); // version mismatch
+		p += 4;
+
+		Identity npid;
+		p += npid.deserialize(b,p);
+		if (!npid)
+			return SharedPtr<Peer>();
+
+		SharedPtr<Peer> np(new Peer(myIdentity,npid));
+
+		np->_lastUsed = b.template at<uint64_t>(p); p += 8;
+		np->_lastReceive = b.template at<uint64_t>(p); p += 8;
+		np->_lastUnicastFrame = b.template at<uint64_t>(p); p += 8;
+		np->_lastMulticastFrame = b.template at<uint64_t>(p); p += 8;
+		np->_lastAnnouncedTo = b.template at<uint64_t>(p); p += 8;
+		np->_lastPathConfirmationSent = b.template at<uint64_t>(p); p += 8;
+		np->_lastDirectPathPush = b.template at<uint64_t>(p); p += 8;
+		np->_lastPathSort = b.template at<uint64_t>(p); p += 8;
+		np->_vProto = b.template at<uint16_t>(p); p += 2;
+		np->_vMajor = b.template at<uint16_t>(p); p += 2;
+		np->_vMinor = b.template at<uint16_t>(p); p += 2;
+		np->_vRevision = b.template at<uint16_t>(p); p += 2;
+		np->_latency = b.template at<uint32_t>(p); p += 4;
+
+		const unsigned int numPaths = b.template at<uint32_t>(p); p += 2;
+		for(unsigned int i=0;i<numPaths;++i) {
+			if (i < ZT_MAX_PEER_NETWORK_PATHS) {
+				p += np->_paths[np->_numPaths++].deserialize(b,p);
+			} else {
+				// Skip any paths beyond max, but still read stream
+				RemotePath foo;
+				p += foo.deserialize(b,p);
+			}
+		}
+
+		const unsigned int numNetworkComs = b.template at<uint32_t>(p); p += 4;
+		for(unsigned int i=0;i<numNetworkComs;++i) {
+			_NetworkCom &c = np->_networkComs[b.template at<uint64_t>(p)]; p += 8;
+			c.ts = b.template at<uint64_t>(p); p += 8;
+			p += c.com.deserialize(b,p);
+		}
+
+		const unsigned int numLastPushed = b.template at<uint32_t>(p); p += 4;
+		for(unsigned int i=0;i<numLastPushed;++i) {
+			const uint64_t nwid = b.template at<uint64_t>(p); p += 8;
+			const uint64_t ts = b.template at<uint64_t>(p); p += 8;
+			np->_lastPushedComs.set(nwid,ts);
+		}
+	}
+
 private:
 	void _sortPaths(const uint64_t now);
 	RemotePath *_getBestPath(const uint64_t now);

node/RemotePath.hpp

@@ -39,6 +39,8 @@
 #include "AntiRecursion.hpp"
 #include "RuntimeEnvironment.hpp"
 
+#define ZT_REMOTEPATH_FLAG_FIXED 0x0001
+
 namespace ZeroTier {
 
 /**
@@ -54,14 +56,14 @@ public:
 		_lastSend(0),
 		_lastReceived(0),
 		_localAddress(),
-		_fixed(false) {}
+		_flags(0) {}
 
 	RemotePath(const InetAddress &localAddress,const InetAddress &addr,bool fixed) :
 		Path(addr,0,TRUST_NORMAL),
 		_lastSend(0),
 		_lastReceived(0),
 		_localAddress(localAddress),
-		_fixed(fixed) {}
+		_flags(fixed ? ZT_REMOTEPATH_FLAG_FIXED : 0) {}
 
 	inline const InetAddress &localAddress() const throw() { return _localAddress; }
 
@@ -71,7 +73,7 @@ public:
 	/**
 	 * @return Is this a fixed path?
 	 */
-	inline bool fixed() const throw() { return _fixed; }
+	inline bool fixed() const throw() { return ((_flags & ZT_REMOTEPATH_FLAG_FIXED) != 0); }
 
 	/**
 	 * @param f New value of fixed flag
@@ -79,7 +81,9 @@ public:
 	inline void setFixed(const bool f)
 		throw()
 	{
-		_fixed = f;
+		if (f)
+			_flags |= ZT_REMOTEPATH_FLAG_FIXED;
+		else _flags &= ~ZT_REMOTEPATH_FLAG_FIXED;
 	}
 
 	/**
@@ -113,7 +117,7 @@ public:
 	inline bool active(uint64_t now) const
 		throw()
 	{
-		return ( (_fixed) || ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT) );
+		return ( ((_flags & ZT_REMOTEPATH_FLAG_FIXED) != 0) || ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT) );
 	}
 
 	/**
@@ -135,11 +139,39 @@ public:
 		return false;
 	}
 
-private:
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b) const
+	{
+		b.append((uint8_t)1); // version
+		_addr.serialize(b);
+		b.append((uint8_t)_trust);
+		b.append((uint64_t)_lastSend);
+		b.append((uint64_t)_lastReceived);
+		_localAddress.serialize(b);
+		b.append((uint16_t)_flags);
+	}
+
+	template<unsigned int C>
+	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
+	{
+		unsigned int p = startAt;
+		if (b[p++] != 1)
+			throw std::invalid_argument("invalid serialized RemotePath");
+		p += _addr.deserialize(b,p);
+		_ipScope = _addr.ipScope();
+		_trust = (Path::Trust)b[p++];
+		_lastSend = b.template at<uint64_t>(p); p += 8;
+		_lastReceived = b.template at<uint64_t>(p); p += 8;
+		p += _localAddress.deserialize(b,p);
+		_flags = b.template at<uint16_t>(p); p += 4;
+		return (startAt - p);
+	}
+
+protected:
 	uint64_t _lastSend;
 	uint64_t _lastReceived;
 	InetAddress _localAddress;
-	bool _fixed;
+	uint16_t _flags;
 };
 
 } // namespace ZeroTier

service/OneService.cpp

@@ -489,13 +489,12 @@ public:
 		OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S + "zerotier-one.port").c_str(),std::string(portstr));
 
 #ifdef ZT_USE_MINIUPNPC
-		// Bind a random secondary port for use with uPnP, since some NAT routers
+		// Bind a secondary port for use with uPnP, since some NAT routers
 		// (cough Ubiquity Edge cough) barf up a lung if you do both conventional
 		// NAT-t and uPnP from behind the same port. I think this is a bug, but
 		// everyone else's router bugs are our problem. :P
 		for(int k=0;k<512;++k) {
-			unsigned int upnport = 40000 + (((port + 1) * (k + 1)) % 25500);
+			const unsigned int upnport = 40000 + (((port + 1) * (k + 1)) % 25500);
-
 			_v4UpnpLocalAddress = InetAddress(0,upnport);
 			_v4UpnpUdpSocket = _phy.udpBind((const struct sockaddr *)&_v4UpnpLocalAddress,reinterpret_cast<void *>(&_v4UpnpLocalAddress),131072);
 			if (_v4UpnpUdpSocket) {