Multicast bandwidth accounting work in progress, and some config field changes and cleanup.

netconf-service/netconf.cpp

@@ -245,13 +245,14 @@ int main(int argc,char **argv)
 
 				Dictionary netconf;
 
-				netconf["peer"] = peerIdentity.address().toString();
 				sprintf(buf,"%.16llx",(unsigned long long)nwid);
 				netconf["nwid"] = buf;
-				netconf["isOpen"] = (isOpen ? "1" : "0");
+				netconf["isOpen"] = (isOpen ? "1" : "0"); // TODO: remove, old name
+				netconf["o"] = (isOpen ? "1" : "0");
 				netconf["name"] = name;
 				netconf["desc"] = desc;
-				netconf["etherTypes"] = etherTypeWhitelist;
+				netconf["etherTypes"] = etherTypeWhitelist; // TODO: remove, old name
+				netconf["et"] = etherTypeWhitelist;
 				sprintf(buf,"%llx",(unsigned long long)Utils::now());
 				netconf["ts"] = buf;
 
@@ -326,12 +327,16 @@ int main(int argc,char **argv)
 					}
 				}
 
-				if (ipv4Static.length())
+				// Add static assignments to netconf, if any
-					netconf["ipv4Static"] = ipv4Static;
+				if (ipv4Static.length()) {
-				if (ipv6Static.length())
+					netconf["ipv4Static"] = ipv4Static; // TODO: remove, old name
-					netconf["ipv6Static"] = ipv6Static;
+					netconf["v4s"] = ipv4Static;
+				}
+				if (ipv6Static.length()) {
+					netconf["v6s"] = ipv6Static;
+				}
 
-				{
+				{ // Create and send service bus response with payload attached as 'netconf'
 					Dictionary response;
 					response["peer"] = peerIdentity.address().toString();
 					response["nwid"] = request.get("nwid");

node/BandwidthAccount.hpp

@@ -25,8 +25,8 @@
  * LLC. Start here: http://www.zerotier.com/
  */
 
-#ifndef _ZT_RATELIMITER_HPP
+#ifndef _ZT_BWACCOUNT_HPP
-#define _ZT_RATELIMITER_HPP
+#define _ZT_BWACCOUNT_HPP
 
 #include <math.h>
 
@@ -41,7 +41,7 @@
 namespace ZeroTier {
 
 /**
- * Data transfer accounting used for multicast groups
+ * Bandwidth account used for rate limiting multicast groups
  *
  * This is used to apply a bank account model to multicast groups. Each
  * multicast packet counts against a balance, which accrues at a given
@@ -53,13 +53,13 @@ namespace ZeroTier {
  * spew lots of multicast messages at once, wait a while, then do it
  * again. A consistent bandwidth limit model doesn't fit.
  */
-class RateLimiter
+class BandwidthAccount
 {
 public:
 	/**
-	 * Rate and min/max to apply on rate limiter update
+	 * Rate of balance accrual and min/max
 	 */
-	struct Rate
+	struct Accrual
 	{
 		/**
 		 * Rate of balance accrual in bytes per second
@@ -78,25 +78,25 @@ public:
 	};
 
 	/**
-	 * Create an uninitialized rate limiter
+	 * Create an uninitialized account
 	 *
 	 * init() must be called before this is used.
 	 */
-	RateLimiter() throw() {}
+	BandwidthAccount() throw() {}
 
 	/**
-	 * Create an initialize rate limiter
+	 * Create and initialize
 	 *
 	 * @param preload Initial balance to place in account
 	 */
-	RateLimiter(double preload)
+	BandwidthAccount(double preload)
 		throw()
 	{
 		init(preload);
 	}
 
 	/**
-	 * Initialize or re-initialize rate limiter
+	 * Initialize or re-initialize account
 	 *
 	 * @param preload Initial balance to place in account
 	 */
@@ -108,18 +108,18 @@ public:
 	}
 
 	/**
-	 * Update balance based on current clock and supplied rate
+	 * Update balance by accruing and then deducting
 	 *
-	 * @param lim Current limits in effect
+	 * @param ar Current rate of accrual
 	 * @param deduct Amount to deduct, or 0.0 to just update
 	 * @return New balance with deduction applied
 	 */
-	inline double update(const Rate &r,double deduct)
+	inline double update(const Accrual &ar,double deduct)
 		throw()
 	{
 		double lt = _lastTime;
 		double now = _lastTime = Utils::nowf();
-		return (_balance = fmax(r.minBalance,fmin(r.maxBalance,(_balance + (r.bytesPerSecond * (now - lt))) - deduct)));
+		return (_balance = fmax(ar.minBalance,fmin(ar.maxBalance,(_balance + (ar.bytesPerSecond * (now - lt))) - deduct)));
 	}
 
 private:

node/Constants.hpp

@@ -282,26 +282,6 @@ error_no_ZT_ARCH_defined;
  */
 #define ZT_MULTICAST_LOCAL_POLL_PERIOD 10000
 
-/**
- * Default bytes per second limit for multicasts per peer on a network
- */
-#define ZT_MULTICAST_DEFAULT_BYTES_PER_SECOND 100.0
-
-/**
- * Default balance preload for multicast rate limiters on a network
- */
-#define ZT_MULTICAST_DEFAULT_RATE_PRELOAD 25000.0
-
-/**
- * Default maximum balance for multicast rate limiters
- */
-#define ZT_MULTICAST_DEFAULT_RATE_MAX_BALANCE 25000.0
-
-/**
- * Default minimum balance for multicast rate limiters (max debt)
- */
-#define ZT_MULTICAST_DEFAULT_RATE_MIN_BALANCE -5000.0
-
 /**
  * Delay between scans of the topology active peer DB for peers that need ping
  */

node/MulticastGroup.hpp

@@ -106,7 +106,7 @@ public:
 	inline std::string toString() const
 	{
 		char buf[64];
-		Utils::snprintf(buf,sizeof(buf),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x/%.8lx",(unsigned int)_mac.data[0],(unsigned int)_mac.data[1],(unsigned int)_mac.data[2],(unsigned int)_mac.data[3],(unsigned int)_mac.data[4],(unsigned int)_mac.data[5],(unsigned long)_adi);
+		Utils::snprintf(buf,sizeof(buf),"%.2x%.2x%.2x%.2x%.2x%.2x/%.8lx",(unsigned int)_mac.data[0],(unsigned int)_mac.data[1],(unsigned int)_mac.data[2],(unsigned int)_mac.data[3],(unsigned int)_mac.data[4],(unsigned int)_mac.data[5],(unsigned long)_adi);
 		return std::string(buf);
 	}
 

node/Network.cpp

@@ -110,6 +110,9 @@ bool Network::Certificate::qualifyMembership(const Network::Certificate &mc) con
 	return true;
 }
 
+// A low default global rate, fast enough for something like ARP
+const Network::MulticastRates::Rate Network::MulticastRates::GLOBAL_DEFAULT_RATE(256.0,-32.0,256.0,64.0);
+
 const char *Network::statusString(const Status s)
 	throw()
 {
@@ -166,24 +169,31 @@ SharedPtr<Network> Network::newInstance(const RuntimeEnvironment *renv,uint64_t
 void Network::setConfiguration(const Network::Config &conf)
 {
 	Mutex::Lock _l(_lock);
-	if ((conf.networkId() == _id)&&(conf.peerAddress() == _r->identity.address())) { // sanity check
+	try {
-		//TRACE("network %.16llx got netconf:\n%s",(unsigned long long)_id,conf.toString().c_str());
+		if (conf.networkId() == _id) { // sanity check
-		_configuration = conf;
+			//TRACE("network %.16llx got netconf:\n%s",(unsigned long long)_id,conf.toString().c_str());
-		_myCertificate = conf.certificateOfMembership();
+			_configuration = conf;
-		_lastConfigUpdate = Utils::now();
+			_myCertificate = conf.certificateOfMembership();
+			_lastConfigUpdate = Utils::now();
 
-		_tap->setIps(conf.staticAddresses());
+			_tap->setIps(conf.staticAddresses());
-		_tap->setDisplayName((std::string("ZeroTier One [") + conf.name() + "]").c_str());
+			_tap->setDisplayName((std::string("ZeroTier One [") + conf.name() + "]").c_str());
 
-		memset(_etWhitelist,0,sizeof(_etWhitelist));
+			memset(_etWhitelist,0,sizeof(_etWhitelist));
-		std::set<unsigned int> wl(conf.etherTypes());
+			std::set<unsigned int> wl(conf.etherTypes());
-		for(std::set<unsigned int>::const_iterator t(wl.begin());t!=wl.end();++t)
+			for(std::set<unsigned int>::const_iterator t(wl.begin());t!=wl.end();++t)
-			_etWhitelist[*t / 8] |= (unsigned char)(1 << (*t % 8));
+				_etWhitelist[*t / 8] |= (unsigned char)(1 << (*t % 8));
 
-		std::string confPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + toString() + ".conf");
+			std::string confPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + toString() + ".conf");
-		if (!Utils::writeFile(confPath.c_str(),conf.toString())) {
+			if (!Utils::writeFile(confPath.c_str(),conf.toString())) {
-			LOG("error: unable to write network configuration file at: %s",confPath.c_str());
+				LOG("error: unable to write network configuration file at: %s",confPath.c_str());
+			}
 		}
+	} catch ( ... ) {
+		_configuration = Config();
+		_myCertificate = Certificate();
+		_lastConfigUpdate = 0;
+		LOG("unexpected exception handling config for network %.16llx, retrying fetch...",(unsigned long long)_id);
 	}
 }
 
@@ -275,7 +285,7 @@ void Network::clean()
 Network::Status Network::status() const
 {
 	Mutex::Lock _l(_lock);
-	if (_configuration.containsAllFields())
+	if (_configuration)
 		return NETWORK_OK;
 	return NETWORK_WAITING_FOR_FIRST_AUTOCONF;
 }
@@ -302,11 +312,8 @@ void Network::_restoreState()
 	std::string confs;
 	if (Utils::readFile(confPath.c_str(),confs)) {
 		try {
-			if (confs.length()) {
+			if (confs.length())
-				Config conf(confs);
+				setConfiguration(Config(confs));
-				if (conf.containsAllFields())
-					setConfiguration(conf);
-			}
 		} catch ( ... ) {} // ignore invalid config on disk, we will re-request
 	} else {
 		// If the conf file isn't present, "touch" it so we'll remember

node/Network.hpp

@@ -47,6 +47,7 @@
 #include "Dictionary.hpp"
 #include "Identity.hpp"
 #include "InetAddress.hpp"
+#include "BandwidthAccount.hpp"
 
 namespace ZeroTier {
 
@@ -85,24 +86,10 @@ public:
 	class Certificate : private Dictionary
 	{
 	public:
-		Certificate()
+		Certificate() {}
-		{
+		Certificate(const char *s) : Dictionary(s) {}
-		}
+		Certificate(const std::string &s) : Dictionary(s) {}
-
+		inline std::string toString() const { return Dictionary::toString(); }
-		Certificate(const char *s) :
-			Dictionary(s)
-		{
-		}
-
-		Certificate(const std::string &s) :
-			Dictionary(s)
-		{
-		}
-
-		inline std::string toString() const
-		{
-			return Dictionary::toString();
-		}
 
 		inline void setNetworkId(uint64_t id)
 		{
@@ -192,59 +179,160 @@ public:
 		void _shaForSignature(unsigned char *dig) const;
 	};
 
+	/**
+	 * Preload and rates of accrual for multicast group bandwidth limits
+	 *
+	 * Key is multicast group in lower case hex format: MAC (without :s) /
+	 * ADI (hex). Value is a comma-delimited list of: preload, min, max,
+	 * rate of accrual for bandwidth accounts. A key called '*' indicates
+	 * the default for unlisted groups.
+	 */
+	class MulticastRates : private Dictionary
+	{
+	public:
+		/**
+		 * Preload and accrual parameter tuple
+		 */
+		struct Rate
+		{
+			Rate() {}
+			Rate(double pl,double minr,double maxr,double bps)
+			{
+				preload = pl;
+				accrual.bytesPerSecond = bps;
+				accrual.maxBalance = maxr;
+				accrual.minBalance = minr;
+			}
+
+			double preload;
+			BandwidthAccount::Accrual accrual;
+		};
+
+		MulticastRates() {}
+		MulticastRates(const char *s) : Dictionary(s) {}
+		MulticastRates(const std::string &s) : Dictionary(s) {}
+		inline std::string toString() const { return Dictionary::toString(); }
+
+		/**
+		 * A very minimal default rate, fast enough for ARP
+		 */
+		static const Rate GLOBAL_DEFAULT_RATE;
+
+		/**
+		 * @return Default rate, or GLOBAL_DEFAULT_RATE if not specified
+		 */
+		Rate defaultRate() const
+		{
+			Rate r;
+			const_iterator dfl(find("*"));
+			if (dfl == end())
+				return GLOBAL_DEFAULT_RATE;
+			return _toRate(dfl->second);
+		}
+
+		/**
+		 * Get the rate for a given multicast group
+		 *
+		 * @param mg Multicast group
+		 * @return Rate or default() rate if not specified
+		 */
+		Rate get(const MulticastGroup &mg) const
+		{
+			const_iterator r(find(mg.toString()));
+			if (r == end())
+				return defaultRate();
+			return _toRate(r->second);
+		}
+
+	private:
+		static inline Rate _toRate(const std::string &s)
+		{
+			char tmp[16384];
+			Utils::scopy(tmp,sizeof(tmp),s.c_str());
+			Rate r;
+			r.preload = 0.0;
+			r.accrual.bytesPerSecond = 0.0;
+			r.accrual.maxBalance = 0.0;
+			r.accrual.minBalance = 0.0;
+			char *saveptr = (char *)0;
+			unsigned int fn = 0;
+			for(char *f=Utils::stok(tmp,",",&saveptr);(f);f=Utils::stok((char *)0,",",&saveptr)) {
+				switch(fn++) {
+					case 0:
+						r.preload = Utils::strToDouble(f);
+						break;
+					case 1:
+						r.accrual.minBalance = Utils::strToDouble(f);
+						break;
+					case 2:
+						r.accrual.maxBalance = Utils::strToDouble(f);
+						break;
+					case 3:
+						r.accrual.bytesPerSecond = Utils::strToDouble(f);
+						break;
+				}
+			}
+			return r;
+		}
+	};
+
 	/**
 	 * A network configuration for a given node
+	 *
+	 * Configuration fields:
+	 *
+	 * nwid=<hex network ID> (required)
+	 * name=short name
+	 * desc=long(er) description
+	 * com=Certificate (serialized dictionary)
+	 * mr=MulticastRates (serialized dictionary)
+	 * o=open network? (1 or 0, default false if missing)
+	 * et=ethertype whitelist (comma-delimited list of ethertypes in decimal)
+	 * v4s=IPv4 static assignments / netmasks (comma-delimited)
+	 * v6s=IPv6 static assignments / netmasks (comma-delimited)
 	 */
 	class Config : private Dictionary
 	{
 	public:
-		Config()
+		Config() {}
-		{
+		Config(const char *s) : Dictionary(s) {}
-		}
+		Config(const std::string &s) : Dictionary(s) {}
+		inline std::string toString() const { return Dictionary::toString(); }
 
-		Config(const char *s) :
+		/**
-			Dictionary(s)
+		 * @return True if configuration is valid and contains required fields
-		{
+		 */
-		}
+		inline operator bool() const throw() { return (find("nwid") != end()); }
-
-		Config(const std::string &s) :
-			Dictionary(s)
-		{
-		}
-
-		inline bool containsAllFields() const
-		{
-			return (contains("nwid")&&contains("peer"));
-		}
-
-		inline std::string toString() const
-		{
-			return Dictionary::toString();
-		}
 
+		/**
+		 * @return Network ID
+		 * @throws std::invalid_argument Network ID field missing
+		 */
 		inline uint64_t networkId() const
 			throw(std::invalid_argument)
 		{
-#ifdef __WINDOWS__
+			return Utils::hexStrToU64(get("nwid").c_str());
-			return _strtoui64(get("nwid").c_str(),(char **)0,16);
-#else
-			return strtoull(get("nwid").c_str(),(char **)0,16);
-#endif
 		}
 
+		/**
+		 * Get this network's short name, or its ID in hex if unspecified
+		 *
+		 * @return Short name of this network (e.g. "earth")
+		 */
 		inline std::string name() const
 		{
-			if (contains("name"))
+			const_iterator n(find("name"));
-				return get("name");
+			if (n == end())
-			char buf[32];
+				return get("nwid");
-			Utils::snprintf(buf,sizeof(buf),"%.16llx",(unsigned long long)networkId());
+			return n->second;
-			return std::string(buf);
 		}
 
-		inline Address peerAddress() const
+		/**
-			throw(std::invalid_argument)
+		 * @return Long description of network or empty string if not present
+		 */
+		inline std::string desc() const
 		{
-			return Address(get("peer"));
+			return get("desc",std::string());
 		}
 
 		/**
@@ -258,12 +346,28 @@ public:
 			else return Certificate(cm->second);
 		}
 
+		/**
+		 * @return Multicast rates for this network
+		 */
+		inline MulticastRates multicastRates() const
+		{
+			const_iterator mr(find("mr"));
+			if (mr == end())
+				return MulticastRates();
+			else return MulticastRates(mr->second);
+		}
+
 		/**
 		 * @return True if this is an open non-access-controlled network
 		 */
 		inline bool isOpen() const
 		{
-			return (get("isOpen","0") == "1");
+			const_iterator o(find("o"));
+			if (o == end())
+				return false;
+			else if (!o->second.length())
+				return false;
+			else return (o->second[0] == '1');
 		}
 
 		/**
@@ -274,10 +378,10 @@ public:
 			char tmp[16384];
 			char *saveptr = (char *)0;
 			std::set<unsigned int> et;
-			if (!Utils::scopy(tmp,sizeof(tmp),get("etherTypes","").c_str()))
+			if (!Utils::scopy(tmp,sizeof(tmp),get("et","").c_str()))
-				return et; // sanity check
+				return et; // sanity check, packet can't really be that big
 			for(char *f=Utils::stok(tmp,",",&saveptr);(f);f=Utils::stok((char *)0,",",&saveptr)) {
-				unsigned int t = Utils::stoui(f);
+				unsigned int t = Utils::strToUInt(f);
 				if (t)
 					et.insert(t);
 			}
@@ -290,10 +394,10 @@ public:
 		inline std::set<InetAddress> staticAddresses() const
 		{
 			std::set<InetAddress> sa;
-			std::vector<std::string> ips(Utils::split(get("ipv4Static","").c_str(),",","",""));
+			std::vector<std::string> ips(Utils::split(get("v4s","").c_str(),",","",""));
 			for(std::vector<std::string>::const_iterator i(ips.begin());i!=ips.end();++i)
 				sa.insert(InetAddress(*i));
-			ips = Utils::split(get("ipv6Static","").c_str(),",","","");
+			ips = Utils::split(get("v6s","").c_str(),",","","");
 			for(std::vector<std::string>::const_iterator i(ips.begin());i!=ips.end();++i)
 				sa.insert(InetAddress(*i));
 			return sa;
@@ -462,6 +566,8 @@ public:
 	Status status() const;
 
 	/**
+	 * Determine whether frames of a given ethernet type are allowed on this network
+	 *
 	 * @param etherType Ethernet frame type
 	 * @return True if network permits this type
 	 */
@@ -475,17 +581,26 @@ public:
 		else return ((_etWhitelist[etherType / 8] & (unsigned char)(1 << (etherType % 8))) != 0);
 	}
 
+	inline bool updateAndCheckMulticastBalance(const Address &a,const MulticastGroup &mg,unsigned int bytes)
+	{
+		Mutex::Lock _l(_lock);
+		std::map< std::pair<Address,MulticastGroup>,BandwidthAccount >::iterator bal(_multicastRateAccounts.find(std::pair<Address,MulticastGroup>(a,mg)));
+	}
+
 private:
 	static void _CBhandleTapData(void *arg,const MAC &from,const MAC &to,unsigned int etherType,const Buffer<4096> &data);
 	void _restoreState();
 
 	const RuntimeEnvironment *_r;
 
-	// Tap and tap multicast memberships
+	// Multicast bandwidth accounting for peers on this network
+	std::map< std::pair<Address,MulticastGroup>,BandwidthAccount > _multicastRateAccounts;
+
+	// Tap and tap multicast memberships for this node on this network
 	EthernetTap *_tap;
 	std::set<MulticastGroup> _multicastGroups;
 
-	// Membership certificates supplied by peers
+	// Membership certificates supplied by other peers on this network
 	std::map<Address,Certificate> _membershipCertificates;
 
 	// Configuration from network master node

node/PacketDecoder.cpp

@@ -319,10 +319,8 @@ bool PacketDecoder::_doOK(const RuntimeEnvironment *_r,const SharedPtr<Peer> &pe
 					std::string dict((const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT,dictlen),dictlen);
 					if (dict.length()) {
 						Network::Config netconf(dict);
-						if ((netconf.networkId() == nw->id())&&(netconf.peerAddress() == _r->identity.address())) { // sanity check
+						LOG("got network configuration for network %.16llx from %s",(unsigned long long)nw->id(),source().toString().c_str());
-							LOG("got network configuration for network %.16llx from %s",(unsigned long long)nw->id(),source().toString().c_str());
+						nw->setConfiguration(netconf);
-							nw->setConfiguration(netconf);
-						}
 					}
 				}
 			}	break;

node/Utils.hpp

@@ -461,36 +461,49 @@ public:
 #endif
 	}
 
-	// String to int converters (and hex string to int)
+	// String to number converters
-	static inline unsigned int stoui(const char *s)
+	static inline unsigned int strToUInt(const char *s)
 		throw()
 	{
 		return (unsigned int)strtoul(s,(char **)0,10);
 	}
-	static inline unsigned long stoul(const char *s)
+	static inline unsigned long strToULong(const char *s)
 		throw()
 	{
 		return strtoul(s,(char **)0,10);
 	}
-	static inline unsigned long long stoull(const char *s)
+	static inline unsigned long long strToU64(const char *s)
 		throw()
 	{
+#ifdef __WINDOWS__
+		return _strtoui64(s,(char **)0,10);
+#else
 		return strtoull(s,(char **)0,10);
+#endif
 	}
-	static inline unsigned int hstoui(const char *s)
+	static inline unsigned int hexStrToUInt(const char *s)
 		throw()
 	{
 		return (unsigned int)strtoul(s,(char **)0,16);
 	}
-	static inline unsigned long hstoul(const char *s)
+	static inline unsigned long hexStrToULong(const char *s)
 		throw()
 	{
 		return strtoul(s,(char **)0,16);
 	}
-	static inline unsigned long long hstoull(const char *s)
+	static inline unsigned long long hexStrToU64(const char *s)
 		throw()
 	{
+#ifdef __WINDOWS__
+		return _strtoui64(s,(char **)0,16);
+#else
 		return strtoull(s,(char **)0,16);
+#endif
+	}
+	static inline double strToDouble(const char *s)
+		throw()
+	{
+		return strtod(s,(char **)0);
 	}
 
 	/**