Formatting

node/Bond.cpp

@@ -140,7 +140,7 @@ SharedPtr<Path> Bond::getAppropriatePath(int64_t now, int32_t flowId)
 					}
 				}
 			}
-			//fprintf(stderr, "resultant _rrIdx=%d\n", _rrIdx);
+			fprintf(stderr, "_rrIdx=%d\n", _rrIdx);
 			if (_paths[_bondedIdx[_rrIdx]]) {
 				return _paths[_bondedIdx[_rrIdx]];
 			}
@@ -385,7 +385,7 @@ SharedPtr<Flow> Bond::createFlow(const SharedPtr<Path> &path, int32_t flowId, un
 	}
 	if (_flows.size() >= ZT_FLOW_MAX_COUNT) {
 		fprintf(stderr, "max number of flows reached (%d), forcibly forgetting oldest flow\n", ZT_FLOW_MAX_COUNT);
-        forgetFlowsWhenNecessary(0,true,now);
+		forgetFlowsWhenNecessary(0,true,now);
 	}
 	SharedPtr<Flow> flow = new Flow(flowId, now);
 	_flows[flowId] = flow;
@@ -588,7 +588,7 @@ void Bond::sendQOS_MEASUREMENT(void *tPtr,const SharedPtr<Path> &path,const int6
 	} else {
 		RR->sw->send(tPtr,outp,false);
 	}
-    // Account for the fact that a VERB_QOS_MEASUREMENT was just sent. Reset timers.
+	// Account for the fact that a VERB_QOS_MEASUREMENT was just sent. Reset timers.
 	path->_packetsReceivedSinceLastQoS = 0;
 	path->_lastQoSMeasurement = now;
 }
@@ -854,8 +854,8 @@ void Bond::estimatePathQuality(const int64_t now)
 	float plr[ZT_MAX_PEER_NETWORK_PATHS];
 	float per[ZT_MAX_PEER_NETWORK_PATHS];
 	float thr[ZT_MAX_PEER_NETWORK_PATHS];
-    float thm[ZT_MAX_PEER_NETWORK_PATHS];
-    float thv[ZT_MAX_PEER_NETWORK_PATHS];
+	float thm[ZT_MAX_PEER_NETWORK_PATHS];
+	float thv[ZT_MAX_PEER_NETWORK_PATHS];
 
 	float maxLAT = 0;
 	float maxPDV = 0;
@@ -1011,8 +1011,8 @@ void Bond::estimatePathQuality(const int64_t now)
 			if (_paths[i]) {
 				_paths[i]->address().toString(pathStr);
 				fprintf(stdout, "%s, %s, %8.3f, %8.3f, %8.3f, %5.3f, %5.3f, %5.3f, %8f, %5.3f, %5.3f, %d, %5.3f, %d, %d, %d, %d, %d, %d, ",
-					              getSlave(_paths[i])->ifname().c_str(), pathStr, _paths[i]->latencyMean, lat[i],pdv[i], _paths[i]->packetLossRatio, plr[i],per[i],thr[i],thm[i],thv[i],(now - _paths[i]->lastIn()),quality[i],alloc[i],
-					              _paths[i]->relativeByteLoad, _paths[i]->assignedFlowCount, _paths[i]->alive(now, true), _paths[i]->eligible(now,_ackSendInterval), _paths[i]->qosStatsOut.size());
+								  getSlave(_paths[i])->ifname().c_str(), pathStr, _paths[i]->latencyMean, lat[i],pdv[i], _paths[i]->packetLossRatio, plr[i],per[i],thr[i],thm[i],thv[i],(now - _paths[i]->lastIn()),quality[i],alloc[i],
+								  _paths[i]->relativeByteLoad, _paths[i]->assignedFlowCount, _paths[i]->alive(now, true), _paths[i]->eligible(now,_ackSendInterval), _paths[i]->qosStatsOut.size());
 			}
 		}
 		fprintf(stdout, "\n");
@@ -1022,7 +1022,144 @@ void Bond::estimatePathQuality(const int64_t now)
 
 void Bond::processBalanceTasks(const int64_t now)
 {
-	// Omitted
+	//fprintf(stderr, "processBalanceTasks\n");
+	char curPathStr[128];
+	if (_allowFlowHashing) {
+		/**
+		 * Clean up and reset flows if necessary
+		 */
+		if ((now - _lastFlowExpirationCheck) > ZT_MULTIPATH_FLOW_CHECK_INTERVAL) {
+			Mutex::Lock _l(_flows_m);
+			forgetFlowsWhenNecessary(ZT_MULTIPATH_FLOW_EXPIRATION_INTERVAL,false,now);
+			_lastFlowExpirationCheck = now;
+		}
+		if ((now - _lastFlowStatReset) > ZT_FLOW_STATS_RESET_INTERVAL) {
+			Mutex::Lock _l(_flows_m);
+			_lastFlowStatReset = now;
+			std::map<int32_t,SharedPtr<Flow> >::iterator it = _flows.begin();
+			while (it != _flows.end()) {
+				it->second->resetByteCounts();
+				++it;
+			}
+		}
+		/**
+		 * Re-allocate flows from dead paths
+		 */
+		if (_bondingPolicy== ZT_BONDING_POLICY_BALANCE_XOR || _bondingPolicy== ZT_BONDING_POLICY_BALANCE_AWARE) {
+			Mutex::Lock _l(_flows_m);
+			for (int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
+				if (!_paths[i]) {
+					continue;
+				}
+				if (!_paths[i]->eligible(now,_ackSendInterval) && _paths[i]->_shouldReallocateFlows) {
+					_paths[i]->address().toString(curPathStr);
+					fprintf(stderr, "%d reallocating flows from dead path %s on %s\n", (RR->node->now() - RR->bc->getBondStartTime()), curPathStr, getSlave(_paths[i])->ifname().c_str());
+					std::map<int32_t,SharedPtr<Flow> >::iterator flow_it = _flows.begin();
+					while (flow_it != _flows.end()) {
+						if (flow_it->second->assignedPath() == _paths[i]) {
+							if(assignFlowToBondedPath(flow_it->second, now)) {
+								_paths[i]->_assignedFlowCount--;
+							}
+						}
+						++flow_it;
+					}
+					_paths[i]->_shouldReallocateFlows = false;
+				}
+			}
+		}
+	}
+	/**
+	 * Tasks specific to (Balance Round Robin)
+	 */
+	if (_bondingPolicy== ZT_BONDING_POLICY_BALANCE_RR) {
+		if (_allowFlowHashing) {
+			// TODO: Should ideally failover from (idx) to a random slave, this is so that (idx+1) isn't overloaded
+		}
+		else if (!_allowFlowHashing) {
+			// Nothing
+		}
+	}
+	/**
+	 * Tasks specific to (Balance XOR)
+	 */
+	if (_bondingPolicy== ZT_BONDING_POLICY_BALANCE_XOR) {
+		// Nothing specific for XOR
+	}
+	/**
+	 * Tasks specific to (Balance Aware)
+	 */
+	if ((_bondingPolicy== ZT_BONDING_POLICY_BALANCE_AWARE)) {
+		if (_allowFlowHashing) {
+			Mutex::Lock _l(_flows_m);
+			/**
+			 * Re-balance flows in proportion to slave capacity (or when eligibility changes)
+			 */
+			if ((now - _lastFlowRebalance) > ZT_FLOW_REBALANCE_INTERVAL) {
+				/**
+				 * Determine "load" for bonded paths
+				 */
+				uint64_t totalBytes = 0;
+				for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { // first pass: compute absolute byte load and total
+					if (_paths[i] && _paths[i]->bonded()) {
+						_paths[i]->_byteLoad = 0;
+						std::map<int32_t,SharedPtr<Flow> >::iterator flow_it = _flows.begin();
+						while (flow_it != _flows.end()) {
+							if (flow_it->second->assignedPath() == _paths[i]) {
+								_paths[i]->_byteLoad += flow_it->second->totalBytes();
+							}
+							++flow_it;
+						}
+						totalBytes += _paths[i]->_byteLoad;
+					}
+				}
+				/**
+				 * Determine "affinity" for bonded path
+				 */
+				//fprintf(stderr, "\n\n");
+				_totalBondUnderload = 0;
+
+				for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { // second pass: compute relative byte loads and total imbalance
+					if (_paths[i] && _paths[i]->bonded()) {
+						if (totalBytes) {
+							uint8_t relativeByteLoad = std::ceil(((float)_paths[i]->_byteLoad / (float)totalBytes) * (float)255);
+							//fprintf(stderr, "lastComputedAllocation  = %d\n", _paths[i]->allocation);
+							//fprintf(stderr, "  relativeByteLoad      = %d\n", relativeByteLoad);
+							_paths[i]->_relativeByteLoad = relativeByteLoad;
+							uint8_t relativeUnderload = std::max(0, (int)_paths[i]->_allocation - (int)relativeByteLoad);
+							//fprintf(stderr, "    relativeUnderload  = %d\n", relativeUnderload);
+							_totalBondUnderload += relativeUnderload;
+							//fprintf(stderr, "    _totalBondUnderload = %d\n\n", _totalBondUnderload);
+							//_paths[i]->affinity = (relativeUnderload > 0 ? relativeUnderload : _paths[i]->_allocation);
+						}
+						else { // set everything to base values
+							_totalBondUnderload = 0;
+							//_paths[i]->affinity = 0;
+						}
+					}
+				}
+
+				//fprintf(stderr, "_totalBondUnderload=%d (end)\n\n", _totalBondUnderload);
+
+				/**
+				 *
+				 */
+				//fprintf(stderr, "_lastFlowRebalance\n");
+				std::map<int32_t, SharedPtr<Flow> >::iterator it = _flows.begin();
+				while (it != _flows.end()) {
+					int32_t flowId = it->first;
+					SharedPtr<Flow> flow = it->second;
+					if ((now - flow->_lastPathReassignment) > ZT_FLOW_MIN_REBALANCE_INTERVAL) {
+						//fprintf(stdout, "  could move : %x\n", flowId);
+					}
+					++it;
+				}
+				_lastFlowRebalance = now;
+			}
+		}
+		else if (!_allowFlowHashing) {
+			// Nothing
+		}
+	}
 }
 
 void Bond::dequeueNextActiveBackupPath(const uint64_t now)
@@ -1445,7 +1582,7 @@ void Bond::setReasonableDefaults(int policy)
 		case ZT_BONDING_POLICY_BALANCE_RR:
 			_failoverInterval = 5000;
 			_allowFlowHashing = false;
-			_packetsPerSlave = 8;
+			_packetsPerSlave = 512;
 			_slaveMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC;
 			_qualityWeights[ZT_QOS_LAT_IDX] = 0.4f;
 			_qualityWeights[ZT_QOS_LTM_IDX] = 0.0f;

node/Bond.hpp

@@ -45,48 +45,48 @@ class Bond
 
 public:
 
-    // TODO: Remove
-    bool _header;
-    int64_t _lastLogTS;
-    int64_t _lastPrintTS;
-    void dumpInfo(const int64_t now);
-    bool relevant();
+	// TODO: Remove
+	bool _header;
+	int64_t _lastLogTS;
+	int64_t _lastPrintTS;
+	void dumpInfo(const int64_t now);
+	bool relevant();
 
-    SharedPtr<Slave> getSlave(const SharedPtr<Path>& path);
+	SharedPtr<Slave> getSlave(const SharedPtr<Path>& path);
 
-    /**
-     * Constructor. For use only in first initialization in Node
-     *
-     * @param renv Runtime environment
-     */
-    Bond(const RuntimeEnvironment *renv);
+	/**
+	 * Constructor. For use only in first initialization in Node
+	 *
+	 * @param renv Runtime environment
+	 */
+	Bond(const RuntimeEnvironment *renv);
 
-    /**
-     * Constructor. Creates a bond based off of ZT defaults
-     *
-     * @param renv Runtime environment
-     * @param policy Bonding policy
-     * @param peer
-     */
-    Bond(const RuntimeEnvironment *renv, int policy, const SharedPtr<Peer>& peer);
+	/**
+	 * Constructor. Creates a bond based off of ZT defaults
+	 *
+	 * @param renv Runtime environment
+	 * @param policy Bonding policy
+	 * @param peer
+	 */
+	Bond(const RuntimeEnvironment *renv, int policy, const SharedPtr<Peer>& peer);
 
-    /**
-     * Constructor. For use when user intends to manually specify parameters
-     *
-     * @param basePolicy
-     * @param policyAlias
-     * @param peer
-     */
-    Bond(std::string& basePolicy, std::string& policyAlias, const SharedPtr<Peer>& peer);
+	/**
+	 * Constructor. For use when user intends to manually specify parameters
+	 *
+	 * @param basePolicy
+	 * @param policyAlias
+	 * @param peer
+	 */
+	Bond(std::string& basePolicy, std::string& policyAlias, const SharedPtr<Peer>& peer);
 
-    /**
-     * Constructor. Creates a bond based off of a user-defined bond template
-     *
-     * @param renv Runtime environment
-     * @param original
-     * @param peer
-     */
-    Bond(const RuntimeEnvironment *renv, const Bond &original, const SharedPtr<Peer>& peer);
+	/**
+	 * Constructor. Creates a bond based off of a user-defined bond template
+	 *
+	 * @param renv Runtime environment
+	 * @param original
+	 * @param peer
+	 */
+	Bond(const RuntimeEnvironment *renv, const Bond &original, const SharedPtr<Peer>& peer);
 
 	/**
 	 *
@@ -183,7 +183,7 @@ public:
 	 * @param now Current time
 	 */
 	void recordIncomingPacket(const SharedPtr<Path>& path, uint64_t packetId, uint16_t payloadLength,
-	        Packet::Verb verb, int32_t flowId, int64_t now);
+			Packet::Verb verb, int32_t flowId, int64_t now);
 
 	/**
 	 * Determines the most appropriate path for packet and flow egress. This decision is made by
@@ -223,7 +223,7 @@ public:
 	 */
 	bool assignFlowToBondedPath(SharedPtr<Flow> &flow, int64_t now);
 
-    /**
+	/**
 	 * Determine whether a path change should occur given the remote peer's reported utility and our
 	 * local peer's known utility. This has the effect of assigning inbound and outbound traffic to
 	 * the same path.
@@ -253,7 +253,7 @@ public:
 	 * @param now Current time
 	 */
 	void sendACK(void *tPtr,const SharedPtr<Path> &path,int64_t localSocket,
-	        const InetAddress &atAddress,int64_t now);
+			const InetAddress &atAddress,int64_t now);
 
 	/**
 	 * Sends a VERB_QOS_MEASUREMENT to the remote peer.
@@ -265,7 +265,7 @@ public:
 	 * @param now Current time
 	 */
 	void sendQOS_MEASUREMENT(void *tPtr,const SharedPtr<Path> &path,int64_t localSocket,
-	        const InetAddress &atAddress,int64_t now);
+			const InetAddress &atAddress,int64_t now);
 
 	/**
 	 * Sends a VERB_PATH_NEGOTIATION_REQUEST to the remote peer.
@@ -296,12 +296,12 @@ public:
 	 */
 	void dequeueNextActiveBackupPath(uint64_t now);
 
-    /**
-     * Set bond parameters to reasonable defaults, these may later be overwritten by
+	/**
+	 * Set bond parameters to reasonable defaults, these may later be overwritten by
 	 * user-specified parameters.
-     *
-     * @param policy Bonding policy
-     */
+	 *
+	 * @param policy Bonding policy
+	 */
 	void setReasonableDefaults(int policy);
 
 	/**
@@ -450,19 +450,19 @@ public:
 	 */
 	inline uint16_t getUpDelay() { return _upDelay; }
 
-    /**
-     * @param upDelay Length of time before a newly-discovered path is admitted to the bond
-     */
+	/**
+	 * @param upDelay Length of time before a newly-discovered path is admitted to the bond
+	 */
 	inline void setUpDelay(int upDelay) { if (upDelay >= 0) { _upDelay = upDelay; } }
 
-    /**
-     * @return Length of time before a newly-failed path is removed from the bond
-     */
+	/**
+	 * @return Length of time before a newly-failed path is removed from the bond
+	 */
 	inline uint16_t getDownDelay() { return _downDelay; }
 
-    /**
-     * @param downDelay Length of time before a newly-failed path is removed from the bond
-     */
+	/**
+	 * @param downDelay Length of time before a newly-failed path is removed from the bond
+	 */
 	inline void setDownDelay(int downDelay) { if (downDelay >= 0) { _downDelay = downDelay; } }
 
 	/**
@@ -470,11 +470,11 @@ public:
 	 */
 	inline uint16_t getBondMonitorInterval() { return _bondMonitorInterval; }
 
-    /**
-     * Set the current monitoring interval for the bond (can be overridden with intervals specific to certain slaves.)
-     *
-     * @param monitorInterval How often gratuitous VERB_HELLO(s) are sent to remote peer.
-     */
+	/**
+	 * Set the current monitoring interval for the bond (can be overridden with intervals specific to certain slaves.)
+	 *
+	 * @param monitorInterval How often gratuitous VERB_HELLO(s) are sent to remote peer.
+	 */
 	inline void setBondMonitorInterval(uint16_t interval) { _bondMonitorInterval = interval; }
 
 	/**
@@ -487,10 +487,10 @@ public:
 	 */
 	inline uint8_t getPolicy() { return _bondingPolicy; }
 
-    /**
-     *
-     * @param allowFlowHashing
-     */
+	/**
+	 *
+	 * @param allowFlowHashing
+	 */
 	inline void setFlowHashing(bool allowFlowHashing) { _allowFlowHashing = allowFlowHashing; }
 
 	/**
@@ -498,10 +498,10 @@ public:
 	 */
 	bool flowHashingEnabled() { return _allowFlowHashing; }
 
-    /**
-     *
-     * @param packetsPerSlave
-     */
+	/**
+	 *
+	 * @param packetsPerSlave
+	 */
 	inline void setPacketsPerSlave(int packetsPerSlave) { _packetsPerSlave = packetsPerSlave; }
 
 	/**
@@ -514,7 +514,7 @@ public:
 	 *
 	 * @return
 	 */
-    inline uint8_t getSlaveSelectMethod() { return _abSlaveSelectMethod; }
+	inline uint8_t getSlaveSelectMethod() { return _abSlaveSelectMethod; }
 
 	/**
 	 *

node/BondController.hpp

@@ -55,10 +55,10 @@ public:
 	 */
 	bool inUse() { return !_bondPolicyTemplates.empty() || _defaultBondingPolicy; }
 
-    /**
-     * @param basePolicyName Bonding policy name (See ZeroTierOne.h)
-     * @return The bonding policy code for a given human-readable bonding policy name
-     */
+	/**
+	 * @param basePolicyName Bonding policy name (See ZeroTierOne.h)
+	 * @return The bonding policy code for a given human-readable bonding policy name
+	 */
 	static int getPolicyCodeByStr(const std::string& basePolicyName)
 	{
 		if (basePolicyName == "active-backup") { return 1; }
@@ -83,18 +83,18 @@ public:
 		return "none";
 	}
 
-    /**
-     * Sets the default bonding policy for new or undefined bonds.
+	/**
+	 * Sets the default bonding policy for new or undefined bonds.
 	 *
-     * @param bp Bonding policy
-     */
+	 * @param bp Bonding policy
+	 */
 	void setBondingLayerDefaultPolicy(uint8_t bp) { _defaultBondingPolicy = bp; }
 
-    /**
-     * Sets the default (custom) bonding policy for new or undefined bonds.
+	/**
+	 * Sets the default (custom) bonding policy for new or undefined bonds.
 	 *
-     * @param alias Human-readable string alias for bonding policy
-     */
+	 * @param alias Human-readable string alias for bonding policy
+	 */
 	void setBondingLayerDefaultPolicyStr(std::string alias) { _defaultBondingPolicyStr = alias; }
 
 	/**

node/Flow.hpp

@@ -24,10 +24,10 @@ namespace ZeroTier {
  */
 struct Flow
 {
-    /**
-     * @param flowId Given flow ID
-     * @param now Current time
-     */
+	/**
+	 * @param flowId Given flow ID
+	 * @param now Current time
+	 */
 	Flow(int32_t flowId, int64_t now) :
 		_flowId(flowId),
 		_bytesInPerUnitTime(0),

node/Path.hpp

@@ -28,7 +28,6 @@
 #include "Utils.hpp"
 #include "Packet.hpp"
 #include "RingBuffer.hpp"
-//#include "Bond.hpp"
 
 #include "../osdep/Slave.hpp"
 
@@ -48,7 +47,6 @@ class Path
 {
 	friend class SharedPtr<Path>;
 	friend class Bond;
-	//friend class SharedPtr<Bond>;
 
 public:
 	/**

osdep/Slave.hpp

@@ -230,7 +230,6 @@ private:
 	bool _isUserSpecified;
 
 	AtomicCounter __refCount;
-
 };
 
 } // namespace ZeroTier