Merge branch 'dev' of Z:\ZeroTier\Local\ZeroTier-v1 into dev

ext/installfiles/mac/ZeroTier One.pkgproj

@@ -701,7 +701,7 @@
 				<key>USE_HFS+_COMPRESSION</key>
 				<false/>
 				<key>VERSION</key>
-				<string>1.10.6</string>
+				<string>1.12.0</string>
 			</dict>
 			<key>TYPE</key>
 			<integer>0</integer>

include/ZeroTierOne.h

@@ -1327,6 +1327,11 @@ typedef struct
 	 */
 	float packetErrorRatio;
 
+	/**
+	 * Number of flows assigned to this path
+	 */
+	uint16_t assignedFlowCount;
+
 	/**
 	 * Address scope
 	 */

java/src/com/zerotier/sdk/Event.java

@@ -29,14 +29,14 @@ package com.zerotier.sdk;
 
 /**
  * Status codes sent to status update callback when things happen
- *
+ * <p>
  * Defined in ZeroTierOne.h as ZT_Event
  */
 public enum Event {
 
     /**
      * Node has been initialized
-     *
+     * <p>
      * This is the first event generated, and is always sent. It may occur
      * before Node's constructor returns.
      */
@@ -49,7 +49,7 @@ public enum Event {
 
     /**
      * Node is online -- at least one upstream node appears reachable
-     *
+     * <p>
      * Meta-data: none
      */
     EVENT_ONLINE(2),
@@ -104,7 +104,7 @@ public enum Event {
 
     /**
      * VERB_USER_MESSAGE received
-     *
+     * <p>
      * These are generated when a VERB_USER_MESSAGE packet is received via
      * ZeroTier VL1.
      */
@@ -112,7 +112,7 @@ public enum Event {
 
     /**
      * Remote trace received
-     *
+     * <p>
      * These are generated when a VERB_REMOTE_TRACE is received. Note
      * that any node can fling one of these at us. It is your responsibility
      * to filter and determine if it's worth paying attention to. If it's

java/src/com/zerotier/sdk/Node.java

@@ -75,8 +75,8 @@ public class Node {
             EventListener eventListener,
             VirtualNetworkFrameListener frameListener,
             VirtualNetworkConfigListener configListener,
-            PathChecker pathChecker) throws NodeException {
+            PathChecker pathChecker) {
-        ResultCode rc = node_init(
+        return node_init(
                 nodeId,
                 getListener,
                 putListener,
@@ -85,10 +85,6 @@ public class Node {
                 frameListener,
                 configListener,
                 pathChecker);
-        if(rc != ResultCode.RESULT_OK) {
-            throw new NodeException(rc.toString());
-        }
-        return rc;
     }
 
     public boolean isInited() {
@@ -299,7 +295,7 @@ public class Node {
 
     /**
      * Add or update a moon
-     *
+     * <p>
      * Moons are persisted in the data store in moons.d/, so this can persist
      * across invocations if the contents of moon.d are scanned and orbit is
      * called for each on startup.

java/src/com/zerotier/sdk/NodeException.java

@@ -1,37 +0,0 @@
-/*
- * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2015  ZeroTier, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-package com.zerotier.sdk;
-
-public class NodeException extends Exception {
-
-    private static final long serialVersionUID = 6268040509883125819L;
-    
-    public NodeException(String message) {
-        super(message);
-    }
-}

java/src/com/zerotier/sdk/NodeStatus.java

@@ -31,7 +31,7 @@ import com.zerotier.sdk.util.StringUtils;
 
 /**
  * Current node status
- *
+ * <p>
  * Defined in ZeroTierOne.h as ZT_NodeStatus
  */
 public class NodeStatus {

java/src/com/zerotier/sdk/PathChecker.java

@@ -11,9 +11,9 @@ public interface PathChecker {
 
     /**
      * Callback to check whether a path should be used for ZeroTier traffic
-     *
+     * <p>
      * This function must return true if the path should be used.
-     *
+     * <p>
      * If no path check function is specified, ZeroTier will still exclude paths
      * that overlap with ZeroTier-assigned and managed IP address blocks. But the
      * use of a path check function is recommended to ensure that recursion does
@@ -22,7 +22,7 @@ public interface PathChecker {
      * all configured ZeroTier interfaces and check to ensure that the supplied
      * addresses will not result in ZeroTier traffic being sent over a ZeroTier
      * interface (recursion).
-     *
+     * <p>
      * Obviously this is not required in configurations where this can't happen,
      * such as network containers or embedded.
      *
@@ -35,7 +35,7 @@ public interface PathChecker {
 
     /**
      * Function to get physical addresses for ZeroTier peers
-     *
+     * <p>
      * If provided this function will be occasionally called to get physical
      * addresses that might be tried to reach a ZeroTier address.
      *

java/src/com/zerotier/sdk/Peer.java

@@ -33,7 +33,7 @@ import java.util.Arrays;
 
 /**
  * Peer status result buffer
- *
+ * <p>
  * Defined in ZeroTierOne.h as ZT_Peer
  */
 public class Peer {

java/src/com/zerotier/sdk/PeerPhysicalPath.java

@@ -31,7 +31,7 @@ import java.net.InetSocketAddress;
 
 /**
  * Physical network path to a peer
- *
+ * <p>
  * Defined in ZeroTierOne.h as ZT_PeerPhysicalPath
  */
 public class PeerPhysicalPath {

java/src/com/zerotier/sdk/PeerRole.java

@@ -29,7 +29,7 @@ package com.zerotier.sdk;
 
 /**
  * What trust hierarchy role does this peer have?
- *
+ * <p>
  * Defined in ZeroTierOne.h as ZT_PeerRole
  */
 public enum PeerRole {

java/src/com/zerotier/sdk/VirtualNetworkConfig.java

@@ -38,7 +38,7 @@ import java.util.Collections;
 
 /**
  * Virtual network configuration
- *
+ * <p>
  * Defined in ZeroTierOne.h as ZT_VirtualNetworkConfig
  */
 public class VirtualNetworkConfig implements Comparable<VirtualNetworkConfig> {
@@ -107,100 +107,108 @@ public class VirtualNetworkConfig implements Comparable<VirtualNetworkConfig> {
     @Override
     public boolean equals(Object o) {
 
+        if (o == null) {
+            Log.i(TAG, "Old is null");
+
+            return false;
+        }
+
         if (!(o instanceof VirtualNetworkConfig)) {
-            return false;
+            Log.i(TAG, "Old is not an instance of VirtualNetworkConfig: " + o);
-        }
-
-        VirtualNetworkConfig cfg = (VirtualNetworkConfig) o;
-
-        if (this.nwid != cfg.nwid) {
-            Log.i(TAG, "NetworkID Changed. Old: " + StringUtils.networkIdToString(this.nwid) + " (" + this.nwid + "), " +
-                    "New: " + StringUtils.networkIdToString(cfg.nwid) + " (" + cfg.nwid + ")");
 
             return false;
         }
 
-        if (this.mac != cfg.mac) {
+        VirtualNetworkConfig old = (VirtualNetworkConfig) o;
-            Log.i(TAG, "MAC Changed. Old: " + StringUtils.macAddressToString(this.mac) + ", New: " + StringUtils.macAddressToString(cfg.mac));
+
+        if (this.nwid != old.nwid) {
+            Log.i(TAG, "NetworkID Changed. New: " + StringUtils.networkIdToString(this.nwid) + " (" + this.nwid + "), " +
+                    "Old: " + StringUtils.networkIdToString(old.nwid) + " (" + old.nwid + ")");
 
             return false;
         }
 
-        if (!this.name.equals(cfg.name)) {
+        if (this.mac != old.mac) {
-            Log.i(TAG, "Name Changed. Old: " + this.name + ", New: " + cfg.name);
+            Log.i(TAG, "MAC Changed. New: " + StringUtils.macAddressToString(this.mac) + ", Old: " + StringUtils.macAddressToString(old.mac));
 
             return false;
         }
 
-        if (this.status != cfg.status) {
+        if (!this.name.equals(old.name)) {
-            Log.i(TAG, "Status Changed. Old: " + this.status + ", New: " + cfg.status);
+            Log.i(TAG, "Name Changed. New: " + this.name + ", Old: " + old.name);
 
             return false;
         }
 
-        if (this.type != cfg.type) {
+        if (this.status != old.status) {
-            Log.i(TAG, "Type changed. Old " + this.type + ", New: " + cfg.type);
+            Log.i(TAG, "Status Changed. New: " + this.status + ", Old: " + old.status);
 
             return false;
         }
 
-        if (this.mtu != cfg.mtu) {
+        if (this.type != old.type) {
-            Log.i(TAG, "MTU Changed. Old: " + this.mtu + ", New: " + cfg.mtu);
+            Log.i(TAG, "Type changed. New: " + this.type + ", Old: " + old.type);
 
             return false;
         }
 
-        if (this.dhcp != cfg.dhcp) {
+        if (this.mtu != old.mtu) {
-            Log.i(TAG, "DHCP Flag Changed. Old: " + this.dhcp + ", New: " + cfg.dhcp);
+            Log.i(TAG, "MTU Changed. New: " + this.mtu + ", Old: " + old.mtu);
 
             return false;
         }
 
-        if (this.bridge != cfg.bridge) {
+        if (this.dhcp != old.dhcp) {
-            Log.i(TAG, "Bridge Flag Changed. Old: " + this.bridge + ", New: " + cfg.bridge);
+            Log.i(TAG, "DHCP Flag Changed. New: " + this.dhcp + ", Old: " + old.dhcp);
 
             return false;
         }
 
-        if (this.broadcastEnabled != cfg.broadcastEnabled) {
+        if (this.bridge != old.bridge) {
-            Log.i(TAG, "Broadcast Flag Changed. Old: "+ this.broadcastEnabled + ", New: " + cfg.broadcastEnabled);
+            Log.i(TAG, "Bridge Flag Changed. New: " + this.bridge + ", Old: " + old.bridge);
 
             return false;
         }
 
-        if (this.portError != cfg.portError) {
+        if (this.broadcastEnabled != old.broadcastEnabled) {
-            Log.i(TAG, "Port Error Changed. Old: " + this.portError + ", New: " + cfg.portError);
+            Log.i(TAG, "Broadcast Flag Changed. New: "+ this.broadcastEnabled + ", Old: " + old.broadcastEnabled);
 
             return false;
         }
 
-        if (this.netconfRevision != cfg.netconfRevision) {
+        if (this.portError != old.portError) {
-            Log.i(TAG, "NetConfRevision Changed. Old: " + this.netconfRevision + ", New: " + cfg.netconfRevision);
+            Log.i(TAG, "Port Error Changed. New: " + this.portError + ", Old: " + old.portError);
 
             return false;
         }
 
-        if (!Arrays.equals(assignedAddresses, cfg.assignedAddresses)) {
+        if (this.netconfRevision != old.netconfRevision) {
+            Log.i(TAG, "NetConfRevision Changed. New: " + this.netconfRevision + ", Old: " + old.netconfRevision);
+
+            return false;
+        }
+
+        if (!Arrays.equals(assignedAddresses, old.assignedAddresses)) {
 
-            ArrayList<String> aaCurrent = new ArrayList<>();
             ArrayList<String> aaNew = new ArrayList<>();
+            ArrayList<String> aaOld = new ArrayList<>();
             for (InetSocketAddress s : assignedAddresses) {
-                aaCurrent.add(s.toString());
-            }
-            for (InetSocketAddress s : cfg.assignedAddresses) {
                 aaNew.add(s.toString());
             }
-            Collections.sort(aaCurrent);
+            for (InetSocketAddress s : old.assignedAddresses) {
+                aaOld.add(s.toString());
+            }
             Collections.sort(aaNew);
+            Collections.sort(aaOld);
 
             Log.i(TAG, "Assigned Addresses Changed");
-            Log.i(TAG, "Old:");
+            Log.i(TAG, "New:");
-            for (String s : aaCurrent) {
+            for (String s : aaNew) {
                 Log.i(TAG, "    " + s);
             }
             Log.i(TAG, "");
-            Log.i(TAG, "New:");
+            Log.i(TAG, "Old:");
-            for (String s : aaNew) {
+            for (String s : aaOld) {
                 Log.i(TAG, "    " +s);
             }
             Log.i(TAG, "");
@@ -208,27 +216,27 @@ public class VirtualNetworkConfig implements Comparable<VirtualNetworkConfig> {
             return false;
         }
 
-        if (!Arrays.equals(routes, cfg.routes)) {
+        if (!Arrays.equals(routes, old.routes)) {
 
-            ArrayList<String> rCurrent = new ArrayList<>();
             ArrayList<String> rNew = new ArrayList<>();
+            ArrayList<String> rOld = new ArrayList<>();
             for (VirtualNetworkRoute r : routes) {
-                rCurrent.add(r.toString());
-            }
-            for (VirtualNetworkRoute r : cfg.routes) {
                 rNew.add(r.toString());
             }
-            Collections.sort(rCurrent);
+            for (VirtualNetworkRoute r : old.routes) {
+                rOld.add(r.toString());
+            }
             Collections.sort(rNew);
+            Collections.sort(rOld);
 
             Log.i(TAG, "Managed Routes Changed");
-            Log.i(TAG, "Old:");
+            Log.i(TAG, "New:");
-            for (String s : rCurrent) {
+            for (String s : rNew) {
                 Log.i(TAG, "    " + s);
             }
             Log.i(TAG, "");
-            Log.i(TAG, "New:");
+            Log.i(TAG, "Old:");
-            for (String s : rNew) {
+            for (String s : rOld) {
                 Log.i(TAG, "    " + s);
             }
             Log.i(TAG, "");
@@ -239,20 +247,22 @@ public class VirtualNetworkConfig implements Comparable<VirtualNetworkConfig> {
         boolean dnsEquals;
         if (this.dns == null) {
             //noinspection RedundantIfStatement
-            if (cfg.dns == null) {
+            if (old.dns == null) {
                 dnsEquals = true;
             } else {
                 dnsEquals = false;
             }
         } else {
-            if (cfg.dns == null) {
+            if (old.dns == null) {
                 dnsEquals = false;
             } else {
-                dnsEquals = this.dns.equals(cfg.dns);
+                dnsEquals = this.dns.equals(old.dns);
             }
         }
 
         if (!dnsEquals) {
+            Log.i(TAG, "DNS Changed. New: " + this.dns + ", Old: " + old.dns);
+
             return false;
         }
 
@@ -374,11 +384,11 @@ public class VirtualNetworkConfig implements Comparable<VirtualNetworkConfig> {
 
     /**
      * ZeroTier-assigned addresses (in {@link InetSocketAddress} objects)
-     *
+     * <p>
      * For IP, the port number of the sockaddr_XX structure contains the number
      * of bits in the address netmask. Only the IP address and port are used.
      * Other fields like interface number can be ignored.
-     *
+     * <p>
      * This is only used for ZeroTier-managed address assignments sent by the
      * virtual network's configuration master.
      */

java/src/com/zerotier/sdk/VirtualNetworkConfigListener.java

@@ -36,14 +36,14 @@ public interface VirtualNetworkConfigListener {
      * <p>This can be called at any time to update the configuration of a virtual
      * network port. The parameter after the network ID specifies whether this
      * port is being brought up, updated, brought down, or permanently deleted.
-     *
+     * <p>
      * This in turn should be used by the underlying implementation to create
      * and configure tap devices at the OS (or virtual network stack) layer.</P>
      *
      * This should not call {@link Node#multicastSubscribe(long, long)} or other network-modifying
      * methods, as this could cause a deadlock in multithreaded or interrupt
      * driven environments.
-     *
+     * <p>
      * This must return 0 on success. It can return any OS-dependent error code
      * on failure, and this results in the network being placed into the
      * PORT_ERROR state.

java/src/com/zerotier/sdk/VirtualNetworkConfigOperation.java

@@ -29,7 +29,7 @@ package com.zerotier.sdk;
 
 /**
  * Virtual network configuration update type
- *
+ * <p>
  * Defined in ZeroTierOne.h as ZT_VirtualNetworkConfigOperation
  */
 public enum VirtualNetworkConfigOperation {

java/src/com/zerotier/sdk/VirtualNetworkDNS.java

@@ -10,7 +10,7 @@ import java.util.ArrayList;
 
 /**
  * DNS configuration to be pushed on a virtual network
- *
+ * <p>
  * Defined in ZeroTierOne.h as ZT_VirtualNetworkDNS
  */
 public class VirtualNetworkDNS implements Comparable<VirtualNetworkDNS> {
@@ -45,6 +45,7 @@ public class VirtualNetworkDNS implements Comparable<VirtualNetworkDNS> {
             return false;
         }
 
+        //noinspection RedundantIfStatement
         if (!servers.equals(d.servers)) {
             return false;
         }

java/src/com/zerotier/sdk/VirtualNetworkRoute.java

@@ -31,7 +31,7 @@ import java.net.InetSocketAddress;
 
 /**
  * A route to be pushed on a virtual network
- *
+ * <p>
  * Defined in ZeroTierOne.h as ZT_VirtualNetworkRoute
  */
 public class VirtualNetworkRoute implements Comparable<VirtualNetworkRoute>
@@ -126,6 +126,7 @@ public class VirtualNetworkRoute implements Comparable<VirtualNetworkRoute>
             return false;
         }
 
+        //noinspection RedundantIfStatement
         if (metric != other.metric) {
             return false;
         }

java/src/com/zerotier/sdk/VirtualNetworkStatus.java

@@ -29,7 +29,7 @@ package com.zerotier.sdk;
 
 /**
  * Virtual network status codes
- *
+ * <p>
  * Defined in ZeroTierOne.h as ZT_VirtualNetworkStatus
  */
 public enum VirtualNetworkStatus {

java/src/com/zerotier/sdk/VirtualNetworkType.java

@@ -29,7 +29,7 @@ package com.zerotier.sdk;
 
 /**
  * Virtual network type codes
- *
+ * <p>
  * Defined in ZeroTierOne.h as ZT_VirtualNetworkType
  */
 public enum VirtualNetworkType {

node/Bond.cpp

@@ -102,6 +102,43 @@ SharedPtr<Bond> Bond::getBondByPeerId(int64_t identity)
 	return _bonds.count(identity) ? _bonds[identity] : SharedPtr<Bond>();
 }
 
+bool Bond::setAllMtuByTuple(uint16_t mtu, const std::string& ifStr, const std::string& ipStr)
+{
+	Mutex::Lock _l(_bonds_m);
+	std::map<int64_t, SharedPtr<Bond> >::iterator bondItr = _bonds.begin();
+	bool found = false;
+	while (bondItr != _bonds.end()) {
+		if (bondItr->second->setMtuByTuple(mtu,ifStr,ipStr)) {
+			found = true;
+		}
+		++bondItr;
+	}
+	return found;
+}
+
+bool Bond::setMtuByTuple(uint16_t mtu, const std::string& ifStr, const std::string& ipStr)
+{
+	Mutex::Lock _lp(_paths_m);
+	bool found = false;
+	for (int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS; ++i) {
+		if (_paths[i].p) {
+			SharedPtr<Link> sl = getLink(_paths[i].p);
+			if (sl) {
+				if (sl->ifname() == ifStr) {
+					char ipBuf[64] = { 0 };
+					_paths[i].p->address().toIpString(ipBuf);
+					std::string newString = std::string(ipBuf);
+					if (newString == ipStr) {
+						_paths[i].p->_mtu = mtu;
+						found = true;
+					}
+				}
+			}
+		}
+	}
+	return found;
+}
+
 SharedPtr<Bond> Bond::createBond(const RuntimeEnvironment* renv, const SharedPtr<Peer>& peer)
 {
 	Mutex::Lock _l(_bonds_m);
@@ -162,8 +199,8 @@ void Bond::destroyBond(uint64_t peerId)
 	auto iter = _bonds.find(peerId);
 	if (iter != _bonds.end()) {
 		iter->second->stopBond();
+		_bonds.erase(iter);
 	}
-	_bonds.erase(peerId);
 }
 
 void Bond::stopBond()
@@ -978,7 +1015,7 @@ void Bond::curateBond(int64_t now, bool rebuildBond)
 		// Whether we've waited long enough since the link last came online
 		bool satisfiedUpDelay = (now - _paths[i].lastAliveToggle) >= _upDelay;
 		// How long since the last QoS was received (Must be less than ZT_PEER_PATH_EXPIRATION since the remote peer's _qosSendInterval isn't known)
-		bool acceptableQoSAge = _paths[i].lastQoSReceived == 0 || ((now - _paths[i].lastQoSReceived) < ZT_PEER_EXPIRED_PATH_TRIAL_PERIOD);
+		bool acceptableQoSAge = (_paths[i].lastQoSReceived == 0 && inTrial) || ((now - _paths[i].lastQoSReceived) < ZT_PEER_EXPIRED_PATH_TRIAL_PERIOD);
 		currEligibility = _paths[i].allowed() && ((acceptableAge && satisfiedUpDelay && acceptableQoSAge) || inTrial);
 
 		if (currEligibility) {
@@ -1070,7 +1107,7 @@ void Bond::curateBond(int64_t now, bool rebuildBond)
 
 				// Bond a spare link if required (no viable primary links left)
 				if (! foundUsablePrimaryPath) {
-					debug("no usable primary links remain, will attempt to use spare if available");
+					// debug("no usable primary links remain, will attempt to use spare if available");
 					for (int j = 0; j < it->second.size(); j++) {
 						int idx = it->second.at(j);
 						if (! _paths[idx].p || ! _paths[idx].eligible || ! _paths[idx].allowed() || ! _paths[idx].isSpare()) {
@@ -1244,7 +1281,8 @@ void Bond::estimatePathQuality(int64_t now)
 			if (link) {
 				int linkSpeed = link->capacity();
 				_paths[i].p->_givenLinkSpeed = linkSpeed;
-				_paths[i].p->_mtu = link->mtu();
+				_paths[i].p->_mtu = link->mtu() ? link->mtu() : _paths[i].p->_mtu;
+				_paths[i].p->_assignedFlowCount = _paths[i].assignedFlowCount;
 				maxObservedLinkCap = linkSpeed > maxObservedLinkCap ? linkSpeed : maxObservedLinkCap;
 			}
 		}

node/Bond.hpp

@@ -456,6 +456,26 @@ class Bond {
 	 */
 	static SharedPtr<Bond> getBondByPeerId(int64_t identity);
 
+	/**
+	 * Set MTU for link by given interface name and IP address (across all bonds)
+	 *
+	 * @param mtu MTU to be used on this link
+	 * @param ifStr interface name to match
+	 * @param ipStr IP address to match
+	 * @return Whether the MTU was set
+	 */
+	static bool setAllMtuByTuple(uint16_t mtu, const std::string& ifStr, const std::string& ipStr);
+
+	/**
+	 * Set MTU for link by given interface name and IP address
+	 *
+	 * @param mtu MTU to be used on this link
+	 * @param ifStr interface name to match
+	 * @param ipStr IP address to match
+	 * @return Whether the MTU was set
+	 */
+	bool setMtuByTuple(uint16_t mtu, const std::string& ifStr, const std::string& ipStr);
+
 	/**
 	 * Add a new bond to the bond controller.
 	 *

node/IncomingPacket.cpp

@@ -317,8 +317,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
 bool IncomingPacket::_doACK(const RuntimeEnvironment* RR, void* tPtr, const SharedPtr<Peer>& peer)
 {
 	/*
-	SharedPtr<Bond> bond = peer->bond();
+	if (! peer->rateGateACK(RR->node->now())) {
-	if (! bond || ! bond->rateGateACK(RR->node->now())) {
 		return true;
 	}
 	int32_t ackedBytes;
@@ -326,9 +325,7 @@ bool IncomingPacket::_doACK(const RuntimeEnvironment* RR, void* tPtr, const Shar
 		return true;   // ignore
 	}
 	memcpy(&ackedBytes, payload(), sizeof(ackedBytes));
-	if (bond) {
+	peer->receivedAck(_path, RR->node->now(), Utils::ntoh(ackedBytes));
-		bond->receivedAck(_path, RR->node->now(), Utils::ntoh(ackedBytes));
-	}
 	*/
 	Metrics::pkt_ack_in++;
 	return true;
@@ -338,7 +335,7 @@ bool IncomingPacket::_doQOS_MEASUREMENT(const RuntimeEnvironment* RR, void* tPtr
 {
 	Metrics::pkt_qos_in++;
 	SharedPtr<Bond> bond = peer->bond();
-	if (! bond || ! bond->rateGateQoS(RR->node->now(), _path)) {
+	if (! peer->rateGateQoS(RR->node->now(), _path)) {
 		return true;
 	}
 	if (payloadLength() > ZT_QOS_MAX_PACKET_SIZE || payloadLength() < ZT_QOS_MIN_PACKET_SIZE) {
@@ -359,9 +356,7 @@ bool IncomingPacket::_doQOS_MEASUREMENT(const RuntimeEnvironment* RR, void* tPtr
 		ptr += sizeof(uint16_t);
 		count++;
 	}
-	if (bond) {
+	peer->receivedQoS(_path, now, count, rx_id, rx_ts);
-		bond->receivedQoS(_path, now, count, rx_id, rx_ts);
-	}
 	return true;
 }
 
@@ -626,11 +621,8 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP
 			}
 
 			if (!hops()) {
-				SharedPtr<Bond> bond = peer->bond();
-				if (!bond) {
 				_path->updateLatency((unsigned int)latency,RR->node->now());
 			}
-			}
 
 			peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision);
 
@@ -801,8 +793,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const Shar
 {
 	Metrics::pkt_frame_in++;
 	int32_t _flowId = ZT_QOS_NO_FLOW;
-	SharedPtr<Bond> bond = peer->bond();
+	if (peer->flowHashingSupported()) {
-	if (bond && bond->flowHashingSupported()) {
 		if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) {
 			const unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE);
 			const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
@@ -1481,8 +1472,7 @@ bool IncomingPacket::_doPATH_NEGOTIATION_REQUEST(const RuntimeEnvironment *RR,vo
 {
 	Metrics::pkt_path_negotiation_request_in++;
 	uint64_t now = RR->node->now();
-	SharedPtr<Bond> bond = peer->bond();
+	if (!peer->rateGatePathNegotiation(now, _path)) {
-	if (!bond || !bond->rateGatePathNegotiation(now, _path)) {
 		return true;
 	}
 	if (payloadLength() != sizeof(int16_t)) {
@@ -1490,9 +1480,7 @@ bool IncomingPacket::_doPATH_NEGOTIATION_REQUEST(const RuntimeEnvironment *RR,vo
 	}
 	int16_t remoteUtility = 0;
 	memcpy(&remoteUtility, payload(), sizeof(int16_t));
-	if (peer->bond()) {
+	peer->processIncomingPathNegotiationRequest(now, _path, Utils::ntoh(remoteUtility));
-		peer->bond()->processIncomingPathNegotiationRequest(now, _path, Utils::ntoh(remoteUtility));
-	}
 	return true;
 }
 

node/Node.cpp

@@ -589,6 +589,7 @@ ZT_PeerList *Node::peers() const
 					p->paths[p->pathCount].latencyVariance = (*path)->latencyVariance();
 					p->paths[p->pathCount].packetLossRatio = (*path)->packetLossRatio();
 					p->paths[p->pathCount].packetErrorRatio = (*path)->packetErrorRatio();
+					p->paths[p->pathCount].assignedFlowCount = (*path)->assignedFlowCount();
 					p->paths[p->pathCount].relativeQuality = (*path)->relativeQuality();
 					p->paths[p->pathCount].linkSpeed = (*path)->givenLinkSpeed();
 					p->paths[p->pathCount].bonded = (*path)->bonded();
@@ -602,9 +603,9 @@ ZT_PeerList *Node::peers() const
 		}
 		if (pi->second->bond()) {
 			p->isBonded = pi->second->bond();
-			p->bondingPolicy = pi->second->bond()->policy();
+			p->bondingPolicy = pi->second->bondingPolicy();
-			p->numAliveLinks = pi->second->bond()->getNumAliveLinks();
+			p->numAliveLinks = pi->second->getNumAliveLinks();
-			p->numTotalLinks = pi->second->bond()->getNumTotalLinks();
+			p->numTotalLinks = pi->second->getNumTotalLinks();
 		}
 	}
 

node/Path.hpp

@@ -89,6 +89,7 @@ public:
 		_latencyVariance(0.0),
 		_packetLossRatio(0.0),
 		_packetErrorRatio(0.0),
+		_assignedFlowCount(0),
 		_valid(true),
 		_eligible(false),
 		_bonded(false),
@@ -110,6 +111,7 @@ public:
 		_latencyVariance(0.0),
 		_packetLossRatio(0.0),
 		_packetErrorRatio(0.0),
+		_assignedFlowCount(0),
 		_valid(true),
 		_eligible(false),
 		_bonded(false),
@@ -320,6 +322,11 @@ public:
 	 */
 	inline float packetErrorRatio() const { return _packetErrorRatio; }
 
+	/**
+	 * @return Number of flows assigned to this path
+	 */
+	inline unsigned int assignedFlowCount() const { return _assignedFlowCount; }
+
 	/**
 	 * @return Whether this path is valid as reported by the bonding layer. The bonding layer
 	 * actually checks with Phy to see if the interface is still up
@@ -374,6 +381,7 @@ private:
 	volatile float _latencyVariance;
 	volatile float _packetLossRatio;
 	volatile float _packetErrorRatio;
+	volatile uint16_t _assignedFlowCount;
 	volatile bool _valid;
 	volatile bool _eligible;
 	volatile bool _bonded;

node/Peer.cpp

@@ -487,19 +487,28 @@ void Peer::tryMemorizedPath(void *tPtr,int64_t now)
 void Peer::performMultipathStateCheck(void *tPtr, int64_t now)
 {
 	Mutex::Lock _l(_bond_m);
-	if (_bond) {
-		// Once enabled the Bond object persists, no need to update state
-		return;
-	}
 	/**
 	 * Check for conditions required for multipath bonding and create a bond
 	 * if allowed.
 	 */
 	int numAlivePaths = 0;
+	bool atLeastOneNonExpired = false;
 	for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
-		if (_paths[i].p && _paths[i].p->alive(now)) {
+		if (_paths[i].p) {
+			if(_paths[i].p->alive(now)) {
 				numAlivePaths++;
 			}
+			if ((now - _paths[i].lr) < ZT_PEER_PATH_EXPIRATION) {
+				atLeastOneNonExpired = true;
+			}
+		}
+	}
+	if (_bond) {
+		if (numAlivePaths == 0 && !atLeastOneNonExpired) {
+			_bond = SharedPtr<Bond>();
+			RR->bc->destroyBond(_id.address().toInt());
+		}
+		return;
 	}
 	_localMultipathSupported = ((numAlivePaths >= 1) && (RR->bc->inUse()) && (ZT_PROTO_VERSION > 9));
 	if (_localMultipathSupported && !_bond) {

node/Peer.hpp

@@ -56,7 +56,6 @@ private:
 public:
 	~Peer() {
 		Utils::burn(_key,sizeof(_key));
-		RR->bc->destroyBond(_id.address().toInt());
 	}
 
 	/**
@@ -434,6 +433,64 @@ public:
 		return false;
 	}
 
+	/**
+	 * See definition in Bond
+	 */
+	inline bool rateGateQoS(int64_t now, SharedPtr<Path>& path)
+	{
+		Mutex::Lock _l(_bond_m);
+		if(_bond) {
+			return _bond->rateGateQoS(now, path);
+		}
+		return false; // Default behavior. If there is no bond, we drop these
+	}
+
+	/**
+	 * See definition in Bond
+	 */
+	void receivedQoS(const SharedPtr<Path>& path, int64_t now, int count, uint64_t* rx_id, uint16_t* rx_ts)
+	{
+		Mutex::Lock _l(_bond_m);
+		if(_bond) {
+			_bond->receivedQoS(path, now, count, rx_id, rx_ts);
+		}
+	}
+
+	/**
+	 * See definition in Bond
+	 */
+	void processIncomingPathNegotiationRequest(uint64_t now, SharedPtr<Path>& path, int16_t remoteUtility)
+	{
+		Mutex::Lock _l(_bond_m);
+		if(_bond) {
+			_bond->processIncomingPathNegotiationRequest(now, path, remoteUtility);
+		}
+	}
+
+	/**
+	 * See definition in Bond
+	 */
+	inline bool rateGatePathNegotiation(int64_t now, SharedPtr<Path>& path)
+	{
+		Mutex::Lock _l(_bond_m);
+		if(_bond) {
+			return _bond->rateGatePathNegotiation(now, path);
+		}
+		return false; // Default behavior. If there is no bond, we drop these
+	}
+
+	/**
+	 * See definition in Bond
+	 */
+	bool flowHashingSupported()
+	{
+		Mutex::Lock _l(_bond_m);
+		if(_bond) {
+			return _bond->flowHashingSupported();
+		}
+		return false;
+	}
+
 	/**
 	 * Serialize a peer for storage in local cache
 	 *
@@ -533,6 +590,28 @@ public:
 		return ZT_BOND_POLICY_NONE;
 	}
 
+	/**
+	 * @return the number of links in this bond which are considered alive
+	 */
+	inline uint8_t getNumAliveLinks() {
+		Mutex::Lock _l(_paths_m);
+		if (_bond) {
+			return _bond->getNumAliveLinks();
+		}
+		return 0;
+	}
+
+	/**
+	 * @return the number of links in this bond
+	 */
+	inline uint8_t getNumTotalLinks() {
+		Mutex::Lock _l(_paths_m);
+		if (_bond) {
+			return _bond->getNumTotalLinks();
+		}
+		return 0;
+	}
+
 	//inline const AES *aesKeysIfSupported() const
 	//{ return (const AES *)0; }
 

one.cpp

@@ -171,7 +171,7 @@ static int cli(int argc,char **argv)
 #endif
 {
 	unsigned int port = 0;
-	std::string homeDir,command,arg1,arg2,authToken;
+	std::string homeDir,command,arg1,arg2,arg3,arg4,authToken;
 	std::string ip("127.0.0.1");
 	bool json = false;
 	for(int i=1;i<argc;++i) {
@@ -569,9 +569,35 @@ static int cli(int argc,char **argv)
 				return 1;
 			}
 		}
+		else if (arg1 == "setmtu") { /* zerotier-cli bond setmtu <mtu> <iface> <ip> */
+			requestHeaders["Content-Type"] = "application/json";
+			requestHeaders["Content-Length"] = "2";
+			if (argc == 8) {
+				arg2 = argv[5];
+				arg3 = argv[6];
+				arg4 = argv[7];
+			}
+			unsigned int scode = Http::POST(
+				1024 * 1024 * 16,
+				60000,
+				(const struct sockaddr *)&addr,
+				(std::string("/bond/") + arg1 + "/" + arg2 + "/" + arg3 + "/" + arg4).c_str(),
+				requestHeaders,
+				"{}",
+				2,
+				responseHeaders,
+				responseBody);
+			if (scode == 200) {
+				printf("200 setmtu OK" ZT_EOL_S);
+				return 0;
+			} else {
+				printf("no link match found, new MTU was not applied" ZT_EOL_S);
+				return 1;
+			}
+			return 0;
+		}
 		else if (arg1.length() == 10) {
 			if (arg2 == "rotate") { /* zerotier-cli bond <peerId> rotate */
-				fprintf(stderr, "zerotier-cli bond <peerId> rotate\n");
 				requestHeaders["Content-Type"] = "application/json";
 				requestHeaders["Content-Length"] = "2";
 				unsigned int scode = Http::POST(
@@ -588,7 +614,7 @@ static int cli(int argc,char **argv)
 					if (json) {
 						printf("%s",cliFixJsonCRs(responseBody).c_str());
 					} else {
-						printf("200 bond OK" ZT_EOL_S);
+						printf("200 rotate OK" ZT_EOL_S);
 					}
 					return 0;
 				} else {

service/OneService.cpp

@@ -644,6 +644,7 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer, SharedPtr<Bond>
 			j["latencyVariance"] = peer->paths[i].latencyVariance;
 			j["packetLossRatio"] = peer->paths[i].packetLossRatio;
 			j["packetErrorRatio"] = peer->paths[i].packetErrorRatio;
+			j["assignedFlowCount"] = peer->paths[i].assignedFlowCount;
 			j["lastInAge"] = (now - lastReceive);
 			j["lastOutAge"] = (now - lastSend);
 			j["bonded"] = peer->paths[i].bonded;
@@ -1659,11 +1660,8 @@ public:
 				res.status = 400;
 				return;
 			}
-
 			auto bondID = req.matches[1];
 			uint64_t id = Utils::hexStrToU64(bondID.str().c_str());
-
-			exit(0);
 			SharedPtr<Bond> bond = _node->bondController()->getBondByPeerId(id);
 			if (bond) {
 				if (bond->abForciblyRotateLink()) {
@@ -1680,6 +1678,19 @@ public:
 		_controlPlane.Post("/bond/rotate/([0-9a-fA-F]{10})", bondRotate);
 		_controlPlane.Put("/bond/rotate/([0-9a-fA-F]{10})", bondRotate);
 
+		auto setMtu = [&, setContent](const httplib::Request &req, httplib::Response &res) {
+			if (!_node->bondController()->inUse()) {
+				setContent(req, res, "");
+				res.status = 400;
+				return;
+			}
+			uint16_t mtu = atoi(req.matches[1].str().c_str());
+			res.status = _node->bondController()->setAllMtuByTuple(mtu, req.matches[2].str().c_str(), req.matches[3].str().c_str()) ? 200 : 400;
+			setContent(req, res, "{}");
+		};
+		_controlPlane.Post("/bond/setmtu/([0-9]{3,5})/([a-zA-Z0-9_]{1,16})/([0-9a-fA-F\\.\\:]{1,39})", setMtu);
+		_controlPlane.Put("/bond/setmtu/([0-9]{3,5})/([a-zA-Z0-9_]{1,16})/([0-9a-fA-F\\.\\:]{1,39})", setMtu);
+
 		_controlPlane.Get("/config", [&, setContent](const httplib::Request &req, httplib::Response &res) {
 			std::string config;
 			{
@@ -2750,46 +2761,8 @@ public:
 			TcpConnection *tc = reinterpret_cast<TcpConnection *>(*uptr);
 			tc->lastReceive = OSUtils::now();
 			switch(tc->type) {
-
+				case TcpConnection::TCP_UNCATEGORIZED_INCOMING:
-                // TODO: Remove Me
+					return;
-				// case TcpConnection::TCP_UNCATEGORIZED_INCOMING:
-				// 	switch(reinterpret_cast<uint8_t *>(data)[0]) {
-				// 		// HTTP: GET, PUT, POST, HEAD, DELETE
-				// 		case 'G':
-				// 		case 'P':
-				// 		case 'D':
-				// 		case 'H': {
-				// 			// This is only allowed from IPs permitted to access the management
-				// 			// backplane, which is just 127.0.0.1/::1 unless otherwise configured.
-				// 			bool allow;
-				// 			{
-				// 				Mutex::Lock _l(_localConfig_m);
-				// 				if (_allowManagementFrom.empty()) {
-				// 					allow = (tc->remoteAddr.ipScope() == InetAddress::IP_SCOPE_LOOPBACK);
-				// 				} else {
-				// 					allow = false;
-				// 					for(std::vector<InetAddress>::const_iterator i(_allowManagementFrom.begin());i!=_allowManagementFrom.end();++i) {
-				// 						if (i->containsAddress(tc->remoteAddr)) {
-				// 							allow = true;
-				// 							break;
-				// 						}
-				// 					}
-				// 				}
-				// 			}
-				// 			if (allow) {
-				// 				tc->type = TcpConnection::TCP_HTTP_INCOMING;
-				// 				phyOnTcpData(sock,uptr,data,len);
-				// 			} else {
-				// 				_phy.close(sock);
-				// 			}
-				// 		}	break;
-
-				// 		// Drop unknown protocols
-				// 		default:
-				// 			_phy.close(sock);
-				// 			break;
-				// 	}
-				// 	return;
 
 				case TcpConnection::TCP_HTTP_INCOMING:
 				case TcpConnection::TCP_HTTP_OUTGOING: