Replace long callback arg list with struct, and implement path whitelisting, path blacklisting, and local.conf support for roles.

include/ZeroTierOne.h

@@ -1495,8 +1495,9 @@ typedef int (*ZT_WirePacketSendFunction)(
  * Paramters:
  *  (1) Node
  *  (2) User pointer
- *  (3) Local interface address
+ *  (3) ZeroTier address or 0 for none/any
- *  (4) Remote address
+ *  (4) Local interface address
+ *  (5) Remote address
  *
  * This function must return nonzero (true) if the path should be used.
  *
@@ -1515,13 +1516,87 @@ typedef int (*ZT_WirePacketSendFunction)(
 typedef int (*ZT_PathCheckFunction)(
 	ZT_Node *,                        /* Node */
 	void *,                           /* User ptr */
+	uint64_t,                         /* ZeroTier address */
 	const struct sockaddr_storage *,  /* Local address */
 	const struct sockaddr_storage *); /* Remote address */
 
+/**
+ * Function to get physical addresses for ZeroTier peers
+ *
+ * Parameters:
+ *  (1) Node
+ *  (2) User pointer
+ *  (3) ZeroTier address (least significant 40 bits)
+ *  (4) Desried address family or -1 for any
+ *  (5) Buffer to fill with result
+ *
+ * If provided this function will be occasionally called to get physical
+ * addresses that might be tried to reach a ZeroTier address. It must
+ * return a nonzero (true) value if the result buffer has been filled
+ * with an address.
+ */
+typedef int (*ZT_PathLookupFunction)(
+	ZT_Node *,                        /* Node */
+	void *,                           /* User ptr */
+	uint64_t,                         /* ZeroTier address (40 bits) */
+	int,                              /* Desired ss_family or -1 for any */
+	struct sockaddr_storage *);       /* Result buffer */
+
 /****************************************************************************/
 /* C Node API                                                               */
 /****************************************************************************/
 
+/**
+ * Structure for configuring ZeroTier core callback functions
+ */
+struct ZT_Node_Callbacks
+{
+	/**
+	 * Struct version -- must currently be 0
+	 */
+	long version;
+
+	/**
+	 * REQUIRED: Function to get objects from persistent storage
+	 */
+	ZT_DataStoreGetFunction dataStoreGetFunction;
+
+	/**
+	 * REQUIRED: Function to store objects in persistent storage
+	 */
+	ZT_DataStorePutFunction dataStorePutFunction;
+
+	/**
+	 * REQUIRED: Function to send packets over the physical wire
+	 */
+	ZT_WirePacketSendFunction wirePacketSendFunction;
+
+	/**
+	 * REQUIRED: Function to inject frames into a virtual network's TAP
+	 */
+	ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction;
+
+	/**
+	 * REQUIRED: Function to be called when virtual networks are configured or changed
+	 */
+	ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction;
+
+	/**
+	 * REQUIRED: Function to be called to notify external code of important events
+	 */
+	ZT_EventCallback eventCallback;
+
+	/**
+	 * OPTIONAL: Function to check whether a given physical path should be used
+	 */
+	ZT_PathCheckFunction pathCheckFunction;
+
+	/**
+	 * OPTIONAL: Function to get hints to physical paths to ZeroTier addresses
+	 */
+	ZT_PathLookupFunction pathLookupFunction;
+};
+
 /**
  * Create a new ZeroTier One node
  *
@@ -1533,25 +1608,11 @@ typedef int (*ZT_PathCheckFunction)(
  *
  * @param node Result: pointer is set to new node instance on success
  * @param uptr User pointer to pass to functions/callbacks
+ * @param callbacks Callback function configuration
  * @param now Current clock in milliseconds
- * @param dataStoreGetFunction Function called to get objects from persistent storage
- * @param dataStorePutFunction Function called to put objects in persistent storage
- * @param virtualNetworkConfigFunction Function to be called when virtual LANs are created, deleted, or their config parameters change
- * @param pathCheckFunction A function to check whether a path should be used for ZeroTier traffic, or NULL to allow any path
- * @param eventCallback Function to receive status updates and non-fatal error notices
  * @return OK (0) or error code if a fatal error condition has occurred
  */
-enum ZT_ResultCode ZT_Node_new(
+enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now);
-	ZT_Node **node,
-	void *uptr,
-	uint64_t now,
-	ZT_DataStoreGetFunction dataStoreGetFunction,
-	ZT_DataStorePutFunction dataStorePutFunction,
-	ZT_WirePacketSendFunction wirePacketSendFunction,
-	ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
-	ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-	ZT_PathCheckFunction pathCheckFunction,
-	ZT_EventCallback eventCallback);
 
 /**
  * Delete a node and free all resources it consumes

node/IncomingPacket.cpp

@@ -552,7 +552,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr<
 				const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN];
 				if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) {
 					const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port);
-					if (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),atAddr)) {
+					if (RR->node->shouldUsePathForZeroTierTraffic(with,_path->localAddress(),atAddr)) {
 						RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls
 						rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now());
 						TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
@@ -1120,7 +1120,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 						redundant = peer->hasActivePathTo(now,a);
 					}
 
-					if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),a)) ) {
+					if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(peer->address(),_path->localAddress(),a)) ) {
 						if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
 							TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
 							peer->attemptToContactAt(InetAddress(),a,now);
@@ -1139,7 +1139,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 						redundant = peer->hasActivePathTo(now,a);
 					}
 
-					if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),a)) ) {
+					if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(peer->address(),_path->localAddress(),a)) ) {
 						if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
 							TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
 							peer->attemptToContactAt(InetAddress(),a,now);

node/Node.cpp

@@ -46,34 +46,20 @@ namespace ZeroTier {
 /* Public Node interface (C++, exposed via CAPI bindings)                   */
 /****************************************************************************/
 
-Node::Node(
+Node::Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now) :
-	uint64_t now,
-	void *uptr,
-	ZT_DataStoreGetFunction dataStoreGetFunction,
-	ZT_DataStorePutFunction dataStorePutFunction,
-	ZT_WirePacketSendFunction wirePacketSendFunction,
-	ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
-	ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-	ZT_PathCheckFunction pathCheckFunction,
-	ZT_EventCallback eventCallback) :
 	_RR(this),
 	RR(&_RR),
 	_uPtr(uptr),
-	_dataStoreGetFunction(dataStoreGetFunction),
-	_dataStorePutFunction(dataStorePutFunction),
-	_wirePacketSendFunction(wirePacketSendFunction),
-	_virtualNetworkFrameFunction(virtualNetworkFrameFunction),
-	_virtualNetworkConfigFunction(virtualNetworkConfigFunction),
-	_pathCheckFunction(pathCheckFunction),
-	_eventCallback(eventCallback),
-	_networks(),
-	_networks_m(),
 	_prngStreamPtr(0),
 	_now(now),
 	_lastPingCheck(0),
 	_lastHousekeepingRun(0),
 	_relayPolicy(ZT_RELAY_POLICY_TRUSTED)
 {
+	if (callbacks->version != 0)
+		throw std::runtime_error("callbacks struct version mismatch");
+	memcpy(&_cb,callbacks,sizeof(ZT_Node_Callbacks));
+
 	_online = false;
 
 	memset(_expectingRepliesToBucketPtr,0,sizeof(_expectingRepliesToBucketPtr));
@@ -81,30 +67,26 @@ Node::Node(
 	memset(_lastIdentityVerification,0,sizeof(_lastIdentityVerification));
 
 	// Use Salsa20 alone as a high-quality non-crypto PRNG
-	{
+	char foo[32];
-		char foo[32];
+	Utils::getSecureRandom(foo,32);
-		Utils::getSecureRandom(foo,32);
+	_prng.init(foo,256,foo);
-		_prng.init(foo,256,foo);
+	memset(_prngStream,0,sizeof(_prngStream));
-		memset(_prngStream,0,sizeof(_prngStream));
+	_prng.encrypt12(_prngStream,_prngStream,sizeof(_prngStream));
-		_prng.encrypt12(_prngStream,_prngStream,sizeof(_prngStream));
-	}
 
-	{
+	std::string idtmp(dataStoreGet("identity.secret"));
-		std::string idtmp(dataStoreGet("identity.secret"));
+	if ((!idtmp.length())||(!RR->identity.fromString(idtmp))||(!RR->identity.hasPrivate())) {
-		if ((!idtmp.length())||(!RR->identity.fromString(idtmp))||(!RR->identity.hasPrivate())) {
+		TRACE("identity.secret not found, generating...");
-			TRACE("identity.secret not found, generating...");
+		RR->identity.generate();
-			RR->identity.generate();
+		idtmp = RR->identity.toString(true);
-			idtmp = RR->identity.toString(true);
+		if (!dataStorePut("identity.secret",idtmp,true))
-			if (!dataStorePut("identity.secret",idtmp,true))
+			throw std::runtime_error("unable to write identity.secret");
-				throw std::runtime_error("unable to write identity.secret");
+	}
-		}
+	RR->publicIdentityStr = RR->identity.toString(false);
-		RR->publicIdentityStr = RR->identity.toString(false);
+	RR->secretIdentityStr = RR->identity.toString(true);
-		RR->secretIdentityStr = RR->identity.toString(true);
+	idtmp = dataStoreGet("identity.public");
-		idtmp = dataStoreGet("identity.public");
+	if (idtmp != RR->publicIdentityStr) {
-		if (idtmp != RR->publicIdentityStr) {
+		if (!dataStorePut("identity.public",RR->publicIdentityStr,false))
-			if (!dataStorePut("identity.public",RR->publicIdentityStr,false))
+			throw std::runtime_error("unable to write identity.public");
-				throw std::runtime_error("unable to write identity.public");
-		}
 	}
 
 	try {
@@ -638,7 +620,7 @@ std::string Node::dataStoreGet(const char *name)
 	std::string r;
 	unsigned long olen = 0;
 	do {
-		long n = _dataStoreGetFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,name,buf,sizeof(buf),(unsigned long)r.length(),&olen);
+		long n = _cb.dataStoreGetFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,name,buf,sizeof(buf),(unsigned long)r.length(),&olen);
 		if (n <= 0)
 			return std::string();
 		r.append(buf,n);
@@ -646,7 +628,7 @@ std::string Node::dataStoreGet(const char *name)
 	return r;
 }
 
-bool Node::shouldUsePathForZeroTierTraffic(const InetAddress &localAddress,const InetAddress &remoteAddress)
+bool Node::shouldUsePathForZeroTierTraffic(const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress)
 {
 	if (!Path::isAddressValidForPath(remoteAddress))
 		return false;
@@ -663,9 +645,7 @@ bool Node::shouldUsePathForZeroTierTraffic(const InetAddress &localAddress,const
 		}
 	}
 
-	if (_pathCheckFunction)
+	return ( (_cb.pathCheckFunction) ? (_cb.pathCheckFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,ztaddr.toInt(),reinterpret_cast<const struct sockaddr_storage *>(&localAddress),reinterpret_cast<const struct sockaddr_storage *>(&remoteAddress)) != 0) : true);
-		return (_pathCheckFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,reinterpret_cast<const struct sockaddr_storage *>(&localAddress),reinterpret_cast<const struct sockaddr_storage *>(&remoteAddress)) != 0);
-	else return true;
 }
 
 #ifdef ZT_TRACE
@@ -822,21 +802,11 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des
 
 extern "C" {
 
-enum ZT_ResultCode ZT_Node_new(
+enum ZT_ResultCode ZT_Node_new(ZT_Node **node,void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now)
-	ZT_Node **node,
-	void *uptr,
-	uint64_t now,
-	ZT_DataStoreGetFunction dataStoreGetFunction,
-	ZT_DataStorePutFunction dataStorePutFunction,
-	ZT_WirePacketSendFunction wirePacketSendFunction,
-	ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
-	ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-	ZT_PathCheckFunction pathCheckFunction,
-	ZT_EventCallback eventCallback)
 {
 	*node = (ZT_Node *)0;
 	try {
-		*node = reinterpret_cast<ZT_Node *>(new ZeroTier::Node(now,uptr,dataStoreGetFunction,dataStorePutFunction,wirePacketSendFunction,virtualNetworkFrameFunction,virtualNetworkConfigFunction,pathCheckFunction,eventCallback));
+		*node = reinterpret_cast<ZT_Node *>(new ZeroTier::Node(uptr,callbacks,now));
 		return ZT_RESULT_OK;
 	} catch (std::bad_alloc &exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;

node/Node.hpp

@@ -59,17 +59,7 @@ namespace ZeroTier {
 class Node : public NetworkController::Sender
 {
 public:
-	Node(
+	Node(void *uptr,const struct ZT_Node_Callbacks *callbacks,uint64_t now);
-		uint64_t now,
-		void *uptr,
-		ZT_DataStoreGetFunction dataStoreGetFunction,
-		ZT_DataStorePutFunction dataStorePutFunction,
-		ZT_WirePacketSendFunction wirePacketSendFunction,
-		ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
-		ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-		ZT_PathCheckFunction pathCheckFunction,
-		ZT_EventCallback eventCallback);
-
 	virtual ~Node();
 
 	// Public API Functions ----------------------------------------------------
@@ -127,24 +117,11 @@ public:
 
 	// Internal functions ------------------------------------------------------
 
-	/**
-	 * @return Time as of last call to run()
-	 */
 	inline uint64_t now() const throw() { return _now; }
 
-	/**
-	 * Enqueue a ZeroTier message to be sent
-	 *
-	 * @param localAddress Local address
-	 * @param addr Destination address
-	 * @param data Packet data
-	 * @param len Packet length
-	 * @param ttl Desired TTL (default: 0 for unchanged/default TTL)
-	 * @return True if packet appears to have been sent
-	 */
 	inline bool putPacket(const InetAddress &localAddress,const InetAddress &addr,const void *data,unsigned int len,unsigned int ttl = 0)
 	{
-		return (_wirePacketSendFunction(
+		return (_cb.wirePacketSendFunction(
 			reinterpret_cast<ZT_Node *>(this),
 			_uPtr,
 			reinterpret_cast<const struct sockaddr_storage *>(&localAddress),
@@ -154,21 +131,9 @@ public:
 			ttl) == 0);
 	}
 
-	/**
-	 * Enqueue a frame to be injected into a tap device (port)
-	 *
-	 * @param nwid Network ID
-	 * @param nuptr Network user ptr
-	 * @param source Source MAC
-	 * @param dest Destination MAC
-	 * @param etherType 16-bit ethernet type
-	 * @param vlanId VLAN ID or 0 if none
-	 * @param data Frame data
-	 * @param len Frame length
-	 */
 	inline void putFrame(uint64_t nwid,void **nuptr,const MAC &source,const MAC &dest,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
 	{
-		_virtualNetworkFrameFunction(
+		_cb.virtualNetworkFrameFunction(
 			reinterpret_cast<ZT_Node *>(this),
 			_uPtr,
 			nwid,
@@ -181,13 +146,6 @@ public:
 			len);
 	}
 
-	/**
-	 * @param localAddress Local address
-	 * @param remoteAddress Remote address
-	 * @return True if path should be used
-	 */
-	bool shouldUsePathForZeroTierTraffic(const InetAddress &localAddress,const InetAddress &remoteAddress);
-
 	inline SharedPtr<Network> network(uint64_t nwid) const
 	{
 		Mutex::Lock _l(_networks_m);
@@ -214,37 +172,20 @@ public:
 		return nw;
 	}
 
-	/**
-	 * @return Potential direct paths to me a.k.a. local interface addresses
-	 */
 	inline std::vector<InetAddress> directPaths() const
 	{
 		Mutex::Lock _l(_directPaths_m);
 		return _directPaths;
 	}
 
-	inline bool dataStorePut(const char *name,const void *data,unsigned int len,bool secure) { return (_dataStorePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,name,data,len,(int)secure) == 0); }
+	inline bool dataStorePut(const char *name,const void *data,unsigned int len,bool secure) { return (_cb.dataStorePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,name,data,len,(int)secure) == 0); }
 	inline bool dataStorePut(const char *name,const std::string &data,bool secure) { return dataStorePut(name,(const void *)data.data(),(unsigned int)data.length(),secure); }
-	inline void dataStoreDelete(const char *name) { _dataStorePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,name,(const void *)0,0,0); }
+	inline void dataStoreDelete(const char *name) { _cb.dataStorePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,name,(const void *)0,0,0); }
 	std::string dataStoreGet(const char *name);
 
-	/**
+	inline void postEvent(ZT_Event ev,const void *md = (const void *)0) { _cb.eventCallback(reinterpret_cast<ZT_Node *>(this),_uPtr,ev,md); }
-	 * Post an event to the external user
-	 *
-	 * @param ev Event type
-	 * @param md Meta-data (default: NULL/none)
-	 */
-	inline void postEvent(ZT_Event ev,const void *md = (const void *)0) { _eventCallback(reinterpret_cast<ZT_Node *>(this),_uPtr,ev,md); }
 
-	/**
+	inline int configureVirtualNetworkPort(uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _cb.virtualNetworkConfigFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,nwid,nuptr,op,nc); }
-	 * Update virtual network port configuration
-	 *
-	 * @param nwid Network ID
-	 * @param nuptr Network user ptr
-	 * @param op Configuration operation
-	 * @param nc Network configuration
-	 */
-	inline int configureVirtualNetworkPort(uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _virtualNetworkConfigFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,nwid,nuptr,op,nc); }
 
 	inline bool online() const throw() { return _online; }
 	inline ZT_RelayPolicy relayPolicy() const { return _relayPolicy; }
@@ -253,6 +194,9 @@ public:
 	void postTrace(const char *module,unsigned int line,const char *fmt,...);
 #endif
 
+	bool shouldUsePathForZeroTierTraffic(const Address &ztaddr,const InetAddress &localAddress,const InetAddress &remoteAddress);
+	inline bool getPathHint(const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,ztaddr.toInt(),family,reinterpret_cast<struct sockaddr_storage *>(&addr)) != 0) : false ); }
+
 	uint64_t prng();
 	void postCircuitTestReport(const ZT_CircuitTestReport *report);
 	void setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count);
@@ -317,8 +261,8 @@ private:
 
 	RuntimeEnvironment _RR;
 	RuntimeEnvironment *RR;
-
 	void *_uPtr; // _uptr (lower case) is reserved in Visual Studio :P
+	ZT_Node_Callbacks _cb;
 
 	// For tracking packet IDs to filter out OK/ERROR replies to packets we did not send
 	uint8_t _expectingRepliesToBucketPtr[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1];
@@ -327,14 +271,6 @@ private:
 	// Time of last identity verification indexed by InetAddress.rateGateHash()
 	uint64_t _lastIdentityVerification[16384];
 
-	ZT_DataStoreGetFunction _dataStoreGetFunction;
-	ZT_DataStorePutFunction _dataStorePutFunction;
-	ZT_WirePacketSendFunction _wirePacketSendFunction;
-	ZT_VirtualNetworkFrameFunction _virtualNetworkFrameFunction;
-	ZT_VirtualNetworkConfigFunction _virtualNetworkConfigFunction;
-	ZT_PathCheckFunction _pathCheckFunction;
-	ZT_EventCallback _eventCallback;
-
 	std::vector< std::pair< uint64_t, SharedPtr<Network> > > _networks;
 	Mutex _networks_m;
 

node/Peer.cpp

@@ -160,7 +160,7 @@ void Peer::received(
 			}
 		}
 
-		if ( (!pathIsConfirmed) && (RR->node->shouldUsePathForZeroTierTraffic(path->localAddress(),path->address())) ) {
+		if ( (!pathIsConfirmed) && (RR->node->shouldUsePathForZeroTierTraffic(_id.address(),path->localAddress(),path->address())) ) {
 			if (verb == Packet::VERB_OK) {
 				Mutex::Lock _l(_paths_m);
 

node/Switch.cpp

@@ -85,7 +85,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from
 			Address beaconAddr(reinterpret_cast<const char *>(data) + 8,5);
 			if (beaconAddr == RR->identity.address())
 				return;
-			if (!RR->node->shouldUsePathForZeroTierTraffic(localAddr,fromAddr))
+			if (!RR->node->shouldUsePathForZeroTierTraffic(beaconAddr,localAddr,fromAddr))
 				return;
 			SharedPtr<Peer> peer(RR->topology->getPeer(beaconAddr));
 			if (peer) { // we'll only respond to beacons from known peers

service/OneService.cpp

@@ -160,7 +160,6 @@ static uint64_t _jI(const json &jv,const uint64_t dfl)
 	}
 	return dfl;
 }
-/*
 static bool _jB(const json &jv,const bool dfl)
 {
 	if (jv.is_boolean()) {
@@ -181,7 +180,6 @@ static bool _jB(const json &jv,const bool dfl)
 	}
 	return dfl;
 }
-*/
 static std::string _jS(const json &jv,const char *dfl)
 {
 	if (jv.is_string()) {
@@ -452,7 +450,8 @@ static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,const char *name,
 static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,const char *name,const void *data,unsigned long len,int secure);
 static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl);
 static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
-static int SnodePathCheckFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr);
+static int SnodePathCheckFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr);
+static int SnodePathLookupFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,int family,struct sockaddr_storage *result);
 
 #ifdef ZT_ENABLE_CLUSTER
 static void SclusterSendFunction(void *uptr,unsigned int toMemberId,const void *data,unsigned int len);
@@ -536,11 +535,20 @@ public:
 	const std::string _homePath;
 	BackgroundResolver _tcpFallbackResolver;
 	InetAddress _allowManagementFrom;
-	json _localConfig;
 	EmbeddedNetworkController *_controller;
 	Phy<OneServiceImpl *> _phy;
 	Node *_node;
 
+	// Local configuration and memo-ized static path definitions
+	json _localConfig;
+	Hashtable< uint64_t,std::vector<InetAddress> > _v4Hints;
+	Hashtable< uint64_t,std::vector<InetAddress> > _v6Hints;
+	Hashtable< uint64_t,std::vector<InetAddress> > _v4Blacklists;
+	Hashtable< uint64_t,std::vector<InetAddress> > _v6Blacklists;
+	std::vector< InetAddress > _globalV4Blacklist;
+	std::vector< InetAddress > _globalV6Blacklist;
+	Mutex _localConfig_m;
+
 	/*
 	 * To attempt to handle NAT/gateway craziness we use three local UDP ports:
 	 *
@@ -552,7 +560,6 @@ public:
 	 * destructively with uPnP port mapping behavior in very weird buggy ways.
 	 * It's only used if uPnP/NAT-PMP is enabled in this build.
 	 */
-
 	Binder _bindings[3];
 	unsigned int _ports[3];
 	uint16_t _portsBE[3]; // ports in big-endian network byte order as in sockaddr
@@ -756,16 +763,19 @@ public:
 			// Clean up any legacy files if present
 			OSUtils::rm((_homePath + ZT_PATH_SEPARATOR_S + "peers.save").c_str());
 
-			_node = new Node(
+			{
-				OSUtils::now(),
+				struct ZT_Node_Callbacks cb;
-				this,
+				cb.version = 0;
-				SnodeDataStoreGetFunction,
+				cb.dataStoreGetFunction = SnodeDataStoreGetFunction;
-				SnodeDataStorePutFunction,
+				cb.dataStorePutFunction = SnodeDataStorePutFunction;
-				SnodeWirePacketSendFunction,
+				cb.wirePacketSendFunction = SnodeWirePacketSendFunction;
-				SnodeVirtualNetworkFrameFunction,
+				cb.virtualNetworkFrameFunction = SnodeVirtualNetworkFrameFunction;
-				SnodeVirtualNetworkConfigFunction,
+				cb.virtualNetworkConfigFunction = SnodeVirtualNetworkConfigFunction;
-				SnodePathCheckFunction,
+				cb.eventCallback = SnodeEventCallback;
-				SnodeEventCallback);
+				cb.pathCheckFunction = SnodePathCheckFunction;
+				cb.pathLookupFunction = SnodePathLookupFunction;
+				_node = new Node(this,&cb,OSUtils::now());
+			}
 
 			// Attempt to bind to a secondary port chosen from our ZeroTier address.
 			// This exists because there are buggy NATs out there that fail if more
@@ -842,6 +852,7 @@ public:
 				}
 
 				// Read local config file
+				Mutex::Lock _l2(_localConfig_m);
 				std::string lcbuf;
 				if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + "local.conf").c_str(),lcbuf)) {
 					try {
@@ -854,19 +865,18 @@ public:
 					}
 				}
 
-				// Get any trusted paths in local.conf
+				// Get any trusted paths in local.conf (we'll parse the rest of physical[] elsewhere)
 				json &physical = _localConfig["physical"];
 				if (physical.is_object()) {
 					for(json::iterator phy(physical.begin());phy!=physical.end();++phy) {
-						std::string nstr = phy.key();
+						InetAddress net(_jS(phy.key(),""));
-						if (nstr.length()) {
+						if (net) {
 							if (phy.value().is_object()) {
-								uint64_t tpid = 0;
+								uint64_t tpid;
-								if ((tpid = _jI(phy.value()["trustedPathId"],0ULL))) {
+								if ((tpid = _jI(phy.value()["trustedPathId"],0ULL)) != 0ULL) {
-									InetAddress trustedPathNetwork(nstr);
+									if ( ((net.ss_family == AF_INET)||(net.ss_family == AF_INET6)) && (trustedPathCount < ZT_MAX_TRUSTED_PATHS) && (net.ipScope() != InetAddress::IP_SCOPE_GLOBAL) && (net.netmaskBits() > 0) ) {
-									if ( ((trustedPathNetwork.ss_family == AF_INET)||(trustedPathNetwork.ss_family == AF_INET6)) && (trustedPathCount < ZT_MAX_TRUSTED_PATHS) && (trustedPathNetwork.ipScope() != InetAddress::IP_SCOPE_GLOBAL) && (trustedPathNetwork.netmaskBits() > 0) ) {
 										trustedPathIds[trustedPathCount] = tpid;
-										trustedPathNetworks[trustedPathCount] = trustedPathNetwork;
+										trustedPathNetworks[trustedPathCount] = net;
 										++trustedPathCount;
 									}
 								}
@@ -878,31 +888,8 @@ public:
 				// Set trusted paths if there are any
 				if (trustedPathCount)
 					_node->setTrustedPaths(reinterpret_cast<const struct sockaddr_storage *>(trustedPathNetworks),trustedPathIds,trustedPathCount);
-
-				// Set any roles (upstream/federation)
-				json &virt = _localConfig["virtual"];
-				if (virt.is_object()) {
-					for(json::iterator v(virt.begin());v!=virt.end();++v) {
-						const std::string nstr = v.key();
-						if ((nstr.length() == ZT_ADDRESS_LENGTH_HEX)&&(v.value().is_object())) {
-							const Address ztaddr(nstr.c_str());
-							if (ztaddr)
-								_node->setRole(ztaddr.toInt(),(_jS(v.value()["role"],"") == "upstream") ? ZT_PEER_ROLE_UPSTREAM : ZT_PEER_ROLE_LEAF);
-						}
-					}
-				}
-
-				// Set any other local config stuff
-				json &settings = _localConfig["settings"];
-				if (settings.is_object()) {
-					const std::string rp(_jS(settings["relayPolicy"],""));
-					if (rp == "always")
-						_node->setRelayPolicy(ZT_RELAY_POLICY_ALWAYS);
-					else if (rp == "never")
-						_node->setRelayPolicy(ZT_RELAY_POLICY_NEVER);
-					else _node->setRelayPolicy(ZT_RELAY_POLICY_TRUSTED);
-				}
 			}
+			applyLocalConfig();
 
 			_controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S + ZT_CONTROLLER_DB_PATH).c_str());
 			_node->setNetconfMaster((void *)_controller);
@@ -1174,7 +1161,90 @@ public:
 		return true;
 	}
 
-	// Begin private implementation methods
+	// Internal implementation methods -----------------------------------------
+
+	void applyLocalConfig()
+	{
+		Mutex::Lock _l(_localConfig_m);
+
+		_v4Hints.clear();
+		_v6Hints.clear();
+		_v4Blacklists.clear();
+		_v6Blacklists.clear();
+		json &virt = _localConfig["virtual"];
+		if (virt.is_object()) {
+			for(json::iterator v(virt.begin());v!=virt.end();++v) {
+				const std::string nstr = v.key();
+				if ((nstr.length() == ZT_ADDRESS_LENGTH_HEX)&&(v.value().is_object())) {
+					const Address ztaddr(nstr.c_str());
+					if (ztaddr) {
+						_node->setRole(ztaddr.toInt(),(_jS(v.value()["role"],"") == "upstream") ? ZT_PEER_ROLE_UPSTREAM : ZT_PEER_ROLE_LEAF);
+
+						const uint64_t ztaddr2 = ztaddr.toInt();
+						std::vector<InetAddress> &v4h = _v4Hints[ztaddr2];
+						std::vector<InetAddress> &v6h = _v6Hints[ztaddr2];
+						std::vector<InetAddress> &v4b = _v4Blacklists[ztaddr2];
+						std::vector<InetAddress> &v6b = _v6Blacklists[ztaddr2];
+
+						json &tryAddrs = v.value()["try"];
+						if (tryAddrs.is_array()) {
+							for(unsigned long i=0;i<tryAddrs.size();++i) {
+								const InetAddress ip(_jS(tryAddrs[i],""));
+								if (ip.ss_family == AF_INET)
+									v4h.push_back(ip);
+								else if (ip.ss_family == AF_INET6)
+									v6h.push_back(ip);
+							}
+						}
+						json &blAddrs = v.value()["blacklist"];
+						if (blAddrs.is_array()) {
+							for(unsigned long i=0;i<blAddrs.size();++i) {
+								const InetAddress ip(_jS(tryAddrs[i],""));
+								if (ip.ss_family == AF_INET)
+									v4b.push_back(ip);
+								else if (ip.ss_family == AF_INET6)
+									v6b.push_back(ip);
+							}
+						}
+
+						if (v4h.empty()) _v4Hints.erase(ztaddr2);
+						if (v6h.empty()) _v6Hints.erase(ztaddr2);
+						if (v4b.empty()) _v4Blacklists.erase(ztaddr2);
+						if (v6b.empty()) _v6Blacklists.erase(ztaddr2);
+					}
+				}
+			}
+		}
+
+		_globalV4Blacklist.clear();
+		_globalV6Blacklist.clear();
+		json &physical = _localConfig["physical"];
+		if (physical.is_object()) {
+			for(json::iterator phy(physical.begin());phy!=physical.end();++phy) {
+				const InetAddress net(_jS(phy.key(),""));
+				if ((net)&&(net.netmaskBits() > 0)) {
+					if (phy.value().is_object()) {
+						if (_jB(phy.value()["blacklist"],false)) {
+							if (net.ss_family == AF_INET)
+								_globalV4Blacklist.push_back(net);
+							else if (net.ss_family == AF_INET6)
+								_globalV6Blacklist.push_back(net);
+						}
+					}
+				}
+			}
+		}
+
+		json &settings = _localConfig["settings"];
+		if (settings.is_object()) {
+			const std::string rp(_jS(settings["relayPolicy"],""));
+			if (rp == "always")
+				_node->setRelayPolicy(ZT_RELAY_POLICY_ALWAYS);
+			else if (rp == "never")
+				_node->setRelayPolicy(ZT_RELAY_POLICY_NEVER);
+			else _node->setRelayPolicy(ZT_RELAY_POLICY_TRUSTED);
+		}
+	}
 
 	// Checks if a managed IP or route target is allowed
 	bool checkIfManagedIsAllowed(const NetworkState &n,const InetAddress &target)
@@ -1306,6 +1376,8 @@ public:
 		}
 	}
 
+	// Handlers for Node and Phy<> callbacks -----------------------------------
+
 	inline void phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *localAddr,const struct sockaddr *from,void *data,unsigned long len)
 	{
 #ifdef ZT_ENABLE_CLUSTER
@@ -1783,21 +1855,48 @@ public:
 		n->tap->put(MAC(sourceMac),MAC(destMac),etherType,data,len);
 	}
 
-	inline int nodePathCheckFunction(const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr)
+	inline int nodePathCheckFunction(uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr)
 	{
-		Mutex::Lock _l(_nets_m);
+		// Make sure we're not trying to do ZeroTier-over-ZeroTier
-	
+		{
-		for(std::map<uint64_t,NetworkState>::const_iterator n(_nets.begin());n!=_nets.end();++n) {
+			Mutex::Lock _l(_nets_m);
-			if (n->second.tap) {
+			for(std::map<uint64_t,NetworkState>::const_iterator n(_nets.begin());n!=_nets.end();++n) {
-				std::vector<InetAddress> ips(n->second.tap->ips());
+				if (n->second.tap) {
-				for(std::vector<InetAddress>::const_iterator i(ips.begin());i!=ips.end();++i) {
+					std::vector<InetAddress> ips(n->second.tap->ips());
-					if (i->containsAddress(*(reinterpret_cast<const InetAddress *>(remoteAddr)))) {
+					for(std::vector<InetAddress>::const_iterator i(ips.begin());i!=ips.end();++i) {
-						return 0;
+						if (i->containsAddress(*(reinterpret_cast<const InetAddress *>(remoteAddr)))) {
+							return 0;
+						}
 					}
 				}
 			}
 		}
-	
+
+		// Check blacklists
+		const Hashtable< uint64_t,std::vector<InetAddress> > *blh = (const Hashtable< uint64_t,std::vector<InetAddress> > *)0;
+		const std::vector<InetAddress> *gbl = (const std::vector<InetAddress> *)0;
+		if (remoteAddr->ss_family == AF_INET) {
+			blh = &_v4Blacklists;
+			gbl = &_globalV4Blacklist;
+		} else if (remoteAddr->ss_family == AF_INET6) {
+			blh = &_v6Blacklists;
+			gbl = &_globalV6Blacklist;
+		}
+		if (blh) {
+			Mutex::Lock _l(_localConfig_m);
+			const std::vector<InetAddress> *l = blh->get(ztaddr);
+			if (l) {
+				for(std::vector<InetAddress>::const_iterator a(l->begin());a!=l->end();++a) {
+					if (a->containsAddress(*reinterpret_cast<const InetAddress *>(remoteAddr)))
+						return 0;
+				}
+			}
+			for(std::vector<InetAddress>::const_iterator a(gbl->begin());a!=gbl->end();++a) {
+				if (a->containsAddress(*reinterpret_cast<const InetAddress *>(remoteAddr)))
+					return 0;
+			}
+		}
+
 		/* Note: I do not think we need to scan for overlap with managed routes
 		 * because of the "route forking" and interface binding that we do. This
 		 * ensures (we hope) that ZeroTier traffic will still take the physical
@@ -1807,6 +1906,23 @@ public:
 		return 1;
 	}
 
+	inline int nodePathLookupFunction(uint64_t ztaddr,int family,struct sockaddr_storage *result)
+	{
+		const Hashtable< uint64_t,std::vector<InetAddress> > *lh = (const Hashtable< uint64_t,std::vector<InetAddress> > *)0;
+		if (family < 0)
+			lh = (_node->prng() & 1) ? &_v4Hints : &_v6Hints;
+		else if (family == AF_INET)
+			lh = &_v4Hints;
+		else if (family == AF_INET6)
+			lh = &_v6Hints;
+		else return 0;
+		const std::vector<InetAddress> *l = lh->get(ztaddr);
+		if ((l)&&(l->size() > 0)) {
+			memcpy(result,&((*l)[(unsigned long)_node->prng() % l->size()]),sizeof(struct sockaddr_storage));
+			return 1;
+		} else return 0;
+	}
+
 	inline void tapFrameHandler(uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
 	{
 		_node->processVirtualNetworkFrame(OSUtils::now(),nwid,from.toInt(),to.toInt(),etherType,vlanId,data,len,&_nextBackgroundTaskDeadline);
@@ -1956,8 +2072,10 @@ static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct soc
 { return reinterpret_cast<OneServiceImpl *>(uptr)->nodeWirePacketSendFunction(localAddr,addr,data,len,ttl); }
 static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
 { reinterpret_cast<OneServiceImpl *>(uptr)->nodeVirtualNetworkFrameFunction(nwid,nuptr,sourceMac,destMac,etherType,vlanId,data,len); }
-static int SnodePathCheckFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr)
+static int SnodePathCheckFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *remoteAddr)
-{ return reinterpret_cast<OneServiceImpl *>(uptr)->nodePathCheckFunction(localAddr,remoteAddr); }
+{ return reinterpret_cast<OneServiceImpl *>(uptr)->nodePathCheckFunction(ztaddr,localAddr,remoteAddr); }
+static int SnodePathLookupFunction(ZT_Node *node,void *uptr,uint64_t ztaddr,int family,struct sockaddr_storage *result)
+{ return reinterpret_cast<OneServiceImpl *>(uptr)->nodePathLookupFunction(ztaddr,family,result); }
 
 #ifdef ZT_ENABLE_CLUSTER
 static void SclusterSendFunction(void *uptr,unsigned int toMemberId,const void *data,unsigned int len)