Clean up remote tracing code, add per-network remote trace settings, add remote trace level, and make local trace output readable again.

include/ZeroTierOne.h

@@ -293,7 +293,6 @@ extern "C" {
 #define ZT_REMOTE_TRACE_FIELD__PACKET_TRUSTED_PATH_ID "packetTrustedPathId"
 #define ZT_REMOTE_TRACE_FIELD__PACKET_TRUSTED_PATH_APPROVED "packetTrustedPathApproved"
 #define ZT_REMOTE_TRACE_FIELD__PACKET_HOPS "packetHops"
-#define ZT_REMOTE_TRACE_FIELD__OLD_REMOTE_PHYADDR "oldRemotePhyAddr"
 #define ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR "remoteZtAddr"
 #define ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR "remotePhyAddr"
 #define ZT_REMOTE_TRACE_FIELD__LOCAL_ZTADDR "localZtAddr"
@@ -326,12 +325,11 @@ extern "C" {
 
 // Event types in remote traces
 #define ZT_REMOTE_TRACE_EVENT__RESETTING_PATHS_IN_SCOPE 0x1000
-#define ZT_REMOTE_TRACE_EVENT__TX_TIMED_OUT 0x1001
+#define ZT_REMOTE_TRACE_EVENT__PEER_CONFIRMING_UNKNOWN_PATH 0x1001
-#define ZT_REMOTE_TRACE_EVENT__PEER_CONFIRMING_UNKNOWN_PATH 0x1002
+#define ZT_REMOTE_TRACE_EVENT__PEER_LEARNED_NEW_PATH 0x1002
-#define ZT_REMOTE_TRACE_EVENT__PEER_LEARNED_NEW_PATH 0x1003
+#define ZT_REMOTE_TRACE_EVENT__PEER_REDIRECTED 0x1003
-#define ZT_REMOTE_TRACE_EVENT__PEER_REDIRECTED 0x1004
+#define ZT_REMOTE_TRACE_EVENT__PACKET_MAC_FAILURE 0x1004
-#define ZT_REMOTE_TRACE_EVENT__PACKET_MAC_FAILURE 0x1005
+#define ZT_REMOTE_TRACE_EVENT__PACKET_INVALID 0x1005
-#define ZT_REMOTE_TRACE_EVENT__PACKET_INVALID 0x1006
 #define ZT_REMOTE_TRACE_EVENT__DROPPED_HELLO 0x1006
 #define ZT_REMOTE_TRACE_EVENT__OUTGOING_NETWORK_FRAME_DROPPED 0x2000
 #define ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_ACCESS_DENIED 0x2001
@@ -343,12 +341,11 @@ extern "C" {
 
 // Event types in remote traces in hex string form
 #define ZT_REMOTE_TRACE_EVENT__RESETTING_PATHS_IN_SCOPE_S "1000"
-#define ZT_REMOTE_TRACE_EVENT__TX_TIMED_OUT_S "1001"
+#define ZT_REMOTE_TRACE_EVENT__PEER_CONFIRMING_UNKNOWN_PATH_S "1001"
-#define ZT_REMOTE_TRACE_EVENT__PEER_CONFIRMING_UNKNOWN_PATH_S "1002"
+#define ZT_REMOTE_TRACE_EVENT__PEER_LEARNED_NEW_PATH_S "1002"
-#define ZT_REMOTE_TRACE_EVENT__PEER_LEARNED_NEW_PATH_S "1003"
+#define ZT_REMOTE_TRACE_EVENT__PEER_REDIRECTED_S "1003"
-#define ZT_REMOTE_TRACE_EVENT__PEER_REDIRECTED_S "1004"
+#define ZT_REMOTE_TRACE_EVENT__PACKET_MAC_FAILURE_S "1004"
-#define ZT_REMOTE_TRACE_EVENT__PACKET_MAC_FAILURE_S "1005"
+#define ZT_REMOTE_TRACE_EVENT__PACKET_INVALID_S "1005"
-#define ZT_REMOTE_TRACE_EVENT__PACKET_INVALID_S "1006"
 #define ZT_REMOTE_TRACE_EVENT__DROPPED_HELLO_S "1006"
 #define ZT_REMOTE_TRACE_EVENT__OUTGOING_NETWORK_FRAME_DROPPED_S "2000"
 #define ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_ACCESS_DENIED_S "2001"

node/Hashtable.hpp

@@ -255,6 +255,24 @@ public:
 	}
 	inline const V *get(const K &k) const { return const_cast<Hashtable *>(this)->get(k); }
 
+	/**
+	 * @param k Key
+	 * @param v Value to fill with result
+	 * @return True if value was found and set (if false, v is not modified)
+	 */
+	inline bool get(const K &k,V &v) const
+	{
+		_Bucket *b = _t[_hc(k) % _bc];
+		while (b) {
+			if (b->k == k) {
+				v = b->v;
+				return true;
+			}
+			b = b->next;
+		}
+		return false;
+	}
+
 	/**
 	 * @param k Key to check
 	 * @return True if key is present

node/MAC.hpp

@@ -200,6 +200,29 @@ public:
 
 	inline unsigned long hashCode() const { return (unsigned long)_m; }
 
+	inline char *toString(char buf[18]) const
+	{
+		buf[0] = Utils::HEXCHARS[(_m >> 44) & 0xf];
+		buf[1] = Utils::HEXCHARS[(_m >> 40) & 0xf];
+		buf[2] = ':';
+		buf[3] = Utils::HEXCHARS[(_m >> 36) & 0xf];
+		buf[4] = Utils::HEXCHARS[(_m >> 32) & 0xf];
+		buf[5] = ':';
+		buf[6] = Utils::HEXCHARS[(_m >> 28) & 0xf];
+		buf[7] = Utils::HEXCHARS[(_m >> 24) & 0xf];
+		buf[8] = ':';
+		buf[9] = Utils::HEXCHARS[(_m >> 20) & 0xf];
+		buf[10] = Utils::HEXCHARS[(_m >> 16) & 0xf];
+		buf[11] = ':';
+		buf[12] = Utils::HEXCHARS[(_m >> 12) & 0xf];
+		buf[13] = Utils::HEXCHARS[(_m >> 8) & 0xf];
+		buf[14] = ':';
+		buf[15] = Utils::HEXCHARS[(_m >> 4) & 0xf];
+		buf[16] = Utils::HEXCHARS[_m & 0xf];
+		buf[17] = (char)0;
+		return buf;
+	}
+
 	inline MAC &operator=(const MAC &m)
 	{
 		_m = m._m;

node/NetworkConfig.cpp

@@ -49,6 +49,7 @@ bool NetworkConfig::toDictionary(Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> &d,b
 		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REVISION,this->revision)) return false;
 		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,this->issuedTo.toString(tmp2))) return false;
 		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_TARGET,this->remoteTraceTarget.toString(tmp2))) return false;
+		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_LEVEL,(uint64_t)this->remoteTraceLevel)) return false;
 		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_FLAGS,this->flags)) return false;
 		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,(uint64_t)this->multicastLimit)) return false;
 		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TYPE,(uint64_t)this->type)) return false;
@@ -220,6 +221,7 @@ bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACI
 			return false;
 		}
 		this->remoteTraceTarget = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_TARGET);
+		this->remoteTraceLevel = (Trace::Level)d.getUI(ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_LEVEL);
 		this->multicastLimit = (unsigned int)d.getUI(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,0);
 		d.get(ZT_NETWORKCONFIG_DICT_KEY_NAME,this->name,sizeof(this->name));
 

node/NetworkConfig.hpp

@@ -49,6 +49,7 @@
 #include "Dictionary.hpp"
 #include "Identity.hpp"
 #include "Utils.hpp"
+#include "Trace.hpp"
 
 /**
  * Default maximum time delta for COMs, tags, and capabilities
@@ -161,6 +162,8 @@ namespace ZeroTier {
 #define ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO "id"
 // remote trace target
 #define ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_TARGET "tt"
+// remote trace level
+#define ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_LEVEL "tl"
 // flags(hex)
 #define ZT_NETWORKCONFIG_DICT_KEY_FLAGS "f"
 // integer(hex)
@@ -189,8 +192,6 @@ namespace ZeroTier {
 #define ZT_NETWORKCONFIG_DICT_KEY_TAGS "TAG"
 // tags (binary blobs)
 #define ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATES_OF_OWNERSHIP "COO"
-// curve25519 signature
-#define ZT_NETWORKCONFIG_DICT_KEY_SIGNATURE "C25519"
 
 // Legacy fields -- these are obsoleted but are included when older clients query
 
@@ -445,6 +446,11 @@ public:
 	 */
 	uint64_t flags;
 
+	/**
+	 * Remote trace level
+	 */
+	Trace::Level remoteTraceLevel;
+
 	/**
 	 * Network MTU
 	 */

node/Node.cpp

@@ -61,7 +61,8 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 	_networks(8),
 	_now(now),
 	_lastPingCheck(0),
-	_lastHousekeepingRun(0)
+	_lastHousekeepingRun(0),
+	_lastMemoizedTraceSettings(0)
 {
 	if (callbacks->version != 0)
 		throw ZT_EXCEPTION_INVALID_ARGUMENT;
@@ -299,6 +300,11 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64
 		timeUntilNextPingCheck -= (unsigned long)timeSinceLastPingCheck;
 	}
 
+	if ((now - _lastMemoizedTraceSettings) >= 10000) {
+		_lastMemoizedTraceSettings = now;
+		RR->t->updateMemoizedSettings();
+	}
+
 	if ((now - _lastHousekeepingRun) >= ZT_HOUSEKEEPING_PERIOD) {
 		_lastHousekeepingRun = now;
 		try {

node/Node.hpp

@@ -258,6 +258,7 @@ public:
 	virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode);
 
 	inline const Address &remoteTraceTarget() const { return _remoteTraceTarget; }
+	inline Trace::Level remoteTraceLevel() const { return _remoteTraceLevel; }
 
 private:
 	RuntimeEnvironment _RR;
@@ -281,9 +282,12 @@ private:
 	Mutex _backgroundTasksLock;
 
 	Address _remoteTraceTarget;
+	enum Trace::Level _remoteTraceLevel;
+
 	int64_t _now;
 	int64_t _lastPingCheck;
 	int64_t _lastHousekeepingRun;
+	int64_t _lastMemoizedTraceSettings;
 	volatile int64_t _prngState[2];
 	bool _online;
 };

node/Peer.cpp

@@ -149,7 +149,7 @@ void Peer::received(
 
 				if (replacePath != ZT_MAX_PEER_NETWORK_PATHS) {
 					if (verb == Packet::VERB_OK) {
-						RR->t->peerLearnedNewPath(tPtr,networkId,*this,_paths[replacePath].p,path,packetId);
+						RR->t->peerLearnedNewPath(tPtr,networkId,*this,path,packetId);
 						_paths[replacePath].lr = now;
 						_paths[replacePath].p = path;
 						_paths[replacePath].priority = 1;

node/Switch.cpp

@@ -579,7 +579,6 @@ unsigned long Switch::doTimerTasks(void *tPtr,int64_t now)
 			if (_trySend(tPtr,txi->packet,txi->encrypt)) {
 				_txQueue.erase(txi++);
 			} else if ((now - txi->creationTime) > ZT_TRANSMIT_QUEUE_TIMEOUT) {
-				RR->t->txTimedOut(tPtr,txi->dest);
 				_txQueue.erase(txi++);
 			} else {
 				if (!RR->topology->getPeer(tPtr,txi->dest))

node/Trace.cpp

@@ -24,6 +24,10 @@
  * of your own application.
  */
 
+#define ZT_TRACE
+
+#include <stdio.h>
+
 #include "Trace.hpp"
 #include "RuntimeEnvironment.hpp"
 #include "Switch.hpp"
@@ -38,180 +42,292 @@
 
 namespace ZeroTier {
 
+#ifdef ZT_TRACE
+static void ZT_LOCAL_TRACE(void *const tPtr,const RuntimeEnvironment *const RR,const char *const fmt,...)
+{
+	char traceMsgBuf[1024];
+	va_list ap;
+	va_start(ap,fmt);
+	vsnprintf(traceMsgBuf,sizeof(traceMsgBuf),fmt,ap);
+	va_end(ap);
+	traceMsgBuf[sizeof(traceMsgBuf) - 1] = (char)0;
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,traceMsgBuf);
+}
+#else
+#define ZT_LOCAL_TRACE(...)
+#endif
+
 void Trace::resettingPathsInScope(void *const tPtr,const Address &reporter,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,const InetAddress::IpScope scope)
 {
 	char tmp[128];
+
+	ZT_LOCAL_TRACE(tPtr,RR,"RESET and revalidate paths in scope %d; new phy address %s reported by trusted peer %.10llx",(int)scope,myPhysicalAddress.toIpString(tmp),reporter.toInt());
+
 	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
 	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__RESETTING_PATHS_IN_SCOPE_S);
 	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,reporter);
 	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,reporterPhysicalAddress.toString(tmp));
 	d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_PHYADDR,myPhysicalAddress.toString(tmp));
 	d.add(ZT_REMOTE_TRACE_FIELD__IP_SCOPE,(uint64_t)scope);
-	_send(tPtr,d,0);
-}
 
-void Trace::txTimedOut(void *const tPtr,const Address &destination)
+	if (_globalTarget)
-{
+		_send(tPtr,d,_globalTarget);
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+	_spamToAllNetworks(tPtr,d,Trace::NORMAL);
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__TX_TIMED_OUT_S);
-	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,destination);
-	_send(tPtr,d,0);
 }
 
 void Trace::peerConfirmingUnknownPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr<Path> &path,const uint64_t packetId,const Packet::Verb verb)
 {
 	char tmp[128];
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+	if (!path) return; // sanity check
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PEER_CONFIRMING_UNKNOWN_PATH_S);
+
-	d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId);
+	ZT_LOCAL_TRACE(tPtr,RR,"trying unknown path %s to %.10llx (packet %.16llx verb %d local socket %lld network %.16llx)",path->address().toString(tmp),peer.address().toInt(),packetId,(double)verb,path->localSocket(),networkId);
-	d.add(ZT_REMOTE_TRACE_FIELD__PACKET_VERB,(uint64_t)verb);
+
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,networkId);
+	std::pair<Address,Trace::Level> byn;
-	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,peer.address());
+	if (networkId) { Mutex::Lock l(_byNet_m); _byNet.get(networkId,byn); }
-	if (path) {
+
-		d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp));
+	if ((_globalTarget)||(byn.first)) {
-		d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket());
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PEER_CONFIRMING_UNKNOWN_PATH_S);
+		d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId);
+		d.add(ZT_REMOTE_TRACE_FIELD__PACKET_VERB,(uint64_t)verb);
+		if (networkId)
+			d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,networkId);
+		d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,peer.address());
+		if (path) {
+			d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp));
+			d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket());
+		}
+
+		if (_globalTarget)
+			_send(tPtr,d,_globalTarget);
+		if (byn.first)
+			_send(tPtr,d,byn.first);
 	}
-	_send(tPtr,d,networkId);
 }
 
-void Trace::peerLearnedNewPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr<Path> &oldPath,const SharedPtr<Path> &newPath,const uint64_t packetId)
+void Trace::peerLearnedNewPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr<Path> &newPath,const uint64_t packetId)
 {
 	char tmp[128];
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+	if (!newPath) return; // sanity check
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PEER_LEARNED_NEW_PATH_S);
+
-	d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId);
+	ZT_LOCAL_TRACE(tPtr,RR,"learned new path %s to %.10llx (packet %.16llx local socket %lld network %.16llx)",newPath->address().toString(tmp),peer.address().toInt(),packetId,newPath->localSocket(),networkId);
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,networkId);
+
-	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,peer.address());
+	std::pair<Address,Trace::Level> byn;
-	if (oldPath) {
+	if (networkId) { Mutex::Lock l(_byNet_m); _byNet.get(networkId,byn); }
-		d.add(ZT_REMOTE_TRACE_FIELD__OLD_REMOTE_PHYADDR,oldPath->address().toString(tmp));
+
-	}
+	if ((_globalTarget)||(byn.first)) {
-	if (newPath) {
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PEER_LEARNED_NEW_PATH_S);
+		d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId);
+		if (networkId)
+			d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,networkId);
+		d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,peer.address());
 		d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,newPath->address().toString(tmp));
 		d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,newPath->localSocket());
+
+		if (_globalTarget)
+			_send(tPtr,d,_globalTarget);
+		if (byn.first)
+			_send(tPtr,d,byn.first);
 	}
-	_send(tPtr,d,networkId);
 }
 
 void Trace::peerRedirected(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr<Path> &newPath)
 {
 	char tmp[128];
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+	if (!newPath) return; // sanity check
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PEER_REDIRECTED_S);
+
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,networkId);
+	ZT_LOCAL_TRACE(tPtr,RR,"explicit redirect from %.10llx to path %s",peer.address().toInt(),newPath->address().toString(tmp));
-	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,peer.address());
+
-	if (newPath) {
+	std::pair<Address,Trace::Level> byn;
+	if (networkId) { Mutex::Lock l(_byNet_m); _byNet.get(networkId,byn); }
+
+	if ((_globalTarget)||(byn.first)) {
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PEER_REDIRECTED_S);
+		if (networkId)
+			d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,networkId);
+		d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,peer.address());
 		d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,newPath->address().toString(tmp));
 		d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,newPath->localSocket());
+
+		if (_globalTarget)
+			_send(tPtr,d,_globalTarget);
+		if (byn.first)
+			_send(tPtr,d,byn.first);
 	}
-	_send(tPtr,d,networkId);
 }
 
 void Trace::outgoingNetworkFrameDropped(void *const tPtr,const SharedPtr<Network> &network,const MAC &sourceMac,const MAC &destMac,const unsigned int etherType,const unsigned int vlanId,const unsigned int frameLen,const char *reason)
 {
+	char tmp[128],tmp2[128];
 	if (!network) return; // sanity check
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__OUTGOING_NETWORK_FRAME_DROPPED_S);
+	ZT_LOCAL_TRACE(tPtr,RR,"%.16llx DROP frame %s -> %s etherType %.4x size %u (%s)",network->id(),sourceMac.toString(tmp),destMac.toString(tmp2),etherType,frameLen,(reason) ? reason : "unknown reason");
-	d.add(ZT_REMOTE_TRACE_FIELD__SOURCE_MAC,sourceMac.toInt());
+
-	d.add(ZT_REMOTE_TRACE_FIELD__DEST_MAC,destMac.toInt());
+	std::pair<Address,Trace::Level> byn;
-	d.add(ZT_REMOTE_TRACE_FIELD__ETHERTYPE,(uint64_t)etherType);
+	{ Mutex::Lock l(_byNet_m); _byNet.get(network->id(),byn); }
-	d.add(ZT_REMOTE_TRACE_FIELD__VLAN_ID,(uint64_t)vlanId);
+
-	d.add(ZT_REMOTE_TRACE_FIELD__FRAME_LENGTH,(uint64_t)frameLen);
+	if ( ((_globalTarget)&&((int)_globalLevel >= (int)Trace::VERBOSE)) || ((byn.first)&&((int)byn.second >= (int)Trace::VERBOSE)) ) {
-	if (reason) {
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-		d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__OUTGOING_NETWORK_FRAME_DROPPED_S);
+		d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network->id());
+		d.add(ZT_REMOTE_TRACE_FIELD__SOURCE_MAC,sourceMac.toInt());
+		d.add(ZT_REMOTE_TRACE_FIELD__DEST_MAC,destMac.toInt());
+		d.add(ZT_REMOTE_TRACE_FIELD__ETHERTYPE,(uint64_t)etherType);
+		d.add(ZT_REMOTE_TRACE_FIELD__VLAN_ID,(uint64_t)vlanId);
+		d.add(ZT_REMOTE_TRACE_FIELD__FRAME_LENGTH,(uint64_t)frameLen);
+		if (reason)
+			d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+
+		if ((_globalTarget)&&((int)_globalLevel >= (int)Trace::VERBOSE))
+			_send(tPtr,d,_globalTarget);
+		if ((byn.first)&&((int)byn.second >= (int)Trace::VERBOSE))
+			_send(tPtr,d,byn.first);
 	}
-	_send(tPtr,d,network);
 }
 
 void Trace::incomingNetworkAccessDenied(void *const tPtr,const SharedPtr<Network> &network,const SharedPtr<Path> &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,bool credentialsRequested)
 {
-	if (!network) return; // sanity check
 	char tmp[128];
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+	if (!network) return; // sanity check
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_ACCESS_DENIED_S);
+
-	d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId);
+	ZT_LOCAL_TRACE(tPtr,RR,"%.16llx DENIED packet from %.10llx(%s) verb %d size %u%s",network->id(),source.toInt(),(path) ? (path->address().toString(tmp)) : "???",(int)verb,packetLength,credentialsRequested ? " (credentials requested)" : " (credentials not requested)");
-	d.add(ZT_REMOTE_TRACE_FIELD__PACKET_VERB,(uint64_t)verb);
+
-	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source);
+	std::pair<Address,Trace::Level> byn;
-	if (path) {
+	{ Mutex::Lock l(_byNet_m); _byNet.get(network->id(),byn); }
-		d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp));
+
-		d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket());
+	if ( ((_globalTarget)&&((int)_globalLevel >= (int)Trace::VERBOSE)) || ((byn.first)&&((int)byn.second >= (int)Trace::VERBOSE)) ) {
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_ACCESS_DENIED_S);
+		d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId);
+		d.add(ZT_REMOTE_TRACE_FIELD__PACKET_VERB,(uint64_t)verb);
+		d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source);
+		if (path) {
+			d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp));
+			d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket());
+		}
+		d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network->id());
+
+		if ((_globalTarget)&&((int)_globalLevel >= (int)Trace::VERBOSE))
+			_send(tPtr,d,_globalTarget);
+		if ((byn.first)&&((int)byn.second >= (int)Trace::VERBOSE))
+			_send(tPtr,d,byn.first);
 	}
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network->id());
-	_send(tPtr,d,*network);
 }
 
 void Trace::incomingNetworkFrameDropped(void *const tPtr,const SharedPtr<Network> &network,const SharedPtr<Path> &path,const uint64_t packetId,const unsigned int packetLength,const Address &source,const Packet::Verb verb,const MAC &sourceMac,const MAC &destMac,const char *reason)
 {
-	if (!network) return; // sanity check
 	char tmp[128];
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+	if (!network) return; // sanity check
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_FRAME_DROPPED_S);
+
-	d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId);
+	ZT_LOCAL_TRACE(tPtr,RR,"%.16llx DROPPED frame from %.10llx(%s) verb %d size %u",network->id(),source.toInt(),(path) ? (path->address().toString(tmp)) : "???",(int)verb,packetLength);
-	d.add(ZT_REMOTE_TRACE_FIELD__PACKET_VERB,(uint64_t)verb);
+
-	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source);
+	std::pair<Address,Trace::Level> byn;
-	if (path) {
+	{ Mutex::Lock l(_byNet_m); _byNet.get(network->id(),byn); }
-		d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp));
+
-		d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket());
+	if ( ((_globalTarget)&&((int)_globalLevel >= (int)Trace::VERBOSE)) || ((byn.first)&&((int)byn.second >= (int)Trace::VERBOSE)) ) {
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__INCOMING_NETWORK_FRAME_DROPPED_S);
+		d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId);
+		d.add(ZT_REMOTE_TRACE_FIELD__PACKET_VERB,(uint64_t)verb);
+		d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source);
+		if (path) {
+			d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp));
+			d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket());
+		}
+		d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network->id());
+		d.add(ZT_REMOTE_TRACE_FIELD__SOURCE_MAC,sourceMac.toInt());
+		d.add(ZT_REMOTE_TRACE_FIELD__DEST_MAC,destMac.toInt());
+		if (reason)
+			d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+
+		if ((_globalTarget)&&((int)_globalLevel >= (int)Trace::VERBOSE))
+			_send(tPtr,d,_globalTarget);
+		if ((byn.first)&&((int)byn.second >= (int)Trace::VERBOSE))
+			_send(tPtr,d,byn.first);
 	}
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network->id());
-	d.add(ZT_REMOTE_TRACE_FIELD__SOURCE_MAC,sourceMac.toInt());
-	d.add(ZT_REMOTE_TRACE_FIELD__DEST_MAC,destMac.toInt());
-	if (reason)
-		d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
-	_send(tPtr,d,*network);
 }
 
 void Trace::incomingPacketMessageAuthenticationFailure(void *const tPtr,const SharedPtr<Path> &path,const uint64_t packetId,const Address &source,const unsigned int hops,const char *reason)
 {
 	char tmp[128];
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PACKET_MAC_FAILURE_S);
+	ZT_LOCAL_TRACE(tPtr,RR,"MAC failed for packet %.16llx from %.10llx(%s)",packetId,source.toInt(),(path) ? path->address().toString(tmp) : "???");
-	d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId);
+
-	d.add(ZT_REMOTE_TRACE_FIELD__PACKET_HOPS,(uint64_t)hops);
+	if ((_globalTarget)&&((int)_globalLevel >= Trace::DEBUG)) {
-	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source);
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp));
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PACKET_MAC_FAILURE_S);
-	d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket());
+		d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId);
-	if (reason)
+		d.add(ZT_REMOTE_TRACE_FIELD__PACKET_HOPS,(uint64_t)hops);
-		d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+		d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source);
-	_send(tPtr,d,0);
+		if (path) {
+			d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp));
+			d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket());
+		}
+		if (reason)
+			d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+
+		_send(tPtr,d,_globalTarget);
+	}
 }
 
 void Trace::incomingPacketInvalid(void *const tPtr,const SharedPtr<Path> &path,const uint64_t packetId,const Address &source,const unsigned int hops,const Packet::Verb verb,const char *reason)
 {
 	char tmp[128];
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PACKET_INVALID_S);
+	ZT_LOCAL_TRACE(tPtr,RR,"INVALID packet %.16llx from %.10llx(%s) (%s)",packetId,source.toInt(),(path) ? path->address().toString(tmp) : "???",(reason) ? reason : "unknown reason");
-	d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId);
+
-	d.add(ZT_REMOTE_TRACE_FIELD__PACKET_VERB,(uint64_t)verb);
+	if ((_globalTarget)&&((int)_globalLevel >= Trace::DEBUG)) {
-	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source);
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp));
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PACKET_INVALID_S);
-	d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket());
+		d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId);
-	d.add(ZT_REMOTE_TRACE_FIELD__PACKET_HOPS,(uint64_t)hops);
+		d.add(ZT_REMOTE_TRACE_FIELD__PACKET_VERB,(uint64_t)verb);
-	if (reason)
+		d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source);
-		d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+		if (path) {
-	_send(tPtr,d,0);
+			d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp));
+			d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket());
+		}
+		d.add(ZT_REMOTE_TRACE_FIELD__PACKET_HOPS,(uint64_t)hops);
+		if (reason)
+			d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+
+		_send(tPtr,d,_globalTarget);
+	}
 }
 
 void Trace::incomingPacketDroppedHELLO(void *const tPtr,const SharedPtr<Path> &path,const uint64_t packetId,const Address &source,const char *reason)
 {
 	char tmp[128];
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PACKET_INVALID_S);
+	ZT_LOCAL_TRACE(tPtr,RR,"DROPPED HELLO from %.10llx(%s) (%s)",source.toInt(),(path) ? path->address().toString(tmp) : "???",(reason) ? reason : "???");
-	d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId);
+
-	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source);
+	if ((_globalTarget)&&((int)_globalLevel >= Trace::DEBUG)) {
-	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp));
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket());
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__PACKET_INVALID_S);
-	if (reason)
+		d.add(ZT_REMOTE_TRACE_FIELD__PACKET_ID,packetId);
-		d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+		d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source);
-	_send(tPtr,d,0);
+		if (path) {
+			d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp));
+			d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket());
+		}
+		if (reason)
+			d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+
+		_send(tPtr,d,_globalTarget);
+	}
 }
 
 void Trace::networkConfigRequestSent(void *const tPtr,const Network &network,const Address &controller)
 {
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+	ZT_LOCAL_TRACE(tPtr,RR,"requesting configuration for network %.16llx",network.id());
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__NETWORK_CONFIG_REQUEST_SENT_S);
+	if ((_globalTarget)&&((int)_globalLevel >= Trace::DEBUG)) {
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network.id());
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_CONTROLLER_ID,controller);
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__NETWORK_CONFIG_REQUEST_SENT_S);
-	_send(tPtr,d,network);
+		d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network.id());
+		d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_CONTROLLER_ID,controller);
+		_send(tPtr,d,_globalTarget);
+	}
 }
 
 void Trace::networkFilter(
@@ -232,153 +348,189 @@ void Trace::networkFilter(
 	const bool inbound,
 	const int accept)
 {
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+	std::pair<Address,Trace::Level> byn;
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__NETWORK_FILTER_TRACE_S);
+	{ Mutex::Lock l(_byNet_m); _byNet.get(network.id(),byn); }
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network.id());
+
-	d.add(ZT_REMOTE_TRACE_FIELD__SOURCE_ZTADDR,ztSource);
+	if ( ((_globalTarget)&&((int)_globalLevel >= (int)Trace::RULES)) || ((byn.first)&&((int)byn.second >= (int)Trace::RULES)) ) {
-	d.add(ZT_REMOTE_TRACE_FIELD__DEST_ZTADDR,ztDest);
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__SOURCE_MAC,macSource.toInt());
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__NETWORK_FILTER_TRACE_S);
-	d.add(ZT_REMOTE_TRACE_FIELD__DEST_MAC,macDest.toInt());
+		d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,network.id());
-	d.add(ZT_REMOTE_TRACE_FIELD__ETHERTYPE,(uint64_t)etherType);
+		d.add(ZT_REMOTE_TRACE_FIELD__SOURCE_ZTADDR,ztSource);
-	d.add(ZT_REMOTE_TRACE_FIELD__VLAN_ID,(uint64_t)vlanId);
+		d.add(ZT_REMOTE_TRACE_FIELD__DEST_ZTADDR,ztDest);
-	d.add(ZT_REMOTE_TRACE_FIELD__FILTER_FLAG_NOTEE,noTee ? "1" : "0");
+		d.add(ZT_REMOTE_TRACE_FIELD__SOURCE_MAC,macSource.toInt());
-	d.add(ZT_REMOTE_TRACE_FIELD__FILTER_FLAG_INBOUND,inbound ? "1" : "0");
+		d.add(ZT_REMOTE_TRACE_FIELD__DEST_MAC,macDest.toInt());
-	d.add(ZT_REMOTE_TRACE_FIELD__FILTER_RESULT,(int64_t)accept);
+		d.add(ZT_REMOTE_TRACE_FIELD__ETHERTYPE,(uint64_t)etherType);
-	d.add(ZT_REMOTE_TRACE_FIELD__FILTER_BASE_RULE_LOG,(const char *)primaryRuleSetLog.data(),(int)primaryRuleSetLog.sizeBytes());
+		d.add(ZT_REMOTE_TRACE_FIELD__VLAN_ID,(uint64_t)vlanId);
-	if (matchingCapabilityRuleSetLog)
+		d.add(ZT_REMOTE_TRACE_FIELD__FILTER_FLAG_NOTEE,noTee ? "1" : "0");
-		d.add(ZT_REMOTE_TRACE_FIELD__FILTER_CAP_RULE_LOG,(const char *)matchingCapabilityRuleSetLog->data(),(int)matchingCapabilityRuleSetLog->sizeBytes());
+		d.add(ZT_REMOTE_TRACE_FIELD__FILTER_FLAG_INBOUND,inbound ? "1" : "0");
-	if (matchingCapability)
+		d.add(ZT_REMOTE_TRACE_FIELD__FILTER_RESULT,(int64_t)accept);
-		d.add(ZT_REMOTE_TRACE_FIELD__FILTER_CAP_ID,(uint64_t)matchingCapability->id());
+		d.add(ZT_REMOTE_TRACE_FIELD__FILTER_BASE_RULE_LOG,(const char *)primaryRuleSetLog.data(),(int)primaryRuleSetLog.sizeBytes());
-	d.add(ZT_REMOTE_TRACE_FIELD__FRAME_LENGTH,(uint64_t)frameLen);
+		if (matchingCapabilityRuleSetLog)
-	if (frameLen > 0)
+			d.add(ZT_REMOTE_TRACE_FIELD__FILTER_CAP_RULE_LOG,(const char *)matchingCapabilityRuleSetLog->data(),(int)matchingCapabilityRuleSetLog->sizeBytes());
-		d.add(ZT_REMOTE_TRACE_FIELD__FRAME_DATA,(const char *)frameData,(frameLen > 256) ? (int)256 : (int)frameLen);
+		if (matchingCapability)
-	_send(tPtr,d,network);
+			d.add(ZT_REMOTE_TRACE_FIELD__FILTER_CAP_ID,(uint64_t)matchingCapability->id());
+		d.add(ZT_REMOTE_TRACE_FIELD__FRAME_LENGTH,(uint64_t)frameLen);
+		if (frameLen > 0)
+			d.add(ZT_REMOTE_TRACE_FIELD__FRAME_DATA,(const char *)frameData,(frameLen > 256) ? (int)256 : (int)frameLen);
+
+		if ((_globalTarget)&&((int)_globalLevel >= (int)Trace::RULES))
+			_send(tPtr,d,_globalTarget);
+		if ((byn.first)&&((int)byn.second >= (int)Trace::RULES))
+			_send(tPtr,d,byn.first);
+	}
 }
 
 void Trace::credentialRejected(void *const tPtr,const CertificateOfMembership &c,const char *reason)
 {
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+	std::pair<Address,Trace::Level> byn;
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S);
+	if (c.networkId()) { Mutex::Lock l(_byNet_m); _byNet.get(c.networkId(),byn); }
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
+
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
+	if ((_globalTarget)||(byn.first)) {
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp());
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S);
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo());
+		d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
-	if (reason)
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
-		d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
-	_send(tPtr,d,c.networkId());
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp());
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo());
+		if (reason)
+			d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+
+		if (_globalTarget)
+			_send(tPtr,d,_globalTarget);
+		if (byn.first)
+			_send(tPtr,d,byn.first);
+	}
 }
 
 void Trace::credentialRejected(void *const tPtr,const CertificateOfOwnership &c,const char *reason)
 {
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+	std::pair<Address,Trace::Level> byn;
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S);
+	if (c.networkId()) { Mutex::Lock l(_byNet_m); _byNet.get(c.networkId(),byn); }
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
+
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
+	if ((_globalTarget)||(byn.first)) {
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp());
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S);
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo());
+		d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
-	if (reason)
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
-		d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
-	_send(tPtr,d,c.networkId());
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp());
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo());
+		if (reason)
+			d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+
+		if (_globalTarget)
+			_send(tPtr,d,_globalTarget);
+		if (byn.first)
+			_send(tPtr,d,byn.first);
+	}
 }
 
 void Trace::credentialRejected(void *const tPtr,const Capability &c,const char *reason)
 {
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+	std::pair<Address,Trace::Level> byn;
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S);
+	if (c.networkId()) { Mutex::Lock l(_byNet_m); _byNet.get(c.networkId(),byn); }
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
+
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
+	if ((_globalTarget)||(byn.first)) {
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp());
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S);
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo());
+		d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
-	if (reason)
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
-		d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
-	_send(tPtr,d,c.networkId());
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp());
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo());
+		if (reason)
+			d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+
+		if (_globalTarget)
+			_send(tPtr,d,_globalTarget);
+		if (byn.first)
+			_send(tPtr,d,byn.first);
+	}
 }
 
 void Trace::credentialRejected(void *const tPtr,const Tag &c,const char *reason)
 {
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+	std::pair<Address,Trace::Level> byn;
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S);
+	if (c.networkId()) { Mutex::Lock l(_byNet_m); _byNet.get(c.networkId(),byn); }
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
+
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
+	if ((_globalTarget)||(byn.first)) {
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp());
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S);
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo());
+		d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO,(uint64_t)c.value());
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
-	if (reason)
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
-		d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp());
-	_send(tPtr,d,c.networkId());
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo());
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO,(uint64_t)c.value());
+		if (reason)
+			d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+
+		if (_globalTarget)
+			_send(tPtr,d,_globalTarget);
+		if (byn.first)
+			_send(tPtr,d,byn.first);
+	}
 }
 
 void Trace::credentialRejected(void *const tPtr,const Revocation &c,const char *reason)
 {
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
+	std::pair<Address,Trace::Level> byn;
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S);
+	if (c.networkId()) { Mutex::Lock l(_byNet_m); _byNet.get(c.networkId(),byn); }
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
+
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
+	if ((_globalTarget)||(byn.first)) {
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
+		Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET,c.target());
+		d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_REJECTED_S);
-	if (reason)
+		d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
-		d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
-	_send(tPtr,d,c.networkId());
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
+		d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET,c.target());
+		if (reason)
+			d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
+
+		if (_globalTarget)
+			_send(tPtr,d,_globalTarget);
+		if (byn.first)
+			_send(tPtr,d,byn.first);
+	}
 }
 
-void Trace::_send(void *const tPtr,const Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> &d)
+void Trace::updateMemoizedSettings()
 {
-#ifdef ZT_TRACE
+	_globalTarget = RR->node->remoteTraceTarget();
-	unsigned int i = 0;
+	_globalLevel = RR->node->remoteTraceLevel();
-	while (i < (unsigned int)(sizeof(_traceMsgBuf) - 1)) {
+	const std::vector< SharedPtr<Network> > nws(RR->node->allNetworks());
-		const char c = d.data()[i];
+	{
-		if (c == 0) {
+		Mutex::Lock l(_byNet_m);
-			break;
+		_byNet.clear();
-		} else if (c == '\n') {
+		for(std::vector< SharedPtr<Network> >::const_iterator n(nws.begin());n!=nws.end();++n) {
-			_traceMsgBuf[i++] = ' ';
+			const Address dest((*n)->config().remoteTraceTarget);
-		} else if ((c >= 32)&&(c <= 126)) {
+			if (dest) {
-			_traceMsgBuf[i++] = c;
+				std::pair<Address,Trace::Level> &m = _byNet[(*n)->id()];
-		} else {
+				m.first = dest;
-			if ((i + 3) < (unsigned int)(sizeof(_traceMsgBuf) - 1)) {
+				m.second = (*n)->config().remoteTraceLevel;
-				_traceMsgBuf[i++] = '\\';
-				Utils::hex((uint8_t)c,_traceMsgBuf + i);
 			}
 		}
 	}
-	_traceMsgBuf[i] = (char)0;
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,_traceMsgBuf);
-#endif
-
-	const Address rtt(RR->node->remoteTraceTarget());
-	if (rtt) {
-		Packet outp(rtt,RR->identity.address(),Packet::VERB_REMOTE_TRACE);
-		outp.appendCString(d.data());
-		outp.compress();
-		RR->sw->send(tPtr,outp,true);
-	}
 }
 
-void Trace::_send(void *const tPtr,const Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> &d,const uint64_t networkId)
+void Trace::_send(void *const tPtr,const Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> &d,const Address &dest)
 {
-	_send(tPtr,d);
+	Packet outp(dest,RR->identity.address(),Packet::VERB_REMOTE_TRACE);
-	if (networkId) {
+	outp.appendCString(d.data());
-		const SharedPtr<Network> network(RR->node->network(networkId));
+	outp.compress();
-		if ((network)&&(network->config().remoteTraceTarget)) {
+	RR->sw->send(tPtr,outp,true);
-			Packet outp(network->config().remoteTraceTarget,RR->identity.address(),Packet::VERB_REMOTE_TRACE);
-			outp.appendCString(d.data());
-			outp.compress();
-			RR->sw->send(tPtr,outp,true);
-		}
-	}
 }
 
-void Trace::_send(void *const tPtr,const Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> &d,const Network &network)
+void Trace::_spamToAllNetworks(void *const tPtr,const Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> &d,const Level level)
 {
-	_send(tPtr,d);
+	Mutex::Lock l(_byNet_m);
-	if (network.config().remoteTraceTarget) {
+	Hashtable< uint64_t,std::pair< Address,Trace::Level > >::Iterator i(_byNet);
-		Packet outp(network.config().remoteTraceTarget,RR->identity.address(),Packet::VERB_REMOTE_TRACE);
+	uint64_t *k = (uint64_t *)0;
-		outp.appendCString(d.data());
+	std::pair<Address,Trace::Level> *v = (std::pair<Address,Trace::Level> *)0;
-		outp.compress();
+	while (i.next(k,v)) {
-		RR->sw->send(tPtr,outp,true);
+		if ((v)&&(v->first)&&((int)v->second >= (int)level))
+			_send(tPtr,d,v->first);
 	}
 }
 

node/Trace.hpp

@@ -40,6 +40,8 @@
 #include "Credential.hpp"
 #include "InetAddress.hpp"
 #include "Dictionary.hpp"
+#include "Mutex.hpp"
+#include "Hashtable.hpp"
 
 namespace ZeroTier {
 
@@ -63,6 +65,18 @@ class Capability;
 class Trace
 {
 public:
+	/**
+	 * Trace verbosity level
+	 */
+	enum Level
+	{
+		NORMAL = 0,
+		VERBOSE = 10,
+		RULES = 15,
+		DEBUG = 20,
+		INSANE = 30
+	};
+
 	/**
 	 * Filter rule evaluation result log
 	 *
@@ -98,13 +112,16 @@ public:
 		uint8_t _l[ZT_MAX_NETWORK_RULES / 2];
 	};
 
-	Trace(const RuntimeEnvironment *renv) : RR(renv) {}
+	Trace(const RuntimeEnvironment *renv) :
+		RR(renv),
+		_byNet(8)
+	{
+	}
 
 	void resettingPathsInScope(void *const tPtr,const Address &reporter,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,const InetAddress::IpScope scope);
-	void txTimedOut(void *const tPtr,const Address &destination);
 
 	void peerConfirmingUnknownPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr<Path> &path,const uint64_t packetId,const Packet::Verb verb);
-	void peerLearnedNewPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr<Path> &oldPath,const SharedPtr<Path> &newPath,const uint64_t packetId);
+	void peerLearnedNewPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr<Path> &newPath,const uint64_t packetId);
 	void peerRedirected(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr<Path> &newPath);
 
 	void incomingPacketMessageAuthenticationFailure(void *const tPtr,const SharedPtr<Path> &path,const uint64_t packetId,const Address &source,const unsigned int hops,const char *reason);
@@ -140,16 +157,18 @@ public:
 	void credentialRejected(void *const tPtr,const Tag &c,const char *reason);
 	void credentialRejected(void *const tPtr,const Revocation &c,const char *reason);
 
+	void updateMemoizedSettings();
+
 private:
 	const RuntimeEnvironment *const RR;
 
-	void _send(void *const tPtr,const Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> &d);
+	void _send(void *const tPtr,const Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> &d,const Address &dest);
-	void _send(void *const tPtr,const Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> &d,const uint64_t networkId);
+	void _spamToAllNetworks(void *const tPtr,const Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> &d,const Level level);
-	void _send(void *const tPtr,const Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> &d,const Network &network);
 
-#ifdef ZT_TRACE
+	Address _globalTarget;
-	char _traceMsgBuf[4096];
+	Trace::Level _globalLevel;
-#endif
+	Hashtable< uint64_t,std::pair< Address,Trace::Level > > _byNet;
+	Mutex _byNet_m;
 };
 
 } // namespace ZeroTier