From ade52bf81e2c24d9e4fdc0665030e988eee6f256 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 16 Sep 2019 17:04:13 -0700 Subject: [PATCH] Fix RENDEZVOUS issue and possibly improve GeoIP resolution. --- node/Topology.hpp | 247 ++++++++++++++++++++++------------------------ root/root.cpp | 52 ++++++---- 2 files changed, 149 insertions(+), 150 deletions(-) diff --git a/node/Topology.hpp b/node/Topology.hpp index 2a3e44282..ef7e546df 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -52,7 +52,14 @@ private: ZT_ALWAYS_INLINE _RootRankingFunction() : bestRoot(),bestRootLatency(0xffff) {} ZT_ALWAYS_INLINE bool operator()(const SharedPtr &peer,const std::vector &phy) { + const unsigned int lat = peer->latency(now); + if ((!bestRoot)||((lat <= bestRootLatency)&&(peer->getAppropriatePath(now,false)))) { + bestRoot = peer; + bestRootLatency = lat; + } + return true; } + int64_t now; SharedPtr bestRoot; unsigned int bestRootLatency; }; @@ -61,7 +68,8 @@ public: ZT_ALWAYS_INLINE Topology(const RuntimeEnvironment *renv,const Identity &myId) : RR(renv), _myIdentity(myId), - _numConfiguredPhysicalPaths(0) {} + _numConfiguredPhysicalPaths(0), + _lastUpdatedBestRoot(0) {} ZT_ALWAYS_INLINE ~Topology() {} /** @@ -98,25 +106,10 @@ public: { if (zta == _myIdentity.address()) return SharedPtr(); - Mutex::Lock l1(_peers_l); const SharedPtr *const ap = _peers.get(zta); if (ap) return *ap; - -#if 0 - Mutex::Lock l2(_roots_m); - for(std::vector::const_iterator r(_roots.begin());r!=_roots.end();++r) { - if (r->address() == zta) { - try { - SharedPtr rp(new Peer(RR,_myIdentity,r->id())); - _peers[zta] = rp; - return rp; - } catch ( ... ) {} - } - } -#endif - return SharedPtr(); } @@ -160,13 +153,16 @@ public: */ ZT_ALWAYS_INLINE bool isRoot(const Identity &id) const { -#if 0 - Mutex::Lock l(_roots_m); - for(std::vector::const_iterator r(_roots.begin());r!=_roots.end();++r) { - if (r->is(id)) + { + Mutex::Lock l(_dynamicRoots_l); + if (_dynamicRootIdentities.contains(id)) + return true; + } + { + Mutex::Lock l(_staticRoots_l); + if (_staticRoots.contains(id)) return true; } -#endif return false; } @@ -238,75 +234,6 @@ public: } } -#if 0 - /** - * Apply a function or function object to all roots - * - * This locks the root list during execution but other operations - * are fine. - * - * @param f Function to apply - * @tparam F function or function object type - */ - template - ZT_ALWAYS_INLINE void eachRoot(F f) - { - Mutex::Lock l(_roots_m); - SharedPtr rp; - for(std::vector::const_iterator i(_roots.begin());i!=_roots.end();++i) { - { - Mutex::Lock l2(_peers_l); - const SharedPtr *const ap = _peers.get(i->address()); - if (ap) { - rp = *ap; - } else { - rp.set(new Peer(RR,_myIdentity,i->id())); - _peers.set(rp->address(),rp); - } - } - f(*i,rp); - } - } - - /** - * Get the best root, rescanning and re-ranking roots periodically - * - * @param now Current time - * @return Best/fastest currently connected root or NULL if none - */ - inline SharedPtr root(const int64_t now) - { - Mutex::Lock l(_bestRoot_m); - if ((!_bestRoot)||((now - _lastRankedBestRoot) >= ZT_FIND_BEST_ROOT_PERIOD)) { - _bestRoot.zero(); - Mutex::Lock l2(_roots_m); - SharedPtr rp; - long bestQuality = 2147483647; - for(std::vector::const_iterator i(_roots.begin());i!=_roots.end();++i) { - { - Mutex::Lock l2(_peers_l); - const SharedPtr *const ap = _peers.get(i->address()); - if (ap) { - rp = *ap; - } else { - rp.set(new Peer(RR,_myIdentity,i->id())); - _peers.set(rp->address(),rp); - } - } - SharedPtr path(rp->getAppropriatePath(now,false)); - if (path) { - const long pq = path->quality(now); - if (pq < bestQuality) { - bestQuality = pq; - _bestRoot = rp; - } - } - } - } - return _bestRoot; - } -#endif - /** * Apply a function or function object to all roots * @@ -319,33 +246,6 @@ public: template inline void eachRoot(F f) { - { - Mutex::Lock l(_staticRoots_l); - Hashtable< Identity,std::vector >::Iterator i(_staticRoots); - Identity *k = (Identity *)0; - std::vector *v = (std::vector *)0; - while (i.next(k,v)) { - if (!v->empty()) { - const SharedPtr *ap; - { - Mutex::Lock l2(_peers_l); - ap = _peers.get(k->address()); - } - if (ap) { - if (!f(*ap,*v)) - return; - } else { - SharedPtr p(new Peer(RR,_myIdentity,*k)); - { - Mutex::Lock l2(_peers_l); - _peers.set(k->address(),p); - } - if (!f(p,*v)) - return; - } - } - } - } { Mutex::Lock l(_dynamicRoots_l); Hashtable< Str,Locator >::Iterator i(_dynamicRoots); @@ -375,21 +275,49 @@ public: } } } - } - - inline SharedPtr root(const int64_t now) - { - _RootRankingFunction rrf; - eachRoot(rrf); + { + Mutex::Lock l(_staticRoots_l); + Hashtable< Identity,std::vector >::Iterator i(_staticRoots); + Identity *k = (Identity *)0; + std::vector *v = (std::vector *)0; + while (i.next(k,v)) { + if (!v->empty()) { + const SharedPtr *ap; + { + Mutex::Lock l2(_peers_l); + ap = _peers.get(k->address()); + } + if (ap) { + if (!f(*ap,*v)) + return; + } else { + SharedPtr p(new Peer(RR,_myIdentity,*k)); + { + Mutex::Lock l2(_peers_l); + _peers.set(k->address(),p); + } + if (!f(p,*v)) + return; + } + } + } + } } /** - * @return Names of dynamic roots currently known by the system + * @return Current best root (updated automatically each second) */ - ZT_ALWAYS_INLINE std::vector dynamicRootNames() const + inline SharedPtr root(const int64_t now) { - Mutex::Lock l(_dynamicRoots_l); - return _dynamicRoots.keys(); + Mutex::Lock l(_bestRoot_l); + if ((!_bestRoot)||((now - _lastUpdatedBestRoot) > 1000)) { + _lastUpdatedBestRoot = now; + _RootRankingFunction rrf; + rrf.now = now; + eachRoot(rrf); + _bestRoot = rrf.bestRoot; + } + return _bestRoot; } /** @@ -404,6 +332,26 @@ public: _staticRoots[id] = addrs; } + /** + * Remove a static root + * + * @param id Identity to remove + */ + ZT_ALWAYS_INLINE void removeStaticRoot(const Identity &id) + { + Mutex::Lock l(_staticRoots_l); + _staticRoots.erase(id); + } + + /** + * @return Names of dynamic roots currently known by the system + */ + ZT_ALWAYS_INLINE std::vector dynamicRootNames() const + { + Mutex::Lock l(_dynamicRoots_l); + return _dynamicRoots.keys(); + } + /** * Set or update dynamic root if new locator is newer and valid * @@ -420,11 +368,34 @@ public: Locator &ll = _dynamicRoots[dnsName]; if (ll.timestamp() < latestLocator.timestamp()) { ll = latestLocator; + _updateDynamicRootIdentities(); return true; } return false; } + /** + * Remove a dynamic root entry + * + * @param dnsName DNS name to remove + */ + ZT_ALWAYS_INLINE bool removeDynamicRoot(const Str &dnsName) + { + Mutex::Lock l(_dynamicRoots_l); + _dynamicRoots.erase(dnsName); + _updateDynamicRootIdentities(); + } + + /** + * Remove all dynamic roots + */ + ZT_ALWAYS_INLINE bool clearDynamicRoots(const Str &dnsName) + { + Mutex::Lock l(_dynamicRoots_l); + _dynamicRoots.clear(); + _dynamicRootIdentities.clear(); + } + /** * Get the best relay to a given address, which may or may not be a root * @@ -557,6 +528,19 @@ public: } private: + inline void _updateDynamicRootIdentities() + { + // assumes _dynamicRoots_l is locked + _dynamicRootIdentities.clear(); + Hashtable< Str,Locator >::Iterator i(_dynamicRoots); + Str *k = (Str *)0; + Locator *v = (Locator *)0; + while (i.next(k,v)) { + if (v->id()) + _dynamicRootIdentities.set(v->id(),true); + } + } + const RuntimeEnvironment *const RR; const Identity _myIdentity; @@ -566,13 +550,18 @@ private: Hashtable< Address,SharedPtr > _peers; Hashtable< Path::HashKey,SharedPtr > _paths; - Hashtable< Identity,std::vector > _staticRoots; Hashtable< Str,Locator > _dynamicRoots; + Hashtable< Identity,bool > _dynamicRootIdentities; + Hashtable< Identity,std::vector > _staticRoots; + + int64_t _lastUpdatedBestRoot; + SharedPtr _bestRoot; Mutex _peers_l; Mutex _paths_l; - Mutex _staticRoots_l; Mutex _dynamicRoots_l; + Mutex _staticRoots_l; + Mutex _bestRoot_l; }; } // namespace ZeroTier diff --git a/root/root.cpp b/root/root.cpp index 9a45b4853..5275bdbad 100644 --- a/root/root.cpp +++ b/root/root.cpp @@ -431,7 +431,6 @@ static void handlePacket(const int v4s,const int v6s,const InetAddress *const ip const uint64_t nwid = pkt.template at(ptr); const MulticastGroup mg(MAC(pkt.field(ptr + 8,6),6),pkt.template at(ptr + 14)); s_multicastSubscriptions[nwid][mg][source] = now; - //printf("%s %s subscribes to %s/%.8lx on network %.16llx" ZT_EOL_S,ip->toString(ipstr),source.toString(astr),mg.mac().toString(tmpstr),(unsigned long)mg.adi(),(unsigned long long)nwid); } } catch ( ... ) { printf("* unexpected exception handling MULTICAST_LIKE from %s" ZT_EOL_S,ip->toString(ipstr)); @@ -561,7 +560,7 @@ static void handlePacket(const int v4s,const int v6s,const InetAddress *const ip dest.appendTo(outp); outp.append((uint16_t)b->second->ip6.port()); outp.append((uint8_t)16); - outp.append((const uint8_t *)b->second->ip6.rawIpData(),16); + outp.append((const uint8_t *)(b->second->ip6.rawIpData()),16); outp.armor((*a)->key,true); sendto(v6s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&((*a)->ip6),(socklen_t)sizeof(struct sockaddr_in6)); @@ -572,9 +571,9 @@ static void handlePacket(const int v4s,const int v6s,const InetAddress *const ip outp.reset(dest,s_self.address(),Packet::VERB_RENDEZVOUS); outp.append((uint8_t)0); source.appendTo(outp); - outp.append((uint16_t)ip->port()); + outp.append((uint16_t)(*a)->ip6.port()); outp.append((uint8_t)16); - outp.append((const uint8_t *)ip->rawIpData(),16); + outp.append((const uint8_t *)((*a)->ip6.rawIpData()),16); outp.armor(b->second->key,true); sendto(v6s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&(b->second->ip6),(socklen_t)sizeof(struct sockaddr_in6)); @@ -601,9 +600,9 @@ static void handlePacket(const int v4s,const int v6s,const InetAddress *const ip outp.reset(dest,s_self.address(),Packet::VERB_RENDEZVOUS); outp.append((uint8_t)0); source.appendTo(outp); - outp.append((uint16_t)ip->port()); + outp.append((uint16_t)(*a)->ip4.port()); outp.append((uint8_t)4); - outp.append((const uint8_t *)ip->rawIpData(),4); + outp.append((const uint8_t *)((*a)->ip4.rawIpData()),4); outp.armor(b->second->key,true); sendto(v4s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&(b->second->ip4),(socklen_t)sizeof(struct sockaddr_in)); @@ -616,7 +615,6 @@ static void handlePacket(const int v4s,const int v6s,const InetAddress *const ip } for(auto i=toAddrs.begin();i!=toAddrs.end();++i) { - //printf("%s -> %s for %s -> %s (%u bytes)" ZT_EOL_S,ip->toString(ipstr),i->first->toString(ipstr2),source.toString(astr),dest.toString(astr2),pkt.size()); if (sendto(i->first->isV4() ? v4s : v6s,pkt.data(),pkt.size(),SENDTO_FLAGS,(const struct sockaddr *)i->first,(socklen_t)(i->first->isV4() ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6))) > 0) { s_outputRate.log(now,pkt.size()); s_forwardRate.log(now,pkt.size()); @@ -636,13 +634,13 @@ static int bindSocket(struct sockaddr *const bindAddr) return -1; } - int f = 4194304; + int f = 16777216; while (f > 131072) { if (setsockopt(s,SOL_SOCKET,SO_RCVBUF,(const char *)&f,sizeof(f)) == 0) break; f -= 131072; } - f = 4194304; + f = 16777216; while (f > 131072) { if (setsockopt(s,SOL_SOCKET,SO_SNDBUF,(const char *)&f,sizeof(f)) == 0) break; @@ -665,11 +663,13 @@ static int bindSocket(struct sockaddr *const bindAddr) f = IP_PMTUDISC_DONT; setsockopt(s,IPPROTO_IP,IP_MTU_DISCOVER,&f,sizeof(f)); #endif +/* #ifdef SO_NO_CHECK if (bindAddr->sa_family == AF_INET) { f = 1; setsockopt(s,SOL_SOCKET,SO_NO_CHECK,(void *)&f,sizeof(f)); } #endif +*/ #if defined(SO_REUSEPORT) f = 1; setsockopt(s,SOL_SOCKET,SO_REUSEPORT,(void *)&f,sizeof(f)); @@ -1026,6 +1026,7 @@ int main(int argc,char **argv) std::pair< uint32_t,uint32_t > k4(0,0xffffffff); std::pair< std::array< uint64_t,2 >,std::array< uint64_t,2 > > k6; k6.second[0] = 0xffffffffffffffffULL; k6.second[1] = 0xffffffffffffffffULL; + std::unordered_map< InetAddress,std::set
,InetAddressHasher > ips; { std::lock_guard l(s_peers_l); @@ -1036,30 +1037,39 @@ int main(int argc,char **argv) ips[(*p)->ip6].insert((*p)->id.address()); } } + for(auto p=ips.begin();p!=ips.end();++p) { if (p->first.isV4()) { k4.first = ip4ToH32(p->first); auto geo = std::map< std::pair< uint32_t,uint32_t >,std::pair< float,float > >::reverse_iterator(s_geoIp4.upper_bound(k4)); + uint32_t bestRangeSize = 0xffffffff; + std::pair< float,float > bestRangeLatLon; while (geo != s_geoIp4.rend()) { if ((geo->first.first <= k4.first)&&(geo->first.second >= k4.first)) { - if (!firstCoord) - o << ','; - firstCoord = false; - o << "{lat:" << geo->second.first << ",lng:" << geo->second.second << ",_l:\""; - bool firstAddr = true; - for(auto a=p->second.begin();a!=p->second.end();++a) { - if (!firstAddr) - o << ','; - o << a->toString(tmp); - firstAddr = false; + uint32_t range = geo->first.second - geo->first.first; + if (range <= bestRangeSize) { + bestRangeSize = range; + bestRangeLatLon = geo->second; } - o << "\"}"; - break; } else if ((geo->first.first < k4.first)&&(geo->first.second < k4.first)) { break; } ++geo; } + if (bestRangeSize != 0xffffffff) { + if (!firstCoord) + o << ','; + firstCoord = false; + o << "{lat:" << bestRangeLatLon.first << ",lng:" << bestRangeLatLon.second << ",_l:\""; + bool firstAddr = true; + for(auto a=p->second.begin();a!=p->second.end();++a) { + if (!firstAddr) + o << ','; + o << a->toString(tmp); + firstAddr = false; + } + o << "\"}"; + } } else if (p->first.isV6()) { k6.first = ip6ToH128(p->first); auto geo = std::map< std::pair< std::array< uint64_t,2 >,std::array< uint64_t,2 > >,std::pair< float,float > >::reverse_iterator(s_geoIp6.upper_bound(k6));