Add comments to join ("orbit") moons.

include/ZeroTierOne.h

@@ -1779,6 +1779,28 @@ enum ZT_ResultCode ZT_Node_multicastSubscribe(ZT_Node *node,uint64_t nwid,uint64
  */
 enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
 
+/**
+ * Add or update a moon
+ *
+ * Moons are persisted in the data store in moons.d/, so this can persist
+ * across invocations if the contents of moon.d are scanned and orbit is
+ * called for each on startup.
+ *
+ * @param moonWorldId Moon's world ID
+ * @param len Length of moonWorld in bytes
+ * @return Error if moon was invalid or failed to be added
+ */
+enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId);
+
+/**
+ * Remove a moon (does nothing if not present)
+ *
+ * @param node Node instance
+ * @param moonWorldId World ID of moon to remove
+ * @return Error if anything bad happened
+ */
+ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,uint64_t moonWorldId);
+
 /**
  * Get this node's 40-bit ZeroTier address
  *

node/IncomingPacket.cpp

@@ -444,7 +444,8 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 
 				TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_path->address().toString().c_str(),vMajor,vMinor,vRevision,latency,((externalSurfaceAddress) ? externalSurfaceAddress.toString().c_str() : "(none)"));
 
-				peer->addDirectLatencyMeasurment(latency);
+				if (!hops())
+					peer->addDirectLatencyMeasurment(latency);
 				peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision);
 
 				if ((externalSurfaceAddress)&&(hops() == 0))

node/Node.cpp

@@ -168,26 +168,35 @@ public:
 
 	inline void operator()(Topology &t,const SharedPtr<Peer> &p)
 	{
-		const std::vector<InetAddress> *upstreamStableEndpoints = _upstreams.get(p->address());
+		const std::vector<InetAddress> *const upstreamStableEndpoints = _upstreams.get(p->address());
-		if ((upstreamStableEndpoints)&&(upstreamStableEndpoints->size() > 0)) {
+		if (upstreamStableEndpoints) {
+			bool contacted = false;
+
 			if (!p->doPingAndKeepalive(_now,AF_INET)) {
 				for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) {
 					const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()];
 					if (addr.ss_family == AF_INET) {
 						p->sendHELLO(InetAddress(),addr,_now);
+						contacted = true;
 						break;
 					}
 				}
-			}
+			} else contacted = true;
+
 			if (!p->doPingAndKeepalive(_now,AF_INET6)) {
 				for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) {
 					const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()];
 					if (addr.ss_family == AF_INET6) {
 						p->sendHELLO(InetAddress(),addr,_now);
+						contacted = true;
 						break;
 					}
 				}
-			}
+			} else contacted = true;
+
+			if (!contacted)
+				p->sendHELLO(InetAddress(),InetAddress(),_now);
+
 			lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream);
 		} else if (p->isActive(_now)) {
 			p->doPingAndKeepalive(_now,-1);
@@ -224,7 +233,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 			for(std::vector< SharedPtr<Network> >::const_iterator n(needConfig.begin());n!=needConfig.end();++n)
 				(*n)->requestConfiguration();
 
-			// Run WHOIS on upstreams we don't know about
+			// Attempt to get identity for any unknown upstreams
 			const std::vector<Address> upstreams(RR->topology->upstreamAddresses());
 			for(std::vector<Address>::const_iterator a(upstreams.begin());a!=upstreams.end();++a) {
 				if (!RR->topology->getPeer(*a))
@@ -323,6 +332,18 @@ ZT_ResultCode Node::multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,u
 	} else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND;
 }
 
+ZT_ResultCode Node::orbit(uint64_t moonWorldId)
+{
+	RR->topology->addMoon(moonWorldId);
+	return ZT_RESULT_OK;
+}
+
+ZT_ResultCode Node::deorbit(uint64_t moonWorldId)
+{
+	RR->topology->removeMoon(moonWorldId);
+	return ZT_RESULT_OK;
+}
+
 uint64_t Node::address() const
 {
 	return RR->identity.address().toInt();
@@ -893,6 +914,24 @@ enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint
 	}
 }
 
+enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId)
+{
+	try {
+		return reinterpret_cast<ZeroTier::Node *>(node)->orbit(moonWorldId);
+	} catch ( ... ) {
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
+	}
+}
+
+ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,uint64_t moonWorldId)
+{
+	try {
+		return reinterpret_cast<ZeroTier::Node *>(node)->deorbit(moonWorldId);
+	} catch ( ... ) {
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
+	}
+}
+
 uint64_t ZT_Node_address(ZT_Node *node)
 {
 	return reinterpret_cast<ZeroTier::Node *>(node)->address();

node/Node.hpp

@@ -95,6 +95,8 @@ public:
 	ZT_ResultCode leave(uint64_t nwid,void **uptr);
 	ZT_ResultCode multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
 	ZT_ResultCode multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
+	ZT_ResultCode orbit(uint64_t moonWorldId);
+	ZT_ResultCode deorbit(uint64_t moonWorldId);
 	uint64_t address() const;
 	void status(ZT_NodeStatus *status) const;
 	ZT_PeerList *peers() const;

node/Peer.cpp

@@ -366,8 +366,13 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u
 	}
 
 	RR->node->expectReplyTo(outp.packetId());
-	outp.armor(_key,false); // HELLO is sent in the clear
+
-	RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size());
+	if (atAddress) {
+		outp.armor(_key,false);
+		RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size());
+	} else {
+		RR->sw->send(outp,false);
+	}
 }
 
 void Peer::attemptToContactAt(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now)

node/Topology.cpp

@@ -48,13 +48,6 @@ Topology::Topology(const RuntimeEnvironment *renv) :
 	_trustedPathCount(0),
 	_amRoot(false)
 {
-	World defaultPlanet;
-	{
-		Buffer<ZT_DEFAULT_WORLD_LENGTH> wtmp(ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH);
-		defaultPlanet.deserialize(wtmp,0); // throws on error, which would indicate a bad static variable up top
-	}
-	addWorld(defaultPlanet,false);
-
 	try {
 		World cachedPlanet;
 		std::string buf(RR->node->dataStoreGet("planet"));
@@ -64,6 +57,13 @@ Topology::Topology(const RuntimeEnvironment *renv) :
 		}
 		addWorld(cachedPlanet,false);
 	} catch ( ... ) {}
+
+	World defaultPlanet;
+	{
+		Buffer<ZT_DEFAULT_WORLD_LENGTH> wtmp(ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH);
+		defaultPlanet.deserialize(wtmp,0); // throws on error, which would indicate a bad static variable up top
+	}
+	addWorld(defaultPlanet,false);
 }
 
 SharedPtr<Peer> Topology::addPeer(const SharedPtr<Peer> &peer)
@@ -287,7 +287,7 @@ bool Topology::addWorld(const World &newWorld,bool updateOnly)
 
 	char savePath[64];
 	if (existing->type() == World::TYPE_MOON)
-		Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx",existing->id());
+		Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",existing->id());
 	else Utils::scopy(savePath,sizeof(savePath),"planet");
 	try {
 		Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> dswtmp;
@@ -297,22 +297,71 @@ bool Topology::addWorld(const World &newWorld,bool updateOnly)
 		RR->node->dataStoreDelete(savePath);
 	}
 
-	_upstreamAddresses.clear();
+	if (existing->type() == World::TYPE_MOON) {
-	_amRoot = false;
+		std::vector<Address> cm;
-	for(std::vector<World::Root>::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) {
+		for(std::vector<Address>::const_iterator m(_contacingMoons.begin());m!=_contacingMoons.end();++m) {
-		if (i->identity == RR->identity)
+			if (m->toInt() != ((existing->id() >> 24) & 0xffffffffffULL))
-			_amRoot = true;
+				cm.push_back(*m);
-		else _upstreamAddresses.push_back(i->identity.address());
-	}
-	for(std::vector<World>::const_iterator m(_moons.begin());m!=_moons.end();++m) {
-		for(std::vector<World::Root>::const_iterator i(m->roots().begin());i!=m->roots().end();++i) {
-			if (i->identity == RR->identity)
-				_amRoot = true;
-			else _upstreamAddresses.push_back(i->identity.address());
 		}
+		_contacingMoons.swap(cm);
 	}
 
-	return false;
+	_memoizeUpstreams();
+
+	return true;
+}
+
+void Topology::addMoon(const uint64_t id)
+{
+	char savePath[64];
+	Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",id);
+
+	try {
+		std::string moonBin(RR->node->dataStoreGet(savePath));
+		if (moonBin.length() > 1) {
+			Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> wtmp(moonBin.data(),(unsigned int)moonBin.length());
+			World w;
+			w.deserialize(wtmp);
+			if (w.type() == World::TYPE_MOON) {
+				addWorld(w,false);
+				return;
+			}
+		}
+	} catch ( ... ) {}
+
+	{
+		const Address a(id >> 24);
+		Mutex::Lock _l(_lock);
+		if (std::find(_contacingMoons.begin(),_contacingMoons.end(),a) == _contacingMoons.end())
+			_contacingMoons.push_back(a);
+	}
+	RR->node->dataStorePut(savePath,"\0",1,false); // persist that we want to be a member
+}
+
+void Topology::removeMoon(const uint64_t id)
+{
+	Mutex::Lock _l(_lock);
+
+	std::vector<World> nm;
+	for(std::vector<World>::const_iterator m(_moons.begin());m!=_moons.end();++m) {
+		if (m->id() != id) {
+			nm.push_back(*m);
+		} else {
+			char savePath[64];
+			Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx.moon",id);
+			RR->node->dataStoreDelete(savePath);
+		}
+	}
+	_moons.swap(nm);
+
+	std::vector<Address> cm;
+	for(std::vector<Address>::const_iterator m(_contacingMoons.begin());m!=_contacingMoons.end();++m) {
+		if (m->toInt() != ((id >> 24) & 0xffffffffffULL))
+			cm.push_back(*m);
+	}
+	_contacingMoons.swap(cm);
+
+	_memoizeUpstreams();
 }
 
 void Topology::clean(uint64_t now)
@@ -351,4 +400,37 @@ Identity Topology::_getIdentity(const Address &zta)
 	return Identity();
 }
 
+void Topology::_memoizeUpstreams()
+{
+	// assumes _lock is locked
+	_upstreamAddresses.clear();
+	_amRoot = false;
+	for(std::vector<World::Root>::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) {
+		if (i->identity == RR->identity) {
+			_amRoot = true;
+		} else {
+			_upstreamAddresses.push_back(i->identity.address());
+			SharedPtr<Peer> &hp = _peers[i->identity.address()];
+			if (!hp) {
+				hp = new Peer(RR,RR->identity,i->identity);
+				saveIdentity(i->identity);
+			}
+		}
+	}
+	for(std::vector<World>::const_iterator m(_moons.begin());m!=_moons.end();++m) {
+		for(std::vector<World::Root>::const_iterator i(m->roots().begin());i!=m->roots().end();++i) {
+			if (i->identity == RR->identity) {
+				_amRoot = true;
+			} else {
+				_upstreamAddresses.push_back(i->identity.address());
+				SharedPtr<Peer> &hp = _peers[i->identity.address()];
+				if (!hp) {
+					hp = new Peer(RR,RR->identity,i->identity);
+					saveIdentity(i->identity);
+				}
+			}
+		}
+	}
+}
+
 } // namespace ZeroTier

node/Topology.hpp

@@ -169,6 +169,11 @@ public:
 	bool isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipaddr) const;
 
 	/**
+	 * This gets the known stable endpoints for any upstream
+	 *
+	 * It also adds empty entries for any upstreams we are attempting to
+	 * contact.
+	 *
 	 * @param eps Hash table to fill with addresses and their stable endpoints
 	 */
 	inline void getUpstreamStableEndpoints(Hashtable< Address,std::vector<InetAddress> > &eps) const
@@ -190,6 +195,8 @@ public:
 				}
 			}
 		}
+		for(std::vector<Address>::const_iterator m(_contacingMoons.begin());m!=_contacingMoons.end();++m)
+			eps[*m];
 	}
 
 	/**
@@ -198,7 +205,12 @@ public:
 	inline std::vector<Address> upstreamAddresses() const
 	{
 		Mutex::Lock _l(_lock);
-		return _upstreamAddresses;
+		std::vector<Address> u(_upstreamAddresses);
+		for(std::vector<Address>::const_iterator m(_contacingMoons.begin());m!=_contacingMoons.end();++m) {
+			if (std::find(u.begin(),u.end(),*m) == u.end())
+				u.push_back(*m);
+		}
+		return u;
 	}
 
 	/**
@@ -244,6 +256,25 @@ public:
 	 */
 	bool addWorld(const World &newWorld,bool updateOnly);
 
+	/**
+	 * Add a moon
+	 *
+	 * This loads it from moons.d if present, and if not adds it to
+	 * a list of moons that we want to contact. It does not actually
+	 * send anything, though this will happen on the next background
+	 * task loop where pings etc. are checked.
+	 *
+	 * @param id Moon ID
+	 */
+	void addMoon(const uint64_t id);
+
+	/**
+	 * Remove a moon
+	 *
+	 * @param id Moon's world ID
+	 */
+	void removeMoon(const uint64_t id);
+
 	/**
 	 * Clean and flush database
 	 */
@@ -362,6 +393,7 @@ public:
 
 private:
 	Identity _getIdentity(const Address &zta);
+	void _memoizeUpstreams();
 
 	const RuntimeEnvironment *const RR;
 
@@ -375,8 +407,9 @@ private:
 	Hashtable< Address,SharedPtr<Peer> > _peers;
 	Hashtable< Path::HashKey,SharedPtr<Path> > _paths;
 
-	std::vector< Address > _upstreamAddresses; // includes root addresses of both planets and moons
+	std::vector<Address> _contacingMoons;
-	bool _amRoot; // am I a root in a planet or moon?
+	std::vector<Address> _upstreamAddresses;
+	bool _amRoot;
 
 	Mutex _lock;
 };

service/OneService.cpp

@@ -702,6 +702,14 @@ public:
 						_node->join(Utils::hexStrToU64(f->substr(0,dot).c_str()),(void *)0);
 				}
 			}
+			{ // Load existing moons
+				std::vector<std::string> moonsDotD(OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S "moons.d").c_str()));
+				for(std::vector<std::string>::iterator f(moonsDotD.begin());f!=moonsDotD.end();++f) {
+					std::size_t dot = f->find_last_of('.');
+					if ((dot == 16)&&(f->substr(16) == ".moon"))
+						_node->orbit(Utils::hexStrToU64(f->substr(0,dot).c_str()));
+				}
+			}
 
 			_nextBackgroundTaskDeadline = 0;
 			uint64_t clockShouldBe = OSUtils::now();