void/ZeroTierOne
2
0
Fork 0
mirror of https://github.com/zerotier/ZeroTierOne.git synced 2025-06-05 03:53:44 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Reduce memory use and a bit of cleanup.

Browse source
This commit is contained in:
Adam Ierymenko 2017-04-27 19:36:03 -07:00
parent 9e80db0fd1
commit 62578a2162
4 changed files with 67 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
controller/EmbeddedNetworkController.cpp
View file

@ -719,6 +719,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
member["lastModified"] = now; member["lastModified"] = now;
json &revj = member["revision"]; json &revj = member["revision"];
member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL);
_removeMemberNonPersistedFields(member);
_db.saveNetworkMember(nwid,address,member); _db.saveNetworkMember(nwid,address,member);
_pushMemberUpdate(now,nwid,member); _pushMemberUpdate(now,nwid,member);
} }
@ -1017,10 +1018,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
network["id"] = nwids; network["id"] = nwids;
network["nwid"] = nwids; // legacy network["nwid"] = nwids; // legacy
if (true) { if (network != origNetwork) {
json &revj = network["revision"]; json &revj = network["revision"];
network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL);
network["lastModified"] = now; network["lastModified"] = now;
_removeNetworkNonPersistedFields(network);
_db.saveNetwork(nwid,network); _db.saveNetwork(nwid,network);
// Send an update to all members of the network // Send an update to all members of the network
@ -1187,6 +1189,10 @@ void EmbeddedNetworkController::_request(
const Identity &identity, const Identity &identity,
const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData) const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData)
{ {
char nwids[24];
JSONDB::NetworkSummaryInfo ns;
json network,member,origMember;
if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender)) if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender))
return; return;
@ -1200,17 +1206,13 @@ void EmbeddedNetworkController::_request(
lrt = now; lrt = now;
} }
char nwids[24];
Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid);
json network,member;
JSONDB::NetworkSummaryInfo ns;
if (!_db.getNetworkAndMember(nwid,identity.address().toInt(),network,member,ns)) { if (!_db.getNetworkAndMember(nwid,identity.address().toInt(),network,member,ns)) {
_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND); _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND);
return; return;
} }
origMember = member;
const bool newMember = (member.size() == 0); const bool newMember = ((!member.is_object())||(member.size() == 0));
json origMember(member); // for detecting modification later
_initMember(member); _initMember(member);
{ {
@ -1235,10 +1237,12 @@ void EmbeddedNetworkController::_request(
} }
// These are always the same, but make sure they are set // These are always the same, but make sure they are set
const std::string addrs(identity.address().toString()); {
member["id"] = addrs; const std::string addrs(identity.address().toString());
member["address"] = addrs; member["id"] = addrs;
member["nwid"] = nwids; member["address"] = addrs;
member["nwid"] = nwids;
}
// Determine whether and how member is authorized // Determine whether and how member is authorized
const char *authorizedBy = (const char *)0; const char *authorizedBy = (const char *)0;
@ -1347,6 +1351,7 @@ void EmbeddedNetworkController::_request(
if (!authorizedBy) { if (!authorizedBy) {
if (origMember != member) { if (origMember != member) {
member["lastModified"] = now; member["lastModified"] = now;
_removeMemberNonPersistedFields(member);
_db.saveNetworkMember(nwid,identity.address().toInt(),member); _db.saveNetworkMember(nwid,identity.address().toInt(),member);
} }
_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED);
@ -1697,6 +1702,7 @@ void EmbeddedNetworkController::_request(
if (member != origMember) { if (member != origMember) {
member["lastModified"] = now; member["lastModified"] = now;
_removeMemberNonPersistedFields(member);
_db.saveNetworkMember(nwid,identity.address().toInt(),member); _db.saveNetworkMember(nwid,identity.address().toInt(),member);
} }

11
controller/EmbeddedNetworkController.hpp
View file

@ -157,10 +157,21 @@ private:
network["activeMemberCount"] = ns.activeMemberCount; network["activeMemberCount"] = ns.activeMemberCount;
network["totalMemberCount"] = ns.totalMemberCount; network["totalMemberCount"] = ns.totalMemberCount;
} }
inline void _removeNetworkNonPersistedFields(nlohmann::json &network)
{
network.erase("clock");
network.erase("authorizedMemberCount");
network.erase("activeMemberCount");
network.erase("totalMemberCount");
}
inline void _addMemberNonPersistedFields(nlohmann::json &member,uint64_t now) inline void _addMemberNonPersistedFields(nlohmann::json &member,uint64_t now)
{ {
member["clock"] = now; member["clock"] = now;
} }
inline void _removeMemberNonPersistedFields(nlohmann::json &member)
{
member.erase("clock");
}
const uint64_t _startTime; const uint64_t _startTime;

63
controller/JSONDB.cpp
View file

@ -110,24 +110,23 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj)
bool JSONDB::hasNetwork(const uint64_t networkId) const bool JSONDB::hasNetwork(const uint64_t networkId) const
{ {
Mutex::Lock _l(_networks_m); Mutex::Lock _l(_networks_m);
std::unordered_map<uint64_t,_NW>::const_iterator i(_networks.find(networkId)); return (_networks.find(networkId) != _networks.end());
return (i != _networks.end());
} }
bool JSONDB::getNetwork(const uint64_t networkId,nlohmann::json &config) const bool JSONDB::getNetwork(const uint64_t networkId,nlohmann::json &config) const
{ {
Mutex::Lock _l(_networks_m); Mutex::Lock _l(_networks_m);
std::unordered_map<uint64_t,_NW>::const_iterator i(_networks.find(networkId)); const std::unordered_map<uint64_t,_NW>::const_iterator i(_networks.find(networkId));
if (i == _networks.end()) if (i == _networks.end())
return false; return false;
config = i->second.config; config = nlohmann::json::from_msgpack(i->second.config);
return true; return true;
} }
bool JSONDB::getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &ns) const bool JSONDB::getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &ns) const
{ {
Mutex::Lock _l(_networks_m); Mutex::Lock _l(_networks_m);
std::unordered_map<uint64_t,_NW>::const_iterator i(_networks.find(networkId)); const std::unordered_map<uint64_t,_NW>::const_iterator i(_networks.find(networkId));
if (i == _networks.end()) if (i == _networks.end())
return false; return false;
ns = i->second.summaryInfo; ns = i->second.summaryInfo;
@ -137,14 +136,14 @@ bool JSONDB::getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &
int JSONDB::getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &networkConfig,nlohmann::json &memberConfig,NetworkSummaryInfo &ns) const int JSONDB::getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &networkConfig,nlohmann::json &memberConfig,NetworkSummaryInfo &ns) const
{ {
Mutex::Lock _l(_networks_m); Mutex::Lock _l(_networks_m);
std::unordered_map<uint64_t,_NW>::const_iterator i(_networks.find(networkId)); const std::unordered_map<uint64_t,_NW>::const_iterator i(_networks.find(networkId));
if (i == _networks.end()) if (i == _networks.end())
return 0; return 0;
std::unordered_map<uint64_t,nlohmann::json>::const_iterator j(i->second.members.find(nodeId)); const std::unordered_map< uint64_t,std::vector<uint8_t> >::const_iterator j(i->second.members.find(nodeId));
if (j == i->second.members.end()) if (j == i->second.members.end())
return 1; return 1;
networkConfig = i->second.config; networkConfig = nlohmann::json::from_msgpack(i->second.config);
memberConfig = j->second; memberConfig = nlohmann::json::from_msgpack(j->second);
ns = i->second.summaryInfo; ns = i->second.summaryInfo;
return 3; return 3;
} }
@ -152,13 +151,13 @@ int JSONDB::getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,n
bool JSONDB::getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &memberConfig) const bool JSONDB::getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &memberConfig) const
{ {
Mutex::Lock _l(_networks_m); Mutex::Lock _l(_networks_m);
std::unordered_map<uint64_t,_NW>::const_iterator i(_networks.find(networkId)); const std::unordered_map<uint64_t,_NW>::const_iterator i(_networks.find(networkId));
if (i == _networks.end()) if (i == _networks.end())
return false; return false;
std::unordered_map<uint64_t,nlohmann::json>::const_iterator j(i->second.members.find(nodeId)); const std::unordered_map< uint64_t,std::vector<uint8_t> >::const_iterator j(i->second.members.find(nodeId));
if (j == i->second.members.end()) if (j == i->second.members.end())
return false; return false;
memberConfig = j->second; memberConfig = nlohmann::json::from_msgpack(j->second);
return true; return true;
} }
@ -169,7 +168,7 @@ void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkC
writeRaw(n,OSUtils::jsonDump(networkConfig)); writeRaw(n,OSUtils::jsonDump(networkConfig));
{ {
Mutex::Lock _l(_networks_m); Mutex::Lock _l(_networks_m);
_networks[networkId].config = networkConfig; _networks[networkId].config = nlohmann::json::to_msgpack(networkConfig);
} }
_recomputeSummaryInfo(networkId); _recomputeSummaryInfo(networkId);
} }
@ -181,7 +180,7 @@ void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,co
writeRaw(n,OSUtils::jsonDump(memberConfig)); writeRaw(n,OSUtils::jsonDump(memberConfig));
{ {
Mutex::Lock _l(_networks_m); Mutex::Lock _l(_networks_m);
_networks[networkId].members[nodeId] = memberConfig; _networks[networkId].members[nodeId] = nlohmann::json::to_msgpack(memberConfig);
} }
_recomputeSummaryInfo(networkId); _recomputeSummaryInfo(networkId);
} }
@ -192,10 +191,10 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId)
std::vector<uint64_t> memberIds; std::vector<uint64_t> memberIds;
{ {
Mutex::Lock _l(_networks_m); Mutex::Lock _l(_networks_m);
std::unordered_map<uint64_t,_NW>::iterator i(_networks.find(networkId)); const std::unordered_map<uint64_t,_NW>::iterator i(_networks.find(networkId));
if (i == _networks.end()) if (i == _networks.end())
return _EMPTY_JSON; return _EMPTY_JSON;
for(std::unordered_map<uint64_t,nlohmann::json>::iterator m(i->second.members.begin());m!=i->second.members.end();++m) for(std::unordered_map< uint64_t,std::vector<uint8_t> >::iterator m(i->second.members.begin());m!=i->second.members.end();++m)
memberIds.push_back(m->first); memberIds.push_back(m->first);
} }
for(std::vector<uint64_t>::iterator m(memberIds.begin());m!=memberIds.end();++m) for(std::vector<uint64_t>::iterator m(memberIds.begin());m!=memberIds.end();++m)
@ -221,7 +220,7 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId)
std::unordered_map<uint64_t,_NW>::iterator i(_networks.find(networkId)); std::unordered_map<uint64_t,_NW>::iterator i(_networks.find(networkId));
if (i == _networks.end()) if (i == _networks.end())
return _EMPTY_JSON; // sanity check, shouldn't happen return _EMPTY_JSON; // sanity check, shouldn't happen
nlohmann::json tmp(i->second.config); nlohmann::json tmp(nlohmann::json::from_msgpack(i->second.config));
_networks.erase(i); _networks.erase(i);
return tmp; return tmp;
} }
@ -248,7 +247,7 @@ nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_
std::unordered_map<uint64_t,_NW>::iterator i(_networks.find(networkId)); std::unordered_map<uint64_t,_NW>::iterator i(_networks.find(networkId));
if (i == _networks.end()) if (i == _networks.end())
return _EMPTY_JSON; return _EMPTY_JSON;
std::unordered_map<uint64_t,nlohmann::json>::iterator j(i->second.members.find(nodeId)); std::unordered_map< uint64_t,std::vector<uint8_t> >::iterator j(i->second.members.find(nodeId));
if (j == i->second.members.end()) if (j == i->second.members.end())
return _EMPTY_JSON; return _EMPTY_JSON;
nlohmann::json tmp(j->second); nlohmann::json tmp(j->second);
@ -288,13 +287,15 @@ void JSONDB::threadMain()
ns.totalMemberCount = 0; ns.totalMemberCount = 0;
ns.mostRecentDeauthTime = 0; ns.mostRecentDeauthTime = 0;
for(std::unordered_map<uint64_t,nlohmann::json>::const_iterator m(n->second.members.begin());m!=n->second.members.end();++m) { for(std::unordered_map< uint64_t,std::vector<uint8_t> >::const_iterator m(n->second.members.begin());m!=n->second.members.end();++m) {
try { try {
if (OSUtils::jsonBool(m->second["authorized"],false)) { nlohmann::json member(nlohmann::json::from_msgpack(m->second));
if (OSUtils::jsonBool(member["authorized"],false)) {
++ns.authorizedMemberCount; ++ns.authorizedMemberCount;
try { try {
const nlohmann::json &mlog = m->second["recentLog"]; const nlohmann::json &mlog = member["recentLog"];
if ((mlog.is_array())&&(mlog.size() > 0)) { if ((mlog.is_array())&&(mlog.size() > 0)) {
const nlohmann::json &mlog1 = mlog[0]; const nlohmann::json &mlog1 = mlog[0];
if (mlog1.is_object()) { if (mlog1.is_object()) {
@ -305,12 +306,12 @@ void JSONDB::threadMain()
} catch ( ... ) {} } catch ( ... ) {}
try { try {
if (OSUtils::jsonBool(m->second["activeBridge"],false)) if (OSUtils::jsonBool(member["activeBridge"],false))
ns.activeBridges.push_back(Address(m->first)); ns.activeBridges.push_back(Address(m->first));
} catch ( ... ) {} } catch ( ... ) {}
try { try {
const nlohmann::json &mips = m->second["ipAssignments"]; const nlohmann::json &mips = member["ipAssignments"];
if (mips.is_array()) { if (mips.is_array()) {
for(unsigned long i=0;i<mips.size();++i) { for(unsigned long i=0;i<mips.size();++i) {
InetAddress mip(OSUtils::jsonString(mips[i],"")); InetAddress mip(OSUtils::jsonString(mips[i],""));
@ -321,7 +322,7 @@ void JSONDB::threadMain()
} catch ( ... ) {} } catch ( ... ) {}
} else { } else {
try { try {
ns.mostRecentDeauthTime = std::max(ns.mostRecentDeauthTime,OSUtils::jsonInt(m->second["lastDeauthorizedTime"],0ULL)); ns.mostRecentDeauthTime = std::max(ns.mostRecentDeauthTime,OSUtils::jsonInt(member["lastDeauthorizedTime"],0ULL));
} catch ( ... ) {} } catch ( ... ) {}
} }
++ns.totalMemberCount; ++ns.totalMemberCount;
@ -342,6 +343,8 @@ void JSONDB::threadMain()
bool JSONDB::_load(const std::string &p) bool JSONDB::_load(const std::string &p)
{ {
if (_httpAddr) { if (_httpAddr) {
// In HTTP harnessed mode we download our entire working data set on startup.
std::string body; std::string body;
std::map<std::string,std::string> headers; std::map<std::string,std::string> headers;
const unsigned int sc = Http::GET(2147483647,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast<const struct sockaddr *>(&_httpAddr),_basePath.c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); const unsigned int sc = Http::GET(2147483647,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast<const struct sockaddr *>(&_httpAddr),_basePath.c_str(),_ZT_JSONDB_GET_HEADERS,headers,body);
@ -360,12 +363,12 @@ bool JSONDB::_load(const std::string &p)
if ((id.length() == 16)&&(objtype == "network")) { if ((id.length() == 16)&&(objtype == "network")) {
const uint64_t nwid = Utils::hexStrToU64(id.c_str()); const uint64_t nwid = Utils::hexStrToU64(id.c_str());
if (nwid) if (nwid)
_networks[nwid].config = j; _networks[nwid].config = nlohmann::json::to_msgpack(j);
} else if ((id.length() == 10)&&(objtype == "member")) { } else if ((id.length() == 10)&&(objtype == "member")) {
const uint64_t mid = Utils::hexStrToU64(id.c_str()); const uint64_t mid = Utils::hexStrToU64(id.c_str());
const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str());
if ((mid)&&(nwid)) if ((mid)&&(nwid))
_networks[nwid].members[mid] = j; _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j);
} }
} }
} }
@ -374,7 +377,10 @@ bool JSONDB::_load(const std::string &p)
} catch ( ... ) {} // invalid JSON, so maybe incomplete request } catch ( ... ) {} // invalid JSON, so maybe incomplete request
} }
return false; return false;
} else { } else {
// In regular mode we recursively read it from controller.d/ on disk
std::vector<std::string> dl(OSUtils::listDirectory(p.c_str(),true)); std::vector<std::string> dl(OSUtils::listDirectory(p.c_str(),true));
for(std::vector<std::string>::const_iterator di(dl.begin());di!=dl.end();++di) { for(std::vector<std::string>::const_iterator di(dl.begin());di!=dl.end();++di) {
if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) {
@ -389,14 +395,14 @@ bool JSONDB::_load(const std::string &p)
const uint64_t nwid = Utils::strToU64(id.c_str()); const uint64_t nwid = Utils::strToU64(id.c_str());
if (nwid) { if (nwid) {
Mutex::Lock _l(_networks_m); Mutex::Lock _l(_networks_m);
_networks[nwid].config = j; _networks[nwid].config = nlohmann::json::to_msgpack(j);
} }
} else if ((id.length() == 10)&&(objtype == "member")) { } else if ((id.length() == 10)&&(objtype == "member")) {
const uint64_t mid = Utils::strToU64(id.c_str()); const uint64_t mid = Utils::strToU64(id.c_str());
const uint64_t nwid = Utils::strToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); const uint64_t nwid = Utils::strToU64(OSUtils::jsonString(j["nwid"],"0").c_str());
if ((mid)&&(nwid)) { if ((mid)&&(nwid)) {
Mutex::Lock _l(_networks_m); Mutex::Lock _l(_networks_m);
_networks[nwid].members[mid] = j; _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j);
} }
} }
} catch ( ... ) {} } catch ( ... ) {}
@ -406,6 +412,7 @@ bool JSONDB::_load(const std::string &p)
} }
} }
return true; return true;
} }
} }

8
controller/JSONDB.hpp
View file

@ -99,9 +99,9 @@ public:
Mutex::Lock _l(_networks_m); Mutex::Lock _l(_networks_m);
std::unordered_map<uint64_t,_NW>::const_iterator i(_networks.find(networkId)); std::unordered_map<uint64_t,_NW>::const_iterator i(_networks.find(networkId));
if (i != _networks.end()) { if (i != _networks.end()) {
for(std::unordered_map<uint64_t,nlohmann::json>::const_iterator m(i->second.members.begin());m!=i->second.members.end();++m) { for(std::unordered_map< uint64_t,std::vector<uint8_t> >::const_iterator m(i->second.members.begin());m!=i->second.members.end();++m) {
try { try {
func(networkId,m->first,m->second); func(networkId,m->first,nlohmann::json::from_msgpack(m->second));
} catch ( ... ) {} } catch ( ... ) {}
} }
} }
@ -126,10 +126,10 @@ private:
struct _NW struct _NW
{ {
_NW() : summaryInfoLastComputed(0) {} _NW() : summaryInfoLastComputed(0) {}
nlohmann::json config; std::vector<uint8_t> config;
NetworkSummaryInfo summaryInfo; NetworkSummaryInfo summaryInfo;
uint64_t summaryInfoLastComputed; uint64_t summaryInfoLastComputed;
std::unordered_map<uint64_t,nlohmann::json> members; std::unordered_map< uint64_t,std::vector<uint8_t> > members;
}; };
std::unordered_map<uint64_t,_NW> _networks; std::unordered_map<uint64_t,_NW> _networks;