void/ZeroTierOne
2
0
Fork 0
mirror of https://github.com/zerotier/ZeroTierOne.git synced 2025-06-06 12:33:44 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Refactor controller to use split-out DB for better performance and less ugly.

Browse source
This commit is contained in:
Adam Ierymenko 2016-11-04 15:18:31 -07:00
parent ab2ccb094a
commit b03c7b2f30
6 changed files with 224 additions and 173 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

267
controller/EmbeddedNetworkController.cpp
View file

@ -64,6 +64,7 @@ using json = nlohmann::json;
namespace ZeroTier { namespace ZeroTier {
// JSON blob I/O // JSON blob I/O
/*
static json _readJson(const std::string &path) static json _readJson(const std::string &path)
{ {
std::string buf; std::string buf;
@ -78,6 +79,7 @@ static bool _writeJson(const std::string &path,const json &obj)
{ {
return OSUtils::writeFile(path.c_str(),obj.dump(2)); return OSUtils::writeFile(path.c_str(),obj.dump(2));
} }
*/
// Get JSON values as unsigned integers, strings, or booleans, doing type conversion if possible // Get JSON values as unsigned integers, strings, or booleans, doing type conversion if possible
static uint64_t _jI(const json &jv,const uint64_t dfl) static uint64_t _jI(const json &jv,const uint64_t dfl)
@ -475,55 +477,17 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule)
} }
EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) :
_node(node), _db(dbPath),
_path(dbPath), _node(node)
_daemonRun(true)
{ {
OSUtils::mkdir(dbPath); OSUtils::mkdir(dbPath);
OSUtils::lockDownFile(dbPath,true); // networks might contain auth tokens, etc., so restrict directory permissions OSUtils::lockDownFile(dbPath,true); // networks might contain auth tokens, etc., so restrict directory permissions
_daemon = Thread::start(this);
} }
EmbeddedNetworkController::~EmbeddedNetworkController() EmbeddedNetworkController::~EmbeddedNetworkController()
{ {
} }
void EmbeddedNetworkController::threadMain()
throw()
{
uint64_t lastUpdatedNetworkMemberCache = 0;
while (_daemonRun) {
// Every 60 seconds we rescan the filesystem for network members and rebuild our cache
if ((OSUtils::now() - lastUpdatedNetworkMemberCache) >= 60000) {
const std::vector<std::string> networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str()));
for(auto n=networks.begin();n!=networks.end();++n) {
if (n->length() == 16) {
const std::vector<std::string> members(OSUtils::listSubdirectories((*n + ZT_PATH_SEPARATOR_S + "member").c_str()));
std::map<Address,nlohmann::json> newCache;
for(auto m=members.begin();m!=members.end();++m) {
if (m->length() == ZT_ADDRESS_LENGTH_HEX) {
const Address maddr(*m);
try {
const json mj(_readJson((_path + ZT_PATH_SEPARATOR_S + "network" + ZT_PATH_SEPARATOR_S + *n + ZT_PATH_SEPARATOR_S + "member" + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json")));
if ((mj.is_object())&&(mj.size() > 0)) {
newCache[maddr] = mj;
}
} catch ( ... ) {}
}
}
{
Mutex::Lock _l(_networkMemberCache_m);
_networkMemberCache[Utils::hexStrToU64(n->c_str())] = newCache;
}
}
}
lastUpdatedNetworkMemberCache = OSUtils::now();
}
Thread::sleep(25);
}
}
NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData,NetworkConfig &nc) NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData,NetworkConfig &nc)
{ {
if (((!signingId)||(!signingId.hasPrivate()))||(signingId.address().toInt() != (nwid >> 24))) { if (((!signingId)||(!signingId.hasPrivate()))||(signingId.address().toInt() != (nwid >> 24))) {
@ -541,12 +505,17 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(
lrt = now; lrt = now;
} }
json network(_readJson(_networkJP(nwid,false))); char nwids[24];
Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid);
json network;
json member;
{
Mutex::Lock _l(_db_m);
network = _db.get("network",nwids,0);
member = _db.get("network",nwids,"member",identity.address().toString(),0);
}
if (!network.size()) if (!network.size())
return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND;
const std::string memberJP(_memberJP(nwid,identity.address(),true));
json member(_readJson(memberJP));
_initMember(member); _initMember(member);
{ {
@ -673,7 +642,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(
// If they are not authorized, STOP! // If they are not authorized, STOP!
if (!authorizedBy) { if (!authorizedBy) {
_writeJson(memberJP,member); Mutex::Lock _l(_db_m);
_db.put("network",nwids,"member",identity.address().toString(),member);
return NetworkController::NETCONF_QUERY_ACCESS_DENIED; return NetworkController::NETCONF_QUERY_ACCESS_DENIED;
} }
@ -966,7 +936,10 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(
return NETCONF_QUERY_INTERNAL_SERVER_ERROR; return NETCONF_QUERY_INTERNAL_SERVER_ERROR;
} }
_writeJson(memberJP,member); {
Mutex::Lock _l(_db_m);
_db.put("network",nwids,"member",identity.address().toString(),member);
}
return NetworkController::NETCONF_QUERY_OK; return NetworkController::NETCONF_QUERY_OK;
} }
@ -985,7 +958,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
char nwids[24]; char nwids[24];
Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid);
json network(_readJson(_networkJP(nwid,false))); json network;
{
Mutex::Lock _l(_db_m);
network = _db.get("network",nwids,0);
}
if (!network.size()) if (!network.size())
return 404; return 404;
@ -996,7 +973,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
if (path.size() >= 4) { if (path.size() >= 4) {
const uint64_t address = Utils::hexStrToU64(path[3].c_str()); const uint64_t address = Utils::hexStrToU64(path[3].c_str());
json member(_readJson(_memberJP(nwid,Address(address),false))); json member;
{
Mutex::Lock _l(_db_m);
member = _db.get("network",nwids,"member",Address(address).toString(),0);
}
if (!member.size()) if (!member.size())
return 404; return 404;
@ -1007,19 +988,19 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
return 200; return 200;
} else { } else {
Mutex::Lock _l(_db_m);
responseBody = "{"; responseBody = "{";
std::vector<std::string> members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); std::string pfx(std::string("network/") + nwids + "member/");
for(std::vector<std::string>::iterator i(members.begin());i!=members.end();++i) { _db.filter(pfx,120000,[&responseBody](const std::string &n,const json &member) {
if (i->length() == ZT_ADDRESS_LENGTH_HEX) { if (member.size() > 0) {
json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); responseBody.append((responseBody.length() == 1) ? "\"" : ",\"");
if (member.size()) { responseBody.append(_jS(member["id"],""));
responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); responseBody.append("\":");
responseBody.append(*i); responseBody.append(_jS(member["revision"],"0"));
responseBody.append("\":");
responseBody.append(_jS(member["revision"],"0"));
}
} }
} return true; // never delete
});
responseBody.push_back('}'); responseBody.push_back('}');
responseContentType = "application/json"; responseContentType = "application/json";
@ -1056,7 +1037,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
} }
} else if (path.size() == 1) { } else if (path.size() == 1) {
responseBody = "["; /*
std::vector<std::string> networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str())); std::vector<std::string> networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str()));
for(auto i(networks.begin());i!=networks.end();++i) { for(auto i(networks.begin());i!=networks.end();++i) {
if (i->length() == 16) { if (i->length() == 16) {
@ -1065,6 +1046,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
responseBody.append("\""); responseBody.append("\"");
} }
} }
*/
responseBody.push_back(']'); responseBody.push_back(']');
responseContentType = "application/json"; responseContentType = "application/json";
return 200; return 200;
@ -1122,7 +1104,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid);
if (path.size() >= 3) { if (path.size() >= 3) {
json network(_readJson(_networkJP(nwid,false))); json network;
{
Mutex::Lock _l(_db_m);
network = _db.get("network",nwids,0);
}
if (!network.size()) if (!network.size())
return 404; return 404;
@ -1131,7 +1117,13 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
char addrs[24]; char addrs[24];
Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address);
json member(_readJson(_memberJP(nwid,Address(address),true))); json member;
{
Mutex::Lock _l(_db_m);
member = _db.get("network",nwids,"member",Address(address).toString(),0);
}
if (!member.size())
return 404;
_initMember(member); _initMember(member);
try { try {
@ -1154,7 +1146,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
} }
if (b.count("ipAssignments")) { if (b.count("ipAssignments")) {
auto ipa = b["ipAssignments"]; json &ipa = b["ipAssignments"];
if (ipa.is_array()) { if (ipa.is_array()) {
json mipa(json::array()); json mipa(json::array());
for(unsigned long i=0;i<ipa.size();++i) { for(unsigned long i=0;i<ipa.size();++i) {
@ -1169,7 +1161,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
} }
if (b.count("tags")) { if (b.count("tags")) {
auto tags = b["tags"]; json &tags = b["tags"];
if (tags.is_array()) { if (tags.is_array()) {
std::map<uint64_t,uint64_t> mtags; std::map<uint64_t,uint64_t> mtags;
for(unsigned long i=0;i<tags.size();++i) { for(unsigned long i=0;i<tags.size();++i) {
@ -1189,7 +1181,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
} }
if (b.count("capabilities")) { if (b.count("capabilities")) {
auto capabilities = b["capabilities"]; json &capabilities = b["capabilities"];
if (capabilities.is_array()) { if (capabilities.is_array()) {
json mcaps = json::array(); json mcaps = json::array();
for(unsigned long i=0;i<capabilities.size();++i) { for(unsigned long i=0;i<capabilities.size();++i) {
@ -1210,14 +1202,12 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
member["address"] = addrs; // legacy member["address"] = addrs; // legacy
member["nwid"] = nwids; member["nwid"] = nwids;
member["lastModified"] = now; member["lastModified"] = now;
auto revj = member["revision"]; json &revj = member["revision"];
member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL);
_writeJson(_memberJP(nwid,Address(address),true).c_str(),member);
{ {
Mutex::Lock _l(_networkMemberCache_m); Mutex::Lock _l(_db_m);
_networkMemberCache[nwid][Address(address)] = member; _db.put("network",nwids,"member",Address(address).toString(),member);
} }
// Add non-persisted fields // Add non-persisted fields
@ -1281,26 +1271,33 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
} else { } else {
// POST to network ID // POST to network ID
// Magic ID ending with ______ picks a random unused network ID json network;
if (path[1].substr(10) == "______") { {
nwid = 0; Mutex::Lock _l(_db_m);
uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL;
uint64_t nwidPostfix = 0;
for(unsigned long k=0;k<100000;++k) { // sanity limit on trials
Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix));
uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL);
if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL;
Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid);
if (!OSUtils::fileExists(_networkJP(tryNwid,false).c_str())) {
nwid = tryNwid;
break;
}
}
if (!nwid)
return 503;
}
json network(_readJson(_networkJP(nwid,true))); // Magic ID ending with ______ picks a random unused network ID
if (path[1].substr(10) == "______") {
nwid = 0;
uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL;
uint64_t nwidPostfix = 0;
for(unsigned long k=0;k<100000;++k) { // sanity limit on trials
Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix));
uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL);
if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL;
Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid);
if (_db.get("network",nwids,120000).size() <= 0) {
nwid = tryNwid;
break;
}
}
if (!nwid)
return 503;
}
network = _db.get("network",nwids,0);
if (!network.size())
return 404;
}
_initNetwork(network); _initNetwork(network);
try { try {
@ -1481,11 +1478,14 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
network["id"] = nwids; network["id"] = nwids;
network["nwid"] = nwids; // legacy network["nwid"] = nwids; // legacy
auto rev = network["revision"]; json &rev = network["revision"];
network["revision"] = (rev.is_number() ? ((uint64_t)rev + 1ULL) : 1ULL); network["revision"] = (rev.is_number() ? ((uint64_t)rev + 1ULL) : 1ULL);
network["lastModified"] = now; network["lastModified"] = now;
_writeJson(_networkJP(nwid,true),network); {
Mutex::Lock _l(_db_m);
_db.put("network",nwids,network);
}
_NetworkMemberInfo nmi; _NetworkMemberInfo nmi;
_getNetworkMemberInfo(now,nwid,nmi); _getNetworkMemberInfo(now,nwid,nmi);
@ -1518,7 +1518,13 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE(
if ((path.size() >= 2)&&(path[1].length() == 16)) { if ((path.size() >= 2)&&(path[1].length() == 16)) {
const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); const uint64_t nwid = Utils::hexStrToU64(path[1].c_str());
json network(_readJson(_networkJP(nwid,false))); char nwids[24];
Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid);
json network;
{
Mutex::Lock _l(_db_m);
network = _db.get("network",nwids,0);
}
if (!network.size()) if (!network.size())
return 404; return 404;
@ -1526,22 +1532,23 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE(
if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) {
const uint64_t address = Utils::hexStrToU64(path[3].c_str()); const uint64_t address = Utils::hexStrToU64(path[3].c_str());
json member(_readJson(_memberJP(nwid,Address(address),false))); Mutex::Lock _l(_db_m);
json member = _db.get("network",nwids,"member",Address(address).toString(),0);
if (!member.size()) if (!member.size())
return 404; return 404;
_db.erase("network",nwids,"member",Address(address).toString());
OSUtils::rmDashRf(_memberBP(nwid,Address(address),false).c_str());
responseBody = member.dump(2); responseBody = member.dump(2);
responseContentType = "application/json"; responseContentType = "application/json";
return 200; return 200;
} }
} else { } else {
OSUtils::rmDashRf(_networkBP(nwid,false).c_str()); Mutex::Lock _l(_db_m);
{ std::string pfx("network/"); pfx.append(nwids);
Mutex::Lock _l(_networkMemberCache_m); _db.filter(pfx,120000,[](const std::string &n,const json &obj) {
_networkMemberCache.erase(nwid); return false; // delete
} });
responseBody = network.dump(2); responseBody = network.dump(2);
responseContentType = "application/json"; responseContentType = "application/json";
return 200; return 200;
@ -1618,42 +1625,46 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes
void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi) void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi)
{ {
Mutex::Lock _mcl(_networkMemberCache_m); char pfx[256];
std::map< Address,nlohmann::json > &memberCacheEntry = _networkMemberCache[nwid]; Utils::snprintf(pfx,sizeof(pfx),"network/%.16llx/member",nwid);
nmi.totalMemberCount = memberCacheEntry.size();
for(std::map< Address,nlohmann::json >::iterator nm(memberCacheEntry.begin());nm!=memberCacheEntry.end();++nm) {
if (_jB(nm->second["authorized"],false)) {
++nmi.authorizedMemberCount;
if (nm->second.count("recentLog")) { Mutex::Lock _l(_db_m);
json &mlog = nm->second["recentLog"]; _db.filter(pfx,120000,[&nmi,&now](const std::string &n,const json &member) {
if ((mlog.is_array())&&(mlog.size() > 0)) { try {
json &mlog1 = mlog[0]; if (_jB(member["authorized"],false)) {
if (mlog1.is_object()) { ++nmi.authorizedMemberCount;
if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD)
++nmi.activeMemberCount; if (member.count("recentLog")) {
const json &mlog = member["recentLog"];
if ((mlog.is_array())&&(mlog.size() > 0)) {
const json &mlog1 = mlog[0];
if (mlog1.is_object()) {
if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD)
++nmi.activeMemberCount;
}
} }
} }
}
if (_jB(nm->second["activeBridge"],false)) { if (_jB(member["activeBridge"],false)) {
nmi.activeBridges.insert(nm->first); nmi.activeBridges.insert(_jS(member["id"],"0000000000"));
} }
if (nm->second.count("ipAssignments")) { if (member.count("ipAssignments")) {
json &mips = nm->second["ipAssignments"]; const json &mips = member["ipAssignments"];
if (mips.is_array()) { if (mips.is_array()) {
for(unsigned long i=0;i<mips.size();++i) { for(unsigned long i=0;i<mips.size();++i) {
InetAddress mip(_jS(mips[i],"")); InetAddress mip(_jS(mips[i],""));
if ((mip.ss_family == AF_INET)||(mip.ss_family == AF_INET6)) if ((mip.ss_family == AF_INET)||(mip.ss_family == AF_INET6))
nmi.allocatedIps.insert(mip); nmi.allocatedIps.insert(mip);
}
} }
} }
} else {
nmi.mostRecentDeauthTime = std::max(nmi.mostRecentDeauthTime,_jI(member["lastDeauthorizedTime"],0ULL));
} }
} else { } catch ( ... ) {}
nmi.mostRecentDeauthTime = std::max(nmi.mostRecentDeauthTime,_jI(nm->second["lastDeauthorizedTime"],0ULL)); return true;
} });
}
} }
} // namespace ZeroTier } // namespace ZeroTier

14
controller/EmbeddedNetworkController.hpp
View file

@ -40,6 +40,8 @@
#include "../ext/json/json.hpp" #include "../ext/json/json.hpp"
#include "JSONDB.hpp"
namespace ZeroTier { namespace ZeroTier {
class Node; class Node;
@ -50,10 +52,6 @@ public:
EmbeddedNetworkController(Node *node,const char *dbPath); EmbeddedNetworkController(Node *node,const char *dbPath);
virtual ~EmbeddedNetworkController(); virtual ~EmbeddedNetworkController();
// Thread main method -- do not call directly
void threadMain()
throw();
virtual NetworkController::ResultCode doNetworkConfigRequest( virtual NetworkController::ResultCode doNetworkConfigRequest(
const InetAddress &fromAddr, const InetAddress &fromAddr,
const Identity &signingId, const Identity &signingId,
@ -88,6 +86,7 @@ private:
static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report);
// Network base path and network JSON path // Network base path and network JSON path
/*
inline std::string _networkBP(const uint64_t nwid,bool create) inline std::string _networkBP(const uint64_t nwid,bool create)
{ {
char tmp[64]; char tmp[64];
@ -124,6 +123,10 @@ private:
// In-memory cache of network members // In-memory cache of network members
std::map< uint64_t,std::map< Address,nlohmann::json > > _networkMemberCache; std::map< uint64_t,std::map< Address,nlohmann::json > > _networkMemberCache;
Mutex _networkMemberCache_m; Mutex _networkMemberCache_m;
*/
JSONDB _db;
Mutex _db_m;
// Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places // Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places
// This does lock _networkMemberCache_m // This does lock _networkMemberCache_m
@ -203,9 +206,6 @@ private:
// Last request time by address, for rate limitation // Last request time by address, for rate limitation
std::map< std::pair<uint64_t,uint64_t>,uint64_t > _lastRequestTime; std::map< std::pair<uint64_t,uint64_t>,uint64_t > _lastRequestTime;
Mutex _lastRequestTime_m; Mutex _lastRequestTime_m;
Thread _daemon;
volatile bool _daemonRun;
}; };
} // namespace ZeroTier } // namespace ZeroTier

58
controller/JSONDB.cpp
View file

@ -27,7 +27,7 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj)
if (!_isValidObjectName(n)) if (!_isValidObjectName(n))
return false; return false;
std::string path(_genPath(n,false)); std::string path(_genPath(n,true));
if (!path.length()) if (!path.length())
return false; return false;
@ -36,10 +36,9 @@ bool JSONDB::put(const std::string &n,const nlohmann::json &obj)
return false; return false;
_E &e = _db[n]; _E &e = _db[n];
e.obj = obj;
e.lastModifiedOnDisk = OSUtils::getLastModified(path.c_str()); e.lastModifiedOnDisk = OSUtils::getLastModified(path.c_str());
e.lastCheck = OSUtils::now(); e.lastCheck = OSUtils::now();
e.obj = obj;
return true; return true;
} }
@ -56,7 +55,6 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe
if (e != _db.end()) { if (e != _db.end()) {
if ((now - e->second.lastCheck) <= (uint64_t)maxSinceCheck) if ((now - e->second.lastCheck) <= (uint64_t)maxSinceCheck)
return e->second.obj; return e->second.obj;
std::string path(_genPath(n,false)); std::string path(_genPath(n,false));
if (!path.length()) // sanity check if (!path.length()) // sanity check
return _EMPTY_JSON; return _EMPTY_JSON;
@ -64,15 +62,13 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe
// We are somewhat tolerant to momentary disk failures here. This may // We are somewhat tolerant to momentary disk failures here. This may
// occur over e.g. EC2's elastic filesystem (NFS). // occur over e.g. EC2's elastic filesystem (NFS).
const uint64_t lm = OSUtils::getLastModified(path.c_str()); const uint64_t lm = OSUtils::getLastModified(path.c_str());
if ((lm)&&(e->second.lastModifiedOnDisk != lm)) { if (e->second.lastModifiedOnDisk != lm) {
if (OSUtils::readFile(path.c_str(),buf)) { if (OSUtils::readFile(path.c_str(),buf)) {
try { try {
e->second.lastModifiedOnDisk = lm;
e->second.lastCheck = now;
e->second.obj = nlohmann::json::parse(buf); e->second.obj = nlohmann::json::parse(buf);
} catch ( ... ) { e->second.lastModifiedOnDisk = lm; // don't update these if there is a parse error -- try again and again ASAP
e->second.obj = _EMPTY_JSON; e->second.lastCheck = now;
} } catch ( ... ) {} // parse errors result in "holding pattern" behavior
} }
} }
@ -86,22 +82,50 @@ const nlohmann::json &JSONDB::get(const std::string &n,unsigned long maxSinceChe
return _EMPTY_JSON; return _EMPTY_JSON;
const uint64_t lm = OSUtils::getLastModified(path.c_str()); const uint64_t lm = OSUtils::getLastModified(path.c_str());
if (!lm)
return _EMPTY_JSON;
_E &e2 = _db[n]; _E &e2 = _db[n];
e2.lastModifiedOnDisk = lm;
e2.lastCheck = now;
try { try {
e2.obj = nlohmann::json::parse(buf); e2.obj = nlohmann::json::parse(buf);
} catch ( ... ) { } catch ( ... ) {
e2.obj = _EMPTY_JSON; e2.obj = _EMPTY_JSON;
} }
e2.lastModifiedOnDisk = lm;
e2.lastCheck = now;
return e2.obj; return e2.obj;
} }
} }
void JSONDB::erase(const std::string &n)
{
if (!_isValidObjectName(n))
return;
std::string path(_genPath(n,true));
if (!path.length())
return;
OSUtils::rm(path.c_str());
_db.erase(n);
}
void JSONDB::_reload(const std::string &p)
{
std::map<std::string,char> l(OSUtils::listDirectoryFull(p.c_str()));
for(std::map<std::string,char>::iterator li(l.begin());li!=l.end();++li) {
if (li->second == 'f') {
// assume p starts with _basePath, which it always does -- will throw otherwise
std::string n(p.substr(_basePath.length()));
while ((n.length() > 0)&&(n[0] == ZT_PATH_SEPARATOR)) n = n.substr(1);
if (ZT_PATH_SEPARATOR != '/') std::replace(n.begin(),n.end(),ZT_PATH_SEPARATOR,'/');
if ((n.length() > 0)&&(n[n.length() - 1] != '/')) n.push_back('/');
n.append(li->first);
this->get(n,0); // causes load and cache or update
} else if (li->second == 'd') {
this->_reload(p + ZT_PATH_SEPARATOR + li->first);
}
}
}
bool JSONDB::_isValidObjectName(const std::string &n) bool JSONDB::_isValidObjectName(const std::string &n)
{ {
if (n.length() == 0) if (n.length() == 0)
@ -139,8 +163,4 @@ std::string JSONDB::_genPath(const std::string &n,bool create)
return p; return p;
} }
void JSONDB::_reloadAll(const std::string &path)
{
}
} // namespace ZeroTier } // namespace ZeroTier

36
controller/JSONDB.hpp
View file

@ -41,13 +41,21 @@ public:
JSONDB(const std::string &basePath) : JSONDB(const std::string &basePath) :
_basePath(basePath) _basePath(basePath)
{ {
this->_reloadAll(_basePath); _reload(_basePath);
}
inline void reload()
{
_db.clear();
_reload(_basePath);
} }
bool put(const std::string &n,const nlohmann::json &obj); bool put(const std::string &n,const nlohmann::json &obj);
inline bool put(const std::string &n1,const std::string &n2,const nlohmann::json &obj) { return this->put((n1 + "/" + n2),obj); } inline bool put(const std::string &n1,const std::string &n2,const nlohmann::json &obj) { return this->put((n1 + "/" + n2),obj); }
inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3),obj); } inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3),obj); }
inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4),obj); }
inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),obj); }
const nlohmann::json &get(const std::string &n,unsigned long maxSinceCheck = 0); const nlohmann::json &get(const std::string &n,unsigned long maxSinceCheck = 0);
@ -56,27 +64,37 @@ public:
inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4),maxSinceCheck); } inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4),maxSinceCheck); }
inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),maxSinceCheck); } inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,unsigned long maxSinceCheck = 0) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),maxSinceCheck); }
void erase(const std::string &n);
inline void erase(const std::string &n1,const std::string &n2) { this->erase(n1 + "/" + n2); }
inline void erase(const std::string &n1,const std::string &n2,const std::string &n3) { this->erase(n1 + "/" + n2 + "/" + n3); }
inline void erase(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4) { this->erase(n1 + "/" + n2 + "/" + n3 + "/" + n4); }
inline void erase(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5) { this->erase(n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5); }
template<typename F> template<typename F>
inline void each(F func,unsigned long maxSinceCheck = 0) inline void filter(const std::string &prefix,unsigned long maxSinceCheck,F func)
{ {
const uint64_t now = OSUtils::now(); for(std::map<std::string,_E>::iterator i(_db.lower_bound(prefix));i!=_db.end();) {
for(std::map<std::string,_E>::const_iterator i(_db.begin());i!=_db.end();++i) { if (i->first.substr(0,prefix.length()) == prefix) {
if ((now - i->second.lastCheck) > (uint64_t)maxSinceCheck) if (!func(i->first,get(i->second.obj,maxSinceCheck))) {
this->get(i->first); std::map<std::string,_E>::iterator i2(i); ++i2;
func(i->first,i->second.obj); this->erase(i->first);
i = i2;
} else ++i;
} else break;
} }
} }
private: private:
void _reload(const std::string &p);
bool _isValidObjectName(const std::string &n); bool _isValidObjectName(const std::string &n);
std::string _genPath(const std::string &n,bool create); std::string _genPath(const std::string &n,bool create);
void _reloadAll(const std::string &path);
struct _E struct _E
{ {
nlohmann::json obj;
uint64_t lastModifiedOnDisk; uint64_t lastModifiedOnDisk;
uint64_t lastCheck; uint64_t lastCheck;
nlohmann::json obj;
}; };
std::string _basePath; std::string _basePath;

16
osdep/OSUtils.cpp
View file

@ -107,17 +107,18 @@ std::vector<std::string> OSUtils::listDirectory(const char *path)
return r; return r;
} }
std::vector<std::string> OSUtils::listSubdirectories(const char *path) std::map<std::string,char> OSUtils::listDirectoryFull(const char *path)
{ {
std::vector<std::string> r; std::map<std::string,char> r;
#ifdef __WINDOWS__ #ifdef __WINDOWS__
HANDLE hFind; HANDLE hFind;
WIN32_FIND_DATAA ffd; WIN32_FIND_DATAA ffd;
if ((hFind = FindFirstFileA((std::string(path) + "\\*").c_str(),&ffd)) != INVALID_HANDLE_VALUE) { if ((hFind = FindFirstFileA((std::string(path) + "\\*").c_str(),&ffd)) != INVALID_HANDLE_VALUE) {
do { do {
if ((strcmp(ffd.cFileName,"."))&&(strcmp(ffd.cFileName,".."))&&((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0)) if ((strcmp(ffd.cFileName,"."))&&(strcmp(ffd.cFileName,".."))) {
r.push_back(std::string(ffd.cFileName)); r[ffd.cFileName] = ((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) ? 'd' : 'f';
}
} while (FindNextFileA(hFind,&ffd)); } while (FindNextFileA(hFind,&ffd));
FindClose(hFind); FindClose(hFind);
} }
@ -132,8 +133,9 @@ std::vector<std::string> OSUtils::listSubdirectories(const char *path)
if (readdir_r(d,&de,&dptr)) if (readdir_r(d,&de,&dptr))
break; break;
if (dptr) { if (dptr) {
if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))&&(dptr->d_type == DT_DIR)) if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))) {
r.push_back(std::string(dptr->d_name)); r[dptr->d_name] = (dptr->d_type == DT_DIR) ? 'd' : 'f';
}
} else break; } else break;
} }
closedir(d); closedir(d);
@ -178,7 +180,7 @@ bool OSUtils::rmDashRf(const char *path)
std::string p(path); std::string p(path);
p.push_back(ZT_PATH_SEPARATOR); p.push_back(ZT_PATH_SEPARATOR);
p.append(dptr->d_name); p.append(dptr->d_name);
if (unlink(p.c_str()) != 0) { if (unlink(p.c_str()) != 0) { // unlink first will remove symlinks instead of recursing them
if (!rmDashRf(p.c_str())) if (!rmDashRf(p.c_str()))
return false; return false;
} }

6
osdep/OSUtils.hpp
View file

@ -110,12 +110,12 @@ public:
static std::vector<std::string> listDirectory(const char *path); static std::vector<std::string> listDirectory(const char *path);
/** /**
* List a directory's subdirectories * List all contents in a directory
* *
* @param path Path to list * @param path Path to list
* @return Names of subdirectories (without path prepended) * @return Names of things and types, currently just 'f' and 'd'
*/ */
static std::vector<std::string> listSubdirectories(const char *path); static std::map<std::string,char> listDirectoryFull(const char *path);
/** /**
* Delete a directory and all its files and subdirectories recursively * Delete a directory and all its files and subdirectories recursively