From 760ae07d7243b8e7ff6ec03cf508058c287c219f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 8 Aug 2019 16:30:52 -0500 Subject: [PATCH 1/7] Sync bug fix --- controller/LFDB.cpp | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/controller/LFDB.cpp b/controller/LFDB.cpp index 35068321d..d00d8631c 100644 --- a/controller/LFDB.cpp +++ b/controller/LFDB.cpp @@ -220,20 +220,16 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons const uint64_t id = Utils::hexStrToU64(idstr.c_str()); if ((id >> 24) == controllerAddressInt) { // sanity check - std::lock_guard sl(_state_l); - _NetworkState &ns = _state[id]; - if (!ns.dirty) { - nlohmann::json oldNetwork; - if ((timeRangeStart > 0)&&(get(id,oldNetwork))) { - const uint64_t revision = network["revision"]; - const uint64_t prevRevision = oldNetwork["revision"]; - if (prevRevision < revision) { - _networkChanged(oldNetwork,network,timeRangeStart > 0); - } - } else { - nlohmann::json nullJson; - _networkChanged(nullJson,network,timeRangeStart > 0); + nlohmann::json oldNetwork; + if ((timeRangeStart > 0)&&(get(id,oldNetwork))) { + const uint64_t revision = network["revision"]; + const uint64_t prevRevision = oldNetwork["revision"]; + if (prevRevision < revision) { + _networkChanged(oldNetwork,network,timeRangeStart > 0); } + } else { + nlohmann::json nullJson; + _networkChanged(nullJson,network,timeRangeStart > 0); } } @@ -294,17 +290,13 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons const uint64_t id = Utils::hexStrToU64(idstr.c_str()); if ((id)&&((nwid >> 24) == controllerAddressInt)) { // sanity check - std::lock_guard sl(_state_l); - auto ns = _state.find(nwid); - if ((ns == _state.end())||(!ns->second.members[id].dirty)) { - nlohmann::json network,oldMember; - if ((timeRangeStart > 0)&&(get(nwid,network,id,oldMember))) { - const uint64_t revision = member["revision"]; - const uint64_t prevRevision = oldMember["revision"]; - if (prevRevision < revision) - _memberChanged(oldMember,member,timeRangeStart > 0); - } - } else { + nlohmann::json network,oldMember; + if ((timeRangeStart > 0)&&(get(nwid,network,id,oldMember))) { + const uint64_t revision = member["revision"]; + const uint64_t prevRevision = oldMember["revision"]; + if (prevRevision < revision) + _memberChanged(oldMember,member,timeRangeStart > 0); + } else if (network.is_object()) { nlohmann::json nullJson; _memberChanged(nullJson,member,timeRangeStart > 0); } From 55087521f6db9aaa874cccd86e424d77e6c820c3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 8 Aug 2019 17:20:50 -0500 Subject: [PATCH 2/7] Add periodic sync check when syncing LF<>another DB --- controller/DB.hpp | 15 ++++++++++++- controller/DBMirrorSet.cpp | 46 +++++++++++++++++++++++++++++++++++++- controller/DBMirrorSet.hpp | 3 +++ controller/LFDB.cpp | 2 +- 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/controller/DB.hpp b/controller/DB.hpp index aebe4e112..6c518426c 100644 --- a/controller/DB.hpp +++ b/controller/DB.hpp @@ -100,6 +100,19 @@ public: void networks(std::set &networks); + template + inline void each(F f) + { + nlohmann::json nullJson; + std::lock_guard lck(_networks_l); + for(auto nw=_networks.begin();nw!=_networks.end();++nw) { + f(nw->first,nw->second->config,0,nullJson); // first provide network with 0 for member ID + for(auto m=nw->second->members.begin();m!=nw->second->members.end();++m) { + f(nw->first,nw->second->config,m->first,m->second); + } + } + } + virtual bool save(nlohmann::json &record,bool notifyListeners) = 0; virtual void eraseNetwork(const uint64_t networkId) = 0; @@ -114,7 +127,7 @@ public: } protected: - inline bool _compareRecords(const nlohmann::json &a,const nlohmann::json &b) + static inline bool _compareRecords(const nlohmann::json &a,const nlohmann::json &b) { if (a.is_object() == b.is_object()) { if (a.is_object()) { diff --git a/controller/DBMirrorSet.cpp b/controller/DBMirrorSet.cpp index 5b491216d..852e70f30 100644 --- a/controller/DBMirrorSet.cpp +++ b/controller/DBMirrorSet.cpp @@ -29,8 +29,52 @@ namespace ZeroTier { DBMirrorSet::DBMirrorSet(DB::ChangeListener *listener) : - _listener(listener) + _listener(listener), + _running(true) { + _syncCheckerThread = std::thread([this]() { + for(;;) { + for(int i=0;i<120;++i) { // 1 minute delay between checks + if (!_running) + return; + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + + std::vector< std::shared_ptr > dbs; + { + std::lock_guard l(_dbs_l); + if (_dbs.size() <= 1) + continue; // no need to do this if there's only one DB, so skip the iteration + dbs = _dbs; + } + + for(auto db=dbs.begin();db!=dbs.end();++db) { + (*db)->each([this,&dbs,&db](uint64_t networkId,const nlohmann::json &network,uint64_t memberId,const nlohmann::json &member) { + if (memberId == 0) { + for(auto db2=dbs.begin();db2!=dbs.end();++db2) { + if (db->get() != db2->get()) { + nlohmann::json nw2; + if ((!(*db2)->get(networkId,nw2))||(OSUtils::jsonInt(nw2["revision"],0) < OSUtils::jsonInt(network["revision"],0))) { + nw2 = network; + (*db2)->save(nw2,false); + } + } + } + } else { + for(auto db2=dbs.begin();db2!=dbs.end();++db2) { + if (db->get() != db2->get()) { + nlohmann::json nw2,m2; + if ((!(*db2)->get(networkId,nw2,memberId,m2))||(OSUtils::jsonInt(nw2["revision"],0) < OSUtils::jsonInt(network["revision"],0))) { + m2 = member; + (*db2)->save(m2,false); + } + } + } + } + }); + } + } + }); } DBMirrorSet::~DBMirrorSet() diff --git a/controller/DBMirrorSet.hpp b/controller/DBMirrorSet.hpp index aee598af2..23cb25e71 100644 --- a/controller/DBMirrorSet.hpp +++ b/controller/DBMirrorSet.hpp @@ -33,6 +33,7 @@ #include #include #include +#include namespace ZeroTier { @@ -72,6 +73,8 @@ public: private: DB::ChangeListener *const _listener; + std::atomic_bool _running; + std::thread _syncCheckerThread; std::vector< std::shared_ptr< DB > > _dbs; mutable std::mutex _dbs_l; }; diff --git a/controller/LFDB.cpp b/controller/LFDB.cpp index d00d8631c..b6dc66578 100644 --- a/controller/LFDB.cpp +++ b/controller/LFDB.cpp @@ -296,7 +296,7 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons const uint64_t prevRevision = oldMember["revision"]; if (prevRevision < revision) _memberChanged(oldMember,member,timeRangeStart > 0); - } else if (network.is_object()) { + } else if (hasNetwork(nwid)) { nlohmann::json nullJson; _memberChanged(nullJson,member,timeRangeStart > 0); } From f8f2432ece59fbc0913d011423ecc1e369924bcd Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 8 Aug 2019 17:22:20 -0500 Subject: [PATCH 3/7] Shutdown fix --- controller/DBMirrorSet.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/controller/DBMirrorSet.cpp b/controller/DBMirrorSet.cpp index 852e70f30..c1be6b9ce 100644 --- a/controller/DBMirrorSet.cpp +++ b/controller/DBMirrorSet.cpp @@ -79,6 +79,8 @@ DBMirrorSet::DBMirrorSet(DB::ChangeListener *listener) : DBMirrorSet::~DBMirrorSet() { + _running = false; + _syncCheckerThread.join(); } bool DBMirrorSet::hasNetwork(const uint64_t networkId) const From 730305ecc7961752d10dae2bcb195e4de1c45464 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 8 Aug 2019 17:31:32 -0500 Subject: [PATCH 4/7] One more sync fix. --- controller/DBMirrorSet.cpp | 40 +++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/controller/DBMirrorSet.cpp b/controller/DBMirrorSet.cpp index c1be6b9ce..c3cd7841f 100644 --- a/controller/DBMirrorSet.cpp +++ b/controller/DBMirrorSet.cpp @@ -50,27 +50,31 @@ DBMirrorSet::DBMirrorSet(DB::ChangeListener *listener) : for(auto db=dbs.begin();db!=dbs.end();++db) { (*db)->each([this,&dbs,&db](uint64_t networkId,const nlohmann::json &network,uint64_t memberId,const nlohmann::json &member) { - if (memberId == 0) { - for(auto db2=dbs.begin();db2!=dbs.end();++db2) { - if (db->get() != db2->get()) { - nlohmann::json nw2; - if ((!(*db2)->get(networkId,nw2))||(OSUtils::jsonInt(nw2["revision"],0) < OSUtils::jsonInt(network["revision"],0))) { - nw2 = network; - (*db2)->save(nw2,false); + try { + if (network.is_object()) { + if (memberId == 0) { + for(auto db2=dbs.begin();db2!=dbs.end();++db2) { + if (db->get() != db2->get()) { + nlohmann::json nw2; + if ((!(*db2)->get(networkId,nw2))||((nw2.is_object())&&(OSUtils::jsonInt(nw2["revision"],0) < OSUtils::jsonInt(network["revision"],0)))) { + nw2 = network; + (*db2)->save(nw2,false); + } + } + } + } else if (member.is_object()) { + for(auto db2=dbs.begin();db2!=dbs.end();++db2) { + if (db->get() != db2->get()) { + nlohmann::json nw2,m2; + if ((!(*db2)->get(networkId,nw2,memberId,m2))||((m2.is_object())&&(OSUtils::jsonInt(m2["revision"],0) < OSUtils::jsonInt(member["revision"],0)))) { + m2 = member; + (*db2)->save(m2,false); + } + } } } } - } else { - for(auto db2=dbs.begin();db2!=dbs.end();++db2) { - if (db->get() != db2->get()) { - nlohmann::json nw2,m2; - if ((!(*db2)->get(networkId,nw2,memberId,m2))||(OSUtils::jsonInt(nw2["revision"],0) < OSUtils::jsonInt(network["revision"],0))) { - m2 = member; - (*db2)->save(m2,false); - } - } - } - } + } catch ( ... ) {} // skip entries that generate JSON errors }); } } From 6a9a577c599875b821afa285ae3a964dd78d3301 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 8 Aug 2019 17:56:27 -0500 Subject: [PATCH 5/7] rulesSource needs to be required in network or pgsql has issues --- controller/DB.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/controller/DB.cpp b/controller/DB.cpp index a4440f388..0c6274b9c 100644 --- a/controller/DB.cpp +++ b/controller/DB.cpp @@ -52,6 +52,7 @@ void DB::initNetwork(nlohmann::json &network) if (!network.count("mtu")) network["mtu"] = ZT_DEFAULT_MTU; if (!network.count("remoteTraceTarget")) network["remoteTraceTarget"] = nlohmann::json(); if (!network.count("removeTraceLevel")) network["remoteTraceLevel"] = 0; + if (!network.count("rulesSource")) network["rulesSource"] = ""; if (!network.count("rules")) { // If unspecified, rules are set to allow anything and behave like a flat L2 segment network["rules"] = {{ From 72ccd1020a47970e622912b2f30b8a0dbb03bb55 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 8 Aug 2019 18:11:06 -0500 Subject: [PATCH 6/7] Another exception fix --- controller/PostgreSQL.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index 66b49394e..cd913f87a 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -1047,7 +1047,10 @@ void PostgreSQL::commitThread() if (!(*config)["remoteTraceTarget"].is_null()) { remoteTraceTarget = (*config)["remoteTraceTarget"]; } - std::string rulesSource = (*config)["rulesSource"]; + std::string rulesSource; + if ((*config)["rulesSource"].is_string()) { + rulesSource = (*config)["rulesSource"]; + } std::string caps = OSUtils::jsonDump((*config)["capabilitles"], -1); std::string now = std::to_string(OSUtils::now()); std::string mtu = std::to_string((int)(*config)["mtu"]); From d4dc983a01973d4148c5f4344d86b9534fd61432 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 8 Aug 2019 18:31:41 -0500 Subject: [PATCH 7/7] fix bad column name --- controller/PostgreSQL.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index cd913f87a..07bb5825a 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -1084,13 +1084,13 @@ void PostgreSQL::commitThread() PGresult *res = PQexecParams(conn, "INSERT INTO ztc_network (id, controller_id, capabilities, enable_broadcast, " - "last_updated, mtu, multicast_limit, name, private, " + "last_modified, mtu, multicast_limit, name, private, " "remote_trace_level, remote_trace_target, rules, rules_source, " "tags, v4_assign_mode, v6_assign_mode) VALUES (" "$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) " "ON CONFLICT (id) DO UPDATE set controller_id = EXCLUDED.controller_id, " "capabilities = EXCLUDED.capabilities, enable_broadcast = EXCLUDED.enable_broadcast, " - "last_updated = EXCLUDED.last_updated, mtu = EXCLUDED.mtu, " + "last_modified = EXCLUDED.last_modified, mtu = EXCLUDED.mtu, " "multicast_limit = EXCLUDED.multicast_limit, name = EXCLUDED.name, " "private = EXCLUDED.private, remote_trace_level = EXCLUDED.remote_trace_level, " "remote_trace_target = EXCLUDED.remote_trace_target, rules = EXCLUDED.rules, "