From 9de2b90528d1fe7efe963acca3313defd1eb5662 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Wed, 27 Mar 2024 14:40:38 -0700 Subject: [PATCH 1/4] fix rebuild for x64 --- rustybits/zeroidc.vcxproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rustybits/zeroidc.vcxproj b/rustybits/zeroidc.vcxproj index b31ed0ee4..833437238 100644 --- a/rustybits/zeroidc.vcxproj +++ b/rustybits/zeroidc.vcxproj @@ -91,7 +91,7 @@ cargo clean - cargo clean & cargo build --release --target=x86_64-pc-windows-msvc + cargo clean & cargo build -p zeroidc --release --target=x86_64-pc-windows-msvc NDEBUG;$(NMakePreprocessorDefinitions) From ad60d708e1bd4880f341b2622ed808eab038bb9c Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Mon, 17 Jun 2024 16:32:53 -0700 Subject: [PATCH 2/4] fix log line --- controller/PostgreSQL.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index abfdbd31d..010df3b9d 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -1180,7 +1180,7 @@ void PostgreSQL::_membersWatcher_Redis() { _memberChanged(oldConfig,newConfig,(this->_ready >= 2)); } } catch (...) { - fprintf(stderr, "json parse error in networkWatcher_Redis\n"); + fprintf(stderr, "json parse error in _membersWatcher_Redis: %s\n", a.second.c_str()); } } if (_rc->clusterMode) { @@ -1269,8 +1269,8 @@ void PostgreSQL::_networksWatcher_Redis() { if (oldConfig.is_object()||newConfig.is_object()) { _networkChanged(oldConfig,newConfig,(this->_ready >= 2)); } - } catch (...) { - fprintf(stderr, "json parse error in networkWatcher_Redis\n"); + } catch (std::exception &e) { + fprintf(stderr, "json parse error in networkWatcher_Redis: what: %s json: %s\n", e.what(), a.second.c_str()); } } if (_rc->clusterMode) { From 9d57ccd7b145071c662e57d5b7a27c14f5d0f636 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Mon, 17 Jun 2024 16:35:08 -0700 Subject: [PATCH 3/4] deauth all members upon network delete --- controller/DB.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/controller/DB.cpp b/controller/DB.cpp index fe2973990..2c354ae7f 100644 --- a/controller/DB.cpp +++ b/controller/DB.cpp @@ -382,6 +382,24 @@ void DB::_networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool const std::string ids = old["id"]; const uint64_t networkId = Utils::hexStrToU64(ids.c_str()); if (networkId) { + try { + // deauth all members on the network + nlohmann::json network; + std::vector members; + this->get(networkId, network, members); + for(auto i=members.begin();i!=members.end();++i) { + const std::string nodeID = (*i)["id"]; + const uint64_t memberId = Utils::hexStrToU64(nodeID.c_str()); + std::unique_lock ll(_changeListeners_l); + for(auto j=_changeListeners.begin();j!=_changeListeners.end();++j) { + (*j)->onNetworkMemberDeauthorize(this,networkId,memberId); + } + } + } catch (std::exception &e) { + std::cerr << "Error deauthorizing members on network delete: " << e.what() << std::endl; + } + + // delete the network std::unique_lock l(_networks_l); _networks.erase(networkId); } From 508527f7cdcf622b51cedab86d95be79a82d9a37 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Mon, 8 Jul 2024 14:08:54 -0700 Subject: [PATCH 4/4] break up redis tx inserts into smaller chunks --- controller/PostgreSQL.cpp | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index 010df3b9d..e4a31ba28 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -780,11 +780,25 @@ void PostgreSQL::initializeNetworks() fprintf(stderr, "adding networks to redis...\n"); if (_rc->clusterMode) { auto tx = _cluster->transaction(_myAddressStr, true, false); - tx.sadd(setKey, networkSet.begin(), networkSet.end()); + uint64_t count = 0; + for (std::string nwid : networkSet) { + tx.sadd(setKey, nwid); + if (++count % 30000 == 0) { + tx.exec(); + tx = _cluster->transaction(_myAddressStr, true, false); + } + } tx.exec(); } else { auto tx = _redis->transaction(true, false); - tx.sadd(setKey, networkSet.begin(), networkSet.end()); + uint64_t count = 0; + for (std::string nwid : networkSet) { + tx.sadd(setKey, nwid); + if (++count % 30000 == 0) { + tx.exec(); + tx = _redis->transaction(true, false); + } + } tx.exec(); } fprintf(stderr, "done.\n"); @@ -1005,14 +1019,24 @@ void PostgreSQL::initializeMembers() fprintf(stderr, "Load member data into redis...\n"); if (_rc->clusterMode) { auto tx = _cluster->transaction(_myAddressStr, true, false); + uint64_t count = 0; for (auto it : networkMembers) { tx.sadd(it.first, it.second); + if (++count % 30000 == 0) { + tx.exec(); + tx = _cluster->transaction(_myAddressStr, true, false); + } } tx.exec(); } else { auto tx = _redis->transaction(true, false); + uint64_t count = 0; for (auto it : networkMembers) { tx.sadd(it.first, it.second); + if (++count % 30000 == 0) { + tx.exec(); + tx = _redis->transaction(true, false); + } } tx.exec(); }