WIP: refactoring for CV2 db integration

controller/CV1.hpp

@@ -0,0 +1,143 @@
+/*
+ * Copyright (c)2019 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "DB.hpp"
+
+#ifdef ZT_CONTROLLER_USE_LIBPQ
+
+#ifndef ZT_CONTROLLER_LIBPQ_HPP
+#define ZT_CONTROLLER_LIBPQ_HPP
+
+#define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4
+
+#include "ConnectionPool.hpp"
+#include <pqxx/pqxx>
+
+#include <memory>
+#include <redis++/redis++.h>
+
+#include "../node/Metrics.hpp"
+
+#include "PostgreSQL.hpp"
+
+
+namespace smeeclient {
+	struct SmeeClient;
+}
+
+namespace ZeroTier {
+
+struct RedisConfig;
+
+/**
+ * A controller database driver that talks to  PostgreSQL
+ *
+ * This is for use with ZeroTier Central.  Others are free to build and use it
+ * but be aware that we might change it at any time.
+ */
+class CV1 : public DB
+{
+public:
+	CV1(const Identity &myId, const char *path, int listenPort, RedisConfig *rc);
+	virtual ~CV1();
+
+	virtual bool waitForReady();
+	virtual bool isReady();
+	virtual bool save(nlohmann::json &record,bool notifyListeners);
+	virtual void eraseNetwork(const uint64_t networkId);
+	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress &physicalAddress);
+	virtual AuthInfo getSSOAuthInfo(const nlohmann::json &member, const std::string &redirectURL);
+
+	virtual bool ready() {
+		return _ready == 2;
+	}
+
+protected:
+	struct _PairHasher
+	{
+		inline std::size_t operator()(const std::pair<uint64_t,uint64_t> &p) const { return (std::size_t)(p.first ^ p.second); }
+	};
+	virtual void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners) {
+		DB::_memberChanged(old, memberConfig, notifyListeners);
+	}
+
+	virtual void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool notifyListeners) {
+		DB::_networkChanged(old, networkConfig, notifyListeners);
+	}
+
+private:
+	void initializeNetworks();
+	void initializeMembers();
+	void heartbeat();
+	void membersDbWatcher();
+	void _membersWatcher_Postgres();
+	void networksDbWatcher();
+	void _networksWatcher_Postgres();
+
+	void _membersWatcher_Redis();
+	void _networksWatcher_Redis();
+
+	void commitThread();
+	void onlineNotificationThread();
+	void onlineNotification_Postgres();
+	void onlineNotification_Redis();
+	uint64_t _doRedisUpdate(sw::redis::Transaction &tx, std::string &controllerId,
+		std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > &lastOnline);
+
+	void configureSmee();
+	void notifyNewMember(const std::string &networkID, const std::string &memberID);
+
+	enum OverrideMode {
+		ALLOW_PGBOUNCER_OVERRIDE = 0,
+		NO_OVERRIDE = 1
+	};
+
+	std::shared_ptr<ConnectionPool<PostgresConnection> > _pool;
+
+	const Identity _myId;
+	const Address _myAddress;
+	std::string _myAddressStr;
+	std::string _connString;
+
+	BlockingQueue< std::pair<nlohmann::json,bool> > _commitQueue;
+
+	std::thread _heartbeatThread;
+	std::thread _membersDbWatcher;
+	std::thread _networksDbWatcher;
+	std::thread _commitThread[ZT_CENTRAL_CONTROLLER_COMMIT_THREADS];
+	std::thread _onlineNotificationThread;
+
+	std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > _lastOnline;
+
+	mutable std::mutex _lastOnline_l;
+	mutable std::mutex _readyLock;
+	std::atomic<int> _ready, _connected, _run;
+	mutable volatile bool _waitNoticePrinted;
+
+	int _listenPort;
+	uint8_t _ssoPsk[48];
+
+	RedisConfig *_rc;
+	std::shared_ptr<sw::redis::Redis> _redis;
+	std::shared_ptr<sw::redis::RedisCluster> _cluster;
+    bool _redisMemberStatus;
+
+	smeeclient::SmeeClient *_smee;
+};
+
+} // namespace ZeroTier
+
+#endif // ZT_CONTROLLER_LIBPQ_HPP
+
+#endif // ZT_CONTROLLER_USE_LIBPQ

controller/DB.hpp

@@ -75,6 +75,10 @@ public:
  */
 class DB
 {
+#ifdef ZT_CONTROLLER_USE_LIBPQ
+	friend class MemberNotificationReceiver;
+	friend class NetworkNotificationReceiver;
+#endif
 public:
 	class ChangeListener
 	{
@@ -146,6 +150,10 @@ public:
 		_changeListeners.push_back(listener);
 	}
 
+	virtual bool ready() {
+		return true;
+	}
+
 protected:
 	static inline bool _compareRecords(const nlohmann::json &a,const nlohmann::json &b)
 	{

controller/EmbeddedNetworkController.cpp

@@ -41,6 +41,7 @@
 #include "FileDB.hpp"
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 #include "PostgreSQL.hpp"
+#include "CV1.hpp"
 #endif
 
 #include "../node/Node.hpp"
@@ -534,7 +535,7 @@ void EmbeddedNetworkController::init(const Identity &signingId,Sender *sender)
 
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 	if ((_path.length() > 9)&&(_path.substr(0,9) == "postgres:")) {
-		_db.addDB(std::shared_ptr<DB>(new PostgreSQL(_signingId,_path.substr(9).c_str(), _listenPort, _rc)));
+		_db.addDB(std::shared_ptr<DB>(new CV1(_signingId,_path.substr(9).c_str(), _listenPort, _rc)));
 	} else {
 #endif
 		_db.addDB(std::shared_ptr<DB>(new FileDB(_path.c_str())));

controller/PostgreSQL.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c)2019 ZeroTier, Inc.
+ * Copyright (c)2025 ZeroTier, Inc.
  *
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
@@ -11,35 +11,22 @@
  */
 /****/
 
-#include "DB.hpp"
-
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 
-#ifndef ZT_CONTROLLER_LIBPQ_HPP
+#ifndef ZT_CONTROLLER_POSTGRESQL_HPP
-#define ZT_CONTROLLER_LIBPQ_HPP
+#define ZT_CONTROLLER_POSTGRESQL_HPP
-
-#define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4
 
+#include "DB.hpp"
 #include "ConnectionPool.hpp"
 #include <pqxx/pqxx>
-
 #include <memory>
-#include <redis++/redis++.h>
 
-#include "../node/Metrics.hpp"
+namespace ZeroTier {
 
 extern "C" {
 typedef struct pg_conn PGconn;
 }
 
-namespace smeeclient {
-	struct SmeeClient;
-}
-
-namespace ZeroTier {
-
-struct RedisConfig;
-
 
 class PostgresConnection : public Connection {
 public:
@@ -68,129 +55,32 @@ private:
 	std::string m_connString;
 };
 
-class PostgreSQL;
-
 class MemberNotificationReceiver : public pqxx::notification_receiver {
 public: 
-	MemberNotificationReceiver(PostgreSQL *p, pqxx::connection &c, const std::string &channel);
+	MemberNotificationReceiver(DB *p, pqxx::connection &c, const std::string &channel);
 	virtual ~MemberNotificationReceiver() {
 		fprintf(stderr, "MemberNotificationReceiver destroyed\n");
 	}
 
 	virtual void operator() (const std::string &payload, int backendPid);
 private:
-	PostgreSQL *_psql;
+	DB *_psql;
 };
 
 class NetworkNotificationReceiver : public pqxx::notification_receiver {
 public:
-	NetworkNotificationReceiver(PostgreSQL *p, pqxx::connection &c, const std::string &channel);
+	NetworkNotificationReceiver(DB *p, pqxx::connection &c, const std::string &channel);
 	virtual ~NetworkNotificationReceiver() {
 		fprintf(stderr, "NetworkNotificationReceiver destroyed\n");
 	};
 
 	virtual void operator() (const std::string &payload, int packend_pid);
 private:
-	PostgreSQL *_psql;
+	DB *_psql;
-};
-
-/**
- * A controller database driver that talks to PostgreSQL
- *
- * This is for use with ZeroTier Central.  Others are free to build and use it
- * but be aware that we might change it at any time.
- */
-class PostgreSQL : public DB
-{
-	friend class MemberNotificationReceiver;
-	friend class NetworkNotificationReceiver;
-public:
-	PostgreSQL(const Identity &myId, const char *path, int listenPort, RedisConfig *rc);
-	virtual ~PostgreSQL();
-
-	virtual bool waitForReady();
-	virtual bool isReady();
-	virtual bool save(nlohmann::json &record,bool notifyListeners);
-	virtual void eraseNetwork(const uint64_t networkId);
-	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
-	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress &physicalAddress);
-	virtual AuthInfo getSSOAuthInfo(const nlohmann::json &member, const std::string &redirectURL);
-
-protected:
-	struct _PairHasher
-	{
-		inline std::size_t operator()(const std::pair<uint64_t,uint64_t> &p) const { return (std::size_t)(p.first ^ p.second); }
-	};
-	virtual void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners) {
-		DB::_memberChanged(old, memberConfig, notifyListeners);
-	}
-
-	virtual void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool notifyListeners) {
-		DB::_networkChanged(old, networkConfig, notifyListeners);
-	}
-
-private:
-	void initializeNetworks();
-	void initializeMembers();
-	void heartbeat();
-	void membersDbWatcher();
-	void _membersWatcher_Postgres();
-	void networksDbWatcher();
-	void _networksWatcher_Postgres();
-
-	void _membersWatcher_Redis();
-	void _networksWatcher_Redis();
-
-	void commitThread();
-	void onlineNotificationThread();
-	void onlineNotification_Postgres();
-	void onlineNotification_Redis();
-	uint64_t _doRedisUpdate(sw::redis::Transaction &tx, std::string &controllerId,
-		std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > &lastOnline);
-
-	void configureSmee();
-	void notifyNewMember(const std::string &networkID, const std::string &memberID);
-
-	enum OverrideMode {
-		ALLOW_PGBOUNCER_OVERRIDE = 0,
-		NO_OVERRIDE = 1
-	};
-
-	std::shared_ptr<ConnectionPool<PostgresConnection> > _pool;
-
-	const Identity _myId;
-	const Address _myAddress;
-	std::string _myAddressStr;
-	std::string _connString;
-
-	BlockingQueue< std::pair<nlohmann::json,bool> > _commitQueue;
-
-	std::thread _heartbeatThread;
-	std::thread _membersDbWatcher;
-	std::thread _networksDbWatcher;
-	std::thread _commitThread[ZT_CENTRAL_CONTROLLER_COMMIT_THREADS];
-	std::thread _onlineNotificationThread;
-
-	std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > _lastOnline;
-
-	mutable std::mutex _lastOnline_l;
-	mutable std::mutex _readyLock;
-	std::atomic<int> _ready, _connected, _run;
-	mutable volatile bool _waitNoticePrinted;
-
-	int _listenPort;
-	uint8_t _ssoPsk[48];
-
-	RedisConfig *_rc;
-	std::shared_ptr<sw::redis::Redis> _redis;
-	std::shared_ptr<sw::redis::RedisCluster> _cluster;
-    bool _redisMemberStatus;
-
-	smeeclient::SmeeClient *_smee;
 };
 
 } // namespace ZeroTier
 
-#endif // ZT_CONTROLLER_LIBPQ_HPP
+#endif // ZT_CONTROLLER_POSTGRESQL_HPP
 
 #endif // ZT_CONTROLLER_USE_LIBPQ

make-mac.mk

@@ -57,9 +57,9 @@ ONE_OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connec
 ifeq ($(ZT_CONTROLLER),1)
 	MACOS_VERSION_MIN=10.15
 	override CXXFLAGS=$(CFLAGS) -std=c++17 -stdlib=libc++
-	LIBS+=-L/usr/local/opt/libpqxx/lib -L/usr/local/opt/libpq/lib -L/usr/local/opt/openssl/lib/ -lpqxx -lpq -lssl -lcrypto -lgssapi_krb5 ext/redis-plus-plus-1.1.1/install/macos/lib/libredis++.a ext/hiredis-0.14.1/lib/macos/libhiredis.a
+	LIBS+=-L/opt/homebrew/lib -L/usr/local/opt/libpqxx/lib -L/usr/local/opt/libpq/lib -L/usr/local/opt/openssl/lib/ -lpqxx -lpq -lssl -lcrypto -lgssapi_krb5 ext/redis-plus-plus-1.1.1/install/macos/lib/libredis++.a ext/hiredis-0.14.1/lib/macos/libhiredis.a rustybits/target/libsmeeclient.a
 	DEFS+=-DZT_CONTROLLER_USE_LIBPQ -DZT_CONTROLLER_USE_REDIS -DZT_CONTROLLER 
-	INCLUDES+=-I/usr/local/opt/libpq/include -I/usr/local/opt/libpqxx/include -Iext/hiredis-0.14.1/include/ -Iext/redis-plus-plus-1.1.1/install/macos/include/sw/
+	INCLUDES+=-I/opt/homebrew/include -I/opt/homebrew/opt/libpq/include -I/usr/local/opt/libpq/include -I/usr/local/opt/libpqxx/include -Iext/hiredis-0.14.1/include/ -Iext/redis-plus-plus-1.1.1/install/macos/include/sw/ -Irustybits/target/
 else
 	MACOS_VERSION_MIN=10.13
 endif
@@ -115,7 +115,11 @@ mac-agent: FORCE
 osdep/MacDNSHelper.o: osdep/MacDNSHelper.mm
 	$(CXX) $(CXXFLAGS) -c osdep/MacDNSHelper.mm -o osdep/MacDNSHelper.o 
 
+ifeq ($(ZT_CONTROLLER),1)
+one:	zeroidc smeeclient $(CORE_OBJS) $(ONE_OBJS) one.o mac-agent 
+else
 one:	zeroidc $(CORE_OBJS) $(ONE_OBJS) one.o mac-agent 
+endif
 	$(CXX) $(CXXFLAGS) -o zerotier-one $(CORE_OBJS) $(ONE_OBJS) one.o $(LIBS) rustybits/target/libzeroidc.a
 	# $(STRIP) zerotier-one
 	ln -sf zerotier-one zerotier-idtool
@@ -126,6 +130,15 @@ zerotier-one: one
 
 zeroidc: rustybits/target/libzeroidc.a
 
+ifeq ($(ZT_CONTROLLER),1)
+smeeclient: rustybits/target/libsmeeclient.a
+
+rustybits/target/libsmeeclient.a:	FORCE
+	cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p smeeclient --target=x86_64-apple-darwin $(EXTRA_CARGO_FLAGS)
+	cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p smeeclient --target=aarch64-apple-darwin $(EXTRA_CARGO_FLAGS)
+	cd rustybits && lipo -create target/x86_64-apple-darwin/$(RUST_VARIANT)/libsmeeclient.a target/aarch64-apple-darwin/$(RUST_VARIANT)/libsmeeclient.a -output target/libsmeeclient.a
+endif
+
 rustybits/target/libzeroidc.a:	FORCE
 	cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p zeroidc --target=x86_64-apple-darwin $(EXTRA_CARGO_FLAGS)
 	cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p zeroidc --target=aarch64-apple-darwin $(EXTRA_CARGO_FLAGS)

objects.mk

@@ -40,6 +40,7 @@ ONE_OBJS=\
 	controller/FileDB.o \
 	controller/LFDB.o \
 	controller/PostgreSQL.o \
+	controller/CV1.o \
 	osdep/EthernetTap.o \
 	osdep/ManagedRoute.o \
 	osdep/Http.o \