instantiate and configure otel for controllers

make-linux.mk

@@ -317,7 +317,7 @@ ifeq (${ZT_OTEL},1)
 	OTEL_INSTALL_DIR=ext/opentelemetry-cpp-${OTEL_VERSION}/localinstall
 	DEFS+=-DZT_OTEL
 	INCLUDES+=-I${OTEL_INSTALL_DIR}/include
-	LDLIBS+=-L${OTEL_INSTALL_DIR}/lib -lopentelemetry_exporter_in_memory_metric -lopentelemetry_exporter_in_memory -lopentelemetry_exporter_ostream_logs -lopentelemetry_exporter_ostream_metrics -lopentelemetry_exporter_ostream_span -lopentelemetry_trace -lopentelemetry_common -lopentelemetry_resources -lopentelemetry_logs -lopentelemetry_metrics -lopentelemetry_version
+	LDLIBS+=-L${OTEL_INSTALL_DIR}/lib -lopentelemetry_common -lopentelemetry_resources -lopentelemetry_otlp_recordable -lopentelemetry_exporter_in_memory_metric -lopentelemetry_exporter_in_memory -lopentelemetry_exporter_ostream_logs -lopentelemetry_exporter_ostream_metrics -lopentelemetry_exporter_ostream_span -lopentelemetry_exporter_otlp_grpc -lopentelemetry_exporter_otlp_grpc_client -lopentelemetry_exporter_otlp_grpc_log -lopentelemetry_exporter_otlp_grpc_metrics -lopentelemetry_trace -lopentelemetry_common -lopentelemetry_resources -lopentelemetry_logs -lopentelemetry_metrics -lopentelemetry_version
 else
 	OTEL_INSTALL_DIR=ext/opentelemetry-cpp-api-only/localinstall
 	INCLUDES+=-I${OTEL_INSTALL_DIR}/include
@@ -343,7 +343,7 @@ endif
 ifeq ($(ZT_CONTROLLER),1)
 	override CXXFLAGS+=-Wall -Wno-deprecated -std=c++17 -pthread $(INCLUDES) -DNDEBUG $(DEFS)
 	override LDLIBS+=-Lext/libpqxx-7.7.3/install/ubuntu22.04/$(EXT_ARCH)/lib -lpqxx -lpq ext/hiredis-1.0.2/lib/ubuntu22.04/$(EXT_ARCH)/libhiredis.a ext/redis-plus-plus-1.3.3/install/ubuntu22.04/$(EXT_ARCH)/lib/libredis++.a -lssl -lcrypto
-	override DEFS+=-DZT_CONTROLLER_USE_LIBPQ -DZT_NO_PEER_METRICS
+	override DEFS+=-DZT_CONTROLLER_USE_LIBPQ -DZT_NO_PEER_METRICS -DZT_OTEL_EXPORTER
 	override INCLUDES+=-I/usr/include/postgresql -Iext/libpqxx-7.7.3/install/ubuntu22.04/$(EXT_ARCH)/include -Iext/hiredis-1.0.2/include/ -Iext/redis-plus-plus-1.3.3/install/ubuntu22.04/$(EXT_ARCH)/include/sw/
 	ifeq ($(ZT_DEBUG),1)
 		override LDLIBS+=rustybits/target/debug/libsmeeclient.a

service/OneService.cpp

@@ -59,6 +59,11 @@
 
 #include <cpp-httplib/httplib.h>
 
+#ifdef ZT_OTEL_EXPORTER
+#include "opentelemetry/exporters/otlp/otlp_grpc_exporter.h"
+#include "opentelemetry/sdk/resource/resource.h"
+#endif
+
 #if ZT_SSO_ENABLED
 #include <zeroidc.h>
 #endif
@@ -912,6 +917,12 @@ class OneServiceImpl : public OneService {
 	RedisConfig* _rc;
 	std::string _ssoRedirectURL;
 
+#ifdef ZT_OPENTELEMETRY_ENABLED
+	nostd::shared_ptr<sdktrace::TracerProvider> _traceProvider;
+	std::string _exporterEndpoint;
+	double _exporterSampleRate;
+#endif
+
 	// end member variables ----------------------------------------------------
 
 	OneServiceImpl(const char* hp, unsigned int port)
@@ -955,6 +966,11 @@ class OneServiceImpl : public OneService {
 		, _run(true)
 		, _rc(NULL)
 		, _ssoRedirectURL()
+#ifdef ZT_OPENTELEMETRY_ENABLED
+		, _traceProvider(nullptr)
+		, _exporterEndpoint()
+		, _exporterSampleRate(1.0)
+#endif
 	{
 		_ports[0] = 0;
 		_ports[1] = 0;
@@ -1018,6 +1034,58 @@ class OneServiceImpl : public OneService {
 		bool pinning = _cpuPinningEnabled;
 	}
 
+#ifdef ZT_OPENTELEMETRY_ENABLED
+	void initTracing()
+	{
+		if (! _exporterEndpoint.empty() && _exporterSampleRate > 0.0) {
+			// Set up OpenTelemetry exporter and tracer provider
+			opentelemetry::exporter::otlp::OtlpGrpcExporterOptions opts;
+			opts.endpoint = _exporterEndpoint + "/v1/traces";
+			auto exporter = std::unique_ptr<opentelemetry::exporter::otlp::OtlpGrpcExporter>(new opentelemetry::exporter::otlp::OtlpGrpcExporter(opts));
+			auto processor = std::unique_ptr<opentelemetry::sdk::trace::SpanProcessor>(new opentelemetry::sdk::trace::SimpleSpanProcessor(std::move(exporter)));
+
+			char buf[16];
+			auto versionString = std::stringstream();
+			versionString << ZEROTIER_ONE_VERSION_MAJOR << "." << ZEROTIER_ONE_VERSION_MINOR << "." << ZEROTIER_ONE_VERSION_REVISION;
+			auto resource_attributes = opentelemetry::sdk::resource::ResourceAttributes { { "service.name", "zerotier-one" },
+																						  { "service.version", versionString.str() },
+																						  { "service.node_id", _node->identity().address().toString(buf) },
+																						  { "service.namespace", "com.zerotier.zerotier-one" } };
+			auto resource = std::unique_ptr<opentelemetry::sdk::resource::Resource>(new opentelemetry::sdk::resource::Resource(resource_attributes));
+			auto sampler = std::unique_ptr<sdktrace::TraceIdRatioBasedSampler>(new sdktrace::TraceIdRatioBasedSampler(_exporterSampleRate));
+			auto tracer_context = std::make_shared<sdktrace::TracerContext>(std::move(processor), resource, std::move(sampler));
+			_traceProvider = nostd::shared_ptr<sdktrace::TracerProvider>(new sdktrace::TracerProvider(tracer_context));
+			opentelemetry::trace::Provider::SetTracerProvider(_traceProvider);
+		}
+	}
+
+	void initMetrics()
+	{
+		if (! _exporterEndpoint.empty()) {
+			// Set up OpenTelemetry metrics exporter
+			opentelemetry::exporter::otlp::OtlpGrpcExporterOptions opts;
+			opts.endpoint = _exporterEndpoint + "/v1/metrics";
+			auto exporter = std::unique_ptr<opentelemetry::exporter::otlp::OtlpGrpcExporter>(new opentelemetry::exporter::otlp::OtlpGrpcExporter(opts));
+			auto processor = std::unique_ptr<opentelemetry::sdk::metrics::MetricReader>(new opentelemetry::sdk::metrics::PeriodicExportingMetricReader(std::move(exporter), std::chrono::seconds(5)));
+			auto meter_provider = nostd::shared_ptr<sdkmetrics::MeterProvider>(new sdkmetrics::MeterProvider(std::move(processor)));
+			opentelemetry::metrics::Provider::SetMeterProvider(meter_provider);
+		}
+	}
+
+	void initLogging()
+	{
+		if (! _exporterEndpoint.empty()) {
+			// Set up OpenTelemetry logging exporter
+			opentelemetry::exporter::otlp::OtlpGrpcExporterOptions opts;
+			opts.endpoint = _exporterEndpoint + "/v1/logs";
+			auto exporter = std::unique_ptr<opentelemetry::exporter::otlp::OtlpGrpcExporter>(new opentelemetry::exporter::otlp::OtlpGrpcExporter(opts));
+			auto processor = std::unique_ptr<opentelemetry::sdk::logs::LogRecordProcessor>(new opentelemetry::sdk::logs::SimpleLogRecordProcessor(std::move(exporter)));
+			auto logger_provider = nostd::shared_ptr<sdklogs::LoggerProvider>(new sdklogs::LoggerProvider(std::move(processor)));
+			opentelemetry::logs::Provider::SetLoggerProvider(logger_provider);
+		}
+	}
+#endif
+
 	virtual ReasonForTermination run()
 	{
 		try {
@@ -1081,6 +1149,12 @@ class OneServiceImpl : public OneService {
 			readLocalSettings();
 			applyLocalConfig();
 
+#ifdef ZT_OPENTELEMETRY_ENABLED
+			initTracing();
+			initMetrics();
+			initLogging();
+#endif
+
 			// Save original port number to show it if bind error
 			const int _configuredPort = _primaryPort;
 
@@ -1201,6 +1275,7 @@ class OneServiceImpl : public OneService {
 			int64_t lastCleanedPeersDb = 0;
 			int64_t lastLocalConfFileCheck = OSUtils::now();
 			int64_t lastOnline = lastLocalConfFileCheck;
+
 			for (;;) {
 				_run_m.lock();
 				if (! _run) {
@@ -1516,6 +1591,17 @@ class OneServiceImpl : public OneService {
 			}
 #endif
 
+#ifdef ZT_OPENTELEMETRY_ENABLED
+			json& otel = settings["otel"];
+			if (otel.is_object()) {
+				_exporterEndpoint = OSUtils::jsonString(otel["exporterEndpoint"], "");
+				_exporterSampleRate = OSUtils::jsonDouble(otel["exporterSampleRate"], 1.0f);
+				if (_exporterEndpoint.empty()) {
+					fprintf(stderr, "WARNING: OpenTelemetry exporter endpoint is not set. Metrics will not be exported." ZT_EOL_S);
+				}
+			}
+#endif
+
 			// Bind to wildcard instead of to specific interfaces (disables full tunnel capability)
 			json& bind = settings["bind"];
 			if (bind.is_array()) {

version.h

@@ -45,4 +45,9 @@
 #define ZT_BUILD_PLATFORM 0
 #endif
 
+#define _ZT_STR_HELPER(x)		 #x
+#define _ZT_STR(x)				 _ZT_STR_HELPER(x)
+#define ZEROTIER_ONE_VERSION_STR _ZT_STR(ZEROTIER_ONE_VERSION_MAJOR) "." _ZT_STR(ZEROTIER_ONE_VERSION_MINOR) "." _ZT_STR(ZEROTIER_ONE_VERSION_REVISION)
+#define ZEROTIER_ONE_NAME		 "zerotier-one"
+
 #endif