Merge branch 'dev' into brenton/thread-safety-fixes

.drone.jsonnet

@@ -38,6 +38,11 @@ local less_targets = [
       { "os": "linux", distro: "ubuntu", "name": "focal",    "isas": [        "armv7", "amd64", "arm64" ],             "events": [ "push", "tag", "custom" ] },
 ];
 
+
+local native_targets = [
+      { "os": "linux", distro: "debian", "name": "bullseye", "isas": [ "386", "armv7", "amd64", "arm64" ],            "events": [ "push", "tag", "custom" ] },
+];    
+
 local master_targets = [
       //
       // copypasta from here
@@ -239,13 +244,13 @@ std.flattenArrays([
     [
       Index(p)
     ]
-    for p in targets
+    for p in native_targets
  ]) +
  std.flattenArrays([
      [
         Test(p.os, p.distro, p.name, isa, p.events)
          for isa in p.isas
      ]
-     for p in targets
+     for p in native_targets
  ])
  

.github/workflows/report.sh

@@ -13,3 +13,9 @@ echo -e "\nBytes of memory definitely lost: $DEFINITELY_LOST"
 if [[ "$DEFINITELY_LOST" -gt 0 ]]; then
       exit 1
 fi
+
+EXIT_TEST_FAILED=$(cat *test-results/*summary.json | jq .exit_test_failed)
+
+if [[ "$EXIT_TEST_FAILED" -gt 0 ]]; then
+      exit 1
+fi

.github/workflows/validate-1m-linux.sh

@@ -3,12 +3,14 @@
 # This test script joins Earth and pokes some stuff
 
 TEST_NETWORK=8056c2e21c000001
-RUN_LENGTH=10
+RUN_LENGTH=60
 TEST_FINISHED=false
 ZTO_VER=$(git describe --tags $(git rev-list --tags --max-count=1))
 ZTO_COMMIT=$(git rev-parse HEAD)
 ZTO_COMMIT_SHORT=$(git rev-parse --short HEAD)
 TEST_DIR_PREFIX="$ZTO_VER-$ZTO_COMMIT_SHORT-test-results"
+EXIT_TEST_FAILED=0
+
 echo "Performing test on: $ZTO_VER-$ZTO_COMMIT_SHORT"
 TEST_FILEPATH_PREFIX="$TEST_DIR_PREFIX/$ZTO_COMMIT_SHORT"
 mkdir $TEST_DIR_PREFIX
@@ -16,394 +18,361 @@ mkdir $TEST_DIR_PREFIX
 ################################################################################
 # Multi-node connectivity and performance test                                 #
 ################################################################################
+main() {
+	echo -e "\nRunning test for $RUN_LENGTH seconds"
 
-NS1="ip netns exec ns1"
-NS2="ip netns exec ns2"
+	check_exit_on_invalid_identity
 
-ZT1="$NS1 ./zerotier-cli -D$(pwd)/node1"
-# Specify custom port on one node to ensure that feature works
-ZT2="$NS2 ./zerotier-cli -p9997 -D$(pwd)/node2"
+	NS1="ip netns exec ns1"
+	NS2="ip netns exec ns2"
 
-echo -e "Setting up network namespaces..."
-echo "Setting up ns1"
+	ZT1="$NS1 ./zerotier-cli -p9996 -D$(pwd)/node1"
+	# Specify custom port on one node to ensure that feature works
+	ZT2="$NS2 ./zerotier-cli -p9997 -D$(pwd)/node2"
 
-ip netns add ns1
-$NS1 ip link set dev lo up
-ip link add veth0 type veth peer name veth1
-ip link set veth1 netns ns1
-ip addr add 192.168.0.1/24 dev veth0
-ip link set dev veth0 up
+	echo -e "\nSetting up network namespaces..."
+	echo "Setting up ns1"
 
-$NS1 ip addr add 192.168.0.2/24 dev veth1
-$NS1 ip link set dev veth1 up
+	ip netns add ns1
+	$NS1 ip link set dev lo up
+	ip link add veth0 type veth peer name veth1
+	ip link set veth1 netns ns1
+	ip addr add 192.168.0.1/24 dev veth0
+	ip link set dev veth0 up
 
-# Add default route
-$NS1 ip route add default via 192.168.0.1
+	$NS1 ip addr add 192.168.0.2/24 dev veth1
+	$NS1 ip link set dev veth1 up
+
+	# Add default route
+	$NS1 ip route add default via 192.168.0.1
 
-iptables -t nat -A POSTROUTING -s 192.168.0.0/255.255.255.0 \
-      -o eth0 -j MASQUERADE
-iptables -A FORWARD -i eth0 -o veth0 -j ACCEPT
-iptables -A FORWARD -o eth0 -i veth0 -j ACCEPT
+	iptables -t nat -A POSTROUTING -s 192.168.0.0/255.255.255.0 \
+		-o eth0 -j MASQUERADE
+	iptables -A FORWARD -i eth0 -o veth0 -j ACCEPT
+	iptables -A FORWARD -o eth0 -i veth0 -j ACCEPT
 
-echo "Setting up ns2"
-ip netns add ns2
-$NS2 ip link set dev lo up
-ip link add veth2 type veth peer name veth3
-ip link set veth3 netns ns2
-ip addr add 192.168.1.1/24 dev veth2
-ip link set dev veth2 up
+	echo "Setting up ns2"
+	ip netns add ns2
+	$NS2 ip link set dev lo up
+	ip link add veth2 type veth peer name veth3
+	ip link set veth3 netns ns2
+	ip addr add 192.168.1.1/24 dev veth2
+	ip link set dev veth2 up
 
-$NS2 ip addr add 192.168.1.2/24 dev veth3
-$NS2 ip link set dev veth3 up
-$NS2 ip route add default via 192.168.1.1
+	$NS2 ip addr add 192.168.1.2/24 dev veth3
+	$NS2 ip link set dev veth3 up
+	$NS2 ip route add default via 192.168.1.1
+
+	iptables -t nat -A POSTROUTING -s 192.168.1.0/255.255.255.0 \
+		-o eth0 -j MASQUERADE
+	iptables -A FORWARD -i eth0 -o veth2 -j ACCEPT
+	iptables -A FORWARD -o eth0 -i veth2 -j ACCEPT
+
+	# Allow forwarding
+	sysctl -w net.ipv4.ip_forward=1
 
-iptables -t nat -A POSTROUTING -s 192.168.1.0/255.255.255.0 \
-      -o eth0 -j MASQUERADE
-iptables -A FORWARD -i eth0 -o veth2 -j ACCEPT
-iptables -A FORWARD -o eth0 -i veth2 -j ACCEPT
+	echo -e "\nPing from host to namespaces"
 
-# Allow forwarding
-sysctl -w net.ipv4.ip_forward=1
-
-echo -e "\nPing from host to namespaces"
-
-ping -c 3 192.168.0.1
-ping -c 3 192.168.1.1
-
-echo -e "\nPing from namespace to host"
-
-$NS1 ping -c 3 192.168.0.1
-$NS1 ping -c 3 192.168.0.1
-$NS2 ping -c 3 192.168.0.2
-$NS2 ping -c 3 192.168.0.2
-
-echo -e "\nPing from ns1 to ns2"
-
-$NS1 ping -c 3 192.168.0.1
-
-echo -e "\nPing from ns2 to ns1"
-
-$NS2 ping -c 3 192.168.0.1
-
-################################################################################
-# Memory Leak Check                                                            #
-################################################################################
-
-FILENAME_MEMORY_LOG="$TEST_FILEPATH_PREFIX-memory.log"
-
-echo -e "\nStarting a ZeroTier instance in each namespace..."
-
-time_test_start=`date +%s`
-
-# Spam the CLI as ZeroTier is starting
-spam_cli 100
-
-echo "Starting memory leak check"
-$NS1 sudo valgrind --demangle=yes --exit-on-first-error=yes \
-      --error-exitcode=1 \
-      --xml=yes \
-      --xml-file=$FILENAME_MEMORY_LOG \
-      --leak-check=full \
-      ./zerotier-one node1 -U >>node_1.log 2>&1 &
-
-# Second instance, not run in memory profiler
-$NS2 sudo ./zerotier-one node2 -U -p9997 >>node_2.log 2>&1 &
-
-################################################################################
-# Online Check                                                                 #
-################################################################################
-
-spam_cli()
-{
-      echo "Spamming CLI..."
-      # Rapidly spam the CLI with joins/leaves
-
-      MAX_TRIES="${$1:-10}"
-
-      for ((s=0; s<=MAX_TRIES; s++))
-      do
-            $ZT1 status
-            $ZT2 status
-            sleep 0.1
-      done
-
-      SPAM_TRIES=128
-
-      for ((s=0; s<=SPAM_TRIES; s++))
-      do
-            $ZT1 join $TEST_NETWORK
-      done
-
-      for ((s=0; s<=SPAM_TRIES; s++))
-      do
-            $ZT1 leave $TEST_NETWORK
-      done
-
-      for ((s=0; s<=SPAM_TRIES; s++))
-      do
-            $ZT1 leave $TEST_NETWORK
-            $ZT1 join $TEST_NETWORK
-      done
-}
+	ping -c 3 192.168.0.1
+	ping -c 3 192.168.1.1
 
-echo "Waiting for ZeroTier to come online before attempting test..."
-MAX_WAIT_SECS="${MAX_WAIT_SECS:-120}"
-node1_online=false
-node2_online=false
-both_instances_online=false
-time_zt_node1_start=`date +%s`
-time_zt_node2_start=`date +%s`
+	echo -e "\nPing from namespace to host"
 
-for ((s=0; s<=MAX_WAIT_SECS; s++))
-do
-    node1_online="$($ZT1 -j info | jq '.online' 2>/dev/null)"
-    node2_online="$($ZT2 -j info | jq '.online' 2>/dev/null)"
-    echo "Checking for online status: try #$s, node1:$node1_online, node2:$node2_online"
-    if [[ "$node1_online" == "true" ]]
-    then
-        time_zt_node1_online=`date +%s`
-    fi
-    if [[ "$node2_online" == "true" ]]
-    then
-        time_zt_node2_online=`date +%s`
-    fi
-    if [[ "$node2_online" == "true" && "$node1_online" == "true" ]]
-    then
-        both_instances_online=true
-        break
-    fi
-    sleep 1
-done
-
-echo -e "\n\nContents of ZeroTier home paths:"
-
-ls -lga node1
-tree node1
-ls -lga node2
-tree node2
-
-echo -e "\n\nRunning ZeroTier processes:"
-echo -e "\nNode 1:"
-$NS1 ps aux | grep zerotier-one
-echo -e "\nNode 2:"
-$NS2 ps aux | grep zerotier-one
-
-echo -e "\n\nStatus of each instance:"
-
-echo -e "\n\nNode 1:"
-$ZT1 status
-echo -e "\n\nNode 2:"
-$ZT2 status
-
-if [[ "$both_instances_online" != "true" ]]
-then
-    echo "One or more instances of ZeroTier failed to come online. Aborting test."
-    exit 1
-fi
-
-echo -e "\nJoining networks"
-
-$ZT1 join $TEST_NETWORK
-$ZT2 join $TEST_NETWORK
-
-sleep 10
-
-node1_ip4=$($ZT1 get $TEST_NETWORK ip4)
-node2_ip4=$($ZT2 get $TEST_NETWORK ip4)
-
-echo "node1_ip4=$node1_ip4"
-echo "node2_ip4=$node2_ip4"
-
-echo -e "\nPinging each node"
-
-PING12_FILENAME="$TEST_FILEPATH_PREFIX-ping-1-to-2.txt"
-PING21_FILENAME="$TEST_FILEPATH_PREFIX-ping-2-to-1.txt"
-
-$NS1 ping -c 16 $node2_ip4 > $PING12_FILENAME
-$NS2 ping -c 16 $node1_ip4 > $PING21_FILENAME
-
-# Parse ping statistics
-ping_loss_percent_1_to_2="${ping_loss_percent_1_to_2:-100.0}"
-ping_loss_percent_2_to_1="${ping_loss_percent_2_to_1:-100.0}"
-
-ping_loss_percent_1_to_2=$(cat $PING12_FILENAME | \
-      grep "packet loss" | awk '{print $6}' | sed 's/%//')
-ping_loss_percent_2_to_1=$(cat $PING21_FILENAME | \
-      grep "packet loss" | awk '{print $6}' | sed 's/%//')
-
-# Normalize loss value
-ping_loss_percent_1_to_2=$(echo "scale=2; $ping_loss_percent_1_to_2/100.0" | bc)
-ping_loss_percent_2_to_1=$(echo "scale=2; $ping_loss_percent_2_to_1/100.0" | bc)
-
-################################################################################
-# CLI Check                                                                    #
-################################################################################
+	$NS1 ping -c 3 192.168.0.1
+	$NS1 ping -c 3 192.168.0.1
+	$NS2 ping -c 3 192.168.0.2
+	$NS2 ping -c 3 192.168.0.2
 
-echo "Testing basic CLI functionality..."
+	echo -e "\nPing from ns1 to ns2"
 
-spam_cli 10
+	$NS1 ping -c 3 192.168.0.1
 
-$ZT1 join $TEST_NETWORK
+	echo -e "\nPing from ns2 to ns1"
 
-$ZT1 -h
-$ZT1 -v
-$ZT1 status
-$ZT1 info
-$ZT1 listnetworks
-$ZT1 peers
-$ZT1 listpeers
+	$NS2 ping -c 3 192.168.0.1
 
-$ZT1 -j status
-$ZT1 -j info
-$ZT1 -j listnetworks
-$ZT1 -j peers
-$ZT1 -j listpeers
+	################################################################################
+	# Memory Leak Check                                                            #
+	################################################################################
 
-$ZT1 dump
+	FILENAME_MEMORY_LOG="$TEST_FILEPATH_PREFIX-memory.log"
 
-$ZT1 get $TEST_NETWORK allowDNS
-$ZT1 get $TEST_NETWORK allowDefault
-$ZT1 get $TEST_NETWORK allowGlobal
-$ZT1 get $TEST_NETWORK allowManaged
-$ZT1 get $TEST_NETWORK bridge
-$ZT1 get $TEST_NETWORK broadcastEnabled
-$ZT1 get $TEST_NETWORK dhcp
-$ZT1 get $TEST_NETWORK id
-$ZT1 get $TEST_NETWORK mac
-$ZT1 get $TEST_NETWORK mtu
-$ZT1 get $TEST_NETWORK name
-$ZT1 get $TEST_NETWORK netconfRevision
-$ZT1 get $TEST_NETWORK nwid
-$ZT1 get $TEST_NETWORK portDeviceName
-$ZT1 get $TEST_NETWORK portError
-$ZT1 get $TEST_NETWORK status
-$ZT1 get $TEST_NETWORK type
+	echo -e "\nStarting a ZeroTier instance in each namespace..."
 
-# Test an invalid command
-$ZT1 get $TEST_NETWORK derpderp
+	time_test_start=$(date +%s)
 
-# TODO: Validate JSON
+	# Spam the CLI as ZeroTier is starting
+	spam_cli 100
 
-################################################################################
-# Performance Test                                                             #
-################################################################################
+	echo "Starting memory leak check"
+	$NS1 sudo valgrind --demangle=yes --exit-on-first-error=yes \
+		--error-exitcode=1 \
+		--xml=yes \
+		--xml-file=$FILENAME_MEMORY_LOG \
+		--leak-check=full \
+		./zerotier-one node1 -p9996 -U >>node_1.log 2>&1 &
 
-FILENAME_PERF_JSON="$TEST_FILEPATH_PREFIX-iperf.json"
+	# Second instance, not run in memory profiler
+	$NS2 sudo ./zerotier-one node2 -U -p9997 >>node_2.log 2>&1 &
 
-echo -e "\nBeginning performance test:"
+	################################################################################
+	# Online Check                                                                 #
+	################################################################################
 
-echo -e "\nStarting server:"
+	echo "Waiting for ZeroTier to come online before attempting test..."
+	MAX_WAIT_SECS="${MAX_WAIT_SECS:-120}"
+	node1_online=false
+	node2_online=false
+	both_instances_online=false
+	time_zt_node1_start=$(date +%s)
+	time_zt_node2_start=$(date +%s)
 
-echo "$NS1 iperf3 -s &"
-sleep 1
+	for ((s = 0; s <= MAX_WAIT_SECS; s++)); do
+		node1_online="$($ZT1 -j info | jq '.online' 2>/dev/null)"
+		node2_online="$($ZT2 -j info | jq '.online' 2>/dev/null)"
+		echo "Checking for online status: try #$s, node1:$node1_online, node2:$node2_online"
+		if [[ "$node1_online" == "true" ]]; then
+			time_zt_node1_online=$(date +%s)
+		fi
+		if [[ "$node2_online" == "true" ]]; then
+			time_zt_node2_online=$(date +%s)
+		fi
+		if [[ "$node2_online" == "true" && "$node1_online" == "true" ]]; then
+			both_instances_online=true
+			break
+		fi
+		sleep 1
+	done
+
+	echo -e "\n\nContents of ZeroTier home paths:"
+
+	ls -lga node1
+	tree node1
+	ls -lga node2
+	tree node2
+
+	echo -e "\n\nRunning ZeroTier processes:"
+	echo -e "\nNode 1:"
+	$NS1 ps aux | grep zerotier-one
+	echo -e "\nNode 2:"
+	$NS2 ps aux | grep zerotier-one
 
-echo -e "\nStarting client:"
-sleep 1
+	echo -e "\n\nStatus of each instance:"
 
-echo "$NS2 iperf3 --json -c $node1_ip4 > $FILENAME_PERF_JSON"
+	echo -e "\n\nNode 1:"
+	$ZT1 status
+	echo -e "\n\nNode 2:"
+	$ZT2 status
 
-cat $FILENAME_PERF_JSON
+	if [[ "$both_instances_online" != "true" ]]; then
+		echo "One or more instances of ZeroTier failed to come online. Aborting test."
+		exit 1
+	fi
 
-################################################################################
-# Collect ZeroTier dump files                                                  #
-################################################################################
+	echo -e "\nJoining networks"
 
-echo -e "\nCollecting ZeroTier dump files"
+	$ZT1 join $TEST_NETWORK
+	$ZT2 join $TEST_NETWORK
 
-node1_id=$($ZT1 -j status | jq -r .address)
-node2_id=$($ZT2 -j status | jq -r .address)
+	sleep 10
 
-$ZT1 dump
-mv zerotier_dump.txt "$TEST_FILEPATH_PREFIX-node-dump-$node1_id.txt"
+	node1_ip4=$($ZT1 get $TEST_NETWORK ip4)
+	node2_ip4=$($ZT2 get $TEST_NETWORK ip4)
 
-$ZT2 dump
-mv zerotier_dump.txt "$TEST_FILEPATH_PREFIX-node-dump-$node2_id.txt"
+	echo "node1_ip4=$node1_ip4"
+	echo "node2_ip4=$node2_ip4"
 
-################################################################################
-# Let ZeroTier idle long enough for various timers                             #
-################################################################################
+	echo -e "\nPinging each node"
 
-echo -e "\nIdling ZeroTier for $RUN_LENGTH seconds..."
-sleep $RUN_LENGTH
+	PING12_FILENAME="$TEST_FILEPATH_PREFIX-ping-1-to-2.txt"
+	PING21_FILENAME="$TEST_FILEPATH_PREFIX-ping-2-to-1.txt"
 
-echo -e "\nLeaving networks"
+	$NS1 ping -c 16 $node2_ip4 >$PING12_FILENAME
+	$NS2 ping -c 16 $node1_ip4 >$PING21_FILENAME
 
-$ZT1 leave $TEST_NETWORK
-$ZT2 leave $TEST_NETWORK
+	# Parse ping statistics
+	ping_loss_percent_1_to_2="${ping_loss_percent_1_to_2:-100.0}"
+	ping_loss_percent_2_to_1="${ping_loss_percent_2_to_1:-100.0}"
 
-sleep 5
+	ping_loss_percent_1_to_2=$(cat $PING12_FILENAME |
+		grep "packet loss" | awk '{print $6}' | sed 's/%//')
+	ping_loss_percent_2_to_1=$(cat $PING21_FILENAME |
+		grep "packet loss" | awk '{print $6}' | sed 's/%//')
 
-################################################################################
-# Stop test                                                                    #
-################################################################################
+	# Normalize loss value
+	ping_loss_percent_1_to_2=$(echo "scale=2; $ping_loss_percent_1_to_2/100.0" | bc)
+	ping_loss_percent_2_to_1=$(echo "scale=2; $ping_loss_percent_2_to_1/100.0" | bc)
 
-echo -e "\nStopping memory check..."
-sudo pkill -15 -f valgrind
-sleep 10
+	################################################################################
+	# CLI Check                                                                    #
+	################################################################################
 
-time_test_end=`date +%s`
+	echo "Testing basic CLI functionality..."
 
-################################################################################
-# Rename ZeroTier stdout/stderr logs                                           #
-################################################################################
+	spam_cli 10
 
-mv node_1.log "$TEST_FILEPATH_PREFIX-node-log-$node1_id.txt"
-mv node_2.log "$TEST_FILEPATH_PREFIX-node-log-$node2_id.txt"
+	$ZT1 join $TEST_NETWORK
 
-################################################################################
-# Generate report                                                              #
-################################################################################
+	$ZT1 -h
+	$ZT1 -v
+	$ZT1 status
+	$ZT1 info
+	$ZT1 listnetworks
+	$ZT1 peers
+	$ZT1 listpeers
 
-cat $FILENAME_MEMORY_LOG
+	$ZT1 -j status
+	$ZT1 -j info
+	$ZT1 -j listnetworks
+	$ZT1 -j peers
+	$ZT1 -j listpeers
 
-DEFINITELY_LOST=$(xmlstarlet sel -t -v '/valgrindoutput/error/xwhat' \
-      $FILENAME_MEMORY_LOG | grep "definitely" | awk '{print $1;}')
-POSSIBLY_LOST=$(xmlstarlet sel -t -v '/valgrindoutput/error/xwhat' \
-      $FILENAME_MEMORY_LOG | grep "possibly" | awk '{print $1;}')
+	$ZT1 dump
 
-################################################################################
-# Generate coverage report artifact and summary                                #
-################################################################################
+	$ZT1 get $TEST_NETWORK allowDNS
+	$ZT1 get $TEST_NETWORK allowDefault
+	$ZT1 get $TEST_NETWORK allowGlobal
+	$ZT1 get $TEST_NETWORK allowManaged
+	$ZT1 get $TEST_NETWORK bridge
+	$ZT1 get $TEST_NETWORK broadcastEnabled
+	$ZT1 get $TEST_NETWORK dhcp
+	$ZT1 get $TEST_NETWORK id
+	$ZT1 get $TEST_NETWORK mac
+	$ZT1 get $TEST_NETWORK mtu
+	$ZT1 get $TEST_NETWORK name
+	$ZT1 get $TEST_NETWORK netconfRevision
+	$ZT1 get $TEST_NETWORK nwid
+	$ZT1 get $TEST_NETWORK portDeviceName
+	$ZT1 get $TEST_NETWORK portError
+	$ZT1 get $TEST_NETWORK status
+	$ZT1 get $TEST_NETWORK type
 
-FILENAME_COVERAGE_JSON="$TEST_FILEPATH_PREFIX-coverage.json"
-FILENAME_COVERAGE_HTML="$TEST_FILEPATH_PREFIX-coverage.html"
+	# Test an invalid command
+	$ZT1 get $TEST_NETWORK derpderp
 
-echo -e "\nGenerating coverage test report..."
+	# TODO: Validate JSON
 
-gcovr -r . --exclude ext --json-summary $FILENAME_COVERAGE_JSON \
-      --html > $FILENAME_COVERAGE_HTML
+	################################################################################
+	# Performance Test                                                             #
+	################################################################################
 
-cat $FILENAME_COVERAGE_JSON
+	FILENAME_PERF_JSON="$TEST_FILEPATH_PREFIX-iperf.json"
 
-COVERAGE_LINE_COVERED=$(cat $FILENAME_COVERAGE_JSON | jq .line_covered)
-COVERAGE_LINE_TOTAL=$(cat $FILENAME_COVERAGE_JSON | jq .line_total)
-COVERAGE_LINE_PERCENT=$(cat $FILENAME_COVERAGE_JSON | jq .line_percent)
+	echo -e "\nBeginning performance test:"
 
-COVERAGE_LINE_COVERED="${COVERAGE_LINE_COVERED:-0}"
-COVERAGE_LINE_TOTAL="${COVERAGE_LINE_TOTAL:-0}"
-COVERAGE_LINE_PERCENT="${COVERAGE_LINE_PERCENT:-0}"
+	echo -e "\nStarting server:"
 
-################################################################################
-# Default values                                                               #
-################################################################################
+	echo "$NS1 iperf3 -s &"
+	sleep 1
 
-DEFINITELY_LOST="${DEFINITELY_LOST:-0}"
-POSSIBLY_LOST="${POSSIBLY_LOST:-0}"
+	echo -e "\nStarting client:"
+	sleep 1
 
-################################################################################
-# Summarize and emit json for trend reporting                                  #
-################################################################################
+	echo "$NS2 iperf3 --json -c $node1_ip4 > $FILENAME_PERF_JSON"
 
-FILENAME_SUMMARY="$TEST_FILEPATH_PREFIX-summary.json"
+	cat $FILENAME_PERF_JSON
 
-time_length_test=$((time_test_end-time_test_start))
-time_length_zt_node1_online=$((time_zt_node1_online-time_zt_start))
-time_length_zt_node2_online=$((time_zt_node2_online-time_zt_start))
-#time_length_zt_join=$((time_zt_join_end-time_zt_join_start))
-#time_length_zt_leave=$((time_zt_leave_end-time_zt_leave_start))
-#time_length_zt_can_still_ping=$((time_zt_can_still_ping-time_zt_leave_start))
+	################################################################################
+	# Collect ZeroTier dump files                                                  #
+	################################################################################
 
-summary=$(cat <<EOF
+	echo -e "\nCollecting ZeroTier dump files"
+
+	node1_id=$($ZT1 -j status | jq -r .address)
+	node2_id=$($ZT2 -j status | jq -r .address)
+
+	$ZT1 dump
+	mv zerotier_dump.txt "$TEST_FILEPATH_PREFIX-node-dump-$node1_id.txt"
+
+	$ZT2 dump
+	mv zerotier_dump.txt "$TEST_FILEPATH_PREFIX-node-dump-$node2_id.txt"
+
+	################################################################################
+	# Let ZeroTier idle long enough for various timers                             #
+	################################################################################
+
+	echo -e "\nIdling ZeroTier for $RUN_LENGTH seconds..."
+	sleep $RUN_LENGTH
+
+	echo -e "\nLeaving networks"
+
+	$ZT1 leave $TEST_NETWORK
+	$ZT2 leave $TEST_NETWORK
+
+	sleep 5
+
+	################################################################################
+	# Stop test                                                                    #
+	################################################################################
+
+	echo -e "\nStopping memory check..."
+	sudo pkill -15 -f valgrind
+	sleep 10
+
+	time_test_end=$(date +%s)
+
+	################################################################################
+	# Rename ZeroTier stdout/stderr logs                                           #
+	################################################################################
+
+	mv node_1.log "$TEST_FILEPATH_PREFIX-node-log-$node1_id.txt"
+	mv node_2.log "$TEST_FILEPATH_PREFIX-node-log-$node2_id.txt"
+
+	################################################################################
+	# Generate report                                                              #
+	################################################################################
+
+	cat $FILENAME_MEMORY_LOG
+
+	DEFINITELY_LOST=$(xmlstarlet sel -t -v '/valgrindoutput/error/xwhat' \
+		$FILENAME_MEMORY_LOG | grep "definitely" | awk '{print $1;}')
+	POSSIBLY_LOST=$(xmlstarlet sel -t -v '/valgrindoutput/error/xwhat' \
+		$FILENAME_MEMORY_LOG | grep "possibly" | awk '{print $1;}')
+
+	################################################################################
+	# Generate coverage report artifact and summary                                #
+	################################################################################
+
+	FILENAME_COVERAGE_JSON="$TEST_FILEPATH_PREFIX-coverage.json"
+	FILENAME_COVERAGE_HTML="$TEST_FILEPATH_PREFIX-coverage.html"
+
+	echo -e "\nGenerating coverage test report..."
+
+	gcovr -r . --exclude ext --json-summary $FILENAME_COVERAGE_JSON \
+		--html >$FILENAME_COVERAGE_HTML
+
+	cat $FILENAME_COVERAGE_JSON
+
+	COVERAGE_LINE_COVERED=$(cat $FILENAME_COVERAGE_JSON | jq .line_covered)
+	COVERAGE_LINE_TOTAL=$(cat $FILENAME_COVERAGE_JSON | jq .line_total)
+	COVERAGE_LINE_PERCENT=$(cat $FILENAME_COVERAGE_JSON | jq .line_percent)
+
+	COVERAGE_LINE_COVERED="${COVERAGE_LINE_COVERED:-0}"
+	COVERAGE_LINE_TOTAL="${COVERAGE_LINE_TOTAL:-0}"
+	COVERAGE_LINE_PERCENT="${COVERAGE_LINE_PERCENT:-0}"
+
+	################################################################################
+	# Default values                                                               #
+	################################################################################
+
+	DEFINITELY_LOST="${DEFINITELY_LOST:-0}"
+	POSSIBLY_LOST="${POSSIBLY_LOST:-0}"
+
+	################################################################################
+	# Summarize and emit json for trend reporting                                  #
+	################################################################################
+
+	FILENAME_SUMMARY="$TEST_FILEPATH_PREFIX-summary.json"
+
+	time_length_test=$((time_test_end - time_test_start))
+	time_length_zt_node1_online=$((time_zt_node1_online - time_zt_start))
+	time_length_zt_node2_online=$((time_zt_node2_online - time_zt_start))
+	#time_length_zt_join=$((time_zt_join_end-time_zt_join_start))
+	#time_length_zt_leave=$((time_zt_leave_end-time_zt_leave_start))
+	#time_length_zt_can_still_ping=$((time_zt_can_still_ping-time_zt_leave_start))
+
+	summary=$(
+		cat <<EOF
 {
   "version":"$ZTO_VER",
   "commit":"$ZTO_COMMIT",
@@ -426,12 +395,70 @@ summary=$(cat <<EOF
   "mean_latency_ping_netns": $POSSIBLY_LOST,
   "mean_pdv_random": $POSSIBLY_LOST,
   "mean_pdv_netns": $POSSIBLY_LOST,
-  "mean_perf_netns": $POSSIBLY_LOST
+  "mean_perf_netns": $POSSIBLY_LOST,
+  "exit_test_failed": $EXIT_TEST_FAILED
 }
 EOF
-)
+	)
 
-echo $summary > $FILENAME_SUMMARY
-cat $FILENAME_SUMMARY
+	echo $summary >$FILENAME_SUMMARY
+	cat $FILENAME_SUMMARY
+}
 
-"$@"
+################################################################################
+# CLI Check                                                                    #
+################################################################################
+
+spam_cli() {
+	echo "Spamming CLI..."
+	# Rapidly spam the CLI with joins/leaves
+
+	MAX_TRIES="${1:-10}"
+
+	for ((s = 0; s <= MAX_TRIES; s++)); do
+		$ZT1 status
+		$ZT2 status
+		sleep 0.1
+	done
+
+	SPAM_TRIES=128
+
+	for ((s = 0; s <= SPAM_TRIES; s++)); do
+		$ZT1 join $TEST_NETWORK
+	done
+
+	for ((s = 0; s <= SPAM_TRIES; s++)); do
+		$ZT1 leave $TEST_NETWORK
+	done
+
+	for ((s = 0; s <= SPAM_TRIES; s++)); do
+		$ZT1 leave $TEST_NETWORK
+		$ZT1 join $TEST_NETWORK
+	done
+}
+
+check_exit_on_invalid_identity() {
+	echo "Checking ZeroTier exits on invalid identity..."
+	mkdir -p $(pwd)/exit_test
+	ZT1="sudo ./zerotier-one -p9999 $(pwd)/exit_test"
+	echo "asdfasdfasdfasdf" > $(pwd)/exit_test/identity.secret
+	echo "asdfasdfasdfasdf" > $(pwd)/exit_test/authtoken.secret
+
+	echo "Launch ZeroTier with an invalid identity"
+	$ZT1 &
+	my_pid=$!
+
+	echo "Waiting 5 secons"
+	sleep 5
+
+	# check if process is running
+	kill -0 $my_pid
+	if [ $? -eq 0 ]; then
+		EXIT_TEST_FAILED=1
+		echo "Exit test FAILED: Process still running after being fed an invalid identity"
+	else
+		echo "Exit test PASSED"
+	fi
+}
+
+main "$@"

README.md

@@ -37,7 +37,6 @@ The base path contains the ZeroTier One service main entry point (`one.cpp`), se
  - `ext/`: third party libraries, binaries that we ship for convenience on some platforms (Mac and Windows), and installation support files.
  - `include/`: include files for the ZeroTier core.
  - `java/`: a JNI wrapper used with our Android mobile app. (The whole Android app is not open source but may be made so in the future.)
- - `macui/`: a Macintosh menu-bar app for controlling ZeroTier One, written in Objective C.
  - `node/`: the ZeroTier virtual Ethernet switch core, which is designed to be entirely separate from the rest of the code and able to be built as a stand-alone OS-independent library. Note to developers: do not use C++11 features in here, since we want this to build on old embedded platforms that lack C++11 support. C++11 can be used elsewhere.
  - `osdep/`: code to support and integrate with OSes, including platform-specific stuff only built for certain targets.
  - `rule-compiler/`: JavaScript rules language compiler for defining network-level rules.
@@ -113,18 +112,18 @@ Additional help can be found in our [knowledge base](https://zerotier.atlassian.
 
 ### Prometheus Metrics
 
-Prometheus Metrics are available at the `/metrics` API endpoint.  This endpoint is protected by an API key stored in `authtoken.secret` because of the possibility of information leakage.  Information that could be gleaned from the metrics include joined networks and peers your instance is talking to. 
+Prometheus Metrics are available at the `/metrics` API endpoint.  This endpoint is protected by an API key stored in `metricstoken.secret` to prevent unwanted information leakage.  Information that could be gleaned from the metrics include joined networks and peers your instance is talking to. 
 
-Access control is via the ZeroTier control interface itself and `authtoken.secret`. This can be sent as the `X-ZT1-Auth` HTTP header field or appended to the URL as `?auth=<token>`. You can see the current metrics via `cURL` with the following command:
+Access control is via the ZeroTier control interface itself and `metricstoken.secret`. This can be sent as a bearer auth token, via the `X-ZT1-Auth` HTTP header field, or appended to the URL as `?auth=<token>`. You can see the current metrics via `cURL` with the following command:
 
     // Linux
-    curl -H "X-ZT1-Auth: $(sudo cat /var/lib/zerotier-one/authtoken.secret)" http://localhost:9993/metrics
+    curl -H "X-ZT1-Auth: $(sudo cat /var/lib/zerotier-one/metricstoken.secret)" http://localhost:9993/metrics
 
     // macOS
-    curl -H "X-XT1-Auth: $(sudo cat /Library/Application\ Support/ZeroTier/One/authtoken.secret)" http://localhost:9993/metrics
+    curl -H "X-XT1-Auth: $(sudo cat /Library/Application\ Support/ZeroTier/One/metricstoken.secret)" http://localhost:9993/metrics
 
     // Windows PowerShell (Admin)
-    Invoke-RestMethod -Headers @{'X-ZT1-Auth' = "$(Get-Content C:\ProgramData\ZeroTier\One\authtoken.secret)"; } -Uri http://localhost:9993/metrics
+    Invoke-RestMethod -Headers @{'X-ZT1-Auth' = "$(Get-Content C:\ProgramData\ZeroTier\One\metricstoken.secret)"; } -Uri http://localhost:9993/metrics
 
 To configure a scrape job in Prometheus on the machine ZeroTier is running on, add this to your Prometheus `scrape_config`:
 
@@ -137,24 +136,23 @@ To configure a scrape job in Prometheus on the machine ZeroTier is running on, a
         - 127.0.0.1:9993
         labels:
           group: zerotier-one
-      params:
-        auth:
-        - $YOUR_AUTHTOKEN_SECRET
-
-If your Prometheus instance is remote from the machine ZeroTier instance, you'll have to edit your `local.conf` file to allow remote access to the API control port.  If your local lan is `10.0.0.0/24`, edit your `local.conf` as follows:
-
-    {
-      "settings": {
-        "allowManagementFrom:" ["10.0.0.0/24"]
-      }
-    }
-
-Substitute your actual network IP ranges as necessary.
-
-It's also possible to access the metrics & control port over the ZeroTier network itself via the same method shown above.  Just add the address range of your ZeroTier network to the list. NOTE: Using this method means that anyone with your auth token can control your ZeroTier instance, including leaving & joining other networks.
+          node_id: $YOUR_10_CHARACTER_NODE_ID
+      authorization:
+        credentials: $YOUR_METRICS_TOKEN_SECRET
 
 If neither of these methods are desirable, it is probably possible to distribute metrics via [Prometheus Proxy](https://github.com/pambrose/prometheus-proxy) or some other tool.  Note: We have not tested this internally, but will probably work with the correct configuration.
 
+Metrics are also available on disk in ZeroTier's working directory:
+
+   // Linux
+   /var/lib/zerotier-one/metrics.prom
+
+   // macOS
+   /Library/Application\ Support/ZeroTier/One/metrics.prom
+
+   //Windows
+   C:\ProgramData\ZeroTier\One\metrics.prom
+
 #### Available Metrics
 
 | Metric Name | Labels | Metric Type | Description |

attic/world/build.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-c++ -std=c++11 -I../.. -I.. -g -o mkworld ../../node/C25519.cpp ../../node/Salsa20.cpp ../../node/SHA512.cpp ../../node/Identity.cpp ../../node/Utils.cpp ../../node/InetAddress.cpp ../../osdep/OSUtils.cpp mkworld.cpp -lm
+c++ -std=c++11 -I../.. -I../../ext -I.. -g -o mkworld ../../node/C25519.cpp ../../node/Salsa20.cpp ../../node/SHA512.cpp ../../node/Identity.cpp ../../node/Utils.cpp ../../node/InetAddress.cpp ../../osdep/OSUtils.cpp mkworld.cpp -lm

controller/EmbeddedNetworkController.cpp

@@ -466,11 +466,21 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *ztPa
 	, _node(node)
 	, _ztPath(ztPath)
 	, _path(dbPath)
+	, _signingId()
+	, _signingIdAddressString()
 	, _sender((NetworkController::Sender *)0)
 	, _db(this)
+	, _queue()
+	, _threads()
+	, _threads_l()
+	, _memberStatus()
+	, _memberStatus_l()
+	, _expiringSoon()
+	, _expiringSoon_l()
+	, _rc(rc)
 	, _ssoExpiryRunning(true)
 	, _ssoExpiry(std::thread(&EmbeddedNetworkController::_ssoExpiryThread, this))
-	, _rc(rc)
+
 #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK	
 	, _member_status_lookup{"nc_member_status_lookup",""}
 	, _member_status_lookup_count{"nc_member_status_lookup_count",""}

controller/EmbeddedNetworkController.hpp

@@ -139,9 +139,6 @@ private:
 	std::vector<std::thread> _threads;
 	std::mutex _threads_l;
 
-	bool _ssoExpiryRunning;
-	std::thread _ssoExpiry;
-
 	std::unordered_map< _MemberStatusKey,_MemberStatus,_MemberStatusHash > _memberStatus;
 	std::mutex _memberStatus_l;
 
@@ -151,6 +148,9 @@ private:
 	RedisConfig *_rc;
 	std::string _ssoRedirectURL;
 
+	bool _ssoExpiryRunning;
+	std::thread _ssoExpiry;
+
 #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
 	prometheus::simpleapi::benchmark_family_t _member_status_lookup;
 	prometheus::simpleapi::counter_family_t   _member_status_lookup_count;

controller/PostgreSQL.cpp

@@ -460,7 +460,7 @@ AuthInfo PostgreSQL::getSSOAuthInfo(const nlohmann::json &member, const std::str
 			    "LEFT OUTER JOIN ztc_network_oidc_config noc "
 				"  ON noc.network_id = n.id "
 				"LEFT OUTER JOIN ztc_oidc_config oc "
-				"  ON noc.client_id = oc.client_id AND noc.org_id = o.org_id "
+				"  ON noc.client_id = oc.client_id AND oc.org_id = o.org_id "
 				"WHERE n.id = $1 AND n.sso_enabled = true", networkId);
 		
 			std::string client_id = "";
@@ -527,9 +527,6 @@ AuthInfo PostgreSQL::getSSOAuthInfo(const nlohmann::json &member, const std::str
 
 		_pool->unborrow(c);
 	} catch (std::exception &e) {
-		if (c) {
-			_pool->unborrow(c);
-		}
 		fprintf(stderr, "ERROR: Error updating member on load for network %s: %s\n", networkId.c_str(), e.what());
 	}
 
@@ -1051,7 +1048,6 @@ void PostgreSQL::heartbeat()
 				w.commit();
 			} catch (std::exception &e) {
 				fprintf(stderr, "%s: Heartbeat update failed: %s\n", controllerId, e.what());
-				_pool->unborrow(c);
 				std::this_thread::sleep_for(std::chrono::milliseconds(1000));
 				continue;
 			}

entrypoint.sh.release

@@ -107,7 +107,7 @@ log_params "Writing healthcheck for networks:" $@
 
 cat >/healthcheck.sh <<EOF
 #!/bin/bash
-for i in $@
+for i in $@ $ZEROTIER_JOIN_NETWORKS
 do
   [ "\$(zerotier-cli get \$i status)" = "OK" ] || exit 1
 done

ext/central-controller-docker/Dockerfile

@@ -1,10 +1,10 @@
 # Dockerfile for ZeroTier Central Controllers
-FROM registry.zerotier.com/zerotier/controller-builder:latest as builder
+FROM registry.zerotier.com/zerotier/ctlbuild:latest as builder
 MAINTAINER Adam Ierymekno <adam.ierymenko@zerotier.com>, Grant Limberg <grant.limberg@zerotier.com>
 ADD . /ZeroTierOne
 RUN export PATH=$PATH:~/.cargo/bin && cd ZeroTierOne && make clean && make central-controller -j8
 
-FROM registry.zerotier.com/zerotier/controller-run:latest
+FROM registry.zerotier.com/zerotier/ctlrun:latest
 COPY --from=builder /ZeroTierOne/zerotier-one /usr/local/bin/zerotier-one
 RUN chmod a+x /usr/local/bin/zerotier-one
 RUN echo "/usr/local/lib64" > /etc/ld.so.conf.d/usr-local-lib64.conf && ldconfig

ext/central-controller-docker/Makefile

@@ -0,0 +1,16 @@
+registry = registry.zerotier.com/zerotier
+
+all:	controller-builder controller-runbase
+
+buildx:
+	@echo "docker buildx create"
+	# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+	docker run --privileged --rm tonistiigi/binfmt --install all
+	@echo docker buildx create --name multiarch --driver docker-container --use
+	@echo docker buildx inspect --bootstrap
+
+controller-builder:	buildx
+	docker buildx build --no-cache --platform linux/amd64,linux/arm64 -t $(registry)/ctlbuild:latest -f Dockerfile.builder . --push
+
+controller-runbase: buildx
+	docker buildx build --no-cache --platform linux/amd64,linux/arm64 -t $(registry)/ctlrun:latest -f Dockerfile.run_base . --push

ext/central-controller-docker/main.sh

@@ -64,6 +64,7 @@ fi
 popd
 
 DEFAULT_PORT=9993
+DEFAULT_LB_MODE=false
 
 APP_NAME="controller-$(cat /var/lib/zerotier-one/identity.public | cut -d ':' -f 1)"
 
@@ -76,6 +77,7 @@ echo "{
             \"inot\",
             \"nat64\"
         ],
+        \"lowBandwidthMode\": ${ZT_LB_MODE:-$DEFAULT_LB_MODE},
         \"ssoRedirectURL\": \"${ZT_SSO_REDIRECT_URL}\",
         \"allowManagementFrom\": [\"127.0.0.1\", \"::1\", \"10.0.0.0/8\"],
         ${REDIS}