Make main() more rustic, fix IPv6 fragmentation bit on MacOS, etc.

osdep/rust-osdep.h

@@ -42,7 +42,10 @@
 #include <net/if_dl.h>
 #include <net/if_media.h>
 #include <net/ndrv.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
 #include <netinet/in_var.h>
+#include <netinet/in.h>
 #include <netinet/icmp6.h>
 #include <netinet6/in6_var.h>
 #include <netinet6/nd6.h>
@@ -65,6 +68,9 @@ extern const unsigned long c_SIOCAUTOCONF_STOP;
 #ifdef __cplusplus
 }
 #endif
+#ifndef IPV6_DONTFRAG
+#define IPV6_DONTFRAG 62
+#endif
 #endif /* __APPLE__ */
 
 /********************************************************************************************************************/

rust-zerotier-core/src/endpoint.rs

@@ -21,7 +21,7 @@ use num_traits::FromPrimitive;
 use crate::*;
 use crate::capi as ztcore;
 
-#[derive(FromPrimitive, ToPrimitive, PartialEq, Eq)]
+#[derive(FromPrimitive, ToPrimitive, PartialEq, Eq, Clone, Copy)]
 pub enum EndpointType {
     Nil = ztcore::ZT_EndpointType_ZT_ENDPOINT_TYPE_NIL as isize,
     ZeroTier = ztcore::ZT_EndpointType_ZT_ENDPOINT_TYPE_ZEROTIER as isize,
@@ -34,6 +34,7 @@ pub enum EndpointType {
     IpTcpWs = ztcore::ZT_EndpointType_ZT_ENDPOINT_TYPE_IP_TCP_WS as isize,
 }
 
+#[derive(Clone)]
 pub struct Endpoint {
     pub type_: EndpointType,
     pub(crate) capi: ztcore::ZT_Endpoint

rust-zerotier-core/src/fingerprint.rs

@@ -19,7 +19,7 @@ use std::ptr::copy_nonoverlapping;
 use crate::*;
 use crate::capi as ztcore;
 
-#[derive(PartialEq, Eq)]
+#[derive(PartialEq, Eq, Clone)]
 pub struct Fingerprint {
     pub address: Address,
     pub hash: [u8; 48]
@@ -55,7 +55,7 @@ impl Fingerprint {
 
     pub fn new_from_bytes(bytes: &[u8]) -> Result<Fingerprint, ResultCode> {
         if bytes.len() >= (5 + 48) {
-            let mut fp = Fingerprint {
+            Ok(Fingerprint {
                 address: Address::from(bytes),
                 hash: {
                     let mut h: MaybeUninit<[u8; 48]> = MaybeUninit::uninit();
@@ -64,8 +64,7 @@ impl Fingerprint {
                         h.assume_init()
                     }
                 },
-            };
+            })
-            Ok(fp)
         } else {
             Err(ResultCode::ErrorBadParameter)
         }

service/src/commands/cert.rs

@@ -13,6 +13,7 @@
 
 use std::io::Write;
 use std::str::FromStr;
+use std::sync::Arc;
 
 use clap::ArgMatches;
 use dialoguer::Input;
@@ -20,15 +21,15 @@ use zerotier_core::*;
 
 use crate::store::Store;
 
-fn list(store: &Store, auth_token: &Option<String>) -> i32 {
+fn list(store: &Arc<Store>) -> i32 {
     0
 }
 
-fn show<'a>(store: &Store, cli_args: &ArgMatches<'a>, auth_token: &Option<String>) -> i32 {
+fn show<'a>(store: &Arc<Store>, cli_args: &ArgMatches<'a>) -> i32 {
     0
 }
 
-fn newsid<'a>(store: &Store, cli_args: Option<&ArgMatches<'a>>, auth_token: &Option<String>) -> i32 {
+fn newsid(cli_args: Option<&ArgMatches>) -> i32 {
     let sid = CertificateSubjectUniqueIdSecret::new(CertificateUniqueIdType::NistP384); // right now there's only one type
     let sid = sid.to_json();
     let path = cli_args.map_or("", |cli_args| { cli_args.value_of("path").unwrap_or("") });
@@ -45,7 +46,7 @@ fn newsid<'a>(store: &Store, cli_args: Option<&ArgMatches<'a>>, auth_token: &Opt
     }
 }
 
-fn newcsr<'a>(store: &Store, cli_args: &ArgMatches<'a>, auth_token: &Option<String>) -> i32 {
+fn newcsr(cli_args: &ArgMatches) -> i32 {
     let theme = &dialoguer::theme::SimpleTheme;
 
     let subject_unique_id: String = Input::with_theme(theme)
@@ -245,47 +246,47 @@ fn newcsr<'a>(store: &Store, cli_args: &ArgMatches<'a>, auth_token: &Option<Stri
     })
 }
 
-fn sign<'a>(store: &Store, cli_args: &ArgMatches<'a>, auth_token: &Option<String>) -> i32 {
+fn sign<'a>(store: &Arc<Store>, cli_args: &ArgMatches<'a>) -> i32 {
     0
 }
 
-fn verify<'a>(store: &Store, cli_args: &ArgMatches<'a>, auth_token: &Option<String>) -> i32 {
+fn verify<'a>(store: &Arc<Store>, cli_args: &ArgMatches<'a>) -> i32 {
     0
 }
 
-fn dump<'a>(store: &Store, cli_args: &ArgMatches<'a>, auth_token: &Option<String>) -> i32 {
+fn dump<'a>(store: &Arc<Store>, cli_args: &ArgMatches<'a>) -> i32 {
     0
 }
 
-fn import<'a>(store: &Store, cli_args: &ArgMatches<'a>, auth_token: &Option<String>) -> i32 {
+fn import<'a>(store: &Arc<Store>, cli_args: &ArgMatches<'a>) -> i32 {
     0
 }
 
-fn factoryreset(store: &Store, auth_token: &Option<String>) -> i32 {
+fn factoryreset(store: &Arc<Store>) -> i32 {
     0
 }
 
-fn export<'a>(store: &Store, cli_args: &ArgMatches<'a>, auth_token: &Option<String>) -> i32 {
+fn export<'a>(store: &Arc<Store>, cli_args: &ArgMatches<'a>) -> i32 {
     0
 }
 
-fn delete<'a>(store: &Store, cli_args: &ArgMatches<'a>, auth_token: &Option<String>) -> i32 {
+fn delete<'a>(store: &Arc<Store>, cli_args: &ArgMatches<'a>) -> i32 {
     0
 }
 
-pub(crate) fn run<'a>(store: &Store, cli_args: &ArgMatches<'a>, auth_token: &Option<String>) -> i32 {
+pub(crate) fn run<'a>(store: Arc<Store>, cli_args: &ArgMatches<'a>) -> i32 {
     match cli_args.subcommand() {
-        ("list", None) => list(store, auth_token),
+        ("list", None) => list(&store),
-        ("show", Some(sub_cli_args)) => show(store, sub_cli_args, auth_token),
+        ("show", Some(sub_cli_args)) => show(&store, sub_cli_args),
-        ("newsid", sub_cli_args) => newsid(store, sub_cli_args, auth_token),
+        ("newsid", sub_cli_args) => newsid(sub_cli_args),
-        ("newcsr", Some(sub_cli_args)) => newcsr(store, sub_cli_args, auth_token),
+        ("newcsr", Some(sub_cli_args)) => newcsr(sub_cli_args),
-        ("sign", Some(sub_cli_args)) => sign(store, sub_cli_args, auth_token),
+        ("sign", Some(sub_cli_args)) => sign(&store, sub_cli_args),
-        ("verify", Some(sub_cli_args)) => verify(store, sub_cli_args, auth_token),
+        ("verify", Some(sub_cli_args)) => verify(&store, sub_cli_args),
-        ("dump", Some(sub_cli_args)) => dump(store, sub_cli_args, auth_token),
+        ("dump", Some(sub_cli_args)) => dump(&store, sub_cli_args),
-        ("import", Some(sub_cli_args)) => import(store, sub_cli_args, auth_token),
+        ("import", Some(sub_cli_args)) => import(&store, sub_cli_args),
-        ("factoryreset", None) => factoryreset(store, auth_token),
+        ("factoryreset", None) => factoryreset(&store),
-        ("export", Some(sub_cli_args)) => export(store, sub_cli_args, auth_token),
+        ("export", Some(sub_cli_args)) => export(&store, sub_cli_args),
-        ("delete", Some(sub_cli_args)) => delete(store, sub_cli_args, auth_token),
+        ("delete", Some(sub_cli_args)) => delete(&store, sub_cli_args),
         _ => {
             crate::cli::print_help();
             1

service/src/commands/identity.rs

@@ -14,6 +14,7 @@
 use clap::ArgMatches;
 use crate::store::Store;
 use zerotier_core::{IdentityType, Identity};
+use std::sync::Arc;
 
 fn new_(cli_args: &ArgMatches) -> i32 {
     let id_type = cli_args.value_of("type").map_or(IdentityType::Curve25519, |idt| {
@@ -118,7 +119,7 @@ fn verify(cli_args: &ArgMatches) -> i32 {
     })
 }
 
-pub(crate) fn run<'a>(_: &Store, cli_args: &ArgMatches<'a>, _: &Option<String>) -> i32 {
+pub(crate) fn run<'a>(cli_args: &ArgMatches<'a>) -> i32 {
     match cli_args.subcommand() {
         ("new", Some(sub_cli_args)) => new_(sub_cli_args),
         ("getpublic", Some(sub_cli_args)) => getpublic(sub_cli_args),

service/src/commands/locator.rs

@@ -105,7 +105,7 @@ fn show(cli_args: &ArgMatches) -> i32 {
     0
 }
 
-pub(crate) fn run<'a>(_: &Store, cli_args: &ArgMatches<'a>, _: &Option<String>) -> i32 {
+pub(crate) fn run<'a>(cli_args: &ArgMatches<'a>) -> i32 {
     match cli_args.subcommand() {
         ("new", Some(sub_cli_args)) => new_(sub_cli_args),
         ("verify", Some(sub_cli_args)) => verify(sub_cli_args),

service/src/fastudpsocket.rs

@@ -11,14 +11,15 @@
  */
 /****/
 
+use std::os::raw::c_int;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
-use zerotier_core::{Buffer, InetAddress, InetAddressFamily};
-use num_traits::cast::AsPrimitive;
-use std::os::raw::c_int;
-use crate::osdep as osdep;
 
-#[cfg(target_os = "linux")]
+use num_traits::cast::AsPrimitive;
+
+use zerotier_core::{Buffer, InetAddress, InetAddressFamily};
+
+use crate::osdep as osdep;
 
 /*
  * This is a threaded UDP socket listener for high performance. The fastest way to receive UDP
@@ -26,29 +27,10 @@ use crate::osdep as osdep;
  * for each thread using options like SO_REUSEPORT and concurrent packet listening.
  */
 
-#[cfg(windows)]
+#[cfg(windows)] use winapi::um::winsock2 as winsock2;
-use winapi::um::winsock2 as winsock2;
 
-#[cfg(windows)]
+#[cfg(windows)] pub(crate) type FastUDPRawOsSocket = winsock2::SOCKET;
-pub(crate) type FastUDPRawOsSocket = winsock2::SOCKET;
+#[cfg(unix)] pub(crate) type FastUDPRawOsSocket = c_int;
-
-#[cfg(unix)]
-pub(crate) type FastUDPRawOsSocket = c_int;
-
-/// Test bind UDP to a port at 0.0.0.0 and ::0, returning whether IPv4 and/or IPv6 succeeded (respectively).
-pub(crate) fn test_bind_udp(port: u16) -> (bool, bool) {
-    let v4 = InetAddress::new_ipv4_any(port);
-    let v6 = InetAddress::new_ipv6_any(port);
-    let v4b = bind_udp_socket("", &v4);
-    if v4b.is_ok() {
-        fast_udp_socket_close(v4b.as_ref().unwrap());
-    }
-    let v6b = bind_udp_socket("", &v6);
-    if v6b.is_ok() {
-        fast_udp_socket_close(v6b.as_ref().unwrap());
-    }
-    (v4b.is_ok(), v6b.is_ok())
-}
 
 #[cfg(unix)]
 fn bind_udp_socket(_device_name: &str, address: &InetAddress) -> Result<FastUDPRawOsSocket, &'static str> {
@@ -70,9 +52,10 @@ fn bind_udp_socket(_device_name: &str, address: &InetAddress) -> Result<FastUDPR
         };
 
         #[cfg(not(target_os = "linux"))]
-            let s = osdep::socket(af.as_(), osdep::SOCK_DGRAM.as_(), 0);
+        let s = osdep::socket(af.as_(), osdep::SOCK_DGRAM.as_(), 0);
         #[cfg(target_os = "linux")]
-            let s = osdep::socket(af.as_(), 2, 0);
+        let s = osdep::socket(af.as_(), 2, 0);
+
         if s < 0 {
             return Err("unable to create socket");
         }
@@ -97,17 +80,15 @@ fn bind_udp_socket(_device_name: &str, address: &InetAddress) -> Result<FastUDPR
             setsockopt_results |= osdep::setsockopt(s, osdep::SOL_SOCKET.as_(), osdep::SO_NOSIGPIPE.as_(), (&mut fl as *mut c_int).cast(), fl_size)
         }
 
+        // On linux we bind to device so default route override can work. This
+        // is not required on BSDs as they behave this way if the socket is
+        // bound to the device's address.
         #[cfg(target_os = "linux")] {
             if !_device_name.is_empty() {
                 unsafe {
                     let _ = std::ffi::CString::new(_device_name).map(|dn| {
                         let dnb = dn.as_bytes_with_nul();
-                        let _ = osdep::setsockopt(
+                        let _ = osdep::setsockopt(s.as_(), osdep::SOL_SOCKET.as_(), osdep::SO_BINDTODEVICE.as_(), dnb.as_ptr().cast(), (dnb.len() - 1).as_());
-                            s.as_(),
-                            osdep::SOL_SOCKET.as_(),
-                            osdep::SO_BINDTODEVICE.as_(),
-                            dnb.as_ptr().cast(),
-                            (dnb.len() - 1).as_());
                     });
                 }
             }
@@ -119,25 +100,19 @@ fn bind_udp_socket(_device_name: &str, address: &InetAddress) -> Result<FastUDPR
         }
 
         if af == osdep::AF_INET {
-            #[cfg(any(target_os = "macos", target_os = "ios"))] {
+            #[cfg(not(target_os = "linux"))] {
                 fl = 0;
-                osdep::setsockopt(s, osdep::IPPROTO_IP.as_(), 0x4000 /* IP_DF */, (&mut fl as *mut c_int).cast(), fl_size);
+                osdep::setsockopt(s, osdep::IPPROTO_IP.as_(), osdep::IP_DF.as_(), (&mut fl as *mut c_int).cast(), fl_size);
             }
             #[cfg(target_os = "linux")] {
-                fl = osdep::IP_PMTUDISC_DONT.as_();
+                fl = osdep::IP_PMTUDISC_DONT as c_int;
                 osdep::setsockopt(s, osdep::IPPROTO_IP.as_(), osdep::IP_MTU_DISCOVER.as_(), (&mut fl as *mut c_int).cast(), fl_size);
             }
         }
 
         if af == osdep::AF_INET6 {
-            #[cfg(any(target_os = "macos", target_os = "ios"))] {
+            fl = 0;
-                fl = 0;
+            osdep::setsockopt(s, osdep::IPPROTO_IPV6.as_(), osdep::IPV6_DONTFRAG.as_(), (&mut fl as *mut c_int).cast(), fl_size);
-                osdep::setsockopt(s, osdep::IPPROTO_IPV6.as_(), 62 /* IPV6_DONTFRAG */, (&mut fl as *mut c_int).cast(), fl_size);
-            }
-            #[cfg(target_os = "linux")] {
-                fl = 0;
-                osdep::setsockopt(s, osdep::IPPROTO_IPV6.as_(), osdep::IPV6_DONTFRAG.as_(), (&mut fl as *mut c_int).cast(), fl_size);
-            }
         }
 
         fl = 1048576;
@@ -345,10 +320,12 @@ impl Drop for FastUDPSocket {
 
 #[cfg(test)]
 mod tests {
-    use crate::fastudpsocket::*;
-    use zerotier_core::{InetAddress, Buffer};
     use std::sync::atomic::{AtomicU32, Ordering};
 
+    use zerotier_core::{Buffer, InetAddress};
+
+    use crate::fastudpsocket::*;
+
     #[test]
     fn test_udp_bind_and_transfer() {
         {

service/src/main.rs

@@ -29,92 +29,48 @@ mod weblistener;
 mod osdep; // bindgen generated
 
 use std::boxed::Box;
+use std::rc::Rc;
 use std::sync::Arc;
 
 use crate::store::Store;
 use clap::ArgMatches;
 
 fn main() {
-    let mut process_exit_value: i32 = 0;
+    let cli_args = cli::parse_cli_args();
-
+    let store = || {
-    let cli_args = Box::new(cli::parse_cli_args());
+        //let json_output = cli_args.is_present("json"); // TODO
-    let mut zerotier_path = unsafe { zerotier_core::cstr_to_string(osdep::platformDefaultHomePath(), -1) };
+        let zerotier_path = cli_args.value_of("path").map_or_else(|| unsafe { zerotier_core::cstr_to_string(osdep::platformDefaultHomePath(), -1) }, |ztp| ztp.to_string());
-    let mut auth_token: Option<String> = None;
+        let store = Store::new(zerotier_path.as_str(), cli_args.value_of("token_path").map_or(None, |tp| Some(tp.to_string())), cli_args.value_of("token").map_or(None, |tok| Some(tok.trim().to_string())));
-    let mut auth_token_path: Option<String> = None;
+        if store.is_err() {
-    //let json_output = cli_args.is_present("json");
+            eprintln!("FATAL: error accessing directory '{}': {}", zerotier_path, store.err().unwrap().to_string());
-    let v = cli_args.value_of("path");
+            std::process::exit(1);
-    if v.is_some() {
-        zerotier_path = String::from(v.unwrap());
-    }
-    let v = cli_args.value_of("token");
-    if v.is_some() {
-        auth_token = Some(v.unwrap().trim().to_string());
-    }
-    let v = cli_args.value_of("token_path");
-    if v.is_some() {
-        auth_token_path = Some(v.unwrap().to_string());
-    }
-
-    let store = Store::new(zerotier_path.as_str());
-    if store.is_err() {
-        eprintln!("FATAL: error accessing directory '{}': {}", zerotier_path, store.err().unwrap().to_string());
-        std::process::exit(1);
-    }
-    let store = Arc::new(store.unwrap());
-
-    // From this point on we shouldn't call std::process::exit() since that would
-    // fail to erase zerotier.pid from the working directory.
-
-    if auth_token.is_none() {
-        let t;
-        if auth_token_path.is_some() {
-            t = store.read_file_str(auth_token_path.unwrap().trim());
-        } else {
-            t = store.read_authtoken_secret();
         }
-        if t.is_ok() {
+        Arc::new(store.unwrap())
-            auth_token = Some(t.unwrap().trim().to_string());
+    };
-        }
+    std::process::exit(match cli_args.subcommand() {
-    } else {
-        drop(auth_token_path);
-        auth_token = Some(auth_token.unwrap().trim().to_string());
-    }
-
-    drop(zerotier_path);
-
-    match cli_args.subcommand() {
         ("help", None) => {
-            cli::print_help()
+            cli::print_help();
+            0
         }
         ("version", None) => {
             let ver = zerotier_core::version();
             println!("{}.{}.{}", ver.0, ver.1, ver.2);
+            0
         }
-        ("status", None) => {}
+        ("status", None) => { 0 }
-        ("set", Some(sub_cli_args)) => {}
+        ("set", Some(sub_cli_args)) => { 0 }
-        ("peer", Some(sub_cli_args)) => {}
+        ("peer", Some(sub_cli_args)) => { 0 }
-        ("network", Some(sub_cli_args)) => {}
+        ("network", Some(sub_cli_args)) => { 0 }
-        ("join", Some(sub_cli_args)) => {}
+        ("join", Some(sub_cli_args)) => { 0 }
-        ("leave", Some(sub_cli_args)) => {}
+        ("leave", Some(sub_cli_args)) => { 0 }
-        ("service", None) => {
+        ("service", None) => service::run(store()),
-            drop(cli_args); // free unnecssary memory before launching service
+        ("controller", Some(sub_cli_args)) => { 0 }
-            process_exit_value = service::run(&store, auth_token);
+        ("identity", Some(sub_cli_args)) => crate::commands::identity::run(sub_cli_args),
-        }
+        ("locator", Some(sub_cli_args)) => crate::commands::locator::run(sub_cli_args),
-        ("controller", Some(sub_cli_args)) => {}
+        ("cert", Some(sub_cli_args)) => crate::commands::cert::run(store(), sub_cli_args),
-        ("identity", Some(sub_cli_args)) => {
-            process_exit_value = crate::commands::identity::run(&store, sub_cli_args, &auth_token);
-        }
-        ("locator", Some(sub_cli_args)) => {
-            process_exit_value = crate::commands::locator::run(&store, sub_cli_args, &auth_token);
-        }
-        ("cert", Some(sub_cli_args)) => {
-            process_exit_value = crate::commands::cert::run(&store, sub_cli_args, &auth_token);
-        }
         _ => {
             cli::print_help();
-            process_exit_value = 1;
+            1
         }
-    }
+    });
-
-    std::process::exit(process_exit_value);
 }

service/src/service.rs

@@ -36,7 +36,6 @@ use crate::weblistener::WebListener;
 const CONFIG_CHECK_INTERVAL: i64 = 5000;
 
 struct ServiceIntl {
-    auth_token: String,
     interrupt: Mutex<futures::channel::mpsc::Sender<()>>,
     local_config: Mutex<Arc<LocalConfig>>,
     store: Arc<Store>,
@@ -191,7 +190,7 @@ unsafe impl Send for Service {}
 
 unsafe impl Sync for Service {}
 
-async fn run_async(store: &Arc<Store>, auth_token: String, log: &Arc<Log>, local_config: Arc<LocalConfig>) -> i32 {
+async fn run_async(store: Arc<Store>, log: Arc<Log>, local_config: Arc<LocalConfig>) -> i32 {
     let mut process_exit_value: i32 = 0;
 
     let mut udp_sockets: BTreeMap<InetAddress, FastUDPSocket> = BTreeMap::new();
@@ -203,7 +202,6 @@ async fn run_async(store: &Arc<Store>, auth_token: String, log: &Arc<Log>, local
         log: log.clone(),
         _node: Weak::new(),
         intl: Arc::new(ServiceIntl {
-            auth_token,
             interrupt: Mutex::new(interrupt_tx),
             local_config: Mutex::new(local_config),
             store: store.clone(),
@@ -417,7 +415,7 @@ async fn run_async(store: &Arc<Store>, auth_token: String, log: &Arc<Log>, local
     process_exit_value
 }
 
-pub(crate) fn run(store: &Arc<Store>, auth_token: Option<String>) -> i32 {
+pub(crate) fn run(store: Arc<Store>) -> i32 {
     let local_config = Arc::new(store.read_local_conf(false).unwrap_or_else(|_| { LocalConfig::default() }));
 
     let log = Arc::new(Log::new(
@@ -432,36 +430,18 @@ pub(crate) fn run(store: &Arc<Store>, auth_token: Option<String>) -> i32 {
         "",
     ));
 
-    let auth_token = auth_token.unwrap_or_else(|| -> String {
+    if store.auth_token(true).is_err() {
-        d!(log, "authtoken.secret not found, generating new...");
+        eprintln!("FATAL: error writing new web API authorization token (likely permission problem).");
-        let mut rb = [0_u8; 32];
-        unsafe { crate::osdep::getSecureRandom(rb.as_mut_ptr().cast(), 64) };
-        let mut generated_auth_token = String::new();
-        generated_auth_token.reserve(rb.len());
-        for b in rb.iter() {
-            if *b > 127_u8 {
-                generated_auth_token.push((65 + (*b % 26)) as char); // A..Z
-            } else {
-                generated_auth_token.push((97 + (*b % 26)) as char); // a..z
-            }
-        }
-        if store.write_authtoken_secret(generated_auth_token.as_str()).is_err() {
-            generated_auth_token.clear();
-        }
-        generated_auth_token
-    });
-    if auth_token.is_empty() {
-        log.fatal(format!("unable to write authtoken.secret to '{}'", store.base_path.to_str().unwrap()));
         return 1;
     }
-
     if store.write_pid().is_err() {
-        eprintln!("FATAL: error writing to directory '{}': unable to write zerotier.pid", store.base_path.to_str().unwrap());
+        eprintln!("FATAL: error writing to directory '{}': unable to write zerotier.pid (likely permission problem).", store.base_path.to_str().unwrap());
         return 1;
     }
 
     let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
-    let process_exit_value = rt.block_on(async move { run_async(store, auth_token, &log, local_config).await });
+    let store2 = store.clone();
+    let process_exit_value = rt.block_on(async move { run_async(store2, log, local_config).await });
     rt.shutdown_timeout(Duration::from_millis(500));
 
     store.erase_pid();

service/src/store.rs

@@ -30,6 +30,8 @@ pub(crate) struct Store {
     controller_path: Box<Path>,
     networks_path: Box<Path>,
     certs_path: Box<Path>,
+    auth_token_path: Mutex<Box<Path>>,
+    auth_token: Mutex<String>,
 }
 
 /// Restrict file permissions using OS-specific code in osdep/OSUtils.cpp.
@@ -46,7 +48,7 @@ pub fn lock_down_file(path: &str) {
 impl Store {
     const MAX_OBJECT_SIZE: usize = 262144; // sanity limit
 
-    pub fn new(base_path: &str) -> std::io::Result<Store> {
+    pub fn new(base_path: &str, auth_token_path_override: Option<String>, auth_token_override: Option<String>) -> std::io::Result<Store> {
         let bp = Path::new(base_path);
         let _ = std::fs::create_dir_all(bp);
         let md = bp.metadata()?;
@@ -62,6 +64,16 @@ impl Store {
             controller_path: bp.join("controller.d").into_boxed_path(),
             networks_path: bp.join("networks.d").into_boxed_path(),
             certs_path: bp.join("certs.d").into_boxed_path(),
+            auth_token_path: Mutex::new(auth_token_path_override.map_or_else(|| {
+                bp.join("authtoken.secret").into_boxed_path()
+            }, |auth_token_path_override| {
+                PathBuf::from(auth_token_path_override).into_boxed_path()
+            })),
+            auth_token: Mutex::new(auth_token_override.map_or_else(|| {
+                String::new()
+            }, |auth_token_override| {
+                auth_token_override
+            })),
         };
 
         let _ = std::fs::create_dir_all(&s.peers_path);
@@ -72,7 +84,7 @@ impl Store {
         Ok(s)
     }
 
-    fn make_obj_path(&self, obj_type: &StateObjectType, obj_id: &[u64]) -> Option<PathBuf> {
+    fn make_obj_path_internal(&self, obj_type: &StateObjectType, obj_id: &[u64]) -> Option<PathBuf> {
         match obj_type {
             StateObjectType::IdentityPublic => {
                 Some(self.base_path.join("identity.public"))
@@ -127,6 +139,46 @@ impl Store {
         Err(std::io::Error::new(std::io::ErrorKind::NotFound, "does not exist or is not readable"))
     }
 
+    /// Get content of authtoken.secret or optionally generate and save if missing.
+    pub fn auth_token(&self, generate_if_missing: bool) -> std::io::Result<String> {
+        let mut token = self.auth_token.lock().unwrap();
+        if token.is_empty() {
+            let p = self.auth_token_path.lock().unwrap();
+            let ps = p.to_str().unwrap();
+
+            let token2 = self.read_file(ps).map_or(String::new(), |sb| { String::from_utf8(sb).unwrap_or(String::new()).trim().to_string() });
+            if token2.is_empty() {
+                if generate_if_missing {
+                    let mut rb = [0_u8; 32];
+                    unsafe { crate::osdep::getSecureRandom(rb.as_mut_ptr().cast(), 64) };
+                    token.reserve(rb.len());
+                    for b in rb.iter() {
+                        if *b > 127_u8 {
+                            token.push((65 + (*b % 26)) as char); // A..Z
+                        } else {
+                            token.push((97 + (*b % 26)) as char); // a..z
+                        }
+                    }
+                    let res = self.write_file(ps, token.as_bytes());
+                    if res.is_err() {
+                        token.clear();
+                        Err(res.err().unwrap())
+                    } else {
+                        lock_down_file(ps);
+                        Ok(token.clone())
+                    }
+                } else {
+                    Err(std::io::Error::new(std::io::ErrorKind::NotFound, ""))
+                }
+            } else {
+                *token = token2;
+                Ok(token.clone())
+            }
+        } else {
+            Ok(token.clone())
+        }
+    }
+
     /// Get a list of the network IDs to which this node is joined.
     /// This is used to recall networks on startup by enumerating networks.d
     /// and telling the core to (re)join them all.
@@ -212,20 +264,6 @@ impl Store {
         })
     }
 
-    /// Reads the authtoken.secret file in the home directory.
-    #[inline(always)]
-    pub fn read_authtoken_secret(&self) -> std::io::Result<String> {
-        Ok(self.read_file_str("authtoken.secret")?)
-    }
-
-    /// Write authtoken.secret and lock down file permissions.
-    pub fn write_authtoken_secret(&self, sec: &str) -> std::io::Result<()> {
-        let p = self.base_path.join("authtoken.secret");
-        let _ = std::fs::OpenOptions::new().write(true).truncate(true).create(true).open(&p)?.write_all(sec.as_bytes())?;
-        lock_down_file(p.to_str().unwrap());
-        Ok(())
-    }
-
     /// Write zerotier.pid file with current process's PID.
     #[cfg(unix)]
     pub fn write_pid(&self) -> std::io::Result<()> {
@@ -240,7 +278,7 @@ impl Store {
 
     /// Load a ZeroTier core object.
     pub fn load_object(&self, obj_type: &StateObjectType, obj_id: &[u64]) -> std::io::Result<Vec<u8>> {
-        let obj_path = self.make_obj_path(&obj_type, obj_id);
+        let obj_path = self.make_obj_path_internal(&obj_type, obj_id);
         if obj_path.is_some() {
             return self.read_internal(obj_path.unwrap());
         }
@@ -249,7 +287,7 @@ impl Store {
 
     /// Erase a ZeroTier core object.
     pub fn erase_object(&self, obj_type: &StateObjectType, obj_id: &[u64]) {
-        let obj_path = self.make_obj_path(obj_type, obj_id);
+        let obj_path = self.make_obj_path_internal(obj_type, obj_id);
         if obj_path.is_some() {
             let _ = std::fs::remove_file(obj_path.unwrap());
         }
@@ -258,7 +296,7 @@ impl Store {
     /// Store a ZeroTier core object.
     /// Permissions will also be restricted for some object types.
     pub fn store_object(&self, obj_type: &StateObjectType, obj_id: &[u64], obj_data: &[u8]) -> std::io::Result<()> {
-        let obj_path = self.make_obj_path(obj_type, obj_id);
+        let obj_path = self.make_obj_path_internal(obj_type, obj_id);
         if obj_path.is_some() {
             let obj_path = obj_path.unwrap();
             std::fs::OpenOptions::new().write(true).truncate(true).create(true).open(&obj_path)?.write_all(obj_data)?;