More Rusty MacEthernetTap and Rust binding hacks.

osdep/CMakeLists.txt

@@ -5,6 +5,7 @@ set(src
 	EthernetTap.cpp
 	ManagedRoute.cpp
 	OSUtils.cpp
+	rust-osdep.cpp
 )
 
 set(headers

osdep/MacEthernetTapAgent.c

@@ -225,7 +225,6 @@ int main(int argc,char **argv)
 	const char *mac = argv[2];
 	const char *mtu = argv[3];
 	const char *metric = argv[4];
-	int i = NDRV_SETDMXSPEC;
 
 	s_ndrvfd = socket(AF_NDRV,SOCK_RAW,0);
 	if (s_ndrvfd < 0) {

osdep/rust-osdep.cpp

@@ -0,0 +1,28 @@
+#include "../core/Constants.hpp"
+#include "rust-osdep.h"
+
+#ifdef __APPLE__
+#ifndef SIOCAUTOCONF_START
+#define SIOCAUTOCONF_START _IOWR('i', 132, struct in6_ifreq)    /* accept rtadvd on this interface */
+#endif
+#ifndef SIOCAUTOCONF_STOP
+#define SIOCAUTOCONF_STOP _IOWR('i', 133, struct in6_ifreq)    /* stop accepting rtadv for this interface */
+#endif
+#endif
+
+extern "C" {
+
+#ifdef __APPLE__
+extern const unsigned long c_BIOCSBLEN = BIOCSBLEN;
+extern const unsigned long c_BIOCIMMEDIATE = BIOCIMMEDIATE;
+extern const unsigned long c_BIOCSSEESENT = BIOCSSEESENT;
+extern const unsigned long c_BIOCSETIF = BIOCSETIF;
+extern const unsigned long c_BIOCSHDRCMPLT = BIOCSHDRCMPLT;
+extern const unsigned long c_BIOCPROMISC = BIOCPROMISC;
+extern const unsigned long c_SIOCGIFINFO_IN6 = SIOCGIFINFO_IN6;
+extern const unsigned long c_SIOCSIFINFO_FLAGS = SIOCSIFINFO_FLAGS;
+extern const unsigned long c_SIOCAUTOCONF_START = SIOCAUTOCONF_START;
+extern const unsigned long c_SIOCAUTOCONF_STOP = SIOCAUTOCONF_STOP;
+#endif
+
+}

osdep/rust-osdep.h

@@ -25,10 +25,21 @@
 #include <netinet6/in6_var.h>
 #include <netinet6/nd6.h>
 #include <ifaddrs.h>
-#ifndef SIOCAUTOCONF_START
+#ifdef __cplusplus
-#define SIOCAUTOCONF_START _IOWR('i', 132, struct in6_ifreq)    /* accept rtadvd on this interface */
+extern "C" {
 #endif
-#ifndef SIOCAUTOCONF_STOP
+/* These complex macros don't translate well with Rust bindgen, so compute them with the C compiler and export them. */
-#define SIOCAUTOCONF_STOP _IOWR('i', 133, struct in6_ifreq)    /* stop accepting rtadv for this interface */
+extern const unsigned long c_BIOCSBLEN;
+extern const unsigned long c_BIOCIMMEDIATE;
+extern const unsigned long c_BIOCSSEESENT;
+extern const unsigned long c_BIOCSETIF;
+extern const unsigned long c_BIOCSHDRCMPLT;
+extern const unsigned long c_BIOCPROMISC;
+extern const unsigned long c_SIOCGIFINFO_IN6;
+extern const unsigned long c_SIOCSIFINFO_FLAGS;
+extern const unsigned long c_SIOCAUTOCONF_START;
+extern const unsigned long c_SIOCAUTOCONF_STOP;
+#ifdef __cplusplus
+}
 #endif
 #endif

rust-zerotier-core/src/inetaddress.rs

@@ -172,6 +172,20 @@ impl InetAddress {
         }
     }
 
+    #[inline(always)]
+    pub fn is_v4(&self) -> bool {
+        unsafe {
+            ztcore::ZT_InetAddress_isV4(self.as_capi_ptr()) != 0
+        }
+    }
+
+    #[inline(always)]
+    pub fn is_v6(&self) -> bool {
+        unsafe {
+            ztcore::ZT_InetAddress_isV6(self.as_capi_ptr()) != 0
+        }
+    }
+
     /// Get the address family of this InetAddress.
     pub fn family(&self) -> InetAddressFamily {
         if !self.is_nil() {

rust-zerotier-service/src/physicallink.rs

@@ -15,6 +15,7 @@ use zerotier_core::InetAddress;
 use std::ffi::CStr;
 use std::ptr::{null_mut, copy_nonoverlapping};
 use std::mem::size_of;
+use num_traits::cast::AsPrimitive;
 use crate::osdep as osdep;
 
 pub struct PhysicalLink {
@@ -22,6 +23,11 @@ pub struct PhysicalLink {
     pub device: String
 }
 
+#[inline(always)]
+fn s6_addr_as_ptr<A>(a: &A) -> *A {
+    a as *A
+}
+
 impl PhysicalLink {
     #[cfg(unix)]
     pub fn map<F: FnMut(PhysicalLink)>(mut f: F) {
@@ -32,14 +38,36 @@ impl PhysicalLink {
                 while !i.is_null() {
                     if !(*i).ifa_addr.is_null() {
                         let mut a = InetAddress::new();
-                        if (*(*i).ifa_addr).sa_family == osdep::AF_INET as u8 {
+
+                        let sa_family = (*(*i).ifa_addr).sa_family;
+                        if sa_family == osdep::AF_INET.as_() {
                             copy_nonoverlapping((*i).ifa_addr.cast::<u8>(), (&mut a as *mut InetAddress).cast::<u8>(), size_of::<osdep::sockaddr_in>());
-                        } else if (*(*i).ifa_addr).sa_family == osdep::AF_INET6 as u8 {
+                        } else if sa_family == osdep::AF_INET6.as_() {
                             copy_nonoverlapping((*i).ifa_addr.cast::<u8>(), (&mut a as *mut InetAddress).cast::<u8>(), size_of::<osdep::sockaddr_in6>());
                         } else {
                             continue;
                         }
-                        a.set_port(0);
+
+                        let mut netmask_bits: u16 = 0;
+                        if !(*i).ifa_netmask.is_null() {
+                            if sa_family == osdep::AF_INET.as_() {
+                                let mut a = (*(*i).ifa_netmask.cast::<osdep::sockaddr_in>()).sin_addr.s_addr as u32;
+                                netmask_bits = a.leading_ones() as u16;
+                            } else if sa_family == osdep::AF_INET6.as_() {
+                                let a = s6_addr_as_ptr(&((*(*i).ifa_netmask.cast::<osdep::sockaddr_in6>()).sin6_addr)).cast::<u8>();
+                                for i in 0..16 as isize {
+                                    let mut b = *a.offset(i);
+                                    if b == 0xff {
+                                        netmask_bits += 8;
+                                    } else {
+                                        netmask_bits += b.leading_ones() as u16;
+                                        break;
+                                    }
+                                }
+                            }
+                        }
+                        a.set_port(netmask_bits);
+
                         f(PhysicalLink{
                             address: a,
                             device: if (*i).ifa_name.is_null() { String::new() } else { String::from(CStr::from_ptr((*i).ifa_name).to_str().unwrap()) }

rust-zerotier-service/src/vnp/mac_feth_tap.rs

@@ -1,21 +1,71 @@
-use crate::osdep as osdep;
+/*
-use crate::vnp::Port;
+ * This creates a pair of feth devices with the lower numbered device
-use std::error::Error;
+ * being the ZeroTier virtual interface and the other being the device
-use std::os::raw::c_int;
+ * used to actually read and write packets. The latter gets no IP config
-use std::ffi::CString;
+ * and is only used for I/O. The behavior of feth is similar to the
-use std::thread::JoinHandle;
+ * veth pairs that exist on Linux.
-use std::sync::Mutex;
+ *
+ * The feth device has only existed since MacOS Sierra, but that's fairly
+ * long ago in Mac terms.
+ *
+ * I/O with feth must be done using two different sockets. The BPF socket
+ * is used to receive packets, while an AF_NDRV (low-level network driver
+ * access) socket must be used to inject. AF_NDRV can't read IP frames
+ * since BSD doesn't forward packets out the NDRV tap if they've already
+ * been handled, and while BPF can inject its MTU for injected packets
+ * is limited to 2048. AF_NDRV packet injection is required to inject
+ * ZeroTier's large MTU frames.
+ *
+ * All this stuff is completely undocumented. A lot of tracing through
+ * the Darwin/XNU kernel source was required to find feth in the first place
+ * and figure out how to make this work.
+ */
+
 use std::cell::Cell;
-use zerotier_core::NetworkId;
+use std::error::Error;
+use std::ffi::CString;
+use std::os::raw::{c_int, c_ulong, c_void};
+use std::process::Command;
+use std::sync::Mutex;
+use std::thread::JoinHandle;
+use std::intrinsics::copy_nonoverlapping;
+
 use lazy_static::lazy_static;
+
+use zerotier_core::{NetworkId, MAC, InetAddress, MulticastGroup};
+
+use crate::osdep as osdep;
 use crate::physicallink::PhysicalLink;
+use crate::vnp::Port;
+use std::collections::BTreeSet;
 
 const BPF_BUFFER_SIZE: usize = 131072;
+const IFCONFIG: &str = "/sbin/ifconfig";
+
+// Holds names of feth devices and destroys them on Drop.
+struct MacFethDevice {
+    pub name: String,
+    pub peer_name: String
+}
+
+impl Drop for MacFethDevice {
+    fn drop(&mut self) {
+        if self.name.len() > 0 && self.peer_name.len() > 0 {
+            let destroy_peer = Command::new(IFCONFIG).arg(self.peer_name.as_str()).arg("destroy").spawn();
+            if destroy_peer.is_ok() {
+                let _ = destroy_peer.unwrap().wait();
+            }
+            let destroy = Command::new(IFCONFIG).arg(self.name.as_str()).arg("destroy").spawn();
+            if destroy.is_ok() {
+                let _ = destroy.unwrap().wait();
+            }
+        }
+    }
+}
 
 pub struct MacFethTap {
     network_id: u64,
-    device_name: String,
+    device: MacFethDevice,
-    peer_device_name: String,
     bpf_fd: c_int,
     bpf_read_thread: Cell<Option<JoinHandle<()>>>,
 }
@@ -38,8 +88,10 @@ impl MacFethTap {
     /// Create a new MacFethTap with a function to call for Ethernet frames.
     /// The function F should return as quickly as possible. It should pass copies
     /// of frames elsewhere if anything needs to be done with them. The slice it's
-    /// given will not remain valid after it returns.
+    /// given will not remain valid after it returns. Also note that F will be called
-    pub fn new<F: Fn(&[u8]) + Send + Sync + 'static>(nwid: &NetworkId, eth_frame_func: F) -> Result<MacFethTap, String> {
+    /// from another thread that is spawned here, so all its bound references must
+    /// be "Send" and "Sync" e.g. Arc<>.
+    pub fn new<F: Fn(&[u8]) + Send + Sync + 'static>(nwid: &NetworkId, mac: &MAC, mtu: i32, metric: i32, eth_frame_func: F) -> Result<MacFethTap, String> {
         let _one_at_a_time = unsafe { MAC_FETH_GLOBAL_LOCK.lock().unwrap() };
 
         if unsafe { osdep::getuid() } != 0 {
@@ -71,6 +123,45 @@ impl MacFethTap {
             device_feth_ctr += 1;
         }
 
+        let cmd = Command::new(IFCONFIG).arg(&device_name).arg("create").spawn();
+        if cmd.is_err() {
+            return Err(format!("unable to create device '{}': {}", device_name.as_str(), cmd.err().unwrap().to_string()));
+        }
+        let _ = cmd.unwrap().wait();
+        let cmd = Command::new(IFCONFIG).arg(&peer_device_name).arg("create").spawn();
+        if cmd.is_err() {
+            return Err(format!("unable to create device '{}': {}", peer_device_name.as_str(), cmd.err().unwrap().to_string()));
+        }
+        let _ = cmd.unwrap().wait();
+
+        let device = MacFethDevice{
+            name: device_name,
+            peer_name: peer_device_name,
+        };
+
+        let cmd = Command::new(IFCONFIG).arg(&device.name).arg("lladdr").arg(mac.to_string()).spawn();
+        if cmd.is_err() {
+            return Err(format!("unable to configure device '{}': {}", &device.name, cmd.err().unwrap().to_string()));
+        }
+        let _ = cmd.unwrap().wait();
+
+        let cmd = Command::new(IFCONFIG).arg(&device.peer_name).arg("peer").arg(device.name.as_str()).spawn();
+        if cmd.is_err() {
+            return Err(format!("unable to configure device '{}': {}", &device.peer_name, cmd.err().unwrap().to_string()));
+        }
+        let _ = cmd.unwrap().wait();
+        let cmd = Command::new(IFCONFIG).arg(&device.peer_name).arg("mtu").arg("16370").arg("up").spawn();
+        if cmd.is_err() {
+            return Err(format!("unable to configure device '{}': {}", &device.peer_name, cmd.err().unwrap().to_string()));
+        }
+        let _ = cmd.unwrap().wait();
+
+        let cmd = Command::new(IFCONFIG).arg(&device.name).arg("mtu").arg(mtu.to_string()).arg("metric").arg(metric.to_string()).arg("up").spawn();
+        if cmd.is_err() {
+            return Err(format!("unable to configure device '{}': {}", &device.name.as_str(), cmd.err().unwrap().to_string()));
+        }
+        let _ = cmd.unwrap().wait();
+
         let mut bpf_no: u32 = 1; // start at 1 since some software hard-codes /dev/bpf0
         let mut bpf_fd: c_int = -1;
         loop {
@@ -82,16 +173,63 @@ impl MacFethTap {
             }
             bpf_no += 1;
             if bpf_no > 1000 {
+                break;
+            }
+        }
+        if bpf_fd < 0 {
             return Err(String::from("unable to open /dev/bpf## where attempted ## from 1 to 1000"));
         }
+
+        // Set/get buffer length.
+        let mut fl: c_int = BPF_BUFFER_SIZE as c_int;
+        if unsafe { osdep::ioctl(bpf_fd as c_int, osdep::c_BIOCSBLEN, (&mut fl as *mut c_int).cast::<c_void>()) } != 0 {
+            unsafe { osdep::close(bpf_fd); }
+            return Err(String::from("unable to configure BPF device"));
+        }
+        let bpf_read_size = fl as osdep::size_t;
+
+        // Set immediate mode.
+        fl = 1;
+        if unsafe { osdep::ioctl(bpf_fd as c_int, osdep::c_BIOCIMMEDIATE, (&mut fl as *mut c_int).cast::<c_void>()) } != 0 {
+            unsafe { osdep::close(bpf_fd); }
+            return Err(String::from("unable to configure BPF device"));
         }
 
-        let bpf_fd_copy = bpf_fd;
+        // We don't want to see packets we inject.
-        let t = std::thread::Builder::new().stack_size(zerotier_core::RECOMMENDED_THREAD_STACK_SIZE).spawn(move || {
+        fl = 0;
+        if unsafe { osdep::ioctl(bpf_fd as c_int, osdep::c_BIOCSSEESENT, (&mut fl as *mut c_int).cast::<c_void>()) } != 0 {
+            unsafe { osdep::close(bpf_fd); }
+            return Err(String::from("unable to configure BPF device"));
+        }
+
+        // Set device name that we're monitoring.
+        let mut bpf_ifr: osdep::ifreq = unsafe { std::mem::zeroed() };
+        let peer_dev_name_bytes = device.peer_name.as_bytes();
+        unsafe { copy_nonoverlapping(peer_dev_name_bytes.as_ptr(), bpf_ifr.ifr_name.as_mut_ptr().cast::<u8>(), if peer_dev_name_bytes.len() > (bpf_ifr.ifr_name.len() - 1) { bpf_ifr.ifr_name.len() - 1 } else { peer_dev_name_bytes.len() }); }
+        if unsafe { osdep::ioctl(bpf_fd as c_int, BIOCSETIF, (&mut bpf_ifr as *mut osdep::ifreq).cast::<c_void>()) } != 0 {
+            unsafe { osdep::close(bpf_fd); }
+            return Err(String::from("unable to configure BPF device"));
+        }
+
+        // Include Ethernet header.
+        fl = 1;
+        if unsafe { osdep::ioctl(bpf_fd as c_int, osdep::c_BIOCSHDRCMPLT, (&mut fl as *mut c_int).cast::<c_void>()) } != 0 {
+            unsafe { osdep::close(bpf_fd); }
+            return Err(String::from("unable to configure BPF device"));
+        }
+
+        // Set promiscuous mode so bridging could work, etc.
+        fl = 1;
+        if unsafe { osdep::ioctl(bpf_fd as c_int, osdep::c_BIOCPROMISC, (&mut fl as *mut c_int).cast::<c_void>()) } != 0 {
+            unsafe { osdep::close(bpf_fd); }
+            return Err(String::from("unable to configure BPF device"));
+        }
+
+        let t = std::thread::Builder::new().stack_size(zerotier_core::RECOMMENDED_THREAD_STACK_SIZE).spawn(|| {
             let mut buf: [u8; BPF_BUFFER_SIZE] = [0_u8; BPF_BUFFER_SIZE];
             let hdr_struct_size = std::mem::size_of::<osdep::bpf_hdr>() as isize;
             loop {
-                let n = unsafe { osdep::read(bpf_fd_copy, buf.as_mut_ptr().cast(), BPF_BUFFER_SIZE as osdep::size_t) } as isize;
+                let n = unsafe { osdep::read(bpf_fd, buf.as_mut_ptr().cast(), bpf_read_size) } as isize;
                 if n >= 0 {
                     let mut p: isize = 0;
                     while (p + hdr_struct_size) < n {
@@ -118,12 +256,68 @@ impl MacFethTap {
 
         Ok(MacFethTap {
             network_id: nwid.0,
-            device_name: device_name,
+            device: device,
-            peer_device_name: peer_device_name,
             bpf_fd: bpf_fd,
             bpf_read_thread: Cell::new(Some(t.unwrap()))
         })
     }
+
+    fn have_ip(&self, ip: &InetAddress) -> bool {
+        let mut have_ip = false;
+        PhysicalLink::map(|link: PhysicalLink| {
+            if link.device.eq(dev) && link.address.eq(ip) {
+                have_ip = true;
+            }
+        });
+        have_ip
+    }
+}
+
+impl Port for MacFethTap {
+    fn add_ip(&self, ip: &InetAddress) -> bool {
+        if !self.have_ip(ip) {
+            let cmd = Command::new(IFCONFIG).arg(&self.device.name).arg(if ip.is_v6() { "inet6" } else { "inet" }).arg(ip.to_string()).arg("alias").spawn();
+            if cmd.is_ok() {
+                let _ = cmd.unwrap().wait();
+            }
+            return self.have_ip(ip);
+        }
+        true
+    }
+
+    fn remove_ip(&self, ip: &InetAddress) -> bool {
+        if self.have_ip(ip) {
+            let cmd = Command::new(IFCONFIG).arg(&self.device.name).arg(if ip.is_v6() { "inet6" } else { "inet" }).arg(ip.to_string()).arg("-alias").spawn();
+            if cmd.is_ok() {
+                let _ = cmd.unwrap().wait();
+            }
+            return !self.have_ip(ip);
+        }
+        true // if we don't have it it's successfully removed
+    }
+
+    fn ips(&self) -> Vec<InetAddress> {
+        let mut ipv: Vec<InetAddress> = Vec::new();
+        ipv.reserve(8);
+        let dev = self.device.name.as_str();
+        PhysicalLink::map(|link: PhysicalLink| {
+            if link.device.eq(dev) {
+                ipv.push(link.address.clone());
+            }
+        });
+        ipv.sort();
+        ipv
+    }
+
+    #[inline(always)]
+    fn device_name(&self) -> String {
+        self.device.name.clone()
+    }
+
+    fn get_multicast_groups(&self) -> BTreeSet<MulticastGroup> {
+        let groups: BTreeSet<MulticastGroup> = BTreeSet::new();
+        groups
+    }
 }
 
 impl Drop for MacFethTap {

rust-zerotier-service/src/vnp/mod.rs

@@ -4,14 +4,9 @@
 mod mac_feth_tap;
 
 pub trait Port {
-    fn set_enabled(&self, enabled: bool);
+    fn add_ip(&self, ip: &zerotier_core::InetAddress) -> bool;
-    fn is_enabled(&self, ) -> bool;
+    fn remove_ip(&self, ip: &zerotier_core::InetAddress) -> bool;
-    fn set_ips(&self, ip: &zerotier_core::InetAddress);
     fn ips(&self) -> Vec<zerotier_core::InetAddress>;
     fn device_name(&self) -> String;
-    fn routing_device_name(&self) -> String;
-    fn set_friendly_name(&self, friendly_name: &str);
-    fn friendly_name(&self) -> String;
     fn get_multicast_groups(&self) -> std::collections::BTreeSet<zerotier_core::MulticastGroup>;
 }
-