From 05e2c63d6a146fa607bd2037bf37ffaf65821052 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 26 Jan 2021 22:51:21 -0500 Subject: [PATCH] More Rusty MacEthernetTap and Rust binding hacks. --- {osdep => attic}/Arp.cpp | 0 {osdep => attic}/Arp.hpp | 0 {osdep => attic}/MacKextEthernetTap.cpp | 0 {osdep => attic}/MacKextEthernetTap.hpp | 0 {osdep => attic}/NeighborDiscovery.cpp | 0 {osdep => attic}/NeighborDiscovery.hpp | 0 osdep/CMakeLists.txt | 1 + osdep/MacEthernetTapAgent.c | 1 - osdep/rust-osdep.cpp | 28 +++ osdep/rust-osdep.h | 19 +- rust-zerotier-core/src/inetaddress.rs | 14 ++ rust-zerotier-service/src/physicallink.rs | 34 ++- rust-zerotier-service/src/vnp/mac_feth_tap.rs | 230 ++++++++++++++++-- rust-zerotier-service/src/vnp/mod.rs | 9 +- 14 files changed, 303 insertions(+), 33 deletions(-) rename {osdep => attic}/Arp.cpp (100%) rename {osdep => attic}/Arp.hpp (100%) rename {osdep => attic}/MacKextEthernetTap.cpp (100%) rename {osdep => attic}/MacKextEthernetTap.hpp (100%) rename {osdep => attic}/NeighborDiscovery.cpp (100%) rename {osdep => attic}/NeighborDiscovery.hpp (100%) create mode 100644 osdep/rust-osdep.cpp diff --git a/osdep/Arp.cpp b/attic/Arp.cpp similarity index 100% rename from osdep/Arp.cpp rename to attic/Arp.cpp diff --git a/osdep/Arp.hpp b/attic/Arp.hpp similarity index 100% rename from osdep/Arp.hpp rename to attic/Arp.hpp diff --git a/osdep/MacKextEthernetTap.cpp b/attic/MacKextEthernetTap.cpp similarity index 100% rename from osdep/MacKextEthernetTap.cpp rename to attic/MacKextEthernetTap.cpp diff --git a/osdep/MacKextEthernetTap.hpp b/attic/MacKextEthernetTap.hpp similarity index 100% rename from osdep/MacKextEthernetTap.hpp rename to attic/MacKextEthernetTap.hpp diff --git a/osdep/NeighborDiscovery.cpp b/attic/NeighborDiscovery.cpp similarity index 100% rename from osdep/NeighborDiscovery.cpp rename to attic/NeighborDiscovery.cpp diff --git a/osdep/NeighborDiscovery.hpp b/attic/NeighborDiscovery.hpp similarity index 100% rename from osdep/NeighborDiscovery.hpp rename to attic/NeighborDiscovery.hpp diff --git a/osdep/CMakeLists.txt b/osdep/CMakeLists.txt index 1443dcf32..20ac93b25 100644 --- a/osdep/CMakeLists.txt +++ b/osdep/CMakeLists.txt @@ -5,6 +5,7 @@ set(src EthernetTap.cpp ManagedRoute.cpp OSUtils.cpp + rust-osdep.cpp ) set(headers diff --git a/osdep/MacEthernetTapAgent.c b/osdep/MacEthernetTapAgent.c index 871ec7e46..3a44eadd2 100644 --- a/osdep/MacEthernetTapAgent.c +++ b/osdep/MacEthernetTapAgent.c @@ -225,7 +225,6 @@ int main(int argc,char **argv) const char *mac = argv[2]; const char *mtu = argv[3]; const char *metric = argv[4]; - int i = NDRV_SETDMXSPEC; s_ndrvfd = socket(AF_NDRV,SOCK_RAW,0); if (s_ndrvfd < 0) { diff --git a/osdep/rust-osdep.cpp b/osdep/rust-osdep.cpp new file mode 100644 index 000000000..74355cf7c --- /dev/null +++ b/osdep/rust-osdep.cpp @@ -0,0 +1,28 @@ +#include "../core/Constants.hpp" +#include "rust-osdep.h" + +#ifdef __APPLE__ +#ifndef SIOCAUTOCONF_START +#define SIOCAUTOCONF_START _IOWR('i', 132, struct in6_ifreq) /* accept rtadvd on this interface */ +#endif +#ifndef SIOCAUTOCONF_STOP +#define SIOCAUTOCONF_STOP _IOWR('i', 133, struct in6_ifreq) /* stop accepting rtadv for this interface */ +#endif +#endif + +extern "C" { + +#ifdef __APPLE__ +extern const unsigned long c_BIOCSBLEN = BIOCSBLEN; +extern const unsigned long c_BIOCIMMEDIATE = BIOCIMMEDIATE; +extern const unsigned long c_BIOCSSEESENT = BIOCSSEESENT; +extern const unsigned long c_BIOCSETIF = BIOCSETIF; +extern const unsigned long c_BIOCSHDRCMPLT = BIOCSHDRCMPLT; +extern const unsigned long c_BIOCPROMISC = BIOCPROMISC; +extern const unsigned long c_SIOCGIFINFO_IN6 = SIOCGIFINFO_IN6; +extern const unsigned long c_SIOCSIFINFO_FLAGS = SIOCSIFINFO_FLAGS; +extern const unsigned long c_SIOCAUTOCONF_START = SIOCAUTOCONF_START; +extern const unsigned long c_SIOCAUTOCONF_STOP = SIOCAUTOCONF_STOP; +#endif + +} diff --git a/osdep/rust-osdep.h b/osdep/rust-osdep.h index 71e5dfd62..54c4fee90 100644 --- a/osdep/rust-osdep.h +++ b/osdep/rust-osdep.h @@ -25,10 +25,21 @@ #include #include #include -#ifndef SIOCAUTOCONF_START -#define SIOCAUTOCONF_START _IOWR('i', 132, struct in6_ifreq) /* accept rtadvd on this interface */ +#ifdef __cplusplus +extern "C" { #endif -#ifndef SIOCAUTOCONF_STOP -#define SIOCAUTOCONF_STOP _IOWR('i', 133, struct in6_ifreq) /* stop accepting rtadv for this interface */ +/* These complex macros don't translate well with Rust bindgen, so compute them with the C compiler and export them. */ +extern const unsigned long c_BIOCSBLEN; +extern const unsigned long c_BIOCIMMEDIATE; +extern const unsigned long c_BIOCSSEESENT; +extern const unsigned long c_BIOCSETIF; +extern const unsigned long c_BIOCSHDRCMPLT; +extern const unsigned long c_BIOCPROMISC; +extern const unsigned long c_SIOCGIFINFO_IN6; +extern const unsigned long c_SIOCSIFINFO_FLAGS; +extern const unsigned long c_SIOCAUTOCONF_START; +extern const unsigned long c_SIOCAUTOCONF_STOP; +#ifdef __cplusplus +} #endif #endif diff --git a/rust-zerotier-core/src/inetaddress.rs b/rust-zerotier-core/src/inetaddress.rs index cb66d87f7..aa39dac6a 100644 --- a/rust-zerotier-core/src/inetaddress.rs +++ b/rust-zerotier-core/src/inetaddress.rs @@ -172,6 +172,20 @@ impl InetAddress { } } + #[inline(always)] + pub fn is_v4(&self) -> bool { + unsafe { + ztcore::ZT_InetAddress_isV4(self.as_capi_ptr()) != 0 + } + } + + #[inline(always)] + pub fn is_v6(&self) -> bool { + unsafe { + ztcore::ZT_InetAddress_isV6(self.as_capi_ptr()) != 0 + } + } + /// Get the address family of this InetAddress. pub fn family(&self) -> InetAddressFamily { if !self.is_nil() { diff --git a/rust-zerotier-service/src/physicallink.rs b/rust-zerotier-service/src/physicallink.rs index b4228e8e0..44747796f 100644 --- a/rust-zerotier-service/src/physicallink.rs +++ b/rust-zerotier-service/src/physicallink.rs @@ -15,6 +15,7 @@ use zerotier_core::InetAddress; use std::ffi::CStr; use std::ptr::{null_mut, copy_nonoverlapping}; use std::mem::size_of; +use num_traits::cast::AsPrimitive; use crate::osdep as osdep; pub struct PhysicalLink { @@ -22,6 +23,11 @@ pub struct PhysicalLink { pub device: String } +#[inline(always)] +fn s6_addr_as_ptr(a: &A) -> *A { + a as *A +} + impl PhysicalLink { #[cfg(unix)] pub fn map(mut f: F) { @@ -32,14 +38,36 @@ impl PhysicalLink { while !i.is_null() { if !(*i).ifa_addr.is_null() { let mut a = InetAddress::new(); - if (*(*i).ifa_addr).sa_family == osdep::AF_INET as u8 { + + let sa_family = (*(*i).ifa_addr).sa_family; + if sa_family == osdep::AF_INET.as_() { copy_nonoverlapping((*i).ifa_addr.cast::(), (&mut a as *mut InetAddress).cast::(), size_of::()); - } else if (*(*i).ifa_addr).sa_family == osdep::AF_INET6 as u8 { + } else if sa_family == osdep::AF_INET6.as_() { copy_nonoverlapping((*i).ifa_addr.cast::(), (&mut a as *mut InetAddress).cast::(), size_of::()); } else { continue; } - a.set_port(0); + + let mut netmask_bits: u16 = 0; + if !(*i).ifa_netmask.is_null() { + if sa_family == osdep::AF_INET.as_() { + let mut a = (*(*i).ifa_netmask.cast::()).sin_addr.s_addr as u32; + netmask_bits = a.leading_ones() as u16; + } else if sa_family == osdep::AF_INET6.as_() { + let a = s6_addr_as_ptr(&((*(*i).ifa_netmask.cast::()).sin6_addr)).cast::(); + for i in 0..16 as isize { + let mut b = *a.offset(i); + if b == 0xff { + netmask_bits += 8; + } else { + netmask_bits += b.leading_ones() as u16; + break; + } + } + } + } + a.set_port(netmask_bits); + f(PhysicalLink{ address: a, device: if (*i).ifa_name.is_null() { String::new() } else { String::from(CStr::from_ptr((*i).ifa_name).to_str().unwrap()) } diff --git a/rust-zerotier-service/src/vnp/mac_feth_tap.rs b/rust-zerotier-service/src/vnp/mac_feth_tap.rs index 137ff5648..d863f5825 100644 --- a/rust-zerotier-service/src/vnp/mac_feth_tap.rs +++ b/rust-zerotier-service/src/vnp/mac_feth_tap.rs @@ -1,21 +1,71 @@ -use crate::osdep as osdep; -use crate::vnp::Port; -use std::error::Error; -use std::os::raw::c_int; -use std::ffi::CString; -use std::thread::JoinHandle; -use std::sync::Mutex; +/* + * This creates a pair of feth devices with the lower numbered device + * being the ZeroTier virtual interface and the other being the device + * used to actually read and write packets. The latter gets no IP config + * and is only used for I/O. The behavior of feth is similar to the + * veth pairs that exist on Linux. + * + * The feth device has only existed since MacOS Sierra, but that's fairly + * long ago in Mac terms. + * + * I/O with feth must be done using two different sockets. The BPF socket + * is used to receive packets, while an AF_NDRV (low-level network driver + * access) socket must be used to inject. AF_NDRV can't read IP frames + * since BSD doesn't forward packets out the NDRV tap if they've already + * been handled, and while BPF can inject its MTU for injected packets + * is limited to 2048. AF_NDRV packet injection is required to inject + * ZeroTier's large MTU frames. + * + * All this stuff is completely undocumented. A lot of tracing through + * the Darwin/XNU kernel source was required to find feth in the first place + * and figure out how to make this work. + */ + use std::cell::Cell; -use zerotier_core::NetworkId; +use std::error::Error; +use std::ffi::CString; +use std::os::raw::{c_int, c_ulong, c_void}; +use std::process::Command; +use std::sync::Mutex; +use std::thread::JoinHandle; +use std::intrinsics::copy_nonoverlapping; + use lazy_static::lazy_static; + +use zerotier_core::{NetworkId, MAC, InetAddress, MulticastGroup}; + +use crate::osdep as osdep; use crate::physicallink::PhysicalLink; +use crate::vnp::Port; +use std::collections::BTreeSet; const BPF_BUFFER_SIZE: usize = 131072; +const IFCONFIG: &str = "/sbin/ifconfig"; + +// Holds names of feth devices and destroys them on Drop. +struct MacFethDevice { + pub name: String, + pub peer_name: String +} + +impl Drop for MacFethDevice { + fn drop(&mut self) { + if self.name.len() > 0 && self.peer_name.len() > 0 { + let destroy_peer = Command::new(IFCONFIG).arg(self.peer_name.as_str()).arg("destroy").spawn(); + if destroy_peer.is_ok() { + let _ = destroy_peer.unwrap().wait(); + } + let destroy = Command::new(IFCONFIG).arg(self.name.as_str()).arg("destroy").spawn(); + if destroy.is_ok() { + let _ = destroy.unwrap().wait(); + } + } + } +} pub struct MacFethTap { network_id: u64, - device_name: String, - peer_device_name: String, + device: MacFethDevice, bpf_fd: c_int, bpf_read_thread: Cell>>, } @@ -38,8 +88,10 @@ impl MacFethTap { /// Create a new MacFethTap with a function to call for Ethernet frames. /// The function F should return as quickly as possible. It should pass copies /// of frames elsewhere if anything needs to be done with them. The slice it's - /// given will not remain valid after it returns. - pub fn new(nwid: &NetworkId, eth_frame_func: F) -> Result { + /// given will not remain valid after it returns. Also note that F will be called + /// from another thread that is spawned here, so all its bound references must + /// be "Send" and "Sync" e.g. Arc<>. + pub fn new(nwid: &NetworkId, mac: &MAC, mtu: i32, metric: i32, eth_frame_func: F) -> Result { let _one_at_a_time = unsafe { MAC_FETH_GLOBAL_LOCK.lock().unwrap() }; if unsafe { osdep::getuid() } != 0 { @@ -71,6 +123,45 @@ impl MacFethTap { device_feth_ctr += 1; } + let cmd = Command::new(IFCONFIG).arg(&device_name).arg("create").spawn(); + if cmd.is_err() { + return Err(format!("unable to create device '{}': {}", device_name.as_str(), cmd.err().unwrap().to_string())); + } + let _ = cmd.unwrap().wait(); + let cmd = Command::new(IFCONFIG).arg(&peer_device_name).arg("create").spawn(); + if cmd.is_err() { + return Err(format!("unable to create device '{}': {}", peer_device_name.as_str(), cmd.err().unwrap().to_string())); + } + let _ = cmd.unwrap().wait(); + + let device = MacFethDevice{ + name: device_name, + peer_name: peer_device_name, + }; + + let cmd = Command::new(IFCONFIG).arg(&device.name).arg("lladdr").arg(mac.to_string()).spawn(); + if cmd.is_err() { + return Err(format!("unable to configure device '{}': {}", &device.name, cmd.err().unwrap().to_string())); + } + let _ = cmd.unwrap().wait(); + + let cmd = Command::new(IFCONFIG).arg(&device.peer_name).arg("peer").arg(device.name.as_str()).spawn(); + if cmd.is_err() { + return Err(format!("unable to configure device '{}': {}", &device.peer_name, cmd.err().unwrap().to_string())); + } + let _ = cmd.unwrap().wait(); + let cmd = Command::new(IFCONFIG).arg(&device.peer_name).arg("mtu").arg("16370").arg("up").spawn(); + if cmd.is_err() { + return Err(format!("unable to configure device '{}': {}", &device.peer_name, cmd.err().unwrap().to_string())); + } + let _ = cmd.unwrap().wait(); + + let cmd = Command::new(IFCONFIG).arg(&device.name).arg("mtu").arg(mtu.to_string()).arg("metric").arg(metric.to_string()).arg("up").spawn(); + if cmd.is_err() { + return Err(format!("unable to configure device '{}': {}", &device.name.as_str(), cmd.err().unwrap().to_string())); + } + let _ = cmd.unwrap().wait(); + let mut bpf_no: u32 = 1; // start at 1 since some software hard-codes /dev/bpf0 let mut bpf_fd: c_int = -1; loop { @@ -82,16 +173,63 @@ impl MacFethTap { } bpf_no += 1; if bpf_no > 1000 { - return Err(String::from("unable to open /dev/bpf## where attempted ## from 1 to 1000")); + break; } } + if bpf_fd < 0 { + return Err(String::from("unable to open /dev/bpf## where attempted ## from 1 to 1000")); + } - let bpf_fd_copy = bpf_fd; - let t = std::thread::Builder::new().stack_size(zerotier_core::RECOMMENDED_THREAD_STACK_SIZE).spawn(move || { + // Set/get buffer length. + let mut fl: c_int = BPF_BUFFER_SIZE as c_int; + if unsafe { osdep::ioctl(bpf_fd as c_int, osdep::c_BIOCSBLEN, (&mut fl as *mut c_int).cast::()) } != 0 { + unsafe { osdep::close(bpf_fd); } + return Err(String::from("unable to configure BPF device")); + } + let bpf_read_size = fl as osdep::size_t; + + // Set immediate mode. + fl = 1; + if unsafe { osdep::ioctl(bpf_fd as c_int, osdep::c_BIOCIMMEDIATE, (&mut fl as *mut c_int).cast::()) } != 0 { + unsafe { osdep::close(bpf_fd); } + return Err(String::from("unable to configure BPF device")); + } + + // We don't want to see packets we inject. + fl = 0; + if unsafe { osdep::ioctl(bpf_fd as c_int, osdep::c_BIOCSSEESENT, (&mut fl as *mut c_int).cast::()) } != 0 { + unsafe { osdep::close(bpf_fd); } + return Err(String::from("unable to configure BPF device")); + } + + // Set device name that we're monitoring. + let mut bpf_ifr: osdep::ifreq = unsafe { std::mem::zeroed() }; + let peer_dev_name_bytes = device.peer_name.as_bytes(); + unsafe { copy_nonoverlapping(peer_dev_name_bytes.as_ptr(), bpf_ifr.ifr_name.as_mut_ptr().cast::(), if peer_dev_name_bytes.len() > (bpf_ifr.ifr_name.len() - 1) { bpf_ifr.ifr_name.len() - 1 } else { peer_dev_name_bytes.len() }); } + if unsafe { osdep::ioctl(bpf_fd as c_int, BIOCSETIF, (&mut bpf_ifr as *mut osdep::ifreq).cast::()) } != 0 { + unsafe { osdep::close(bpf_fd); } + return Err(String::from("unable to configure BPF device")); + } + + // Include Ethernet header. + fl = 1; + if unsafe { osdep::ioctl(bpf_fd as c_int, osdep::c_BIOCSHDRCMPLT, (&mut fl as *mut c_int).cast::()) } != 0 { + unsafe { osdep::close(bpf_fd); } + return Err(String::from("unable to configure BPF device")); + } + + // Set promiscuous mode so bridging could work, etc. + fl = 1; + if unsafe { osdep::ioctl(bpf_fd as c_int, osdep::c_BIOCPROMISC, (&mut fl as *mut c_int).cast::()) } != 0 { + unsafe { osdep::close(bpf_fd); } + return Err(String::from("unable to configure BPF device")); + } + + let t = std::thread::Builder::new().stack_size(zerotier_core::RECOMMENDED_THREAD_STACK_SIZE).spawn(|| { let mut buf: [u8; BPF_BUFFER_SIZE] = [0_u8; BPF_BUFFER_SIZE]; let hdr_struct_size = std::mem::size_of::() as isize; loop { - let n = unsafe { osdep::read(bpf_fd_copy, buf.as_mut_ptr().cast(), BPF_BUFFER_SIZE as osdep::size_t) } as isize; + let n = unsafe { osdep::read(bpf_fd, buf.as_mut_ptr().cast(), bpf_read_size) } as isize; if n >= 0 { let mut p: isize = 0; while (p + hdr_struct_size) < n { @@ -118,12 +256,68 @@ impl MacFethTap { Ok(MacFethTap { network_id: nwid.0, - device_name: device_name, - peer_device_name: peer_device_name, + device: device, bpf_fd: bpf_fd, bpf_read_thread: Cell::new(Some(t.unwrap())) }) } + + fn have_ip(&self, ip: &InetAddress) -> bool { + let mut have_ip = false; + PhysicalLink::map(|link: PhysicalLink| { + if link.device.eq(dev) && link.address.eq(ip) { + have_ip = true; + } + }); + have_ip + } +} + +impl Port for MacFethTap { + fn add_ip(&self, ip: &InetAddress) -> bool { + if !self.have_ip(ip) { + let cmd = Command::new(IFCONFIG).arg(&self.device.name).arg(if ip.is_v6() { "inet6" } else { "inet" }).arg(ip.to_string()).arg("alias").spawn(); + if cmd.is_ok() { + let _ = cmd.unwrap().wait(); + } + return self.have_ip(ip); + } + true + } + + fn remove_ip(&self, ip: &InetAddress) -> bool { + if self.have_ip(ip) { + let cmd = Command::new(IFCONFIG).arg(&self.device.name).arg(if ip.is_v6() { "inet6" } else { "inet" }).arg(ip.to_string()).arg("-alias").spawn(); + if cmd.is_ok() { + let _ = cmd.unwrap().wait(); + } + return !self.have_ip(ip); + } + true // if we don't have it it's successfully removed + } + + fn ips(&self) -> Vec { + let mut ipv: Vec = Vec::new(); + ipv.reserve(8); + let dev = self.device.name.as_str(); + PhysicalLink::map(|link: PhysicalLink| { + if link.device.eq(dev) { + ipv.push(link.address.clone()); + } + }); + ipv.sort(); + ipv + } + + #[inline(always)] + fn device_name(&self) -> String { + self.device.name.clone() + } + + fn get_multicast_groups(&self) -> BTreeSet { + let groups: BTreeSet = BTreeSet::new(); + groups + } } impl Drop for MacFethTap { diff --git a/rust-zerotier-service/src/vnp/mod.rs b/rust-zerotier-service/src/vnp/mod.rs index 5f0af71da..69bac6fab 100644 --- a/rust-zerotier-service/src/vnp/mod.rs +++ b/rust-zerotier-service/src/vnp/mod.rs @@ -4,14 +4,9 @@ mod mac_feth_tap; pub trait Port { - fn set_enabled(&self, enabled: bool); - fn is_enabled(&self, ) -> bool; - fn set_ips(&self, ip: &zerotier_core::InetAddress); + fn add_ip(&self, ip: &zerotier_core::InetAddress) -> bool; + fn remove_ip(&self, ip: &zerotier_core::InetAddress) -> bool; fn ips(&self) -> Vec; fn device_name(&self) -> String; - fn routing_device_name(&self) -> String; - fn set_friendly_name(&self, friendly_name: &str); - fn friendly_name(&self) -> String; fn get_multicast_groups(&self) -> std::collections::BTreeSet; } -