Tons and tons of work on the controller, and some other cleanup.

controller/src/controller.rs

@@ -1,175 +0,0 @@
-// (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md.
-
-use std::sync::Arc;
-
-use tokio::time::{Duration, Instant};
-
-use zerotier_network_hypervisor::protocol::{verbs, PacketBuffer};
-use zerotier_network_hypervisor::vl1::{HostSystem, Identity, InnerProtocol, PacketHandlerResult, Path, PathFilter, Peer};
-use zerotier_network_hypervisor::vl2::NetworkId;
-use zerotier_utils::dictionary::Dictionary;
-use zerotier_utils::reaper::Reaper;
-use zerotier_utils::tokio;
-
-use crate::database::Database;
-
-const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
-
-pub struct Controller<DatabaseImpl: Database> {
-    database: Arc<DatabaseImpl>,
-    reaper: Reaper,
-    runtime: tokio::runtime::Handle,
-}
-
-impl<DatabaseImpl: Database> Controller<DatabaseImpl> {
-    pub fn new(database: Arc<DatabaseImpl>, runtime: tokio::runtime::Handle) -> Arc<Self> {
-        Arc::new(Self { database, reaper: Reaper::new(&runtime), runtime })
-    }
-
-    async fn handle_network_config_request<HostSystemImpl: HostSystem>(
-        database: Arc<DatabaseImpl>,
-        source: Arc<Peer<HostSystemImpl>>,
-        source_path: Arc<Path<HostSystemImpl>>,
-        network_id: NetworkId,
-        meta_data: Dictionary,
-        have_revision: Option<u64>,
-        have_timestamp: Option<u64>,
-    ) {
-        println!(
-            "handle_network_config_request {} {} {}",
-            source.identity.to_string(),
-            source_path.endpoint.to_string(),
-            network_id.to_string()
-        );
-        if let Ok(Some(network)) = database.get_network(network_id).await {}
-    }
-}
-
-impl<DatabaseImpl: Database> PathFilter for Controller<DatabaseImpl> {
-    fn should_use_physical_path<HostSystemImpl: HostSystem>(
-        &self,
-        _id: &Identity,
-        _endpoint: &zerotier_network_hypervisor::vl1::Endpoint,
-        _local_socket: Option<&HostSystemImpl::LocalSocket>,
-        _local_interface: Option<&HostSystemImpl::LocalInterface>,
-    ) -> bool {
-        true
-    }
-
-    fn get_path_hints<HostSystemImpl: HostSystem>(
-        &self,
-        _id: &Identity,
-    ) -> Option<
-        Vec<(
-            zerotier_network_hypervisor::vl1::Endpoint,
-            Option<HostSystemImpl::LocalSocket>,
-            Option<HostSystemImpl::LocalInterface>,
-        )>,
-    > {
-        None
-    }
-}
-
-impl<DatabaseImpl: Database> InnerProtocol for Controller<DatabaseImpl> {
-    fn handle_packet<HostSystemImpl: HostSystem>(
-        &self,
-        source: &Arc<Peer<HostSystemImpl>>,
-        source_path: &Arc<Path<HostSystemImpl>>,
-        verb: u8,
-        payload: &PacketBuffer,
-    ) -> PacketHandlerResult {
-        match verb {
-            verbs::VL2_VERB_NETWORK_CONFIG_REQUEST => {
-                let mut cursor = 1;
-
-                let network_id = payload.read_u64(&mut cursor);
-                if network_id.is_err() {
-                    return PacketHandlerResult::Error;
-                }
-                let network_id = NetworkId::from_u64(network_id.unwrap());
-                if network_id.is_none() {
-                    return PacketHandlerResult::Error;
-                }
-                let network_id = network_id.unwrap();
-
-                let meta_data = if (cursor + 2) < payload.len() {
-                    let meta_data_len = payload.read_u16(&mut cursor);
-                    if meta_data_len.is_err() {
-                        return PacketHandlerResult::Error;
-                    }
-                    if let Ok(d) = payload.read_bytes(meta_data_len.unwrap() as usize, &mut cursor) {
-                        let d = Dictionary::from_bytes(d);
-                        if d.is_none() {
-                            return PacketHandlerResult::Error;
-                        }
-                        d.unwrap()
-                    } else {
-                        return PacketHandlerResult::Error;
-                    }
-                } else {
-                    Dictionary::new()
-                };
-
-                let (have_revision, have_timestamp) = if (cursor + 16) <= payload.len() {
-                    let r = payload.read_u64(&mut cursor);
-                    let t = payload.read_u64(&mut cursor);
-                    if r.is_err() || t.is_err() {
-                        return PacketHandlerResult::Error;
-                    }
-                    (Some(r.unwrap()), Some(t.unwrap()))
-                } else {
-                    (None, None)
-                };
-
-                if let Some(deadline) = Instant::now().checked_add(REQUEST_TIMEOUT) {
-                    self.reaper.add(
-                        self.runtime.spawn(Self::handle_network_config_request(
-                            self.database.clone(),
-                            source.clone(),
-                            source_path.clone(),
-                            network_id,
-                            meta_data,
-                            have_revision,
-                            have_timestamp,
-                        )),
-                        deadline,
-                    );
-                } else {
-                    eprintln!("WARNING: Instant::now() + REQUEST_TIMEOUT overflowed! should be impossible.");
-                }
-
-                PacketHandlerResult::Ok
-            }
-            _ => PacketHandlerResult::NotHandled,
-        }
-    }
-
-    fn handle_error<HostSystemImpl: HostSystem>(
-        &self,
-        _source: &Arc<Peer<HostSystemImpl>>,
-        _source_path: &Arc<Path<HostSystemImpl>>,
-        _in_re_verb: u8,
-        _in_re_message_id: u64,
-        _error_code: u8,
-        _payload: &PacketBuffer,
-        _cursor: &mut usize,
-    ) -> PacketHandlerResult {
-        PacketHandlerResult::NotHandled
-    }
-
-    fn handle_ok<HostSystemImpl: HostSystem>(
-        &self,
-        _source: &Arc<Peer<HostSystemImpl>>,
-        _source_path: &Arc<Path<HostSystemImpl>>,
-        _in_re_verb: u8,
-        _in_re_message_id: u64,
-        _payload: &PacketBuffer,
-        _cursor: &mut usize,
-    ) -> PacketHandlerResult {
-        PacketHandlerResult::NotHandled
-    }
-
-    fn should_communicate_with(&self, _: &Identity) -> bool {
-        true
-    }
-}

controller/src/database.rs

@@ -2,7 +2,7 @@ use std::error::Error;
 
 use async_trait::async_trait;
 
-use zerotier_network_hypervisor::vl1::{Address, NodeStorage};
+use zerotier_network_hypervisor::vl1::{Address, InetAddress, NodeStorage};
 use zerotier_network_hypervisor::vl2::NetworkId;
 
 use crate::model::*;
@@ -14,7 +14,40 @@ pub trait Database: Sync + Send + NodeStorage + 'static {
 
     async fn list_members(&self, network_id: NetworkId) -> Result<Vec<Address>, Box<dyn Error>>;
     async fn get_member(&self, network_id: NetworkId, node_id: Address) -> Result<Option<Member>, Box<dyn Error>>;
-    async fn save_member(&self, obj: &Member) -> Result<(), Box<dyn Error>>;
+    async fn save_member(&self, obj: Member) -> Result<(), Box<dyn Error>>;
+
+    /// List members deauthorized after a given time (milliseconds since epoch).
+    ///
+    /// The default trait implementation uses a brute force method. This should be reimplemented if a
+    /// more efficient way is available.
+    async fn list_members_deauthorized_after(&self, network_id: NetworkId, cutoff: i64) -> Result<Vec<Address>, Box<dyn Error>> {
+        let mut v = Vec::new();
+        let members = self.list_members(network_id).await?;
+        for a in members.iter() {
+            if let Some(m) = self.get_member(network_id, *a).await? {
+                if m.last_deauthorized_time.unwrap_or(i64::MIN) >= cutoff {
+                    v.push(m.node_id);
+                }
+            }
+        }
+        Ok(v)
+    }
+
+    /// Check if any member of a network has a given static IP assignment.
+    ///
+    /// The default trait implementation uses a brute force method. This should be reimplemented if a
+    /// more efficient way is available.
+    async fn is_ip_assigned(&self, network_id: NetworkId, ip: &InetAddress) -> Result<bool, Box<dyn Error>> {
+        let members = self.list_members(network_id).await?;
+        for a in members.iter() {
+            if let Some(m) = self.get_member(network_id, *a).await? {
+                if m.ip_assignments.iter().any(|ip2| ip2.ip_bytes().eq(ip.ip_bytes())) {
+                    return Ok(true);
+                }
+            }
+        }
+        return Ok(false);
+    }
 
     async fn log_request(&self, obj: &RequestLogItem) -> Result<(), Box<dyn Error>>;
 }

controller/src/filedatabase.rs

@@ -169,15 +169,15 @@ impl Database for FileDatabase {
         }
     }
 
-    async fn save_member(&self, obj: &Member) -> Result<(), Box<dyn Error>> {
+    async fn save_member(&self, obj: Member) -> Result<(), Box<dyn Error>> {
         let base_member_path = member_path(&self.base_path, obj.network_id, obj.node_id);
         if !fs::metadata(&base_member_path).await.is_ok() {
-            fs::write(base_member_path, to_json_pretty(obj).as_bytes()).await?;
+            fs::write(base_member_path, to_json_pretty(&obj).as_bytes()).await?;
         }
 
         fs::write(
             member_path(&self.cache_path, obj.network_id, obj.node_id),
-            serde_json::to_vec(obj)?.as_slice(),
+            serde_json::to_vec(&obj)?.as_slice(),
         )
         .await?;
         Ok(())

controller/src/handler.rs

@@ -0,0 +1,303 @@
+// (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md.
+
+use std::error::Error;
+use std::sync::Arc;
+
+use tokio::time::{Duration, Instant};
+
+use zerotier_network_hypervisor::protocol::{verbs, PacketBuffer, DEFAULT_MULTICAST_LIMIT, ZEROTIER_VIRTUAL_NETWORK_DEFAULT_MTU};
+use zerotier_network_hypervisor::vl1::{HostSystem, Identity, InnerProtocol, Node, PacketHandlerResult, Path, PathFilter, Peer};
+use zerotier_network_hypervisor::vl2::{CertificateOfMembership, CertificateOfOwnership, NetworkConfig, NetworkId, Tag};
+use zerotier_utils::dictionary::Dictionary;
+use zerotier_utils::error::UnexpectedError;
+use zerotier_utils::ms_since_epoch;
+use zerotier_utils::reaper::Reaper;
+use zerotier_utils::tokio;
+
+use crate::database::Database;
+use crate::model::{AuthorizationResult, Member, CREDENTIAL_WINDOW_SIZE_DEFAULT};
+
+const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
+
+pub struct Handler<DatabaseImpl: Database> {
+    reaper: Reaper,
+    runtime: tokio::runtime::Handle,
+    inner: Arc<Inner<DatabaseImpl>>,
+}
+
+impl<DatabaseImpl: Database> Handler<DatabaseImpl> {
+    pub fn new(database: Arc<DatabaseImpl>, runtime: tokio::runtime::Handle, local_identity: Identity) -> Arc<Self> {
+        assert!(local_identity.secret.is_some());
+        Arc::new(Self {
+            reaper: Reaper::new(&runtime),
+            runtime,
+            inner: Arc::new(Inner::<DatabaseImpl> { database, local_identity }),
+        })
+    }
+}
+
+impl<DatabaseImpl: Database> PathFilter for Handler<DatabaseImpl> {
+    fn should_use_physical_path<HostSystemImpl: HostSystem>(
+        &self,
+        _id: &Identity,
+        _endpoint: &zerotier_network_hypervisor::vl1::Endpoint,
+        _local_socket: Option<&HostSystemImpl::LocalSocket>,
+        _local_interface: Option<&HostSystemImpl::LocalInterface>,
+    ) -> bool {
+        true
+    }
+
+    fn get_path_hints<HostSystemImpl: HostSystem>(
+        &self,
+        _id: &Identity,
+    ) -> Option<
+        Vec<(
+            zerotier_network_hypervisor::vl1::Endpoint,
+            Option<HostSystemImpl::LocalSocket>,
+            Option<HostSystemImpl::LocalInterface>,
+        )>,
+    > {
+        None
+    }
+}
+
+impl<DatabaseImpl: Database> InnerProtocol for Handler<DatabaseImpl> {
+    fn handle_packet<HostSystemImpl: HostSystem>(
+        &self,
+        _node: &Node<HostSystemImpl>,
+        source: &Arc<Peer<HostSystemImpl>>,
+        source_path: &Arc<Path<HostSystemImpl>>,
+        message_id: u64,
+        verb: u8,
+        payload: &PacketBuffer,
+    ) -> PacketHandlerResult {
+        match verb {
+            verbs::VL2_VERB_NETWORK_CONFIG_REQUEST => {
+                let mut cursor = 1;
+
+                let network_id = payload.read_u64(&mut cursor);
+                if network_id.is_err() {
+                    return PacketHandlerResult::Error;
+                }
+                let network_id = NetworkId::from_u64(network_id.unwrap());
+                if network_id.is_none() {
+                    return PacketHandlerResult::Error;
+                }
+                let network_id = network_id.unwrap();
+
+                let meta_data = if (cursor + 2) < payload.len() {
+                    let meta_data_len = payload.read_u16(&mut cursor);
+                    if meta_data_len.is_err() {
+                        return PacketHandlerResult::Error;
+                    }
+                    if let Ok(d) = payload.read_bytes(meta_data_len.unwrap() as usize, &mut cursor) {
+                        let d = Dictionary::from_bytes(d);
+                        if d.is_none() {
+                            return PacketHandlerResult::Error;
+                        }
+                        d.unwrap()
+                    } else {
+                        return PacketHandlerResult::Error;
+                    }
+                } else {
+                    Dictionary::new()
+                };
+
+                let (have_revision, have_timestamp) = if (cursor + 16) <= payload.len() {
+                    let r = payload.read_u64(&mut cursor);
+                    let t = payload.read_u64(&mut cursor);
+                    if r.is_err() || t.is_err() {
+                        return PacketHandlerResult::Error;
+                    }
+                    (Some(r.unwrap()), Some(t.unwrap()))
+                } else {
+                    (None, None)
+                };
+
+                // Launch handler as an async background task.
+                let (inner, source2, source_path2) = (self.inner.clone(), source.clone(), source_path.clone());
+                self.reaper.add(
+                    self.runtime.spawn(async move {
+                        // TODO: log errors
+                        let _ = inner.handle_network_config_request(
+                            source2,
+                            source_path2,
+                            message_id,
+                            network_id,
+                            meta_data,
+                            have_revision,
+                            have_timestamp,
+                        );
+                    }),
+                    Instant::now().checked_add(REQUEST_TIMEOUT).unwrap(),
+                );
+
+                PacketHandlerResult::Ok
+            }
+            _ => PacketHandlerResult::NotHandled,
+        }
+    }
+
+    fn handle_error<HostSystemImpl: HostSystem>(
+        &self,
+        _node: &Node<HostSystemImpl>,
+        _source: &Arc<Peer<HostSystemImpl>>,
+        _source_path: &Arc<Path<HostSystemImpl>>,
+        _message_id: u64,
+        _in_re_verb: u8,
+        _in_re_message_id: u64,
+        _error_code: u8,
+        _payload: &PacketBuffer,
+        _cursor: &mut usize,
+    ) -> PacketHandlerResult {
+        PacketHandlerResult::NotHandled
+    }
+
+    fn handle_ok<HostSystemImpl: HostSystem>(
+        &self,
+        _node: &Node<HostSystemImpl>,
+        _source: &Arc<Peer<HostSystemImpl>>,
+        _source_path: &Arc<Path<HostSystemImpl>>,
+        _message_id: u64,
+        _in_re_verb: u8,
+        _in_re_message_id: u64,
+        _payload: &PacketBuffer,
+        _cursor: &mut usize,
+    ) -> PacketHandlerResult {
+        PacketHandlerResult::NotHandled
+    }
+
+    fn should_communicate_with(&self, _: &Identity) -> bool {
+        true
+    }
+}
+
+struct Inner<DatabaseImpl: Database> {
+    database: Arc<DatabaseImpl>,
+    local_identity: Identity,
+}
+
+impl<DatabaseImpl: Database> Inner<DatabaseImpl> {
+    async fn handle_network_config_request<HostSystemImpl: HostSystem>(
+        self: Arc<Self>,
+        source: Arc<Peer<HostSystemImpl>>,
+        _source_path: Arc<Path<HostSystemImpl>>,
+        _message_id: u64,
+        network_id: NetworkId,
+        _meta_data: Dictionary,
+        _have_revision: Option<u64>,
+        _have_timestamp: Option<u64>,
+    ) -> Result<(AuthorizationResult, Option<NetworkConfig>), Box<dyn Error>> {
+        let network = self.database.get_network(network_id).await?;
+        if network.is_none() {
+            // TODO: send error
+            return Ok((AuthorizationResult::Rejected, None));
+        }
+        let network = network.unwrap();
+
+        let mut member = self.database.get_member(network_id, source.identity.address).await?;
+        let mut member_changed = false;
+        let legacy_v1 = source.identity.p384.is_none();
+
+        // If we have a member object and a pinned identity, check to make sure it matches.
+        if let Some(member) = member.as_ref() {
+            if let Some(pinned_identity) = member.identity.as_ref() {
+                if !pinned_identity.eq(&source.identity) {
+                    return Ok((AuthorizationResult::RejectedIdentityMismatch, None));
+                }
+            }
+        }
+
+        let now = ms_since_epoch();
+
+        let mut authorization_result = AuthorizationResult::Rejected;
+        let mut authorized = member.as_ref().map_or(false, |m| m.authorized());
+        if !authorized {
+            if member.is_none() {
+                if network.learn_members {
+                    let _ = member.insert(Member::new_with_identity(source.identity.clone(), network_id));
+                    member_changed = true;
+                } else {
+                    return Ok((AuthorizationResult::Rejected, None));
+                }
+            }
+
+            if !network.private {
+                authorization_result = AuthorizationResult::ApprovedOnPublicNetwork;
+                authorized = true;
+                member.as_mut().unwrap().last_authorized_time = Some(now);
+                member_changed = true;
+            }
+        }
+
+        let mut member = member.unwrap();
+
+        let nc: Option<NetworkConfig> = if authorized {
+            // ====================================================================================
+            // Authorized requests are handled here
+            // ====================================================================================
+
+            // TODO: check SSO
+
+            // Figure out time bounds for the certificate to generate.
+            let max_delta = network.credential_window_size.unwrap_or(CREDENTIAL_WINDOW_SIZE_DEFAULT);
+
+            // Get a list of all network members that were deauthorized but are still within the time window.
+            // These will be issued revocations to remind the node not to speak to them until they fall off.
+            let deauthed_members_still_in_window = self.database.list_members_deauthorized_after(network.id, now - max_delta).await;
+
+            // Check and if necessary auto-assign static IPs for this member.
+            member_changed |= network.check_zt_ip_assignments(self.database.as_ref(), &mut member).await;
+
+            let mut nc = NetworkConfig::new(network_id, source.identity.address);
+
+            nc.name = member.name.clone();
+            nc.private = network.private;
+            nc.timestamp = now;
+            nc.max_delta = max_delta;
+            nc.revision = now as u64;
+            nc.mtu = network.mtu.unwrap_or(ZEROTIER_VIRTUAL_NETWORK_DEFAULT_MTU as u16);
+            nc.multicast_limit = network.multicast_limit.unwrap_or(DEFAULT_MULTICAST_LIMIT as u32);
+            nc.routes = network.ip_routes;
+            nc.static_ips = member.ip_assignments.clone();
+            nc.rules = network.rules;
+            nc.dns = network.dns;
+
+            nc.certificate_of_membership = Some(
+                CertificateOfMembership::new(&self.local_identity, network_id, &source.identity, now, max_delta, legacy_v1)
+                    .ok_or(UnexpectedError)?,
+            );
+
+            let mut coo = CertificateOfOwnership::new(network_id, now, source.identity.address, legacy_v1);
+            for ip in nc.static_ips.iter() {
+                coo.add_ip(ip);
+            }
+            if !coo.sign(&self.local_identity, &source.identity) {
+                return Err(Box::new(UnexpectedError));
+            }
+            nc.certificates_of_ownership.push(coo);
+
+            for (id, value) in member.tags.iter() {
+                let _ = nc.tags.insert(
+                    *id,
+                    Tag::new(*id, *value, &self.local_identity, network_id, &source.identity, now, legacy_v1).ok_or(UnexpectedError)?,
+                );
+            }
+
+            // TODO: node info, which isn't supported in v1 so not needed yet
+
+            // TODO: revocations!
+
+            Some(nc)
+            // ====================================================================================
+        } else {
+            None
+        };
+
+        if member_changed {
+            self.database.save_member(member).await?;
+        }
+
+        Ok((authorization_result, nc))
+    }
+}

controller/src/lib.rs

@@ -1,6 +1,6 @@
 // (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md.
 
-pub mod controller;
 pub mod database;
 pub mod filedatabase;
+pub mod handler;
 pub mod model;

controller/src/main.rs

@@ -6,9 +6,9 @@ use std::time::Duration;
 
 use clap::{Arg, Command};
 
-use zerotier_network_controller::controller::Controller;
 use zerotier_network_controller::database::Database;
 use zerotier_network_controller::filedatabase::FileDatabase;
+use zerotier_network_controller::handler::Handler;
 
 use zerotier_network_hypervisor::{VERSION_MAJOR, VERSION_MINOR, VERSION_REVISION};
 use zerotier_utils::exitcode;
@@ -16,12 +16,12 @@ use zerotier_utils::tokio::runtime::Runtime;
 use zerotier_vl1_service::VL1Service;
 
 async fn run<DatabaseImpl: Database>(database: Arc<DatabaseImpl>, runtime: &Runtime) -> i32 {
-    let controller = Controller::new(database.clone(), runtime.handle().clone());
+    let handler = Handler::new(database.clone(), runtime.handle().clone(), todo!());
 
     let svc = VL1Service::new(
         database.clone(),
-        controller.clone(),
+        handler.clone(),
-        controller.clone(),
+        handler.clone(),
         zerotier_vl1_service::VL1Settings::default(),
     );
     if svc.is_ok() {

controller/src/model.rs

@@ -1,225 +0,0 @@
-// (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md.
-
-use std::collections::{HashMap, HashSet};
-use std::hash::Hash;
-
-use serde::{Deserialize, Serialize};
-
-use zerotier_network_hypervisor::vl1::{Address, Endpoint, Identity, InetAddress};
-use zerotier_network_hypervisor::vl2::NetworkId;
-
-/// Static string included in JSON-serializable objects to indicate their object type through the API.
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub enum ObjectType {
-    #[serde(rename = "network")]
-    Network,
-    #[serde(rename = "member")]
-    Member,
-}
-
-impl ObjectType {
-    fn network() -> ObjectType {
-        Self::Network
-    }
-    fn member() -> ObjectType {
-        Self::Member
-    }
-}
-
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Ipv4AssignMode {
-    pub zt: bool,
-}
-
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Ipv6AssignMode {
-    pub zt: bool,
-    pub rfc4193: bool,
-    #[serde(rename = "6plane")]
-    pub _6plane: bool,
-}
-
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct IpAssignmentPool {
-    #[serde(rename = "ipRangeStart")]
-    ip_range_start: InetAddress,
-    #[serde(rename = "ipRangeEnd")]
-    ip_range_end: InetAddress,
-}
-
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Tag {
-    pub id: u32,
-    pub value: u32,
-}
-
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Network {
-    pub id: NetworkId,
-    pub name: String,
-
-    #[serde(rename = "creationTime")]
-    pub creation_time: i64,
-
-    #[serde(rename = "multicastLimit")]
-    pub multicast_limit: u64,
-    #[serde(rename = "enableBroadcast")]
-    pub enable_broadcast: bool,
-
-    #[serde(rename = "v4AssignMode")]
-    pub v4_assign_mode: Ipv4AssignMode,
-    #[serde(rename = "v6AssignMode")]
-    pub v6_assign_mode: Ipv6AssignMode,
-    #[serde(rename = "ipAssignmentPools")]
-    pub ip_assignment_pools: Vec<IpAssignmentPool>,
-
-    #[serde(rename = "rulesSource")]
-    pub rules_source: String,
-
-    pub mtu: u16,
-    pub private: bool,
-
-    #[serde(default = "ObjectType::network")]
-    pub objtype: ObjectType,
-}
-
-impl Hash for Network {
-    #[inline]
-    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
-        self.id.hash(state)
-    }
-}
-
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Member {
-    #[serde(rename = "address")]
-    pub node_id: Address,
-    #[serde(rename = "networkId")]
-    pub network_id: NetworkId,
-    pub identity: Option<Identity>,
-    pub name: String,
-    pub description: String,
-
-    #[serde(rename = "creationTime")]
-    pub creation_time: i64,
-
-    #[serde(rename = "revision")]
-    pub last_modified_time: i64,
-
-    pub authorized: bool,
-    #[serde(rename = "lastAuthorizedTime")]
-    pub last_authorized_time: Option<i64>,
-    #[serde(rename = "lastDeauthorizedTime")]
-    pub last_deauthorized_time: Option<i64>,
-
-    #[serde(rename = "ipAssignments")]
-    pub ip_assignments: HashSet<InetAddress>,
-    #[serde(rename = "noAutoAssignIps")]
-    pub no_auto_assign_ips: bool,
-
-    /// If true this member is a full Ethernet bridge.
-    #[serde(rename = "activeBridge")]
-    pub bridge: bool,
-
-    pub tags: Vec<Tag>,
-
-    #[serde(rename = "ssoExempt")]
-    pub sso_exempt: bool,
-
-    /// If true this node is explicitly listed in every member's network configuration.
-    #[serde(rename = "advertised")]
-    pub advertised: bool,
-
-    /// Most recently generated and signed network configuration for this member in binary format.
-    #[serde(rename = "networkConfig")]
-    pub network_config: Option<Vec<u8>>,
-
-    /// API object type documentation field, not actually edited/used.
-    #[serde(default = "ObjectType::member")]
-    pub objtype: ObjectType,
-}
-
-impl Hash for Member {
-    #[inline]
-    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
-        self.node_id.hash(state);
-        self.network_id.hash(state);
-    }
-}
-
-/// A complete network with all member configuration information for import/export or blob storage.
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct NetworkExport {
-    pub network: Network,
-    pub members: HashMap<Address, Member>,
-}
-
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[repr(u8)]
-pub enum AuthorizationResult {
-    #[serde(rename = "r")]
-    Rejected = 0,
-    #[serde(rename = "rs")]
-    RejectedViaSSO = 1,
-    #[serde(rename = "rt")]
-    RejectedViaToken = 2,
-    #[serde(rename = "ro")]
-    RejectedTooOld = 3,
-    #[serde(rename = "a")]
-    Approved = 16,
-    #[serde(rename = "as")]
-    ApprovedViaSSO = 17,
-    #[serde(rename = "at")]
-    ApprovedViaToken = 18,
-}
-
-impl ToString for AuthorizationResult {
-    fn to_string(&self) -> String {
-        match self {
-            Self::Rejected => "rejected",
-            _ => "",
-        }
-        .to_string()
-    }
-}
-
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct RequestLogItem {
-    #[serde(rename = "nwid")]
-    pub network_id: NetworkId,
-    #[serde(rename = "nid")]
-    pub node_id: Address,
-    #[serde(rename = "cid")]
-    pub controller_node_id: Address,
-    #[serde(rename = "md")]
-    pub metadata: Vec<u8>,
-    #[serde(rename = "ts")]
-    pub timestamp: i64,
-    #[serde(rename = "v")]
-    pub version: (u16, u16, u16, u16),
-    #[serde(rename = "s")]
-    pub source_remote_endpoint: Endpoint,
-    #[serde(rename = "sh")]
-    pub source_hops: u8,
-    #[serde(rename = "r")]
-    pub result: AuthorizationResult,
-}
-
-impl ToString for RequestLogItem {
-    fn to_string(&self) -> String {
-        format!(
-            "{} {} {} ts={} v={}.{}.{},{} s={},{} {}",
-            self.controller_node_id.to_string(),
-            self.network_id.to_string(),
-            self.node_id.to_string(),
-            self.timestamp,
-            self.version.0,
-            self.version.1,
-            self.version.2,
-            self.version.3,
-            self.source_remote_endpoint.to_string(),
-            self.source_hops,
-            self.result.to_string()
-        )
-    }
-}

controller/src/model/member.rs

@@ -0,0 +1,108 @@
+// (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md.
+
+use std::collections::{HashMap, HashSet};
+use std::hash::Hash;
+
+use serde::{Deserialize, Serialize};
+
+use zerotier_network_hypervisor::vl1::{Address, Identity, InetAddress};
+use zerotier_network_hypervisor::vl2::NetworkId;
+
+use crate::model::ObjectType;
+
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Member {
+    #[serde(rename = "address")]
+    pub node_id: Address,
+    #[serde(rename = "networkId")]
+    pub network_id: NetworkId,
+
+    /// Pinned full member identity, if known.
+    pub identity: Option<Identity>,
+
+    /// A short name that can also be used for DNS, etc.
+    #[serde(default)]
+    pub name: String,
+
+    /// Time member was most recently authorized, None for 'never'.
+    #[serde(rename = "lastAuthorizedTime")]
+    pub last_authorized_time: Option<i64>,
+
+    /// Time member was most recently deauthorized, None for 'never'.
+    #[serde(rename = "lastDeauthorizedTime")]
+    pub last_deauthorized_time: Option<i64>,
+
+    /// ZeroTier-managed IP assignments.
+    #[serde(rename = "ipAssignments")]
+    #[serde(default)]
+    pub ip_assignments: HashSet<InetAddress>,
+
+    /// If true, do not auto-assign IPs in the controller.
+    #[serde(rename = "noAutoAssignIps")]
+    #[serde(default)]
+    pub no_auto_assign_ips: bool,
+
+    /// If true this member is a full Ethernet bridge.
+    #[serde(rename = "activeBridge")]
+    #[serde(default)]
+    pub bridge: bool,
+
+    /// Tags that can be used in rule evaluation for ACL-like behavior.
+    #[serde(default)]
+    pub tags: HashMap<u32, u32>,
+
+    /// Member is exempt from SSO, authorization managed conventionally.
+    #[serde(rename = "ssoExempt")]
+    #[serde(default)]
+    pub sso_exempt: bool,
+
+    /// If true this node is explicitly listed in every member's network configuration.
+    /// This is only supported for V2 nodes.
+    #[serde(rename = "advertised")]
+    #[serde(default)]
+    pub advertised: bool,
+
+    /// API object type documentation field, not actually edited/used.
+    #[serde(default = "ObjectType::member")]
+    pub objtype: ObjectType,
+}
+
+impl Member {
+    pub fn new_with_identity(identity: Identity, network_id: NetworkId) -> Self {
+        Self {
+            node_id: identity.address,
+            network_id,
+            identity: Some(identity),
+            name: String::new(),
+            last_authorized_time: None,
+            last_deauthorized_time: None,
+            ip_assignments: HashSet::new(),
+            no_auto_assign_ips: false,
+            bridge: false,
+            tags: HashMap::new(),
+            sso_exempt: false,
+            advertised: false,
+            objtype: ObjectType::Member,
+        }
+    }
+
+    /// Check whether this member is authorized, which is true if the last authorized time is after last deauthorized time.
+    pub fn authorized(&self) -> bool {
+        self.last_authorized_time
+            .map_or(false, |la| self.last_deauthorized_time.map_or(true, |ld| la > ld))
+    }
+}
+
+impl Hash for Member {
+    #[inline]
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.node_id.hash(state);
+        self.network_id.hash(state);
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Tag {
+    pub id: u32,
+    pub value: u32,
+}

controller/src/model/mod.rs

@@ -0,0 +1,130 @@
+// (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md.
+
+mod member;
+mod network;
+
+pub use member::*;
+pub use network::*;
+
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+use zerotier_network_hypervisor::vl1::{Address, Endpoint};
+use zerotier_network_hypervisor::vl2::NetworkId;
+
+/// A complete network with all member configuration information for import/export or blob storage.
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct NetworkExport {
+    pub network: Network,
+    pub members: HashMap<Address, Member>,
+}
+
+/// Static string included in JSON-serializable objects to indicate their object type through the API.
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum ObjectType {
+    #[serde(rename = "network")]
+    Network,
+    #[serde(rename = "member")]
+    Member,
+}
+
+impl ObjectType {
+    fn network() -> ObjectType {
+        Self::Network
+    }
+    fn member() -> ObjectType {
+        Self::Member
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[repr(u8)]
+pub enum AuthorizationResult {
+    #[serde(rename = "r")]
+    Rejected = 0,
+    #[serde(rename = "rs")]
+    RejectedViaSSO = 1,
+    #[serde(rename = "rt")]
+    RejectedViaToken = 2,
+    #[serde(rename = "ro")]
+    RejectedTooOld = 3,
+    #[serde(rename = "re")]
+    RejectedDueToError = 4,
+    #[serde(rename = "rm")]
+    RejectedIdentityMismatch = 5,
+    #[serde(rename = "a")]
+    Approved = 128,
+    #[serde(rename = "as")]
+    ApprovedViaSSO = 129,
+    #[serde(rename = "at")]
+    ApprovedViaToken = 130,
+    #[serde(rename = "ap")]
+    ApprovedOnPublicNetwork = 131,
+}
+
+impl AuthorizationResult {
+    pub fn as_str(&self) -> &'static str {
+        // These short codes should match the serde enum names above.
+        match self {
+            Self::Rejected => "r",
+            Self::RejectedViaSSO => "rs",
+            Self::RejectedViaToken => "rt",
+            Self::RejectedTooOld => "ro",
+            Self::RejectedDueToError => "re",
+            Self::RejectedIdentityMismatch => "rm",
+            Self::Approved => "a",
+            Self::ApprovedViaSSO => "as",
+            Self::ApprovedViaToken => "at",
+            Self::ApprovedOnPublicNetwork => "ap",
+        }
+    }
+}
+
+impl ToString for AuthorizationResult {
+    #[inline(always)]
+    fn to_string(&self) -> String {
+        self.as_str().to_string()
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RequestLogItem {
+    #[serde(rename = "nwid")]
+    pub network_id: NetworkId,
+    #[serde(rename = "nid")]
+    pub node_id: Address,
+    #[serde(rename = "cid")]
+    pub controller_node_id: Address,
+    #[serde(rename = "md")]
+    pub metadata: Vec<u8>,
+    #[serde(rename = "ts")]
+    pub timestamp: i64,
+    #[serde(rename = "v")]
+    pub version: (u16, u16, u16, u16),
+    #[serde(rename = "s")]
+    pub source_remote_endpoint: Endpoint,
+    #[serde(rename = "sh")]
+    pub source_hops: u8,
+    #[serde(rename = "r")]
+    pub result: AuthorizationResult,
+}
+
+impl ToString for RequestLogItem {
+    fn to_string(&self) -> String {
+        format!(
+            "{} {} {} ts={} v={}.{}.{},{} s={},{} {}",
+            self.controller_node_id.to_string(),
+            self.network_id.to_string(),
+            self.node_id.to_string(),
+            self.timestamp,
+            self.version.0,
+            self.version.1,
+            self.version.2,
+            self.version.3,
+            self.source_remote_endpoint.to_string(),
+            self.source_hops,
+            self.result.to_string()
+        )
+    }
+}

controller/src/model/network.rs

@@ -0,0 +1,180 @@
+// (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md.
+
+use std::collections::{HashMap, HashSet};
+use std::hash::Hash;
+
+use serde::{Deserialize, Serialize};
+
+use zerotier_network_hypervisor::vl1::InetAddress;
+use zerotier_network_hypervisor::vl2::networkconfig::IpRoute;
+use zerotier_network_hypervisor::vl2::{NetworkId, Rule};
+
+use crate::database::Database;
+use crate::model::{Member, ObjectType};
+
+pub const CREDENTIAL_WINDOW_SIZE_DEFAULT: i64 = 1000 * 60 * 60;
+
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
+pub struct Ipv4AssignMode {
+    pub zt: bool,
+}
+
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
+pub struct Ipv6AssignMode {
+    pub zt: bool,
+    pub rfc4193: bool,
+    #[serde(rename = "6plane")]
+    pub _6plane: bool,
+}
+
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Hash)]
+pub struct IpAssignmentPool {
+    #[serde(rename = "ipRangeStart")]
+    ip_range_start: InetAddress,
+    #[serde(rename = "ipRangeEnd")]
+    ip_range_end: InetAddress,
+}
+
+/// Virtual network configuration.
+///
+/// This contains only fields of relevance to the controller. Other fields can be tracked by various
+/// database implementations such as row last modified, creation time, ownership in an admin panel, etc.
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Network {
+    pub id: NetworkId,
+
+    /// Network name that's sent to network members
+    #[serde(default)]
+    pub name: String,
+
+    /// Guideline for the maximum number of multicast recipients on a network (not a hard limit).
+    /// Setting to zero disables multicast entirely. The default is used if this is not set.
+    #[serde(rename = "multicastLimit")]
+    pub multicast_limit: Option<u32>,
+
+    /// If true, this network supports ff:ff:ff:ff:ff:ff Ethernet broadcast.
+    #[serde(rename = "enableBroadcast")]
+    #[serde(default = "troo")]
+    pub enable_broadcast: bool,
+
+    /// Auto IP assignment mode(s) for IPv4 addresses.
+    #[serde(rename = "v4AssignMode")]
+    #[serde(default)]
+    pub v4_assign_mode: Ipv4AssignMode,
+
+    /// Auto IP assignment mode(s) for IPv6 addresses.
+    #[serde(rename = "v6AssignMode")]
+    #[serde(default)]
+    pub v6_assign_mode: Ipv6AssignMode,
+
+    /// IPv4 or IPv6 auto-assignment pools available, must be present to use 'zt' mode.
+    #[serde(rename = "ipAssignmentPools")]
+    #[serde(default)]
+    pub ip_assignment_pools: HashSet<IpAssignmentPool>,
+
+    /// IPv4 or IPv6 routes to advertise.
+    #[serde(default)]
+    pub ip_routes: HashSet<IpRoute>,
+
+    /// DNS records to push to members.
+    pub dns: HashMap<String, HashSet<InetAddress>>,
+
+    /// Network rule set.
+    #[serde(default)]
+    pub rules: Vec<Rule>,
+
+    /// If set this overrides the default "agreement" window for certificates and credentials.
+    ///
+    /// Making it smaller causes deauthorized nodes to fall out of the window more rapidly but can
+    /// come at the expense of reliability if it's too short for everyone to update their certs
+    /// on time from the controller. Note that revocations are also used to deauthorize nodes
+    /// promptly, so nodes will still deauthorize quickly even if the window is long.
+    ///
+    /// Usually this does not need to be changed.
+    #[serde(rename = "credentialWindowSize")]
+    pub credential_window_size: Option<i64>,
+
+    /// MTU inside the virtual network, default of 2800 is used if not set.
+    pub mtu: Option<u16>,
+
+    /// If true the network has access control, which is usually what you want.
+    #[serde(default = "troo")]
+    pub private: bool,
+
+    /// If true this network will add not-authorized members for anyone who requests a config.
+    #[serde(default = "troo")]
+    pub learn_members: bool,
+
+    /// Static object type field for use with API.
+    #[serde(default = "ObjectType::network")]
+    pub objtype: ObjectType,
+}
+
+impl Hash for Network {
+    #[inline]
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.id.hash(state)
+    }
+}
+
+#[inline(always)]
+fn troo() -> bool {
+    true
+}
+
+impl Network {
+    /// Check member IP assignments and return 'true' if IP assignments were created or modified.
+    pub async fn check_zt_ip_assignments<DatabaseImpl: Database>(&self, database: &DatabaseImpl, member: &mut Member) -> bool {
+        let mut modified = false;
+
+        if self.v4_assign_mode.zt {
+            if !member.ip_assignments.iter().any(|ip| ip.is_ipv4()) {
+                'ip_search: for pool in self.ip_assignment_pools.iter() {
+                    if pool.ip_range_start.is_ipv4() && pool.ip_range_end.is_ipv4() {
+                        let mut ip_ptr = u32::from_be_bytes(pool.ip_range_start.ip_bytes().try_into().unwrap());
+                        let ip_end = u32::from_be_bytes(pool.ip_range_end.ip_bytes().try_into().unwrap());
+                        while ip_ptr < ip_end {
+                            for route in self.ip_routes.iter() {
+                                let ip = InetAddress::from_ip_port(&ip_ptr.to_be_bytes(), route.target.port()); // IP/bits
+                                if ip.is_within(&route.target) {
+                                    if !database.is_ip_assigned(self.id, &ip).await.unwrap_or(true) {
+                                        modified = true;
+                                        let _ = member.ip_assignments.insert(ip);
+                                        break 'ip_search;
+                                    }
+                                }
+                            }
+                            ip_ptr += 1;
+                        }
+                    }
+                }
+            }
+        }
+
+        if self.v6_assign_mode.zt {
+            if !member.ip_assignments.iter().any(|ip| ip.is_ipv6()) {
+                'ip_search: for pool in self.ip_assignment_pools.iter() {
+                    if pool.ip_range_start.is_ipv6() && pool.ip_range_end.is_ipv6() {
+                        let mut ip_ptr = u128::from_be_bytes(pool.ip_range_start.ip_bytes().try_into().unwrap());
+                        let ip_end = u128::from_be_bytes(pool.ip_range_end.ip_bytes().try_into().unwrap());
+                        while ip_ptr < ip_end {
+                            for route in self.ip_routes.iter() {
+                                let ip = InetAddress::from_ip_port(&ip_ptr.to_be_bytes(), route.target.port()); // IP/bits
+                                if ip.is_within(&route.target) {
+                                    if !database.is_ip_assigned(self.id, &ip).await.unwrap_or(true) {
+                                        modified = true;
+                                        let _ = member.ip_assignments.insert(ip);
+                                        break 'ip_search;
+                                    }
+                                }
+                            }
+                            ip_ptr += 1;
+                        }
+                    }
+                }
+            }
+        }
+
+        modified
+    }
+}

network-hypervisor/src/protocol.rs

@@ -111,6 +111,12 @@ pub mod verbs {
 /// two fragments.
 pub const UDP_DEFAULT_MTU: usize = 1432;
 
+/// Default MTU inside VL2 virtual networks.
+pub const ZEROTIER_VIRTUAL_NETWORK_DEFAULT_MTU: usize = 2800;
+
+/// Default multicast limit if not set in the network.
+pub const DEFAULT_MULTICAST_LIMIT: usize = 32;
+
 /// Length of an address in bytes.
 pub const ADDRESS_SIZE: usize = 5;
 

network-hypervisor/src/vl1/identity.rs

@@ -156,6 +156,8 @@ impl Identity {
         + P384_ECDSA_SIGNATURE_SIZE
         + P384_ECDSA_SIGNATURE_SIZE;
 
+    pub const FINGERPRINT_SIZE: usize = IDENTITY_FINGERPRINT_SIZE;
+
     const ALGORITHM_X25519: u8 = 0x01;
     const ALGORITHM_EC_NIST_P384: u8 = 0x02;
     const FLAG_INCLUDES_SECRETS: u8 = 0x80;

network-hypervisor/src/vl1/node.rs

@@ -134,8 +134,10 @@ pub trait InnerProtocol: Sync + Send + 'static {
     /// Do not attempt to handle OK or ERROR. Instead implement handle_ok() and handle_error().
     fn handle_packet<HostSystemImpl: HostSystem>(
         &self,
+        node: &Node<HostSystemImpl>,
         source: &Arc<Peer<HostSystemImpl>>,
         source_path: &Arc<Path<HostSystemImpl>>,
+        message_id: u64,
         verb: u8,
         payload: &PacketBuffer,
     ) -> PacketHandlerResult;
@@ -143,8 +145,10 @@ pub trait InnerProtocol: Sync + Send + 'static {
     /// Handle errors, returning true if the error was recognized.
     fn handle_error<HostSystemImpl: HostSystem>(
         &self,
+        node: &Node<HostSystemImpl>,
         source: &Arc<Peer<HostSystemImpl>>,
         source_path: &Arc<Path<HostSystemImpl>>,
+        message_id: u64,
         in_re_verb: u8,
         in_re_message_id: u64,
         error_code: u8,
@@ -155,8 +159,10 @@ pub trait InnerProtocol: Sync + Send + 'static {
     /// Handle an OK, returing true if the OK was recognized.
     fn handle_ok<HostSystemImpl: HostSystem>(
         &self,
+        node: &Node<HostSystemImpl>,
         source: &Arc<Peer<HostSystemImpl>>,
         source_path: &Arc<Path<HostSystemImpl>>,
+        message_id: u64,
         in_re_verb: u8,
         in_re_message_id: u64,
         payload: &PacketBuffer,
@@ -1044,8 +1050,10 @@ impl InnerProtocol for DummyInnerProtocol {
     #[inline(always)]
     fn handle_packet<HostSystemImpl: HostSystem>(
         &self,
+        _node: &Node<HostSystemImpl>,
         _source: &Arc<Peer<HostSystemImpl>>,
         _source_path: &Arc<Path<HostSystemImpl>>,
+        _message_id: u64,
         _verb: u8,
         _payload: &PacketBuffer,
     ) -> PacketHandlerResult {
@@ -1055,8 +1063,10 @@ impl InnerProtocol for DummyInnerProtocol {
     #[inline(always)]
     fn handle_error<HostSystemImpl: HostSystem>(
         &self,
+        _node: &Node<HostSystemImpl>,
         _source: &Arc<Peer<HostSystemImpl>>,
         _source_path: &Arc<Path<HostSystemImpl>>,
+        _message_id: u64,
         _in_re_verb: u8,
         _in_re_message_id: u64,
         _error_code: u8,
@@ -1069,8 +1079,10 @@ impl InnerProtocol for DummyInnerProtocol {
     #[inline(always)]
     fn handle_ok<HostSystemImpl: HostSystem>(
         &self,
+        _node: &Node<HostSystemImpl>,
         _source: &Arc<Peer<HostSystemImpl>>,
         _source_path: &Arc<Path<HostSystemImpl>>,
+        _message_id: u64,
         _in_re_verb: u8,
         _in_re_message_id: u64,
         _payload: &PacketBuffer,

network-hypervisor/src/vl1/peer.rs

@@ -293,7 +293,7 @@ impl<HostSystemImpl: HostSystem> Peer<HostSystemImpl> {
             };
 
             let mut aes_gmac_siv = self.v1_proto_static_secret.aes_gmac_siv.get();
-            aes_gmac_siv.encrypt_init(&self.v1_proto_next_message_id().to_ne_bytes());
+            aes_gmac_siv.encrypt_init(&self.v1_proto_next_message_id().to_be_bytes());
             aes_gmac_siv.encrypt_set_aad(&v1::get_packet_aad_bytes(
                 self.identity.address,
                 node.identity.address,
@@ -326,7 +326,7 @@ impl<HostSystemImpl: HostSystem> Peer<HostSystemImpl> {
                 &self.v1_proto_static_secret,
                 {
                     let header = packet.struct_mut_at::<v1::PacketHeader>(0).unwrap();
-                    header.id = self.v1_proto_next_message_id().to_ne_bytes();
+                    header.id = self.v1_proto_next_message_id().to_be_bytes();
                     header.dest = self.identity.address.to_bytes();
                     header.src = node.identity.address.to_bytes();
                     header.flags_cipher_hops = flags_cipher_hops;
@@ -531,13 +531,14 @@ impl<HostSystemImpl: HostSystem> Peer<HostSystemImpl> {
                         packet_header.hops(),
                         &payload,
                     ),
-                    verbs::VL1_ERROR => self.handle_incoming_error(host_system, inner, node, time_ticks, source_path, &payload),
+                    verbs::VL1_ERROR => self.handle_incoming_error(host_system, inner, node, time_ticks, source_path, message_id, &payload),
                     verbs::VL1_OK => self.handle_incoming_ok(
                         host_system,
                         inner,
                         node,
                         time_ticks,
                         source_path,
+                        message_id,
                         packet_header.hops(),
                         path_is_known,
                         &payload,
@@ -551,7 +552,7 @@ impl<HostSystemImpl: HostSystem> Peer<HostSystemImpl> {
                         self.handle_incoming_push_direct_paths(host_system, node, time_ticks, source_path, &payload)
                     }
                     verbs::VL1_USER_MESSAGE => self.handle_incoming_user_message(host_system, node, time_ticks, source_path, &payload),
-                    _ => inner.handle_packet(self, &source_path, verb, &payload),
+                    _ => inner.handle_packet(node, self, &source_path, message_id, verb, &payload),
                 };
             }
         }
@@ -623,20 +624,23 @@ impl<HostSystemImpl: HostSystem> Peer<HostSystemImpl> {
         self: &Arc<Self>,
         _: &HostSystemImpl,
         inner: &InnerProtocolImpl,
-        _: &Node<HostSystemImpl>,
+        node: &Node<HostSystemImpl>,
         _: i64,
         source_path: &Arc<Path<HostSystemImpl>>,
+        message_id: u64,
         payload: &PacketBuffer,
     ) -> PacketHandlerResult {
         let mut cursor = 0;
         if let Ok(error_header) = payload.read_struct::<v1::message_component_structs::ErrorHeader>(&mut cursor) {
-            let in_re_message_id: MessageId = u64::from_ne_bytes(error_header.in_re_message_id);
+            let in_re_message_id: MessageId = u64::from_be_bytes(error_header.in_re_message_id);
             if self.message_id_counter.load(Ordering::Relaxed).wrapping_sub(in_re_message_id) <= PACKET_RESPONSE_COUNTER_DELTA_MAX {
                 match error_header.in_re_verb {
                     _ => {
                         return inner.handle_error(
+                            node,
                             self,
                             &source_path,
+                            message_id,
                             error_header.in_re_verb,
                             in_re_message_id,
                             error_header.error_code,
@@ -657,6 +661,7 @@ impl<HostSystemImpl: HostSystem> Peer<HostSystemImpl> {
         node: &Node<HostSystemImpl>,
         time_ticks: i64,
         source_path: &Arc<Path<HostSystemImpl>>,
+        message_id: u64,
         hops: u8,
         path_is_known: bool,
         payload: &PacketBuffer,
@@ -730,7 +735,16 @@ impl<HostSystemImpl: HostSystem> Peer<HostSystemImpl> {
                     }
 
                     _ => {
-                        return inner.handle_ok(self, &source_path, ok_header.in_re_verb, in_re_message_id, payload, &mut cursor);
+                        return inner.handle_ok(
+                            node,
+                            self,
+                            &source_path,
+                            message_id,
+                            ok_header.in_re_verb,
+                            in_re_message_id,
+                            payload,
+                            &mut cursor,
+                        );
                     }
                 }
             }
@@ -890,7 +904,7 @@ fn v1_proto_try_aead_decrypt(
             let (mut salsa, poly1305_key) = v1_proto_salsa_poly_create(secret, packet_header, payload.len() + v1::HEADER_SIZE);
             let mac = poly1305::compute(&poly1305_key, &payload.as_bytes());
             if mac[0..8].eq(&packet_header.mac) {
-                let message_id = u64::from_ne_bytes(packet_header.id);
+                let message_id = u64::from_be_bytes(packet_header.id);
                 if cipher == v1::CIPHER_SALSA2012_POLY1305 {
                     salsa.crypt_in_place(payload.as_bytes_mut());
                     Some(message_id)
@@ -941,7 +955,7 @@ fn v1_proto_try_aead_decrypt(
                 // AES-GMAC-SIV encrypts the packet ID too as part of its computation of a single
                 // opaque 128-bit tag, so to get the original packet ID we have to grab it from the
                 // decrypted tag.
-                Some(u64::from_ne_bytes(*array_range::<u8, 16, 0, 8>(tag)))
+                Some(u64::from_be_bytes(*array_range::<u8, 16, 0, 8>(tag)))
             } else {
                 None
             }

network-hypervisor/src/vl2/certificateofmembership.rs

@@ -8,25 +8,63 @@ use crate::vl2::NetworkId;
 use serde::{Deserialize, Serialize};
 
 use zerotier_crypto::hash::SHA384;
+use zerotier_crypto::verified::Verified;
 use zerotier_utils::arrayvec::ArrayVec;
 use zerotier_utils::blob::Blob;
+use zerotier_utils::error::InvalidParameterError;
 use zerotier_utils::memory;
 
 #[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct CertificateOfMembership {
     pub network_id: NetworkId,
-    pub issued_to: Address,
     pub timestamp: i64,
     pub max_delta: i64,
-    pub issued_to_fingerprint: Blob<48>,
+    pub issued_to: Address,
-    pub v1: Option<[u8; 32]>,
+    pub issued_to_fingerprint: Blob<{ identity::IDENTITY_FINGERPRINT_SIZE }>,
-    pub signed_by: Address,
     pub signature: ArrayVec<u8, { identity::IDENTITY_MAX_SIGNATURE_SIZE }>,
+    pub version: u8,
 }
 
 impl CertificateOfMembership {
-    /// Generate the first three "qualifiers" for V1 nodes.
+    /// Create a new signed certificate of membership.
-    fn v1_proto_write_first_3_qualifiers(&self, q: &mut [u64]) {
+    /// None is returned if an error occurs, such as the issuer missing its secrets.
+    pub fn new(
+        issuer: &Identity,
+        network_id: NetworkId,
+        issued_to: &Identity,
+        timestamp: i64,
+        max_delta: i64,
+        legacy_v1: bool,
+    ) -> Option<Self> {
+        let mut com = CertificateOfMembership {
+            network_id,
+            timestamp,
+            max_delta,
+            issued_to: issued_to.address,
+            issued_to_fingerprint: Blob::default(),
+            signature: ArrayVec::new(),
+            version: 0,
+        };
+        if legacy_v1 {
+            com.issued_to_fingerprint = Blob::from(Self::v1_proto_issued_to_fingerprint(issued_to, Some(issuer.address)));
+            com.version = 1;
+            if let Some(signature) = issuer.sign(&com.v1_proto_get_qualifier_bytes(), true) {
+                com.signature = signature;
+                Some(com)
+            } else {
+                None
+            }
+        } else {
+            com.issued_to_fingerprint = Blob::from(issued_to.fingerprint);
+            com.version = 2;
+            todo!()
+        }
+    }
+
+    fn v1_proto_get_qualifier_bytes(&self) -> [u8; 168] {
+        assert_eq!(self.version, 1);
+        let mut q = [0u64; 21];
+
         q[0] = 0;
         q[1] = self.timestamp.to_be() as u64;
         q[2] = self.max_delta.to_be() as u64;
@@ -38,68 +76,130 @@ impl CertificateOfMembership {
         let a: u64 = self.issued_to.into();
         q[7] = a.to_be();
         q[8] = 0xffffffffffffffffu64; // no to_be needed
-    }
-
-    /// Generate all the qualifiers for V1 nodes, which is part of marshaling for those.
-    fn v1_proto_get_qualifier_bytes(&self) -> Option<[u8; 168]> {
-        self.v1.as_ref().map(|v1| {
-            let mut q = [0u64; 21];
-
-            self.v1_proto_write_first_3_qualifiers(&mut q);
 
+        let fp = self.issued_to_fingerprint.as_bytes();
         q[9] = 3;
-            q[10] = u64::from_ne_bytes(v1[0..8].try_into().unwrap());
+        q[10] = u64::from_ne_bytes(fp[0..8].try_into().unwrap());
         q[11] = 0xffffffffffffffffu64;
         q[12] = 4;
-            q[13] = u64::from_ne_bytes(v1[8..16].try_into().unwrap());
+        q[13] = u64::from_ne_bytes(fp[8..16].try_into().unwrap());
         q[14] = 0xffffffffffffffffu64;
         q[15] = 5;
-            q[16] = u64::from_ne_bytes(v1[16..24].try_into().unwrap());
+        q[16] = u64::from_ne_bytes(fp[16..24].try_into().unwrap());
         q[17] = 0xffffffffffffffffu64;
         q[18] = 6;
-            q[19] = u64::from_ne_bytes(v1[24..32].try_into().unwrap());
+        q[19] = u64::from_ne_bytes(fp[24..32].try_into().unwrap());
         q[20] = 0xffffffffffffffffu64;
 
         *memory::as_byte_array(&q)
-        })
     }
 
-    /// Sign this certificate of membership for use by V1 nodes.
+    fn v1_proto_issued_to_fingerprint(issued_to: &Identity, signed_by: Option<Address>) -> [u8; 48] {
-    ///
-    /// This should be used in conjunction with v1_proto_to_bytes() to generate a COM for v1
-    /// nodes. This sets the issued_to and v1 fields.
-    pub fn v1_proto_sign(&mut self, issuer: &Identity, issued_to: &Identity) -> bool {
         let mut v1_signee_hasher = SHA384::new();
         v1_signee_hasher.update(&issued_to.address.to_bytes());
         v1_signee_hasher.update(&issued_to.x25519);
         v1_signee_hasher.update(&issued_to.ed25519);
-        let v1_signee_hash = v1_signee_hasher.finish();
+        let mut fp = v1_signee_hasher.finish();
-
+        fp[32..].fill(0);
-        let mut to_sign = [0u64; 9];
+        if let Some(signed_by) = signed_by {
-        self.v1_proto_write_first_3_qualifiers(&mut to_sign);
+            fp[32..38].copy_from_slice(&signed_by.to_bytes());
-        if let Some(signature) = issuer.sign(memory::as_byte_array::<[u64; 9], 27>(&to_sign), true) {
-            self.issued_to = issued_to.address;
-            self.v1 = Some(v1_signee_hash[..32].try_into().unwrap());
-            self.signed_by = issuer.address;
-            self.signature = signature;
-            true
-        } else {
-            false
         }
+        fp
     }
 
-    /// Get this certificate of membership encoded in the format expected by old V1 nodes.
+    /// Get this certificate of membership in byte encoded format.
-    pub fn v1_proto_to_bytes(&self) -> Option<Vec<u8>> {
+    pub fn to_bytes(&self) -> Option<Vec<u8>> {
-        if self.signature.is_empty() || self.v1.is_none() {
+        if self.version == 1 {
-            return None;
+            if self.signature.len() == 96 {
-        }
+                let mut v: Vec<u8> = Vec::with_capacity(3 + 168 + 5 + 96);
-        let mut v: Vec<u8> = Vec::with_capacity(384);
+                v.push(1); // version byte from v1 protocol
-        v.push(1);
                 v.push(0);
                 v.push(7); // 7 qualifiers, big-endian 16-bit
-        let _ = v.write_all(&self.v1_proto_get_qualifier_bytes().unwrap());
+                let _ = v.write_all(&self.v1_proto_get_qualifier_bytes());
-        let _ = v.write_all(&self.signed_by.to_bytes());
+                let _ = v.write_all(&self.issued_to_fingerprint.as_bytes()[32..38]); // issuer address
                 let _ = v.write_all(self.signature.as_bytes());
                 return Some(v);
             }
+        } else if self.version == 2 {
+            todo!()
+        }
+        return None;
+    }
+
+    /// Decode a V1 legacy format certificate of membership in byte format.
+    pub fn v1_proto_from_bytes(mut b: &[u8]) -> Result<Self, InvalidParameterError> {
+        if b.len() <= 3 || b[0] != 1 {
+            return Err(InvalidParameterError("version mismatch"));
+        }
+        let qualifier_count = (b[1] as usize).wrapping_shl(8) | (b[2] as usize);
+        b = &b[3..];
+
+        if b.len() < ((qualifier_count * 24) + (5 + 96)) {
+            return Err(InvalidParameterError("incomplete"));
+        }
+
+        let (mut network_id, mut issued_to, mut timestamp, mut max_delta, mut v1_fingerprint) = (0, 0, 0, 0, [0u8; 48]);
+        for _ in 0..qualifier_count {
+            let qt = u64::from_be_bytes(b[..8].try_into().unwrap());
+            let q: [u8; 8] = b[8..16].try_into().unwrap();
+            let qd = u64::from_be_bytes(b[16..24].try_into().unwrap());
+            match qt {
+                0 => {
+                    timestamp = i64::from_be_bytes(q);
+                    max_delta = qd as i64;
+                }
+                1 => {
+                    network_id = u64::from_be_bytes(q);
+                }
+                2 => {
+                    issued_to = u64::from_be_bytes(q);
+                }
+                3 => {
+                    v1_fingerprint[0..8].copy_from_slice(&q);
+                }
+                4 => {
+                    v1_fingerprint[8..16].copy_from_slice(&q);
+                }
+                5 => {
+                    v1_fingerprint[16..24].copy_from_slice(&q);
+                }
+                6 => {
+                    v1_fingerprint[24..32].copy_from_slice(&q);
+                }
+                _ => {}
+            }
+            b = &b[24..];
+        }
+
+        v1_fingerprint[32..38].copy_from_slice(&b[..5]); // issuer address
+        b = &b[5..];
+
+        Ok(Self {
+            network_id: NetworkId::from_u64(network_id).ok_or(InvalidParameterError("invalid network ID"))?,
+            timestamp,
+            max_delta,
+            issued_to: Address::from_u64(issued_to).ok_or(InvalidParameterError("invalid issued to address"))?,
+            issued_to_fingerprint: Blob::from(v1_fingerprint),
+            signature: {
+                let mut s = ArrayVec::new();
+                s.push_slice(&b[..96]);
+                s
+            },
+            version: 1,
+        })
+    }
+
+    /// Verify this certificate of membership.
+    pub fn verify(self, issuer: &Identity, expect_issued_to: &Identity) -> Option<Verified<Self>> {
+        if self.version == 1 {
+            if Self::v1_proto_issued_to_fingerprint(expect_issued_to, None).eq(&self.issued_to_fingerprint.as_bytes()[..32]) {
+                if issuer.verify(&self.v1_proto_get_qualifier_bytes(), self.signature.as_bytes()) {
+                    return Some(Verified(self));
+                }
+            }
+        } else if self.version == 2 {
+            todo!()
+        }
+        return None;
+    }
 }

network-hypervisor/src/vl2/certificateofownership.rs

@@ -1,3 +1,4 @@
+use std::collections::HashSet;
 use std::io::Write;
 
 use crate::vl1::{Address, Identity, InetAddress, MAC};
@@ -6,27 +7,71 @@ use crate::vl2::NetworkId;
 use serde::{Deserialize, Serialize};
 
 use zerotier_utils::arrayvec::ArrayVec;
+use zerotier_utils::blob::Blob;
+use zerotier_utils::error::InvalidParameterError;
 
-#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
 pub enum Thing {
-    Ip(InetAddress),
+    Ipv4([u8; 4]),
+    Ipv6([u8; 16]),
     Mac(MAC),
 }
 
+impl Thing {
+    /// Get the type ID for this "thing."
+    pub fn type_id(&self) -> u8 {
+        match self {
+            Self::Mac(_) => 1,
+            Self::Ipv4(_) => 2,
+            Self::Ipv6(_) => 3,
+        }
+    }
+}
+
 #[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct CertificateOfOwnership {
     pub network_id: NetworkId,
     pub timestamp: i64,
-    pub flags: u64,
+    pub things: HashSet<Thing>,
-    pub id: u32,
-    pub things: Vec<Thing>,
     pub issued_to: Address,
-    pub signed_by: Address,
+    pub issued_to_fingerprint: Blob<{ Identity::FINGERPRINT_SIZE }>,
     pub signature: ArrayVec<u8, { crate::vl1::identity::IDENTITY_MAX_SIGNATURE_SIZE }>,
+    pub version: u8,
 }
 
 impl CertificateOfOwnership {
-    fn internal_v1_proto_to_bytes(&self, for_sign: bool) -> Option<Vec<u8>> {
+    /// Create a new empty and unsigned certificate.
+    pub fn new(network_id: NetworkId, timestamp: i64, issued_to: Address, legacy_v1: bool) -> Self {
+        Self {
+            network_id,
+            timestamp,
+            things: HashSet::with_capacity(4),
+            issued_to,
+            issued_to_fingerprint: Blob::default(),
+            signature: ArrayVec::new(),
+            version: if legacy_v1 {
+                1
+            } else {
+                2
+            },
+        }
+    }
+
+    /// Add an IP address to this certificate.
+    pub fn add_ip(&mut self, ip: &InetAddress) {
+        if ip.is_ipv4() {
+            let _ = self.things.insert(Thing::Ipv4(ip.ip_bytes().try_into().unwrap()));
+        } else if ip.is_ipv6() {
+            let _ = self.things.insert(Thing::Ipv6(ip.ip_bytes().try_into().unwrap()));
+        }
+    }
+
+    /// Add a MAC address to this certificate.
+    pub fn add_mac(&mut self, mac: MAC) {
+        let _ = self.things.insert(Thing::Mac(mac));
+    }
+
+    fn internal_v1_proto_to_bytes(&self, for_sign: bool, signed_by: Address) -> Option<Vec<u8>> {
         if self.things.len() > 0xffff || self.signature.len() != 96 {
             return None;
         }
@@ -36,23 +81,18 @@ impl CertificateOfOwnership {
         }
         let _ = v.write_all(&self.network_id.to_bytes());
         let _ = v.write_all(&self.timestamp.to_be_bytes());
-        let _ = v.write_all(&self.flags.to_be_bytes());
+        let _ = v.write_all(&[0u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]); // obsolete flags and ID fields
-        let _ = v.write_all(&self.id.to_be_bytes());
         let _ = v.write_all(&(self.things.len() as u16).to_be_bytes());
         for t in self.things.iter() {
             match t {
-                Thing::Ip(ip) => {
+                Thing::Ipv4(ip) => {
-                    if ip.is_ipv4() {
                     v.push(2);
-                        let mut tmp = [0u8; 16];
+                    let _ = v.write_all(ip);
-                        tmp[..4].copy_from_slice(&ip.ip_bytes());
+                    let _ = v.write_all(&[0u8; 12]);
-                        let _ = v.write_all(&tmp);
-                    } else if ip.is_ipv6() {
-                        v.push(3);
-                        let _ = v.write_all(ip.ip_bytes());
-                    } else {
-                        return None;
                 }
+                Thing::Ipv6(ip) => {
+                    v.push(3);
+                    let _ = v.write_all(ip);
                 }
                 Thing::Mac(m) => {
                     v.push(1);
@@ -63,7 +103,7 @@ impl CertificateOfOwnership {
             }
         }
         let _ = v.write_all(&self.issued_to.to_bytes());
-        let _ = v.write_all(&self.signed_by.to_bytes());
+        let _ = v.write_all(&signed_by.to_bytes());
         if for_sign {
             v.push(0);
             v.push(0);
@@ -80,19 +120,76 @@ impl CertificateOfOwnership {
     }
 
     #[inline(always)]
-    pub fn v1_proto_to_bytes(&self) -> Option<Vec<u8>> {
+    pub fn v1_proto_to_bytes(&self, signed_by: Address) -> Option<Vec<u8>> {
-        self.internal_v1_proto_to_bytes(false)
+        self.internal_v1_proto_to_bytes(false, signed_by)
     }
 
-    pub fn v1_proto_sign(&mut self, issuer: &Identity, issued_to: &Identity) -> bool {
+    /// Decode a V1 legacy format certificate of ownership in byte format.
+    /// The certificate and the current position slice are returned so multiple certs can be easily read from a buffer.
+    pub fn v1_proto_from_bytes(mut b: &[u8]) -> Result<(Self, &[u8]), InvalidParameterError> {
+        if b.len() < 30 {
+            return Err(InvalidParameterError("incomplete"));
+        }
+        let network_id = u64::from_be_bytes(b[0..8].try_into().unwrap());
+        let timestamp = i64::from_be_bytes(b[8..16].try_into().unwrap());
+        let thing_count = u16::from_be_bytes(b[28..30].try_into().unwrap());
+        let mut things: HashSet<Thing> = HashSet::with_capacity(thing_count as usize);
+        b = &b[30..];
+        for _ in 0..thing_count {
+            if b.len() < 17 {
+                return Err(InvalidParameterError("incomplete"));
+            }
+            match b[0] {
+                1 => {
+                    let _ = things.insert(Thing::Mac(MAC::from_bytes(&b[1..7]).ok_or(InvalidParameterError("invalid MAC"))?));
+                }
+                2 => {
+                    let _ = things.insert(Thing::Ipv4(b[1..5].try_into().unwrap()));
+                }
+                3 => {
+                    let _ = things.insert(Thing::Ipv6(b[1..17].try_into().unwrap()));
+                }
+                _ => {
+                    return Err(InvalidParameterError("unknown thing type"));
+                }
+            }
+            b = &b[17..];
+        }
+        const END_LEN: usize = 5 + 5 + 3 + 96 + 2;
+        if b.len() < END_LEN {
+            return Err(InvalidParameterError("incomplete"));
+        }
+        Ok((
+            Self {
+                network_id: NetworkId::from_u64(network_id).ok_or(InvalidParameterError("invalid network ID"))?,
+                timestamp,
+                things,
+                issued_to: Address::from_bytes(&b[..5]).ok_or(InvalidParameterError("invalid address"))?,
+                issued_to_fingerprint: Blob::default(),
+                signature: {
+                    let mut s = ArrayVec::new();
+                    s.push_slice(&b[13..109]);
+                    s
+                },
+                version: 1,
+            },
+            &b[END_LEN..],
+        ))
+    }
+
+    /// Sign certificate of ownership for use by V1 nodes.
+    pub fn sign(&mut self, issuer: &Identity, issued_to: &Identity) -> bool {
+        if self.version == 1 {
             self.issued_to = issued_to.address;
-        self.signed_by = issuer.address;
+            if let Some(to_sign) = self.internal_v1_proto_to_bytes(true, issuer.address) {
-        if let Some(to_sign) = self.internal_v1_proto_to_bytes(true) {
                 if let Some(signature) = issuer.sign(&to_sign.as_slice(), true) {
                     self.signature = signature;
                     return true;
                 }
             }
+        } else if self.version == 2 {
+            todo!()
+        }
         return false;
     }
 }

network-hypervisor/src/vl2/mod.rs

@@ -3,12 +3,13 @@
 mod certificateofmembership;
 mod certificateofownership;
 mod multicastgroup;
-mod networkconfig;
 mod networkid;
 mod rule;
 mod switch;
 mod tag;
 
+pub mod networkconfig;
+
 pub use certificateofmembership::CertificateOfMembership;
 pub use certificateofownership::CertificateOfOwnership;
 pub use multicastgroup::MulticastGroup;

network-hypervisor/src/vl2/multicastgroup.rs

@@ -25,7 +25,6 @@ impl From<MAC> for MulticastGroup {
 }
 
 impl Ord for MulticastGroup {
-    #[inline(always)]
     fn cmp(&self, other: &Self) -> Ordering {
         let o = self.mac.cmp(&other.mac);
         match o {

network-hypervisor/src/vl2/networkconfig.rs

@@ -1,19 +1,329 @@
 // (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md.
 
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::io::Write;
+use std::str::FromStr;
 
 use serde::{Deserialize, Serialize};
 
-use crate::vl1::{Address, InetAddress};
+use crate::vl1::{Address, Identity, InetAddress};
 use crate::vl2::certificateofmembership::CertificateOfMembership;
 use crate::vl2::certificateofownership::CertificateOfOwnership;
 use crate::vl2::rule::Rule;
 use crate::vl2::tag::Tag;
+use crate::vl2::NetworkId;
 
 use zerotier_utils::buffer::Buffer;
 use zerotier_utils::dictionary::Dictionary;
-use zerotier_utils::marshalable::{Marshalable, UnmarshalError};
+use zerotier_utils::error::InvalidParameterError;
+use zerotier_utils::marshalable::Marshalable;
+
+/// Network configuration object sent to nodes by network controllers.
+#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct NetworkConfig {
+    pub network_id: NetworkId,
+    pub issued_to: Address,
+
+    pub name: String,
+    pub motd: String,
+    pub private: bool,
+
+    pub timestamp: i64,
+    pub max_delta: i64,
+    pub revision: u64,
+
+    pub mtu: u16,
+    pub multicast_limit: u32,
+    pub routes: HashSet<IpRoute>,
+    pub static_ips: HashSet<InetAddress>,
+    pub rules: Vec<Rule>,
+    pub dns: HashMap<String, HashSet<InetAddress>>,
+
+    pub certificate_of_membership: Option<CertificateOfMembership>, // considered invalid if None
+    pub certificates_of_ownership: Vec<CertificateOfOwnership>,
+    pub tags: HashMap<u32, Tag>,
+
+    pub node_info: HashMap<Address, NodeInfo>,
+
+    pub central_url: String,
+    pub sso: Option<SSOAuthConfiguration>,
+}
+
+impl NetworkConfig {
+    pub fn new(network_id: NetworkId, issued_to: Address) -> Self {
+        Self {
+            network_id,
+            issued_to,
+            name: String::new(),
+            motd: String::new(),
+            private: true,
+            timestamp: 0,
+            max_delta: 0,
+            revision: 0,
+            mtu: 0,
+            multicast_limit: 0,
+            routes: HashSet::new(),
+            static_ips: HashSet::new(),
+            rules: Vec::new(),
+            dns: HashMap::new(),
+            certificate_of_membership: None,
+            certificates_of_ownership: Vec::new(),
+            tags: HashMap::new(),
+            node_info: HashMap::new(),
+            central_url: String::new(),
+            sso: None,
+        }
+    }
+
+    /// Encode a network configuration for sending to V1 nodes.
+    pub fn v1_proto_to_dictionary(&self, controller_identity: &Identity) -> Option<Dictionary> {
+        let mut d = Dictionary::new();
+
+        d.set_str(
+            proto_v1_field_name::network_config::NETWORK_ID,
+            self.network_id.to_string().as_str(),
+        );
+        if !self.name.is_empty() {
+            d.set_str(proto_v1_field_name::network_config::NAME, self.name.as_str());
+        }
+        d.set_str(proto_v1_field_name::network_config::ISSUED_TO, self.issued_to.to_string().as_str());
+        d.set_str(
+            proto_v1_field_name::network_config::TYPE,
+            if self.private {
+                "0"
+            } else {
+                "1"
+            },
+        );
+        d.set_u64(proto_v1_field_name::network_config::TIMESTAMP, self.timestamp as u64);
+        d.set_u64(proto_v1_field_name::network_config::MAX_DELTA, self.max_delta as u64);
+        d.set_u64(proto_v1_field_name::network_config::REVISION, self.revision);
+        d.set_u64(proto_v1_field_name::network_config::MTU, self.mtu as u64);
+        d.set_u64(proto_v1_field_name::network_config::MULTICAST_LIMIT, self.multicast_limit as u64);
+
+        if !self.routes.is_empty() {
+            let r: Vec<IpRoute> = self.routes.iter().cloned().collect();
+            d.set_bytes(
+                proto_v1_field_name::network_config::ROUTES,
+                IpRoute::marshal_multiple_to_bytes(r.as_slice()).unwrap(),
+            );
+        }
+
+        if !self.static_ips.is_empty() {
+            let ips: Vec<InetAddress> = self.static_ips.iter().cloned().collect();
+            d.set_bytes(
+                proto_v1_field_name::network_config::STATIC_IPS,
+                InetAddress::marshal_multiple_to_bytes(ips.as_slice()).unwrap(),
+            );
+        }
+
+        if !self.rules.is_empty() {
+            d.set_bytes(
+                proto_v1_field_name::network_config::RULES,
+                Rule::marshal_multiple_to_bytes(self.rules.as_slice()).unwrap(),
+            );
+        }
+
+        if !self.dns.is_empty() {
+            // NOTE: v1 nodes only support one DNS server per network! If there is more than
+            // one the first will be picked, whichever that is (it's a set). The UI should not
+            // allow a user to add more than one unless this is a v2-only network.
+            let mut dns_bin: Vec<u8> = Vec::with_capacity(256);
+            if let Some((name, servers)) = self.dns.iter().next() {
+                let mut name_bytes = name.as_bytes();
+                name_bytes = &name_bytes[..name_bytes.len().min(127)];
+                let _ = dns_bin.write_all(name_bytes);
+                for _ in 0..(128 - name_bytes.len()) {
+                    dns_bin.push(0);
+                }
+                for s in servers.iter() {
+                    if let Ok(s) = s.to_buffer::<64>() {
+                        let _ = dns_bin.write_all(s.as_bytes());
+                    }
+                }
+            }
+            d.set_bytes(proto_v1_field_name::network_config::DNS, dns_bin);
+        }
+
+        d.set_bytes(
+            proto_v1_field_name::network_config::CERTIFICATE_OF_MEMBERSHIP,
+            self.certificate_of_membership.as_ref()?.to_bytes()?,
+        );
+
+        if !self.certificates_of_ownership.is_empty() {
+            let mut certs = Vec::with_capacity(self.certificates_of_ownership.len() * 256);
+            for c in self.certificates_of_ownership.iter() {
+                let _ = certs.write_all(c.v1_proto_to_bytes(controller_identity.address)?.as_slice());
+            }
+            d.set_bytes(proto_v1_field_name::network_config::CERTIFICATES_OF_OWNERSHIP, certs);
+        }
+
+        if !self.tags.is_empty() {
+            let mut certs = Vec::with_capacity(self.tags.len() * 256);
+            for (_, t) in self.tags.iter() {
+                let _ = certs.write_all(t.v1_proto_to_bytes(controller_identity.address)?.as_slice());
+            }
+            d.set_bytes(proto_v1_field_name::network_config::TAGS, certs);
+        }
+
+        // node_info is not supported by V1 nodes
+
+        if !self.central_url.is_empty() {
+            d.set_str(proto_v1_field_name::network_config::CENTRAL_URL, self.central_url.as_str());
+        }
+
+        if let Some(sso) = self.sso.as_ref() {
+            d.set_bool(proto_v1_field_name::network_config::SSO_ENABLED, true);
+            d.set_u64(proto_v1_field_name::network_config::SSO_VERSION, sso.version as u64);
+            d.set_str(
+                proto_v1_field_name::network_config::SSO_AUTHENTICATION_URL,
+                sso.authentication_url.as_str(),
+            );
+            d.set_u64(
+                proto_v1_field_name::network_config::SSO_AUTHENTICATION_EXPIRY_TIME,
+                sso.authentication_expiry_time as u64,
+            );
+            d.set_str(proto_v1_field_name::network_config::SSO_ISSUER_URL, sso.issuer_url.as_str());
+            d.set_str(proto_v1_field_name::network_config::SSO_NONCE, sso.nonce.as_str());
+            d.set_str(proto_v1_field_name::network_config::SSO_STATE, sso.state.as_str());
+            d.set_str(proto_v1_field_name::network_config::SSO_CLIENT_ID, sso.client_id.as_str());
+        } else {
+            d.set_bool(proto_v1_field_name::network_config::SSO_ENABLED, false);
+        }
+
+        Some(d)
+    }
+
+    /// Decode a V1 format network configuration.
+    pub fn v1_proto_from_dictionary(d: &Dictionary) -> Result<NetworkConfig, InvalidParameterError> {
+        let nwid = NetworkId::from_str(
+            d.get_str(proto_v1_field_name::network_config::NETWORK_ID)
+                .ok_or(InvalidParameterError("missing network ID"))?,
+        )
+        .map_err(|_| InvalidParameterError("invalid network ID"))?;
+        let issued_to_address = Address::from_str(
+            d.get_str(proto_v1_field_name::network_config::ISSUED_TO)
+                .ok_or(InvalidParameterError("missing address"))?,
+        )
+        .map_err(|_| InvalidParameterError("invalid address"))?;
+
+        let mut nc = Self::new(nwid, issued_to_address);
+
+        d.get_str(proto_v1_field_name::network_config::NAME)
+            .map(|x| nc.name = x.to_string());
+        nc.private = d.get_str(proto_v1_field_name::network_config::TYPE).map_or(true, |x| x == "1");
+        nc.timestamp = d
+            .get_i64(proto_v1_field_name::network_config::TIMESTAMP)
+            .ok_or(InvalidParameterError("missing timestamp"))?;
+        nc.max_delta = d.get_i64(proto_v1_field_name::network_config::MAX_DELTA).unwrap_or(0);
+        nc.revision = d.get_u64(proto_v1_field_name::network_config::REVISION).unwrap_or(0);
+        nc.mtu = d
+            .get_u64(proto_v1_field_name::network_config::MTU)
+            .unwrap_or(crate::protocol::ZEROTIER_VIRTUAL_NETWORK_DEFAULT_MTU as u64) as u16;
+        nc.multicast_limit = d.get_u64(proto_v1_field_name::network_config::MULTICAST_LIMIT).unwrap_or(0) as u32;
+
+        if let Some(routes_bin) = d.get_bytes(proto_v1_field_name::network_config::ROUTES) {
+            for r in IpRoute::unmarshal_multiple_from_bytes(routes_bin)
+                .map_err(|_| InvalidParameterError("invalid route object(s)"))?
+                .drain(..)
+            {
+                let _ = nc.routes.insert(r);
+            }
+        }
+
+        if let Some(static_ips_bin) = d.get_bytes(proto_v1_field_name::network_config::STATIC_IPS) {
+            for ip in InetAddress::unmarshal_multiple_from_bytes(static_ips_bin)
+                .map_err(|_| InvalidParameterError("invalid route object(s)"))?
+                .drain(..)
+            {
+                let _ = nc.static_ips.insert(ip);
+            }
+        }
+
+        if let Some(rules_bin) = d.get_bytes(proto_v1_field_name::network_config::RULES) {
+            nc.rules = Rule::unmarshal_multiple_from_bytes(rules_bin).map_err(|_| InvalidParameterError("invalid route object(s)"))?;
+        }
+
+        if let Some(dns_bin) = d.get_bytes(proto_v1_field_name::network_config::DNS) {
+            if dns_bin.len() > 128 && dns_bin.len() < 1024 {
+                let mut name = String::with_capacity(64);
+                for i in 0..128 {
+                    if dns_bin[i] == 0 {
+                        break;
+                    } else {
+                        name.push(dns_bin[i] as char);
+                    }
+                }
+                if !name.is_empty() {
+                    let mut tmp: Buffer<1024> = Buffer::new();
+                    let _ = tmp.append_bytes(&dns_bin[128..]);
+                    let mut servers = HashSet::new();
+                    let mut cursor = 0;
+                    while cursor < tmp.len() {
+                        if let Ok(s) = InetAddress::unmarshal(&tmp, &mut cursor) {
+                            let _ = servers.insert(s);
+                        } else {
+                            break;
+                        }
+                    }
+                    if !servers.is_empty() {
+                        let _ = nc.dns.insert(name, servers);
+                    }
+                }
+            }
+        }
+
+        nc.certificate_of_membership = Some(CertificateOfMembership::v1_proto_from_bytes(
+            d.get_bytes(proto_v1_field_name::network_config::CERTIFICATE_OF_MEMBERSHIP)
+                .ok_or(InvalidParameterError("missing certificate of membership"))?,
+        )?);
+
+        if let Some(mut coo_bin) = d.get_bytes(proto_v1_field_name::network_config::CERTIFICATES_OF_OWNERSHIP) {
+            while !coo_bin.is_empty() {
+                let c = CertificateOfOwnership::v1_proto_from_bytes(coo_bin)?;
+                nc.certificates_of_ownership.push(c.0);
+                coo_bin = c.1;
+            }
+        }
+
+        if let Some(mut tag_bin) = d.get_bytes(proto_v1_field_name::network_config::TAGS) {
+            while !tag_bin.is_empty() {
+                let t = Tag::v1_proto_from_bytes(tag_bin)?;
+                let _ = nc.tags.insert(t.0.id, t.0);
+                tag_bin = t.1;
+            }
+        }
+
+        if let Some(central_url) = d.get_str(proto_v1_field_name::network_config::CENTRAL_URL) {
+            nc.central_url = central_url.to_string();
+        }
+
+        if d.get_bool(proto_v1_field_name::network_config::SSO_ENABLED).unwrap_or(false) {
+            nc.sso = Some(SSOAuthConfiguration {
+                version: d.get_u64(proto_v1_field_name::network_config::SSO_VERSION).unwrap_or(0) as u32,
+                authentication_url: d
+                    .get_str(proto_v1_field_name::network_config::SSO_AUTHENTICATION_URL)
+                    .unwrap_or("")
+                    .to_string(),
+                authentication_expiry_time: d
+                    .get_i64(proto_v1_field_name::network_config::SSO_AUTHENTICATION_EXPIRY_TIME)
+                    .unwrap_or(0),
+                issuer_url: d
+                    .get_str(proto_v1_field_name::network_config::SSO_ISSUER_URL)
+                    .unwrap_or("")
+                    .to_string(),
+                nonce: d.get_str(proto_v1_field_name::network_config::SSO_NONCE).unwrap_or("").to_string(),
+                state: d.get_str(proto_v1_field_name::network_config::SSO_STATE).unwrap_or("").to_string(),
+                client_id: d
+                    .get_str(proto_v1_field_name::network_config::SSO_CLIENT_ID)
+                    .unwrap_or("")
+                    .to_string(),
+            })
+        }
+
+        Ok(nc)
+    }
+}
 
 #[allow(unused)]
 mod proto_v1_field_name {
@@ -27,7 +337,6 @@ mod proto_v1_field_name {
         pub const MULTICAST_LIMIT: &'static str = "ml";
         pub const TYPE: &'static str = "t";
         pub const NAME: &'static str = "n";
-        pub const MOTD: &'static str = "motd";
         pub const MTU: &'static str = "mtu";
         pub const MAX_DELTA: &'static str = "ctmd";
         pub const CERTIFICATE_OF_MEMBERSHIP: &'static str = "C";
@@ -72,129 +381,6 @@ pub struct SSOAuthConfiguration {
     pub client_id: String,
 }
 
-/// Network configuration object sent to nodes by network controllers.
-#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub struct NetworkConfig {
-    pub id: u64,
-    pub name: String,
-    pub motd: String,
-    pub issued_to: Address,
-    pub private: bool,
-
-    pub timestamp: i64,
-    pub max_delta: i64,
-    pub revision: u64,
-
-    pub mtu: u16,
-    pub multicast_limit: u32,
-    pub routes: Vec<IpRoute>,
-    pub static_ips: Vec<InetAddress>,
-    pub rules: Vec<Rule>,
-    pub dns: Vec<Nameserver>,
-
-    pub certificate_of_membership: CertificateOfMembership,
-    pub certificates_of_ownership: Vec<CertificateOfOwnership>,
-    pub tags: Vec<Tag>,
-
-    pub node_info: HashMap<Address, NodeInfo>,
-
-    pub central_url: String,
-
-    pub sso: Option<SSOAuthConfiguration>,
-}
-
-impl NetworkConfig {
-    pub fn v1_proto_to_dictionary(&self) -> Option<Dictionary> {
-        let mut d = Dictionary::new();
-        d.set_u64(proto_v1_field_name::network_config::NETWORK_ID, self.id);
-        if !self.name.is_empty() {
-            d.set_str(proto_v1_field_name::network_config::NAME, self.name.as_str());
-        }
-        if !self.motd.is_empty() {
-            d.set_str(proto_v1_field_name::network_config::MOTD, self.motd.as_str());
-        }
-        d.set_str(proto_v1_field_name::network_config::ISSUED_TO, self.issued_to.to_string().as_str());
-        d.set_str(
-            proto_v1_field_name::network_config::TYPE,
-            if self.private {
-                "0"
-            } else {
-                "1"
-            },
-        );
-        d.set_u64(proto_v1_field_name::network_config::TIMESTAMP, self.timestamp as u64);
-        d.set_u64(proto_v1_field_name::network_config::MAX_DELTA, self.max_delta as u64);
-        d.set_u64(proto_v1_field_name::network_config::REVISION, self.revision);
-        d.set_u64(proto_v1_field_name::network_config::MTU, self.mtu as u64);
-        d.set_u64(proto_v1_field_name::network_config::MULTICAST_LIMIT, self.multicast_limit as u64);
-        if !self.routes.is_empty() {
-            d.set_bytes(
-                proto_v1_field_name::network_config::ROUTES,
-                IpRoute::marshal_multiple_to_bytes(self.routes.as_slice()).unwrap(),
-            );
-        }
-        if !self.static_ips.is_empty() {
-            d.set_bytes(
-                proto_v1_field_name::network_config::STATIC_IPS,
-                InetAddress::marshal_multiple_to_bytes(self.static_ips.as_slice()).unwrap(),
-            );
-        }
-        if !self.rules.is_empty() {
-            d.set_bytes(
-                proto_v1_field_name::network_config::RULES,
-                Rule::marshal_multiple_to_bytes(self.rules.as_slice()).unwrap(),
-            );
-        }
-        if !self.dns.is_empty() {
-            d.set_bytes(
-                proto_v1_field_name::network_config::DNS,
-                Nameserver::marshal_multiple_to_bytes(self.dns.as_slice()).unwrap(),
-            );
-        }
-        d.set_bytes(
-            proto_v1_field_name::network_config::CERTIFICATE_OF_MEMBERSHIP,
-            self.certificate_of_membership.v1_proto_to_bytes()?,
-        );
-        if !self.certificates_of_ownership.is_empty() {
-            let mut certs = Vec::with_capacity(self.certificates_of_ownership.len() * 256);
-            for c in self.certificates_of_ownership.iter() {
-                let _ = certs.write_all(c.v1_proto_to_bytes()?.as_slice());
-            }
-            d.set_bytes(proto_v1_field_name::network_config::CERTIFICATES_OF_OWNERSHIP, certs);
-        }
-        if !self.tags.is_empty() {
-            let mut certs = Vec::with_capacity(self.certificates_of_ownership.len() * 256);
-            for t in self.tags.iter() {
-                let _ = certs.write_all(t.v1_proto_to_bytes()?.as_slice());
-            }
-            d.set_bytes(proto_v1_field_name::network_config::TAGS, certs);
-        }
-        // node_info is not supported by V1 nodes
-        if !self.central_url.is_empty() {
-            d.set_str(proto_v1_field_name::network_config::CENTRAL_URL, self.central_url.as_str());
-        }
-        if let Some(sso) = self.sso.as_ref() {
-            d.set_bool(proto_v1_field_name::network_config::SSO_ENABLED, true);
-            d.set_u64(proto_v1_field_name::network_config::SSO_VERSION, sso.version as u64);
-            d.set_str(
-                proto_v1_field_name::network_config::SSO_AUTHENTICATION_URL,
-                sso.authentication_url.as_str(),
-            );
-            d.set_u64(
-                proto_v1_field_name::network_config::SSO_AUTHENTICATION_EXPIRY_TIME,
-                sso.authentication_expiry_time as u64,
-            );
-            d.set_str(proto_v1_field_name::network_config::SSO_ISSUER_URL, sso.issuer_url.as_str());
-            d.set_str(proto_v1_field_name::network_config::SSO_NONCE, sso.nonce.as_str());
-            d.set_str(proto_v1_field_name::network_config::SSO_STATE, sso.state.as_str());
-            d.set_str(proto_v1_field_name::network_config::SSO_CLIENT_ID, sso.client_id.as_str());
-        } else {
-            d.set_bool(proto_v1_field_name::network_config::SSO_ENABLED, false);
-        }
-        Some(d)
-    }
-}
-
 /// Information about nodes on the network that can be included in a network config.
 #[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct NodeInfo {
@@ -205,7 +391,7 @@ pub struct NodeInfo {
 }
 
 /// Statically pushed L3 IP routes included with a network configuration.
-#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
 pub struct IpRoute {
     pub target: InetAddress,
     pub via: Option<InetAddress>,
@@ -250,42 +436,3 @@ impl Marshalable for IpRoute {
         })
     }
 }
-
-/// ZeroTier-pushed DNS nameserver configuration.
-#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub struct Nameserver {
-    pub ip: Vec<InetAddress>,
-    pub domain: String,
-}
-
-impl Marshalable for Nameserver {
-    const MAX_MARSHAL_SIZE: usize = 128 + InetAddress::MAX_MARSHAL_SIZE;
-
-    fn marshal<const BL: usize>(&self, buf: &mut Buffer<BL>) -> Result<(), UnmarshalError> {
-        let domain_bytes = self.domain.as_bytes();
-        let domain_bytes_len = domain_bytes.len().min(127);
-        let mut domain_bytes_pad128 = [0_u8; 128];
-        domain_bytes_pad128[..domain_bytes_len].copy_from_slice(&domain_bytes[..domain_bytes_len]);
-        buf.append_bytes_fixed(&domain_bytes_pad128)?;
-        buf.append_bytes(InetAddress::marshal_multiple_to_bytes(self.ip.as_slice()).unwrap().as_slice())?;
-        Ok(())
-    }
-
-    fn unmarshal<const BL: usize>(buf: &Buffer<BL>, cursor: &mut usize) -> Result<Self, UnmarshalError> {
-        let domain_bytes_pad128: &[u8; 128] = buf.read_bytes_fixed(cursor)?;
-        let mut domain_bytes_len = 0;
-        for i in 0..128 {
-            if domain_bytes_pad128[i] == 0 {
-                domain_bytes_len = i;
-                break;
-            }
-        }
-        if domain_bytes_len == 0 {
-            return Err(UnmarshalError::InvalidData);
-        }
-        Ok(Nameserver {
-            ip: InetAddress::unmarshal_multiple(buf, cursor, buf.len())?,
-            domain: String::from_utf8_lossy(&domain_bytes_pad128[..domain_bytes_len]).to_string(),
-        })
-    }
-}

network-hypervisor/src/vl2/rule.rs

@@ -740,7 +740,7 @@ struct HumanReadableRule<'a> {
 
 impl<'a> HumanReadableRule<'a> {
     fn to_rule(&self) -> Option<Rule> {
-        if let Some(t) = HR_NAME_TO_RULE_TYPE.get(self._type) {
+        if let Some(t) = HR_NAME_TO_RULE_TYPE.get(self._type.to_uppercase().as_str()) {
             let mut r = Rule::default();
             r.t =
                 *t | if self.not.unwrap_or(false) {

network-hypervisor/src/vl2/switch.rs

@@ -3,7 +3,7 @@
 use std::sync::Arc;
 
 use crate::protocol::PacketBuffer;
-use crate::vl1::node::{HostSystem, InnerProtocol, PacketHandlerResult};
+use crate::vl1::node::{HostSystem, InnerProtocol, Node, PacketHandlerResult};
 use crate::vl1::{Identity, Path, Peer};
 
 pub trait SwitchInterface: Sync + Send {}
@@ -13,8 +13,10 @@ pub struct Switch {}
 impl InnerProtocol for Switch {
     fn handle_packet<HostSystemImpl: HostSystem>(
         &self,
+        node: &Node<HostSystemImpl>,
         source: &Arc<Peer<HostSystemImpl>>,
         source_path: &Arc<Path<HostSystemImpl>>,
+        message_id: u64,
         verb: u8,
         payload: &PacketBuffer,
     ) -> PacketHandlerResult {
@@ -23,8 +25,10 @@ impl InnerProtocol for Switch {
 
     fn handle_error<HostSystemImpl: HostSystem>(
         &self,
+        node: &Node<HostSystemImpl>,
         source: &Arc<Peer<HostSystemImpl>>,
         source_path: &Arc<Path<HostSystemImpl>>,
+        message_id: u64,
         in_re_verb: u8,
         in_re_message_id: u64,
         error_code: u8,
@@ -36,8 +40,10 @@ impl InnerProtocol for Switch {
 
     fn handle_ok<HostSystemImpl: HostSystem>(
         &self,
+        node: &Node<HostSystemImpl>,
         source: &Arc<Peer<HostSystemImpl>>,
         source_path: &Arc<Path<HostSystemImpl>>,
+        message_id: u64,
         in_re_verb: u8,
         in_re_message_id: u64,
         payload: &PacketBuffer,

network-hypervisor/src/vl2/tag.rs

@@ -8,6 +8,8 @@ use crate::vl2::NetworkId;
 use serde::{Deserialize, Serialize};
 
 use zerotier_utils::arrayvec::ArrayVec;
+use zerotier_utils::blob::Blob;
+use zerotier_utils::error::InvalidParameterError;
 
 #[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct Tag {
@@ -16,13 +18,49 @@ pub struct Tag {
     pub network_id: NetworkId,
     pub timestamp: i64,
     pub issued_to: Address,
-    pub signed_by: Address,
+    pub issued_to_fingerprint: Blob<{ Identity::FINGERPRINT_SIZE }>,
     pub signature: ArrayVec<u8, { identity::IDENTITY_MAX_SIGNATURE_SIZE }>,
+    pub version: u8,
 }
 
 impl Tag {
-    fn internal_v1_proto_to_bytes(&self, for_sign: bool) -> Option<Vec<u8>> {
+    pub fn new(
-        if self.signature.len() == 96 {
+        id: u32,
+        value: u32,
+        issuer: &Identity,
+        network_id: NetworkId,
+        issued_to: &Identity,
+        timestamp: i64,
+        legacy_v1: bool,
+    ) -> Option<Self> {
+        let mut tag = Self {
+            id,
+            value,
+            network_id,
+            timestamp,
+            issued_to: issued_to.address,
+            issued_to_fingerprint: Blob::from(issued_to.fingerprint),
+            signature: ArrayVec::new(),
+            version: if legacy_v1 {
+                1
+            } else {
+                2
+            },
+        };
+        if legacy_v1 {
+            let to_sign = tag.internal_v1_proto_to_bytes(true, issuer.address)?;
+            if let Some(signature) = issuer.sign(to_sign.as_slice(), true) {
+                tag.signature = signature;
+                return Some(tag);
+            }
+        } else {
+            todo!()
+        }
+        return None;
+    }
+
+    fn internal_v1_proto_to_bytes(&self, for_sign: bool, signed_by: Address) -> Option<Vec<u8>> {
+        if self.version == 1 && self.signature.len() == 96 {
             let mut v = Vec::with_capacity(256);
             if for_sign {
                 let _ = v.write_all(&[0x7f; 8]);
@@ -32,7 +70,7 @@ impl Tag {
             let _ = v.write_all(&self.id.to_be_bytes());
             let _ = v.write_all(&self.value.to_be_bytes());
             let _ = v.write_all(&self.issued_to.to_bytes());
-            let _ = v.write_all(&self.signed_by.to_bytes());
+            let _ = v.write_all(&signed_by.to_bytes());
             if !for_sign {
                 v.push(1);
                 v.push(0);
@@ -50,14 +88,13 @@ impl Tag {
     }
 
     #[inline(always)]
-    pub fn v1_proto_to_bytes(&self) -> Option<Vec<u8>> {
+    pub fn v1_proto_to_bytes(&self, signed_by: Address) -> Option<Vec<u8>> {
-        self.internal_v1_proto_to_bytes(false)
+        self.internal_v1_proto_to_bytes(false, signed_by)
     }
 
     pub fn v1_proto_sign(&mut self, issuer: &Identity, issued_to: &Identity) -> bool {
         self.issued_to = issued_to.address;
-        self.signed_by = issuer.address;
+        if let Some(to_sign) = self.internal_v1_proto_to_bytes(true, issuer.address) {
-        if let Some(to_sign) = self.internal_v1_proto_to_bytes(true) {
             if let Some(signature) = issuer.sign(&to_sign.as_slice(), true) {
                 self.signature = signature;
                 return true;
@@ -65,4 +102,28 @@ impl Tag {
         }
         return false;
     }
+
+    pub fn v1_proto_from_bytes(b: &[u8]) -> Result<(Self, &[u8]), InvalidParameterError> {
+        const LEN: usize = 8 + 8 + 4 + 4 + 5 + 5 + 3 + 96 + 2;
+        if b.len() < LEN {
+            return Err(InvalidParameterError("incomplete"));
+        }
+        Ok((
+            Self {
+                id: u32::from_be_bytes(b[16..20].try_into().unwrap()),
+                value: u32::from_be_bytes(b[20..24].try_into().unwrap()),
+                network_id: NetworkId::from_bytes(&b[0..8]).ok_or(InvalidParameterError("invalid network ID"))?,
+                timestamp: i64::from_be_bytes(b[8..16].try_into().unwrap()),
+                issued_to: Address::from_bytes(&b[24..29]).ok_or(InvalidParameterError("invalid address"))?,
+                issued_to_fingerprint: Blob::default(),
+                signature: {
+                    let mut s = ArrayVec::new();
+                    s.push_slice(&b[37..133]);
+                    s
+                },
+                version: 1,
+            },
+            &b[LEN..],
+        ))
+    }
 }

utils/src/gatherarray.rs

@@ -64,7 +64,7 @@ impl<T, const C: usize> GatherArray<T, C> {
 }
 
 impl<T, const C: usize> Drop for GatherArray<T, C> {
-    #[inline(always)]
+    #[inline]
     fn drop(&mut self) {
         let have = self.have_bits;
         for i in 0..self.goal {