Now mostly fully parses OK(HELLO) and also HELLO

aes-gmac-siv/src/impl_macos.rs

@@ -327,3 +327,6 @@ impl AesGmacSiv {
         }
     }
 }
+
+unsafe impl Send for AesGmacSiv {}
+unsafe impl Send for AesCtr {}

aes-gmac-siv/src/impl_openssl.rs

@@ -245,3 +245,6 @@ impl AesGmacSiv {
         }
     }
 }
+
+unsafe impl Send for AesGmacSiv {}
+unsafe impl Send for AesCtr {}

zerotier-network-hypervisor/src/lib.rs

@@ -2,7 +2,7 @@
 
 pub const VERSION_MAJOR: u8 = 1;
 pub const VERSION_MINOR: u8 = 99;
-pub const VERSION_REVISION: u8 = 1;
+pub const VERSION_REVISION: u16 = 1;
 
 pub mod error;
 pub mod util;

zerotier-network-hypervisor/src/util/buffer.rs

@@ -112,7 +112,6 @@ impl<const L: usize> Buffer<L> {
         self.1.as_mut_ptr()
     }
 
-    /// Get all bytes after a given position.
     #[inline(always)]
     pub fn as_bytes_starting_at(&self, start: usize) -> std::io::Result<&[u8]> {
         if start <= self.0 {
@@ -122,6 +121,15 @@ impl<const L: usize> Buffer<L> {
         }
     }
 
+    #[inline(always)]
+    pub fn as_bytes_starting_at_mut(&mut self, start: usize) -> std::io::Result<&mut [u8]> {
+        if start <= self.0 {
+            Ok(&mut self.1[start..self.0])
+        } else {
+            Err(overflow_err())
+        }
+    }
+
     pub fn clear(&mut self) {
         self.1[0..self.0].fill(0);
         self.0 = 0;

zerotier-network-hypervisor/src/vl1/node.rs

@@ -94,8 +94,6 @@ pub trait InnerProtocolInterface: Sync + Send + 'static {
     /// Handle a packet, returning true if it was handled by the next layer.
     ///
     /// Do not attempt to handle OK or ERROR. Instead implement handle_ok() and handle_error().
-    /// The return values of these must follow the same semantic of returning true if the message
-    /// was handled.
     async fn handle_packet<SI: SystemInterface>(&self, source: &Peer<SI>, source_path: &Path<SI>, forward_secrecy: bool, extended_authentication: bool, verb: u8, payload: &PacketBuffer) -> bool;
 
     /// Handle errors, returning true if the error was recognized.

zerotier-network-hypervisor/src/vl1/peer.rs

@@ -103,7 +103,20 @@ fn try_aead_decrypt(secret: &SymmetricSecret, packet_frag0_payload_bytes: &[u8],
         security_constants::CIPHER_AES_GMAC_SIV => {
             let mut aes_gmac_siv = secret.aes_gmac_siv.get();
             aes_gmac_siv.decrypt_init(&packet_header.aes_gmac_siv_tag());
-            aes_gmac_siv.decrypt_set_aad(&packet_header.aad_bytes());
+
+            aes_gmac_siv.decrypt_set_aad(&[
+                packet_header.dest[0],
+                packet_header.dest[1],
+                packet_header.dest[2],
+                packet_header.dest[3],
+                packet_header.dest[4],
+                packet_header.src[0],
+                packet_header.src[1],
+                packet_header.src[2],
+                packet_header.src[3],
+                packet_header.src[4],
+                packet_header.flags_cipher_hops & packet_constants::FLAGS_FIELD_MASK_HIDE_HOPS,
+            ]);
 
             if let Ok(b) = payload.append_bytes_get_mut(packet_frag0_payload_bytes.len()) {
                 aes_gmac_siv.decrypt(packet_frag0_payload_bytes, b);
@@ -187,7 +200,7 @@ impl<SI: SystemInterface> Peer<SI> {
     pub fn version(&self) -> Option<[u16; 4]> {
         let rv = self.remote_version.load(Ordering::Relaxed);
         if rv != 0 {
-            Some([(rv >> 48) as u16, (rv >> 32) as u16, (rv >> 16) as u16, rv as u16])
+            Some([rv.wrapping_shr(48) as u16, rv.wrapping_shr(32) as u16, rv.wrapping_shr(16) as u16, rv as u16])
         } else {
             None
         }
@@ -334,14 +347,47 @@ impl<SI: SystemInterface> Peer<SI> {
         }
     }
 
-    /// Send a packet to this peer.
+    fn create_session_metadata(&self, node: &Node<SI>, destination: &Endpoint) -> Vec<u8> {
+        let mut session_metadata = Dictionary::new();
+        session_metadata.set_bytes(session_metadata::INSTANCE_ID, node.instance_id.to_vec());
+        session_metadata.set_bytes(session_metadata::CARE_OF, node.care_of_bytes());
+        session_metadata.set_bytes(session_metadata::SENT_TO, destination.to_buffer::<{ Endpoint::MAX_MARSHAL_SIZE }>().unwrap().as_bytes().to_vec());
+        if let Some(my_root_sets) = node.my_root_sets() {
+            session_metadata.set_bytes(session_metadata::MY_ROOT_SETS, my_root_sets);
+        }
+        session_metadata.to_bytes()
+    }
+
+    /// Send a packet to this peer, returning true on (potential) success.
     ///
     /// This will go directly if there is an active path, or otherwise indirectly
     /// via a root or some other route.
-    #[allow(unused)]
+    ///
-    pub(crate) async fn send(&self, si: &SI, node: &Node<SI>, time_ticks: i64, packet: &PacketBuffer) -> bool {
+    /// It encrypts and sets the MAC and cipher fields and packet ID and other things.
+    pub(crate) async fn send(&self, si: &SI, node: &Node<SI>, time_ticks: i64, message_id: MessageId, packet: &mut PacketBuffer) -> bool {
         if let Some(path) = self.path(node) {
-            if self.internal_send(si, &path.endpoint, Some(&path.local_socket), Some(&path.local_interface), if path.endpoint.requires_fragmentation() { UDP_DEFAULT_MTU } else { usize::MAX }, packet).await {
+            let max_fragment_size = if path.endpoint.requires_fragmentation() { UDP_DEFAULT_MTU } else { usize::MAX };
+            let flags_cipher_hops = if packet.len() > max_fragment_size { packet_constants::HEADER_FLAG_FRAGMENTED | security_constants::CIPHER_AES_GMAC_SIV } else { security_constants::CIPHER_AES_GMAC_SIV };
+
+            let mut aes_gmac_siv = if let Some(ephemeral_key) = self.ephemeral_symmetric_key.read().as_ref() { ephemeral_key.secret.aes_gmac_siv.get() } else { self.identity_symmetric_key.aes_gmac_siv.get() };
+            aes_gmac_siv.encrypt_init(&message_id.to_ne_bytes());
+            aes_gmac_siv.encrypt_set_aad(&get_packet_aad_bytes(self.identity.address, node.identity.address, flags_cipher_hops));
+            if let Ok(payload) = packet.as_bytes_starting_at_mut(packet_constants::HEADER_SIZE) {
+                aes_gmac_siv.encrypt_first_pass(payload);
+                aes_gmac_siv.encrypt_first_pass_finish();
+                aes_gmac_siv.encrypt_second_pass_in_place(payload);
+                let tag = aes_gmac_siv.encrypt_second_pass_finish();
+                let header = packet.struct_mut_at::<PacketHeader>(0).unwrap();
+                header.id = *byte_array_range::<16, 0, 8>(tag);
+                header.dest = self.identity.address.to_bytes();
+                header.src = node.identity.address.to_bytes();
+                header.flags_cipher_hops = flags_cipher_hops;
+                header.mac = *byte_array_range::<16, 8, 8>(tag);
+            } else {
+                return false;
+            }
+
+            if self.internal_send(si, &path.endpoint, Some(&path.local_socket), Some(&path.local_interface), max_fragment_size, packet).await {
                 self.last_send_time_ticks.store(time_ticks, Ordering::Relaxed);
                 return true;
             }
@@ -396,7 +442,7 @@ impl<SI: SystemInterface> Peer<SI> {
 
             {
                 let packet_header: &mut PacketHeader = packet.append_struct_get_mut().unwrap();
-                packet_header.id = message_id.to_ne_bytes(); // packet ID and message ID are the same when Poly1305 MAC is used
+                packet_header.id = message_id.to_ne_bytes();
                 packet_header.dest = self.identity.address.to_bytes();
                 packet_header.src = node.identity.address.to_bytes();
                 packet_header.flags_cipher_hops = security_constants::CIPHER_NOCRYPT_POLY1305;
@@ -408,28 +454,20 @@ impl<SI: SystemInterface> Peer<SI> {
                 hello_fixed_headers.version_proto = PROTOCOL_VERSION;
                 hello_fixed_headers.version_major = VERSION_MAJOR;
                 hello_fixed_headers.version_minor = VERSION_MINOR;
-                hello_fixed_headers.version_revision = (VERSION_REVISION as u16).to_be_bytes();
+                hello_fixed_headers.version_revision = VERSION_REVISION.to_be_bytes();
                 hello_fixed_headers.timestamp = (time_ticks as u64).wrapping_add(self.random_ticks_offset).to_be_bytes();
             }
 
-            assert_eq!(packet.len(), 41);
+            debug_assert_eq!(packet.len(), 41);
 
             // Full identity of this node.
             assert!(node.identity.marshal_with_options(&mut packet, Identity::ALGORITHM_ALL, false).is_ok());
 
-            // Create session meta-data.
+            // Create session meta-data and append length of this section.
-            let mut session_metadata = Dictionary::new();
+            let session_metadata = self.create_session_metadata(node, destination);
-            session_metadata.set_bytes(session_metadata::INSTANCE_ID, node.instance_id.to_vec());
+            let session_metadata_len = session_metadata.len() + 16; // plus nonce
-            session_metadata.set_bytes(session_metadata::CARE_OF, node.care_of_bytes());
+            assert!(session_metadata_len <= 0xffff); // sanity check, should be impossible
-            session_metadata.set_bytes(session_metadata::SENT_TO, destination.to_buffer::<{ Endpoint::MAX_MARSHAL_SIZE }>().unwrap().as_bytes().to_vec());
+            assert!(packet.append_u16(session_metadata_len as u16).is_ok());
-            if let Some(my_root_sets) = node.my_root_sets() {
-                session_metadata.set_bytes(session_metadata::MY_ROOT_SETS, my_root_sets);
-            }
-            let session_metadata = session_metadata.to_bytes();
-
-            // Prefix encrypted session metadata with its size (in cleartext).
-            assert!(session_metadata.len() <= 0xffff); // sanity check, should be impossible
-            assert!(packet.append_u16(session_metadata.len() as u16).is_ok());
 
             // Append a 16-byte AES-CTR nonce. LEGACY: for compatibility the last two bytes of this nonce
             // are in fact an encryption of two zeroes with Salsa20/12, which old nodes will interpret as
@@ -440,7 +478,7 @@ impl<SI: SystemInterface> Peer<SI> {
             Salsa::<12>::new(&self.identity_symmetric_key.key.0[0..32], &salsa_iv).crypt(&crate::util::ZEROES[..2], &mut nonce[14..]);
             assert!(packet.append_bytes_fixed(&nonce).is_ok());
 
-            // Write session meta-data in encrypted form.
+            // Write session meta-data, encrypted.
             nonce[12] &= 0x7f; // mask off the MSB of the 32-bit counter part of the CTR nonce for compatibility with AES libraries that don't wrap
             let salted_key = Secret(hmac_sha384(&message_id.to_ne_bytes(), self.identity_symmetric_key.hello_private_section_key.as_bytes()));
             let mut aes = AesCtr::new(&salted_key.as_bytes()[0..32]);
@@ -489,7 +527,7 @@ impl<SI: SystemInterface> Peer<SI> {
     /// those fragments after the main packet header and first chunk.
     ///
     /// This returns true if the packet decrypted and passed authentication.
-    pub(crate) async fn receive<PH: InnerProtocolInterface>(&self, node: &Node<SI>, si: &SI, ph: &PH, time_ticks: i64, source_path: &Arc<Path<SI>>, packet_header: &PacketHeader, frag0: &PacketBuffer, fragments: &[Option<PooledPacketBuffer>]) {
+    pub(crate) async fn receive<PH: InnerProtocolInterface>(&self, node: &Node<SI>, si: &SI, ph: &PH, time_ticks: i64, source_path: &Arc<Path<SI>>, packet_header: &PacketHeader, frag0: &PacketBuffer, fragments: &[Option<PooledPacketBuffer>]) -> bool {
         if let Ok(packet_frag0_payload_bytes) = frag0.as_bytes_starting_at(packet_constants::VERB_INDEX) {
             //let mut payload = unsafe { PacketBuffer::new_without_memzero() };
             let mut payload = PacketBuffer::new();
@@ -517,7 +555,7 @@ impl<SI: SystemInterface> Peer<SI> {
                 } else {
                     // Packet failed to decrypt using either ephemeral or permament key, reject.
                     debug_event!(si, "[vl1] #{:0>16x} failed authentication", u64::from_be_bytes(packet_header.id));
-                    return;
+                    return false;
                 }
             }
 
@@ -532,11 +570,11 @@ impl<SI: SystemInterface> Peer<SI> {
                         hmac.update(&message_id.to_ne_bytes());
                         hmac.update(&payload.as_bytes()[..actual_end_of_payload]);
                         if !hmac.finish().eq(&payload.as_bytes()[actual_end_of_payload..]) {
-                            return;
+                            return false;
                         }
                         payload.set_size(actual_end_of_payload);
                     } else {
-                        return;
+                        return false;
                     }
                 }
 
@@ -552,7 +590,7 @@ impl<SI: SystemInterface> Peer<SI> {
                     if let Ok(dlen) = lz4_flex::block::decompress_into(&payload.as_bytes()[1..], &mut decompressed_payload[1..]) {
                         payload.set_to(&decompressed_payload[..(dlen + 1)]);
                     } else {
-                        return;
+                        return false;
                     }
                 }
 
@@ -567,53 +605,82 @@ impl<SI: SystemInterface> Peer<SI> {
                     }
                 }
 
-                // For performance reasons we let VL2 handle packets first. It returns false
-                // if it didn't handle the packet, in which case it's handled at VL1. This is
-                // because the most performance critical path is the handling of the ???_FRAME
-                // verbs, which are in VL2.
                 verb &= packet_constants::VERB_MASK; // mask off flags
-                if !ph.handle_packet(self, &source_path, forward_secrecy, extended_authentication, verb, &payload).await {
+                return match verb {
-                    match verb {
+                    verbs::VL1_NOP => true,
-                        //VERB_VL1_NOP => {}
+                    verbs::VL1_HELLO => self.handle_incoming_hello(si, node, time_ticks, message_id, source_path, &payload).await,
-                        verbs::VL1_HELLO => self.handle_incoming_hello(si, node, time_ticks, source_path, &payload).await,
+                    verbs::VL1_ERROR => self.handle_incoming_error(si, ph, node, time_ticks, source_path, forward_secrecy, extended_authentication, &payload).await,
-                        verbs::VL1_ERROR => self.handle_incoming_error(si, ph, node, time_ticks, source_path, forward_secrecy, extended_authentication, &payload).await,
+                    verbs::VL1_OK => self.handle_incoming_ok(si, ph, node, time_ticks, source_path, packet_header.hops(), path_is_known, forward_secrecy, extended_authentication, &payload).await,
-                        verbs::VL1_OK => self.handle_incoming_ok(si, ph, node, time_ticks, source_path, packet_header.hops(), path_is_known, forward_secrecy, extended_authentication, &payload).await,
+                    verbs::VL1_WHOIS => self.handle_incoming_whois(si, node, time_ticks, source_path, &payload).await,
-                        verbs::VL1_WHOIS => self.handle_incoming_whois(si, node, time_ticks, source_path, &payload).await,
+                    verbs::VL1_RENDEZVOUS => self.handle_incoming_rendezvous(si, node, time_ticks, source_path, &payload).await,
-                        verbs::VL1_RENDEZVOUS => self.handle_incoming_rendezvous(si, node, time_ticks, source_path, &payload).await,
+                    verbs::VL1_ECHO => self.handle_incoming_echo(si, node, time_ticks, source_path, &payload).await,
-                        verbs::VL1_ECHO => self.handle_incoming_echo(si, node, time_ticks, source_path, &payload).await,
+                    verbs::VL1_PUSH_DIRECT_PATHS => self.handle_incoming_push_direct_paths(si, node, time_ticks, source_path, &payload).await,
-                        verbs::VL1_PUSH_DIRECT_PATHS => self.handle_incoming_push_direct_paths(si, node, time_ticks, source_path, &payload).await,
+                    verbs::VL1_USER_MESSAGE => self.handle_incoming_user_message(si, node, time_ticks, source_path, &payload).await,
-                        verbs::VL1_USER_MESSAGE => self.handle_incoming_user_message(si, node, time_ticks, source_path, &payload).await,
+                    _ => ph.handle_packet(self, &source_path, forward_secrecy, extended_authentication, verb, &payload).await,
-                        _ => {}
+                };
-                    }
-                }
             }
         }
+        return false;
+    }
+
+    async fn handle_incoming_hello(&self, si: &SI, node: &Node<SI>, time_ticks: i64, message_id: MessageId, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) -> bool {
+        let mut cursor = 0;
+        if let Ok(hello_fixed_headers) = payload.read_struct::<message_component_structs::HelloFixedHeaderFields>(&mut cursor) {
+            self.remote_protocol_version.store(hello_fixed_headers.version_proto, Ordering::Relaxed);
+            self.remote_version
+                .store((hello_fixed_headers.version_major as u64).wrapping_shl(48) | (hello_fixed_headers.version_minor as u64).wrapping_shl(32) | (u16::from_be_bytes(hello_fixed_headers.version_revision) as u64).wrapping_shl(16), Ordering::Relaxed);
+
+            let mut packet = PacketBuffer::new();
+
+            packet.set_size(packet_constants::HEADER_SIZE);
+            {
+                let fixed_fields: &mut message_component_structs::OkHeader = packet.append_struct_get_mut().unwrap();
+                fixed_fields.verb = verbs::VL1_OK;
+                fixed_fields.in_re_verb = verbs::VL1_HELLO;
+                fixed_fields.in_re_message_id = message_id.to_ne_bytes();
+            }
+            {
+                let fixed_fields: &mut message_component_structs::OkHelloFixedHeaderFields = packet.append_struct_get_mut().unwrap();
+                fixed_fields.timestamp_echo = hello_fixed_headers.timestamp;
+                fixed_fields.version_proto = PROTOCOL_VERSION;
+                fixed_fields.version_major = VERSION_MAJOR;
+                fixed_fields.version_minor = VERSION_MINOR;
+                fixed_fields.version_revision = VERSION_REVISION.to_be_bytes();
+            }
+
+            if hello_fixed_headers.version_proto >= 20 {
+                let session_metadata = self.create_session_metadata(node, &source_path.endpoint);
+                assert!(session_metadata.len() <= 0xffff); // sanity check, should be impossible
+                assert!(packet.append_u16(session_metadata.len() as u16).is_ok());
+                assert!(packet.append_bytes(session_metadata.as_slice()).is_ok());
+            }
+
+            return self.send(si, node, time_ticks, self.next_message_id(), &mut packet).await;
+        }
+        return false;
     }
 
     #[allow(unused)]
-    async fn handle_incoming_hello(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) {}
+    async fn handle_incoming_error<PH: InnerProtocolInterface>(&self, si: &SI, ph: &PH, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, forward_secrecy: bool, extended_authentication: bool, payload: &PacketBuffer) -> bool {
-
+        let mut cursor = 0;
-    #[allow(unused)]
-    async fn handle_incoming_error<PH: InnerProtocolInterface>(&self, si: &SI, ph: &PH, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, forward_secrecy: bool, extended_authentication: bool, payload: &PacketBuffer) {
-        let mut cursor: usize = 1;
         if let Ok(error_header) = payload.read_struct::<message_component_structs::ErrorHeader>(&mut cursor) {
             let in_re_message_id: MessageId = u64::from_ne_bytes(error_header.in_re_message_id);
             if self.message_id_counter.load(Ordering::Relaxed).wrapping_sub(in_re_message_id) <= PACKET_RESPONSE_COUNTER_DELTA_MAX {
                 match error_header.in_re_verb {
                     _ => {
-                        ph.handle_error(self, &source_path, forward_secrecy, extended_authentication, error_header.in_re_verb, in_re_message_id, error_header.error_code, payload, &mut cursor).await;
+                        return ph.handle_error(self, &source_path, forward_secrecy, extended_authentication, error_header.in_re_verb, in_re_message_id, error_header.error_code, payload, &mut cursor).await;
                     }
                 }
             }
         }
+        return false;
     }
 
-    #[allow(unused)]
     async fn handle_incoming_ok<PH: InnerProtocolInterface>(
         &self,
         si: &SI,
         ph: &PH,
-        node: &Node<SI>,
+        _node: &Node<SI>,
         time_ticks: i64,
         source_path: &Arc<Path<SI>>,
         hops: u8,
@@ -621,8 +688,8 @@ impl<SI: SystemInterface> Peer<SI> {
         forward_secrecy: bool,
         extended_authentication: bool,
         payload: &PacketBuffer,
-    ) {
+    ) -> bool {
-        let mut cursor: usize = 1;
+        let mut cursor = 0;
         if let Ok(ok_header) = payload.read_struct::<message_component_structs::OkHeader>(&mut cursor) {
             let in_re_message_id: MessageId = u64::from_ne_bytes(ok_header.in_re_message_id);
             if self.message_id_counter.load(Ordering::Relaxed).wrapping_sub(in_re_message_id) <= PACKET_RESPONSE_COUNTER_DELTA_MAX {
@@ -633,7 +700,7 @@ impl<SI: SystemInterface> Peer<SI> {
                                 if let Ok(session_metadata_len) = payload.read_u16(&mut cursor) {
                                     if session_metadata_len > 0 {
                                         if let Ok(session_metadata) = payload.read_bytes(session_metadata_len as usize, &mut cursor) {
-                                            if let Some(session_metadata) = Dictionary::from_bytes(session_metadata) {
+                                            if let Some(_session_metadata) = Dictionary::from_bytes(session_metadata) {
                                                 // TODO
                                             }
                                         }
@@ -654,27 +721,38 @@ impl<SI: SystemInterface> Peer<SI> {
                     verbs::VL1_WHOIS => {}
 
                     _ => {
-                        ph.handle_ok(self, &source_path, forward_secrecy, extended_authentication, ok_header.in_re_verb, in_re_message_id, payload, &mut cursor).await;
+                        return ph.handle_ok(self, &source_path, forward_secrecy, extended_authentication, ok_header.in_re_verb, in_re_message_id, payload, &mut cursor).await;
                     }
                 }
             }
         }
+        return false;
     }
 
     #[allow(unused)]
-    async fn handle_incoming_whois(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) {}
+    async fn handle_incoming_whois(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) -> bool {
+        false
+    }
 
     #[allow(unused)]
-    async fn handle_incoming_rendezvous(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) {}
+    async fn handle_incoming_rendezvous(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) -> bool {
+        false
+    }
 
     #[allow(unused)]
-    async fn handle_incoming_echo(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) {}
+    async fn handle_incoming_echo(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) -> bool {
+        false
+    }
 
     #[allow(unused)]
-    async fn handle_incoming_push_direct_paths(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) {}
+    async fn handle_incoming_push_direct_paths(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) -> bool {
+        false
+    }
 
     #[allow(unused)]
-    async fn handle_incoming_user_message(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) {}
+    async fn handle_incoming_user_message(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) -> bool {
+        false
+    }
 }
 
 impl<SI: SystemInterface> PartialEq for Peer<SI> {

zerotier-network-hypervisor/src/vl1/protocol.rs

@@ -46,6 +46,9 @@ use crate::vl1::Address;
  */
 pub const PROTOCOL_VERSION: u8 = 20;
 
+/// Minimum peer protocol version supported.
+pub const PROTOCOL_VERSION_MIN: u8 = 11;
+
 /// Buffer sized for ZeroTier packets.
 pub type PacketBuffer = Buffer<{ packet_constants::SIZE_MAX }>;
 
@@ -205,7 +208,7 @@ pub mod security_constants {
     pub const CIPHER_AES_GMAC_SIV: u8 = 0x18;
 
     /// KBKDF usage label indicating a key used to HMAC packets for extended authentication.
-    pub const KBKDF_KEY_USAGE_LABEL_PACKET_HMAC: u8 = b'M';
+    pub const KBKDF_KEY_USAGE_LABEL_PACKET_HMAC: u8 = b'm';
 
     /// KBKDF usage label for the first AES-GMAC-SIV key.
     pub const KBKDF_KEY_USAGE_LABEL_AES_GMAC_SIV_K0: u8 = b'0';
@@ -216,8 +219,8 @@ pub mod security_constants {
     /// KBKDF usage label for the private section of HELLOs.
     pub const KBKDF_KEY_USAGE_LABEL_HELLO_PRIVATE_SECTION: u8 = b'h';
 
-    /// KBKDF usage label for the key used to advance the ratchet.
+    /// KBKDF usage label for a unique ID for ephemeral keys (not actually a key).
-    pub const KBKDF_KEY_USAGE_LABEL_EPHEMERAL_RATCHET_KEY: u8 = b'e';
+    pub const KBKDF_KEY_USAGE_LABEL_EPHEMERAL_KEY_ID: u8 = b'e';
 
     /// Try to re-key ephemeral keys after this time.
     pub const EPHEMERAL_SECRET_REKEY_AFTER_TIME: i64 = 300000; // 5 minutes
@@ -365,15 +368,6 @@ impl PacketHeader {
         unsafe { &*(self as *const Self).cast::<[u8; packet_constants::HEADER_SIZE]>() }
     }
 
-    #[inline(always)]
-    pub fn aad_bytes(&self) -> [u8; 11] {
-        let mut id = unsafe { MaybeUninit::<[u8; 11]>::uninit().assume_init() };
-        id[0..5].copy_from_slice(&self.dest);
-        id[5..10].copy_from_slice(&self.src);
-        id[10] = self.flags_cipher_hops & packet_constants::FLAGS_FIELD_MASK_HIDE_HOPS;
-        id
-    }
-
     #[inline(always)]
     pub fn aes_gmac_siv_tag(&self) -> [u8; 16] {
         let mut id = unsafe { MaybeUninit::<[u8; 16]>::uninit().assume_init() };
@@ -383,6 +377,15 @@ impl PacketHeader {
     }
 }
 
+#[inline(always)]
+pub fn get_packet_aad_bytes(destination: Address, source: Address, flags_cipher_hops: u8) -> [u8; 11] {
+    let mut id = unsafe { MaybeUninit::<[u8; 11]>::uninit().assume_init() };
+    id[0..5].copy_from_slice(&destination.to_bytes());
+    id[5..10].copy_from_slice(&source.to_bytes());
+    id[10] = flags_cipher_hops & packet_constants::FLAGS_FIELD_MASK_HIDE_HOPS;
+    id
+}
+
 /// ZeroTier fragment header
 ///
 /// Fragments are indicated by byte 0xff at the start of the source address, which
@@ -439,10 +442,12 @@ impl FragmentHeader {
     }
 }
 
+/// Flat packed structs for fixed length header blocks in messages.
 pub(crate) mod message_component_structs {
     #[derive(Clone, Copy)]
     #[repr(C, packed)]
     pub struct OkHeader {
+        pub verb: u8,
         pub in_re_verb: u8,
         pub in_re_message_id: [u8; 8],
     }
@@ -450,6 +455,7 @@ pub(crate) mod message_component_structs {
     #[derive(Clone, Copy)]
     #[repr(C, packed)]
     pub struct ErrorHeader {
+        pub verb: u8,
         pub in_re_verb: u8,
         pub in_re_message_id: [u8; 8],
         pub error_code: u8,

zerotier-network-hypervisor/src/vl1/symmetricsecret.rs

@@ -42,12 +42,27 @@ impl SymmetricSecret {
 }
 
 /// An ephemeral symmetric secret with usage timers and counters.
+#[allow(unused)]
 pub(crate) struct EphemeralSymmetricSecret {
     pub secret: SymmetricSecret,
-    #[allow(unused)]
+    pub key_hash: [u8; 16],
+    pub create_time_ticks: i64,
     pub encrypt_uses: AtomicUsize,
 }
 
+impl EphemeralSymmetricSecret {
+    #[allow(unused)]
+    pub fn new(key: Secret<48>, create_time_ticks: i64) -> EphemeralSymmetricSecret {
+        let key_hash: [u8; 16] = zt_kbkdf_hmac_sha384(key.as_bytes(), security_constants::KBKDF_KEY_USAGE_LABEL_EPHEMERAL_KEY_ID).0[0..16].try_into().unwrap();
+        Self {
+            secret: SymmetricSecret::new(key),
+            key_hash,
+            create_time_ticks,
+            encrypt_uses: AtomicUsize::new(0),
+        }
+    }
+}
+
 pub(crate) struct AesGmacSivPoolFactory(Secret<32>, Secret<32>);
 
 impl PoolFactory<AesGmacSiv> for AesGmacSivPoolFactory {