Lots of work including notes and preliminary sketches of session.

aes-gmac-siv/src/impl_macos.rs

@@ -31,6 +31,84 @@ extern "C" {
     fn CCCryptorGCMReset(cryptor_ref: *mut c_void) -> i32;
 }
 
+pub struct Aes(*mut c_void, *mut c_void);
+
+impl Drop for Aes {
+    fn drop(&mut self) {
+        if !self.0.is_null() {
+            unsafe {
+                CCCryptorRelease(self.0);
+            }
+        }
+        if !self.1.is_null() {
+            unsafe {
+                CCCryptorRelease(self.1);
+            }
+        }
+    }
+}
+
+impl Aes {
+    pub fn new(k: &[u8]) -> Self {
+        unsafe {
+            if k.len() != 32 && k.len() != 24 && k.len() != 16 {
+                panic!("AES supports 128, 192, or 256 bits keys");
+            }
+            let mut aes: Self = std::mem::zeroed();
+            let enc = CCCryptorCreateWithMode(kCCEncrypt, kCCModeECB, kCCAlgorithmAES, 0, crate::ZEROES.as_ptr().cast(), k.as_ptr().cast(), k.len(), null(), 0, 0, kCCOptionECBMode, &mut aes.0);
+            if enc != 0 {
+                panic!("CCCryptorCreateWithMode for ECB encrypt mode returned {}", enc);
+            }
+            let dec = CCCryptorCreateWithMode(kCCDecrypt, kCCModeECB, kCCAlgorithmAES, 0, crate::ZEROES.as_ptr().cast(), k.as_ptr().cast(), k.len(), null(), 0, 0, kCCOptionECBMode, &mut aes.1);
+            if dec != 0 {
+                panic!("CCCryptorCreateWithMode for ECB decrypt mode returned {}", dec);
+            }
+            aes
+        }
+    }
+
+    #[inline(always)]
+    pub fn encrypt_block(&self, plaintext: &[u8], ciphertext: &mut [u8]) {
+        assert_eq!(plaintext.len(), 16);
+        assert_eq!(ciphertext.len(), 16);
+        unsafe {
+            let mut data_out_written = 0;
+            CCCryptorUpdate(self.0, plaintext.as_ptr().cast(), 16, ciphertext.as_mut_ptr().cast(), 16, &mut data_out_written);
+        }
+    }
+
+    #[inline(always)]
+    pub fn encrypt_block_in_place(&self, data: &mut [u8]) {
+        assert_eq!(data.len(), 16);
+        unsafe {
+            let mut data_out_written = 0;
+            CCCryptorUpdate(self.0, data.as_ptr().cast(), 16, data.as_mut_ptr().cast(), 16, &mut data_out_written);
+        }
+    }
+
+    #[inline(always)]
+    pub fn decrypt_block(&self, ciphertext: &[u8], plaintext: &mut [u8]) {
+        assert_eq!(plaintext.len(), 16);
+        assert_eq!(ciphertext.len(), 16);
+        unsafe {
+            let mut data_out_written = 0;
+            CCCryptorUpdate(self.1, ciphertext.as_ptr().cast(), 16, plaintext.as_mut_ptr().cast(), 16, &mut data_out_written);
+        }
+    }
+
+    #[inline(always)]
+    pub fn decrypt_block_in_place(&self, data: &mut [u8]) {
+        assert_eq!(data.len(), 16);
+        unsafe {
+            let mut data_out_written = 0;
+            CCCryptorUpdate(self.1, data.as_ptr().cast(), 16, data.as_mut_ptr().cast(), 16, &mut data_out_written);
+        }
+    }
+}
+
+unsafe impl Send for Aes {}
+unsafe impl Sync for Aes {}
+
 pub struct AesCtr(*mut c_void);
 
 impl Drop for AesCtr {
@@ -46,7 +124,6 @@ impl Drop for AesCtr {
 impl AesCtr {
     /// Construct a new AES-CTR cipher.
     /// Key must be 16, 24, or 32 bytes in length or a panic will occur.
-    #[inline(always)]
     pub fn new(k: &[u8]) -> Self {
         if k.len() != 32 && k.len() != 24 && k.len() != 16 {
             panic!("AES supports 128, 192, or 256 bits keys");
@@ -63,7 +140,6 @@ impl AesCtr {
 
     /// Initialize AES-CTR for encryption or decryption with the given IV.
     /// If it's already been used, this also resets the cipher. There is no separate reset.
-    #[inline(always)]
     pub fn init(&mut self, iv: &[u8]) {
         unsafe {
             if iv.len() == 16 {
@@ -102,6 +178,8 @@ impl AesCtr {
     }
 }
 
+unsafe impl Send for AesCtr {}
+
 #[repr(align(8))]
 pub struct AesGmacSiv {
     tag: [u8; 16],
@@ -267,7 +345,7 @@ impl AesGmacSiv {
             if CCCryptorReset(self.ctr, self.tmp.as_ptr().cast()) != 0 {
                 panic!("CCCryptorReset for CTR mode failed (old MacOS bug)");
             }
-            let mut data_out_written: usize = 0;
+            let mut data_out_written = 0;
             CCCryptorUpdate(self.ecb_dec, self.tag.as_ptr().cast(), 16, self.tag.as_mut_ptr().cast(), 16, &mut data_out_written);
             let tmp = self.tmp.as_mut_ptr().cast::<u64>();
             *tmp = *self.tag.as_mut_ptr().cast::<u64>();
@@ -296,7 +374,7 @@ impl AesGmacSiv {
     #[inline(always)]
     pub fn decrypt(&mut self, ciphertext: &[u8], plaintext: &mut [u8]) {
         unsafe {
-            let mut data_out_written: usize = 0;
+            let mut data_out_written = 0;
             CCCryptorUpdate(self.ctr, ciphertext.as_ptr().cast(), ciphertext.len(), plaintext.as_mut_ptr().cast(), plaintext.len(), &mut data_out_written);
             CCCryptorGCMAddAAD(self.gmac, plaintext.as_ptr().cast(), plaintext.len());
         }
@@ -307,7 +385,7 @@ impl AesGmacSiv {
     #[inline(always)]
     pub fn decrypt_in_place(&mut self, ciphertext_to_plaintext: &mut [u8]) {
         unsafe {
-            let mut data_out_written: usize = 0;
+            let mut data_out_written = 0;
             CCCryptorUpdate(self.ctr, ciphertext_to_plaintext.as_ptr().cast(), ciphertext_to_plaintext.len(), ciphertext_to_plaintext.as_mut_ptr().cast(), ciphertext_to_plaintext.len(), &mut data_out_written);
             CCCryptorGCMAddAAD(self.gmac, ciphertext_to_plaintext.as_ptr().cast(), ciphertext_to_plaintext.len());
         }
@@ -329,4 +407,3 @@ impl AesGmacSiv {
 }
 
 unsafe impl Send for AesGmacSiv {}
-unsafe impl Send for AesCtr {}

aes-gmac-siv/src/impl_openssl.rs

@@ -4,6 +4,7 @@
 
 use openssl::symm::{Cipher, Crypter, Mode};
 
+#[inline(always)]
 fn aes_ctr_by_key_size(ks: usize) -> Cipher {
     match ks {
         16 => Cipher::aes_128_ctr(),
@@ -15,6 +16,7 @@ fn aes_ctr_by_key_size(ks: usize) -> Cipher {
     }
 }
 
+#[inline(always)]
 fn aes_gcm_by_key_size(ks: usize) -> Cipher {
     match ks {
         16 => Cipher::aes_128_gcm(),
@@ -26,6 +28,7 @@ fn aes_gcm_by_key_size(ks: usize) -> Cipher {
     }
 }
 
+#[inline(always)]
 fn aes_ecb_by_key_size(ks: usize) -> Cipher {
     match ks {
         16 => Cipher::aes_128_ecb(),
@@ -37,6 +40,55 @@ fn aes_ecb_by_key_size(ks: usize) -> Cipher {
     }
 }
 
+pub struct Aes(Crypter, Crypter);
+
+impl Aes {
+    pub fn new(k: &[u8]) -> Self {
+        let mut aes = Self(Crypter::new(aes_ecb_by_key_size(k.len()), Mode::Encrypt, k, None).unwrap(), Crypter::new(aes_ecb_by_key_size(k.len()), Mode::Decrypt, k, None).unwrap());
+        aes.0.pad(false);
+        aes.1.pad(false);
+    }
+
+    #[inline(always)]
+    pub fn encrypt_block(&self, plaintext: &[u8], ciphertext: &mut [u8]) {
+        let mut tmp = [0_u8; 32];
+        if self.0.update(plaintext, &mut tmp).unwrap() != 16 {
+            assert_eq!(ecb.finalize(&mut tmp).unwrap(), 16);
+        }
+        ciphertext[..16].copy_from_slice(&tmp[..16]);
+    }
+
+    #[inline(always)]
+    pub fn encrypt_block_in_place(&self, data: &mut [u8]) {
+        let mut tmp = [0_u8; 32];
+        if self.0.update(data, &mut tmp).unwrap() != 16 {
+            assert_eq!(ecb.finalize(&mut tmp).unwrap(), 16);
+        }
+        data[..16].copy_from_slice(&tmp[..16]);
+    }
+
+    #[inline(always)]
+    pub fn decrypt_block(&self, ciphertext: &[u8], plaintext: &mut [u8]) {
+        let mut tmp = [0_u8; 32];
+        if self.1.update(plaintext, &mut tmp).unwrap() != 16 {
+            assert_eq!(ecb.finalize(&mut tmp).unwrap(), 16);
+        }
+        ciphertext[..16].copy_from_slice(&tmp[..16]);
+    }
+
+    #[inline(always)]
+    pub fn decrypt_block_in_place(&self, data: &mut [u8]) {
+        let mut tmp = [0_u8; 32];
+        if self.1.update(data, &mut tmp).unwrap() != 16 {
+            assert_eq!(ecb.finalize(&mut tmp).unwrap(), 16);
+        }
+        data[..16].copy_from_slice(&tmp[..16]);
+    }
+}
+
+unsafe impl Send for Aes {}
+unsafe impl Sync for Aes {}
+
 pub struct AesCtr(Vec<u8>, Option<Crypter>);
 
 impl AesCtr {
@@ -69,6 +121,8 @@ impl AesCtr {
     }
 }
 
+unsafe impl Send for AesCtr {}
+
 /// AES-GMAC-SIV encryptor/decryptor.
 pub struct AesGmacSiv {
     tag: [u8; 16],
@@ -247,4 +301,3 @@ impl AesGmacSiv {
 }
 
 unsafe impl Send for AesGmacSiv {}
-unsafe impl Send for AesCtr {}

aes-gmac-siv/src/lib.rs

@@ -7,10 +7,10 @@ mod impl_macos;
 mod impl_openssl;
 
 #[cfg(any(target_os = "macos", target_os = "ios"))]
-pub use impl_macos::{AesCtr, AesGmacSiv};
+pub use impl_macos::{Aes, AesCtr, AesGmacSiv};
 
 #[cfg(not(any(target_os = "macos", target_os = "ios")))]
-pub use impl_openssl::{AesCtr, AesGmacSiv};
+pub use impl_openssl::{Aes, AesCtr, AesGmacSiv};
 
 pub(crate) const ZEROES: [u8; 16] = [0_u8; 16];
 

rustfmt.toml

@@ -1,4 +1,4 @@
-max_width = 256
+max_width = 180
 use_small_heuristics = "Max"
 tab_spaces = 4
 newline_style = "Unix"

zerotier-core-crypto/Cargo.toml

@@ -7,6 +7,7 @@ authors = ["ZeroTier, Inc. <contact@zerotier.com>", "Adam Ierymenko <adam.ieryme
 
 [dependencies]
 rand_core = "0.5.1"
+rand_core_062 = { package = "rand_core", version = "0.6.2" }
 aes-gmac-siv = { path = "../aes-gmac-siv" }
 x25519-dalek = { version = "1.2.0", features = ["std", "u64_backend"], default-features = false }
 ed25519-dalek = { version = "1.0.1", features = ["std", "u64_backend"], default-features = false }
@@ -15,6 +16,7 @@ openssl = { version = "^0", features = [], default-features = false }
 lazy_static = "^1"
 foreign-types = "0.3.1"
 poly1305 = { version = "0.7.2", features = [], default-features = false }
+pqc_kyber = { version = "0.2.0", features = ["kyber512"], default-features = false }
 
 [dev-dependencies]
 quickcheck = "1.0.3"

zerotier-core-crypto/src/lib.rs

@@ -12,5 +12,5 @@ pub mod varint;
 pub mod x25519;
 
 pub use aes_gmac_siv;
-pub use rand_core;
+pub use pqc_kyber;
 pub use subtle;

zerotier-core-crypto/src/random.rs

@@ -83,6 +83,30 @@ impl rand_core::RngCore for SecureRandom {
 
 impl rand_core::CryptoRng for SecureRandom {}
 
+impl rand_core_062::RngCore for SecureRandom {
+    #[inline(always)]
+    fn next_u32(&mut self) -> u32 {
+        next_u32_secure()
+    }
+
+    #[inline(always)]
+    fn next_u64(&mut self) -> u64 {
+        next_u64_secure()
+    }
+
+    #[inline(always)]
+    fn fill_bytes(&mut self, dest: &mut [u8]) {
+        fill_bytes_secure(dest);
+    }
+
+    #[inline(always)]
+    fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), rand_core_062::Error> {
+        rand_bytes(dest).map_err(|e| rand_core_062::Error::new(Box::new(e)))
+    }
+}
+
+impl rand_core_062::CryptoRng for SecureRandom {}
+
 unsafe impl Sync for SecureRandom {}
 
 unsafe impl Send for SecureRandom {}

zerotier-network-hypervisor/src/vl1/identity.rs

@@ -296,16 +296,17 @@ impl Identity {
     /// An error can occur if this identity does not hold its secret portion or if either key is invalid.
     ///
     /// If both sides have NIST P-384 keys then key agreement is performed using both Curve25519 and
-    /// NIST P-384 and the result is HMAC(Curve25519 secret, NIST P-384 secret).
-    pub fn agree(&self, other: &Identity) -> Option<Secret<48>> {
+    /// NIST P-384 and the result is HMAC-SHA512(Curve25519 secret, NIST P-384 secret). This is FIPS
+    /// compliant since the Curve25519 secret is treated as a "salt" in HKDF.
+    pub fn agree(&self, other: &Identity) -> Option<Secret<64>> {
         if let Some(secret) = self.secret.as_ref() {
-            let c25519_secret: Secret<48> = Secret((&SHA512::hash(&secret.c25519.agree(&other.c25519).0)[..48]).try_into().unwrap());
+            let c25519_secret: Secret<64> = Secret(SHA512::hash(&secret.c25519.agree(&other.c25519).0));
 
             // FIPS note: FIPS-compliant exchange algorithms must be the last algorithms in any HKDF chain
             // for the final result to be technically FIPS compliant. Non-FIPS algorithm secrets are considered
             // a salt in the HMAC(salt, key) HKDF construction.
             if secret.p384.is_some() && other.p384.is_some() {
-                secret.p384.as_ref().unwrap().ecdh.agree(&other.p384.as_ref().unwrap().ecdh).map(|p384_secret| Secret(hmac_sha384(&c25519_secret.0, &p384_secret.0)))
+                secret.p384.as_ref().unwrap().ecdh.agree(&other.p384.as_ref().unwrap().ecdh).map(|p384_secret| Secret(hmac_sha512(&c25519_secret.0, &p384_secret.0)))
             } else {
                 Some(c25519_secret)
             }

zerotier-network-hypervisor/src/vl1/mod.rs

@@ -11,6 +11,7 @@ mod mac;
 mod path;
 mod peer;
 mod rootset;
+mod session;
 mod symmetricsecret;
 mod whoisqueue;
 

zerotier-network-hypervisor/src/vl1/node.rs

@@ -68,7 +68,14 @@ pub trait SystemInterface: Sync + Send + 'static {
     /// For endpoint types that support a packet TTL, the implementation may set the TTL
     /// if the 'ttl' parameter is not zero. If the parameter is zero or TTL setting is not
     /// supported, the default TTL should be used.
-    async fn wire_send(&self, endpoint: &Endpoint, local_socket: Option<&Self::LocalSocket>, local_interface: Option<&Self::LocalInterface>, data: &[&[u8]], packet_ttl: u8) -> bool;
+    async fn wire_send(
+        &self,
+        endpoint: &Endpoint,
+        local_socket: Option<&Self::LocalSocket>,
+        local_interface: Option<&Self::LocalInterface>,
+        data: &[&[u8]],
+        packet_ttl: u8,
+    ) -> bool;
 
     /// Called to check and see if a physical address should be used for ZeroTier traffic to a node.
     async fn check_path(&self, id: &Identity, endpoint: &Endpoint, local_socket: Option<&Self::LocalSocket>, local_interface: Option<&Self::LocalInterface>) -> bool;
@@ -94,7 +101,15 @@ pub trait InnerProtocolInterface: Sync + Send + 'static {
     /// Handle a packet, returning true if it was handled by the next layer.
     ///
     /// Do not attempt to handle OK or ERROR. Instead implement handle_ok() and handle_error().
-    async fn handle_packet<SI: SystemInterface>(&self, source: &Peer<SI>, source_path: &Path<SI>, forward_secrecy: bool, extended_authentication: bool, verb: u8, payload: &PacketBuffer) -> bool;
+    async fn handle_packet<SI: SystemInterface>(
+        &self,
+        source: &Peer<SI>,
+        source_path: &Path<SI>,
+        forward_secrecy: bool,
+        extended_authentication: bool,
+        verb: u8,
+        payload: &PacketBuffer,
+    ) -> bool;
 
     /// Handle errors, returning true if the error was recognized.
     async fn handle_error<SI: SystemInterface>(
@@ -111,7 +126,17 @@ pub trait InnerProtocolInterface: Sync + Send + 'static {
     ) -> bool;
 
     /// Handle an OK, returing true if the OK was recognized.
-    async fn handle_ok<SI: SystemInterface>(&self, source: &Peer<SI>, source_path: &Path<SI>, forward_secrecy: bool, extended_authentication: bool, in_re_verb: u8, in_re_message_id: u64, payload: &PacketBuffer, cursor: &mut usize) -> bool;
+    async fn handle_ok<SI: SystemInterface>(
+        &self,
+        source: &Peer<SI>,
+        source_path: &Path<SI>,
+        forward_secrecy: bool,
+        extended_authentication: bool,
+        in_re_verb: u8,
+        in_re_message_id: u64,
+        payload: &PacketBuffer,
+        cursor: &mut usize,
+    ) -> bool;
 
     /// Check if this remote peer has a trust relationship with this node.
     ///
@@ -334,7 +359,14 @@ impl<SI: SystemInterface> Node<SI> {
         let tt = si.time_ticks();
         let (root_sync, root_hello, mut root_spam_hello, peer_service, path_service, whois_service) = {
             let mut intervals = self.intervals.lock();
-            (intervals.root_sync.gate(tt), intervals.root_hello.gate(tt), intervals.root_spam_hello.gate(tt), intervals.peer_service.gate(tt), intervals.path_service.gate(tt), intervals.whois_service.gate(tt))
+            (
+                intervals.root_sync.gate(tt),
+                intervals.root_hello.gate(tt),
+                intervals.root_spam_hello.gate(tt),
+                intervals.peer_service.gate(tt),
+                intervals.path_service.gate(tt),
+                intervals.whois_service.gate(tt),
+            )
         };
 
         // We only "spam" if we are offline.
@@ -383,7 +415,9 @@ impl<SI: SystemInterface> Node<SI> {
                             for m in rs.members.iter() {
                                 if m.identity.eq(&self.identity) {
                                     let _ = my_root_sets.get_or_insert_with(|| Vec::new()).write_all(rs.to_bytes().as_slice());
-                                } else if self.peers.read().get(&m.identity.address).map_or(false, |p| !p.identity.eq(&m.identity)) || address_collision_check.insert(m.identity.address, &m.identity).map_or(false, |old_id| !old_id.eq(&m.identity)) {
+                                } else if self.peers.read().get(&m.identity.address).map_or(false, |p| !p.identity.eq(&m.identity))
+                                    || address_collision_check.insert(m.identity.address, &m.identity).map_or(false, |old_id| !old_id.eq(&m.identity))
+                                {
                                     address_collisions.push(m.identity.address);
                                 }
                             }
@@ -399,7 +433,10 @@ impl<SI: SystemInterface> Node<SI> {
                                     new_roots.insert(peer.clone(), m.endpoints.as_ref().unwrap().iter().cloned().collect());
                                 } else {
                                     if let Some(peer) = Peer::<SI>::new(&self.identity, m.identity.clone(), tt) {
-                                        new_roots.insert(parking_lot::RwLockUpgradableReadGuard::upgrade(peers).entry(m.identity.address).or_insert_with(|| Arc::new(peer)).clone(), m.endpoints.as_ref().unwrap().iter().cloned().collect());
+                                        new_roots.insert(
+                                            parking_lot::RwLockUpgradableReadGuard::upgrade(peers).entry(m.identity.address).or_insert_with(|| Arc::new(peer)).clone(),
+                                            m.endpoints.as_ref().unwrap().iter().cloned().collect(),
+                                        );
                                     } else {
                                         bad_identities.push(m.identity.clone());
                                     }
@@ -412,10 +449,16 @@ impl<SI: SystemInterface> Node<SI> {
                 };
 
                 for c in address_collisions.iter() {
-                    si.event(Event::SecurityWarning(format!("address/identity collision in root sets! address {} collides across root sets or with an existing peer and is being ignored as a root!", c.to_string())));
+                    si.event(Event::SecurityWarning(format!(
+                        "address/identity collision in root sets! address {} collides across root sets or with an existing peer and is being ignored as a root!",
+                        c.to_string()
+                    )));
                 }
                 for i in bad_identities.iter() {
-                    si.event(Event::SecurityWarning(format!("bad identity detected for address {} in at least one root set, ignoring (error creating peer object)", i.address.to_string())));
+                    si.event(Event::SecurityWarning(format!(
+                        "bad identity detected for address {} in at least one root set, ignoring (error creating peer object)",
+                        i.address.to_string()
+                    )));
                 }
 
                 let mut new_root_identities: Vec<Identity> = new_roots.iter().map(|(p, _)| p.identity.clone()).collect();
@@ -516,7 +559,15 @@ impl<SI: SystemInterface> Node<SI> {
         Duration::from_millis(1000)
     }
 
-    pub async fn handle_incoming_physical_packet<PH: InnerProtocolInterface>(&self, si: &SI, ph: &PH, source_endpoint: &Endpoint, source_local_socket: &SI::LocalSocket, source_local_interface: &SI::LocalInterface, mut data: PooledPacketBuffer) {
+    pub async fn handle_incoming_physical_packet<PH: InnerProtocolInterface>(
+        &self,
+        si: &SI,
+        ph: &PH,
+        source_endpoint: &Endpoint,
+        source_local_socket: &SI::LocalSocket,
+        source_local_interface: &SI::LocalInterface,
+        mut data: PooledPacketBuffer,
+    ) {
         debug_event!(
             si,
             "[vl1] {} -> #{} {}->{} length {} (on socket {}@{})",
@@ -539,9 +590,17 @@ impl<SI: SystemInterface> Node<SI> {
                     if fragment_header.is_fragment() {
                         #[cfg(debug_assertions)]
                         let fragment_header_id = u64::from_be_bytes(fragment_header.id);
-                        debug_event!(si, "[vl1] #{:0>16x} fragment {} of {} received", u64::from_be_bytes(fragment_header.id), fragment_header.fragment_no(), fragment_header.total_fragments());
+                        debug_event!(
+                            si,
+                            "[vl1] #{:0>16x} fragment {} of {} received",
+                            u64::from_be_bytes(fragment_header.id),
+                            fragment_header.fragment_no(),
+                            fragment_header.total_fragments()
+                        );
 
-                        if let Some(assembled_packet) = path.receive_fragment(fragment_header.packet_id(), fragment_header.fragment_no(), fragment_header.total_fragments(), data, time_ticks) {
+                        if let Some(assembled_packet) =
+                            path.receive_fragment(fragment_header.packet_id(), fragment_header.fragment_no(), fragment_header.total_fragments(), data, time_ticks)
+                        {
                             if let Some(frag0) = assembled_packet.frags[0].as_ref() {
                                 #[cfg(debug_assertions)]
                                 debug_event!(si, "[vl1] #{:0>16x} packet fully assembled!", fragment_header_id);
@@ -549,7 +608,8 @@ impl<SI: SystemInterface> Node<SI> {
                                 if let Ok(packet_header) = frag0.struct_at::<PacketHeader>(0) {
                                     if let Some(source) = Address::from_bytes(&packet_header.src) {
                                         if let Some(peer) = self.peer(source) {
-                                            peer.receive(self, si, ph, time_ticks, &path, &packet_header, frag0, &assembled_packet.frags[1..(assembled_packet.have as usize)]).await;
+                                            peer.receive(self, si, ph, time_ticks, &path, &packet_header, frag0, &assembled_packet.frags[1..(assembled_packet.have as usize)])
+                                                .await;
                                         } else {
                                             self.whois.query(self, si, source, Some(QueuedPacket::Fragmented(assembled_packet)));
                                         }
@@ -673,6 +733,11 @@ impl<SI: SystemInterface> Node<SI> {
         if let Some(path) = self.paths.read().get(&PathKey::Ref(ep, local_socket)) {
             return path.clone();
         }
-        return self.paths.write().entry(PathKey::Copied(ep.clone(), local_socket.clone())).or_insert_with(|| Arc::new(Path::new(ep.clone(), local_socket.clone(), local_interface.clone(), time_ticks))).clone();
+        return self
+            .paths
+            .write()
+            .entry(PathKey::Copied(ep.clone(), local_socket.clone()))
+            .or_insert_with(|| Arc::new(Path::new(ep.clone(), local_socket.clone(), local_interface.clone(), time_ticks)))
+            .clone();
     }
 }

zerotier-network-hypervisor/src/vl1/peer.rs

@@ -22,7 +22,7 @@ use crate::vl1::careof::CareOf;
 use crate::vl1::node::*;
 use crate::vl1::protocol::*;
 use crate::vl1::rootset::RootSet;
-use crate::vl1::symmetricsecret::{EphemeralSymmetricSecret, SymmetricSecret};
+use crate::vl1::symmetricsecret::SymmetricSecret;
 use crate::vl1::{Dictionary, Endpoint, Identity, Path};
 use crate::{VERSION_MAJOR, VERSION_MINOR, VERSION_REVISION};
 
@@ -55,9 +55,6 @@ pub struct Peer<SI: SystemInterface> {
     // Static shared secret computed from agreement with identity.
     identity_symmetric_key: SymmetricSecret,
 
-    // Latest ephemeral session key or None if no current session.
-    ephemeral_symmetric_key: RwLock<Option<EphemeralSymmetricSecret>>,
-
     // Paths sorted in descending order of quality / preference.
     paths: Mutex<Vec<PeerPath<SI>>>,
 
@@ -80,7 +77,13 @@ pub struct Peer<SI: SystemInterface> {
 }
 
 /// Attempt AEAD packet encryption and MAC validation. Returns message ID on success.
-fn try_aead_decrypt(secret: &SymmetricSecret, packet_frag0_payload_bytes: &[u8], packet_header: &PacketHeader, fragments: &[Option<PooledPacketBuffer>], payload: &mut PacketBuffer) -> Option<MessageId> {
+fn try_aead_decrypt(
+    secret: &SymmetricSecret,
+    packet_frag0_payload_bytes: &[u8],
+    packet_header: &PacketHeader,
+    fragments: &[Option<PooledPacketBuffer>],
+    payload: &mut PacketBuffer,
+) -> Option<MessageId> {
     let cipher = packet_header.cipher();
     match cipher {
         security_constants::CIPHER_NOCRYPT_POLY1305 | security_constants::CIPHER_SALSA2012_POLY1305 => {
@@ -192,7 +195,6 @@ impl<SI: SystemInterface> Peer<SI> {
                 canonical: CanonicalObject::new(),
                 identity: id,
                 identity_symmetric_key: SymmetricSecret::new(static_secret),
-                ephemeral_symmetric_key: RwLock::new(None),
                 paths: Mutex::new(Vec::with_capacity(4)),
                 last_send_time_ticks: AtomicI64::new(crate::util::NEVER_HAPPENED_TICKS),
                 last_receive_time_ticks: AtomicI64::new(crate::util::NEVER_HAPPENED_TICKS),
@@ -280,7 +282,13 @@ impl<SI: SystemInterface> Peer<SI> {
                         match &p.endpoint {
                             Endpoint::IpUdp(existing_ip) => {
                                 if existing_ip.ip_bytes().eq(new_ip.ip_bytes()) {
-                                    debug_event!(si, "[vl1] {} replacing path {} with {} (same IP, different port)", self.identity.address.to_string(), p.endpoint.to_string(), new_path.endpoint.to_string());
+                                    debug_event!(
+                                        si,
+                                        "[vl1] {} replacing path {} with {} (same IP, different port)",
+                                        self.identity.address.to_string(),
+                                        p.endpoint.to_string(),
+                                        new_path.endpoint.to_string()
+                                    );
                                     pi.path = Arc::downgrade(new_path);
                                     pi.canonical_instance_id = new_path.canonical.canonical_instance_id();
                                     pi.last_receive_time_ticks = time_ticks;
@@ -327,7 +335,15 @@ impl<SI: SystemInterface> Peer<SI> {
     ///
     /// This does not set the fragmentation field in the packet header, MAC, or encrypt the packet. The sender
     /// must do that while building the packet. The fragmentation flag must be set if fragmentation will be needed.
-    async fn internal_send(&self, si: &SI, endpoint: &Endpoint, local_socket: Option<&SI::LocalSocket>, local_interface: Option<&SI::LocalInterface>, max_fragment_size: usize, packet: &PacketBuffer) -> bool {
+    async fn internal_send(
+        &self,
+        si: &SI,
+        endpoint: &Endpoint,
+        local_socket: Option<&SI::LocalSocket>,
+        local_interface: Option<&SI::LocalInterface>,
+        max_fragment_size: usize,
+        packet: &PacketBuffer,
+    ) -> bool {
         let packet_size = packet.len();
         if packet_size > max_fragment_size {
             let bytes = packet.as_bytes();
@@ -337,7 +353,8 @@ impl<SI: SystemInterface> Peer<SI> {
             let mut pos = UDP_DEFAULT_MTU;
 
             let overrun_size = (packet_size - UDP_DEFAULT_MTU) as u32;
-            let fragment_count = (overrun_size / (UDP_DEFAULT_MTU - packet_constants::FRAGMENT_HEADER_SIZE) as u32) + (((overrun_size % (UDP_DEFAULT_MTU - packet_constants::FRAGMENT_HEADER_SIZE) as u32) != 0) as u32);
+            let fragment_count = (overrun_size / (UDP_DEFAULT_MTU - packet_constants::FRAGMENT_HEADER_SIZE) as u32)
+                + (((overrun_size % (UDP_DEFAULT_MTU - packet_constants::FRAGMENT_HEADER_SIZE) as u32) != 0) as u32);
             debug_assert!(fragment_count <= packet_constants::FRAGMENT_COUNT_MAX as u32);
 
             let mut header = FragmentHeader {
@@ -387,9 +404,13 @@ impl<SI: SystemInterface> Peer<SI> {
         };
 
         let max_fragment_size = if path.endpoint.requires_fragmentation() { UDP_DEFAULT_MTU } else { usize::MAX };
-        let flags_cipher_hops = if packet.len() > max_fragment_size { packet_constants::HEADER_FLAG_FRAGMENTED | security_constants::CIPHER_AES_GMAC_SIV } else { security_constants::CIPHER_AES_GMAC_SIV };
+        let flags_cipher_hops = if packet.len() > max_fragment_size {
+            packet_constants::HEADER_FLAG_FRAGMENTED | security_constants::CIPHER_AES_GMAC_SIV
+        } else {
+            security_constants::CIPHER_AES_GMAC_SIV
+        };
 
-        let mut aes_gmac_siv = if let Some(ephemeral_key) = self.ephemeral_symmetric_key.read().as_ref() { ephemeral_key.secret.aes_gmac_siv.get() } else { self.identity_symmetric_key.aes_gmac_siv.get() };
+        let mut aes_gmac_siv = self.identity_symmetric_key.aes_gmac_siv.get();
         aes_gmac_siv.encrypt_init(&self.next_message_id().to_ne_bytes());
         aes_gmac_siv.encrypt_set_aad(&get_packet_aad_bytes(self.identity.address, node.identity.address, flags_cipher_hops));
         if let Ok(payload) = packet.as_bytes_starting_at_mut(packet_constants::HEADER_SIZE) {
@@ -556,37 +577,29 @@ impl<SI: SystemInterface> Peer<SI> {
     /// those fragments after the main packet header and first chunk.
     ///
     /// This returns true if the packet decrypted and passed authentication.
-    pub(crate) async fn receive<PH: InnerProtocolInterface>(&self, node: &Node<SI>, si: &SI, ph: &PH, time_ticks: i64, source_path: &Arc<Path<SI>>, packet_header: &PacketHeader, frag0: &PacketBuffer, fragments: &[Option<PooledPacketBuffer>]) -> bool {
+    pub(crate) async fn receive<PH: InnerProtocolInterface>(
+        &self,
+        node: &Node<SI>,
+        si: &SI,
+        ph: &PH,
+        time_ticks: i64,
+        source_path: &Arc<Path<SI>>,
+        packet_header: &PacketHeader,
+        frag0: &PacketBuffer,
+        fragments: &[Option<PooledPacketBuffer>],
+    ) -> bool {
         if let Ok(packet_frag0_payload_bytes) = frag0.as_bytes_starting_at(packet_constants::VERB_INDEX) {
             let mut payload = PacketBuffer::new();
 
-            // First try decrypting and authenticating with an ephemeral secret if one is negotiated.
-            let (forward_secrecy, mut message_id) = if let Some(ephemeral_secret) = self.ephemeral_symmetric_key.read().as_ref() {
-                if let Some(message_id) = try_aead_decrypt(&ephemeral_secret.secret, packet_frag0_payload_bytes, packet_header, fragments, &mut payload) {
-                    // Decryption successful with ephemeral secret
-                    (true, message_id)
-                } else {
-                    // Decryption failed with ephemeral secret, which may indicate that it's obsolete.
-                    (false, 0)
-                }
+            let message_id = if let Some(message_id2) = try_aead_decrypt(&self.identity_symmetric_key, packet_frag0_payload_bytes, packet_header, fragments, &mut payload) {
+                // Decryption successful with static secret.
+                message_id2
             } else {
-                // There is no ephemeral secret negotiated (yet?).
-                (false, 0)
+                // Packet failed to decrypt using either ephemeral or permament key, reject.
+                debug_event!(si, "[vl1] #{:0>16x} failed authentication", u64::from_be_bytes(packet_header.id));
+                return false;
             };
 
-            // If forward_secrecy is false it means the ephemeral key failed. Try decrypting with the permanent key.
-            if !forward_secrecy {
-                payload.clear();
-                if let Some(message_id2) = try_aead_decrypt(&self.identity_symmetric_key, packet_frag0_payload_bytes, packet_header, fragments, &mut payload) {
-                    // Decryption successful with static secret.
-                    message_id = message_id2;
-                } else {
-                    // Packet failed to decrypt using either ephemeral or permament key, reject.
-                    debug_event!(si, "[vl1] #{:0>16x} failed authentication", u64::from_be_bytes(packet_header.id));
-                    return false;
-                }
-            }
-
             if let Ok(mut verb) = payload.u8_at(0) {
                 let extended_authentication = (verb & packet_constants::VERB_FLAG_EXTENDED_AUTHENTICATION) != 0;
                 if extended_authentication {
@@ -636,15 +649,19 @@ impl<SI: SystemInterface> Peer<SI> {
 
                 if match verb {
                     verbs::VL1_NOP => true,
-                    verbs::VL1_HELLO => self.handle_incoming_hello(si, ph, node, time_ticks, message_id, source_path, packet_header.hops(), extended_authentication, &payload).await,
-                    verbs::VL1_ERROR => self.handle_incoming_error(si, ph, node, time_ticks, source_path, forward_secrecy, extended_authentication, &payload).await,
-                    verbs::VL1_OK => self.handle_incoming_ok(si, ph, node, time_ticks, source_path, packet_header.hops(), path_is_known, forward_secrecy, extended_authentication, &payload).await,
+                    verbs::VL1_HELLO => {
+                        self.handle_incoming_hello(si, ph, node, time_ticks, message_id, source_path, packet_header.hops(), extended_authentication, &payload).await
+                    }
+                    verbs::VL1_ERROR => self.handle_incoming_error(si, ph, node, time_ticks, source_path, false, extended_authentication, &payload).await,
+                    verbs::VL1_OK => {
+                        self.handle_incoming_ok(si, ph, node, time_ticks, source_path, packet_header.hops(), path_is_known, false, extended_authentication, &payload).await
+                    }
                     verbs::VL1_WHOIS => self.handle_incoming_whois(si, ph, node, time_ticks, message_id, &payload).await,
                     verbs::VL1_RENDEZVOUS => self.handle_incoming_rendezvous(si, node, time_ticks, message_id, source_path, &payload).await,
                     verbs::VL1_ECHO => self.handle_incoming_echo(si, ph, node, time_ticks, message_id, &payload).await,
                     verbs::VL1_PUSH_DIRECT_PATHS => self.handle_incoming_push_direct_paths(si, node, time_ticks, source_path, &payload).await,
                     verbs::VL1_USER_MESSAGE => self.handle_incoming_user_message(si, node, time_ticks, source_path, &payload).await,
-                    _ => ph.handle_packet(self, &source_path, forward_secrecy, extended_authentication, verb, &payload).await,
+                    _ => ph.handle_packet(self, &source_path, false, extended_authentication, verb, &payload).await,
                 } {
                     // This needs to be saved AFTER processing the packet since some message types may use it to try to filter for replays.
                     self.last_incoming_message_id.store(message_id, Ordering::Relaxed);
@@ -656,7 +673,18 @@ impl<SI: SystemInterface> Peer<SI> {
         return false;
     }
 
-    async fn handle_incoming_hello<PH: InnerProtocolInterface>(&self, si: &SI, ph: &PH, node: &Node<SI>, time_ticks: i64, message_id: MessageId, source_path: &Arc<Path<SI>>, hops: u8, extended_authentication: bool, payload: &PacketBuffer) -> bool {
+    async fn handle_incoming_hello<PH: InnerProtocolInterface>(
+        &self,
+        si: &SI,
+        ph: &PH,
+        node: &Node<SI>,
+        time_ticks: i64,
+        message_id: MessageId,
+        source_path: &Arc<Path<SI>>,
+        hops: u8,
+        extended_authentication: bool,
+        payload: &PacketBuffer,
+    ) -> bool {
         if !(ph.has_trust_relationship(&self.identity) || node.this_node_is_root() || node.is_peer_root(self)) {
             debug_event!(si, "[vl1] dropping HELLO from {} due to lack of trust relationship", self.identity.address.to_string());
             return true; // packet wasn't invalid, just ignored
@@ -671,8 +699,9 @@ impl<SI: SystemInterface> Peer<SI> {
 
                         remote_node_info.remote_protocol_version = hello_fixed_headers.version_proto;
                         remote_node_info.hello_extended_authentication = extended_authentication;
-                        remote_node_info.remote_version =
-                            (hello_fixed_headers.version_major as u64).wrapping_shl(48) | (hello_fixed_headers.version_minor as u64).wrapping_shl(32) | (u16::from_be_bytes(hello_fixed_headers.version_revision) as u64).wrapping_shl(16);
+                        remote_node_info.remote_version = (hello_fixed_headers.version_major as u64).wrapping_shl(48)
+                            | (hello_fixed_headers.version_minor as u64).wrapping_shl(32)
+                            | (u16::from_be_bytes(hello_fixed_headers.version_revision) as u64).wrapping_shl(16);
 
                         if hello_fixed_headers.version_proto >= 20 {
                             let mut session_metadata_len = payload.read_u16(&mut cursor).unwrap_or(0) as usize;
@@ -741,14 +770,36 @@ impl<SI: SystemInterface> Peer<SI> {
         return false;
     }
 
-    async fn handle_incoming_error<PH: InnerProtocolInterface>(&self, _si: &SI, ph: &PH, _node: &Node<SI>, _time_ticks: i64, source_path: &Arc<Path<SI>>, forward_secrecy: bool, extended_authentication: bool, payload: &PacketBuffer) -> bool {
+    async fn handle_incoming_error<PH: InnerProtocolInterface>(
+        &self,
+        _si: &SI,
+        ph: &PH,
+        _node: &Node<SI>,
+        _time_ticks: i64,
+        source_path: &Arc<Path<SI>>,
+        forward_secrecy: bool,
+        extended_authentication: bool,
+        payload: &PacketBuffer,
+    ) -> bool {
         let mut cursor = 0;
         if let Ok(error_header) = payload.read_struct::<message_component_structs::ErrorHeader>(&mut cursor) {
             let in_re_message_id: MessageId = u64::from_ne_bytes(error_header.in_re_message_id);
             if self.message_id_counter.load(Ordering::Relaxed).wrapping_sub(in_re_message_id) <= PACKET_RESPONSE_COUNTER_DELTA_MAX {
                 match error_header.in_re_verb {
                     _ => {
-                        return ph.handle_error(self, &source_path, forward_secrecy, extended_authentication, error_header.in_re_verb, in_re_message_id, error_header.error_code, payload, &mut cursor).await;
+                        return ph
+                            .handle_error(
+                                self,
+                                &source_path,
+                                forward_secrecy,
+                                extended_authentication,
+                                error_header.in_re_verb,
+                                in_re_message_id,
+                                error_header.error_code,
+                                payload,
+                                &mut cursor,
+                            )
+                            .await;
                     }
                 }
             }
@@ -791,7 +842,12 @@ impl<SI: SystemInterface> Peer<SI> {
                                                             let reported_endpoint2 = reported_endpoint.clone();
                                                             if remote_node_info.reported_local_endpoints.insert(reported_endpoint, time_ticks).is_none() {
                                                                 #[cfg(debug_assertions)]
-                                                                debug_event!(si, "[vl1] {} reported new remote perspective, local endpoint: {}", self.identity.address.to_string(), reported_endpoint2.to_string());
+                                                                debug_event!(
+                                                                    si,
+                                                                    "[vl1] {} reported new remote perspective, local endpoint: {}",
+                                                                    self.identity.address.to_string(),
+                                                                    reported_endpoint2.to_string()
+                                                                );
                                                             }
                                                         }
                                                     }
@@ -836,7 +892,12 @@ impl<SI: SystemInterface> Peer<SI> {
                                         let reported_endpoint2 = reported_endpoint.clone();
                                         if self.remote_node_info.write().reported_local_endpoints.insert(reported_endpoint, time_ticks).is_none() {
                                             #[cfg(debug_assertions)]
-                                            debug_event!(si, "[vl1] {} reported new remote perspective, local endpoint: {}", self.identity.address.to_string(), reported_endpoint2.to_string());
+                                            debug_event!(
+                                                si,
+                                                "[vl1] {} reported new remote perspective, local endpoint: {}",
+                                                self.identity.address.to_string(),
+                                                reported_endpoint2.to_string()
+                                            );
                                         }
                                     }
                                 }

zerotier-network-hypervisor/src/vl1/protocol.rs

@@ -224,6 +224,9 @@ pub mod security_constants {
     /// KBKDF usage label for a unique ID for ephemeral keys (not actually a key).
     pub const KBKDF_KEY_USAGE_LABEL_EPHEMERAL_KEY_ID: u8 = b'e';
 
+    /// KBKDF usage label for a unique ID for ephemeral keys (not actually a key).
+    pub const KBKDF_KEY_USAGE_LABEL_RATCHET_KEY: u8 = b'+';
+
     /// Try to re-key ephemeral keys after this time.
     pub const EPHEMERAL_SECRET_REKEY_AFTER_TIME: i64 = 300000; // 5 minutes
 

zerotier-network-hypervisor/src/vl1/session.rs

@@ -0,0 +1,508 @@
+// (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md.
+
+use std::mem::size_of;
+use std::sync::atomic::AtomicU32;
+
+use zerotier_core_crypto::aes_gmac_siv::{Aes, AesCtr};
+use zerotier_core_crypto::hash::{hmac_sha384, SHA384};
+use zerotier_core_crypto::kbkdf::zt_kbkdf_hmac_sha512;
+use zerotier_core_crypto::p384::*;
+use zerotier_core_crypto::pqc_kyber;
+use zerotier_core_crypto::random;
+use zerotier_core_crypto::secret::Secret;
+use zerotier_core_crypto::x25519::*;
+
+use crate::util::buffer::Buffer;
+use crate::util::marshalable::Marshalable;
+use crate::vl1::identity::Identity;
+use crate::vl1::symmetricsecret::SymmetricSecret;
+
+use parking_lot::RwLock;
+
+/*
+
+Basic outline of the ZeroTier V2 session protocol:
+
+*** Three-way connection setup handshake:
+
+(1) Initiator sends INIT:
+
+[16]  random IV
+[4]   session ID
+[1]   FFFFTTTT where F == flags, T == message type (1 for INIT)
+[1]   ZeroTier protocol version
+[4]   reserved, always zero
+[1]   field ID of unencrypted initial ephemeral key
+[...] outer ephemeral public key (currently always NIST P-384)
+-- begin AES-CTR encryption using ephemeral/static AES key
+[...] additional tuples of field ID and field data
+-- end AES-CTR encryption
+[48]  HMAC-SHA384 using static/static HMAC key
+
+Additional fields in INIT:
+ - Optional: additional ephemeral public keys
+ - Optional: first 16 bytes of SHA384 of "current" session key
+ - Required: static ZeroTier identity of initiator
+ - Required: timestamp
+
+(2) Responder sends ACK:
+
+[16]  random IV
+[4]   session ID
+[1]   FFFFTTTT where F == flags, T == message type (2 for ACK)
+[1]   ZeroTier protocol version
+[4]   reserved, always zero
+-- begin AES-CTR encryption using same ephemeral/static AES key as INIT
+[...] tuples of field ID and field data
+-- end AES-CTR encryption
+[48]  HMAC-SHA384 using static/static HMAC key
+
+Fields in ACK:
+ - Required: ephemeral public key matching at least one ephemeral sent
+ - Optional: additional matching ephemeral keys
+ - Optional: first 16 bytes of SHA384 of "current" session key
+ - Required: timestamp
+ - Required: echo of timestamp from INIT
+
+(3) Initiator sends CONFIRM:
+
+[16]  AES-GMAC-SIV opaque tag
+[4]   session ID
+[1]   FFFFTTTT where F == flags, T == message type (3 for CONFIRM)
+-- begin AES-GMAC-SIV encryption
+[...] tuples of field ID and field data
+
+Fields in CONFIRM:
+ - Required: echo of timestamp from ACK
+
+CONFIRM is technically optional as a valid DATA message also counts as
+confirmation, but having an explicit message allows for mutual exchange
+of latency information and in the future other things.
+
+*** DATA packets:
+
+[16]  AES-GMAC-SIV opaque tag
+[4]   session ID
+[1]   FFFFTTTT where F == flags, T == message type (0 for DATA)
+-- begin AES-GMAC-SIV encrypted data packet
+[1]   LNNNNNNN where N == fragment number and L is set if it's the last fragment
+[...] data payload, typically starting with a ZeroTier VL1/VL2 protocol verb
+
+When AES-GMAC-SIV packets are decrypted and authenticated, a sequential
+message ID is exposed. This is used as a counter to prohibit replay attacks
+within a session.
+
+*** SINGLETON packets:
+
+A singleton packet has the same format as an INIT packet, but includes no
+additional public keys or session key info. Instead it includes a data payload
+field and it elicits no ACK response. The session ID must be zero.
+
+Singleton packets can be used to send unidirectional sparse messages without
+incurring the overhead of a full session. There is no replay attack prevention
+in this case, so these messages should only be used for things that are
+idempotent or have their own resistance to replay. There is also no automatic
+fragmentation, so the full packet must fit in the underlying transport.
+
+*** Notes:
+
+The initiator creates one or more ephemeral public keys and sends the first of
+these ephemeral keys in unencrypted form. Key agreement (or KEX if applicable) is
+performed against the responder's static identity key by both the initiator and the
+responder to create an ephemeral/static key that is only used for INIT and ACK and
+not afterwords. (The ephemeral sent in the clear must have a counterpart in the
+recipient's static identity.)
+
+When the responder receives INIT it computes the session key as follows:
+
+(1) A starting ratchet key is chosen. If INIT contains a hash of the current
+    (being replaced) session key and it matches the one at the responder, a
+    derived ratchet key from the current session is used. Otherwise a ratchet
+    key derived from the static/static key (the permanent key) is used.
+(2) For each ephemeral key supplied by the initiator, the responder optionally
+    generates its own ephemeral counterpart. While the responder is not required
+    to match all supplied keys it must compute and supply at least one to create
+    a valid forward-secure session. The responder then sends these keys in an
+    ACK message encrypted using the same key as INIT but authenticated via HMAC
+    using the new session key. Once the responder generates its own ephemeral
+    keys it may compute the session key in the same manner as the initiator.
+(3) When the initiator receives ACK it can compute the session key. Starting
+    with the ratchet key from step (1) the initator performs key agreement using
+    each ephemeral key pair for which both sides have furnished a key. These are
+    chained together using HMAC-SHA512(last, next) where the last key is the
+    "key" in HMAC and the next key is the "message."
+
+Key agreements in (3) are performed in the following order, skipping any where both
+sides have not furnished a key:
+
+(1) Curve25519 ECDH
+(2) Kyber (768)
+(3) NIST P-384 ECDH
+
+The NIST key must be last for FIPS compliance reasons as it's a FIPS-compliant
+algorithm and elliptic curve. FIPS allows HKDF using HMAC(salt, key) and allows
+the salt to be anything, so we can use the results of previous non-FIPS agreements
+as this "salt."
+
+Kyber is a post-quantum algorithm, the first to be standardized by NIST. Its
+purpose is to provide long-term forward secrecy against adversaries who warehouse
+data in anticipation of future quantum computing capability. When enabled a future
+QC adversary could de-anonymize identities by breaking e.g. NIST P-384 but could
+still not decrypt actual session payload.
+
+Kyber is a key encapsulation algorithm rather than a Diffie-Hellman style
+algorithm. When used the initiator generates a key pair and then sends its public
+key to the responder. The responder then uses this public key to generate a shared
+secret that is sent back to the initiator. The responder does not have to generate
+its own key pair for this exchange. The raw Kyber algorithm is used since the
+authentication in this session protocol is provided by HMAC-SHA384 using identity
+keys.
+
+*** Flags:
+
+ - 0x80 - use extended authentication: this flag is only used in DATA and is ignored
+   in setup exchanges. It indicates that the packet is terminated by a 48-byte full
+   HMAC-SHA384 using the HMAC key derived from the session key. Enabling this slows
+   things down but significantly strengthens the authentication posture of the
+   protocol. It's generally only used if required for compliance.
+
+*** Anti-DPI Obfuscation:
+
+Obfuscation is applied to all session packets by AES encrypting a single block (ECB)
+starting at byte index 12 in each packet. This single block is then decrypted by
+the receiving end. The key for AES encryption is the first 32 bytes of the fingerprint
+of the receiving side's ZeroTier identity.
+
+This technically serves no purpose in terms of cryptographic security or authentication.
+Its purpose is to make it difficult for deep packet inspectors to easily detect ZeroTier
+traffic. For a DPI to correctly classify ZeroTier traffic it must know the identity of
+the recipient and perform one single AES decrypt per packet.
+
+Starting at byte index 12 randomizes this AES block even if other fields such as the
+session ID are the same, as this incorporates four bytes of the random IV or tag field.
+
+*** Credits:
+
+Designed by Adam Ierymenko with heavy influence from the Noise protocol specification by
+Trevor Perrin and the Wireguard VPN protocol by Jason Donenfeld.
+
+*/
+
+pub const SESSION_SETUP_PACKET_SIZE_MAX: usize = 1400;
+pub const SESSION_PACKET_SIZE_MIN: usize = 28;
+pub const SESSION_DATA_HEADER_SIZE: usize = 22;
+pub const SESSION_DATA_PAYLOAD_SIZE_MIN: usize = SESSION_PACKET_SIZE_MIN - SESSION_DATA_HEADER_SIZE;
+
+const FLAGS_TYPE_INDEX: usize = 20;
+const FLAGS_TYPE_TYPE_MASK: u8 = 0x0f;
+
+const MESSAGE_TYPE_DATA: u8 = 0x00;
+const MESSAGE_TYPE_INIT: u8 = 0x01;
+const MESSAGE_TYPE_ACK: u8 = 0x02;
+const MESSAGE_TYPE_CONFIRM: u8 = 0x03;
+const MESSAGE_TYPE_SINGLETON: u8 = 0x04;
+
+const MESSAGE_FLAGS_EXTENDED_AUTH: u8 = 0x80;
+
+const FIELD_DATA: u8 = 0x00;
+const FIELD_INITIATOR_IDENTITY: u8 = 0x01;
+const FIELD_EPHEMERAL_C25519: u8 = 0x02;
+const FIELD_EPHEMERAL_NISTP384: u8 = 0x03;
+const FIELD_EPHEMERAL_KYBER_PUBLIC: u8 = 0x04;
+const FIELD_EPHEMERAL_KYBER_ENCAPSULATED_SECRET: u8 = 0x05;
+const FIELD_CURRENT_SESSION_KEY_HASH: u8 = 0x06;
+const FIELD_TIMESTAMP: u8 = 0x07;
+const FIELD_TIMESTAMP_ECHO: u8 = 0x08;
+
+#[derive(Clone, Copy)]
+#[repr(C, packed)]
+struct InitAckSingletonHeader {
+    iv: [u8; 16],
+    session_id: u32,
+    flags_type: u8,
+    protocol_version: u8,
+    zero: [u8; 4],
+}
+
+#[derive(Clone, Copy)]
+#[repr(C, packed)]
+struct InitSingletonHeader {
+    h: InitAckSingletonHeader,
+    outer_ephemeral_field_id: u8,
+    outer_ephemeral: [u8; P384_PUBLIC_KEY_SIZE],
+}
+
+#[derive(Clone, Copy)]
+#[repr(C, packed)]
+struct ConfirmDataHeader {
+    tag: [u8; 16],
+    session_id: u32,
+    flags_type: u8,
+}
+
+struct InitiatorOfferedKeys {
+    p384: P384KeyPair,
+    kyber: Option<pqc_kyber::Keypair>,
+    ratchet_starting_key: Secret<64>,
+}
+
+struct Keys {
+    /// Keys offered by local node and sent to remote, generated by initiate().
+    local_offered: Option<Box<InitiatorOfferedKeys>>,
+
+    /// Key resulting from agreement between the outer (unencrypted) ephemeral sent with INIT and the recipient's static identity key.
+    setup_key: Option<Secret<32>>,
+
+    /// Final key ratcheted from previous or starting key via agreement between all matching ephemeral pairs.
+    session_key: Option<SymmetricSecret>,
+}
+
+#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64"))]
+#[inline(always)]
+fn zero_is_zero(z: &[u8; 4]) -> bool {
+    unsafe { *(z as *const [u8; 4]).cast::<u32>() == 0 }
+}
+
+#[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64")))]
+#[inline(always)]
+fn zero_is_zero(z: &[u8; 4]) -> bool {
+    u32::from_ne_bytes(*z) == 0
+}
+
+/// ZeroTier V2 forward-secure session
+///
+/// The current version always uses NIST P-384 as the outer ephemeral key and optionally
+/// Kyber for the internal ephemeral key. Curve25519 is supported if sent by the remote
+/// side though.
+///
+/// The RD template argument is used to specify a type to be attached to the session such
+/// as a ZeroTier peer.
+#[allow(unused)]
+pub(crate) struct Session<RD> {
+    /// Arbitrary object that may be attached by external code to this session (e.g. a peer).
+    pub related_data: RwLock<Option<RD>>,
+
+    /// Session keys of various types.
+    keys: RwLock<Keys>,
+
+    /// Timestamp when session was created.
+    creation_time: i64,
+
+    /// A random number added to sent timestamps to not reveal exact local tick counter.
+    latency_timestamp_delta: u32,
+
+    /// Number of times session key has been used to encrypt data.
+    encrypt_uses: AtomicU32,
+
+    /// Number of times session key has been used to decrypt data.
+    decrypt_uses: AtomicU32,
+
+    /// Most recent measured latency in milliseconds.
+    latency: AtomicU32,
+
+    /// Random session ID generated by initiator.
+    pub id: u32,
+}
+
+pub(crate) trait SessionContext<RD> {
+    /// Iterate through all sessions matching an ID until the supplied function returns false.
+    fn sessions_with_id<F: FnMut(&Session<RD>) -> bool>(&self, id: u32, f: F);
+}
+
+impl<RD> Session<RD> {
+    /// Create an initiator session and return it and the packet to be sent.
+    pub fn initiate(
+        local_identity: &Identity,
+        remote_identity: &Identity,
+        obfuscation_key: &Aes,
+        static_key: &SymmetricSecret,
+        current_session: Option<&Self>,
+        current_time: i64,
+    ) -> Option<(Self, Buffer<SESSION_SETUP_PACKET_SIZE_MAX>)> {
+        let mut packet: Buffer<SESSION_SETUP_PACKET_SIZE_MAX> = Buffer::new();
+
+        let mut id = random::next_u32_secure();
+        id |= (id == 0) as u32;
+
+        let ephemeral_p384 = P384KeyPair::generate();
+        {
+            let h: &mut InitSingletonHeader = packet.append_struct_get_mut().unwrap();
+            random::fill_bytes_secure(&mut h.h.iv);
+            h.h.session_id = id; // actually [u8; 4] so endian is irrelevant
+            h.h.flags_type = MESSAGE_FLAGS_EXTENDED_AUTH | MESSAGE_TYPE_INIT;
+            h.h.protocol_version = crate::vl1::protocol::PROTOCOL_VERSION;
+            h.outer_ephemeral_field_id = FIELD_EPHEMERAL_NISTP384;
+            h.outer_ephemeral = *ephemeral_p384.public_key_bytes();
+        }
+
+        assert!(packet.append_u8(FIELD_INITIATOR_IDENTITY).is_ok());
+        assert!(local_identity.marshal(&mut packet).is_ok());
+
+        let ephemeral_kyber = pqc_kyber::keypair(&mut random::SecureRandom::get());
+        assert!(packet.append_u8(FIELD_EPHEMERAL_KYBER_PUBLIC).is_ok());
+        assert!(packet.append_bytes_fixed(&ephemeral_kyber.public).is_ok());
+
+        let ratchet_starting_key = current_session
+            .and_then(|cs| {
+                cs.keys.read().session_key.as_ref().map(|cs_key| {
+                    assert!(packet.append_u8(FIELD_CURRENT_SESSION_KEY_HASH).is_ok());
+                    assert!(packet.append_bytes_fixed(&cs_key.key_hash).is_ok());
+                    zt_kbkdf_hmac_sha512(cs_key.key.as_bytes(), crate::vl1::protocol::security_constants::KBKDF_KEY_USAGE_LABEL_RATCHET_KEY)
+                })
+            })
+            .unwrap_or_else(|| zt_kbkdf_hmac_sha512(static_key.key.as_bytes(), crate::vl1::protocol::security_constants::KBKDF_KEY_USAGE_LABEL_RATCHET_KEY));
+
+        let latency_timestamp_delta = random::next_u32_secure();
+        assert!(packet.append_u8(FIELD_TIMESTAMP).is_ok());
+        assert!(packet.append_u64((current_time as u64).wrapping_add(latency_timestamp_delta as u64)).is_ok());
+
+        let setup_key;
+        if let Some(responder_p384) = remote_identity.p384.as_ref() {
+            if let Some(sk) = ephemeral_p384.agree(&responder_p384.ecdh) {
+                setup_key = Secret(SHA384::hash(sk.as_bytes())[..32].try_into().unwrap());
+                AesCtr::new(setup_key.as_bytes()).crypt_in_place(&mut packet.as_bytes_mut()[size_of::<InitSingletonHeader>()..]);
+            } else {
+                return None;
+            }
+        } else {
+            return None;
+        };
+
+        assert!(packet.append_bytes(&hmac_sha384(static_key.packet_hmac_key.as_bytes(), packet.as_bytes())).is_ok());
+
+        obfuscation_key.encrypt_block_in_place(&mut packet.as_bytes_mut()[12..28]);
+
+        return Some((
+            Self {
+                related_data: RwLock::new(None),
+                keys: RwLock::new(Keys {
+                    local_offered: Some(Box::new(InitiatorOfferedKeys {
+                        p384: ephemeral_p384,
+                        kyber: Some(ephemeral_kyber),
+                        ratchet_starting_key,
+                    })),
+                    setup_key: Some(setup_key),
+                    session_key: None,
+                }),
+                creation_time: current_time,
+                latency_timestamp_delta,
+                encrypt_uses: AtomicU32::new(0),
+                decrypt_uses: AtomicU32::new(0),
+                latency: AtomicU32::new(0),
+                id,
+            },
+            packet,
+        ));
+    }
+
+    pub fn receive<const L: usize, SC: SessionContext<RD>>(
+        local_identity: &Identity,
+        obfuscation_key: &Aes,
+        static_key: &SymmetricSecret,
+        current_time: i64,
+        sc: &SC,
+        packet: &mut Buffer<L>,
+    ) -> bool {
+        if packet.len() >= SESSION_PACKET_SIZE_MIN {
+            obfuscation_key.decrypt_block_in_place(&mut packet.as_bytes_mut()[12..28]);
+            let flags = packet.u8_at(FLAGS_TYPE_INDEX).unwrap();
+            let message_type = flags & FLAGS_TYPE_TYPE_MASK;
+            match message_type {
+                MESSAGE_TYPE_DATA | MESSAGE_TYPE_CONFIRM => if let Ok(header) = packet.struct_at::<ConfirmDataHeader>(0) {},
+
+                MESSAGE_TYPE_INIT | MESSAGE_TYPE_ACK | MESSAGE_TYPE_SINGLETON => {
+                    if let Ok(header) = packet.struct_at::<InitAckSingletonHeader>(0) {
+                        if zero_is_zero(&header.zero) {
+                            let (
+                                mut remote_identity,
+                                mut remote_offered_c25519,
+                                mut remote_offered_nistp384,
+                                mut remote_offered_kyber_public,
+                                mut remote_timestamp,
+                                mut remote_session_key_hash,
+                            ) = (None, None, None, None, -1, None);
+
+                            let mut cursor = size_of::<InitAckSingletonHeader>();
+                            loop {
+                                if let Ok(field_type) = packet.read_u8(&mut cursor) {
+                                    match field_type {
+                                        FIELD_DATA => {}
+                                        FIELD_INITIATOR_IDENTITY => {
+                                            if let Ok(id) = Identity::unmarshal(packet, &mut cursor) {
+                                                remote_identity = Some(id);
+                                            } else {
+                                                return false;
+                                            }
+                                        }
+                                        FIELD_EPHEMERAL_C25519 => {
+                                            if let Ok(k) = packet.read_bytes_fixed::<C25519_PUBLIC_KEY_SIZE>(&mut cursor) {
+                                                remote_offered_c25519 = Some(k);
+                                            } else {
+                                                return false;
+                                            }
+                                        }
+                                        FIELD_EPHEMERAL_NISTP384 => {
+                                            if let Ok(k) = packet.read_bytes_fixed::<P384_PUBLIC_KEY_SIZE>(&mut cursor) {
+                                                remote_offered_nistp384 = Some(k);
+                                            } else {
+                                                return false;
+                                            }
+                                        }
+                                        FIELD_EPHEMERAL_KYBER_PUBLIC => {
+                                            if let Ok(k) = packet.read_bytes_fixed::<{ pqc_kyber::KYBER_PUBLICKEYBYTES }>(&mut cursor) {
+                                                remote_offered_kyber_public = Some(k);
+                                            } else {
+                                                return false;
+                                            }
+                                        }
+                                        FIELD_EPHEMERAL_KYBER_ENCAPSULATED_SECRET => {}
+                                        FIELD_CURRENT_SESSION_KEY_HASH => {
+                                            if let Ok(k) = packet.read_bytes_fixed::<16>(&mut cursor) {
+                                                remote_session_key_hash = Some(k);
+                                            } else {
+                                                return false;
+                                            }
+                                        }
+                                        FIELD_TIMESTAMP => {
+                                            if let Ok(ts) = packet.read_varint(&mut cursor) {
+                                                remote_timestamp = ts as i64;
+                                            } else {
+                                                return false;
+                                            }
+                                        }
+                                        FIELD_TIMESTAMP_ECHO => {
+                                            if let Ok(ts) = packet.read_varint(&mut cursor) {
+                                            } else {
+                                                return false;
+                                            }
+                                        }
+                                        _ => {}
+                                    }
+
+                                    if message_type == MESSAGE_TYPE_INIT {}
+                                } else {
+                                    break;
+                                }
+                            }
+                        }
+                    }
+                }
+
+                _ => {}
+            }
+        }
+        return false;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn sizing() {
+        assert_eq!(size_of::<InitAckSingletonHeader>(), 26);
+        assert_eq!(size_of::<InitSingletonHeader>(), 26 + 1 + P384_PUBLIC_KEY_SIZE);
+        assert_eq!(size_of::<ConfirmDataHeader>(), 21);
+    }
+}

zerotier-network-hypervisor/src/vl1/symmetricsecret.rs

@@ -1,8 +1,7 @@
 // (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md.
 
-use std::sync::atomic::AtomicUsize;
-
 use zerotier_core_crypto::aes_gmac_siv::AesGmacSiv;
+use zerotier_core_crypto::hash::SHA384;
 use zerotier_core_crypto::kbkdf::*;
 use zerotier_core_crypto::secret::Secret;
 
@@ -14,7 +13,10 @@ use crate::vl1::protocol::*;
 /// This contains the key and several sub-keys and ciphers keyed with sub-keys.
 pub(crate) struct SymmetricSecret {
     /// Master key from which other keys are derived.
-    pub key: Secret<48>,
+    pub key: Secret<64>,
+
+    /// First 16 bytes of SHA384(key), used to identify sessions for ratcheting.
+    pub key_hash: [u8; 16],
 
     /// Key for private fields in HELLO packets.
     pub hello_private_section_key: Secret<48>,
@@ -28,12 +30,15 @@ pub(crate) struct SymmetricSecret {
 
 impl SymmetricSecret {
     /// Create a new symmetric secret, deriving all sub-keys and such.
-    pub fn new(key: Secret<48>) -> SymmetricSecret {
-        let hello_private_section_key = zt_kbkdf_hmac_sha384(&key.0, security_constants::KBKDF_KEY_USAGE_LABEL_HELLO_PRIVATE_SECTION);
-        let packet_hmac_key = zt_kbkdf_hmac_sha384(&key.0, security_constants::KBKDF_KEY_USAGE_LABEL_PACKET_HMAC);
-        let aes_factory = AesGmacSivPoolFactory(zt_kbkdf_hmac_sha384(&key.0, security_constants::KBKDF_KEY_USAGE_LABEL_AES_GMAC_SIV_K0).first_n(), zt_kbkdf_hmac_sha384(&key.0[..48], security_constants::KBKDF_KEY_USAGE_LABEL_AES_GMAC_SIV_K1).first_n());
+    pub fn new(key: Secret<64>) -> SymmetricSecret {
+        let hello_private_section_key = zt_kbkdf_hmac_sha384(&key.0[..48], security_constants::KBKDF_KEY_USAGE_LABEL_HELLO_PRIVATE_SECTION);
+        let packet_hmac_key = zt_kbkdf_hmac_sha384(&key.0[..48], security_constants::KBKDF_KEY_USAGE_LABEL_PACKET_HMAC);
+        let aes_factory =
+            AesGmacSivPoolFactory(zt_kbkdf_hmac_sha384(&key.0[..48], security_constants::KBKDF_KEY_USAGE_LABEL_AES_GMAC_SIV_K0).first_n(), zt_kbkdf_hmac_sha384(&key.0[..48], security_constants::KBKDF_KEY_USAGE_LABEL_AES_GMAC_SIV_K1).first_n());
+        let key_hash = SHA384::hash(key.as_bytes())[..16].try_into().unwrap();
         SymmetricSecret {
             key,
+            key_hash,
             hello_private_section_key,
             packet_hmac_key,
             aes_gmac_siv: Pool::new(2, aes_factory),
@@ -41,28 +46,6 @@ impl SymmetricSecret {
     }
 }
 
-/// An ephemeral symmetric secret with usage timers and counters.
-#[allow(unused)]
-pub(crate) struct EphemeralSymmetricSecret {
-    pub secret: SymmetricSecret,
-    pub key_hash: [u8; 16],
-    pub create_time_ticks: i64,
-    pub encrypt_uses: AtomicUsize,
-}
-
-impl EphemeralSymmetricSecret {
-    #[allow(unused)]
-    pub fn new(key: Secret<48>, create_time_ticks: i64) -> EphemeralSymmetricSecret {
-        let key_hash: [u8; 16] = zt_kbkdf_hmac_sha384(key.as_bytes(), security_constants::KBKDF_KEY_USAGE_LABEL_EPHEMERAL_KEY_ID).0[0..16].try_into().unwrap();
-        Self {
-            secret: SymmetricSecret::new(key),
-            key_hash,
-            create_time_ticks,
-            encrypt_uses: AtomicUsize::new(0),
-        }
-    }
-}
-
 pub(crate) struct AesGmacSivPoolFactory(Secret<32>, Secret<32>);
 
 impl PoolFactory<AesGmacSiv> for AesGmacSivPoolFactory {