Dictionary, and cleanup and other random stuff.

network-hypervisor/src/util/hex.rs

@@ -12,37 +12,59 @@ pub fn to_string(b: &[u8]) -> String {
     s
 }
 
+/// Encode an unsigned 64-bit value as a string.
+pub fn to_string_u64(mut i: u64, skip_leading_zeroes: bool) -> String {
+    let mut s = String::new();
+    s.reserve(16);
+    for _ in 0..16 {
+        let ii = i >> 60;
+        if ii != 0 || !s.is_empty() || !skip_leading_zeroes {
+            s.push(HEX_CHARS[ii as usize] as char);
+        }
+        i = i.wrapping_shl(4);
+    }
+    s
+}
+
+/// Encode an unsigned 64-bit value as a string.
+pub fn to_vec_u64(mut i: u64, skip_leading_zeroes: bool) -> Vec<u8> {
+    let mut s = Vec::new();
+    s.reserve(16);
+    for _ in 0..16 {
+        let ii = i >> 60;
+        if ii != 0 || !s.is_empty() || !skip_leading_zeroes {
+            s.push(HEX_CHARS[ii as usize]);
+        }
+        i = i.wrapping_shl(4);
+    }
+    s
+}
+
 /// Decode a hex string, ignoring non-hexadecimal characters.
 pub fn from_string(s: &str) -> Vec<u8> {
     let mut b: Vec<u8> = Vec::new();
     b.reserve((s.len() / 2) + 1);
 
-    let mut byte = 0;
+    let mut byte = 0_u8;
     let mut have_8: bool = false;
     for cc in s.as_bytes() {
         let c = *cc;
         if c >= 48 && c <= 57 {
-            byte <<= 4;
-            byte |= c - 48;
+            byte = (byte.wrapping_shl(4)) | (c - 48);
             if have_8 {
                 b.push(byte);
-                byte = 0;
             }
             have_8 = !have_8;
         } else if c >= 65 && c <= 70 {
-            byte <<= 4;
-            byte |= c - 55;
+            byte = (byte.wrapping_shl(4)) | (c - 55);
             if have_8 {
                 b.push(byte);
-                byte = 0;
             }
             have_8 = !have_8;
         } else if c >= 97 && c <= 102 {
-            byte <<= 4;
-            byte |= c - 87;
+            byte = (byte.wrapping_shl(4)) | (c - 87);
             if have_8 {
                 b.push(byte);
-                byte = 0;
             }
             have_8 = !have_8;
         }

network-hypervisor/src/util/mod.rs

@@ -1,5 +1,8 @@
 pub mod hex;
 
+pub(crate) const ZEROES: [u8; 64] = [0_u8; 64];
+
+#[inline(always)]
 pub(crate) unsafe fn equal_bytes(a: *const u8, b: *const u8, l: usize) -> bool {
     for i in 0..l {
         if *a.offset(i as isize) != *b.offset(i as isize) {
@@ -49,5 +52,3 @@ pub(crate) fn integer_load_be_u32(d: &[u8]) -> u32 {
 pub(crate) fn integer_load_be_u64(d: &[u8]) -> u64 {
     (d[0] as u64) << 56 | (d[1] as u64) << 48 | (d[2] as u64) << 40 | (d[3] as u64) << 32 | (d[4] as u64) << 24 | (d[5] as u64) << 16 | (d[6] as u64) << 8 | (d[7] as u64)
 }
-
-pub(crate) const ZEROES: [u8; 64] = [0_u8; 64];

network-hypervisor/src/vl1/address.rs

@@ -6,7 +6,6 @@ use crate::error::InvalidFormatError;
 use crate::util::hex::HEX_CHARS;
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-#[repr(transparent)]
 pub struct Address(u64);
 
 impl Address {

network-hypervisor/src/vl1/buffer.rs

@@ -1,7 +1,6 @@
 use std::mem::{size_of, MaybeUninit, transmute};
 use std::marker::PhantomData;
 use std::io::Write;
-use std::hash::{Hash, Hasher};
 
 const OVERFLOW_ERR_MSG: &'static str = "overflow";
 
@@ -30,7 +29,7 @@ unsafe impl<H: RawObject, const L: usize> RawObject for Buffer<H, L> {}
 impl<H: RawObject, const L: usize> Default for Buffer<H, L> {
     #[inline(always)]
     fn default() -> Self {
-        assert!(size_of::<H>() <= L);
+        debug_assert!(size_of::<H>() <= L);
         Buffer(size_of::<H>(), [0_u8; L], PhantomData::default())
     }
 }
@@ -38,31 +37,30 @@ impl<H: RawObject, const L: usize> Default for Buffer<H, L> {
 impl<H: RawObject, const L: usize> Buffer<H, L> {
     #[inline(always)]
     pub fn new() -> Self {
-        assert!(size_of::<H>() <= L);
+        debug_assert!(size_of::<H>() <= L);
         Self::default()
     }
 
     /// Change the header "personality" of this buffer.
-    /// Note that the new buffer must be of the same size and both the old and
-    /// new headers must be RawObjects with size less than or equal to this size.
-    pub fn change_header_type<NH: RawObject>(self) -> Buffer<NH, L> {
-        assert!(size_of::<NH>() <= L);
-        unsafe { transmute(self) }
+    /// This is a free operation, but the returned reference obviously only lives as long as the source.
+    #[inline(always)]
+    pub fn transmute_header<NH: RawObject>(&self) -> &Buffer<NH, L> {
+        debug_assert!(size_of::<H>() <= L);
+        debug_assert!(size_of::<NH>() <= L);
+        unsafe {
+            &*(self as *const Self).cast::<Buffer<NH, L>>()
+        }
     }
 
-    /// Create a buffer that contains a copy of a slice.
-    /// If the slice is larger than the maximum size L of the buffer, only the first L bytes
-    /// are copied and the rest is ignored.
+    /// Create a buffer that contains a copy of a slice, truncating if the slice is too long.
     #[inline(always)]
-    pub fn from_bytes_truncate(b: &[u8]) -> Self {
+    pub fn from_bytes_lossy(b: &[u8]) -> Self {
         let l = b.len().min(L);
-        unsafe {
-            let mut tmp = MaybeUninit::<Self>::uninit().assume_init();
-            tmp.0 = l;
-            tmp.1[0..l].copy_from_slice(b);
-            tmp.1[l..L].fill(0);
-            tmp
-        }
+        let mut tmp = unsafe { MaybeUninit::<Self>::uninit().assume_init() };
+        tmp.0 = l;
+        tmp.1[0..l].copy_from_slice(b);
+        tmp.1[l..L].fill(0);
+        tmp
     }
 
     /// Get a slice containing the entire buffer in raw form including the header.
@@ -239,7 +237,7 @@ impl<H: RawObject, const L: usize> Buffer<H, L> {
     /// Get the index of the start of the payload after the header.
     #[inline(always)]
     pub fn cursor_after_header(&self) -> usize {
-        size_of::<usize>()
+        size_of::<H>()
     }
 
     /// Get a structure at a given position in the buffer and advance the cursor.
@@ -386,14 +384,6 @@ impl<H: RawObject, const L: usize> AsMut<H> for Buffer<H, L> {
     }
 }
 
-impl<H: RawObject, const L: usize> Hash for Buffer<H, L> {
-    #[inline(always)]
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        state.write_u32(self.0 as u32);
-        state.write(&self.1[0..self.0]);
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use std::mem::size_of;

network-hypervisor/src/vl1/dictionary.rs

@@ -0,0 +1,212 @@
+use std::collections::BTreeMap;
+use std::io::Write;
+use crate::util::hex::HEX_CHARS;
+
+/// Dictionary is an extremely simple key=value serialization format.
+/// It's designed for extreme parsing simplicity and is human readable if keys and values are strings.
+/// It also supports binary keys and values which will be minimally escaped but render the result not
+/// entirely human readable. Keys are serialized in natural sort order so the result can be consistently
+/// checksummed or hashed.
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub struct Dictionary(BTreeMap<String, Vec<u8>>);
+
+impl Default for Dictionary {
+    #[inline(always)]
+    fn default() -> Self {
+        Self(BTreeMap::new())
+    }
+}
+
+fn write_escaped<W: Write>(b: &[u8], w: &mut W) -> std::io::Result<()> {
+    let mut i = 0_usize;
+    let l = b.len();
+    while i < l {
+        let ii = i + 1;
+        match unsafe { b.get_unchecked(i) } {
+            0 => { w.write_all(&[b'\\', b'0'])?; }
+            b'\n' => { w.write_all(&[b'\\', b'n'])?; }
+            b'\r' => { w.write_all(&[b'\\', b'r'])?; }
+            b'=' => { w.write_all(&[b'\\', b'e'])?; }
+            b'\\' => { w.write_all(&[b'\\', b'\\'])?; }
+            _ => { w.write_all(&b[i..ii])?; }
+        }
+        i = ii;
+    }
+    Ok(())
+}
+
+fn append_printable(s: &mut String, b: &[u8]) {
+    for c in b {
+        let c = *c as char;
+        if c.is_alphanumeric() || c.is_whitespace() {
+            s.push(c);
+        } else {
+            s.push('\\');
+            s.push('x');
+            s.push(HEX_CHARS[((c as u8) >> 4) as usize] as char);
+            s.push(HEX_CHARS[((c as u8) & 0xf) as usize] as char);
+        }
+    }
+}
+
+impl Dictionary {
+    #[inline(always)]
+    pub fn new() -> Self {
+        Self(BTreeMap::new())
+    }
+
+    #[inline(always)]
+    pub fn clear(&mut self) {
+        self.0.clear()
+    }
+
+    #[inline(always)]
+    pub fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    pub fn get_str(&self, k: &str) -> Option<&str> {
+        self.0.get(k).map_or(None, |v| std::str::from_utf8(v.as_slice()).map_or(None, |s| Some(s)))
+    }
+
+    pub fn get_bytes(&self, k: &str) -> Option<&[u8]> {
+        self.0.get(k).map_or(None, |v| Some(v.as_slice()))
+    }
+
+    pub fn get_u64(&self, k: &str) -> Option<u64> {
+        self.get_str(k).map_or(None, |s| u64::from_str_radix(s, 16).map_or(None, |i| Some(i)))
+    }
+
+    pub fn get_i64(&self, k: &str) -> Option<i64> {
+        self.get_str(k).map_or(None, |s| i64::from_str_radix(s, 16).map_or(None, |i| Some(i)))
+    }
+
+    pub fn get_bool(&self, k: &str) -> Option<bool> {
+        self.0.get(k).map_or(None, |v| {
+            if v.is_empty() {
+                Some(false)
+            } else {
+                Some(match unsafe { v.get_unchecked(0) } {
+                    b'1' | b't' | b'T' | b'y' | b'Y' => true,
+                    _ => false
+                })
+            }
+        })
+    }
+
+    #[inline(always)]
+    pub fn set_str(&mut self, k: &str, v: &str) {
+        let _ = self.0.insert(String::from(k), v.as_bytes().to_vec());
+    }
+
+    #[inline(always)]
+    pub fn set_u64(&mut self, k: &str, v: u64) {
+        let _ = self.0.insert(String::from(k), crate::util::hex::to_vec_u64(v, true));
+    }
+
+    #[inline(always)]
+    pub fn set_bytes(&mut self, k: &str, v: Vec<u8>) {
+        let _ = self.0.insert(String::from(k), v);
+    }
+
+    #[inline(always)]
+    pub fn set_bool(&mut self, k: &str, v: bool) {
+        let _ = self.0.insert(String::from(k), (if v { [b'1'] } else { [b'0'] }).to_vec());
+    }
+
+    /// Write a dictionary in transport format to a writer.
+    pub fn write_to<W: Write>(&self, w: &mut W) -> std::io::Result<()> {
+        for kv in self.0.iter() {
+            write_escaped(kv.0.as_bytes(), w)?;
+            w.write_all(&[b'='])?;
+            write_escaped(kv.1.as_slice(), w)?;
+            w.write_all(&[b'\n'])?;
+        }
+        Ok(())
+    }
+
+    /// Write a dictionary in transport format to a byte vector.
+    pub fn to_bytes(&self) -> Vec<u8> {
+        let mut b: Vec<u8> = Vec::new();
+        b.reserve(32 * self.0.len());
+        let _ = self.write_to(&mut b);
+        b
+    }
+
+    /// Decode a dictionary in byte format, or return None if the input is invalid.
+    pub fn from_bytes(b: &[u8]) -> Option<Dictionary> {
+        let mut d = Dictionary::new();
+        let mut kv: [Vec<u8>; 2] = [Vec::new(), Vec::new()];
+        let mut state = 0;
+        let mut escape = false;
+        for c in b {
+            let c = *c;
+            if escape {
+                escape = false;
+                kv[state].push(match c {
+                    b'0' => 0,
+                    b'n' => b'\n',
+                    b'r' => b'\r',
+                    _ => c // =, \, and escapes before other characters are unnecessary but not errors
+                });
+            } else if c == b'\\' {
+                escape = true;
+            } else if c == b'=' {
+                if state != 0 {
+                    return None;
+                }
+                state = 1;
+            } else if c == b'\n' {
+                if state != 1 {
+                    return None;
+                }
+                state = 0;
+                if !kv[0].is_empty() {
+                    if String::from_utf8(kv[0].clone()).map_or(true, |key| {
+                        d.0.insert(key, kv[1].clone());
+                        false
+                    }) {
+                        return None;
+                    }
+                }
+                kv[0].clear();
+                kv[1].clear();
+            } else if c != b'\r' {
+                kv[state].push(c);
+            }
+        }
+        Some(d)
+    }
+}
+
+impl ToString for Dictionary {
+    /// Get the dictionary in an always readable format with non-printable characters replaced by '\xXX'.
+    fn to_string(&self) -> String {
+        let mut s = String::new();
+        for kv in self.0.iter() {
+            append_printable(&mut s, kv.0.as_bytes());
+            s.push('=');
+            append_printable(&mut s, kv.1.as_slice());
+            s.push('\n');
+        }
+        s
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::vl1::dictionary::Dictionary;
+
+    #[test]
+    fn dictionary() {
+        let mut d = Dictionary::new();
+        d.set_str("foo", "bar");
+        d.set_u64("bar", 0xfeedcafebabebeef);
+        d.set_bytes("baz", vec![1,2,3,4,5,6,7,8,9]);
+        d.set_bool("lala", true);
+        d.set_bool("haha", false);
+        let bytes = d.to_bytes();
+        let d2 = Dictionary::from_bytes(bytes.as_slice()).unwrap();
+        assert!(d.eq(&d2));
+    }
+}

network-hypervisor/src/vl1/endpoint.rs

@@ -1,6 +1,18 @@
 use crate::vl1::{Address, MAC};
 use crate::vl1::inetaddress::InetAddress;
 use crate::vl1::buffer::{RawObject, Buffer};
+use std::hash::{Hash, Hasher};
+
+const TYPE_NIL: u8 = 0;
+const TYPE_ZEROTIER: u8 = 1;
+const TYPE_ETHERNET: u8 = 2;
+const TYPE_WIFIDIRECT: u8 = 3;
+const TYPE_BLUETOOTH: u8 = 4;
+const TYPE_IP: u8 = 5;
+const TYPE_IPUDP: u8 = 6;
+const TYPE_IPTCP: u8 = 7;
+const TYPE_HTTP: u8 = 8;
+const TYPE_WEBRTC: u8 = 9;
 
 #[repr(u8)]
 pub enum Type {
@@ -16,7 +28,7 @@ pub enum Type {
     WebRTC = 9,
 }
 
-#[derive(Clone, PartialEq, Eq)]
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)]
 pub enum Endpoint {
     Nil,
     ZeroTier(Address),
@@ -120,23 +132,70 @@ impl Endpoint {
                 Ok(Endpoint::IpUdp(ip))
             }
         } else {
-            match (type_byte - 16) as Type {
-                Type::Nil => Ok(Endpoint::Nil),
-                Type::ZeroTier => Ok(Endpoint::ZeroTier(Address::from(buf.get_bytes_fixed(cursor)?))),
-                Type::Ethernet => Ok(Endpoint::Ethernet(MAC::from(buf.get_bytes_fixed(cursor)?))),
-                Type::WifiDirect => Ok(Endpoint::WifiDirect(MAC::from(buf.get_bytes_fixed(cursor)?))),
-                Type::Bluetooth => Ok(Endpoint::Bluetooth(MAC::from(buf.get_bytes_fixed(cursor)?))),
-                Type::Ip => Ok(Endpoint::Ip(InetAddress::unmarshal(buf, cursor)?)),
-                Type::IpUdp => Ok(Endpoint::IpUdp(InetAddress::unmarshal(buf, cursor)?)),
-                Type::IpTcp => Ok(Endpoint::IpTcp(InetAddress::unmarshal(buf, cursor)?)),
-                Type::Http => {
+            match type_byte - 16 {
+                TYPE_NIL => Ok(Endpoint::Nil),
+                TYPE_ZEROTIER => Ok(Endpoint::ZeroTier(Address::from(buf.get_bytes_fixed(cursor)?))),
+                TYPE_ETHERNET => Ok(Endpoint::Ethernet(MAC::from(buf.get_bytes_fixed(cursor)?))),
+                TYPE_WIFIDIRECT => Ok(Endpoint::WifiDirect(MAC::from(buf.get_bytes_fixed(cursor)?))),
+                TYPE_BLUETOOTH => Ok(Endpoint::Bluetooth(MAC::from(buf.get_bytes_fixed(cursor)?))),
+                TYPE_IP => Ok(Endpoint::Ip(InetAddress::unmarshal(buf, cursor)?)),
+                TYPE_IPUDP => Ok(Endpoint::IpUdp(InetAddress::unmarshal(buf, cursor)?)),
+                TYPE_IPTCP => Ok(Endpoint::IpTcp(InetAddress::unmarshal(buf, cursor)?)),
+                TYPE_HTTP => {
                     let l = buf.get_u16(cursor)?;
                     Ok(Endpoint::Http(String::from_utf8_lossy(buf.get_bytes(l as usize, cursor)?).to_string()))
                 }
-                Type::WebRTC => {
+                TYPE_WEBRTC => {
                     let l = buf.get_u16(cursor)?;
                     Ok(Endpoint::WebRTC(String::from_utf8_lossy(buf.get_bytes(l as usize, cursor)?).to_string()))
                 }
+                _ => std::io::Result::Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "unrecognized endpoint type in stream"))
+            }
+        }
+    }
+}
+
+impl Hash for Endpoint {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        match self {
+            Endpoint::Nil => {
+                state.write_u8(Type::Nil as u8);
+            },
+            Endpoint::ZeroTier(a) => {
+                state.write_u8(Type::ZeroTier as u8);
+                state.write_u64(a.to_u64())
+            },
+            Endpoint::Ethernet(m) => {
+                state.write_u8(Type::Ethernet as u8);
+                state.write_u64(m.to_u64())
+            },
+            Endpoint::WifiDirect(m) => {
+                state.write_u8(Type::WifiDirect as u8);
+                state.write_u64(m.to_u64())
+            },
+            Endpoint::Bluetooth(m) => {
+                state.write_u8(Type::Bluetooth as u8);
+                state.write_u64(m.to_u64())
+            },
+            Endpoint::Ip(ip) => {
+                state.write_u8(Type::Ip as u8);
+                ip.hash(state);
+            },
+            Endpoint::IpUdp(ip) => {
+                state.write_u8(Type::IpUdp as u8);
+                ip.hash(state);
+            },
+            Endpoint::IpTcp(ip) => {
+                state.write_u8(Type::IpTcp as u8);
+                ip.hash(state);
+            },
+            Endpoint::Http(url) => {
+                state.write_u8(Type::Http as u8);
+                url.hash(state);
+            },
+            Endpoint::WebRTC(offer) => {
+                state.write_u8(Type::WebRTC as u8);
+                offer.hash(state);
             }
         }
     }
@@ -148,13 +207,13 @@ impl ToString for Endpoint {
             Endpoint::Nil => format!("nil"),
             Endpoint::ZeroTier(a) => format!("zt:{}", a.to_string()),
             Endpoint::Ethernet(m) => format!("eth:{}", m.to_string()),
-            Endpoint::WifiDirect(m) => format!("wifid:{}", m.to_string()),
+            Endpoint::WifiDirect(m) => format!("wifip2p:{}", m.to_string()),
             Endpoint::Bluetooth(m) => format!("bt:{}", m.to_string()),
             Endpoint::Ip(ip) => format!("ip:{}", ip.to_ip_string()),
             Endpoint::IpUdp(ip) => format!("udp:{}", ip.to_string()),
             Endpoint::IpTcp(ip) => format!("tcp:{}", ip.to_string()),
-            Endpoint::Http(url) => url,
-            Endpoint::WebRTC(offer) => format!("webrtc:offer:{}", urlencoding::encode(offer.as_str())),
+            Endpoint::Http(url) => url.clone(),
+            Endpoint::WebRTC(offer) => format!("webrtc:{}", urlencoding::encode(offer.as_str())),
         }
     }
 }

network-hypervisor/src/vl1/identity.rs

@@ -422,7 +422,7 @@ impl Identity {
     /// On success the identity and the number of bytes actually read from the slice are
     /// returned.
     pub fn unmarshal_from_bytes(bytes: &[u8]) -> std::io::Result<(Identity, usize)> {
-        let buf = Buffer::<NoHeader, 2048>::from_bytes_truncate(bytes);
+        let buf = Buffer::<NoHeader, 2048>::from_bytes_lossy(bytes);
         let mut cursor: usize = 0;
         let id = Self::unmarshal(&buf, &mut cursor)?;
         Ok((id, cursor))

network-hypervisor/src/vl1/mac.rs

@@ -4,7 +4,6 @@ use std::hash::{Hash, Hasher};
 use crate::error::InvalidFormatError;
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-#[repr(transparent)]
 pub struct MAC(u64);
 
 impl MAC {

network-hypervisor/src/vl1/mod.rs

@@ -1,10 +1,12 @@
 pub(crate) mod protocol;
 pub(crate) mod packet;
 pub(crate) mod buffer;
+pub(crate) mod node;
+
+pub mod dictionary;
 pub mod identity;
 pub mod inetaddress;
 pub mod endpoint;
-pub(crate) mod node;
 
 mod address;
 mod mac;

network-hypervisor/src/vl1/packet.rs

@@ -3,9 +3,6 @@ use std::ops::Not;
 
 type PacketID = u64;
 
-/// A packet buffer, but with no header "personality."
-pub type Buffer = crate::vl1::buffer::Buffer<crate::vl1::buffer::NoHeader, { crate::vl1::protocol::PACKET_SIZE_MAX }>;
-
 #[derive(Clone)]
 #[repr(packed)]
 pub struct Header {