From 07e1923cc7c3fcddb8d0031308e62054e3059ac6 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 29 Mar 2023 18:21:29 -0400 Subject: [PATCH] Implement base62 and replace in Identity etc. --- {utils/src => attic}/proquint.rs | 0 network-hypervisor/src/vl1/address.rs | 17 ++- network-hypervisor/src/vl1/identity.rs | 54 +++----- network-hypervisor/src/vl1/node.rs | 2 +- utils/src/base62.rs | 181 +++++++++++++++++++++++++ utils/src/base64.rs | 115 ---------------- utils/src/basex.rs | 62 --------- utils/src/lib.rs | 8 +- 8 files changed, 225 insertions(+), 214 deletions(-) rename {utils/src => attic}/proquint.rs (100%) create mode 100644 utils/src/base62.rs delete mode 100644 utils/src/base64.rs delete mode 100644 utils/src/basex.rs diff --git a/utils/src/proquint.rs b/attic/proquint.rs similarity index 100% rename from utils/src/proquint.rs rename to attic/proquint.rs diff --git a/network-hypervisor/src/vl1/address.rs b/network-hypervisor/src/vl1/address.rs index 7aca0d4fb..e72ec5f9d 100644 --- a/network-hypervisor/src/vl1/address.rs +++ b/network-hypervisor/src/vl1/address.rs @@ -360,7 +360,7 @@ impl PartialAddress { /// This returns None if there is no match or if this partial matches more than one entry, in which /// case it's ambiguous and may be unsafe to use. This should be prohibited at other levels of the /// system but is checked for here as well. - #[inline] + #[inline(always)] pub fn find_unique_match_mut<'a, T>(&self, map: &'a mut BTreeMap) -> Option<&'a mut T> { // This not only saves some repetition but is in fact the only way to easily do this. The same code as // find_unique_match() but with range_mut() doesn't compile because the second range_mut() would @@ -477,3 +477,18 @@ impl<'de> Deserialize<'de> for PartialAddress { } } } + +#[cfg(test)] +mod tests { + use super::*; + use zerotier_crypto::random; + + #[test] + fn to_from_string() { + for _ in 0..64 { + let mut tmp = Address::new_uninitialized(); + random::fill_bytes_secure(&mut tmp.0); + println!("{}", tmp.to_string()); + } + } +} diff --git a/network-hypervisor/src/vl1/identity.rs b/network-hypervisor/src/vl1/identity.rs index 4c0b8fb37..4107db8f7 100644 --- a/network-hypervisor/src/vl1/identity.rs +++ b/network-hypervisor/src/vl1/identity.rs @@ -17,7 +17,7 @@ use zerotier_crypto::secret::Secret; use zerotier_crypto::typestate::Valid; use zerotier_crypto::x25519::*; use zerotier_utils::arrayvec::ArrayVec; -use zerotier_utils::base64; +use zerotier_utils::base62; use zerotier_utils::buffer::{Buffer, OutOfBoundsError}; use zerotier_utils::error::InvalidFormatError; use zerotier_utils::marshalable::{Marshalable, UnmarshalError}; @@ -284,17 +284,17 @@ impl ToString for Identity { let mut s = String::with_capacity(1024); s.push_str(self.address.to_string().as_str()); s.push_str(":1:"); - base64::encode_into(&self.x25519.ecdh, &mut s); + base62::encode_into(&self.x25519.ecdh, &mut s, 0); s.push(':'); - base64::encode_into(&self.x25519.eddsa, &mut s); + base62::encode_into(&self.x25519.eddsa, &mut s, 0); s.push(':'); - base64::encode_into(p384.ecdh.as_bytes(), &mut s); + base62::encode_into(p384.ecdh.as_bytes(), &mut s, 0); s.push(':'); - base64::encode_into(p384.ecdsa.as_bytes(), &mut s); + base62::encode_into(p384.ecdsa.as_bytes(), &mut s, 0); s.push(':'); - base64::encode_into(&p384.ed25519_self_signature, &mut s); + base62::encode_into(&p384.ed25519_self_signature, &mut s, 0); s.push(':'); - base64::encode_into(&p384.p384_self_signature, &mut s); + base62::encode_into(&p384.p384_self_signature, &mut s, 0); s } else { format!( @@ -317,28 +317,16 @@ impl FromStr for Identity { return Ok(Self { address: Address::from_str(ss[0]).map_err(|_| InvalidFormatError)?, x25519: X25519 { - ecdh: base64::decode(ss[2].as_bytes()) - .map_err(|_| InvalidFormatError)? - .try_into() - .map_err(|_| InvalidFormatError)?, - eddsa: base64::decode(ss[3].as_bytes()) - .map_err(|_| InvalidFormatError)? - .try_into() - .map_err(|_| InvalidFormatError)?, + ecdh: base62::decode(ss[2].as_bytes()).ok_or(InvalidFormatError)?, + eddsa: base62::decode(ss[3].as_bytes()).ok_or(InvalidFormatError)?, }, p384: Some(P384 { - ecdh: P384PublicKey::from_bytes(base64::decode(ss[4].as_bytes()).map_err(|_| InvalidFormatError)?.as_slice()) + ecdh: P384PublicKey::from_bytes(&base62::decode::(ss[4].as_bytes()).ok_or(InvalidFormatError)?) .ok_or(InvalidFormatError)?, - ecdsa: P384PublicKey::from_bytes(base64::decode(ss[5].as_bytes()).map_err(|_| InvalidFormatError)?.as_slice()) + ecdsa: P384PublicKey::from_bytes(&base62::decode::(ss[5].as_bytes()).ok_or(InvalidFormatError)?) .ok_or(InvalidFormatError)?, - ed25519_self_signature: base64::decode(ss[6].as_bytes()) - .map_err(|_| InvalidFormatError)? - .try_into() - .map_err(|_| InvalidFormatError)?, - p384_self_signature: base64::decode(ss[7].as_bytes()) - .map_err(|_| InvalidFormatError)? - .try_into() - .map_err(|_| InvalidFormatError)?, + ed25519_self_signature: base62::decode(ss[6].as_bytes()).ok_or(InvalidFormatError)?, + p384_self_signature: base62::decode(ss[7].as_bytes()).ok_or(InvalidFormatError)?, }), }); } else if ss[1] == "0" && ss.len() >= 3 { @@ -495,13 +483,13 @@ impl ToString for IdentitySecret { let mut s = self.public.to_string(); if let Some(p384) = self.p384.as_ref() { s.push(':'); - base64::encode_into(self.x25519.ecdh.secret_bytes().as_bytes(), &mut s); + base62::encode_into(self.x25519.ecdh.secret_bytes().as_bytes(), &mut s, 0); s.push(':'); - base64::encode_into(self.x25519.eddsa.secret_bytes().as_bytes(), &mut s); + base62::encode_into(self.x25519.eddsa.secret_bytes().as_bytes(), &mut s, 0); s.push(':'); - base64::encode_into(p384.ecdh.secret_key_bytes().as_bytes(), &mut s); + base62::encode_into(p384.ecdh.secret_key_bytes().as_bytes(), &mut s, 0); s.push(':'); - base64::encode_into(p384.ecdsa.secret_key_bytes().as_bytes(), &mut s); + base62::encode_into(p384.ecdsa.secret_key_bytes().as_bytes(), &mut s, 0); } else { s.push(':'); s.push_str(hex::to_string(self.x25519.ecdh.secret_bytes().as_bytes()).as_str()); @@ -521,22 +509,22 @@ impl FromStr for IdentitySecret { if ss[1] == "1" && ss.len() >= 12 && public.p384.is_some() { let x25519_ecdh = X25519KeyPair::from_bytes( &public.x25519.ecdh, - base64::decode(ss[8].as_bytes()).map_err(|_| InvalidFormatError)?.as_slice(), + &base62::decode::(ss[8].as_bytes()).ok_or(InvalidFormatError)?, ) .ok_or(InvalidFormatError)?; let x25519_eddsa = Ed25519KeyPair::from_bytes( &public.x25519.ecdh, - base64::decode(ss[9].as_bytes()).map_err(|_| InvalidFormatError)?.as_slice(), + &base62::decode::(ss[9].as_bytes()).ok_or(InvalidFormatError)?, ) .ok_or(InvalidFormatError)?; let p384_ecdh = P384KeyPair::from_bytes( public.p384.as_ref().unwrap().ecdh.as_bytes(), - base64::decode(ss[10].as_bytes()).map_err(|_| InvalidFormatError)?.as_slice(), + &base62::decode::(ss[10].as_bytes()).ok_or(InvalidFormatError)?, ) .ok_or(InvalidFormatError)?; let p384_ecdsa = P384KeyPair::from_bytes( public.p384.as_ref().unwrap().ecdh.as_bytes(), - base64::decode(ss[11].as_bytes()).map_err(|_| InvalidFormatError)?.as_slice(), + &base62::decode::(ss[11].as_bytes()).ok_or(InvalidFormatError)?, ) .ok_or(InvalidFormatError)?; return Ok(Self { diff --git a/network-hypervisor/src/vl1/node.rs b/network-hypervisor/src/vl1/node.rs index d61d51f14..3bd2f429a 100644 --- a/network-hypervisor/src/vl1/node.rs +++ b/network-hypervisor/src/vl1/node.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::hash::Hash; +use std::io::Write; use std::sync::atomic::Ordering; use std::sync::{Arc, Mutex, RwLock}; use std::time::Duration; @@ -22,7 +23,6 @@ use zerotier_crypto::typestate::{Valid, Verified}; use zerotier_utils::gate::IntervalGate; use zerotier_utils::hex; use zerotier_utils::marshalable::Marshalable; -use zerotier_utils::tokio::io::AsyncWriteExt; /// A VL1 node on the ZeroTier global peer to peer network. /// diff --git a/utils/src/base62.rs b/utils/src/base62.rs new file mode 100644 index 000000000..4c48fc847 --- /dev/null +++ b/utils/src/base62.rs @@ -0,0 +1,181 @@ +use std::io::Write; + +use super::arrayvec::ArrayVec; +use super::memory; + +const MAX_LENGTH_WORDS: usize = 128; + +/// Encode a byte array into a base62 string. +/// +/// The pad_output_to_length parameter outputs base62 zeroes at the end to ensure that the output +/// string is at least a given length. Set this to zero if you don't want to pad the output. This +/// has no effect on decoded output length. +pub fn encode_into(b: &[u8], s: &mut String, pad_output_to_length: usize) { + assert!(b.len() <= MAX_LENGTH_WORDS * 4); + let mut n: ArrayVec = ArrayVec::new(); + + let mut i = 0; + let len_words = b.len() & usize::MAX.wrapping_shl(2); + while i < len_words { + n.push(u32::from_le(memory::load_raw(&b[i..]))); + i += 4; + } + if i < b.len() { + let mut w = 0u32; + let mut shift = 0u32; + while i < b.len() { + w |= (b[i] as u32).wrapping_shl(shift); + i += 1; + shift += 8; + } + n.push(w); + } + + let mut string_len = 0; + while !n.is_empty() { + s.push(b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"[big_div_rem::(&mut n) as usize] as char); + string_len += 1; + } + while string_len < pad_output_to_length { + s.push('0'); + string_len += 1; + } +} + +/// Decode Base62 into a vector or other output. +/// +/// Note that base62 doesn't have a way to know the output length. Decoding may be short if there were +/// trailing zeroes in the input. The output length parameter specifies the expected length of the +/// output, which will be zero padded if decoded data does not reach it. If decoded data exceeds this +/// length an error is returned. +pub fn decode_into(s: &[u8], b: &mut W, output_length: usize) -> std::io::Result<()> { + let mut n: ArrayVec = ArrayVec::new(); + + for c in s.iter().rev() { + let mut c = *c as u32; + // 0..9, A..Z, or a..z + if c >= 48 && c <= 57 { + c -= 48; + } else if c >= 65 && c <= 90 { + c -= 65 - 10; + } else if c >= 97 && c <= 122 { + c -= 97 - (10 + 26); + } else { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid base62")); + } + big_mul::(&mut n); + big_add(&mut n, c); + } + + let mut bc = output_length; + for w in n.iter() { + if bc > 0 { + let l = bc.min(4); + b.write_all(&w.to_le_bytes()[..l])?; + bc -= l; + } else { + return Err(std::io::Error::new(std::io::ErrorKind::Other, "data too large")); + } + } + while bc > 0 { + b.write_all(&[0])?; + bc -= 1; + } + + return Ok(()); +} + +/// Decode into and return an array whose length is the desired output_length. +/// None is returned if there is an error. +#[inline] +pub fn decode(s: &[u8]) -> Option<[u8; L]> { + let mut buf = [0u8; L]; + let mut w = &mut buf[..]; + if decode_into(s, &mut w, L).is_ok() { + Some(buf) + } else { + None + } +} + +#[inline(always)] +fn big_div_rem(n: &mut ArrayVec) -> u32 { + while let Some(&0) = n.last() { + n.pop(); + } + let mut rem = 0; + for word in n.iter_mut().rev() { + let temp = (rem as u64).wrapping_shl(32) | (*word as u64); + let (a, b) = (temp / D, temp % D); + *word = a as u32; + rem = b as u32; + } + while let Some(&0) = n.last() { + n.pop(); + } + rem +} + +#[inline(always)] +fn big_add(n: &mut ArrayVec, i: u32) { + let mut carry = i as u64; + for word in n.iter_mut() { + let res = (*word as u64).wrapping_add(carry); + *word = res as u32; + carry = res.wrapping_shr(32); + } + if carry > 0 { + n.push(carry as u32); + } +} + +#[inline(always)] +fn big_mul(n: &mut ArrayVec) { + while let Some(&0) = n.last() { + n.pop(); + } + let mut carry = 0; + for word in n.iter_mut() { + let temp = (*word as u64).wrapping_mul(M).wrapping_add(carry); + *word = temp as u32; + carry = temp.wrapping_shr(32); + } + if carry != 0 { + n.push(carry as u32); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn div_rem() { + let mut n = ArrayVec::::new(); + n.push_slice(&[0xdeadbeef, 0xfeedfeed, 0xcafebabe, 0xf00dd00d]); + let rem = big_div_rem::<4, 63>(&mut n); + let nn = n.as_ref(); + assert!(nn[0] == 0xaa23440b && nn[1] == 0xa696103c && nn[2] == 0x89513fea && nn[3] == 0x03cf7514 && rem == 58); + } + + #[test] + fn encode_decode() { + let mut test = [0xff; 64]; + for tl in 1..64 { + let test = &mut test[..tl]; + test.fill(0xff); + let mut b = Vec::with_capacity(1024); + for _ in 0..10 { + let mut s = String::with_capacity(1024); + encode_into(&test, &mut s, 86); + b.clear(); + //println!("{}", s); + assert!(decode_into(s.as_bytes(), &mut b, test.len()).is_ok()); + assert_eq!(b.as_slice(), test); + for c in test.iter_mut() { + *c = crate::rand() as u8; + } + } + } + } +} diff --git a/utils/src/base64.rs b/utils/src/base64.rs deleted file mode 100644 index 5fee0bb48..000000000 --- a/utils/src/base64.rs +++ /dev/null @@ -1,115 +0,0 @@ -use crate::error::InvalidParameterError; - -/// URL-safe base64 alphabet -const ALPHABET: [u8; 64] = *b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; -const ALPHABET_INV: [u8; 256] = [ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, - 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 63, - 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, -]; - -pub fn encode_into(mut b: &[u8], s: &mut String) { - while b.len() >= 3 { - let bits = (b[0] as usize) | (b[1] as usize).wrapping_shl(8) | (b[2] as usize).wrapping_shl(16); - b = &b[3..]; - let (i0, i1, i2, i3) = (bits & 63, bits.wrapping_shr(6) & 63, bits.wrapping_shr(12) & 63, bits.wrapping_shr(18)); - s.push(ALPHABET[i0] as char); - s.push(ALPHABET[i1] as char); - s.push(ALPHABET[i2] as char); - s.push(ALPHABET[i3] as char); - } - if b.len() == 2 { - let bits = (b[0] as usize) | (b[1] as usize).wrapping_shl(8); - s.push(ALPHABET[bits & 63] as char); - s.push(ALPHABET[bits.wrapping_shr(6) & 63] as char); - s.push(ALPHABET[bits.wrapping_shr(12)] as char); - } else if b.len() == 1 { - let bits = b[0] as usize; - s.push(ALPHABET[bits & 63] as char); - s.push(ALPHABET[bits.wrapping_shr(6)] as char); - } -} - -pub fn decode_into(mut s: &[u8], b: &mut Vec) -> Result<(), InvalidParameterError> { - while s.len() >= 4 { - let (i0, i1, i2, i3) = ( - ALPHABET_INV[s[0] as usize], - ALPHABET_INV[s[1] as usize], - ALPHABET_INV[s[2] as usize], - ALPHABET_INV[s[3] as usize], - ); - s = &s[4..]; - if (i0 | i1 | i2 | i3) > 64 { - return Err(InvalidParameterError("invalid base64 string")); - } - let bits = (i0 as usize) | (i1 as usize).wrapping_shl(6) | (i2 as usize).wrapping_shl(12) | (i3 as usize).wrapping_shl(18); - b.push((bits & 0xff) as u8); - b.push((bits.wrapping_shr(8) & 0xff) as u8); - b.push((bits.wrapping_shr(16) & 0xff) as u8); - } - match s.len() { - 1 => return Err(InvalidParameterError("invalid base64 string")), - 2 => { - let (i0, i1) = (ALPHABET_INV[s[0] as usize], ALPHABET_INV[s[1] as usize]); - if (i0 | i1) > 64 { - return Err(InvalidParameterError("invalid base64 string")); - } - let bits = (i0 as usize) | (i1 as usize).wrapping_shl(6); - b.push((bits & 0xff) as u8); - } - 3 => { - let (i0, i1, i2) = (ALPHABET_INV[s[0] as usize], ALPHABET_INV[s[1] as usize], ALPHABET_INV[s[2] as usize]); - if (i0 | i1 | i2) > 64 { - return Err(InvalidParameterError("invalid base64 string")); - } - let bits = (i0 as usize) | (i1 as usize).wrapping_shl(6) | (i2 as usize).wrapping_shl(12); - b.push((bits & 0xff) as u8); - b.push((bits.wrapping_shr(8) & 0xff) as u8); - } - _ => {} - } - Ok(()) -} - -pub fn encode(b: &[u8]) -> String { - let mut tmp = String::with_capacity(((b.len() / 3) * 4) + 3); - encode_into(b, &mut tmp); - tmp -} - -pub fn decode(s: &[u8]) -> Result, InvalidParameterError> { - let mut tmp = Vec::with_capacity(((s.len() / 4) * 3) + 3); - decode_into(s, &mut tmp)?; - Ok(tmp) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn encode_decode() { - let mut tmp = [0xffu8; 256]; - for _ in 0..7 { - let mut s = String::with_capacity(1024); - let mut v: Vec = Vec::with_capacity(256); - for i in 1..256 { - s.clear(); - encode_into(&tmp[..i], &mut s); - //println!("{}", s); - v.clear(); - decode_into(s.as_str().as_bytes(), &mut v).expect("decode error"); - assert!(v.as_slice().eq(&tmp[..i])); - } - for b in tmp.iter_mut() { - *b -= 13; - } - } - } -} diff --git a/utils/src/basex.rs b/utils/src/basex.rs deleted file mode 100644 index 8f5a7fe92..000000000 --- a/utils/src/basex.rs +++ /dev/null @@ -1,62 +0,0 @@ -use super::arrayvec::ArrayVec; - -fn big_div_rem(n: &mut ArrayVec, d: u32) -> u32 { - while let Some(&0) = n.last() { - n.pop(); - } - let d = d as u64; - let mut rem = 0; - for word in n.iter_mut().rev() { - let temp = (rem as u64).wrapping_shl(32) | (*word as u64); - let (a, b) = (temp / d, temp % d); - *word = a as u32; - rem = b as u32; - } - while let Some(&0) = n.last() { - n.pop(); - } - rem -} - -fn big_add(n: &mut ArrayVec, i: u32) { - debug_assert!(i <= (u32::MAX - 1)); - debug_assert!(!n.is_empty()); - debug_assert!(n.iter().any(|x| *x != 0)); - let mut carry = false; - for word in n.iter_mut() { - (*word, carry) = word.overflowing_add(i.wrapping_add(carry as u32)); - } - if carry { - n.push(1); - } -} - -fn big_mul(n: &mut ArrayVec, m: u32) { - while let Some(&0) = n.last() { - n.pop(); - } - let m = m as u64; - let mut carry = 0; - for word in n.iter_mut() { - let temp = (*word as u64).wrapping_mul(m).wrapping_add(carry); - *word = (temp & 0xffffffff) as u32; - carry = temp.wrapping_shr(32); - } - if carry > 0 { - n.push(carry as u32); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn div_rem() { - let mut n = ArrayVec::::new(); - n.push_slice(&[0xdeadbeef, 0xfeedfeed, 0xcafebabe, 0xf00dd00d]); - let rem = big_div_rem(&mut n, 63); - let nn = n.as_ref(); - assert!(nn[0] == 0xaa23440b && nn[1] == 0xa696103c && nn[2] == 0x89513fea && nn[3] == 0x03cf7514 && rem == 58); - } -} diff --git a/utils/src/lib.rs b/utils/src/lib.rs index dbb3ad353..fa941773d 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -8,7 +8,7 @@ pub mod arrayvec; pub mod base24; -pub mod base64; +pub mod base62; pub mod blob; pub mod buffer; pub mod cast; @@ -25,7 +25,6 @@ pub mod json; pub mod marshalable; pub mod memory; pub mod pool; -pub mod proquint; #[cfg(feature = "tokio")] pub mod reaper; pub mod ringbuffer; @@ -89,6 +88,11 @@ pub fn wait_for_process_abort() { #[inline(never)] pub extern "C" fn unlikely_branch() {} +#[cfg(unix)] +pub fn rand() -> u32 { + unsafe { (libc::rand() as u32) ^ (libc::rand() as u32).wrapping_shr(8) } +} + #[cfg(test)] mod tests { use super::ms_monotonic;