void/ZeroTierOne
2
0
Fork 0
mirror of https://github.com/zerotier/ZeroTierOne.git synced 2025-06-07 13:03:45 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

rustfmt

Browse source
This commit is contained in:
Adam Ierymenko 2022-09-13 10:48:36 -04:00
parent 4763404d98
commit 07fc8b2d2b
No known key found for this signature in database
GPG key ID: C8877CF2D7A5D7F3
87 changed files with 8325 additions and 6871 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

153
attic/fastudpsocket.rs
View file

@ -44,25 +44,55 @@ fn bind_udp_socket(_device_name: &str, address: &InetAddress) -> Result<FastUDPR
let mut setsockopt_results: c_int = 0; let mut setsockopt_results: c_int = 0;
fl = 1; fl = 1;
setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_REUSEPORT.as_(), (&mut fl as *mut c_int).cast(), fl_size); setsockopt_results |= libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_REUSEPORT.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
);
fl = 0; fl = 0;
setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_LINGER.as_(), (&mut fl as *mut c_int).cast(), fl_size); setsockopt_results |= libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_LINGER.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
);
//fl = 1; //fl = 1;
//setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET, libc::SO_REUSEADDR, (&mut fl as *mut c_int).cast(), fl_size); //setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET, libc::SO_REUSEADDR, (&mut fl as *mut c_int).cast(), fl_size);
fl = 1; fl = 1;
setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_BROADCAST.as_(), (&mut fl as *mut c_int).cast(), fl_size); setsockopt_results |= libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_BROADCAST.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
);
if af == libc::AF_INET6 { if af == libc::AF_INET6 {
fl = 1; fl = 1;
setsockopt_results |= libc::setsockopt(s, libc::IPPROTO_IPV6.as_(), libc::IPV6_V6ONLY.as_(), (&mut fl as *mut c_int).cast(), fl_size); setsockopt_results |= libc::setsockopt(
s,
libc::IPPROTO_IPV6.as_(),
libc::IPV6_V6ONLY.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
);
} }
#[cfg(any(target_os = "macos", target_os = "ios"))] #[cfg(any(target_os = "macos", target_os = "ios"))]
{ {
fl = 1; fl = 1;
setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_NOSIGPIPE.as_(), (&mut fl as *mut c_int).cast(), fl_size) setsockopt_results |= libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_NOSIGPIPE.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
)
} }
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
@ -70,7 +100,13 @@ fn bind_udp_socket(_device_name: &str, address: &InetAddress) -> Result<FastUDPR
if !_device_name.is_empty() { if !_device_name.is_empty() {
let _ = std::ffi::CString::new(_device_name).map(|dn| { let _ = std::ffi::CString::new(_device_name).map(|dn| {
let dnb = dn.as_bytes_with_nul(); let dnb = dn.as_bytes_with_nul();
let _ = libc::setsockopt(s.as_(), libc::SOL_SOCKET.as_(), libc::SO_BINDTODEVICE.as_(), dnb.as_ptr().cast(), (dnb.len() - 1).as_()); let _ = libc::setsockopt(
s.as_(),
libc::SOL_SOCKET.as_(),
libc::SO_BINDTODEVICE.as_(),
dnb.as_ptr().cast(),
(dnb.len() - 1).as_(),
);
}); });
} }
} }
@ -84,30 +120,62 @@ fn bind_udp_socket(_device_name: &str, address: &InetAddress) -> Result<FastUDPR
#[cfg(not(target_os = "linux"))] #[cfg(not(target_os = "linux"))]
{ {
fl = 0; fl = 0;
libc::setsockopt(s, libc::IPPROTO_IP.as_(), libc::IP_DF.as_(), (&mut fl as *mut c_int).cast(), fl_size); libc::setsockopt(
s,
libc::IPPROTO_IP.as_(),
libc::IP_DF.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
);
} }
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
{ {
fl = libc::IP_PMTUDISC_DONT as c_int; fl = libc::IP_PMTUDISC_DONT as c_int;
libc::setsockopt(s, libc::IPPROTO_IP.as_(), libc::IP_MTU_DISCOVER.as_(), (&mut fl as *mut c_int).cast(), fl_size); libc::setsockopt(
s,
libc::IPPROTO_IP.as_(),
libc::IP_MTU_DISCOVER.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
);
} }
} }
if af == libc::AF_INET6 { if af == libc::AF_INET6 {
fl = 0; fl = 0;
libc::setsockopt(s, libc::IPPROTO_IPV6.as_(), libc::IPV6_DONTFRAG.as_(), (&mut fl as *mut c_int).cast(), fl_size); libc::setsockopt(
s,
libc::IPPROTO_IPV6.as_(),
libc::IPV6_DONTFRAG.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
);
} }
fl = 1048576; fl = 1048576;
while fl >= 131072 { while fl >= 131072 {
if libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_RCVBUF.as_(), (&mut fl as *mut c_int).cast(), fl_size) == 0 { if libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_RCVBUF.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
) == 0
{
break; break;
} }
fl -= 65536; fl -= 65536;
} }
fl = 1048576; fl = 1048576;
while fl >= 131072 { while fl >= 131072 {
if libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_SNDBUF.as_(), (&mut fl as *mut c_int).cast(), fl_size) == 0 { if libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_SNDBUF.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
) == 0
{
break; break;
} }
fl -= 65536; fl -= 65536;
@ -172,10 +240,22 @@ pub(crate) fn fast_udp_socket_sendto(socket: &FastUDPRawOsSocket, to_address: &I
let _ = libc::sendmsg(socket.get().as_(), transmute(&mhdr as *const libc::msghdr), 0); let _ = libc::sendmsg(socket.get().as_(), transmute(&mhdr as *const libc::msghdr), 0);
} else { } else {
let mut ttl = packet_ttl as c_int; let mut ttl = packet_ttl as c_int;
libc::setsockopt(socket.get().as_(), libc::IPPROTO_IP.as_(), libc::IP_TTL.as_(), (&mut ttl as *mut c_int).cast(), std::mem::size_of::<c_int>().as_()); libc::setsockopt(
socket.get().as_(),
libc::IPPROTO_IP.as_(),
libc::IP_TTL.as_(),
(&mut ttl as *mut c_int).cast(),
std::mem::size_of::<c_int>().as_(),
);
let _ = libc::sendmsg(socket.get().as_(), transmute(&mhdr as *const libc::msghdr), 0); let _ = libc::sendmsg(socket.get().as_(), transmute(&mhdr as *const libc::msghdr), 0);
ttl = 255; ttl = 255;
libc::setsockopt(socket.get().as_(), libc::IPPROTO_IP.as_(), libc::IP_TTL.as_(), (&mut ttl as *mut c_int).cast(), std::mem::size_of::<c_int>().as_()); libc::setsockopt(
socket.get().as_(),
libc::IPPROTO_IP.as_(),
libc::IP_TTL.as_(),
(&mut ttl as *mut c_int).cast(),
std::mem::size_of::<c_int>().as_(),
);
} }
} }
} }
@ -185,7 +265,14 @@ pub(crate) fn fast_udp_socket_sendto(socket: &FastUDPRawOsSocket, to_address: &I
fn fast_udp_socket_recvfrom(socket: &FastUDPRawOsSocket, buf: &mut PacketBuffer, from_address: &mut InetAddress) -> isize { fn fast_udp_socket_recvfrom(socket: &FastUDPRawOsSocket, buf: &mut PacketBuffer, from_address: &mut InetAddress) -> isize {
unsafe { unsafe {
let mut addrlen = std::mem::size_of::<InetAddress>() as libc::socklen_t; let mut addrlen = std::mem::size_of::<InetAddress>() as libc::socklen_t;
let s = libc::recvfrom(socket.get().as_(), buf.as_mut_ptr().cast(), buf.capacity().as_(), 0, (from_address as *mut InetAddress).cast(), &mut addrlen) as isize; let s = libc::recvfrom(
socket.get().as_(),
buf.as_mut_ptr().cast(),
buf.capacity().as_(),
0,
(from_address as *mut InetAddress).cast(),
&mut addrlen,
) as isize;
if s > 0 { if s > 0 {
buf.set_size_unchecked(s as usize); buf.set_size_unchecked(s as usize);
} }
@ -194,7 +281,12 @@ fn fast_udp_socket_recvfrom(socket: &FastUDPRawOsSocket, buf: &mut PacketBuffer,
} }
impl FastUDPSocket { impl FastUDPSocket {
pub fn new<F: Fn(&FastUDPRawOsSocket, &InetAddress, PacketBuffer) + Send + Sync + Clone + 'static>(device_name: &str, address: &InetAddress, packet_buffer_pool: &Arc<PacketBufferPool>, handler: F) -> Result<Self, String> { pub fn new<F: Fn(&FastUDPRawOsSocket, &InetAddress, PacketBuffer) + Send + Sync + Clone + 'static>(
device_name: &str,
address: &InetAddress,
packet_buffer_pool: &Arc<PacketBufferPool>,
handler: F,
) -> Result<Self, String> {
let thread_count = num_cpus::get_physical().clamp(1, FAST_UDP_SOCKET_MAX_THREADS); let thread_count = num_cpus::get_physical().clamp(1, FAST_UDP_SOCKET_MAX_THREADS);
let mut s = Self { let mut s = Self {
@ -273,7 +365,14 @@ impl Drop for FastUDPSocket {
self.thread_run.store(false, Ordering::Relaxed); self.thread_run.store(false, Ordering::Relaxed);
for s in self.sockets.iter() { for s in self.sockets.iter() {
unsafe { unsafe {
libc::sendto(s.get().as_(), tmp.as_ptr().cast(), 0, 0, (&self.bind_address as *const InetAddress).cast(), std::mem::size_of::<InetAddress>() as libc::socklen_t); libc::sendto(
s.get().as_(),
tmp.as_ptr().cast(),
0,
0,
(&self.bind_address as *const InetAddress).cast(),
std::mem::size_of::<InetAddress>() as libc::socklen_t,
);
} }
} }
for s in self.sockets.iter() { for s in self.sockets.iter() {
@ -309,9 +408,14 @@ mod tests {
let ba0 = ba0.unwrap(); let ba0 = ba0.unwrap();
let cnt0 = Arc::new(AtomicU32::new(0)); let cnt0 = Arc::new(AtomicU32::new(0));
let cnt0c = cnt0.clone(); let cnt0c = cnt0.clone();
let s0 = FastUDPSocket::new("", &ba0, &pool, move |sock: &FastUDPRawOsSocket, _: &InetAddress, data: PacketBuffer| { let s0 = FastUDPSocket::new(
cnt0c.fetch_add(1, Ordering::Relaxed); "",
}); &ba0,
&pool,
move |sock: &FastUDPRawOsSocket, _: &InetAddress, data: PacketBuffer| {
cnt0c.fetch_add(1, Ordering::Relaxed);
},
);
assert!(s0.is_ok()); assert!(s0.is_ok());
let s0 = s0.unwrap(); let s0 = s0.unwrap();
@ -320,9 +424,14 @@ mod tests {
let ba1 = ba1.unwrap(); let ba1 = ba1.unwrap();
let cnt1 = Arc::new(AtomicU32::new(0)); let cnt1 = Arc::new(AtomicU32::new(0));
let cnt1c = cnt1.clone(); let cnt1c = cnt1.clone();
let s1 = FastUDPSocket::new("", &ba1, &pool, move |sock: &FastUDPRawOsSocket, _: &InetAddress, data: PacketBuffer| { let s1 = FastUDPSocket::new(
cnt1c.fetch_add(1, Ordering::Relaxed); "",
}); &ba1,
&pool,
move |sock: &FastUDPRawOsSocket, _: &InetAddress, data: PacketBuffer| {
cnt1c.fetch_add(1, Ordering::Relaxed);
},
);
assert!(s1.is_ok()); assert!(s1.is_ok());
let s1 = s1.unwrap(); let s1 = s1.unwrap();

32
attic/iblt/src/lib.rs
View file

@ -55,7 +55,11 @@ impl<const BUCKETS: usize, const ITEM_BYTES: usize, const HASHES: usize> Clone f
// of implementing clone() may not since check_hash[] is u32. // of implementing clone() may not since check_hash[] is u32.
unsafe { unsafe {
let mut tmp: Self = std::mem::MaybeUninit::uninit().assume_init(); let mut tmp: Self = std::mem::MaybeUninit::uninit().assume_init();
std::ptr::copy_nonoverlapping((self as *const Self).cast::<u8>(), (&mut tmp as *mut Self).cast::<u8>(), Self::SIZE_BYTES); std::ptr::copy_nonoverlapping(
(self as *const Self).cast::<u8>(),
(&mut tmp as *mut Self).cast::<u8>(),
Self::SIZE_BYTES,
);
tmp tmp
} }
} }
@ -159,7 +163,10 @@ impl<const BUCKETS: usize, const ITEM_BYTES: usize, const HASHES: usize> IBLT<BU
/// Subtract another IBLT from this one to get a set difference. /// Subtract another IBLT from this one to get a set difference.
pub fn subtract(&mut self, other: &Self) { pub fn subtract(&mut self, other: &Self) {
self.check_hash.iter_mut().zip(other.check_hash.iter()).for_each(|(a, b)| *a ^= *b); self.check_hash.iter_mut().zip(other.check_hash.iter()).for_each(|(a, b)| *a ^= *b);
self.count.iter_mut().zip(other.count.iter()).for_each(|(a, b)| *a = a.wrapping_sub(*b)); self.count
.iter_mut()
.zip(other.count.iter())
.for_each(|(a, b)| *a = a.wrapping_sub(*b));
self.key.iter_mut().zip(other.key.iter()).for_each(|(a, b)| xor_with(a, b)); self.key.iter_mut().zip(other.key.iter()).for_each(|(a, b)| xor_with(a, b));
} }
@ -310,7 +317,14 @@ mod tests {
assert!(d); assert!(d);
}); });
println!("inserted: {}\tlisted: {}\tcapacity: {}\tscore: {:.4}\tfill: {:.4}", count, list_count, CAPACITY, (list_count as f64) / (count as f64), (count as f64) / (CAPACITY as f64)); println!(
"inserted: {}\tlisted: {}\tcapacity: {}\tscore: {:.4}\tfill: {:.4}",
count,
list_count,
CAPACITY,
(list_count as f64) / (count as f64),
(count as f64) / (CAPACITY as f64)
);
count += 32; count += 32;
} }
} }
@ -358,7 +372,17 @@ mod tests {
cnt += 1; cnt += 1;
}); });
println!("total: {} missing: {:5} recovered: {:5} size: {:5} score: {:.4} bytes/item: {:.2} extract(fill): {:.4} 100%: {}", REMOTE_SIZE, missing.len(), cnt, bytes, (cnt as f64) / (missing.len() as f64), (bytes as f64) / (cnt as f64), (cnt as f64) / (CAPACITY as f64), all_success); println!(
"total: {} missing: {:5} recovered: {:5} size: {:5} score: {:.4} bytes/item: {:.2} extract(fill): {:.4} 100%: {}",
REMOTE_SIZE,
missing.len(),
cnt,
bytes,
(cnt as f64) / (missing.len() as f64),
(bytes as f64) / (cnt as f64),
(cnt as f64) / (CAPACITY as f64),
all_success
);
missing_count += STEP; missing_count += STEP;
} }

16
attic/syncwhole/src/datastore.rs
View file

@ -84,10 +84,22 @@ pub trait DataStore: Sync + Send {
/// Iterate through keys in a range. /// Iterate through keys in a range.
/// ///
/// Keys MUST be output in ascending binary sort order. /// Keys MUST be output in ascending binary sort order.
async fn keys<F: Send + FnMut(&[u8]) -> bool>(&self, subset: Option<&[u8]>, range_start: &[u8; KEY_SIZE], range_end: &[u8; KEY_SIZE], f: F); async fn keys<F: Send + FnMut(&[u8]) -> bool>(
&self,
subset: Option<&[u8]>,
range_start: &[u8; KEY_SIZE],
range_end: &[u8; KEY_SIZE],
f: F,
);
/// Iterate through values in a range. /// Iterate through values in a range.
/// ///
/// Entries MUST be output in ascending binary sort order. /// Entries MUST be output in ascending binary sort order.
async fn values<F: Send + FnMut(&[u8], &[u8]) -> bool>(&self, subset: Option<&[u8]>, range_start: &[u8; KEY_SIZE], range_end: &[u8; KEY_SIZE], f: F); async fn values<F: Send + FnMut(&[u8], &[u8]) -> bool>(
&self,
subset: Option<&[u8]>,
range_start: &[u8; KEY_SIZE],
range_end: &[u8; KEY_SIZE],
f: F,
);
} }

85
attic/syncwhole/src/main.rs
View file

@ -140,46 +140,57 @@ impl DataStore for TestNodeHost {
} }
fn main() { fn main() {
tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap().block_on(async { tokio::runtime::Builder::new_current_thread()
println!("Running syncwhole local self-test network with {} nodes starting at 127.0.0.1:{}", TEST_NODE_COUNT, TEST_PORT_RANGE_START); .enable_all()
println!(); .build()
.unwrap()
.block_on(async {
println!(
"Running syncwhole local self-test network with {} nodes starting at 127.0.0.1:{}",
TEST_NODE_COUNT, TEST_PORT_RANGE_START
);
println!();
println!("Starting nodes on 127.0.0.1..."); println!("Starting nodes on 127.0.0.1...");
let mut nodes: Vec<Node<TestNodeHost, TestNodeHost>> = Vec::with_capacity(TEST_NODE_COUNT); let mut nodes: Vec<Node<TestNodeHost, TestNodeHost>> = Vec::with_capacity(TEST_NODE_COUNT);
for port in TEST_PORT_RANGE_START..(TEST_PORT_RANGE_START + (TEST_NODE_COUNT as u16)) { for port in TEST_PORT_RANGE_START..(TEST_PORT_RANGE_START + (TEST_NODE_COUNT as u16)) {
let mut peers: Vec<SocketAddr> = Vec::with_capacity(TEST_NODE_COUNT); let mut peers: Vec<SocketAddr> = Vec::with_capacity(TEST_NODE_COUNT);
for port2 in TEST_PORT_RANGE_START..(TEST_PORT_RANGE_START + (TEST_NODE_COUNT as u16)) { for port2 in TEST_PORT_RANGE_START..(TEST_PORT_RANGE_START + (TEST_NODE_COUNT as u16)) {
if port != port2 { if port != port2 {
peers.push(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port2))); peers.push(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port2)));
}
}
let mut th = TestNodeHost::new_random(port as usize);
th.config.anchors = peers;
th.config.name = port.to_string();
let nh = Arc::new(th);
//println!("Starting node on 127.0.0.1:{}...", port, nh.db.lock().unwrap().len());
nodes.push(
Node::new(nh.clone(), nh.clone(), SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port)))
.await
.unwrap(),
);
}
print!("Waiting for all connections to be established...");
let _ = stdout().flush();
loop {
tokio::time::sleep(Duration::from_secs(1)).await;
let mut count = 0;
for n in nodes.iter() {
count += n.connection_count().await;
}
if count == (TEST_NODE_COUNT * (TEST_NODE_COUNT - 1)) {
println!(" {} connections up.", count);
break;
} else {
print!(".");
let _ = stdout().flush();
} }
} }
let mut th = TestNodeHost::new_random(port as usize);
th.config.anchors = peers;
th.config.name = port.to_string();
let nh = Arc::new(th);
//println!("Starting node on 127.0.0.1:{}...", port, nh.db.lock().unwrap().len());
nodes.push(Node::new(nh.clone(), nh.clone(), SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port))).await.unwrap());
}
print!("Waiting for all connections to be established..."); loop {
let _ = stdout().flush(); tokio::time::sleep(Duration::from_secs(1)).await;
loop {
tokio::time::sleep(Duration::from_secs(1)).await;
let mut count = 0;
for n in nodes.iter() {
count += n.connection_count().await;
} }
if count == (TEST_NODE_COUNT * (TEST_NODE_COUNT - 1)) { });
println!(" {} connections up.", count);
break;
} else {
print!(".");
let _ = stdout().flush();
}
}
loop {
tokio::time::sleep(Duration::from_secs(1)).await;
}
});
} }

105
attic/syncwhole/src/node.rs
View file

@ -75,7 +75,11 @@ pub struct Node<D: DataStore + 'static, H: Host + 'static> {
impl<D: DataStore + 'static, H: Host + 'static> Node<D, H> { impl<D: DataStore + 'static, H: Host + 'static> Node<D, H> {
pub async fn new(db: Arc<D>, host: Arc<H>, bind_address: SocketAddr) -> std::io::Result<Self> { pub async fn new(db: Arc<D>, host: Arc<H>, bind_address: SocketAddr) -> std::io::Result<Self> {
let listener = if bind_address.is_ipv4() { TcpSocket::new_v4() } else { TcpSocket::new_v6() }?; let listener = if bind_address.is_ipv4() {
TcpSocket::new_v4()
} else {
TcpSocket::new_v6()
}?;
configure_tcp_socket(&listener)?; configure_tcp_socket(&listener)?;
listener.bind(bind_address.clone())?; listener.bind(bind_address.clone())?;
let listener = listener.listen(1024)?; let listener = listener.listen(1024)?;
@ -119,7 +123,12 @@ impl<D: DataStore + 'static, H: Host + 'static> Node<D, H> {
/// to the endpoint. An error is returned if the connection fails. /// to the endpoint. An error is returned if the connection fails.
pub async fn connect(&self, address: &SocketAddr) -> std::io::Result<bool> { pub async fn connect(&self, address: &SocketAddr) -> std::io::Result<bool> {
if self.internal.connecting_to.lock().await.insert(address.clone()) { if self.internal.connecting_to.lock().await.insert(address.clone()) {
self.internal.connect(address, Instant::now().add(Duration::from_millis(self.internal.host.node_config().connection_timeout))).await self.internal
.connect(
address,
Instant::now().add(Duration::from_millis(self.internal.host.node_config().connection_timeout)),
)
.await
} else { } else {
Ok(false) Ok(false)
} }
@ -161,7 +170,12 @@ fn configure_tcp_socket(socket: &TcpSocket) -> std::io::Result<()> {
} }
fn decode_msgpack<'a, T: Deserialize<'a>>(b: &'a [u8]) -> std::io::Result<T> { fn decode_msgpack<'a, T: Deserialize<'a>>(b: &'a [u8]) -> std::io::Result<T> {
rmp_serde::from_slice(b).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, format!("invalid msgpack object: {}", e.to_string()))) rmp_serde::from_slice(b).map_err(|e| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("invalid msgpack object: {}", e.to_string()),
)
})
} }
pub struct NodeInternal<D: DataStore + 'static, H: Host + 'static> { pub struct NodeInternal<D: DataStore + 'static, H: Host + 'static> {
@ -228,7 +242,10 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
let e = j.unwrap().await; let e = j.unwrap().await;
if e.is_ok() { if e.is_ok() {
let e = e.unwrap(); let e = e.unwrap();
host.on_connection_closed(&*cc.info.lock().unwrap(), e.map_or_else(|e| e.to_string(), |_| "unknown error".to_string())); host.on_connection_closed(
&*cc.info.lock().unwrap(),
e.map_or_else(|e| e.to_string(), |_| "unknown error".to_string()),
);
} else { } else {
host.on_connection_closed(&*cc.info.lock().unwrap(), "remote host closed connection".to_string()); host.on_connection_closed(&*cc.info.lock().unwrap(), "remote host closed connection".to_string());
} }
@ -299,7 +316,10 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
background_tasks.spawn(async move { background_tasks.spawn(async move {
// If the connection dies this will either fail or time out in 1s. Usually these execute instantly due to // If the connection dies this will either fail or time out in 1s. Usually these execute instantly due to
// write buffering but a short timeout prevents them from building up too much. // write buffering but a short timeout prevents them from building up too much.
let _ = tokio::time::timeout(announce_timeout, c2.send_msg(MessageType::HaveRecords, have_records.as_slice(), now)); let _ = tokio::time::timeout(
announce_timeout,
c2.send_msg(MessageType::HaveRecords, have_records.as_slice(), now),
);
}); });
have_records = Vec::with_capacity(have_records_est_size); have_records = Vec::with_capacity(have_records_est_size);
} }
@ -332,7 +352,11 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
/// either succeeds or fails. /// either succeeds or fails.
async fn connect(self: &Arc<Self>, address: &SocketAddr, deadline: Instant) -> std::io::Result<bool> { async fn connect(self: &Arc<Self>, address: &SocketAddr, deadline: Instant) -> std::io::Result<bool> {
let f = async { let f = async {
let stream = if address.is_ipv4() { TcpSocket::new_v4() } else { TcpSocket::new_v6() }?; let stream = if address.is_ipv4() {
TcpSocket::new_v4()
} else {
TcpSocket::new_v6()
}?;
configure_tcp_socket(&stream)?; configure_tcp_socket(&stream)?;
stream.bind(self.bind_address.clone())?; stream.bind(self.bind_address.clone())?;
@ -389,7 +413,12 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
ok ok
} }
async fn connection_io_task_main(self: Arc<Self>, connection: &Arc<Connection>, remote_address: SocketAddr, mut reader: OwnedReadHalf) -> std::io::Result<()> { async fn connection_io_task_main(
self: Arc<Self>,
connection: &Arc<Connection>,
remote_address: SocketAddr,
mut reader: OwnedReadHalf,
) -> std::io::Result<()> {
const BUF_CHUNK_SIZE: usize = 4096; const BUF_CHUNK_SIZE: usize = 4096;
const READ_BUF_INITIAL_SIZE: usize = 65536; // should be a multiple of BUF_CHUNK_SIZE const READ_BUF_INITIAL_SIZE: usize = 65536; // should be a multiple of BUF_CHUNK_SIZE
@ -479,20 +508,29 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
info.name = msg.node_name.to_string(); info.name = msg.node_name.to_string();
info.contact = msg.node_contact.to_string(); info.contact = msg.node_contact.to_string();
let _ = msg.explicit_ipv4.map(|pv4| { let _ = msg.explicit_ipv4.map(|pv4| {
info.explicit_addresses.push(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::from(pv4.ip), pv4.port))); info.explicit_addresses
.push(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::from(pv4.ip), pv4.port)));
}); });
let _ = msg.explicit_ipv6.map(|pv6| { let _ = msg.explicit_ipv6.map(|pv6| {
info.explicit_addresses.push(SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::from(pv6.ip), pv6.port, 0, 0))); info.explicit_addresses
.push(SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::from(pv6.ip), pv6.port, 0, 0)));
}); });
let info = info.clone(); let info = info.clone();
let auth_challenge_response = self.host.authenticate(&info, msg.auth_challenge); let auth_challenge_response = self.host.authenticate(&info, msg.auth_challenge);
if auth_challenge_response.is_none() { if auth_challenge_response.is_none() {
return Err(std::io::Error::new(std::io::ErrorKind::Other, "authenticate() returned None, connection dropped")); return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"authenticate() returned None, connection dropped",
));
} }
let auth_challenge_response = auth_challenge_response.unwrap(); let auth_challenge_response = auth_challenge_response.unwrap();
(H::hmac_sha512(&self.anti_loopback_secret, msg.anti_loopback_challenge), H::hmac_sha512(&H::sha512(&[self.host.node_config().domain.as_bytes()]), msg.domain_challenge), auth_challenge_response) (
H::hmac_sha512(&self.anti_loopback_secret, msg.anti_loopback_challenge),
H::hmac_sha512(&H::sha512(&[self.host.node_config().domain.as_bytes()]), msg.domain_challenge),
auth_challenge_response,
)
}; };
connection connection
@ -517,14 +555,27 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
return Err(std::io::Error::new(std::io::ErrorKind::Other, "duplicate init response")); return Err(std::io::Error::new(std::io::ErrorKind::Other, "duplicate init response"));
} }
if msg.anti_loopback_response.eq(&H::hmac_sha512(&self.anti_loopback_secret, &anti_loopback_challenge_sent)) { if msg
.anti_loopback_response
.eq(&H::hmac_sha512(&self.anti_loopback_secret, &anti_loopback_challenge_sent))
{
return Err(std::io::Error::new(std::io::ErrorKind::Other, "rejected connection to self")); return Err(std::io::Error::new(std::io::ErrorKind::Other, "rejected connection to self"));
} }
if !msg.domain_response.eq(&H::hmac_sha512(&H::sha512(&[self.host.node_config().domain.as_bytes()]), &domain_challenge_sent)) { if !msg.domain_response.eq(&H::hmac_sha512(
&H::sha512(&[self.host.node_config().domain.as_bytes()]),
&domain_challenge_sent,
)) {
return Err(std::io::Error::new(std::io::ErrorKind::Other, "domain mismatch")); return Err(std::io::Error::new(std::io::ErrorKind::Other, "domain mismatch"));
} }
if !self.host.authenticate(&info, &auth_challenge_sent).map_or(false, |cr| msg.auth_response.eq(&cr)) { if !self
return Err(std::io::Error::new(std::io::ErrorKind::Other, "challenge/response authentication failed")); .host
.authenticate(&info, &auth_challenge_sent)
.map_or(false, |cr| msg.auth_response.eq(&cr))
{
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"challenge/response authentication failed",
));
} }
initialized = true; initialized = true;
@ -537,7 +588,10 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
// Handle messages other than INIT and INIT_RESPONSE after checking 'initialized' flag. // Handle messages other than INIT and INIT_RESPONSE after checking 'initialized' flag.
_ => { _ => {
if !initialized { if !initialized {
return Err(std::io::Error::new(std::io::ErrorKind::Other, "init exchange must be completed before other messages are sent")); return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"init exchange must be completed before other messages are sent",
));
} }
match message_type { match message_type {
@ -556,7 +610,10 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
} }
} }
StoreResult::Rejected => { StoreResult::Rejected => {
return Err(std::io::Error::new(std::io::ErrorKind::Other, format!("record rejected by data store: {}", to_hex_string(&key)))); return Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!("record rejected by data store: {}", to_hex_string(&key)),
));
} }
_ => {} _ => {}
} }
@ -615,7 +672,14 @@ impl Connection {
let mut header: [u8; 16] = unsafe { MaybeUninit::uninit().assume_init() }; let mut header: [u8; 16] = unsafe { MaybeUninit::uninit().assume_init() };
header[0] = message_type as u8; header[0] = message_type as u8;
let header_size = 1 + varint::encode(&mut header[1..], data.len() as u64); let header_size = 1 + varint::encode(&mut header[1..], data.len() as u64);
if self.writer.lock().await.write_vectored(&[IoSlice::new(&header[0..header_size]), IoSlice::new(data)]).await? == (data.len() + header_size) { if self
.writer
.lock()
.await
.write_vectored(&[IoSlice::new(&header[0..header_size]), IoSlice::new(data)])
.await?
== (data.len() + header_size)
{
self.last_send_time.store(now, Ordering::Relaxed); self.last_send_time.store(now, Ordering::Relaxed);
Ok(()) Ok(())
} else { } else {
@ -628,7 +692,10 @@ impl Connection {
if rmp_serde::encode::write_named(write_buf, obj).is_ok() { if rmp_serde::encode::write_named(write_buf, obj).is_ok() {
self.send_msg(message_type, write_buf.as_slice(), now).await self.send_msg(message_type, write_buf.as_slice(), now).await
} else { } else {
Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "serialize failure (internal error)")) Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"serialize failure (internal error)",
))
} }
} }
} }

9
attic/syncwhole/src/utils.rs
View file

@ -16,12 +16,17 @@ use tokio::task::JoinHandle;
/// Get the real time clock in milliseconds since Unix epoch. /// Get the real time clock in milliseconds since Unix epoch.
pub fn ms_since_epoch() -> i64 { pub fn ms_since_epoch() -> i64 {
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis() as i64 std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as i64
} }
/// Encode a byte slice to a hexadecimal string. /// Encode a byte slice to a hexadecimal string.
pub fn to_hex_string(b: &[u8]) -> String { pub fn to_hex_string(b: &[u8]) -> String {
const HEX_CHARS: [u8; 16] = [b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f']; const HEX_CHARS: [u8; 16] = [
b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f',
];
let mut s = String::new(); let mut s = String::new();
s.reserve(b.len() * 2); s.reserve(b.len() * 2);
for c in b { for c in b {

4
crypto/benches/benchmark_crypto.rs
View file

@ -19,7 +19,9 @@ pub fn criterion_benchmark(c: &mut Criterion) {
let mut group = c.benchmark_group("cryptography"); let mut group = c.benchmark_group("cryptography");
group.measurement_time(Duration::new(10, 0)); group.measurement_time(Duration::new(10, 0));
group.bench_function("ecdhp384", |b| b.iter(|| p384_a.agree(p384_b.public_key()).expect("ecdhp384 failed"))); group.bench_function("ecdhp384", |b| {
b.iter(|| p384_a.agree(p384_b.public_key()).expect("ecdhp384 failed"))
});
group.bench_function("ecdhx25519", |b| b.iter(|| x25519_a.agree(&x25519_b_pub))); group.bench_function("ecdhx25519", |b| b.iter(|| x25519_a.agree(&x25519_b_pub)));
group.bench_function("kyber_encapsulate", |b| { group.bench_function("kyber_encapsulate", |b| {
b.iter(|| pqc_kyber::encapsulate(&kyber_a.public, &mut random::SecureRandom::default()).expect("kyber encapsulate failed")) b.iter(|| pqc_kyber::encapsulate(&kyber_a.public, &mut random::SecureRandom::default()).expect("kyber encapsulate failed"))

72
crypto/src/aes.rs
View file

@ -36,7 +36,14 @@ mod fruit_flavored {
options: i32, options: i32,
cryyptor_ref: *mut *mut c_void, cryyptor_ref: *mut *mut c_void,
) -> i32; ) -> i32;
fn CCCryptorUpdate(cryptor_ref: *mut c_void, data_in: *const c_void, data_in_len: usize, data_out: *mut c_void, data_out_len: usize, data_out_written: *mut usize) -> i32; fn CCCryptorUpdate(
cryptor_ref: *mut c_void,
data_in: *const c_void,
data_in_len: usize,
data_out: *mut c_void,
data_out_len: usize,
data_out_written: *mut usize,
) -> i32;
//fn CCCryptorReset(cryptor_ref: *mut c_void, iv: *const c_void) -> i32; //fn CCCryptorReset(cryptor_ref: *mut c_void, iv: *const c_void) -> i32;
fn CCCryptorRelease(cryptor_ref: *mut c_void) -> i32; fn CCCryptorRelease(cryptor_ref: *mut c_void) -> i32;
fn CCCryptorGCMSetIV(cryptor_ref: *mut c_void, iv: *const c_void, iv_len: usize) -> i32; fn CCCryptorGCMSetIV(cryptor_ref: *mut c_void, iv: *const c_void, iv_len: usize) -> i32;
@ -110,7 +117,14 @@ mod fruit_flavored {
assert_eq!(ciphertext.len(), 16); assert_eq!(ciphertext.len(), 16);
unsafe { unsafe {
let mut data_out_written = 0; let mut data_out_written = 0;
CCCryptorUpdate(self.0, plaintext.as_ptr().cast(), 16, ciphertext.as_mut_ptr().cast(), 16, &mut data_out_written); CCCryptorUpdate(
self.0,
plaintext.as_ptr().cast(),
16,
ciphertext.as_mut_ptr().cast(),
16,
&mut data_out_written,
);
} }
} }
@ -119,7 +133,14 @@ mod fruit_flavored {
assert_eq!(data.len(), 16); assert_eq!(data.len(), 16);
unsafe { unsafe {
let mut data_out_written = 0; let mut data_out_written = 0;
CCCryptorUpdate(self.0, data.as_ptr().cast(), 16, data.as_mut_ptr().cast(), 16, &mut data_out_written); CCCryptorUpdate(
self.0,
data.as_ptr().cast(),
16,
data.as_mut_ptr().cast(),
16,
&mut data_out_written,
);
} }
} }
@ -129,7 +150,14 @@ mod fruit_flavored {
assert_eq!(ciphertext.len(), 16); assert_eq!(ciphertext.len(), 16);
unsafe { unsafe {
let mut data_out_written = 0; let mut data_out_written = 0;
CCCryptorUpdate(self.1, ciphertext.as_ptr().cast(), 16, plaintext.as_mut_ptr().cast(), 16, &mut data_out_written); CCCryptorUpdate(
self.1,
ciphertext.as_ptr().cast(),
16,
plaintext.as_mut_ptr().cast(),
16,
&mut data_out_written,
);
} }
} }
@ -138,7 +166,14 @@ mod fruit_flavored {
assert_eq!(data.len(), 16); assert_eq!(data.len(), 16);
unsafe { unsafe {
let mut data_out_written = 0; let mut data_out_written = 0;
CCCryptorUpdate(self.1, data.as_ptr().cast(), 16, data.as_mut_ptr().cast(), 16, &mut data_out_written); CCCryptorUpdate(
self.1,
data.as_ptr().cast(),
16,
data.as_mut_ptr().cast(),
16,
&mut data_out_written,
);
} }
} }
} }
@ -215,9 +250,15 @@ mod fruit_flavored {
unsafe { unsafe {
assert_eq!(input.len(), output.len()); assert_eq!(input.len(), output.len());
if self.1 { if self.1 {
assert_eq!(CCCryptorGCMEncrypt(self.0, input.as_ptr().cast(), input.len(), output.as_mut_ptr().cast()), 0); assert_eq!(
CCCryptorGCMEncrypt(self.0, input.as_ptr().cast(), input.len(), output.as_mut_ptr().cast()),
0
);
} else { } else {
assert_eq!(CCCryptorGCMDecrypt(self.0, input.as_ptr().cast(), input.len(), output.as_mut_ptr().cast()), 0); assert_eq!(
CCCryptorGCMDecrypt(self.0, input.as_ptr().cast(), input.len(), output.as_mut_ptr().cast()),
0
);
} }
} }
} }
@ -226,9 +267,15 @@ mod fruit_flavored {
pub fn crypt_in_place(&mut self, data: &mut [u8]) { pub fn crypt_in_place(&mut self, data: &mut [u8]) {
unsafe { unsafe {
if self.1 { if self.1 {
assert_eq!(CCCryptorGCMEncrypt(self.0, data.as_ptr().cast(), data.len(), data.as_mut_ptr().cast()), 0); assert_eq!(
CCCryptorGCMEncrypt(self.0, data.as_ptr().cast(), data.len(), data.as_mut_ptr().cast()),
0
);
} else { } else {
assert_eq!(CCCryptorGCMDecrypt(self.0, data.as_ptr().cast(), data.len(), data.as_mut_ptr().cast()), 0); assert_eq!(
CCCryptorGCMDecrypt(self.0, data.as_ptr().cast(), data.len(), data.as_mut_ptr().cast()),
0
);
} }
} }
} }
@ -392,7 +439,12 @@ mod openssl_aes {
/// Encrypt or decrypt in place (same operation with CTR mode) /// Encrypt or decrypt in place (same operation with CTR mode)
#[inline(always)] #[inline(always)]
pub fn crypt_in_place(&mut self, data: &mut [u8]) { pub fn crypt_in_place(&mut self, data: &mut [u8]) {
assert!(self.2.as_mut().unwrap().update(unsafe { &*std::slice::from_raw_parts(data.as_ptr(), data.len()) }, data).is_ok()); assert!(self
.2
.as_mut()
.unwrap()
.update(unsafe { &*std::slice::from_raw_parts(data.as_ptr(), data.len()) }, data)
.is_ok());
} }
#[inline(always)] #[inline(always)]

110
crypto/src/aes_gmac_siv/impl_macos.rs
View file

@ -35,7 +35,14 @@ extern "C" {
options: i32, options: i32,
cryyptor_ref: *mut *mut c_void, cryyptor_ref: *mut *mut c_void,
) -> i32; ) -> i32;
fn CCCryptorUpdate(cryptor_ref: *mut c_void, data_in: *const c_void, data_in_len: usize, data_out: *mut c_void, data_out_len: usize, data_out_written: *mut usize) -> i32; fn CCCryptorUpdate(
cryptor_ref: *mut c_void,
data_in: *const c_void,
data_in_len: usize,
data_out: *mut c_void,
data_out_len: usize,
data_out_written: *mut usize,
) -> i32;
fn CCCryptorReset(cryptor_ref: *mut c_void, iv: *const c_void) -> i32; fn CCCryptorReset(cryptor_ref: *mut c_void, iv: *const c_void) -> i32;
fn CCCryptorRelease(cryptor_ref: *mut c_void) -> i32; fn CCCryptorRelease(cryptor_ref: *mut c_void) -> i32;
fn CCCryptorGCMSetIV(cryptor_ref: *mut c_void, iv: *const c_void, iv_len: usize) -> i32; fn CCCryptorGCMSetIV(cryptor_ref: *mut c_void, iv: *const c_void, iv_len: usize) -> i32;
@ -65,7 +72,20 @@ impl AesCtr {
} }
unsafe { unsafe {
let mut ptr: *mut c_void = null_mut(); let mut ptr: *mut c_void = null_mut();
let result = CCCryptorCreateWithMode(kCCEncrypt, kCCModeCTR, kCCAlgorithmAES, 0, crate::ZEROES.as_ptr().cast(), k.as_ptr().cast(), k.len(), null(), 0, 0, 0, &mut ptr); let result = CCCryptorCreateWithMode(
kCCEncrypt,
kCCModeCTR,
kCCAlgorithmAES,
0,
crate::ZEROES.as_ptr().cast(),
k.as_ptr().cast(),
k.len(),
null(),
0,
0,
0,
&mut ptr,
);
if result != 0 { if result != 0 {
panic!("CCCryptorCreateWithMode for CTR mode returned {}", result); panic!("CCCryptorCreateWithMode for CTR mode returned {}", result);
} }
@ -99,7 +119,14 @@ impl AesCtr {
unsafe { unsafe {
assert!(output.len() >= input.len()); assert!(output.len() >= input.len());
let mut data_out_written: usize = 0; let mut data_out_written: usize = 0;
CCCryptorUpdate(self.0, input.as_ptr().cast(), input.len(), output.as_mut_ptr().cast(), output.len(), &mut data_out_written); CCCryptorUpdate(
self.0,
input.as_ptr().cast(),
input.len(),
output.as_mut_ptr().cast(),
output.len(),
&mut data_out_written,
);
} }
} }
@ -108,7 +135,14 @@ impl AesCtr {
pub fn crypt_in_place(&mut self, data: &mut [u8]) { pub fn crypt_in_place(&mut self, data: &mut [u8]) {
unsafe { unsafe {
let mut data_out_written: usize = 0; let mut data_out_written: usize = 0;
CCCryptorUpdate(self.0, data.as_ptr().cast(), data.len(), data.as_mut_ptr().cast(), data.len(), &mut data_out_written); CCCryptorUpdate(
self.0,
data.as_ptr().cast(),
data.len(),
data.as_mut_ptr().cast(),
data.len(),
&mut data_out_written,
);
} }
} }
} }
@ -164,8 +198,20 @@ impl AesGmacSiv {
gmac: null_mut(), gmac: null_mut(),
}; };
unsafe { unsafe {
let result = let result = CCCryptorCreateWithMode(
CCCryptorCreateWithMode(kCCEncrypt, kCCModeCTR, kCCAlgorithmAES, 0, crate::ZEROES.as_ptr().cast(), k1.as_ptr().cast(), k1.len(), null(), 0, 0, 0, &mut c.ctr); kCCEncrypt,
kCCModeCTR,
kCCAlgorithmAES,
0,
crate::ZEROES.as_ptr().cast(),
k1.as_ptr().cast(),
k1.len(),
null(),
0,
0,
0,
&mut c.ctr,
);
if result != 0 { if result != 0 {
panic!("CCCryptorCreateWithMode for CTR mode returned {}", result); panic!("CCCryptorCreateWithMode for CTR mode returned {}", result);
} }
@ -203,8 +249,20 @@ impl AesGmacSiv {
if result != 0 { if result != 0 {
panic!("CCCryptorCreateWithMode for ECB decrypt mode returned {}", result); panic!("CCCryptorCreateWithMode for ECB decrypt mode returned {}", result);
} }
let result = let result = CCCryptorCreateWithMode(
CCCryptorCreateWithMode(kCCEncrypt, kCCModeGCM, kCCAlgorithmAES, 0, crate::ZEROES.as_ptr().cast(), k0.as_ptr().cast(), k0.len(), null(), 0, 0, 0, &mut c.gmac); kCCEncrypt,
kCCModeGCM,
kCCAlgorithmAES,
0,
crate::ZEROES.as_ptr().cast(),
k0.as_ptr().cast(),
k0.len(),
null(),
0,
0,
0,
&mut c.gmac,
);
if result != 0 { if result != 0 {
panic!("CCCryptorCreateWithMode for GCM (GMAC) mode returned {}", result); panic!("CCCryptorCreateWithMode for GCM (GMAC) mode returned {}", result);
} }
@ -262,7 +320,14 @@ impl AesGmacSiv {
let tmp = self.tmp.as_mut_ptr().cast::<u64>(); let tmp = self.tmp.as_mut_ptr().cast::<u64>();
*self.tag.as_mut_ptr().cast::<u64>().offset(1) = *tmp ^ *tmp.offset(1); *self.tag.as_mut_ptr().cast::<u64>().offset(1) = *tmp ^ *tmp.offset(1);
let mut data_out_written: usize = 0; let mut data_out_written: usize = 0;
CCCryptorUpdate(self.ecb_enc, self.tag.as_ptr().cast(), 16, self.tag.as_mut_ptr().cast(), 16, &mut data_out_written); CCCryptorUpdate(
self.ecb_enc,
self.tag.as_ptr().cast(),
16,
self.tag.as_mut_ptr().cast(),
16,
&mut data_out_written,
);
} }
self.tmp.copy_from_slice(&self.tag); self.tmp.copy_from_slice(&self.tag);
self.tmp[12] &= 0x7f; self.tmp[12] &= 0x7f;
@ -280,7 +345,14 @@ impl AesGmacSiv {
unsafe { unsafe {
assert!(ciphertext.len() >= plaintext.len()); assert!(ciphertext.len() >= plaintext.len());
let mut data_out_written: usize = 0; let mut data_out_written: usize = 0;
CCCryptorUpdate(self.ctr, plaintext.as_ptr().cast(), plaintext.len(), ciphertext.as_mut_ptr().cast(), ciphertext.len(), &mut data_out_written); CCCryptorUpdate(
self.ctr,
plaintext.as_ptr().cast(),
plaintext.len(),
ciphertext.as_mut_ptr().cast(),
ciphertext.len(),
&mut data_out_written,
);
} }
} }
@ -316,7 +388,14 @@ impl AesGmacSiv {
panic!("CCCryptorReset for CTR mode failed (old MacOS bug)"); panic!("CCCryptorReset for CTR mode failed (old MacOS bug)");
} }
let mut data_out_written = 0; let mut data_out_written = 0;
CCCryptorUpdate(self.ecb_dec, self.tag.as_ptr().cast(), 16, self.tag.as_mut_ptr().cast(), 16, &mut data_out_written); CCCryptorUpdate(
self.ecb_dec,
self.tag.as_ptr().cast(),
16,
self.tag.as_mut_ptr().cast(),
16,
&mut data_out_written,
);
let tmp = self.tmp.as_mut_ptr().cast::<u64>(); let tmp = self.tmp.as_mut_ptr().cast::<u64>();
*tmp = *self.tag.as_mut_ptr().cast::<u64>(); *tmp = *self.tag.as_mut_ptr().cast::<u64>();
*tmp.add(1) = 0; *tmp.add(1) = 0;
@ -345,7 +424,14 @@ impl AesGmacSiv {
pub fn decrypt(&mut self, ciphertext: &[u8], plaintext: &mut [u8]) { pub fn decrypt(&mut self, ciphertext: &[u8], plaintext: &mut [u8]) {
unsafe { unsafe {
let mut data_out_written = 0; let mut data_out_written = 0;
CCCryptorUpdate(self.ctr, ciphertext.as_ptr().cast(), ciphertext.len(), plaintext.as_mut_ptr().cast(), plaintext.len(), &mut data_out_written); CCCryptorUpdate(
self.ctr,
ciphertext.as_ptr().cast(),
ciphertext.len(),
plaintext.as_mut_ptr().cast(),
plaintext.len(),
&mut data_out_written,
);
CCCryptorGCMAddAAD(self.gmac, plaintext.as_ptr().cast(), plaintext.len()); CCCryptorGCMAddAAD(self.gmac, plaintext.as_ptr().cast(), plaintext.len());
} }
} }

60
crypto/src/aes_gmac_siv/impl_openssl.rs
View file

@ -56,7 +56,9 @@ impl AesCtr {
/// If it's already been used, this also resets the cipher. There is no separate reset. /// If it's already been used, this also resets the cipher. There is no separate reset.
#[inline(always)] #[inline(always)]
pub fn init(&mut self, iv: &[u8]) { pub fn init(&mut self, iv: &[u8]) {
let _ = self.1.replace(Crypter::new(aes_ctr_by_key_size(self.0.len()), Mode::Encrypt, self.0.as_slice(), Some(iv)).unwrap()); let _ = self
.1
.replace(Crypter::new(aes_ctr_by_key_size(self.0.len()), Mode::Encrypt, self.0.as_slice(), Some(iv)).unwrap());
} }
/// Encrypt or decrypt (same operation with CTR mode) /// Encrypt or decrypt (same operation with CTR mode)
@ -68,7 +70,11 @@ impl AesCtr {
/// Encrypt or decrypt in place (same operation with CTR mode) /// Encrypt or decrypt in place (same operation with CTR mode)
#[inline(always)] #[inline(always)]
pub fn crypt_in_place(&mut self, data: &mut [u8]) { pub fn crypt_in_place(&mut self, data: &mut [u8]) {
let _ = self.1.as_mut().unwrap().update(unsafe { &*std::slice::from_raw_parts(data.as_ptr(), data.len()) }, data); let _ = self
.1
.as_mut()
.unwrap()
.update(unsafe { &*std::slice::from_raw_parts(data.as_ptr(), data.len()) }, data);
} }
} }
@ -116,7 +122,15 @@ impl AesGmacSiv {
pub fn encrypt_init(&mut self, iv: &[u8]) { pub fn encrypt_init(&mut self, iv: &[u8]) {
self.tag[0..8].copy_from_slice(iv); self.tag[0..8].copy_from_slice(iv);
self.tag[8..12].fill(0); self.tag[8..12].fill(0);
let _ = self.gmac.replace(Crypter::new(aes_gcm_by_key_size(self.k0.len()), Mode::Encrypt, self.k0.as_slice(), Some(&self.tag[0..12])).unwrap()); let _ = self.gmac.replace(
Crypter::new(
aes_gcm_by_key_size(self.k0.len()),
Mode::Encrypt,
self.k0.as_slice(),
Some(&self.tag[0..12]),
)
.unwrap(),
);
} }
/// Set additional authenticated data (data to be authenticated but not encrypted). /// Set additional authenticated data (data to be authenticated but not encrypted).
@ -164,7 +178,15 @@ impl AesGmacSiv {
self.tmp.copy_from_slice(&tag_tmp[0..16]); self.tmp.copy_from_slice(&tag_tmp[0..16]);
self.tmp[12] &= 0x7f; self.tmp[12] &= 0x7f;
let _ = self.ctr.replace(Crypter::new(aes_ctr_by_key_size(self.k1.len()), Mode::Encrypt, self.k1.as_slice(), Some(&self.tmp)).unwrap()); let _ = self.ctr.replace(
Crypter::new(
aes_ctr_by_key_size(self.k1.len()),
Mode::Encrypt,
self.k1.as_slice(),
Some(&self.tmp),
)
.unwrap(),
);
} }
/// Feed plaintext for second pass and write ciphertext to supplied buffer. /// Feed plaintext for second pass and write ciphertext to supplied buffer.
@ -178,7 +200,10 @@ impl AesGmacSiv {
/// This may be called more than once. /// This may be called more than once.
#[inline(always)] #[inline(always)]
pub fn encrypt_second_pass_in_place(&mut self, plaintext_to_ciphertext: &mut [u8]) { pub fn encrypt_second_pass_in_place(&mut self, plaintext_to_ciphertext: &mut [u8]) {
let _ = self.ctr.as_mut().unwrap().update(unsafe { std::slice::from_raw_parts(plaintext_to_ciphertext.as_ptr(), plaintext_to_ciphertext.len()) }, plaintext_to_ciphertext); let _ = self.ctr.as_mut().unwrap().update(
unsafe { std::slice::from_raw_parts(plaintext_to_ciphertext.as_ptr(), plaintext_to_ciphertext.len()) },
plaintext_to_ciphertext,
);
} }
/// Finish second pass and return a reference to the tag for this message. /// Finish second pass and return a reference to the tag for this message.
@ -194,7 +219,15 @@ impl AesGmacSiv {
pub fn decrypt_init(&mut self, tag: &[u8]) { pub fn decrypt_init(&mut self, tag: &[u8]) {
self.tmp.copy_from_slice(tag); self.tmp.copy_from_slice(tag);
self.tmp[12] &= 0x7f; self.tmp[12] &= 0x7f;
let _ = self.ctr.replace(Crypter::new(aes_ctr_by_key_size(self.k1.len()), Mode::Decrypt, self.k1.as_slice(), Some(&self.tmp)).unwrap()); let _ = self.ctr.replace(
Crypter::new(
aes_ctr_by_key_size(self.k1.len()),
Mode::Decrypt,
self.k1.as_slice(),
Some(&self.tmp),
)
.unwrap(),
);
let mut tag_tmp = [0_u8; 32]; let mut tag_tmp = [0_u8; 32];
let mut ecb = Crypter::new(aes_ecb_by_key_size(self.k1.len()), Mode::Decrypt, self.k1.as_slice(), None).unwrap(); let mut ecb = Crypter::new(aes_ecb_by_key_size(self.k1.len()), Mode::Decrypt, self.k1.as_slice(), None).unwrap();
@ -204,7 +237,15 @@ impl AesGmacSiv {
} }
self.tag.copy_from_slice(&tag_tmp[0..16]); self.tag.copy_from_slice(&tag_tmp[0..16]);
tag_tmp[8..12].fill(0); tag_tmp[8..12].fill(0);
let _ = self.gmac.replace(Crypter::new(aes_gcm_by_key_size(self.k0.len()), Mode::Encrypt, self.k0.as_slice(), Some(&tag_tmp[0..12])).unwrap()); let _ = self.gmac.replace(
Crypter::new(
aes_gcm_by_key_size(self.k0.len()),
Mode::Encrypt,
self.k0.as_slice(),
Some(&tag_tmp[0..12]),
)
.unwrap(),
);
} }
/// Set additional authenticated data to be checked. /// Set additional authenticated data to be checked.
@ -225,7 +266,10 @@ impl AesGmacSiv {
/// This may be called more than once. /// This may be called more than once.
#[inline(always)] #[inline(always)]
pub fn decrypt_in_place(&mut self, ciphertext_to_plaintext: &mut [u8]) { pub fn decrypt_in_place(&mut self, ciphertext_to_plaintext: &mut [u8]) {
self.decrypt(unsafe { std::slice::from_raw_parts(ciphertext_to_plaintext.as_ptr(), ciphertext_to_plaintext.len()) }, ciphertext_to_plaintext); self.decrypt(
unsafe { std::slice::from_raw_parts(ciphertext_to_plaintext.as_ptr(), ciphertext_to_plaintext.len()) },
ciphertext_to_plaintext,
);
} }
/// Finish decryption and return true if authentication appears valid. /// Finish decryption and return true if authentication appears valid.

533
crypto/src/aes_gmac_siv/mod.rs
View file

@ -20,98 +20,441 @@ mod tests {
use sha2::Digest; use sha2::Digest;
use std::time::SystemTime; use std::time::SystemTime;
const TV0_KEYS: [&'static [u8]; 2] = ["00000000000000000000000000000000".as_bytes(), "11111111111111111111111111111111".as_bytes()]; const TV0_KEYS: [&'static [u8]; 2] = [
"00000000000000000000000000000000".as_bytes(),
"11111111111111111111111111111111".as_bytes(),
];
/// Test vectors consist of a series of input sizes, a SHA384 hash of a resulting ciphertext, and an expected tag. /// Test vectors consist of a series of input sizes, a SHA384 hash of a resulting ciphertext, and an expected tag.
/// Input is a standard byte array consisting of bytes 0, 1, 2, 3, ..., 255 and then cycling back to 0 over and over /// Input is a standard byte array consisting of bytes 0, 1, 2, 3, ..., 255 and then cycling back to 0 over and over
/// and is provided both as ciphertext and associated data (AAD). /// and is provided both as ciphertext and associated data (AAD).
#[allow(unused)] #[allow(unused)]
const TEST_VECTORS: [(usize, &'static str, &'static str); 85] = [ const TEST_VECTORS: [(usize, &'static str, &'static str); 85] = [
(0, "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b", "43847e644239134deccf5538162c861e"), (
(777, "aabf892f18a620b9c3bae91bb03a74c84193e4a7b64916c6bc88b885b9ebed4134495e5f22f12e3046fbb3f26fa111a7", "b8c318b5dcc1d672114a6f7be54ef289"), 0,
(1554, "648f551df29217f0e634b72ba6973c0eb95c7d4be8b135e550d8bcdf65b75980881bc0e03cf22589e04bedc7da1804cd", "535b8ddd51ec82a1e850906fe321b21a"), "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b",
(2331, "bfbfdffea40062e23bbdf0835e1d38d1623bebca7407908bbc6d5b3f2bfd062a2d237f091affda7348094fafda0bd1a7", "4f521876fbb2c563051196b33c20c822"), "43847e644239134deccf5538162c861e",
(3108, "cc6035cab70f3a3298a5c4956ff07f179acf3771bb915c590a8a19fe5133d6d8a81c118148394dfb364af5c2fbdaadeb", "d3adfa578c8bcd738c55ffc527358cef"), ),
(3885, "15ec2760a21c25f9870a84ee757f3da2c261a950c2f692d75ff9e99b2d50c826c21e27e49c4cd3450fedc7e60371589f", "a4c22d6c3d773634c2dc057e1f7c6738"), (
(4662, "c2afad6f034704300c34f143dcdcb86c9b954cec1ebf22e7071f288c58a2ae430d3e3748d214d1021472793d3f337dc6", "c0601cb6cd4883102f70570c2cdc0ab6"), 777,
(5439, "8fee067f5a7a475a630f9db8b2eb80c1edc40eb4246a0f1c078e535df7d06451c6a9bde1a23ba70285690dd7100a8626", "7352239f2302b08844309d28b13fa867"), "aabf892f18a620b9c3bae91bb03a74c84193e4a7b64916c6bc88b885b9ebed4134495e5f22f12e3046fbb3f26fa111a7",
(6216, "60095b4172438aee61e65f5379f4ef276c3632d4ac74eea7723a2201823432614aba7b4670d9bf7a5b9126ca38f3b88a", "c0f0b0aa651965f8514b473c5406285e"), "b8c318b5dcc1d672114a6f7be54ef289",
(6993, "10e754dd08b4d2a6c109fb01fce2b57d54743947e14a7e67d7efd0608baf91f7fc42a53328fe8c18d234abad8ebcdff0", "58444988a62a99060728a7637c8499eb"), ),
(7770, "1abc4a5dcd2696336bd0e8af20fe7fc261aa424b52cfb5ad80ee7c7c793ac44f11db3506cdbbbaed0f80000925d08d52", "e8065c563bc6018cdcbf9aaafef767e6"), (
(8547, "26aaf74ae8bfc6aaf45ceee0476ea0a484304f5c36050d3e2265cb194a2f7c308213314232270608b6d3f1c11b834e33", "ec50e4b3f6e4b3de24b3476623d08157"), 1554,
(9324, "863206305d466aa9c0d0ec674572069f61fe5009767f99ec8832912725c28c49d6a106ad3f55372c922e4e169fc382ce", "0cfac64f49e0f128d0a18d293878f222"), "648f551df29217f0e634b72ba6973c0eb95c7d4be8b135e550d8bcdf65b75980881bc0e03cf22589e04bedc7da1804cd",
(10101, "bd0c0950b947a6c34f1fa6e877433b42c039a8ea7b37634c40fb47efae4958ba74ef0991cfedf3c82a0b87ef59635071", "e0220a02b74259eeebbebede847d50f9"), "535b8ddd51ec82a1e850906fe321b21a",
(10878, "d7b9901af1dacf6a8c369b993ba1c607f9b7f073d02311c72d8449d3494d477ffc8344a1d8b488020ccfc7c80fbd27e1", "ebe3933146734a6ade2b434f2bcd78ae"), ),
(11655, "0ba265e3ef0bebf01a4f3490da462c7730aad6aa6c70bb9ce64a36d26d24fe213660e60e4d3301329170471f11ff8ca2", "ec3dd4bf4cb7d527a86dd559c773a87b"), (
(12432, "c3b6755a1be922ec71c1e187ead36c4e6fc307c72969c64ca1e9b7339d61e1a93a74a315fd73bed8fa5797b78b19dbe5", "5b58dcf392749bcef91056ba9475d0ef"), 2331,
(13209, "2fb1a67151183daa2f0d7f0064534497357f173161349dd008499a8c1a123cc942662ecc426e2ad7743fe0ab9f5d7be1", "c011260d328d310e2ab606aa1ef8afd4"), "bfbfdffea40062e23bbdf0835e1d38d1623bebca7407908bbc6d5b3f2bfd062a2d237f091affda7348094fafda0bd1a7",
(13986, "6afae2a07ce9bfe30fbbfb7dcf32d755bcf357334dc5c309e58cab38ebe559f25b313a0b3ca32ff1dc41f7b99718f653", "011bf43cfbbb7ae5986f8e0fc87771a9"), "4f521876fbb2c563051196b33c20c822",
(14763, "cc6215c115eb6411f4712c2289f5bf0ccb5151635f9f9ceac7c1b62d8d2f4d26498079d0289f83aeb26e97b5b924ffc4", "a015034a8d5bc83cc76c6983a5ba19ab"), ),
(15540, "3cebce794e947341c4ceec444ca43c6ac57c6f58de462bfec7566cbd59a1b6f2eae774120e29521e76120a604d1a12d9", "d373cd2bd9000655141ac632880eca40"), (
(16317, "899147b98d78bb5d137dc7c4f03be7eca82bcca19cc3a701261332923707aed2e6719d35d2f2bf067cd1d193a53529cf", "ed223b64529299c787f49d631ce181c1"), 3108,
(17094, "aecd1830958b994b2c331b90e7d8ff79f27c83a71f5797a65ade3a30b4fa5928e79140bcd03f375591d53df96fea1a4d", "948a7c253d54bb6b65d78530c0eb7aab"), "cc6035cab70f3a3298a5c4956ff07f179acf3771bb915c590a8a19fe5133d6d8a81c118148394dfb364af5c2fbdaadeb",
(17871, "e677ffd4ecaba5899659fefe5fe8e643004392be3be6dc5a801409870ac1e3398f47cc1d83f7a4c41925b6337e01f7fd", "156a600c336f3ac034ca90034aa22635"), "d3adfa578c8bcd738c55ffc527358cef",
(18648, "4ee50f4a98d0bbd160add6acf76765ccdac0c1cd0bb2adbbcb22dd012a1121620b739a120df7dc4091e684ddf28eb726", "75873467b416a7b025f9f1b015bf653a"), ),
(19425, "aa025f32c0575af7209828fc7fc4591b41fa7cfb485e26c5401e63ca1fa05776f8b8af1769a15e81f2c663bca9b02ab3", "5679efa7a4404e1e5c9b372782a41bf2"), (
(20202, "6e77ab62d2affeb27f4ef326191b3df3863c338a629f64a785505f4a5968ff59bc011c7a27951cb00e2e7d9b9bd32fec", "36a9c4515d34f9bb962d8876ab3b5c86"), 3885,
(20979, "1625b4f0e65fc66f11ba3ee6b3e20c732535654c447df6b517ced113107a1057a64477faa2af4a5ede4034bf3cff98ea", "9058044e0f71c28d4f8d3281a3aec024"), "15ec2760a21c25f9870a84ee757f3da2c261a950c2f692d75ff9e99b2d50c826c21e27e49c4cd3450fedc7e60371589f",
(21756, "94efe6aa55bd77bfa58c185dec313a41003f9bef02568e72c337be4de1b46c6e5bb9a9329b4f108686489b8bc9d5f4f0", "8d6d2c90590268a26f5e7d76351f48c1"), "a4c22d6c3d773634c2dc057e1f7c6738",
(22533, "7327a05fdb0ac92433dfc2c85c5e96e6ddcbdb01e079f8dafbee79c14cb4d5fd46047acd6bb0e09a98f6dd03dced2a0a", "4e0f0a394f85bca35c68ef667aa9c244"), ),
(23310, "93da9e356efbc8b5ae366256f4c6fc11c11fc347aaa879d591b7c1262d90adf98925f571914696054f1d09c74783561e", "8c83c157be439280afc790ee3fd667eb"), (
(24087, "99b91be5ffca51b1cbc7410798b1540b5b1a3356f801ed4dc54812919c08ca5a9adc218bc51e594d97b46445a1515506", "9436ff05729a77f673e815e464aeaa75"), 4662,
(24864, "074253ad5d5a5d2b072e7aeaffa04a06119ec812a88ca43481fe5e2dce02cf6736952095cd342ec70b833c12fc1777f4", "69d8951b96866a08efbb65f2bc31cfbc"), "c2afad6f034704300c34f143dcdcb86c9b954cec1ebf22e7071f288c58a2ae430d3e3748d214d1021472793d3f337dc6",
(25641, "c0a301f90597c05cf19e60c35378676764086b7156e455f4800347f8a6e733d644e4cc709fb9d95a9211f3e1e10c762a", "3561c9802143c306ecc5e07e3b976d9e"), "c0601cb6cd4883102f70570c2cdc0ab6",
(26418, "3c839e59d945b841acb604e1b9ae3df36a291444ce0bcae336ee875beaf208bf10af7342b375429ecb92ec54d11a5907", "3032ffdb8daee11b2e739132c6175615"), ),
(27195, "3dc59b16603950dfc26a90bf036712eb088412e8de4d1b27c3fa6be6502ac12d89d194764fb53c3dc7d90fa696ba5a16", "49436717edff7cd67c9a1be16d524f07"), (
(27972, "4fbc0d40ff13376b8ed5382890cdea337b4a0c9c31b477c4008d2ef8299bd5ab771ba70b1b4b743f8f7caa1f0164d1a1", "64a9856a3bb81dc81ff1bc1025192dc9"), 5439,
(28749, "6ab191aa6327f229cc94e8c7b1b7ee30bc723e6aeaf3050eb7d14cb491c3513254e9b19894c2b4f071d298401fd31945", "101f2ffea60f246a3b57c4a530d67cf1"), "8fee067f5a7a475a630f9db8b2eb80c1edc40eb4246a0f1c078e535df7d06451c6a9bde1a23ba70285690dd7100a8626",
(29526, "d06dece58e6c7345986aae4b7f15b3317653f5387d6262f389b5cbbe804568124a876eabb89204e96b3c0f7b552df3c4", "5c0e873adba65a9f4cb24cce4f194b18"), "7352239f2302b08844309d28b13fa867",
(30303, "7a33c1268eafdc1f89ad460fa4ded8d3df9a3cabe4339706877878c64a2c8080cf3fa5ea7f2f24744e3341476b1eb5a5", "b7dc708fc46ce5cde24a31ad549fec83"), ),
(31080, "37bf1f9fca6d705b989b2d63259ca924dc860fc6027e07d9aad79b94841227739774f5d324590df45d8f41249ef742ea", "8ead50308c281e699b79b69dad7ecb91"), (
(31857, "91b120c73be86f9d53326fa707cfa1411e5ac76ab998a2d7ebd73a75e3b1a04c9f0855d102184b8a3fd5d99818b0b134", "6056d09595bd16bfa317c6f87ce64bb7"), 6216,
(32634, "42cc255c06184ead57b27efd0cefb0f2c788c8962a6fd15db3f25533a7f49700bca85af916f9e985f1941a6e66943b38", "3b15e332d2f53bb97e1a9d03e6113b97"), "60095b4172438aee61e65f5379f4ef276c3632d4ac74eea7723a2201823432614aba7b4670d9bf7a5b9126ca38f3b88a",
(33411, "737f8bb8f3fd03a9d13e50abba3a42f4491c36eda3eb215085abda733227ec490cb863ffbd68f915c8fb2926a899fbc3", "b2c647d25c46aab4d4a5ede4a3b4576d"), "c0f0b0aa651965f8514b473c5406285e",
(34188, "e9caa36505e19628175d1ce8b933267380099753a41e503fa2f894cea17b7692f0b27079ed33cdd1293db9a35722d561", "a2882adfd00f22823250215b12b3a1fd"), ),
(34965, "81ddc348ebbdfb963daa5d0c1b51bbb73cacd883d4fc4316db6bd3388779beff7be0655bbac73951f89dc53832199c11", "f33106eb8104f3780350c6d4f82333ad"), (
(35742, "308ce31daf40dab707e2cb4c4a5307bc403e24c971ae1e30e998449f804a167fe5f2cf617d585851b6fe9f2b4209f09c", "44070ac90cbf350ab92289cc063e978c"), 6993,
(36519, "71f51b4bddbe8a52f18be75f9bdb3fca0773901b794de845450fb308c34775ede1a6da9a82b61e9682a29a3ef71274e2", "0e387704298c444bf3afba0edc0c1c1c"), "10e754dd08b4d2a6c109fb01fce2b57d54743947e14a7e67d7efd0608baf91f7fc42a53328fe8c18d234abad8ebcdff0",
(37296, "478ac94eee8c5f96210003fcb478392b91f2ef6fc3a729774e5fe82a2d8d0abc54ae1d25b3eaefb061e2bd43b70ca4ea", "fb65ebeda52cd5848d303c0677cecb7f"), "58444988a62a99060728a7637c8499eb",
(38073, "bc3a9390618da7d644be932627353e2c92024df939d2d8497fba61fae3dd822cdd3e130c1707f4a9d5d4a0cbb4b3e0b3", "d790d529a837ec79f7cc3f66ed9a399f"), ),
(38850, "ef0e63a53a10e56477c47e13320b8a7d330aee3a4363c850edc56c0707a2686478e5a5193f54ceb33467ab7e8a22aa21", "6f2c18742f106f16fc290767342fb62b"), (
(39627, "c16f63533c099d872d9a01c326db7756e7eb488c756b9a6ebf575993d8ea2eb45c572b2e162f061e145710e0e21e8e18", "a57afde7938b223ae5e109a03db4ee4c"), 7770,
(40404, "ade484ae8c13465a73589ef14789bb6891c933453e198df84edd34b4ac5c83aa90f2cf61fa072fa4d8f5b5c4cd68fa9e", "a01d13009db86ac442f7afd39d83309f"), "1abc4a5dcd2696336bd0e8af20fe7fc261aa424b52cfb5ad80ee7c7c793ac44f11db3506cdbbbaed0f80000925d08d52",
(41181, "6c5c7eed0e043a0bd60bcac9b5b546e150028d70c1efefc9ff69037ef4dc1a36878b171b9f2a639df822d11054a0e405", "6321c8622ca5866c875d340206d06a28"), "e8065c563bc6018cdcbf9aaafef767e6",
(41958, "dd311c54222fb0d92858719cf5b1c51bb5e3ca2539ffd68f1dd6c7e38969495be935804855ccdcc4b4cf221fcdbda886", "cf401eb819b5dc5cd8c909aae9b3b34b"), ),
(42735, "31cda9d663199b32eff042dd16c0b909ba999641e77ba751c91752bfc4d595e17ec6467119e74a600b72da72ba287d0a", "12fd6298ab5d744eb6ade3106565afad"), (
(43512, "11b014057d51a8384d549d5d083c4406b575df6a9295853dd8f2f84f078cc241bb90495a119126b10b9510efcb68c0d3", "a48a49eea5dc90359ef21f32132f8604"), 8547,
(44289, "b44f5dbeecd76ee7efe3fb4dfe10ba8135d7a5e4d104149f4a91c5c6ee9446d9be19fb4c9ba668b074466d3892e22228", "07e1cbb7a19174d9b1e4d5a2c741cc14"), "26aaf74ae8bfc6aaf45ceee0476ea0a484304f5c36050d3e2265cb194a2f7c308213314232270608b6d3f1c11b834e33",
(45066, "d87bbba3a3c739cab622386c89aeb685a70009fab1a606bd34622adfa3a75a05b58d56ee6b9874d414db38a6a32927b3", "a27cd252712cd2a1a2d95dea39f888d4"), "ec50e4b3f6e4b3de24b3476623d08157",
(45843, "abb90e60ea13c6cb3b401b8e271637416b87fbede165dde7be1d34abe4427dae4b39b499352cacac909bc43fb94028c8", "df3ae762b9257936feda435a61a9c3a1"), ),
(46620, "56d1132ee6e0f85543950d2d9667244b66b0ce6414eacd1859b128ed0b9026b31a25bfdcce3d1a0ce7c39d99f609c89c", "cfe7c3c3f1cb615e2d210cc8136443e6"), (
(47397, "ecb023ec4c23cf95d1848a38b359f1f590f172dee9d8fb1be6bc9c4fb2ce96f612d60d7b111de539ab8313a87b821176", "501d24752bf55cb12239863981898a07"), 9324,
(48174, "34236ab60f05bb510aa0880fec358fb2002903efa14c912cab8a399e09418f97223ca2f7b8d6798c11d39e79032eaaa8", "4ecaba4eae886aa429927188abab9623"), "863206305d466aa9c0d0ec674572069f61fe5009767f99ec8832912725c28c49d6a106ad3f55372c922e4e169fc382ce",
(48951, "55e8b40fad90a3d8c85a0f4d5bcf5975b8a6e2fb78377109f5b607a5e367187fbbc9a1e978aab3228fbf43ad23d0ad13", "84c43bc30eb4a67230b6c634fe3c7782"), "0cfac64f49e0f128d0a18d293878f222",
(49728, "14b1f896d0d01ecff4e456c3c392b1ca2bad9f1ef07713f84cdd89e663aa27ca77d80213ed57a89431eb992b11d98749", "7f58c2f9a249f70fe1c6f9b4f65e5a1d"), ),
(50505, "1335b1fb56196e0b371fa53ab7445845fdefcea3eb2833478deb3526e2ec888945e95ee8239b52caae5b9920ba4f43bb", "5fd729126b236ce3e0686fc706dce20f"), (
(51282, "0d1983a6cab870c5e78f89a11dd30e7d2c71a3882f8bba3e71dc1b96a2d9fc6cc6d91d683b74456b886de34df792cfda", "7731ae6e6c54dfde12f6116357e812ea"), 10101,
(52059, "9d619fb4aa8441baaefed7b778693c291f2c1441b206ec135930fac3529d26587ac36f4472949e0b198b51c0c5a9d0f8", "39db2c996aea28996e03d576c118630f"), "bd0c0950b947a6c34f1fa6e877433b42c039a8ea7b37634c40fb47efae4958ba74ef0991cfedf3c82a0b87ef59635071",
(52836, "31dca4fa285878ba3efc3b66a248a078b69a11c3c73f81077377c4ffcb7002627aad5faa955e3141c1d8508aad68c8f6", "32ac1e5a09e7e629ff95f30aa9b69c00"), "e0220a02b74259eeebbebede847d50f9",
(53613, "931a9969cf2bb02302c32b1eecd4933805e2da403d85aaf98c82c68129fb95f089eb85c65a6fcbc7d81bedb39de0cabb", "1a6f54b87c12868da530eac94d99eb31"), ),
(54390, "2f0742565801a37810ecb3f50a6f782e73a369a790d1a6a85135e7ffa12fc063db8909ab9eca7cf7308832887a6149d1", "1b18ed6a8f901b7947626216839f0643"), (
(55167, "901defbd308b54deef89acd0d94e4387b370f9d2e6f870d72da2e447ed3ebe69c5f9f144488bd6207a732102160bff47", "1e0e6a05fcc0794121f617e28cfac1a0"), 10878,
(55944, "df984a5f7475250155dc4733a746e98446dc93a56a3f3bff691ddfef7deefb32b1da1b0e7e15cce443831ebfb3e30ada", "876121af882d0ebeae38f111f3d4b6e8"), "d7b9901af1dacf6a8c369b993ba1c607f9b7f073d02311c72d8449d3494d477ffc8344a1d8b488020ccfc7c80fbd27e1",
(56721, "acb693ed837b33561408cb1eed636e0082ac404f3fd72d277fa146ae5cd81a1fde3645f4cdc7babd8ba044b78075cb67", "5b90ed6c7943fc6da623c536e2ff1352"), "ebe3933146734a6ade2b434f2bcd78ae",
(57498, "dffb54bf5938e812076cfbf15cd524d72a189566c7980363a49dd89fb49e230d9742ef0b0e1ac543dca14366d735d152", "22aee072457306e32747fbbbc3ae127c"), ),
(58275, "92dbc245a980fc78974f7a27e62c22b12a00be9d3ef8d3718ff85f6d5fbcbf1d9d1e0f0a3daeb8c2628d090550a0ff6b", "5fa348117faba4ac8c9d9317ff44cd2d"), (
(59052, "57721475cb719691850696d9a8ad4c28ca8ef9a7d45874ca21df4df250cb87ea60c464f4e3252e2d6161ed36c4b56d75", "24d92ae7cac56d9c0276b06f7428d5df"), 11655,
(59829, "d0936026440b5276747cb9fb7dc96de5d4e7846c233ca5f6f9354b2b39f760333483cbe99ffa905facb347242f58a7ef", "05c57068e183f9d835e7f461202f923c"), "0ba265e3ef0bebf01a4f3490da462c7730aad6aa6c70bb9ce64a36d26d24fe213660e60e4d3301329170471f11ff8ca2",
(60606, "7b3bb3527b73a8692f076f6a503b2e09b427119543c7812db73c7c7fb2d43af9ecbd2a8a1452ac8ada96ad0bad7bb185", "f958635a193fec0bfb958e97961381df"), "ec3dd4bf4cb7d527a86dd559c773a87b",
(61383, "ff0d00255a36747eced86acfccd0cf9ef09faa9f44c8cf382efec462e7ead66e562a971060c3f32798ba142d9e1640a2", "838159b222e56aadde8229ed56a14095"), ),
(62160, "15806e088ed1428cd73ede3fecf5b60e2a616f1925004dadd2cab8e847059f795659659e82a4554f270baf88bf60af63", "fed2aa0c9c0a73d499cc970aef21c52f"), (
(62937, "cfad71b23b6da51256bd1ddbd1ac77977fe10b2ad0a830a23a794cef914bf71a9519d78a5f83fc411e8d8db996a45d4e", "e1ea412fd3e1bd91c24b6b6445e8ff43"), 12432,
(63714, "7d03a3698a79b1af1663e3e485c2efdc306ecd87b2644f2e01d83a35999d6cdf12241b6114d60d107c10c0d0c9cc0d23", "e6a3c3f3fd2d9cfcdc06cca2f59e9a83"), "c3b6755a1be922ec71c1e187ead36c4e6fc307c72969c64ca1e9b7339d61e1a93a74a315fd73bed8fa5797b78b19dbe5",
(64491, "e12b168cce0e82ed1db88df549f39b3ff40b5884a09fceae69c4c3db13c1c37ea79531c47b2700d1c27774a1ab7e8b35", "4cbb14d789f5cd8eca49ce9e1d442ea1"), "5b58dcf392749bcef91056ba9475d0ef",
(65268, "056c9d1172cfa76ce7f19c605e5969c284b82dca155dc9c1ed58062ab4d5a7704e27fe69f3aa745b73f45f1cd0ee57df", "8195187f092d52c2a8695b680568b934"), ),
(
13209,
"2fb1a67151183daa2f0d7f0064534497357f173161349dd008499a8c1a123cc942662ecc426e2ad7743fe0ab9f5d7be1",
"c011260d328d310e2ab606aa1ef8afd4",
),
(
13986,
"6afae2a07ce9bfe30fbbfb7dcf32d755bcf357334dc5c309e58cab38ebe559f25b313a0b3ca32ff1dc41f7b99718f653",
"011bf43cfbbb7ae5986f8e0fc87771a9",
),
(
14763,
"cc6215c115eb6411f4712c2289f5bf0ccb5151635f9f9ceac7c1b62d8d2f4d26498079d0289f83aeb26e97b5b924ffc4",
"a015034a8d5bc83cc76c6983a5ba19ab",
),
(
15540,
"3cebce794e947341c4ceec444ca43c6ac57c6f58de462bfec7566cbd59a1b6f2eae774120e29521e76120a604d1a12d9",
"d373cd2bd9000655141ac632880eca40",
),
(
16317,
"899147b98d78bb5d137dc7c4f03be7eca82bcca19cc3a701261332923707aed2e6719d35d2f2bf067cd1d193a53529cf",
"ed223b64529299c787f49d631ce181c1",
),
(
17094,
"aecd1830958b994b2c331b90e7d8ff79f27c83a71f5797a65ade3a30b4fa5928e79140bcd03f375591d53df96fea1a4d",
"948a7c253d54bb6b65d78530c0eb7aab",
),
(
17871,
"e677ffd4ecaba5899659fefe5fe8e643004392be3be6dc5a801409870ac1e3398f47cc1d83f7a4c41925b6337e01f7fd",
"156a600c336f3ac034ca90034aa22635",
),
(
18648,
"4ee50f4a98d0bbd160add6acf76765ccdac0c1cd0bb2adbbcb22dd012a1121620b739a120df7dc4091e684ddf28eb726",
"75873467b416a7b025f9f1b015bf653a",
),
(
19425,
"aa025f32c0575af7209828fc7fc4591b41fa7cfb485e26c5401e63ca1fa05776f8b8af1769a15e81f2c663bca9b02ab3",
"5679efa7a4404e1e5c9b372782a41bf2",
),
(
20202,
"6e77ab62d2affeb27f4ef326191b3df3863c338a629f64a785505f4a5968ff59bc011c7a27951cb00e2e7d9b9bd32fec",
"36a9c4515d34f9bb962d8876ab3b5c86",
),
(
20979,
"1625b4f0e65fc66f11ba3ee6b3e20c732535654c447df6b517ced113107a1057a64477faa2af4a5ede4034bf3cff98ea",
"9058044e0f71c28d4f8d3281a3aec024",
),
(
21756,
"94efe6aa55bd77bfa58c185dec313a41003f9bef02568e72c337be4de1b46c6e5bb9a9329b4f108686489b8bc9d5f4f0",
"8d6d2c90590268a26f5e7d76351f48c1",
),
(
22533,
"7327a05fdb0ac92433dfc2c85c5e96e6ddcbdb01e079f8dafbee79c14cb4d5fd46047acd6bb0e09a98f6dd03dced2a0a",
"4e0f0a394f85bca35c68ef667aa9c244",
),
(
23310,
"93da9e356efbc8b5ae366256f4c6fc11c11fc347aaa879d591b7c1262d90adf98925f571914696054f1d09c74783561e",
"8c83c157be439280afc790ee3fd667eb",
),
(
24087,
"99b91be5ffca51b1cbc7410798b1540b5b1a3356f801ed4dc54812919c08ca5a9adc218bc51e594d97b46445a1515506",
"9436ff05729a77f673e815e464aeaa75",
),
(
24864,
"074253ad5d5a5d2b072e7aeaffa04a06119ec812a88ca43481fe5e2dce02cf6736952095cd342ec70b833c12fc1777f4",
"69d8951b96866a08efbb65f2bc31cfbc",
),
(
25641,
"c0a301f90597c05cf19e60c35378676764086b7156e455f4800347f8a6e733d644e4cc709fb9d95a9211f3e1e10c762a",
"3561c9802143c306ecc5e07e3b976d9e",
),
(
26418,
"3c839e59d945b841acb604e1b9ae3df36a291444ce0bcae336ee875beaf208bf10af7342b375429ecb92ec54d11a5907",
"3032ffdb8daee11b2e739132c6175615",
),
(
27195,
"3dc59b16603950dfc26a90bf036712eb088412e8de4d1b27c3fa6be6502ac12d89d194764fb53c3dc7d90fa696ba5a16",
"49436717edff7cd67c9a1be16d524f07",
),
(
27972,
"4fbc0d40ff13376b8ed5382890cdea337b4a0c9c31b477c4008d2ef8299bd5ab771ba70b1b4b743f8f7caa1f0164d1a1",
"64a9856a3bb81dc81ff1bc1025192dc9",
),
(
28749,
"6ab191aa6327f229cc94e8c7b1b7ee30bc723e6aeaf3050eb7d14cb491c3513254e9b19894c2b4f071d298401fd31945",
"101f2ffea60f246a3b57c4a530d67cf1",
),
(
29526,
"d06dece58e6c7345986aae4b7f15b3317653f5387d6262f389b5cbbe804568124a876eabb89204e96b3c0f7b552df3c4",
"5c0e873adba65a9f4cb24cce4f194b18",
),
(
30303,
"7a33c1268eafdc1f89ad460fa4ded8d3df9a3cabe4339706877878c64a2c8080cf3fa5ea7f2f24744e3341476b1eb5a5",
"b7dc708fc46ce5cde24a31ad549fec83",
),
(
31080,
"37bf1f9fca6d705b989b2d63259ca924dc860fc6027e07d9aad79b94841227739774f5d324590df45d8f41249ef742ea",
"8ead50308c281e699b79b69dad7ecb91",
),
(
31857,
"91b120c73be86f9d53326fa707cfa1411e5ac76ab998a2d7ebd73a75e3b1a04c9f0855d102184b8a3fd5d99818b0b134",
"6056d09595bd16bfa317c6f87ce64bb7",
),
(
32634,
"42cc255c06184ead57b27efd0cefb0f2c788c8962a6fd15db3f25533a7f49700bca85af916f9e985f1941a6e66943b38",
"3b15e332d2f53bb97e1a9d03e6113b97",
),
(
33411,
"737f8bb8f3fd03a9d13e50abba3a42f4491c36eda3eb215085abda733227ec490cb863ffbd68f915c8fb2926a899fbc3",
"b2c647d25c46aab4d4a5ede4a3b4576d",
),
(
34188,
"e9caa36505e19628175d1ce8b933267380099753a41e503fa2f894cea17b7692f0b27079ed33cdd1293db9a35722d561",
"a2882adfd00f22823250215b12b3a1fd",
),
(
34965,
"81ddc348ebbdfb963daa5d0c1b51bbb73cacd883d4fc4316db6bd3388779beff7be0655bbac73951f89dc53832199c11",
"f33106eb8104f3780350c6d4f82333ad",
),
(
35742,
"308ce31daf40dab707e2cb4c4a5307bc403e24c971ae1e30e998449f804a167fe5f2cf617d585851b6fe9f2b4209f09c",
"44070ac90cbf350ab92289cc063e978c",
),
(
36519,
"71f51b4bddbe8a52f18be75f9bdb3fca0773901b794de845450fb308c34775ede1a6da9a82b61e9682a29a3ef71274e2",
"0e387704298c444bf3afba0edc0c1c1c",
),
(
37296,
"478ac94eee8c5f96210003fcb478392b91f2ef6fc3a729774e5fe82a2d8d0abc54ae1d25b3eaefb061e2bd43b70ca4ea",
"fb65ebeda52cd5848d303c0677cecb7f",
),
(
38073,
"bc3a9390618da7d644be932627353e2c92024df939d2d8497fba61fae3dd822cdd3e130c1707f4a9d5d4a0cbb4b3e0b3",
"d790d529a837ec79f7cc3f66ed9a399f",
),
(
38850,
"ef0e63a53a10e56477c47e13320b8a7d330aee3a4363c850edc56c0707a2686478e5a5193f54ceb33467ab7e8a22aa21",
"6f2c18742f106f16fc290767342fb62b",
),
(
39627,
"c16f63533c099d872d9a01c326db7756e7eb488c756b9a6ebf575993d8ea2eb45c572b2e162f061e145710e0e21e8e18",
"a57afde7938b223ae5e109a03db4ee4c",
),
(
40404,
"ade484ae8c13465a73589ef14789bb6891c933453e198df84edd34b4ac5c83aa90f2cf61fa072fa4d8f5b5c4cd68fa9e",
"a01d13009db86ac442f7afd39d83309f",
),
(
41181,
"6c5c7eed0e043a0bd60bcac9b5b546e150028d70c1efefc9ff69037ef4dc1a36878b171b9f2a639df822d11054a0e405",
"6321c8622ca5866c875d340206d06a28",
),
(
41958,
"dd311c54222fb0d92858719cf5b1c51bb5e3ca2539ffd68f1dd6c7e38969495be935804855ccdcc4b4cf221fcdbda886",
"cf401eb819b5dc5cd8c909aae9b3b34b",
),
(
42735,
"31cda9d663199b32eff042dd16c0b909ba999641e77ba751c91752bfc4d595e17ec6467119e74a600b72da72ba287d0a",
"12fd6298ab5d744eb6ade3106565afad",
),
(
43512,
"11b014057d51a8384d549d5d083c4406b575df6a9295853dd8f2f84f078cc241bb90495a119126b10b9510efcb68c0d3",
"a48a49eea5dc90359ef21f32132f8604",
),
(
44289,
"b44f5dbeecd76ee7efe3fb4dfe10ba8135d7a5e4d104149f4a91c5c6ee9446d9be19fb4c9ba668b074466d3892e22228",
"07e1cbb7a19174d9b1e4d5a2c741cc14",
),
(
45066,
"d87bbba3a3c739cab622386c89aeb685a70009fab1a606bd34622adfa3a75a05b58d56ee6b9874d414db38a6a32927b3",
"a27cd252712cd2a1a2d95dea39f888d4",
),
(
45843,
"abb90e60ea13c6cb3b401b8e271637416b87fbede165dde7be1d34abe4427dae4b39b499352cacac909bc43fb94028c8",
"df3ae762b9257936feda435a61a9c3a1",
),
(
46620,
"56d1132ee6e0f85543950d2d9667244b66b0ce6414eacd1859b128ed0b9026b31a25bfdcce3d1a0ce7c39d99f609c89c",
"cfe7c3c3f1cb615e2d210cc8136443e6",
),
(
47397,
"ecb023ec4c23cf95d1848a38b359f1f590f172dee9d8fb1be6bc9c4fb2ce96f612d60d7b111de539ab8313a87b821176",
"501d24752bf55cb12239863981898a07",
),
(
48174,
"34236ab60f05bb510aa0880fec358fb2002903efa14c912cab8a399e09418f97223ca2f7b8d6798c11d39e79032eaaa8",
"4ecaba4eae886aa429927188abab9623",
),
(
48951,
"55e8b40fad90a3d8c85a0f4d5bcf5975b8a6e2fb78377109f5b607a5e367187fbbc9a1e978aab3228fbf43ad23d0ad13",
"84c43bc30eb4a67230b6c634fe3c7782",
),
(
49728,
"14b1f896d0d01ecff4e456c3c392b1ca2bad9f1ef07713f84cdd89e663aa27ca77d80213ed57a89431eb992b11d98749",
"7f58c2f9a249f70fe1c6f9b4f65e5a1d",
),
(
50505,
"1335b1fb56196e0b371fa53ab7445845fdefcea3eb2833478deb3526e2ec888945e95ee8239b52caae5b9920ba4f43bb",
"5fd729126b236ce3e0686fc706dce20f",
),
(
51282,
"0d1983a6cab870c5e78f89a11dd30e7d2c71a3882f8bba3e71dc1b96a2d9fc6cc6d91d683b74456b886de34df792cfda",
"7731ae6e6c54dfde12f6116357e812ea",
),
(
52059,
"9d619fb4aa8441baaefed7b778693c291f2c1441b206ec135930fac3529d26587ac36f4472949e0b198b51c0c5a9d0f8",
"39db2c996aea28996e03d576c118630f",
),
(
52836,
"31dca4fa285878ba3efc3b66a248a078b69a11c3c73f81077377c4ffcb7002627aad5faa955e3141c1d8508aad68c8f6",
"32ac1e5a09e7e629ff95f30aa9b69c00",
),
(
53613,
"931a9969cf2bb02302c32b1eecd4933805e2da403d85aaf98c82c68129fb95f089eb85c65a6fcbc7d81bedb39de0cabb",
"1a6f54b87c12868da530eac94d99eb31",
),
(
54390,
"2f0742565801a37810ecb3f50a6f782e73a369a790d1a6a85135e7ffa12fc063db8909ab9eca7cf7308832887a6149d1",
"1b18ed6a8f901b7947626216839f0643",
),
(
55167,
"901defbd308b54deef89acd0d94e4387b370f9d2e6f870d72da2e447ed3ebe69c5f9f144488bd6207a732102160bff47",
"1e0e6a05fcc0794121f617e28cfac1a0",
),
(
55944,
"df984a5f7475250155dc4733a746e98446dc93a56a3f3bff691ddfef7deefb32b1da1b0e7e15cce443831ebfb3e30ada",
"876121af882d0ebeae38f111f3d4b6e8",
),
(
56721,
"acb693ed837b33561408cb1eed636e0082ac404f3fd72d277fa146ae5cd81a1fde3645f4cdc7babd8ba044b78075cb67",
"5b90ed6c7943fc6da623c536e2ff1352",
),
(
57498,
"dffb54bf5938e812076cfbf15cd524d72a189566c7980363a49dd89fb49e230d9742ef0b0e1ac543dca14366d735d152",
"22aee072457306e32747fbbbc3ae127c",
),
(
58275,
"92dbc245a980fc78974f7a27e62c22b12a00be9d3ef8d3718ff85f6d5fbcbf1d9d1e0f0a3daeb8c2628d090550a0ff6b",
"5fa348117faba4ac8c9d9317ff44cd2d",
),
(
59052,
"57721475cb719691850696d9a8ad4c28ca8ef9a7d45874ca21df4df250cb87ea60c464f4e3252e2d6161ed36c4b56d75",
"24d92ae7cac56d9c0276b06f7428d5df",
),
(
59829,
"d0936026440b5276747cb9fb7dc96de5d4e7846c233ca5f6f9354b2b39f760333483cbe99ffa905facb347242f58a7ef",
"05c57068e183f9d835e7f461202f923c",
),
(
60606,
"7b3bb3527b73a8692f076f6a503b2e09b427119543c7812db73c7c7fb2d43af9ecbd2a8a1452ac8ada96ad0bad7bb185",
"f958635a193fec0bfb958e97961381df",
),
(
61383,
"ff0d00255a36747eced86acfccd0cf9ef09faa9f44c8cf382efec462e7ead66e562a971060c3f32798ba142d9e1640a2",
"838159b222e56aadde8229ed56a14095",
),
(
62160,
"15806e088ed1428cd73ede3fecf5b60e2a616f1925004dadd2cab8e847059f795659659e82a4554f270baf88bf60af63",
"fed2aa0c9c0a73d499cc970aef21c52f",
),
(
62937,
"cfad71b23b6da51256bd1ddbd1ac77977fe10b2ad0a830a23a794cef914bf71a9519d78a5f83fc411e8d8db996a45d4e",
"e1ea412fd3e1bd91c24b6b6445e8ff43",
),
(
63714,
"7d03a3698a79b1af1663e3e485c2efdc306ecd87b2644f2e01d83a35999d6cdf12241b6114d60d107c10c0d0c9cc0d23",
"e6a3c3f3fd2d9cfcdc06cca2f59e9a83",
),
(
64491,
"e12b168cce0e82ed1db88df549f39b3ff40b5884a09fceae69c4c3db13c1c37ea79531c47b2700d1c27774a1ab7e8b35",
"4cbb14d789f5cd8eca49ce9e1d442ea1",
),
(
65268,
"056c9d1172cfa76ce7f19c605e5969c284b82dca155dc9c1ed58062ab4d5a7704e27fe69f3aa745b73f45f1cd0ee57df",
"8195187f092d52c2a8695b680568b934",
),
]; ];
fn to_hex(b: &[u8]) -> String { fn to_hex(b: &[u8]) -> String {
@ -156,8 +499,12 @@ mod tests {
/// Test repeated encrypt/decrypt and run a benchmark. Run with --nocapture to see it. /// Test repeated encrypt/decrypt and run a benchmark. Run with --nocapture to see it.
#[test] #[test]
fn encrypt_decrypt() { fn encrypt_decrypt() {
let aes_key_0: [u8; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; let aes_key_0: [u8; 32] = [
let aes_key_1: [u8; 32] = [2, 3, 4, 5, 6, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
];
let aes_key_1: [u8; 32] = [
2, 3, 4, 5, 6, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
];
let iv: [u8; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; let iv: [u8; 8] = [0, 1, 2, 3, 4, 5, 6, 7];
let mut buf = [0_u8; 12345]; let mut buf = [0_u8; 12345];
@ -205,7 +552,10 @@ mod tests {
let _ = c.encrypt_second_pass_finish(); let _ = c.encrypt_second_pass_finish();
} }
let duration = SystemTime::now().duration_since(start).unwrap(); let duration = SystemTime::now().duration_since(start).unwrap();
println!("AES-GMAC-SIV (legacy) encrypt benchmark: {} MiB/sec", (((benchmark_iterations * buf.len()) as f64) / 1048576.0) / duration.as_secs_f64()); println!(
"AES-GMAC-SIV (legacy) encrypt benchmark: {} MiB/sec",
(((benchmark_iterations * buf.len()) as f64) / 1048576.0) / duration.as_secs_f64()
);
let start = SystemTime::now(); let start = SystemTime::now();
for _ in 0..benchmark_iterations { for _ in 0..benchmark_iterations {
c.reset(); c.reset();
@ -214,6 +564,9 @@ mod tests {
c.decrypt_finish(); c.decrypt_finish();
} }
let duration = SystemTime::now().duration_since(start).unwrap(); let duration = SystemTime::now().duration_since(start).unwrap();
println!("AES-GMAC-SIV (legacy) decrypt benchmark: {} MiB/sec", (((benchmark_iterations * buf.len()) as f64) / 1048576.0) / duration.as_secs_f64()); println!(
"AES-GMAC-SIV (legacy) decrypt benchmark: {} MiB/sec",
(((benchmark_iterations * buf.len()) as f64) / 1048576.0) / duration.as_secs_f64()
);
} }
} }

196
crypto/src/p384.rs
View file

@ -1,6 +1,14 @@
// (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md. // (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md.
#![allow(dead_code, mutable_transmutes, non_camel_case_types, non_snake_case, non_upper_case_globals, unused_assignments, unused_mut)] #![allow(
dead_code,
mutable_transmutes,
non_camel_case_types,
non_snake_case,
non_upper_case_globals,
unused_assignments,
unused_mut
)]
pub const P384_PUBLIC_KEY_SIZE: usize = 49; pub const P384_PUBLIC_KEY_SIZE: usize = 49;
pub const P384_SECRET_KEY_SIZE: usize = 48; pub const P384_SECRET_KEY_SIZE: usize = 48;
@ -116,7 +124,8 @@ mod builtin {
/* Returns nonzero if bit p_bit of p_vli is set. */ /* Returns nonzero if bit p_bit of p_vli is set. */
unsafe fn vli_testBit(mut p_vli: *mut uint64_t, mut p_bit: uint) -> uint64_t { unsafe fn vli_testBit(mut p_vli: *mut uint64_t, mut p_bit: uint) -> uint64_t {
return *p_vli.offset(p_bit.wrapping_div(64 as libc::c_int as libc::c_uint) as isize) & (1 as libc::c_int as uint64_t) << p_bit.wrapping_rem(64 as libc::c_int as libc::c_uint); return *p_vli.offset(p_bit.wrapping_div(64 as libc::c_int as libc::c_uint) as isize)
& (1 as libc::c_int as uint64_t) << p_bit.wrapping_rem(64 as libc::c_int as libc::c_uint);
} }
/* Counts the number of 64-bit "digits" in p_vli. */ /* Counts the number of 64-bit "digits" in p_vli. */
@ -215,7 +224,9 @@ mod builtin {
let mut i: uint = 0; let mut i: uint = 0;
i = 0 as libc::c_int as uint; i = 0 as libc::c_int as uint;
while i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint { while i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint {
let mut l_sum: uint64_t = (*p_left.offset(i as isize)).wrapping_add(*p_right.offset(i as isize)).wrapping_add(l_carry); let mut l_sum: uint64_t = (*p_left.offset(i as isize))
.wrapping_add(*p_right.offset(i as isize))
.wrapping_add(l_carry);
if l_sum != *p_left.offset(i as isize) { if l_sum != *p_left.offset(i as isize) {
l_carry = (l_sum < *p_left.offset(i as isize)) as libc::c_int as uint64_t l_carry = (l_sum < *p_left.offset(i as isize)) as libc::c_int as uint64_t
} }
@ -231,7 +242,9 @@ mod builtin {
let mut i: uint = 0; let mut i: uint = 0;
i = 0 as libc::c_int as uint; i = 0 as libc::c_int as uint;
while i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint { while i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint {
let mut l_diff: uint64_t = (*p_left.offset(i as isize)).wrapping_sub(*p_right.offset(i as isize)).wrapping_sub(l_borrow); let mut l_diff: uint64_t = (*p_left.offset(i as isize))
.wrapping_sub(*p_right.offset(i as isize))
.wrapping_sub(l_borrow);
if l_diff != *p_left.offset(i as isize) { if l_diff != *p_left.offset(i as isize) {
l_borrow = (l_diff > *p_left.offset(i as isize)) as libc::c_int as uint64_t l_borrow = (l_diff > *p_left.offset(i as isize)) as libc::c_int as uint64_t
} }
@ -253,11 +266,13 @@ mod builtin {
let mut l_min: uint = if k < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint { let mut l_min: uint = if k < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint {
0 as libc::c_int as libc::c_uint 0 as libc::c_int as libc::c_uint
} else { } else {
k.wrapping_add(1 as libc::c_int as libc::c_uint).wrapping_sub((48 as libc::c_int / 8 as libc::c_int) as libc::c_uint) k.wrapping_add(1 as libc::c_int as libc::c_uint)
.wrapping_sub((48 as libc::c_int / 8 as libc::c_int) as libc::c_uint)
}; };
i = l_min; i = l_min;
while i <= k && i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint { while i <= k && i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint {
let mut l_product: uint128_t = (*p_left.offset(i as isize) as uint128_t).wrapping_mul(*p_right.offset(k.wrapping_sub(i) as isize) as u128); let mut l_product: uint128_t =
(*p_left.offset(i as isize) as uint128_t).wrapping_mul(*p_right.offset(k.wrapping_sub(i) as isize) as u128);
r01 = (r01 as u128).wrapping_add(l_product) as uint128_t as uint128_t; r01 = (r01 as u128).wrapping_add(l_product) as uint128_t as uint128_t;
r2 = (r2 as libc::c_ulong).wrapping_add((r01 < l_product) as libc::c_int as libc::c_ulong) as uint64_t as uint64_t; r2 = (r2 as libc::c_ulong).wrapping_add((r01 < l_product) as libc::c_int as libc::c_ulong) as uint64_t as uint64_t;
i = i.wrapping_add(1) i = i.wrapping_add(1)
@ -281,11 +296,13 @@ mod builtin {
let mut l_min: uint = if k < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint { let mut l_min: uint = if k < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint {
0 as libc::c_int as libc::c_uint 0 as libc::c_int as libc::c_uint
} else { } else {
k.wrapping_add(1 as libc::c_int as libc::c_uint).wrapping_sub((48 as libc::c_int / 8 as libc::c_int) as libc::c_uint) k.wrapping_add(1 as libc::c_int as libc::c_uint)
.wrapping_sub((48 as libc::c_int / 8 as libc::c_int) as libc::c_uint)
}; };
i = l_min; i = l_min;
while i <= k && i <= k.wrapping_sub(i) { while i <= k && i <= k.wrapping_sub(i) {
let mut l_product: uint128_t = (*p_left.offset(i as isize) as uint128_t).wrapping_mul(*p_left.offset(k.wrapping_sub(i) as isize) as u128); let mut l_product: uint128_t =
(*p_left.offset(i as isize) as uint128_t).wrapping_mul(*p_left.offset(k.wrapping_sub(i) as isize) as u128);
if i < k.wrapping_sub(i) { if i < k.wrapping_sub(i) {
r2 = (r2 as u128).wrapping_add(l_product >> 127 as libc::c_int) as uint64_t as uint64_t; r2 = (r2 as u128).wrapping_add(l_product >> 127 as libc::c_int) as uint64_t as uint64_t;
l_product = (l_product as u128).wrapping_mul(2 as libc::c_int as u128) as uint128_t as uint128_t l_product = (l_product as u128).wrapping_mul(2 as libc::c_int as u128) as uint128_t as uint128_t
@ -333,9 +350,16 @@ mod builtin {
/* Multiply by (2^128 + 2^96 - 2^32 + 1). */ /* Multiply by (2^128 + 2^96 - 2^32 + 1). */
vli_set(p_result, p_right); /* 1 */ vli_set(p_result, p_right); /* 1 */
l_carry = vli_lshift(l_tmp.as_mut_ptr(), p_right, 32 as libc::c_int as uint); /* 2^96 + 1 */ l_carry = vli_lshift(l_tmp.as_mut_ptr(), p_right, 32 as libc::c_int as uint); /* 2^96 + 1 */
*p_result.offset((1 as libc::c_int + 48 as libc::c_int / 8 as libc::c_int) as isize) = *p_result.offset((1 as libc::c_int + 48 as libc::c_int / 8 as libc::c_int) as isize) = l_carry.wrapping_add(vli_add(
l_carry.wrapping_add(vli_add(p_result.offset(1 as libc::c_int as isize), p_result.offset(1 as libc::c_int as isize), l_tmp.as_mut_ptr())); /* 2^128 + 2^96 + 1 */ p_result.offset(1 as libc::c_int as isize),
*p_result.offset((2 as libc::c_int + 48 as libc::c_int / 8 as libc::c_int) as isize) = vli_add(p_result.offset(2 as libc::c_int as isize), p_result.offset(2 as libc::c_int as isize), p_right); /* 2^128 + 2^96 - 2^32 + 1 */ p_result.offset(1 as libc::c_int as isize),
l_tmp.as_mut_ptr(),
)); /* 2^128 + 2^96 + 1 */
*p_result.offset((2 as libc::c_int + 48 as libc::c_int / 8 as libc::c_int) as isize) = vli_add(
p_result.offset(2 as libc::c_int as isize),
p_result.offset(2 as libc::c_int as isize),
p_right,
); /* 2^128 + 2^96 - 2^32 + 1 */
l_carry = (l_carry as libc::c_ulong).wrapping_add(vli_sub(p_result, p_result, l_tmp.as_mut_ptr())) as uint64_t as uint64_t; l_carry = (l_carry as libc::c_ulong).wrapping_add(vli_sub(p_result, p_result, l_tmp.as_mut_ptr())) as uint64_t as uint64_t;
l_diff = (*p_result.offset((48 as libc::c_int / 8 as libc::c_int) as isize)).wrapping_sub(l_carry); l_diff = (*p_result.offset((48 as libc::c_int / 8 as libc::c_int) as isize)).wrapping_sub(l_carry);
if l_diff > *p_result.offset((48 as libc::c_int / 8 as libc::c_int) as isize) { if l_diff > *p_result.offset((48 as libc::c_int / 8 as libc::c_int) as isize) {
@ -365,12 +389,17 @@ mod builtin {
let mut i: uint = 0; /* p = c0 */ let mut i: uint = 0; /* p = c0 */
vli_clear(l_tmp.as_mut_ptr()); vli_clear(l_tmp.as_mut_ptr());
vli_clear(l_tmp.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize)); vli_clear(l_tmp.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize));
omega_mult(l_tmp.as_mut_ptr(), p_product.offset((48 as libc::c_int / 8 as libc::c_int) as isize)); omega_mult(
l_tmp.as_mut_ptr(),
p_product.offset((48 as libc::c_int / 8 as libc::c_int) as isize),
);
vli_clear(p_product.offset((48 as libc::c_int / 8 as libc::c_int) as isize)); vli_clear(p_product.offset((48 as libc::c_int / 8 as libc::c_int) as isize));
/* (c1, c0) = c0 + w * c1 */ /* (c1, c0) = c0 + w * c1 */
i = 0 as libc::c_int as uint; i = 0 as libc::c_int as uint;
while i < (48 as libc::c_int / 8 as libc::c_int + 3 as libc::c_int) as libc::c_uint { while i < (48 as libc::c_int / 8 as libc::c_int + 3 as libc::c_int) as libc::c_uint {
let mut l_sum: uint64_t = (*p_product.offset(i as isize)).wrapping_add(l_tmp[i as usize]).wrapping_add(l_carry); let mut l_sum: uint64_t = (*p_product.offset(i as isize))
.wrapping_add(l_tmp[i as usize])
.wrapping_add(l_carry);
if l_sum != *p_product.offset(i as isize) { if l_sum != *p_product.offset(i as isize) {
l_carry = (l_sum < *p_product.offset(i as isize)) as libc::c_int as uint64_t l_carry = (l_sum < *p_product.offset(i as isize)) as libc::c_int as uint64_t
} }
@ -432,7 +461,8 @@ mod builtin {
vli_rshift1(u.as_mut_ptr()); vli_rshift1(u.as_mut_ptr());
if l_carry != 0 { if l_carry != 0 {
u[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] = u[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] =
(u[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong | 0x8000000000000000 as libc::c_ulonglong) as uint64_t (u[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong
| 0x8000000000000000 as libc::c_ulonglong) as uint64_t
} }
} else if b[0 as libc::c_int as usize] & 1 as libc::c_int as libc::c_ulong == 0 { } else if b[0 as libc::c_int as usize] & 1 as libc::c_int as libc::c_ulong == 0 {
vli_rshift1(b.as_mut_ptr()); vli_rshift1(b.as_mut_ptr());
@ -442,7 +472,8 @@ mod builtin {
vli_rshift1(v.as_mut_ptr()); vli_rshift1(v.as_mut_ptr());
if l_carry != 0 { if l_carry != 0 {
v[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] = v[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] =
(v[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong | 0x8000000000000000 as libc::c_ulonglong) as uint64_t (v[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong
| 0x8000000000000000 as libc::c_ulonglong) as uint64_t
} }
} else if l_cmpResult > 0 as libc::c_int { } else if l_cmpResult > 0 as libc::c_int {
vli_sub(a.as_mut_ptr(), a.as_mut_ptr(), b.as_mut_ptr()); vli_sub(a.as_mut_ptr(), a.as_mut_ptr(), b.as_mut_ptr());
@ -457,7 +488,8 @@ mod builtin {
vli_rshift1(u.as_mut_ptr()); vli_rshift1(u.as_mut_ptr());
if l_carry != 0 { if l_carry != 0 {
u[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] = u[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] =
(u[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong | 0x8000000000000000 as libc::c_ulonglong) as uint64_t (u[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong
| 0x8000000000000000 as libc::c_ulonglong) as uint64_t
} }
} else { } else {
vli_sub(b.as_mut_ptr(), b.as_mut_ptr(), a.as_mut_ptr()); vli_sub(b.as_mut_ptr(), b.as_mut_ptr(), a.as_mut_ptr());
@ -472,7 +504,8 @@ mod builtin {
vli_rshift1(v.as_mut_ptr()); vli_rshift1(v.as_mut_ptr());
if l_carry != 0 { if l_carry != 0 {
v[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] = v[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] =
(v[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong | 0x8000000000000000 as libc::c_ulonglong) as uint64_t (v[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong
| 0x8000000000000000 as libc::c_ulonglong) as uint64_t
} }
} }
} }
@ -538,7 +571,13 @@ mod builtin {
} }
/* P = (x1, y1) => 2P, (x2, y2) => P' */ /* P = (x1, y1) => 2P, (x2, y2) => P' */
unsafe fn XYcZ_initial_double(mut X1: *mut uint64_t, mut Y1: *mut uint64_t, mut X2: *mut uint64_t, mut Y2: *mut uint64_t, mut p_initialZ: *mut uint64_t) { unsafe fn XYcZ_initial_double(
mut X1: *mut uint64_t,
mut Y1: *mut uint64_t,
mut X2: *mut uint64_t,
mut Y2: *mut uint64_t,
mut p_initialZ: *mut uint64_t,
) {
let mut z: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init(); let mut z: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init();
vli_set(X2, X1); vli_set(X2, X1);
vli_set(Y2, Y1); vli_set(Y2, Y1);
@ -606,7 +645,12 @@ mod builtin {
vli_set(X1, t7.as_mut_ptr()); vli_set(X1, t7.as_mut_ptr());
} }
unsafe fn EccPoint_mult(mut p_result: *mut EccPoint, mut p_point: *mut EccPoint, mut p_scalar: *mut uint64_t, mut p_initialZ: *mut uint64_t) { unsafe fn EccPoint_mult(
mut p_result: *mut EccPoint,
mut p_point: *mut EccPoint,
mut p_scalar: *mut uint64_t,
mut p_initialZ: *mut uint64_t,
) {
/* R0 and R1 */ /* R0 and R1 */
let mut Rx: [[uint64_t; 6]; 2] = std::mem::MaybeUninit::uninit().assume_init(); let mut Rx: [[uint64_t; 6]; 2] = std::mem::MaybeUninit::uninit().assume_init();
let mut Ry: [[uint64_t; 6]; 2] = std::mem::MaybeUninit::uninit().assume_init(); let mut Ry: [[uint64_t; 6]; 2] = std::mem::MaybeUninit::uninit().assume_init();
@ -665,7 +709,11 @@ mod builtin {
Rx[(1 as libc::c_int - nb) as usize].as_mut_ptr(), Rx[(1 as libc::c_int - nb) as usize].as_mut_ptr(),
Ry[(1 as libc::c_int - nb) as usize].as_mut_ptr(), Ry[(1 as libc::c_int - nb) as usize].as_mut_ptr(),
); );
apply_z(Rx[0 as libc::c_int as usize].as_mut_ptr(), Ry[0 as libc::c_int as usize].as_mut_ptr(), z.as_mut_ptr()); apply_z(
Rx[0 as libc::c_int as usize].as_mut_ptr(),
Ry[0 as libc::c_int as usize].as_mut_ptr(),
z.as_mut_ptr(),
);
vli_set((*p_result).x.as_mut_ptr(), Rx[0 as libc::c_int as usize].as_mut_ptr()); vli_set((*p_result).x.as_mut_ptr(), Rx[0 as libc::c_int as usize].as_mut_ptr());
vli_set((*p_result).y.as_mut_ptr(), Ry[0 as libc::c_int as usize].as_mut_ptr()); vli_set((*p_result).y.as_mut_ptr(), Ry[0 as libc::c_int as usize].as_mut_ptr());
} }
@ -674,8 +722,11 @@ mod builtin {
let mut i: libc::c_uint = 0; let mut i: libc::c_uint = 0;
i = 0 as libc::c_int as libc::c_uint; i = 0 as libc::c_int as libc::c_uint;
while i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint { while i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint {
let mut p_digit: *const uint8_t = let mut p_digit: *const uint8_t = p_bytes.offset(
p_bytes.offset((8 as libc::c_int as libc::c_uint).wrapping_mul(((48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as libc::c_uint).wrapping_sub(i)) as isize); (8 as libc::c_int as libc::c_uint)
.wrapping_mul(((48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as libc::c_uint).wrapping_sub(i))
as isize,
);
*p_native.offset(i as isize) = (*p_digit.offset(0 as libc::c_int as isize) as uint64_t) << 56 as libc::c_int *p_native.offset(i as isize) = (*p_digit.offset(0 as libc::c_int as isize) as uint64_t) << 56 as libc::c_int
| (*p_digit.offset(1 as libc::c_int as isize) as uint64_t) << 48 as libc::c_int | (*p_digit.offset(1 as libc::c_int as isize) as uint64_t) << 48 as libc::c_int
| (*p_digit.offset(2 as libc::c_int as isize) as uint64_t) << 40 as libc::c_int | (*p_digit.offset(2 as libc::c_int as isize) as uint64_t) << 40 as libc::c_int
@ -692,8 +743,11 @@ mod builtin {
let mut i: libc::c_uint = 0; let mut i: libc::c_uint = 0;
i = 0 as libc::c_int as libc::c_uint; i = 0 as libc::c_int as libc::c_uint;
while i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint { while i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint {
let mut p_digit: *mut uint8_t = let mut p_digit: *mut uint8_t = p_bytes.offset(
p_bytes.offset((8 as libc::c_int as libc::c_uint).wrapping_mul(((48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as libc::c_uint).wrapping_sub(i)) as isize); (8 as libc::c_int as libc::c_uint)
.wrapping_mul(((48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as libc::c_uint).wrapping_sub(i))
as isize,
);
*p_digit.offset(0 as libc::c_int as isize) = (*p_native.offset(i as isize) >> 56 as libc::c_int) as uint8_t; *p_digit.offset(0 as libc::c_int as isize) = (*p_native.offset(i as isize) >> 56 as libc::c_int) as uint8_t;
*p_digit.offset(1 as libc::c_int as isize) = (*p_native.offset(i as isize) >> 48 as libc::c_int) as uint8_t; *p_digit.offset(1 as libc::c_int as isize) = (*p_native.offset(i as isize) >> 48 as libc::c_int) as uint8_t;
*p_digit.offset(2 as libc::c_int as isize) = (*p_native.offset(i as isize) >> 40 as libc::c_int) as uint8_t; *p_digit.offset(2 as libc::c_int as isize) = (*p_native.offset(i as isize) >> 40 as libc::c_int) as uint8_t;
@ -730,11 +784,23 @@ mod builtin {
let mut _3: [uint64_t; 6] = [3 as libc::c_int as uint64_t, 0, 0, 0, 0, 0]; let mut _3: [uint64_t; 6] = [3 as libc::c_int as uint64_t, 0, 0, 0, 0, 0];
ecc_bytes2native((*p_point).x.as_mut_ptr(), p_compressed.offset(1 as libc::c_int as isize)); ecc_bytes2native((*p_point).x.as_mut_ptr(), p_compressed.offset(1 as libc::c_int as isize));
vli_modSquare_fast((*p_point).y.as_mut_ptr(), (*p_point).x.as_mut_ptr()); vli_modSquare_fast((*p_point).y.as_mut_ptr(), (*p_point).x.as_mut_ptr());
vli_modSub((*p_point).y.as_mut_ptr(), (*p_point).y.as_mut_ptr(), _3.as_mut_ptr(), curve_p.as_mut_ptr()); vli_modSub(
(*p_point).y.as_mut_ptr(),
(*p_point).y.as_mut_ptr(),
_3.as_mut_ptr(),
curve_p.as_mut_ptr(),
);
vli_modMult_fast((*p_point).y.as_mut_ptr(), (*p_point).y.as_mut_ptr(), (*p_point).x.as_mut_ptr()); vli_modMult_fast((*p_point).y.as_mut_ptr(), (*p_point).y.as_mut_ptr(), (*p_point).x.as_mut_ptr());
vli_modAdd((*p_point).y.as_mut_ptr(), (*p_point).y.as_mut_ptr(), curve_b.as_mut_ptr(), curve_p.as_mut_ptr()); vli_modAdd(
(*p_point).y.as_mut_ptr(),
(*p_point).y.as_mut_ptr(),
curve_b.as_mut_ptr(),
curve_p.as_mut_ptr(),
);
mod_sqrt((*p_point).y.as_mut_ptr()); mod_sqrt((*p_point).y.as_mut_ptr());
if (*p_point).y[0 as libc::c_int as usize] & 0x1 as libc::c_int as libc::c_ulong != (*p_compressed.offset(0 as libc::c_int as isize) as libc::c_int & 0x1 as libc::c_int) as libc::c_ulong { if (*p_point).y[0 as libc::c_int as usize] & 0x1 as libc::c_int as libc::c_ulong
!= (*p_compressed.offset(0 as libc::c_int as isize) as libc::c_int & 0x1 as libc::c_int) as libc::c_ulong
{
vli_sub((*p_point).y.as_mut_ptr(), curve_p.as_mut_ptr(), (*p_point).y.as_mut_ptr()); vli_sub((*p_point).y.as_mut_ptr(), curve_p.as_mut_ptr(), (*p_point).y.as_mut_ptr());
}; };
} }
@ -763,11 +829,20 @@ mod builtin {
} }
} }
ecc_native2bytes(p_privateKey, l_private.as_mut_ptr() as *const uint64_t); ecc_native2bytes(p_privateKey, l_private.as_mut_ptr() as *const uint64_t);
ecc_native2bytes(p_publicKey.offset(1 as libc::c_int as isize), l_public.x.as_mut_ptr() as *const uint64_t); ecc_native2bytes(
*p_publicKey.offset(0 as libc::c_int as isize) = (2 as libc::c_int as libc::c_ulong).wrapping_add(l_public.y[0 as libc::c_int as usize] & 0x1 as libc::c_int as libc::c_ulong) as uint8_t; p_publicKey.offset(1 as libc::c_int as isize),
l_public.x.as_mut_ptr() as *const uint64_t,
);
*p_publicKey.offset(0 as libc::c_int as isize) = (2 as libc::c_int as libc::c_ulong)
.wrapping_add(l_public.y[0 as libc::c_int as usize] & 0x1 as libc::c_int as libc::c_ulong)
as uint8_t;
return 1 as libc::c_int; return 1 as libc::c_int;
} }
pub unsafe fn ecdh_shared_secret(mut p_publicKey: *const uint8_t, mut p_privateKey: *const uint8_t, mut p_secret: *mut uint8_t) -> libc::c_int { pub unsafe fn ecdh_shared_secret(
mut p_publicKey: *const uint8_t,
mut p_privateKey: *const uint8_t,
mut p_secret: *mut uint8_t,
) -> libc::c_int {
let mut l_public: EccPoint = std::mem::MaybeUninit::uninit().assume_init(); let mut l_public: EccPoint = std::mem::MaybeUninit::uninit().assume_init();
let mut l_private: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init(); let mut l_private: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init();
let mut l_random: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init(); let mut l_random: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init();
@ -794,7 +869,9 @@ mod builtin {
vli_mult(l_product.as_mut_ptr(), p_left, p_right); vli_mult(l_product.as_mut_ptr(), p_left, p_right);
l_productBits = vli_numBits(l_product.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize)); l_productBits = vli_numBits(l_product.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize));
if l_productBits != 0 { if l_productBits != 0 {
l_productBits = (l_productBits as libc::c_uint).wrapping_add((48 as libc::c_int / 8 as libc::c_int * 64 as libc::c_int) as libc::c_uint) as uint as uint l_productBits = (l_productBits as libc::c_uint)
.wrapping_add((48 as libc::c_int / 8 as libc::c_int * 64 as libc::c_int) as libc::c_uint)
as uint as uint
} else { } else {
l_productBits = vli_numBits(l_product.as_mut_ptr()) l_productBits = vli_numBits(l_product.as_mut_ptr())
} }
@ -807,8 +884,12 @@ mod builtin {
power of two possible while still resulting in a number less than p_left. */ power of two possible while still resulting in a number less than p_left. */
vli_clear(l_modMultiple.as_mut_ptr()); vli_clear(l_modMultiple.as_mut_ptr());
vli_clear(l_modMultiple.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize)); vli_clear(l_modMultiple.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize));
l_digitShift = l_productBits.wrapping_sub(l_modBits).wrapping_div(64 as libc::c_int as libc::c_uint); l_digitShift = l_productBits
l_bitShift = l_productBits.wrapping_sub(l_modBits).wrapping_rem(64 as libc::c_int as libc::c_uint); .wrapping_sub(l_modBits)
.wrapping_div(64 as libc::c_int as libc::c_uint);
l_bitShift = l_productBits
.wrapping_sub(l_modBits)
.wrapping_rem(64 as libc::c_int as libc::c_uint);
if l_bitShift != 0 { if l_bitShift != 0 {
l_modMultiple[l_digitShift.wrapping_add((48 as libc::c_int / 8 as libc::c_int) as libc::c_uint) as usize] = l_modMultiple[l_digitShift.wrapping_add((48 as libc::c_int / 8 as libc::c_int) as libc::c_uint) as usize] =
vli_lshift(l_modMultiple.as_mut_ptr().offset(l_digitShift as isize), p_mod, l_bitShift) vli_lshift(l_modMultiple.as_mut_ptr().offset(l_digitShift as isize), p_mod, l_bitShift)
@ -818,12 +899,16 @@ mod builtin {
/* Subtract all multiples of p_mod to get the remainder. */ /* Subtract all multiples of p_mod to get the remainder. */
vli_clear(p_result); /* Use p_result as a temp var to store 1 (for subtraction) */ vli_clear(p_result); /* Use p_result as a temp var to store 1 (for subtraction) */
*p_result.offset(0 as libc::c_int as isize) = 1 as libc::c_int as uint64_t; *p_result.offset(0 as libc::c_int as isize) = 1 as libc::c_int as uint64_t;
while l_productBits > (48 as libc::c_int / 8 as libc::c_int * 64 as libc::c_int) as libc::c_uint || vli_cmp(l_modMultiple.as_mut_ptr(), p_mod) >= 0 as libc::c_int { while l_productBits > (48 as libc::c_int / 8 as libc::c_int * 64 as libc::c_int) as libc::c_uint
|| vli_cmp(l_modMultiple.as_mut_ptr(), p_mod) >= 0 as libc::c_int
{
let mut l_cmp: libc::c_int = vli_cmp( let mut l_cmp: libc::c_int = vli_cmp(
l_modMultiple.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize), l_modMultiple.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize),
l_product.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize), l_product.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize),
); );
if l_cmp < 0 as libc::c_int || l_cmp == 0 as libc::c_int && vli_cmp(l_modMultiple.as_mut_ptr(), l_product.as_mut_ptr()) <= 0 as libc::c_int { if l_cmp < 0 as libc::c_int
|| l_cmp == 0 as libc::c_int && vli_cmp(l_modMultiple.as_mut_ptr(), l_product.as_mut_ptr()) <= 0 as libc::c_int
{
if vli_sub(l_product.as_mut_ptr(), l_product.as_mut_ptr(), l_modMultiple.as_mut_ptr()) != 0 { if vli_sub(l_product.as_mut_ptr(), l_product.as_mut_ptr(), l_modMultiple.as_mut_ptr()) != 0 {
/* borrow */ /* borrow */
vli_sub( vli_sub(
@ -838,7 +923,8 @@ mod builtin {
l_modMultiple.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize), l_modMultiple.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize),
); );
} }
let mut l_carry: uint64_t = (l_modMultiple[(48 as libc::c_int / 8 as libc::c_int) as usize] & 0x1 as libc::c_int as libc::c_ulong) << 63 as libc::c_int; let mut l_carry: uint64_t =
(l_modMultiple[(48 as libc::c_int / 8 as libc::c_int) as usize] & 0x1 as libc::c_int as libc::c_ulong) << 63 as libc::c_int;
vli_rshift1(l_modMultiple.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize)); vli_rshift1(l_modMultiple.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize));
vli_rshift1(l_modMultiple.as_mut_ptr()); vli_rshift1(l_modMultiple.as_mut_ptr());
l_modMultiple[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] |= l_carry; l_modMultiple[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] |= l_carry;
@ -890,7 +976,11 @@ mod builtin {
ecc_native2bytes(p_signature.offset(48 as libc::c_int as isize), l_s.as_mut_ptr() as *const uint64_t); ecc_native2bytes(p_signature.offset(48 as libc::c_int as isize), l_s.as_mut_ptr() as *const uint64_t);
return 1 as libc::c_int; return 1 as libc::c_int;
} }
pub unsafe fn ecdsa_verify(mut p_publicKey: *const uint8_t, mut p_hash: *const uint8_t, mut p_signature: *const uint8_t) -> libc::c_int { pub unsafe fn ecdsa_verify(
mut p_publicKey: *const uint8_t,
mut p_hash: *const uint8_t,
mut p_signature: *const uint8_t,
) -> libc::c_int {
let mut u1: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init(); let mut u1: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init();
let mut u2: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init(); let mut u2: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init();
let mut z: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init(); let mut z: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init();
@ -910,7 +1000,9 @@ mod builtin {
/* r, s must not be 0. */ /* r, s must not be 0. */
return 0 as libc::c_int; return 0 as libc::c_int;
} }
if vli_cmp(curve_n.as_mut_ptr(), l_r.as_mut_ptr()) != 1 as libc::c_int || vli_cmp(curve_n.as_mut_ptr(), l_s.as_mut_ptr()) != 1 as libc::c_int { if vli_cmp(curve_n.as_mut_ptr(), l_r.as_mut_ptr()) != 1 as libc::c_int
|| vli_cmp(curve_n.as_mut_ptr(), l_s.as_mut_ptr()) != 1 as libc::c_int
{
/* r, s must be < n. */ /* r, s must be < n. */
return 0 as libc::c_int; return 0 as libc::c_int;
} }
@ -931,9 +1023,10 @@ mod builtin {
/* Use Shamir's trick to calculate u1*G + u2*Q */ /* Use Shamir's trick to calculate u1*G + u2*Q */
let mut l_points: [*mut EccPoint; 4] = [0 as *mut EccPoint, &mut curve_G, &mut l_public, &mut l_sum]; /* Z = x2 - x1 */ let mut l_points: [*mut EccPoint; 4] = [0 as *mut EccPoint, &mut curve_G, &mut l_public, &mut l_sum]; /* Z = x2 - x1 */
let mut l_numBits: uint = umax(vli_numBits(u1.as_mut_ptr()), vli_numBits(u2.as_mut_ptr())); /* Z = 1/Z */ let mut l_numBits: uint = umax(vli_numBits(u1.as_mut_ptr()), vli_numBits(u2.as_mut_ptr())); /* Z = 1/Z */
let mut l_point: *mut EccPoint = l_points[((vli_testBit(u1.as_mut_ptr(), l_numBits.wrapping_sub(1 as libc::c_int as libc::c_uint)) != 0) as libc::c_int let mut l_point: *mut EccPoint = l_points[((vli_testBit(u1.as_mut_ptr(), l_numBits.wrapping_sub(1 as libc::c_int as libc::c_uint))
| ((vli_testBit(u2.as_mut_ptr(), l_numBits.wrapping_sub(1 as libc::c_int as libc::c_uint)) != 0) as libc::c_int) << 1 as libc::c_int) != 0) as libc::c_int
as usize]; | ((vli_testBit(u2.as_mut_ptr(), l_numBits.wrapping_sub(1 as libc::c_int as libc::c_uint)) != 0) as libc::c_int)
<< 1 as libc::c_int) as usize];
vli_set(rx.as_mut_ptr(), (*l_point).x.as_mut_ptr()); vli_set(rx.as_mut_ptr(), (*l_point).x.as_mut_ptr());
vli_set(ry.as_mut_ptr(), (*l_point).y.as_mut_ptr()); vli_set(ry.as_mut_ptr(), (*l_point).y.as_mut_ptr());
vli_clear(z.as_mut_ptr()); vli_clear(z.as_mut_ptr());
@ -942,7 +1035,8 @@ mod builtin {
i = l_numBits.wrapping_sub(2 as libc::c_int as libc::c_uint) as libc::c_int; i = l_numBits.wrapping_sub(2 as libc::c_int as libc::c_uint) as libc::c_int;
while i >= 0 as libc::c_int { while i >= 0 as libc::c_int {
EccPoint_double_jacobian(rx.as_mut_ptr(), ry.as_mut_ptr(), z.as_mut_ptr()); EccPoint_double_jacobian(rx.as_mut_ptr(), ry.as_mut_ptr(), z.as_mut_ptr());
let mut l_index: libc::c_int = (vli_testBit(u1.as_mut_ptr(), i as uint) != 0) as libc::c_int | ((vli_testBit(u2.as_mut_ptr(), i as uint) != 0) as libc::c_int) << 1 as libc::c_int; let mut l_index: libc::c_int = (vli_testBit(u1.as_mut_ptr(), i as uint) != 0) as libc::c_int
| ((vli_testBit(u2.as_mut_ptr(), i as uint) != 0) as libc::c_int) << 1 as libc::c_int;
let mut l_point_0: *mut EccPoint = l_points[l_index as usize]; let mut l_point_0: *mut EccPoint = l_points[l_index as usize];
if !l_point_0.is_null() { if !l_point_0.is_null() {
vli_set(tx.as_mut_ptr(), (*l_point_0).x.as_mut_ptr()); vli_set(tx.as_mut_ptr(), (*l_point_0).x.as_mut_ptr());
@ -1002,7 +1096,10 @@ mod builtin {
pub fn from_bytes(public_bytes: &[u8], secret_bytes: &[u8]) -> Option<P384KeyPair> { pub fn from_bytes(public_bytes: &[u8], secret_bytes: &[u8]) -> Option<P384KeyPair> {
if public_bytes.len() == 49 && secret_bytes.len() == 48 { if public_bytes.len() == 49 && secret_bytes.len() == 48 {
Some(Self(P384PublicKey(public_bytes.try_into().unwrap()), Secret(secret_bytes.try_into().unwrap()))) Some(Self(
P384PublicKey(public_bytes.try_into().unwrap()),
Secret(secret_bytes.try_into().unwrap()),
))
} else { } else {
None None
} }
@ -1080,7 +1177,9 @@ mod openssl_based {
impl P384PublicKey { impl P384PublicKey {
fn new_from_point(key: &EcPointRef) -> Self { fn new_from_point(key: &EcPointRef) -> Self {
let mut bnc = BigNumContext::new().unwrap(); let mut bnc = BigNumContext::new().unwrap();
let kb = key.to_bytes(GROUP_P384.as_ref(), PointConversionForm::COMPRESSED, &mut bnc).unwrap(); let kb = key
.to_bytes(GROUP_P384.as_ref(), PointConversionForm::COMPRESSED, &mut bnc)
.unwrap();
let mut bytes = [0_u8; 49]; let mut bytes = [0_u8; 49];
bytes[(49 - kb.len())..].copy_from_slice(kb.as_slice()); bytes[(49 - kb.len())..].copy_from_slice(kb.as_slice());
Self { Self {
@ -1219,7 +1318,14 @@ mod openssl_based {
pub fn agree(&self, other_public: &P384PublicKey) -> Option<Secret<P384_ECDH_SHARED_SECRET_SIZE>> { pub fn agree(&self, other_public: &P384PublicKey) -> Option<Secret<P384_ECDH_SHARED_SECRET_SIZE>> {
unsafe { unsafe {
let mut s: Secret<P384_ECDH_SHARED_SECRET_SIZE> = Secret::default(); let mut s: Secret<P384_ECDH_SHARED_SECRET_SIZE> = Secret::default();
if ECDH_compute_key(s.0.as_mut_ptr().cast(), 48, other_public.key.public_key().as_ptr().cast(), self.pair.as_ptr().cast(), null()) == 48 { if ECDH_compute_key(
s.0.as_mut_ptr().cast(),
48,
other_public.key.public_key().as_ptr().cast(),
self.pair.as_ptr().cast(),
null(),
) == 48
{
Some(s) Some(s)
} else { } else {
None None

22
crypto/src/poly1305.rs
View file

@ -10,7 +10,10 @@ pub const POLY1305_MAC_SIZE: usize = 16;
#[inline(always)] #[inline(always)]
pub fn compute(one_time_key: &[u8], message: &[u8]) -> [u8; POLY1305_MAC_SIZE] { pub fn compute(one_time_key: &[u8], message: &[u8]) -> [u8; POLY1305_MAC_SIZE] {
poly1305::Poly1305::new(poly1305::Key::from_slice(one_time_key)).compute_unpadded(message).into_bytes().into() poly1305::Poly1305::new(poly1305::Key::from_slice(one_time_key))
.compute_unpadded(message)
.into_bytes()
.into()
} }
#[cfg(test)] #[cfg(test)]
@ -18,18 +21,25 @@ mod tests {
use crate::poly1305::*; use crate::poly1305::*;
const TV0_INPUT: [u8; 32] = [ const TV0_INPUT: [u8; 32] = [
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
]; ];
const TV0_KEY: [u8; 32] = [ const TV0_KEY: [u8; 32] = [
0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35, 0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f,
0x72, 0x20, 0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35,
];
const TV0_TAG: [u8; 16] = [
0x49, 0xec, 0x78, 0x09, 0x0e, 0x48, 0x1e, 0xc6, 0xc2, 0x6b, 0x33, 0xb9, 0x1c, 0xcc, 0x03, 0x07,
]; ];
const TV0_TAG: [u8; 16] = [0x49, 0xec, 0x78, 0x09, 0x0e, 0x48, 0x1e, 0xc6, 0xc2, 0x6b, 0x33, 0xb9, 0x1c, 0xcc, 0x03, 0x07];
const TV1_INPUT: [u8; 12] = [0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21]; const TV1_INPUT: [u8; 12] = [0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21];
const TV1_KEY: [u8; 32] = [ const TV1_KEY: [u8; 32] = [
0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35, 0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f,
0x72, 0x20, 0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35,
];
const TV1_TAG: [u8; 16] = [
0xa6, 0xf7, 0x45, 0x00, 0x8f, 0x81, 0xc9, 0x16, 0xa2, 0x0d, 0xcc, 0x74, 0xee, 0xf2, 0xb2, 0xf0,
]; ];
const TV1_TAG: [u8; 16] = [0xa6, 0xf7, 0x45, 0x00, 0x8f, 0x81, 0xc9, 0x16, 0xa2, 0x0d, 0xcc, 0x74, 0xee, 0xf2, 0xb2, 0xf0];
#[test] #[test]
fn poly1305() { fn poly1305() {

42
crypto/src/salsa.rs
View file

@ -3,7 +3,12 @@
use std::convert::TryInto; use std::convert::TryInto;
use std::ptr::{slice_from_raw_parts, slice_from_raw_parts_mut}; use std::ptr::{slice_from_raw_parts, slice_from_raw_parts_mut};
const CONSTANTS: [u32; 4] = [u32::from_le_bytes(*b"expa"), u32::from_le_bytes(*b"nd 3"), u32::from_le_bytes(*b"2-by"), u32::from_le_bytes(*b"te k")]; const CONSTANTS: [u32; 4] = [
u32::from_le_bytes(*b"expa"),
u32::from_le_bytes(*b"nd 3"),
u32::from_le_bytes(*b"2-by"),
u32::from_le_bytes(*b"te k"),
];
/// Salsa stream cipher implementation supporting 8, 12, or 20 rounds. /// Salsa stream cipher implementation supporting 8, 12, or 20 rounds.
/// ///
@ -65,8 +70,24 @@ impl<const ROUNDS: usize> Salsa<ROUNDS> {
); );
while !plaintext.is_empty() { while !plaintext.is_empty() {
let (mut x0, mut x1, mut x2, mut x3, mut x4, mut x5, mut x6, mut x7, mut x8, mut x9, mut x10, mut x11, mut x12, mut x13, mut x14, mut x15) = let (
(j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15); mut x0,
mut x1,
mut x2,
mut x3,
mut x4,
mut x5,
mut x6,
mut x7,
mut x8,
mut x9,
mut x10,
mut x11,
mut x12,
mut x13,
mut x14,
mut x15,
) = (j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15);
for _ in 0..(ROUNDS / 2) { for _ in 0..(ROUNDS / 2) {
x4 ^= x0.wrapping_add(x12).rotate_left(7); x4 ^= x0.wrapping_add(x12).rotate_left(7);
@ -206,7 +227,12 @@ impl<const ROUNDS: usize> Salsa<ROUNDS> {
#[inline(always)] #[inline(always)]
pub fn crypt_in_place(&mut self, data: &mut [u8]) { pub fn crypt_in_place(&mut self, data: &mut [u8]) {
unsafe { self.crypt(&*slice_from_raw_parts(data.as_ptr(), data.len()), &mut *slice_from_raw_parts_mut(data.as_mut_ptr(), data.len())) } unsafe {
self.crypt(
&*slice_from_raw_parts(data.as_ptr(), data.len()),
&mut *slice_from_raw_parts_mut(data.as_mut_ptr(), data.len()),
)
}
} }
} }
@ -215,12 +241,14 @@ mod tests {
use crate::salsa::*; use crate::salsa::*;
const SALSA_20_TV0_KEY: [u8; 32] = [ const SALSA_20_TV0_KEY: [u8; 32] = [
0x0f, 0x62, 0xb5, 0x08, 0x5b, 0xae, 0x01, 0x54, 0xa7, 0xfa, 0x4d, 0xa0, 0xf3, 0x46, 0x99, 0xec, 0x3f, 0x92, 0xe5, 0x38, 0x8b, 0xde, 0x31, 0x84, 0xd7, 0x2a, 0x7d, 0xd0, 0x23, 0x76, 0xc9, 0x1c, 0x0f, 0x62, 0xb5, 0x08, 0x5b, 0xae, 0x01, 0x54, 0xa7, 0xfa, 0x4d, 0xa0, 0xf3, 0x46, 0x99, 0xec, 0x3f, 0x92, 0xe5, 0x38, 0x8b, 0xde,
0x31, 0x84, 0xd7, 0x2a, 0x7d, 0xd0, 0x23, 0x76, 0xc9, 0x1c,
]; ];
const SALSA_20_TV0_IV: [u8; 8] = [0x28, 0x8f, 0xf6, 0x5d, 0xc4, 0x2b, 0x92, 0xf9]; const SALSA_20_TV0_IV: [u8; 8] = [0x28, 0x8f, 0xf6, 0x5d, 0xc4, 0x2b, 0x92, 0xf9];
const SALSA_20_TV0_KS: [u8; 64] = [ const SALSA_20_TV0_KS: [u8; 64] = [
0x5e, 0x5e, 0x71, 0xf9, 0x01, 0x99, 0x34, 0x03, 0x04, 0xab, 0xb2, 0x2a, 0x37, 0xb6, 0x62, 0x5b, 0xf8, 0x83, 0xfb, 0x89, 0xce, 0x3b, 0x21, 0xf5, 0x4a, 0x10, 0xb8, 0x10, 0x66, 0xef, 0x87, 0xda, 0x5e, 0x5e, 0x71, 0xf9, 0x01, 0x99, 0x34, 0x03, 0x04, 0xab, 0xb2, 0x2a, 0x37, 0xb6, 0x62, 0x5b, 0xf8, 0x83, 0xfb, 0x89, 0xce, 0x3b,
0x30, 0xb7, 0x76, 0x99, 0xaa, 0x73, 0x79, 0xda, 0x59, 0x5c, 0x77, 0xdd, 0x59, 0x54, 0x2d, 0xa2, 0x08, 0xe5, 0x95, 0x4f, 0x89, 0xe4, 0x0e, 0xb7, 0xaa, 0x80, 0xa8, 0x4a, 0x61, 0x76, 0x66, 0x3f, 0x21, 0xf5, 0x4a, 0x10, 0xb8, 0x10, 0x66, 0xef, 0x87, 0xda, 0x30, 0xb7, 0x76, 0x99, 0xaa, 0x73, 0x79, 0xda, 0x59, 0x5c, 0x77, 0xdd,
0x59, 0x54, 0x2d, 0xa2, 0x08, 0xe5, 0x95, 0x4f, 0x89, 0xe4, 0x0e, 0xb7, 0xaa, 0x80, 0xa8, 0x4a, 0x61, 0x76, 0x66, 0x3f,
]; ];
#[test] #[test]

6
crypto/src/x25519.rs
View file

@ -84,7 +84,11 @@ impl X25519KeyPair {
impl Clone for X25519KeyPair { impl Clone for X25519KeyPair {
fn clone(&self) -> Self { fn clone(&self) -> Self {
Self(x25519_dalek::StaticSecret::from(self.0.to_bytes()), self.1.clone(), x25519_dalek::PublicKey::from(self.1 .0.clone())) Self(
x25519_dalek::StaticSecret::from(self.0.to_bytes()),
self.1.clone(),
x25519_dalek::PublicKey::from(self.1 .0.clone()),
)
} }
} }

265
crypto/src/zssp.rs
View file

@ -117,8 +117,9 @@ const KBKDF_KEY_USAGE_LABEL_RATCHETING: u8 = b'R';
const INITIAL_KEY: [u8; 64] = [ const INITIAL_KEY: [u8; 64] = [
// macOS command line to generate: // macOS command line to generate:
// echo -n 'ZSSP_Noise_IKpsk2_NISTP384_?KYBER1024_AESGCM_SHA512' | shasum -a 512 | cut -d ' ' -f 1 | xxd -r -p | xxd -i // echo -n 'ZSSP_Noise_IKpsk2_NISTP384_?KYBER1024_AESGCM_SHA512' | shasum -a 512 | cut -d ' ' -f 1 | xxd -r -p | xxd -i
0x35, 0x6a, 0x75, 0xc0, 0xbf, 0xbe, 0xc3, 0x59, 0x70, 0x94, 0x50, 0x69, 0x4c, 0xa2, 0x08, 0x40, 0xc7, 0xdf, 0x67, 0xa8, 0x68, 0x52, 0x6e, 0xd5, 0xdd, 0x77, 0xec, 0x59, 0x6f, 0x8e, 0xa1, 0x99, 0x35, 0x6a, 0x75, 0xc0, 0xbf, 0xbe, 0xc3, 0x59, 0x70, 0x94, 0x50, 0x69, 0x4c, 0xa2, 0x08, 0x40, 0xc7, 0xdf, 0x67, 0xa8, 0x68, 0x52,
0xb4, 0x32, 0x85, 0xaf, 0x7f, 0x0d, 0xa9, 0x6c, 0x01, 0xfb, 0x72, 0x46, 0xc0, 0x09, 0x58, 0xb8, 0xe0, 0xa8, 0xcf, 0xb1, 0x58, 0x04, 0x6e, 0x32, 0xba, 0xa8, 0xb8, 0xf9, 0x0a, 0xa4, 0xbf, 0x36, 0x6e, 0xd5, 0xdd, 0x77, 0xec, 0x59, 0x6f, 0x8e, 0xa1, 0x99, 0xb4, 0x32, 0x85, 0xaf, 0x7f, 0x0d, 0xa9, 0x6c, 0x01, 0xfb, 0x72, 0x46,
0xc0, 0x09, 0x58, 0xb8, 0xe0, 0xa8, 0xcf, 0xb1, 0x58, 0x04, 0x6e, 0x32, 0xba, 0xa8, 0xb8, 0xf9, 0x0a, 0xa4, 0xbf, 0x36,
]; ];
pub enum Error { pub enum Error {
@ -232,7 +233,8 @@ impl SessionId {
pub fn new_from_reader<R: Read>(r: &mut R) -> std::io::Result<Option<SessionId>> { pub fn new_from_reader<R: Read>(r: &mut R) -> std::io::Result<Option<SessionId>> {
let mut tmp = [0_u8; 8]; let mut tmp = [0_u8; 8];
r.read_exact(&mut tmp[..SESSION_ID_SIZE]).map(|_| NonZeroU64::new(u64::from_le_bytes(tmp)).map(|i| Self(i))) r.read_exact(&mut tmp[..SESSION_ID_SIZE])
.map(|_| NonZeroU64::new(u64::from_le_bytes(tmp)).map(|i| Self(i)))
} }
} }
@ -278,7 +280,12 @@ pub trait Host: Sized {
/// On success a tuple of local session ID, static secret, and associated object is returned. The /// On success a tuple of local session ID, static secret, and associated object is returned. The
/// static secret is whatever results from agreement between the local and remote static public /// static secret is whatever results from agreement between the local and remote static public
/// keys. /// keys.
fn accept_new_session(&self, receive_context: &ReceiveContext<Self>, remote_static_public: &[u8], remote_metadata: &[u8]) -> Option<(SessionId, Secret<64>, Self::AssociatedObject)>; fn accept_new_session(
&self,
receive_context: &ReceiveContext<Self>,
remote_static_public: &[u8],
remote_metadata: &[u8],
) -> Option<(SessionId, Secret<64>, Self::AssociatedObject)>;
} }
/// ZSSP bi-directional packet transport channel. /// ZSSP bi-directional packet transport channel.
@ -338,7 +345,8 @@ impl<H: Host> Session<H> {
let send_counter = Counter::new(); let send_counter = Counter::new();
let header_check_cipher = Aes::new(kbkdf512(ss.as_bytes(), KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>()); let header_check_cipher = Aes::new(kbkdf512(ss.as_bytes(), KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>());
let remote_s_public_hash = SHA384::hash(remote_s_public); let remote_s_public_hash = SHA384::hash(remote_s_public);
let outgoing_init_header_check_cipher = Aes::new(kbkdf512(&remote_s_public_hash, KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>()); let outgoing_init_header_check_cipher =
Aes::new(kbkdf512(&remote_s_public_hash, KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>());
if let Ok(offer) = create_initial_offer( if let Ok(offer) = create_initial_offer(
&mut send, &mut send,
send_counter.next(), send_counter.next(),
@ -381,7 +389,12 @@ impl<H: Host> Session<H> {
/// ///
/// * `mtu_buffer` - A writable work buffer whose size must be equal to the wire MTU /// * `mtu_buffer` - A writable work buffer whose size must be equal to the wire MTU
/// * `data` - Data to send /// * `data` - Data to send
pub fn send<SendFunction: FnMut(&mut [u8])>(&self, mut send: SendFunction, mtu_buffer: &mut [u8], mut data: &[u8]) -> Result<(), Error> { pub fn send<SendFunction: FnMut(&mut [u8])>(
&self,
mut send: SendFunction,
mtu_buffer: &mut [u8],
mut data: &[u8],
) -> Result<(), Error> {
debug_assert!(mtu_buffer.len() >= MIN_MTU); debug_assert!(mtu_buffer.len() >= MIN_MTU);
let state = self.state.read(); let state = self.state.read();
if let Some(remote_session_id) = state.remote_session_id { if let Some(remote_session_id) = state.remote_session_id {
@ -389,7 +402,14 @@ impl<H: Host> Session<H> {
let mut packet_len = data.len() + HEADER_SIZE + AES_GCM_TAG_SIZE; let mut packet_len = data.len() + HEADER_SIZE + AES_GCM_TAG_SIZE;
let counter = self.send_counter.next(); let counter = self.send_counter.next();
create_packet_header(mtu_buffer, packet_len, mtu_buffer.len(), PACKET_TYPE_DATA, remote_session_id.into(), counter)?; create_packet_header(
mtu_buffer,
packet_len,
mtu_buffer.len(),
PACKET_TYPE_DATA,
remote_session_id.into(),
counter,
)?;
let mut c = key.get_send_cipher(counter)?; let mut c = key.get_send_cipher(counter)?;
c.init(memory::as_byte_array::<Pseudoheader, 12>(&Pseudoheader::make( c.init(memory::as_byte_array::<Pseudoheader, 12>(&Pseudoheader::make(
@ -461,13 +481,24 @@ impl<H: Host> Session<H> {
/// * `mtu` - Physical MTU for sent packets /// * `mtu` - Physical MTU for sent packets
/// * `current_time` - Current monotonic time in milliseconds /// * `current_time` - Current monotonic time in milliseconds
/// * `force_rekey` - Re-key the session now regardless of key aging (still subject to rate limiting) /// * `force_rekey` - Re-key the session now regardless of key aging (still subject to rate limiting)
pub fn service<SendFunction: FnMut(&mut [u8])>(&self, host: &H, mut send: SendFunction, offer_metadata: &[u8], mtu: usize, current_time: i64, force_rekey: bool) { pub fn service<SendFunction: FnMut(&mut [u8])>(
&self,
host: &H,
mut send: SendFunction,
offer_metadata: &[u8],
mtu: usize,
current_time: i64,
force_rekey: bool,
) {
let state = self.state.upgradable_read(); let state = self.state.upgradable_read();
if (force_rekey if (force_rekey
|| state.keys[state.key_ptr] || state.keys[state.key_ptr]
.as_ref() .as_ref()
.map_or(true, |key| key.lifetime.should_rekey(self.send_counter.current(), current_time))) .map_or(true, |key| key.lifetime.should_rekey(self.send_counter.current(), current_time)))
&& state.offer.as_ref().map_or(true, |o| (current_time - o.creation_time) > OFFER_RATE_LIMIT_MS) && state
.offer
.as_ref()
.map_or(true, |o| (current_time - o.creation_time) > OFFER_RATE_LIMIT_MS)
{ {
if let Some(remote_s_public_p384) = P384PublicKey::from_bytes(&self.remote_s_public_p384) { if let Some(remote_s_public_p384) = P384PublicKey::from_bytes(&self.remote_s_public_p384) {
let mut tmp_header_check_cipher = None; let mut tmp_header_check_cipher = None;
@ -485,7 +516,9 @@ impl<H: Host> Session<H> {
if state.remote_session_id.is_some() { if state.remote_session_id.is_some() {
&self.header_check_cipher &self.header_check_cipher
} else { } else {
let _ = tmp_header_check_cipher.insert(Aes::new(kbkdf512(&self.remote_s_public_hash, KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>())); let _ = tmp_header_check_cipher.insert(Aes::new(
kbkdf512(&self.remote_s_public_hash, KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>(),
));
tmp_header_check_cipher.as_ref().unwrap() tmp_header_check_cipher.as_ref().unwrap()
}, },
mtu, mtu,
@ -502,7 +535,9 @@ impl<H: Host> ReceiveContext<H> {
pub fn new(host: &H) -> Self { pub fn new(host: &H) -> Self {
Self { Self {
initial_offer_defrag: Mutex::new(RingBufferMap::new(random::xorshift64_random() as u32)), initial_offer_defrag: Mutex::new(RingBufferMap::new(random::xorshift64_random() as u32)),
incoming_init_header_check_cipher: Aes::new(kbkdf512(host.get_local_s_public_hash(), KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>()), incoming_init_header_check_cipher: Aes::new(
kbkdf512(host.get_local_s_public_hash(), KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>(),
),
} }
} }
@ -529,7 +564,9 @@ impl<H: Host> ReceiveContext<H> {
if let Some(local_session_id) = SessionId::new_from_u64(memory::u64_from_le_bytes(incoming_packet) & SessionId::MAX_BIT_MASK) { if let Some(local_session_id) = SessionId::new_from_u64(memory::u64_from_le_bytes(incoming_packet) & SessionId::MAX_BIT_MASK) {
if let Some(session) = host.session_lookup(local_session_id) { if let Some(session) = host.session_lookup(local_session_id) {
if let Some((packet_type, fragment_count, fragment_no, counter)) = dearmor_header(incoming_packet, &session.header_check_cipher) { if let Some((packet_type, fragment_count, fragment_no, counter)) =
dearmor_header(incoming_packet, &session.header_check_cipher)
{
let pseudoheader = Pseudoheader::make(u64::from(local_session_id), packet_type, counter); let pseudoheader = Pseudoheader::make(u64::from(local_session_id), packet_type, counter);
if fragment_count > 1 { if fragment_count > 1 {
if fragment_count <= (MAX_FRAGMENTS as u8) && fragment_no < fragment_count { if fragment_count <= (MAX_FRAGMENTS as u8) && fragment_no < fragment_count {
@ -576,7 +613,9 @@ impl<H: Host> ReceiveContext<H> {
} }
} else { } else {
unlikely_branch(); unlikely_branch();
if let Some((packet_type, fragment_count, fragment_no, counter)) = dearmor_header(incoming_packet, &self.incoming_init_header_check_cipher) { if let Some((packet_type, fragment_count, fragment_no, counter)) =
dearmor_header(incoming_packet, &self.incoming_init_header_check_cipher)
{
let pseudoheader = Pseudoheader::make(0, packet_type, counter); let pseudoheader = Pseudoheader::make(0, packet_type, counter);
if fragment_count > 1 { if fragment_count > 1 {
let mut defrag = self.initial_offer_defrag.lock(); let mut defrag = self.initial_offer_defrag.lock();
@ -670,13 +709,20 @@ impl<H: Host> ReceiveContext<H> {
key.return_receive_cipher(c); key.return_receive_cipher(c);
return Err(Error::DataBufferTooSmall); return Err(Error::DataBufferTooSmall);
} }
c.crypt(&tail[HEADER_SIZE..(tail.len() - AES_GCM_TAG_SIZE)], &mut data_buf[current_frag_data_start..data_len]); c.crypt(
&tail[HEADER_SIZE..(tail.len() - AES_GCM_TAG_SIZE)],
&mut data_buf[current_frag_data_start..data_len],
);
let ok = c.finish_decrypt(&tail[(tail.len() - AES_GCM_TAG_SIZE)..]); let ok = c.finish_decrypt(&tail[(tail.len() - AES_GCM_TAG_SIZE)..]);
key.return_receive_cipher(c); key.return_receive_cipher(c);
if ok { if ok {
// Select this key as the new default if it's newer than the current key. // Select this key as the new default if it's newer than the current key.
if p > 0 && state.keys[state.key_ptr].as_ref().map_or(true, |old| old.establish_counter < key.establish_counter) { if p > 0
&& state.keys[state.key_ptr]
.as_ref()
.map_or(true, |old| old.establish_counter < key.establish_counter)
{
drop(state); drop(state);
let mut state = session.state.write(); let mut state = session.state.write();
state.key_ptr = key_ptr; state.key_ptr = key_ptr;
@ -741,7 +787,13 @@ impl<H: Host> ReceiveContext<H> {
let hmac1_end = incoming_packet_len - HMAC_SIZE; let hmac1_end = incoming_packet_len - HMAC_SIZE;
// Check that the sender knows this host's identity before doing anything else. // Check that the sender knows this host's identity before doing anything else.
if !hmac_sha384_2(host.get_local_s_public_hash(), pseudoheader, &incoming_packet[HEADER_SIZE..hmac1_end]).eq(&incoming_packet[hmac1_end..]) { if !hmac_sha384_2(
host.get_local_s_public_hash(),
pseudoheader,
&incoming_packet[HEADER_SIZE..hmac1_end],
)
.eq(&incoming_packet[hmac1_end..])
{
return Err(Error::FailedAuthentication); return Err(Error::FailedAuthentication);
} }
@ -755,15 +807,19 @@ impl<H: Host> ReceiveContext<H> {
} }
// Key agreement: alice (remote) ephemeral NIST P-384 <> local static NIST P-384 // Key agreement: alice (remote) ephemeral NIST P-384 <> local static NIST P-384
let (alice_e0_public, e0s) = P384PublicKey::from_bytes(&incoming_packet[(HEADER_SIZE + 1)..(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)]) let (alice_e0_public, e0s) =
.and_then(|pk| host.get_local_s_keypair_p384().agree(&pk).map(move |s| (pk, s))) P384PublicKey::from_bytes(&incoming_packet[(HEADER_SIZE + 1)..(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)])
.ok_or(Error::FailedAuthentication)?; .and_then(|pk| host.get_local_s_keypair_p384().agree(&pk).map(move |s| (pk, s)))
.ok_or(Error::FailedAuthentication)?;
// Initial key derivation from starting point, mixing in alice's ephemeral public and the e0s. // Initial key derivation from starting point, mixing in alice's ephemeral public and the e0s.
let mut key = Secret(hmac_sha512(&hmac_sha512(&INITIAL_KEY, alice_e0_public.as_bytes()), e0s.as_bytes())); let mut key = Secret(hmac_sha512(&hmac_sha512(&INITIAL_KEY, alice_e0_public.as_bytes()), e0s.as_bytes()));
// Decrypt the encrypted part of the packet payload and authenticate the above key exchange via AES-GCM auth. // Decrypt the encrypted part of the packet payload and authenticate the above key exchange via AES-GCM auth.
let mut c = AesGcm::new(kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_ALICE_TO_BOB).first_n::<32>(), false); let mut c = AesGcm::new(
kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_ALICE_TO_BOB).first_n::<32>(),
false,
);
c.init(pseudoheader); c.init(pseudoheader);
c.crypt_in_place(&mut incoming_packet[(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)..payload_end]); c.crypt_in_place(&mut incoming_packet[(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)..payload_end]);
if !c.finish_decrypt(&incoming_packet[payload_end..aes_gcm_tag_end]) { if !c.finish_decrypt(&incoming_packet[payload_end..aes_gcm_tag_end]) {
@ -812,7 +868,10 @@ impl<H: Host> ReceiveContext<H> {
let alice_s_public_p384 = H::extract_p384_static(alice_s_public).ok_or(Error::InvalidPacket)?; let alice_s_public_p384 = H::extract_p384_static(alice_s_public).ok_or(Error::InvalidPacket)?;
// Key agreement: both sides' static P-384 keys. // Key agreement: both sides' static P-384 keys.
let ss = host.get_local_s_keypair_p384().agree(&alice_s_public_p384).ok_or(Error::FailedAuthentication)?; let ss = host
.get_local_s_keypair_p384()
.agree(&alice_s_public_p384)
.ok_or(Error::FailedAuthentication)?;
// Mix result of 'ss' agreement into master key. // Mix result of 'ss' agreement into master key.
key = Secret(hmac_sha512(key.as_bytes(), ss.as_bytes())); key = Secret(hmac_sha512(key.as_bytes(), ss.as_bytes()));
@ -843,7 +902,9 @@ impl<H: Host> ReceiveContext<H> {
let new_session = if session.is_some() { let new_session = if session.is_some() {
None None
} else { } else {
if let Some((new_session_id, psk, associated_object)) = host.accept_new_session(self, alice_s_public, alice_metadata) { if let Some((new_session_id, psk, associated_object)) =
host.accept_new_session(self, alice_s_public, alice_metadata)
{
let header_check_cipher = Aes::new(kbkdf512(ss.as_bytes(), KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>()); let header_check_cipher = Aes::new(kbkdf512(ss.as_bytes(), KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>());
Some(Session::<H> { Some(Session::<H> {
id: new_session_id, id: new_session_id,
@ -877,7 +938,10 @@ impl<H: Host> ReceiveContext<H> {
// fixed size so this shouldn't matter cryptographically. // fixed size so this shouldn't matter cryptographically.
key = Secret(hmac_sha512( key = Secret(hmac_sha512(
session.psk.as_bytes(), session.psk.as_bytes(),
&hmac_sha512(&hmac_sha512(&hmac_sha512(key.as_bytes(), bob_e0_keypair.public_key_bytes()), e0e0.as_bytes()), se0.as_bytes()), &hmac_sha512(
&hmac_sha512(&hmac_sha512(key.as_bytes(), bob_e0_keypair.public_key_bytes()), e0e0.as_bytes()),
se0.as_bytes(),
),
)); ));
// At this point we've completed Noise_IK key derivation with NIST P-384 ECDH, but now for hybrid and ratcheting... // At this point we've completed Noise_IK key derivation with NIST P-384 ECDH, but now for hybrid and ratcheting...
@ -922,12 +986,23 @@ impl<H: Host> ReceiveContext<H> {
REPLY_BUF_LEN - rp.len() REPLY_BUF_LEN - rp.len()
}; };
create_packet_header(&mut reply_buf, reply_len, mtu, PACKET_TYPE_KEY_COUNTER_OFFER, alice_session_id.into(), reply_counter)?; create_packet_header(
let reply_pseudoheader = Pseudoheader::make(alice_session_id.into(), PACKET_TYPE_KEY_COUNTER_OFFER, reply_counter.to_u32()); &mut reply_buf,
reply_len,
mtu,
PACKET_TYPE_KEY_COUNTER_OFFER,
alice_session_id.into(),
reply_counter,
)?;
let reply_pseudoheader =
Pseudoheader::make(alice_session_id.into(), PACKET_TYPE_KEY_COUNTER_OFFER, reply_counter.to_u32());
// Encrypt reply packet using final Noise_IK key BEFORE mixing hybrid or ratcheting, since the other side // Encrypt reply packet using final Noise_IK key BEFORE mixing hybrid or ratcheting, since the other side
// must decrypt before doing these things. // must decrypt before doing these things.
let mut c = AesGcm::new(kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_BOB_TO_ALICE).first_n::<32>(), true); let mut c = AesGcm::new(
kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_BOB_TO_ALICE).first_n::<32>(),
true,
);
c.init(memory::as_byte_array::<Pseudoheader, 12>(&reply_pseudoheader)); c.init(memory::as_byte_array::<Pseudoheader, 12>(&reply_pseudoheader));
c.crypt_in_place(&mut reply_buf[(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)..reply_len]); c.crypt_in_place(&mut reply_buf[(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)..reply_len]);
let c = c.finish_encrypt(); let c = c.finish_encrypt();
@ -985,17 +1060,27 @@ impl<H: Host> ReceiveContext<H> {
if let Some(session) = session { if let Some(session) = session {
let state = session.state.upgradable_read(); let state = session.state.upgradable_read();
if let Some(offer) = state.offer.as_ref() { if let Some(offer) = state.offer.as_ref() {
let (bob_e0_public, e0e0) = P384PublicKey::from_bytes(&incoming_packet[(HEADER_SIZE + 1)..(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)]) let (bob_e0_public, e0e0) =
.and_then(|pk| offer.alice_e0_keypair.agree(&pk).map(move |s| (pk, s))) P384PublicKey::from_bytes(&incoming_packet[(HEADER_SIZE + 1)..(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)])
.and_then(|pk| offer.alice_e0_keypair.agree(&pk).map(move |s| (pk, s)))
.ok_or(Error::FailedAuthentication)?;
let se0 = host
.get_local_s_keypair_p384()
.agree(&bob_e0_public)
.ok_or(Error::FailedAuthentication)?; .ok_or(Error::FailedAuthentication)?;
let se0 = host.get_local_s_keypair_p384().agree(&bob_e0_public).ok_or(Error::FailedAuthentication)?;
let mut key = Secret(hmac_sha512( let mut key = Secret(hmac_sha512(
session.psk.as_bytes(), session.psk.as_bytes(),
&hmac_sha512(&hmac_sha512(&hmac_sha512(offer.key.as_bytes(), bob_e0_public.as_bytes()), e0e0.as_bytes()), se0.as_bytes()), &hmac_sha512(
&hmac_sha512(&hmac_sha512(offer.key.as_bytes(), bob_e0_public.as_bytes()), e0e0.as_bytes()),
se0.as_bytes(),
),
)); ));
let mut c = AesGcm::new(kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_BOB_TO_ALICE).first_n::<32>(), false); let mut c = AesGcm::new(
kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_BOB_TO_ALICE).first_n::<32>(),
false,
);
c.init(pseudoheader); c.init(pseudoheader);
c.crypt_in_place(&mut incoming_packet[(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)..payload_end]); c.crypt_in_place(&mut incoming_packet[(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)..payload_end]);
if !c.finish_decrypt(&incoming_packet[payload_end..aes_gcm_tag_end]) { if !c.finish_decrypt(&incoming_packet[payload_end..aes_gcm_tag_end]) {
@ -1046,10 +1131,21 @@ impl<H: Host> ReceiveContext<H> {
let key = SessionKey::new(key, Role::Alice, current_time, counter, ratchet_count + 1, e1e1.is_some()); let key = SessionKey::new(key, Role::Alice, current_time, counter, ratchet_count + 1, e1e1.is_some());
let mut reply_buf = [0_u8; HEADER_SIZE + AES_GCM_TAG_SIZE]; let mut reply_buf = [0_u8; HEADER_SIZE + AES_GCM_TAG_SIZE];
create_packet_header(&mut reply_buf, HEADER_SIZE + AES_GCM_TAG_SIZE, mtu, PACKET_TYPE_NOP, bob_session_id.into(), counter)?; create_packet_header(
&mut reply_buf,
HEADER_SIZE + AES_GCM_TAG_SIZE,
mtu,
PACKET_TYPE_NOP,
bob_session_id.into(),
counter,
)?;
let mut c = key.get_send_cipher(counter)?; let mut c = key.get_send_cipher(counter)?;
c.init(memory::as_byte_array::<Pseudoheader, 12>(&Pseudoheader::make(bob_session_id.into(), PACKET_TYPE_NOP, counter.to_u32()))); c.init(memory::as_byte_array::<Pseudoheader, 12>(&Pseudoheader::make(
bob_session_id.into(),
PACKET_TYPE_NOP,
counter.to_u32(),
)));
reply_buf[HEADER_SIZE..].copy_from_slice(&c.finish_encrypt()); reply_buf[HEADER_SIZE..].copy_from_slice(&c.finish_encrypt());
key.return_send_cipher(c); key.return_send_cipher(c);
@ -1204,10 +1300,16 @@ fn create_initial_offer<SendFunction: FnMut(&mut [u8])>(
create_packet_header(&mut packet_buf, packet_len, mtu, PACKET_TYPE_KEY_OFFER, bob_session_id, counter)?; create_packet_header(&mut packet_buf, packet_len, mtu, PACKET_TYPE_KEY_OFFER, bob_session_id, counter)?;
let pseudoheader = Pseudoheader::make(bob_session_id, PACKET_TYPE_KEY_OFFER, counter.to_u32()); let pseudoheader = Pseudoheader::make(bob_session_id, PACKET_TYPE_KEY_OFFER, counter.to_u32());
let key = Secret(hmac_sha512(&hmac_sha512(&INITIAL_KEY, alice_e0_keypair.public_key_bytes()), e0s.unwrap().as_bytes())); let key = Secret(hmac_sha512(
&hmac_sha512(&INITIAL_KEY, alice_e0_keypair.public_key_bytes()),
e0s.unwrap().as_bytes(),
));
let gcm_tag = { let gcm_tag = {
let mut c = AesGcm::new(kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_ALICE_TO_BOB).first_n::<32>(), true); let mut c = AesGcm::new(
kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_ALICE_TO_BOB).first_n::<32>(),
true,
);
c.init(memory::as_byte_array::<Pseudoheader, 12>(&pseudoheader)); c.init(memory::as_byte_array::<Pseudoheader, 12>(&pseudoheader));
c.crypt_in_place(&mut packet_buf[(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)..packet_len]); c.crypt_in_place(&mut packet_buf[(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)..packet_len]);
c.finish_encrypt() c.finish_encrypt()
@ -1225,7 +1327,11 @@ fn create_initial_offer<SendFunction: FnMut(&mut [u8])>(
packet_buf[packet_len..(packet_len + HMAC_SIZE)].copy_from_slice(&hmac); packet_buf[packet_len..(packet_len + HMAC_SIZE)].copy_from_slice(&hmac);
packet_len += HMAC_SIZE; packet_len += HMAC_SIZE;
let hmac = hmac_sha384_2(bob_s_public_hash, memory::as_byte_array::<Pseudoheader, 12>(&pseudoheader), &packet_buf[HEADER_SIZE..packet_len]); let hmac = hmac_sha384_2(
bob_s_public_hash,
memory::as_byte_array::<Pseudoheader, 12>(&pseudoheader),
&packet_buf[HEADER_SIZE..packet_len],
);
packet_buf[packet_len..(packet_len + HMAC_SIZE)].copy_from_slice(&hmac); packet_buf[packet_len..(packet_len + HMAC_SIZE)].copy_from_slice(&hmac);
packet_len += HMAC_SIZE; packet_len += HMAC_SIZE;
@ -1243,7 +1349,14 @@ fn create_initial_offer<SendFunction: FnMut(&mut [u8])>(
} }
#[inline(always)] #[inline(always)]
fn create_packet_header(header: &mut [u8], packet_len: usize, mtu: usize, packet_type: u8, recipient_session_id: u64, counter: CounterValue) -> Result<(), Error> { fn create_packet_header(
header: &mut [u8],
packet_len: usize,
mtu: usize,
packet_type: u8,
recipient_session_id: u64,
counter: CounterValue,
) -> Result<(), Error> {
let fragment_count = ((packet_len as f32) / (mtu - HEADER_SIZE) as f32).ceil() as usize; let fragment_count = ((packet_len as f32) / (mtu - HEADER_SIZE) as f32).ceil() as usize;
debug_assert!(header.len() >= HEADER_SIZE); debug_assert!(header.len() >= HEADER_SIZE);
@ -1255,7 +1368,9 @@ fn create_packet_header(header: &mut [u8], packet_len: usize, mtu: usize, packet
debug_assert!(recipient_session_id <= 0xffffffffffff); // session ID is 48 bits debug_assert!(recipient_session_id <= 0xffffffffffff); // session ID is 48 bits
if fragment_count <= MAX_FRAGMENTS { if fragment_count <= MAX_FRAGMENTS {
header[0..8].copy_from_slice(&(recipient_session_id | (packet_type as u64).wrapping_shl(48) | ((fragment_count - 1) as u64).wrapping_shl(52)).to_le_bytes()); header[0..8].copy_from_slice(
&(recipient_session_id | (packet_type as u64).wrapping_shl(48) | ((fragment_count - 1) as u64).wrapping_shl(52)).to_le_bytes(),
);
header[8..12].copy_from_slice(&counter.to_u32().to_le_bytes()); header[8..12].copy_from_slice(&counter.to_u32().to_le_bytes());
header[12..16].fill(0); header[12..16].fill(0);
Ok(()) Ok(())
@ -1265,7 +1380,12 @@ fn create_packet_header(header: &mut [u8], packet_len: usize, mtu: usize, packet
} }
} }
fn send_with_fragmentation<SendFunction: FnMut(&mut [u8])>(send: &mut SendFunction, packet: &mut [u8], mtu: usize, header_check_cipher: &Aes) { fn send_with_fragmentation<SendFunction: FnMut(&mut [u8])>(
send: &mut SendFunction,
packet: &mut [u8],
mtu: usize,
header_check_cipher: &Aes,
) {
let packet_len = packet.len(); let packet_len = packet.len();
let mut fragment_start = 0; let mut fragment_start = 0;
let mut fragment_end = packet_len.min(mtu); let mut fragment_end = packet_len.min(mtu);
@ -1294,7 +1414,10 @@ fn armor_header(packet: &mut [u8], header_check_cipher: &Aes) {
debug_assert!(packet.len() >= MIN_PACKET_SIZE); debug_assert!(packet.len() >= MIN_PACKET_SIZE);
let mut header_pad = 0u128.to_ne_bytes(); let mut header_pad = 0u128.to_ne_bytes();
header_check_cipher.encrypt_block(&packet[16..32], &mut header_pad); header_check_cipher.encrypt_block(&packet[16..32], &mut header_pad);
packet[SESSION_ID_SIZE..HEADER_SIZE].iter_mut().zip(header_pad.iter()).for_each(|(x, y)| *x ^= *y); packet[SESSION_ID_SIZE..HEADER_SIZE]
.iter_mut()
.zip(header_pad.iter())
.for_each(|(x, y)| *x ^= *y);
} }
/// Dearmor the armored part of the header and return it if the 32-bit MAC matches. /// Dearmor the armored part of the header and return it if the 32-bit MAC matches.
@ -1326,7 +1449,10 @@ fn dearmor_header(packet: &[u8], header_check_cipher: &Aes) -> Option<(u8, u8, u
} }
/// Parse KEY_OFFER and KEY_COUNTER_OFFER starting after the unencrypted public key part. /// Parse KEY_OFFER and KEY_COUNTER_OFFER starting after the unencrypted public key part.
fn parse_key_offer_after_header(incoming_packet: &[u8], packet_type: u8) -> Result<([u8; 16], SessionId, &[u8], &[u8], &[u8], Option<[u8; 16]>), Error> { fn parse_key_offer_after_header(
incoming_packet: &[u8],
packet_type: u8,
) -> Result<([u8; 16], SessionId, &[u8], &[u8], &[u8], Option<[u8; 16]>), Error> {
let mut p = &incoming_packet[..]; let mut p = &incoming_packet[..];
let mut offer_id = [0_u8; 16]; let mut offer_id = [0_u8; 16];
p.read_exact(&mut offer_id)?; p.read_exact(&mut offer_id)?;
@ -1381,7 +1507,14 @@ fn parse_key_offer_after_header(incoming_packet: &[u8], packet_type: u8) -> Resu
} else { } else {
None None
}; };
Ok((offer_id, alice_session_id, alice_s_public, alice_metadata, alice_e1_public, alice_ratchet_key_fingerprint)) Ok((
offer_id,
alice_session_id,
alice_s_public,
alice_metadata,
alice_e1_public,
alice_ratchet_key_fingerprint,
))
} }
enum Role { enum Role {
@ -1399,9 +1532,13 @@ struct KeyLifetime {
impl KeyLifetime { impl KeyLifetime {
fn new(current_counter: CounterValue, current_time: i64) -> Self { fn new(current_counter: CounterValue, current_time: i64) -> Self {
Self { Self {
rekey_at_or_after_counter: current_counter.0 + REKEY_AFTER_USES + (random::next_u32_secure() % REKEY_AFTER_USES_MAX_JITTER) as u64, rekey_at_or_after_counter: current_counter.0
+ REKEY_AFTER_USES
+ (random::next_u32_secure() % REKEY_AFTER_USES_MAX_JITTER) as u64,
hard_expire_at_counter: current_counter.0 + EXPIRE_AFTER_USES, hard_expire_at_counter: current_counter.0 + EXPIRE_AFTER_USES,
rekey_at_or_after_timestamp: current_time + REKEY_AFTER_TIME_MS + (random::next_u32_secure() % REKEY_AFTER_TIME_MS_MAX_JITTER) as i64, rekey_at_or_after_timestamp: current_time
+ REKEY_AFTER_TIME_MS
+ (random::next_u32_secure() % REKEY_AFTER_TIME_MS_MAX_JITTER) as i64,
} }
} }
@ -1458,7 +1595,11 @@ impl SessionKey {
#[inline(always)] #[inline(always)]
fn get_send_cipher(&self, counter: CounterValue) -> Result<Box<AesGcm>, Error> { fn get_send_cipher(&self, counter: CounterValue) -> Result<Box<AesGcm>, Error> {
if !self.lifetime.expired(counter) { if !self.lifetime.expired(counter) {
Ok(self.send_cipher_pool.lock().pop().unwrap_or_else(|| Box::new(AesGcm::new(self.send_key.as_bytes(), true)))) Ok(self
.send_cipher_pool
.lock()
.pop()
.unwrap_or_else(|| Box::new(AesGcm::new(self.send_key.as_bytes(), true))))
} else { } else {
Err(Error::MaxKeyLifetimeExceeded) Err(Error::MaxKeyLifetimeExceeded)
} }
@ -1471,7 +1612,10 @@ impl SessionKey {
#[inline(always)] #[inline(always)]
fn get_receive_cipher(&self) -> Box<AesGcm> { fn get_receive_cipher(&self) -> Box<AesGcm> {
self.receive_cipher_pool.lock().pop().unwrap_or_else(|| Box::new(AesGcm::new(self.receive_key.as_bytes(), false))) self.receive_cipher_pool
.lock()
.pop()
.unwrap_or_else(|| Box::new(AesGcm::new(self.receive_key.as_bytes(), false)))
} }
#[inline(always)] #[inline(always)]
@ -1574,7 +1718,12 @@ mod tests {
}) })
} }
fn accept_new_session(&self, _: &ReceiveContext<Self>, _: &[u8], _: &[u8]) -> Option<(SessionId, Secret<64>, Self::AssociatedObject)> { fn accept_new_session(
&self,
_: &ReceiveContext<Self>,
_: &[u8],
_: &[u8],
) -> Option<(SessionId, Secret<64>, Self::AssociatedObject)> {
loop { loop {
let mut new_id = self.session_id_counter.lock(); let mut new_id = self.session_id_counter.lock();
*new_id += 1; *new_id += 1;
@ -1646,7 +1795,13 @@ mod tests {
assert!(!data.iter().any(|x| *x != 0x12)); assert!(!data.iter().any(|x| *x != 0x12));
} }
ReceiveResult::OkNewSession(new_session) => { ReceiveResult::OkNewSession(new_session) => {
println!("zssp: {} => {} ({}): OkNewSession ({})", host.other_name, host.this_name, qi_len, u64::from(new_session.id)); println!(
"zssp: {} => {} ({}): OkNewSession ({})",
host.other_name,
host.this_name,
qi_len,
u64::from(new_session.id)
);
let mut hs = host.session.lock(); let mut hs = host.session.lock();
assert!(hs.is_none()); assert!(hs.is_none());
let _ = hs.insert(Arc::new(new_session)); let _ = hs.insert(Arc::new(new_session));
@ -1656,7 +1811,13 @@ mod tests {
} }
} }
} else { } else {
println!("zssp: {} => {} ({}): error: {}", host.other_name, host.this_name, qi_len, r.err().unwrap().to_string()); println!(
"zssp: {} => {} ({}): error: {}",
host.other_name,
host.this_name,
qi_len,
r.err().unwrap().to_string()
);
panic!(); panic!();
} }
} else { } else {
@ -1683,7 +1844,11 @@ mod tests {
} }
for _ in 0..4 { for _ in 0..4 {
assert!(session assert!(session
.send(send_to_other, &mut mtu_buffer, &data_buf[..((random::xorshift64_random() as usize) % data_buf.len())]) .send(
send_to_other,
&mut mtu_buffer,
&data_buf[..((random::xorshift64_random() as usize) % data_buf.len())]
)
.is_ok()); .is_ok());
} }
if (test_loop % 8) == 0 && test_loop >= 8 && host.this_name.eq("alice") { if (test_loop % 8) == 0 && test_loop >= 8 && host.this_name.eq("alice") {

13
network-hypervisor/src/networkhypervisor.rs
View file

@ -54,8 +54,17 @@ impl<I: Interface> NetworkHypervisor<I> {
/// Process a physical packet received over a network interface. /// Process a physical packet received over a network interface.
#[inline(always)] #[inline(always)]
pub async fn handle_incoming_physical_packet(&self, ii: &I, source_endpoint: &Endpoint, source_local_socket: &I::LocalSocket, source_local_interface: &I::LocalInterface, data: PooledPacketBuffer) { pub async fn handle_incoming_physical_packet(
self.vl1.handle_incoming_physical_packet(ii, &self.vl2, source_endpoint, source_local_socket, source_local_interface, data).await &self,
ii: &I,
source_endpoint: &Endpoint,
source_local_socket: &I::LocalSocket,
source_local_interface: &I::LocalInterface,
data: PooledPacketBuffer,
) {
self.vl1
.handle_incoming_physical_packet(ii, &self.vl2, source_endpoint, source_local_socket, source_local_interface, data)
.await
} }
/// Add or update a root set. /// Add or update a root set.

13
network-hypervisor/src/util/testutil.rs
View file

@ -2,5 +2,16 @@
// from zeronsd // from zeronsd
pub fn randstring(len: u8) -> String { pub fn randstring(len: u8) -> String {
(0..len).map(|_| (rand::random::<u8>() % 26) + 'a' as u8).map(|c| if rand::random::<bool>() { (c as char).to_ascii_uppercase() } else { c as char }).map(|c| c.to_string()).collect::<Vec<String>>().join("") (0..len)
.map(|_| (rand::random::<u8>() % 26) + 'a' as u8)
.map(|c| {
if rand::random::<bool>() {
(c as char).to_ascii_uppercase()
} else {
c as char
}
})
.map(|c| c.to_string())
.collect::<Vec<String>>()
.join("")
} }

15
network-hypervisor/src/vl1/address.rs
View file

@ -78,7 +78,10 @@ impl Marshalable for Address {
#[inline(always)] #[inline(always)]
fn unmarshal<const BL: usize>(buf: &Buffer<BL>, cursor: &mut usize) -> std::io::Result<Self> { fn unmarshal<const BL: usize>(buf: &Buffer<BL>, cursor: &mut usize) -> std::io::Result<Self> {
Self::from_bytes_fixed(buf.read_bytes_fixed(cursor)?).map_or_else(|| Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "cannot be zero")), |a| Ok(a)) Self::from_bytes_fixed(buf.read_bytes_fixed(cursor)?).map_or_else(
|| Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "cannot be zero")),
|a| Ok(a),
)
} }
} }
@ -275,8 +278,14 @@ mod tests {
let addr = safe_address(); let addr = safe_address();
for _ in 0..1000 { for _ in 0..1000 {
assert_eq!(serde_json::from_str::<super::Address>(&serde_json::to_string(&addr).unwrap()).unwrap(), addr); assert_eq!(
assert_eq!(serde_cbor::from_slice::<super::Address>(&serde_cbor::to_vec(&addr).unwrap()).unwrap(), addr); serde_json::from_str::<super::Address>(&serde_json::to_string(&addr).unwrap()).unwrap(),
addr
);
assert_eq!(
serde_cbor::from_slice::<super::Address>(&serde_cbor::to_vec(&addr).unwrap()).unwrap(),
addr
);
} }
} }
} }

22
network-hypervisor/src/vl1/dictionary.rs
View file

@ -81,7 +81,9 @@ impl Dictionary {
} }
pub fn get_str(&self, k: &str) -> Option<&str> { pub fn get_str(&self, k: &str) -> Option<&str> {
self.0.get(k).map_or(None, |v| std::str::from_utf8(v.as_slice()).map_or(None, |s| Some(s))) self.0
.get(k)
.map_or(None, |v| std::str::from_utf8(v.as_slice()).map_or(None, |s| Some(s)))
} }
pub fn get_bytes(&self, k: &str) -> Option<&[u8]> { pub fn get_bytes(&self, k: &str) -> Option<&[u8]> {
@ -89,15 +91,19 @@ impl Dictionary {
} }
pub fn get_u64(&self, k: &str) -> Option<u64> { pub fn get_u64(&self, k: &str) -> Option<u64> {
self.get_str(k).map_or(None, |s| u64::from_str_radix(s, 16).map_or(None, |i| Some(i))) self.get_str(k)
.map_or(None, |s| u64::from_str_radix(s, 16).map_or(None, |i| Some(i)))
} }
pub fn get_i64(&self, k: &str) -> Option<i64> { pub fn get_i64(&self, k: &str) -> Option<i64> {
self.get_str(k).map_or(None, |s| i64::from_str_radix(s, 16).map_or(None, |i| Some(i))) self.get_str(k)
.map_or(None, |s| i64::from_str_radix(s, 16).map_or(None, |i| Some(i)))
} }
pub fn get_bool(&self, k: &str) -> Option<bool> { pub fn get_bool(&self, k: &str) -> Option<bool> {
self.0.get(k).map_or(None, |v| v.first().map_or(Some(false), |c| Some(BOOL_TRUTH.contains(*c as char)))) self.0
.get(k)
.map_or(None, |v| v.first().map_or(Some(false), |c| Some(BOOL_TRUTH.contains(*c as char))))
} }
pub fn set_str(&mut self, k: &str, v: &str) { pub fn set_str(&mut self, k: &str, v: &str) {
@ -238,7 +244,13 @@ mod tests {
match selection { match selection {
0 => d.set_str(&key, &randstring(10)), 0 => d.set_str(&key, &randstring(10)),
1 => d.set_u64(&key, rand::random()), 1 => d.set_u64(&key, rand::random()),
2 => d.set_bytes(&key, (0..((rand::random::<usize>() % 10) + 1)).into_iter().map(|_| rand::random()).collect::<Vec<u8>>()), 2 => d.set_bytes(
&key,
(0..((rand::random::<usize>() % 10) + 1))
.into_iter()
.map(|_| rand::random())
.collect::<Vec<u8>>(),
),
3 => d.set_bool(&key, rand::random::<bool>()), 3 => d.set_bool(&key, rand::random::<bool>()),
_ => unreachable!(), _ => unreachable!(),
} }

50
network-hypervisor/src/vl1/endpoint.rs
View file

@ -190,19 +190,31 @@ impl Marshalable for Endpoint {
if type_byte < 16 { if type_byte < 16 {
if type_byte == 4 { if type_byte == 4 {
let b: &[u8; 6] = buf.read_bytes_fixed(cursor)?; let b: &[u8; 6] = buf.read_bytes_fixed(cursor)?;
Ok(Endpoint::IpUdp(InetAddress::from_ip_port(&b[0..4], u16::from_be_bytes(b[4..6].try_into().unwrap())))) Ok(Endpoint::IpUdp(InetAddress::from_ip_port(
&b[0..4],
u16::from_be_bytes(b[4..6].try_into().unwrap()),
)))
} else if type_byte == 6 { } else if type_byte == 6 {
let b: &[u8; 18] = buf.read_bytes_fixed(cursor)?; let b: &[u8; 18] = buf.read_bytes_fixed(cursor)?;
Ok(Endpoint::IpUdp(InetAddress::from_ip_port(&b[0..16], u16::from_be_bytes(b[16..18].try_into().unwrap())))) Ok(Endpoint::IpUdp(InetAddress::from_ip_port(
&b[0..16],
u16::from_be_bytes(b[16..18].try_into().unwrap()),
)))
} else { } else {
Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "unrecognized endpoint type in stream")) Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"unrecognized endpoint type in stream",
))
} }
} else { } else {
match type_byte - 16 { match type_byte - 16 {
TYPE_NIL => Ok(Endpoint::Nil), TYPE_NIL => Ok(Endpoint::Nil),
TYPE_ZEROTIER => { TYPE_ZEROTIER => {
let zt = Address::unmarshal(buf, cursor)?; let zt = Address::unmarshal(buf, cursor)?;
Ok(Endpoint::ZeroTier(zt, buf.read_bytes_fixed::<IDENTITY_FINGERPRINT_SIZE>(cursor)?.clone())) Ok(Endpoint::ZeroTier(
zt,
buf.read_bytes_fixed::<IDENTITY_FINGERPRINT_SIZE>(cursor)?.clone(),
))
} }
TYPE_ETHERNET => Ok(Endpoint::Ethernet(MAC::unmarshal(buf, cursor)?)), TYPE_ETHERNET => Ok(Endpoint::Ethernet(MAC::unmarshal(buf, cursor)?)),
TYPE_WIFIDIRECT => Ok(Endpoint::WifiDirect(MAC::unmarshal(buf, cursor)?)), TYPE_WIFIDIRECT => Ok(Endpoint::WifiDirect(MAC::unmarshal(buf, cursor)?)),
@ -210,13 +222,20 @@ impl Marshalable for Endpoint {
TYPE_ICMP => Ok(Endpoint::Icmp(InetAddress::unmarshal(buf, cursor)?)), TYPE_ICMP => Ok(Endpoint::Icmp(InetAddress::unmarshal(buf, cursor)?)),
TYPE_IPUDP => Ok(Endpoint::IpUdp(InetAddress::unmarshal(buf, cursor)?)), TYPE_IPUDP => Ok(Endpoint::IpUdp(InetAddress::unmarshal(buf, cursor)?)),
TYPE_IPTCP => Ok(Endpoint::IpTcp(InetAddress::unmarshal(buf, cursor)?)), TYPE_IPTCP => Ok(Endpoint::IpTcp(InetAddress::unmarshal(buf, cursor)?)),
TYPE_HTTP => Ok(Endpoint::Http(String::from_utf8_lossy(buf.read_bytes(buf.read_varint(cursor)? as usize, cursor)?).to_string())), TYPE_HTTP => Ok(Endpoint::Http(
TYPE_WEBRTC => Ok(Endpoint::WebRTC(buf.read_bytes(buf.read_varint(cursor)? as usize, cursor)?.to_vec())), String::from_utf8_lossy(buf.read_bytes(buf.read_varint(cursor)? as usize, cursor)?).to_string(),
)),
TYPE_WEBRTC => Ok(Endpoint::WebRTC(
buf.read_bytes(buf.read_varint(cursor)? as usize, cursor)?.to_vec(),
)),
TYPE_ZEROTIER_ENCAP => { TYPE_ZEROTIER_ENCAP => {
let zt = Address::unmarshal(buf, cursor)?; let zt = Address::unmarshal(buf, cursor)?;
Ok(Endpoint::ZeroTierEncap(zt, buf.read_bytes_fixed(cursor)?.clone())) Ok(Endpoint::ZeroTierEncap(zt, buf.read_bytes_fixed(cursor)?.clone()))
} }
_ => Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "unrecognized endpoint type in stream")), _ => Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"unrecognized endpoint type in stream",
)),
} }
} }
} }
@ -342,7 +361,10 @@ impl FromStr for Endpoint {
if endpoint_type == "zt" { if endpoint_type == "zt" {
return Ok(Endpoint::ZeroTier(Address::from_str(address)?, hash.as_slice().try_into().unwrap())); return Ok(Endpoint::ZeroTier(Address::from_str(address)?, hash.as_slice().try_into().unwrap()));
} else { } else {
return Ok(Endpoint::ZeroTierEncap(Address::from_str(address)?, hash.as_slice().try_into().unwrap())); return Ok(Endpoint::ZeroTierEncap(
Address::from_str(address)?,
hash.as_slice().try_into().unwrap(),
));
} }
} }
} }
@ -538,7 +560,11 @@ mod tests {
for _ in 0..1000 { for _ in 0..1000 {
let mac = crate::vl1::MAC::from_u64(rand::random()).unwrap(); let mac = crate::vl1::MAC::from_u64(rand::random()).unwrap();
for e in [Endpoint::Ethernet(mac.clone()), Endpoint::WifiDirect(mac.clone()), Endpoint::Bluetooth(mac.clone())] { for e in [
Endpoint::Ethernet(mac.clone()),
Endpoint::WifiDirect(mac.clone()),
Endpoint::Bluetooth(mac.clone()),
] {
let mut buf = Buffer::<7>::new(); let mut buf = Buffer::<7>::new();
let res = e.marshal(&mut buf); let res = e.marshal(&mut buf);
@ -563,7 +589,11 @@ mod tests {
let inet = crate::vl1::InetAddress::from_ip_port(&v, 1234); let inet = crate::vl1::InetAddress::from_ip_port(&v, 1234);
for e in [Endpoint::Icmp(inet.clone()), Endpoint::IpTcp(inet.clone()), Endpoint::IpUdp(inet.clone())] { for e in [
Endpoint::Icmp(inet.clone()),
Endpoint::IpTcp(inet.clone()),
Endpoint::IpUdp(inet.clone()),
] {
let mut buf = Buffer::<20>::new(); let mut buf = Buffer::<20>::new();
let res = e.marshal(&mut buf); let res = e.marshal(&mut buf);

106
network-hypervisor/src/vl1/identity.rs
View file

@ -75,7 +75,12 @@ fn concat_arrays_2<const A: usize, const B: usize, const S: usize>(a: &[u8; A],
} }
#[inline(always)] #[inline(always)]
fn concat_arrays_4<const A: usize, const B: usize, const C: usize, const D: usize, const S: usize>(a: &[u8; A], b: &[u8; B], c: &[u8; C], d: &[u8; D]) -> [u8; S] { fn concat_arrays_4<const A: usize, const B: usize, const C: usize, const D: usize, const S: usize>(
a: &[u8; A],
b: &[u8; B],
c: &[u8; C],
d: &[u8; D],
) -> [u8; S] {
assert_eq!(A + B + C + D, S); assert_eq!(A + B + C + D, S);
let mut tmp = [0_u8; S]; let mut tmp = [0_u8; S];
tmp[..A].copy_from_slice(a); tmp[..A].copy_from_slice(a);
@ -137,10 +142,19 @@ impl Identity {
pub const BYTE_LENGTH_X25519_SECRET: usize = Self::BYTE_LENGTH_X25519_PUBLIC + C25519_SECRET_KEY_SIZE + ED25519_SECRET_KEY_SIZE; pub const BYTE_LENGTH_X25519_SECRET: usize = Self::BYTE_LENGTH_X25519_PUBLIC + C25519_SECRET_KEY_SIZE + ED25519_SECRET_KEY_SIZE;
/// Length of a new dual-key public identity in byte array form. /// Length of a new dual-key public identity in byte array form.
pub const BYTE_LENGTH_X25519P384_PUBLIC: usize = Self::BYTE_LENGTH_X25519_PUBLIC + 1 + P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_ECDSA_SIGNATURE_SIZE + ED25519_SIGNATURE_SIZE; pub const BYTE_LENGTH_X25519P384_PUBLIC: usize = Self::BYTE_LENGTH_X25519_PUBLIC
+ 1
+ P384_PUBLIC_KEY_SIZE
+ P384_PUBLIC_KEY_SIZE
+ P384_ECDSA_SIGNATURE_SIZE
+ ED25519_SIGNATURE_SIZE;
/// Length of a new dual-key secret identity in byte array form. /// Length of a new dual-key secret identity in byte array form.
pub const BYTE_LENGTH_X25519P384_SECRET: usize = Self::BYTE_LENGTH_X25519P384_PUBLIC + C25519_SECRET_KEY_SIZE + ED25519_SECRET_KEY_SIZE + P384_SECRET_KEY_SIZE + P384_SECRET_KEY_SIZE; pub const BYTE_LENGTH_X25519P384_SECRET: usize = Self::BYTE_LENGTH_X25519P384_PUBLIC
+ C25519_SECRET_KEY_SIZE
+ ED25519_SECRET_KEY_SIZE
+ P384_SECRET_KEY_SIZE
+ P384_SECRET_KEY_SIZE;
const ALGORITHM_X25519: u8 = 0x01; const ALGORITHM_X25519: u8 = 0x01;
const ALGORITHM_EC_NIST_P384: u8 = 0x02; const ALGORITHM_EC_NIST_P384: u8 = 0x02;
@ -203,14 +217,23 @@ impl Identity {
/// It would be possible to change this in the future, with care. /// It would be possible to change this in the future, with care.
pub fn upgrade(&mut self) -> Result<bool, InvalidParameterError> { pub fn upgrade(&mut self) -> Result<bool, InvalidParameterError> {
if self.secret.is_none() { if self.secret.is_none() {
return Err(InvalidParameterError("an identity can only be upgraded if it includes its private key")); return Err(InvalidParameterError(
"an identity can only be upgraded if it includes its private key",
));
} }
if self.p384.is_none() { if self.p384.is_none() {
let p384_ecdh = P384KeyPair::generate(); let p384_ecdh = P384KeyPair::generate();
let p384_ecdsa = P384KeyPair::generate(); let p384_ecdsa = P384KeyPair::generate();
let mut self_sign_buf: Vec<u8> = let mut self_sign_buf: Vec<u8> = Vec::with_capacity(
Vec::with_capacity(ADDRESS_SIZE + C25519_PUBLIC_KEY_SIZE + ED25519_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_ECDSA_SIGNATURE_SIZE + 4); ADDRESS_SIZE
+ C25519_PUBLIC_KEY_SIZE
+ ED25519_PUBLIC_KEY_SIZE
+ P384_PUBLIC_KEY_SIZE
+ P384_PUBLIC_KEY_SIZE
+ P384_ECDSA_SIGNATURE_SIZE
+ 4,
);
let _ = self_sign_buf.write_all(&self.address.to_bytes()); let _ = self_sign_buf.write_all(&self.address.to_bytes());
let _ = self_sign_buf.write_all(&self.x25519); let _ = self_sign_buf.write_all(&self.x25519);
let _ = self_sign_buf.write_all(&self.ed25519); let _ = self_sign_buf.write_all(&self.ed25519);
@ -233,7 +256,12 @@ impl Identity {
ecdsa_self_signature, ecdsa_self_signature,
ed25519_self_signature, ed25519_self_signature,
}); });
let _ = self.secret.as_mut().unwrap().p384.insert(IdentityP384Secret { ecdh: p384_ecdh, ecdsa: p384_ecdsa }); let _ = self
.secret
.as_mut()
.unwrap()
.p384
.insert(IdentityP384Secret { ecdh: p384_ecdh, ecdsa: p384_ecdsa });
self.fingerprint = SHA384::hash(self.to_public_bytes().as_bytes()); self.fingerprint = SHA384::hash(self.to_public_bytes().as_bytes());
@ -259,7 +287,9 @@ impl Identity {
/// This is somewhat time consuming due to the memory-intensive work algorithm. /// This is somewhat time consuming due to the memory-intensive work algorithm.
pub fn validate_identity(&self) -> bool { pub fn validate_identity(&self) -> bool {
if let Some(p384) = self.p384.as_ref() { if let Some(p384) = self.p384.as_ref() {
let mut self_sign_buf: Vec<u8> = Vec::with_capacity(ADDRESS_SIZE + 4 + C25519_PUBLIC_KEY_SIZE + ED25519_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE); let mut self_sign_buf: Vec<u8> = Vec::with_capacity(
ADDRESS_SIZE + 4 + C25519_PUBLIC_KEY_SIZE + ED25519_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE,
);
let _ = self_sign_buf.write_all(&self.address.to_bytes()); let _ = self_sign_buf.write_all(&self.address.to_bytes());
let _ = self_sign_buf.write_all(&self.x25519); let _ = self_sign_buf.write_all(&self.x25519);
let _ = self_sign_buf.write_all(&self.ed25519); let _ = self_sign_buf.write_all(&self.ed25519);
@ -469,7 +499,11 @@ impl Identity {
} }
IdentityBytes::X25519Secret(b) => { IdentityBytes::X25519Secret(b) => {
let b: &packed::V0S = bytes_as_flat_object(b); let b: &packed::V0S = bytes_as_flat_object(b);
if b.key_type == 0 && b.secret_length == (C25519_SECRET_KEY_SIZE + ED25519_SECRET_KEY_SIZE) as u8 && b.reserved == 0x03 && u16::from_be_bytes(b.ext_len) == 0 { if b.key_type == 0
&& b.secret_length == (C25519_SECRET_KEY_SIZE + ED25519_SECRET_KEY_SIZE) as u8
&& b.reserved == 0x03
&& u16::from_be_bytes(b.ext_len) == 0
{
Some(Self { Some(Self {
address: Address::from_bytes_fixed(&b.address)?, address: Address::from_bytes_fixed(&b.address)?,
x25519: b.x25519, x25519: b.x25519,
@ -570,7 +604,12 @@ impl Identity {
}; };
IdentityBytes::try_from(&buf[..obj_len]).map_or_else( IdentityBytes::try_from(&buf[..obj_len]).map_or_else(
|_| Err(std::io::Error::new(std::io::ErrorKind::Other, "invalid identity")), |_| Err(std::io::Error::new(std::io::ErrorKind::Other, "invalid identity")),
|b| Identity::from_bytes(&b).map_or_else(|| Err(std::io::Error::new(std::io::ErrorKind::Other, "invalid identity")), |id| Ok(id)), |b| {
Identity::from_bytes(&b).map_or_else(
|| Err(std::io::Error::new(std::io::ErrorKind::Other, "invalid identity")),
|id| Ok(id),
)
},
) )
} }
@ -594,14 +633,21 @@ impl Identity {
} }
s.push_str(":2:"); // 2 == IDENTITY_ALGORITHM_EC_NIST_P384 s.push_str(":2:"); // 2 == IDENTITY_ALGORITHM_EC_NIST_P384
let p384_joined: [u8; P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_ECDSA_SIGNATURE_SIZE + ED25519_SIGNATURE_SIZE] = let p384_joined: [u8; P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_ECDSA_SIGNATURE_SIZE + ED25519_SIGNATURE_SIZE] =
concat_arrays_4(p384.ecdh.as_bytes(), p384.ecdsa.as_bytes(), &p384.ecdsa_self_signature, &p384.ed25519_self_signature); concat_arrays_4(
p384.ecdh.as_bytes(),
p384.ecdsa.as_bytes(),
&p384.ecdsa_self_signature,
&p384.ed25519_self_signature,
);
s.push_str(base64::encode_config(p384_joined, base64::URL_SAFE_NO_PAD).as_str()); s.push_str(base64::encode_config(p384_joined, base64::URL_SAFE_NO_PAD).as_str());
if self.secret.is_some() && include_private { if self.secret.is_some() && include_private {
let secret = self.secret.as_ref().unwrap(); let secret = self.secret.as_ref().unwrap();
if secret.p384.is_some() { if secret.p384.is_some() {
let p384_secret = secret.p384.as_ref().unwrap(); let p384_secret = secret.p384.as_ref().unwrap();
let p384_secret_joined: [u8; P384_SECRET_KEY_SIZE + P384_SECRET_KEY_SIZE] = let p384_secret_joined: [u8; P384_SECRET_KEY_SIZE + P384_SECRET_KEY_SIZE] = concat_arrays_2(
concat_arrays_2(p384_secret.ecdh.secret_key_bytes().as_bytes(), p384_secret.ecdsa.secret_key_bytes().as_bytes()); p384_secret.ecdh.secret_key_bytes().as_bytes(),
p384_secret.ecdsa.secret_key_bytes().as_bytes(),
);
s.push(':'); s.push(':');
s.push_str(base64::encode_config(p384_secret_joined, base64::URL_SAFE_NO_PAD).as_str()); s.push_str(base64::encode_config(p384_secret_joined, base64::URL_SAFE_NO_PAD).as_str());
} }
@ -680,7 +726,9 @@ impl FromStr for Identity {
if keys[0].len() != C25519_PUBLIC_KEY_SIZE + ED25519_PUBLIC_KEY_SIZE { if keys[0].len() != C25519_PUBLIC_KEY_SIZE + ED25519_PUBLIC_KEY_SIZE {
return Err(InvalidFormatError); return Err(InvalidFormatError);
} }
if !keys[2].is_empty() && keys[2].len() != P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_ECDSA_SIGNATURE_SIZE + ED25519_SIGNATURE_SIZE { if !keys[2].is_empty()
&& keys[2].len() != P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_ECDSA_SIGNATURE_SIZE + ED25519_SIGNATURE_SIZE
{
return Err(InvalidFormatError); return Err(InvalidFormatError);
} }
if !keys[3].is_empty() && keys[3].len() != P384_SECRET_KEY_SIZE + P384_SECRET_KEY_SIZE { if !keys[3].is_empty() && keys[3].len() != P384_SECRET_KEY_SIZE + P384_SECRET_KEY_SIZE {
@ -710,10 +758,13 @@ impl FromStr for Identity {
Some(IdentityP384Public { Some(IdentityP384Public {
ecdh: ecdh.unwrap(), ecdh: ecdh.unwrap(),
ecdsa: ecdsa.unwrap(), ecdsa: ecdsa.unwrap(),
ecdsa_self_signature: keys[2].as_slice()[(P384_PUBLIC_KEY_SIZE * 2)..((P384_PUBLIC_KEY_SIZE * 2) + P384_ECDSA_SIGNATURE_SIZE)] ecdsa_self_signature: keys[2].as_slice()
[(P384_PUBLIC_KEY_SIZE * 2)..((P384_PUBLIC_KEY_SIZE * 2) + P384_ECDSA_SIGNATURE_SIZE)]
.try_into()
.unwrap(),
ed25519_self_signature: keys[2].as_slice()[((P384_PUBLIC_KEY_SIZE * 2) + P384_ECDSA_SIGNATURE_SIZE)..]
.try_into() .try_into()
.unwrap(), .unwrap(),
ed25519_self_signature: keys[2].as_slice()[((P384_PUBLIC_KEY_SIZE * 2) + P384_ECDSA_SIGNATURE_SIZE)..].try_into().unwrap(),
}) })
}, },
secret: if keys[1].is_empty() { secret: if keys[1].is_empty() {
@ -742,14 +793,20 @@ impl FromStr for Identity {
} else { } else {
Some(IdentityP384Secret { Some(IdentityP384Secret {
ecdh: { ecdh: {
let tmp = P384KeyPair::from_bytes(&keys[2].as_slice()[..P384_PUBLIC_KEY_SIZE], &keys[3].as_slice()[..P384_SECRET_KEY_SIZE]); let tmp = P384KeyPair::from_bytes(
&keys[2].as_slice()[..P384_PUBLIC_KEY_SIZE],
&keys[3].as_slice()[..P384_SECRET_KEY_SIZE],
);
if tmp.is_none() { if tmp.is_none() {
return Err(InvalidFormatError); return Err(InvalidFormatError);
} }
tmp.unwrap() tmp.unwrap()
}, },
ecdsa: { ecdsa: {
let tmp = P384KeyPair::from_bytes(&keys[2].as_slice()[P384_PUBLIC_KEY_SIZE..(P384_PUBLIC_KEY_SIZE * 2)], &keys[3].as_slice()[P384_SECRET_KEY_SIZE..]); let tmp = P384KeyPair::from_bytes(
&keys[2].as_slice()[P384_PUBLIC_KEY_SIZE..(P384_PUBLIC_KEY_SIZE * 2)],
&keys[3].as_slice()[P384_SECRET_KEY_SIZE..],
);
if tmp.is_none() { if tmp.is_none() {
return Err(InvalidFormatError); return Err(InvalidFormatError);
} }
@ -781,7 +838,9 @@ impl Eq for Identity {}
impl Ord for Identity { impl Ord for Identity {
#[inline(always)] #[inline(always)]
fn cmp(&self, other: &Self) -> Ordering { fn cmp(&self, other: &Self) -> Ordering {
self.address.cmp(&other.address).then_with(|| self.fingerprint.cmp(&other.fingerprint)) self.address
.cmp(&other.address)
.then_with(|| self.fingerprint.cmp(&other.fingerprint))
} }
} }
@ -973,7 +1032,8 @@ mod tests {
#[test] #[test]
fn v0_identity() { fn v0_identity() {
let self_agree_expected = hex::from_string("de904fc90ff3a2b96b739b926e623113f5334c80841b654509b77916c4c4a6eb0ca69ec6ed01a7f04aee17c546b30ba4"); let self_agree_expected =
hex::from_string("de904fc90ff3a2b96b739b926e623113f5334c80841b654509b77916c4c4a6eb0ca69ec6ed01a7f04aee17c546b30ba4");
// Test self-agree with a known good x25519-only (v0) identity. // Test self-agree with a known good x25519-only (v0) identity.
let id = Identity::from_str( let id = Identity::from_str(
@ -1020,7 +1080,11 @@ mod tests {
let gen_unmarshaled = Identity::from_bytes(&bytes).unwrap(); let gen_unmarshaled = Identity::from_bytes(&bytes).unwrap();
assert!(gen_unmarshaled.secret.is_some()); assert!(gen_unmarshaled.secret.is_some());
if !gen_unmarshaled.eq(&gen) { if !gen_unmarshaled.eq(&gen) {
println!("{} != {}", hex::to_string(&gen_unmarshaled.fingerprint), hex::to_string(&gen.fingerprint)); println!(
"{} != {}",
hex::to_string(&gen_unmarshaled.fingerprint),
hex::to_string(&gen.fingerprint)
);
} }
assert!(Identity::from_str(string.as_str()).unwrap().secret.is_some()); assert!(Identity::from_str(string.as_str()).unwrap().secret.is_some());

42
network-hypervisor/src/vl1/inetaddress.rs
View file

@ -82,8 +82,10 @@ impl ToSocketAddrs for InetAddress {
#[inline(always)] #[inline(always)]
fn to_socket_addrs(&self) -> std::io::Result<Self::Iter> { fn to_socket_addrs(&self) -> std::io::Result<Self::Iter> {
self.try_into() self.try_into().map_or_else(
.map_or_else(|_| Err(std::io::Error::new(std::io::ErrorKind::Other, "not an IP address")), |sa| Ok(std::iter::once(sa))) |_| Err(std::io::Error::new(std::io::ErrorKind::Other, "not an IP address")),
|sa| Ok(std::iter::once(sa)),
)
} }
} }
@ -199,7 +201,10 @@ impl TryInto<SocketAddrV4> for &InetAddress {
fn try_into(self) -> Result<SocketAddrV4, Self::Error> { fn try_into(self) -> Result<SocketAddrV4, Self::Error> {
unsafe { unsafe {
match self.sa.sa_family { match self.sa.sa_family {
AF_INET => Ok(SocketAddrV4::new(Ipv4Addr::from(self.sin.sin_addr.s_addr.to_ne_bytes()), u16::from_be(self.sin.sin_port as u16))), AF_INET => Ok(SocketAddrV4::new(
Ipv4Addr::from(self.sin.sin_addr.s_addr.to_ne_bytes()),
u16::from_be(self.sin.sin_port as u16),
)),
_ => Err(crate::error::InvalidParameterError("not an IPv4 address")), _ => Err(crate::error::InvalidParameterError("not an IPv4 address")),
} }
} }
@ -222,7 +227,12 @@ impl TryInto<SocketAddrV6> for &InetAddress {
fn try_into(self) -> Result<SocketAddrV6, Self::Error> { fn try_into(self) -> Result<SocketAddrV6, Self::Error> {
unsafe { unsafe {
match self.sa.sa_family { match self.sa.sa_family {
AF_INET6 => Ok(SocketAddrV6::new(Ipv6Addr::from(self.sin6.sin6_addr.s6_addr), u16::from_be(self.sin6.sin6_port as u16), 0, 0)), AF_INET6 => Ok(SocketAddrV6::new(
Ipv6Addr::from(self.sin6.sin6_addr.s6_addr),
u16::from_be(self.sin6.sin6_port as u16),
0,
0,
)),
_ => Err(crate::error::InvalidParameterError("not an IPv6 address")), _ => Err(crate::error::InvalidParameterError("not an IPv6 address")),
} }
} }
@ -597,7 +607,9 @@ impl InetAddress {
AF_INET => { AF_INET => {
if cidr_bits <= 32 { if cidr_bits <= 32 {
let discard_bits = 32 - cidr_bits; let discard_bits = 32 - cidr_bits;
if u32::from_be(self.sin.sin_addr.s_addr as u32).wrapping_shr(discard_bits) == u32::from_be(cidr.sin.sin_addr.s_addr as u32).wrapping_shr(discard_bits) { if u32::from_be(self.sin.sin_addr.s_addr as u32).wrapping_shr(discard_bits)
== u32::from_be(cidr.sin.sin_addr.s_addr as u32).wrapping_shr(discard_bits)
{
return true; return true;
} }
} }
@ -784,7 +796,11 @@ impl Marshalable for InetAddress {
AF_INET6 => { AF_INET6 => {
let b = buf.append_bytes_fixed_get_mut::<19>()?; let b = buf.append_bytes_fixed_get_mut::<19>()?;
b[0] = 6; b[0] = 6;
copy_nonoverlapping((&(self.sin6.sin6_addr) as *const in6_addr).cast::<u8>(), b.as_mut_ptr().offset(1), 16); copy_nonoverlapping(
(&(self.sin6.sin6_addr) as *const in6_addr).cast::<u8>(),
b.as_mut_ptr().offset(1),
16,
);
b[17] = *(&self.sin6.sin6_port as *const u16).cast::<u8>(); b[17] = *(&self.sin6.sin6_port as *const u16).cast::<u8>();
b[18] = *(&self.sin6.sin6_port as *const u16).cast::<u8>().offset(1); b[18] = *(&self.sin6.sin6_port as *const u16).cast::<u8>().offset(1);
Ok(()) Ok(())
@ -801,7 +817,10 @@ impl Marshalable for InetAddress {
Ok(InetAddress::from_ip_port(&b[0..4], u16::from_be_bytes(b[4..6].try_into().unwrap()))) Ok(InetAddress::from_ip_port(&b[0..4], u16::from_be_bytes(b[4..6].try_into().unwrap())))
} else if t == 6 { } else if t == 6 {
let b: &[u8; 18] = buf.read_bytes_fixed(cursor)?; let b: &[u8; 18] = buf.read_bytes_fixed(cursor)?;
Ok(InetAddress::from_ip_port(&b[0..16], u16::from_be_bytes(b[16..18].try_into().unwrap()))) Ok(InetAddress::from_ip_port(
&b[0..16],
u16::from_be_bytes(b[16..18].try_into().unwrap()),
))
} else { } else {
Ok(InetAddress::new()) Ok(InetAddress::new())
} }
@ -880,7 +899,8 @@ impl PartialEq for InetAddress {
AF_INET => self.sin.sin_port == other.sin.sin_port && self.sin.sin_addr.s_addr == other.sin.sin_addr.s_addr, AF_INET => self.sin.sin_port == other.sin.sin_port && self.sin.sin_addr.s_addr == other.sin.sin_addr.s_addr,
AF_INET6 => { AF_INET6 => {
if self.sin6.sin6_port == other.sin6.sin6_port { if self.sin6.sin6_port == other.sin6.sin6_port {
(*(&(self.sin6.sin6_addr) as *const in6_addr).cast::<[u8; 16]>()).eq(&*(&(other.sin6.sin6_addr) as *const in6_addr).cast::<[u8; 16]>()) (*(&(self.sin6.sin6_addr) as *const in6_addr).cast::<[u8; 16]>())
.eq(&*(&(other.sin6.sin6_addr) as *const in6_addr).cast::<[u8; 16]>())
} else { } else {
false false
} }
@ -912,7 +932,8 @@ impl Ord for InetAddress {
match self.sa.sa_family as AddressFamilyType { match self.sa.sa_family as AddressFamilyType {
0 => Ordering::Equal, 0 => Ordering::Equal,
AF_INET => { AF_INET => {
let ip_ordering = u32::from_be(self.sin.sin_addr.s_addr as u32).cmp(&u32::from_be(other.sin.sin_addr.s_addr as u32)); let ip_ordering =
u32::from_be(self.sin.sin_addr.s_addr as u32).cmp(&u32::from_be(other.sin.sin_addr.s_addr as u32));
if ip_ordering == Ordering::Equal { if ip_ordering == Ordering::Equal {
u16::from_be(self.sin.sin_port as u16).cmp(&u16::from_be(other.sin.sin_port as u16)) u16::from_be(self.sin.sin_port as u16).cmp(&u16::from_be(other.sin.sin_port as u16))
} else { } else {
@ -931,7 +952,8 @@ impl Ord for InetAddress {
} }
_ => { _ => {
// This shouldn't be possible, but handle it for correctness. // This shouldn't be possible, but handle it for correctness.
(*slice_from_raw_parts((self as *const Self).cast::<u8>(), size_of::<Self>())).cmp(&*slice_from_raw_parts((other as *const Self).cast::<u8>(), size_of::<Self>())) (*slice_from_raw_parts((self as *const Self).cast::<u8>(), size_of::<Self>()))
.cmp(&*slice_from_raw_parts((other as *const Self).cast::<u8>(), size_of::<Self>()))
} }
} }
} else { } else {

34
network-hypervisor/src/vl1/mac.rs
View file

@ -33,7 +33,15 @@ impl MAC {
#[inline(always)] #[inline(always)]
pub fn from_bytes(b: &[u8]) -> Option<MAC> { pub fn from_bytes(b: &[u8]) -> Option<MAC> {
if b.len() >= 6 { if b.len() >= 6 {
NonZeroU64::new((b[0] as u64) << 40 | (b[1] as u64) << 32 | (b[2] as u64) << 24 | (b[3] as u64) << 16 as u64 | (b[4] as u64) << 8 | b[5] as u64).map(|i| MAC(i)) NonZeroU64::new(
(b[0] as u64) << 40
| (b[1] as u64) << 32
| (b[2] as u64) << 24
| (b[3] as u64) << 16 as u64
| (b[4] as u64) << 8
| b[5] as u64,
)
.map(|i| MAC(i))
} else { } else {
None None
} }
@ -41,13 +49,23 @@ impl MAC {
#[inline(always)] #[inline(always)]
pub fn from_bytes_fixed(b: &[u8; 6]) -> Option<MAC> { pub fn from_bytes_fixed(b: &[u8; 6]) -> Option<MAC> {
NonZeroU64::new((b[0] as u64) << 40 | (b[1] as u64) << 32 | (b[2] as u64) << 24 | (b[3] as u64) << 16 as u64 | (b[4] as u64) << 8 | b[5] as u64).map(|i| MAC(i)) NonZeroU64::new(
(b[0] as u64) << 40 | (b[1] as u64) << 32 | (b[2] as u64) << 24 | (b[3] as u64) << 16 as u64 | (b[4] as u64) << 8 | b[5] as u64,
)
.map(|i| MAC(i))
} }
#[inline(always)] #[inline(always)]
pub fn to_bytes(&self) -> [u8; 6] { pub fn to_bytes(&self) -> [u8; 6] {
let i = self.0.get(); let i = self.0.get();
[(i >> 40) as u8, (i >> 32) as u8, (i >> 24) as u8, (i >> 16) as u8, (i >> 8) as u8, i as u8] [
(i >> 40) as u8,
(i >> 32) as u8,
(i >> 24) as u8,
(i >> 16) as u8,
(i >> 8) as u8,
i as u8,
]
} }
} }
@ -75,14 +93,20 @@ impl Marshalable for MAC {
#[inline(always)] #[inline(always)]
fn unmarshal<const BL: usize>(buf: &Buffer<BL>, cursor: &mut usize) -> std::io::Result<Self> { fn unmarshal<const BL: usize>(buf: &Buffer<BL>, cursor: &mut usize) -> std::io::Result<Self> {
Self::from_bytes_fixed(buf.read_bytes_fixed(cursor)?).map_or_else(|| Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "cannot be zero")), |a| Ok(a)) Self::from_bytes_fixed(buf.read_bytes_fixed(cursor)?).map_or_else(
|| Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "cannot be zero")),
|a| Ok(a),
)
} }
} }
impl ToString for MAC { impl ToString for MAC {
fn to_string(&self) -> String { fn to_string(&self) -> String {
let b: [u8; 6] = self.to_bytes(); let b: [u8; 6] = self.to_bytes();
format!("{:0>2x}:{:0>2x}:{:0>2x}:{:0>2x}:{:0>2x}:{:0>2x}", b[0], b[1], b[2], b[3], b[4], b[5]) format!(
"{:0>2x}:{:0>2x}:{:0>2x}:{:0>2x}:{:0>2x}:{:0>2x}",
b[0], b[1], b[2], b[3], b[4], b[5]
)
} }
} }

108
network-hypervisor/src/vl1/node.rs
View file

@ -70,10 +70,23 @@ pub trait SystemInterface: Sync + Send + 'static {
/// For endpoint types that support a packet TTL, the implementation may set the TTL /// For endpoint types that support a packet TTL, the implementation may set the TTL
/// if the 'ttl' parameter is not zero. If the parameter is zero or TTL setting is not /// if the 'ttl' parameter is not zero. If the parameter is zero or TTL setting is not
/// supported, the default TTL should be used. /// supported, the default TTL should be used.
async fn wire_send(&self, endpoint: &Endpoint, local_socket: Option<&Self::LocalSocket>, local_interface: Option<&Self::LocalInterface>, data: &[&[u8]], packet_ttl: u8) -> bool; async fn wire_send(
&self,
endpoint: &Endpoint,
local_socket: Option<&Self::LocalSocket>,
local_interface: Option<&Self::LocalInterface>,
data: &[&[u8]],
packet_ttl: u8,
) -> bool;
/// Called to check and see if a physical address should be used for ZeroTier traffic to a node. /// Called to check and see if a physical address should be used for ZeroTier traffic to a node.
async fn check_path(&self, id: &Identity, endpoint: &Endpoint, local_socket: Option<&Self::LocalSocket>, local_interface: Option<&Self::LocalInterface>) -> bool; async fn check_path(
&self,
id: &Identity,
endpoint: &Endpoint,
local_socket: Option<&Self::LocalSocket>,
local_interface: Option<&Self::LocalInterface>,
) -> bool;
/// Called to look up any statically defined or memorized paths to known nodes. /// Called to look up any statically defined or memorized paths to known nodes.
async fn get_path_hints(&self, id: &Identity) -> Option<Vec<(Endpoint, Option<Self::LocalSocket>, Option<Self::LocalInterface>)>>; async fn get_path_hints(&self, id: &Identity) -> Option<Vec<(Endpoint, Option<Self::LocalSocket>, Option<Self::LocalInterface>)>>;
@ -96,7 +109,8 @@ pub trait InnerProtocolInterface: Sync + Send + 'static {
/// Handle a packet, returning true if it was handled by the next layer. /// Handle a packet, returning true if it was handled by the next layer.
/// ///
/// Do not attempt to handle OK or ERROR. Instead implement handle_ok() and handle_error(). /// Do not attempt to handle OK or ERROR. Instead implement handle_ok() and handle_error().
async fn handle_packet<SI: SystemInterface>(&self, source: &Peer<SI>, source_path: &Path<SI>, verb: u8, payload: &PacketBuffer) -> bool; async fn handle_packet<SI: SystemInterface>(&self, source: &Peer<SI>, source_path: &Path<SI>, verb: u8, payload: &PacketBuffer)
-> bool;
/// Handle errors, returning true if the error was recognized. /// Handle errors, returning true if the error was recognized.
async fn handle_error<SI: SystemInterface>( async fn handle_error<SI: SystemInterface>(
@ -111,7 +125,15 @@ pub trait InnerProtocolInterface: Sync + Send + 'static {
) -> bool; ) -> bool;
/// Handle an OK, returing true if the OK was recognized. /// Handle an OK, returing true if the OK was recognized.
async fn handle_ok<SI: SystemInterface>(&self, source: &Peer<SI>, source_path: &Path<SI>, in_re_verb: u8, in_re_message_id: u64, payload: &PacketBuffer, cursor: &mut usize) -> bool; async fn handle_ok<SI: SystemInterface>(
&self,
source: &Peer<SI>,
source_path: &Path<SI>,
in_re_verb: u8,
in_re_message_id: u64,
payload: &PacketBuffer,
cursor: &mut usize,
) -> bool;
/// Check if this peer should communicate with another at all. /// Check if this peer should communicate with another at all.
fn should_communicate_with(&self, id: &Identity) -> bool; fn should_communicate_with(&self, id: &Identity) -> bool;
@ -298,7 +320,12 @@ impl<SI: SystemInterface> Node<SI> {
let mut best_root = self.best_root.write(); let mut best_root = self.best_root.write();
if let Some(best_root) = best_root.as_mut() { if let Some(best_root) = best_root.as_mut() {
if !Arc::ptr_eq(best_root, best) { if !Arc::ptr_eq(best_root, best) {
debug_event!(si, "[vl1] new best root: {} (replaced {})", best.identity.address.to_string(), best_root.identity.address.to_string()); debug_event!(
si,
"[vl1] new best root: {} (replaced {})",
best.identity.address.to_string(),
best_root.identity.address.to_string()
);
*best_root = best.clone(); *best_root = best.clone();
} }
} else { } else {
@ -409,8 +436,14 @@ impl<SI: SystemInterface> Node<SI> {
for m in rs.members.iter() { for m in rs.members.iter() {
if m.identity.eq(&self.identity) { if m.identity.eq(&self.identity) {
let _ = my_root_sets.get_or_insert_with(|| Vec::new()).write_all(rs.to_bytes().as_slice()); let _ = my_root_sets.get_or_insert_with(|| Vec::new()).write_all(rs.to_bytes().as_slice());
} else if self.peers.read().get(&m.identity.address).map_or(false, |p| !p.identity.eq(&m.identity)) } else if self
|| address_collision_check.insert(m.identity.address, &m.identity).map_or(false, |old_id| !old_id.eq(&m.identity)) .peers
.read()
.get(&m.identity.address)
.map_or(false, |p| !p.identity.eq(&m.identity))
|| address_collision_check
.insert(m.identity.address, &m.identity)
.map_or(false, |old_id| !old_id.eq(&m.identity))
{ {
address_collisions.push(m.identity.address); address_collisions.push(m.identity.address);
} }
@ -420,7 +453,8 @@ impl<SI: SystemInterface> Node<SI> {
for (_, rs) in roots.sets.iter() { for (_, rs) in roots.sets.iter() {
for m in rs.members.iter() { for m in rs.members.iter() {
if m.endpoints.is_some() && !address_collisions.contains(&m.identity.address) && !m.identity.eq(&self.identity) { if m.endpoints.is_some() && !address_collisions.contains(&m.identity.address) && !m.identity.eq(&self.identity)
{
debug_event!( debug_event!(
si, si,
"[vl1] examining root {} with {} endpoints", "[vl1] examining root {} with {} endpoints",
@ -493,7 +527,12 @@ impl<SI: SystemInterface> Node<SI> {
}; };
for (root, endpoints) in roots.iter() { for (root, endpoints) in roots.iter() {
for ep in endpoints.iter() { for ep in endpoints.iter() {
debug_event!(si, "sending HELLO to root {} (root interval: {})", root.identity.address.to_string(), ROOT_HELLO_INTERVAL); debug_event!(
si,
"sending HELLO to root {} (root interval: {})",
root.identity.address.to_string(),
ROOT_HELLO_INTERVAL
);
root.send_hello(si, self, Some(ep)).await; root.send_hello(si, self, Some(ep)).await;
} }
} }
@ -538,7 +577,8 @@ impl<SI: SystemInterface> Node<SI> {
let ka = [tt as u8]; // send different bytes every time for keepalive in case some things filter zero packets let ka = [tt as u8]; // send different bytes every time for keepalive in case some things filter zero packets
let ka2 = [&ka[..1]]; let ka2 = [&ka[..1]];
for ka in need_keepalive.iter() { for ka in need_keepalive.iter() {
si.wire_send(&ka.endpoint, Some(&ka.local_socket), Some(&ka.local_interface), &ka2, 0).await; si.wire_send(&ka.endpoint, Some(&ka.local_socket), Some(&ka.local_interface), &ka2, 0)
.await;
} }
} }
@ -563,7 +603,8 @@ impl<SI: SystemInterface> Node<SI> {
si, si,
"[vl1] {} -> #{} {}->{} length {} (on socket {}@{})", "[vl1] {} -> #{} {}->{} length {} (on socket {}@{})",
source_endpoint.to_string(), source_endpoint.to_string(),
data.bytes_fixed_at::<8>(0).map_or("????????????????".into(), |pid| hex::to_string(pid)), data.bytes_fixed_at::<8>(0)
.map_or("????????????????".into(), |pid| hex::to_string(pid)),
data.bytes_fixed_at::<5>(13).map_or("??????????".into(), |src| hex::to_string(src)), data.bytes_fixed_at::<5>(13).map_or("??????????".into(), |src| hex::to_string(src)),
data.bytes_fixed_at::<5>(8).map_or("??????????".into(), |dest| hex::to_string(dest)), data.bytes_fixed_at::<5>(8).map_or("??????????".into(), |dest| hex::to_string(dest)),
data.len(), data.len(),
@ -589,7 +630,13 @@ impl<SI: SystemInterface> Node<SI> {
fragment_header.total_fragments() fragment_header.total_fragments()
); );
if let Some(assembled_packet) = path.receive_fragment(fragment_header.packet_id(), fragment_header.fragment_no(), fragment_header.total_fragments(), data, time_ticks) { if let Some(assembled_packet) = path.receive_fragment(
fragment_header.packet_id(),
fragment_header.fragment_no(),
fragment_header.total_fragments(),
data,
time_ticks,
) {
if let Some(frag0) = assembled_packet.frags[0].as_ref() { if let Some(frag0) = assembled_packet.frags[0].as_ref() {
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
debug_event!(si, "[vl1] #{:0>16x} packet fully assembled!", fragment_header_id); debug_event!(si, "[vl1] #{:0>16x} packet fully assembled!", fragment_header_id);
@ -597,8 +644,17 @@ impl<SI: SystemInterface> Node<SI> {
if let Ok(packet_header) = frag0.struct_at::<v1::PacketHeader>(0) { if let Ok(packet_header) = frag0.struct_at::<v1::PacketHeader>(0) {
if let Some(source) = Address::from_bytes(&packet_header.src) { if let Some(source) = Address::from_bytes(&packet_header.src) {
if let Some(peer) = self.peer(source) { if let Some(peer) = self.peer(source) {
peer.receive(self, si, ph, time_ticks, &path, packet_header, frag0, &assembled_packet.frags[1..(assembled_packet.have as usize)]) peer.receive(
.await; self,
si,
ph,
time_ticks,
&path,
packet_header,
frag0,
&assembled_packet.frags[1..(assembled_packet.have as usize)],
)
.await;
} else { } else {
self.whois.query(self, si, source, Some(QueuedPacket::Fragmented(assembled_packet))); self.whois.query(self, si, source, Some(QueuedPacket::Fragmented(assembled_packet)));
} }
@ -613,7 +669,8 @@ impl<SI: SystemInterface> Node<SI> {
if let Some(source) = Address::from_bytes(&packet_header.src) { if let Some(source) = Address::from_bytes(&packet_header.src) {
if let Some(peer) = self.peer(source) { if let Some(peer) = self.peer(source) {
peer.receive(self, si, ph, time_ticks, &path, packet_header, data.as_ref(), &[]).await; peer.receive(self, si, ph, time_ticks, &path, packet_header, data.as_ref(), &[])
.await;
} else { } else {
self.whois.query(self, si, source, Some(QueuedPacket::Unfragmented(data))); self.whois.query(self, si, source, Some(QueuedPacket::Unfragmented(data)));
} }
@ -628,7 +685,12 @@ impl<SI: SystemInterface> Node<SI> {
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
{ {
debug_packet_id = u64::from_be_bytes(fragment_header.id); debug_packet_id = u64::from_be_bytes(fragment_header.id);
debug_event!(si, "[vl1] #{:0>16x} forwarding packet fragment to {}", debug_packet_id, dest.to_string()); debug_event!(
si,
"[vl1] #{:0>16x} forwarding packet fragment to {}",
debug_packet_id,
dest.to_string()
);
} }
if fragment_header.increment_hops() > v1::FORWARD_MAX_HOPS { if fragment_header.increment_hops() > v1::FORWARD_MAX_HOPS {
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
@ -644,7 +706,11 @@ impl<SI: SystemInterface> Node<SI> {
} }
if packet_header.increment_hops() > v1::FORWARD_MAX_HOPS { if packet_header.increment_hops() > v1::FORWARD_MAX_HOPS {
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
debug_event!(si, "[vl1] #{:0>16x} discarded: max hops exceeded!", u64::from_be_bytes(packet_header.id)); debug_event!(
si,
"[vl1] #{:0>16x} discarded: max hops exceeded!",
u64::from_be_bytes(packet_header.id)
);
return; return;
} }
} else { } else {
@ -714,7 +780,13 @@ impl<SI: SystemInterface> Node<SI> {
self.roots.read().my_root_sets.is_some() self.roots.read().my_root_sets.is_some()
} }
pub(crate) fn canonical_path(&self, ep: &Endpoint, local_socket: &SI::LocalSocket, local_interface: &SI::LocalInterface, time_ticks: i64) -> Arc<Path<SI>> { pub(crate) fn canonical_path(
&self,
ep: &Endpoint,
local_socket: &SI::LocalSocket,
local_interface: &SI::LocalInterface,
time_ticks: i64,
) -> Arc<Path<SI>> {
if let Some(path) = self.paths.read().get(&PathKey::Ref(ep, local_socket)) { if let Some(path) = self.paths.read().get(&PathKey::Ref(ep, local_socket)) {
return path.clone(); return path.clone();
} }

13
network-hypervisor/src/vl1/path.rs
View file

@ -50,7 +50,14 @@ impl<SI: SystemInterface> Path<SI> {
/// Receive a fragment and return a FragmentedPacket if the entire packet was assembled. /// Receive a fragment and return a FragmentedPacket if the entire packet was assembled.
/// This returns None if more fragments are needed to assemble the packet. /// This returns None if more fragments are needed to assemble the packet.
pub(crate) fn receive_fragment(&self, packet_id: PacketId, fragment_no: u8, fragment_expecting_count: u8, packet: PooledPacketBuffer, time_ticks: i64) -> Option<FragmentedPacket> { pub(crate) fn receive_fragment(
&self,
packet_id: PacketId,
fragment_no: u8,
fragment_expecting_count: u8,
packet: PooledPacketBuffer,
time_ticks: i64,
) -> Option<FragmentedPacket> {
let mut fp = self.fragmented_packets.lock(); let mut fp = self.fragmented_packets.lock();
// Discard some old waiting packets if the total incoming fragments for a path exceeds a // Discard some old waiting packets if the total incoming fragments for a path exceeds a
@ -90,7 +97,9 @@ impl<SI: SystemInterface> Path<SI> {
} }
pub(crate) fn service(&self, time_ticks: i64) -> PathServiceResult { pub(crate) fn service(&self, time_ticks: i64) -> PathServiceResult {
self.fragmented_packets.lock().retain(|_, frag| (time_ticks - frag.ts_ticks) < v1::FRAGMENT_EXPIRATION); self.fragmented_packets
.lock()
.retain(|_, frag| (time_ticks - frag.ts_ticks) < v1::FRAGMENT_EXPIRATION);
if (time_ticks - self.last_receive_time_ticks.load(Ordering::Relaxed)) < PATH_EXPIRATION_TIME { if (time_ticks - self.last_receive_time_ticks.load(Ordering::Relaxed)) < PATH_EXPIRATION_TIME {
if (time_ticks - self.last_send_time_ticks.load(Ordering::Relaxed)) >= PATH_KEEPALIVE_INTERVAL { if (time_ticks - self.last_send_time_ticks.load(Ordering::Relaxed)) >= PATH_KEEPALIVE_INTERVAL {
self.last_send_time_ticks.store(time_ticks, Ordering::Relaxed); self.last_send_time_ticks.store(time_ticks, Ordering::Relaxed);

231
network-hypervisor/src/vl1/peer.rs
View file

@ -210,7 +210,12 @@ impl<SI: SystemInterface> Peer<SI> {
pub fn version(&self) -> Option<[u16; 4]> { pub fn version(&self) -> Option<[u16; 4]> {
let rv = self.remote_node_info.read().remote_version; let rv = self.remote_node_info.read().remote_version;
if rv != 0 { if rv != 0 {
Some([rv.wrapping_shr(48) as u16, rv.wrapping_shr(32) as u16, rv.wrapping_shr(16) as u16, rv as u16]) Some([
rv.wrapping_shr(48) as u16,
rv.wrapping_shr(32) as u16,
rv.wrapping_shr(16) as u16,
rv as u16,
])
} else { } else {
None None
} }
@ -299,7 +304,12 @@ impl<SI: SystemInterface> Peer<SI> {
} }
// Learn new path if it's not a duplicate or should not replace an existing path. // Learn new path if it's not a duplicate or should not replace an existing path.
debug_event!(si, "[vl1] {} learned new path: {}", self.identity.address.to_string(), new_path.endpoint.to_string()); debug_event!(
si,
"[vl1] {} learned new path: {}",
self.identity.address.to_string(),
new_path.endpoint.to_string()
);
paths.push(PeerPath::<SI> { paths.push(PeerPath::<SI> {
path: Arc::downgrade(new_path), path: Arc::downgrade(new_path),
last_receive_time_ticks: time_ticks, last_receive_time_ticks: time_ticks,
@ -314,7 +324,10 @@ impl<SI: SystemInterface> Peer<SI> {
paths.retain(|p| ((time_ticks - p.last_receive_time_ticks) < PEER_EXPIRATION_TIME) && (p.path.strong_count() > 0)); paths.retain(|p| ((time_ticks - p.last_receive_time_ticks) < PEER_EXPIRATION_TIME) && (p.path.strong_count() > 0));
prioritize_paths(&mut paths); prioritize_paths(&mut paths);
} }
self.remote_node_info.write().reported_local_endpoints.retain(|_, ts| (time_ticks - *ts) < PEER_EXPIRATION_TIME); self.remote_node_info
.write()
.reported_local_endpoints
.retain(|_, ts| (time_ticks - *ts) < PEER_EXPIRATION_TIME);
(time_ticks - self.last_receive_time_ticks.load(Ordering::Relaxed).max(self.create_time_ticks)) < PEER_EXPIRATION_TIME (time_ticks - self.last_receive_time_ticks.load(Ordering::Relaxed).max(self.create_time_ticks)) < PEER_EXPIRATION_TIME
} }
@ -334,13 +347,17 @@ impl<SI: SystemInterface> Peer<SI> {
let packet_size = packet.len(); let packet_size = packet.len();
if packet_size > max_fragment_size { if packet_size > max_fragment_size {
let bytes = packet.as_bytes(); let bytes = packet.as_bytes();
if !si.wire_send(endpoint, local_socket, local_interface, &[&bytes[0..UDP_DEFAULT_MTU]], 0).await { if !si
.wire_send(endpoint, local_socket, local_interface, &[&bytes[0..UDP_DEFAULT_MTU]], 0)
.await
{
return false; return false;
} }
let mut pos = UDP_DEFAULT_MTU; let mut pos = UDP_DEFAULT_MTU;
let overrun_size = (packet_size - UDP_DEFAULT_MTU) as u32; let overrun_size = (packet_size - UDP_DEFAULT_MTU) as u32;
let fragment_count = (overrun_size / (UDP_DEFAULT_MTU - v1::FRAGMENT_HEADER_SIZE) as u32) + (((overrun_size % (UDP_DEFAULT_MTU - v1::FRAGMENT_HEADER_SIZE) as u32) != 0) as u32); let fragment_count = (overrun_size / (UDP_DEFAULT_MTU - v1::FRAGMENT_HEADER_SIZE) as u32)
+ (((overrun_size % (UDP_DEFAULT_MTU - v1::FRAGMENT_HEADER_SIZE) as u32) != 0) as u32);
debug_assert!(fragment_count <= v1::FRAGMENT_COUNT_MAX as u32); debug_assert!(fragment_count <= v1::FRAGMENT_COUNT_MAX as u32);
let mut header = v1::FragmentHeader { let mut header = v1::FragmentHeader {
@ -355,7 +372,16 @@ impl<SI: SystemInterface> Peer<SI> {
loop { loop {
header.total_and_fragment_no += 1; header.total_and_fragment_no += 1;
let next_pos = pos + chunk_size; let next_pos = pos + chunk_size;
if !si.wire_send(endpoint, local_socket, local_interface, &[header.as_bytes(), &bytes[pos..next_pos]], 0).await { if !si
.wire_send(
endpoint,
local_socket,
local_interface,
&[header.as_bytes(), &bytes[pos..next_pos]],
0,
)
.await
{
return false; return false;
} }
pos = next_pos; pos = next_pos;
@ -376,7 +402,14 @@ impl<SI: SystemInterface> Peer<SI> {
/// via a root or some other route. /// via a root or some other route.
/// ///
/// It encrypts and sets the MAC and cipher fields and packet ID and other things. /// It encrypts and sets the MAC and cipher fields and packet ID and other things.
pub(crate) async fn send(&self, si: &SI, path: Option<&Arc<Path<SI>>>, node: &Node<SI>, time_ticks: i64, packet: &mut PacketBuffer) -> bool { pub(crate) async fn send(
&self,
si: &SI,
path: Option<&Arc<Path<SI>>>,
node: &Node<SI>,
time_ticks: i64,
packet: &mut PacketBuffer,
) -> bool {
let mut _path_arc = None; let mut _path_arc = None;
let path = if let Some(path) = path { let path = if let Some(path) = path {
path path
@ -402,7 +435,11 @@ impl<SI: SystemInterface> Peer<SI> {
let mut aes_gmac_siv = self.identity_symmetric_key.aes_gmac_siv.get(); let mut aes_gmac_siv = self.identity_symmetric_key.aes_gmac_siv.get();
aes_gmac_siv.encrypt_init(&self.next_message_id().to_ne_bytes()); aes_gmac_siv.encrypt_init(&self.next_message_id().to_ne_bytes());
aes_gmac_siv.encrypt_set_aad(&v1::get_packet_aad_bytes(self.identity.address, node.identity.address, flags_cipher_hops)); aes_gmac_siv.encrypt_set_aad(&v1::get_packet_aad_bytes(
self.identity.address,
node.identity.address,
flags_cipher_hops,
));
if let Ok(payload) = packet.as_bytes_starting_at_mut(v1::HEADER_SIZE) { if let Ok(payload) = packet.as_bytes_starting_at_mut(v1::HEADER_SIZE) {
aes_gmac_siv.encrypt_first_pass(payload); aes_gmac_siv.encrypt_first_pass(payload);
aes_gmac_siv.encrypt_first_pass_finish(); aes_gmac_siv.encrypt_first_pass_finish();
@ -419,7 +456,14 @@ impl<SI: SystemInterface> Peer<SI> {
} }
if self if self
.internal_send(si, &path.endpoint, Some(&path.local_socket), Some(&path.local_interface), max_fragment_size, packet) .internal_send(
si,
&path.endpoint,
Some(&path.local_socket),
Some(&path.local_interface),
max_fragment_size,
packet,
)
.await .await
{ {
self.last_send_time_ticks.store(time_ticks, Ordering::Relaxed); self.last_send_time_ticks.store(time_ticks, Ordering::Relaxed);
@ -438,7 +482,16 @@ impl<SI: SystemInterface> Peer<SI> {
/// Intermediates don't need to adjust fragmentation. /// Intermediates don't need to adjust fragmentation.
pub(crate) async fn forward(&self, si: &SI, time_ticks: i64, packet: &PacketBuffer) -> bool { pub(crate) async fn forward(&self, si: &SI, time_ticks: i64, packet: &PacketBuffer) -> bool {
if let Some(path) = self.direct_path() { if let Some(path) = self.direct_path() {
if si.wire_send(&path.endpoint, Some(&path.local_socket), Some(&path.local_interface), &[packet.as_bytes()], 0).await { if si
.wire_send(
&path.endpoint,
Some(&path.local_socket),
Some(&path.local_interface),
&[packet.as_bytes()],
0,
)
.await
{
self.last_forward_time_ticks.store(time_ticks, Ordering::Relaxed); self.last_forward_time_ticks.store(time_ticks, Ordering::Relaxed);
return true; return true;
} }
@ -479,7 +532,8 @@ impl<SI: SystemInterface> Peer<SI> {
let message_id = self.next_message_id(); let message_id = self.next_message_id();
{ {
let f: &mut (v1::PacketHeader, v1::message_component_structs::HelloFixedHeaderFields) = packet.append_struct_get_mut().unwrap(); let f: &mut (v1::PacketHeader, v1::message_component_structs::HelloFixedHeaderFields) =
packet.append_struct_get_mut().unwrap();
f.0.id = message_id.to_ne_bytes(); f.0.id = message_id.to_ne_bytes();
f.0.dest = self.identity.address.to_bytes(); f.0.dest = self.identity.address.to_bytes();
f.0.src = node.identity.address.to_bytes(); f.0.src = node.identity.address.to_bytes();
@ -495,17 +549,37 @@ impl<SI: SystemInterface> Peer<SI> {
debug_assert_eq!(packet.len(), 41); debug_assert_eq!(packet.len(), 41);
assert!(packet.append_bytes((&node.identity.to_public_bytes()).into()).is_ok()); assert!(packet.append_bytes((&node.identity.to_public_bytes()).into()).is_ok());
let (_, poly1305_key) = salsa_poly_create(&self.identity_symmetric_key, packet.struct_at::<v1::PacketHeader>(0).unwrap(), packet.len()); let (_, poly1305_key) = salsa_poly_create(
&self.identity_symmetric_key,
packet.struct_at::<v1::PacketHeader>(0).unwrap(),
packet.len(),
);
let mac = poly1305::compute(&poly1305_key, packet.as_bytes_starting_at(v1::HEADER_SIZE).unwrap()); let mac = poly1305::compute(&poly1305_key, packet.as_bytes_starting_at(v1::HEADER_SIZE).unwrap());
packet.as_mut()[v1::MAC_FIELD_INDEX..v1::MAC_FIELD_INDEX + 8].copy_from_slice(&mac[0..8]); packet.as_mut()[v1::MAC_FIELD_INDEX..v1::MAC_FIELD_INDEX + 8].copy_from_slice(&mac[0..8]);
self.last_send_time_ticks.store(time_ticks, Ordering::Relaxed); self.last_send_time_ticks.store(time_ticks, Ordering::Relaxed);
debug_event!(si, "HELLO -> {} @ {} ({} bytes)", self.identity.address.to_string(), destination.to_string(), packet.len()); debug_event!(
si,
"HELLO -> {} @ {} ({} bytes)",
self.identity.address.to_string(),
destination.to_string(),
packet.len()
);
} }
if let Some(p) = path.as_ref() { if let Some(p) = path.as_ref() {
if self.internal_send(si, destination, Some(&p.local_socket), Some(&p.local_interface), max_fragment_size, &packet).await { if self
.internal_send(
si,
destination,
Some(&p.local_socket),
Some(&p.local_interface),
max_fragment_size,
&packet,
)
.await
{
p.log_send_anything(time_ticks); p.log_send_anything(time_ticks);
true true
} else { } else {
@ -536,7 +610,13 @@ impl<SI: SystemInterface> Peer<SI> {
if let Ok(packet_frag0_payload_bytes) = frag0.as_bytes_starting_at(v1::VERB_INDEX) { if let Ok(packet_frag0_payload_bytes) = frag0.as_bytes_starting_at(v1::VERB_INDEX) {
let mut payload = PacketBuffer::new(); let mut payload = PacketBuffer::new();
let message_id = if let Some(message_id2) = try_aead_decrypt(&self.identity_symmetric_key, packet_frag0_payload_bytes, packet_header, fragments, &mut payload) { let message_id = if let Some(message_id2) = try_aead_decrypt(
&self.identity_symmetric_key,
packet_frag0_payload_bytes,
packet_header,
fragments,
&mut payload,
) {
// Decryption successful with static secret. // Decryption successful with static secret.
message_id2 message_id2
} else { } else {
@ -582,13 +662,25 @@ impl<SI: SystemInterface> Peer<SI> {
if match verb { if match verb {
verbs::VL1_NOP => true, verbs::VL1_NOP => true,
verbs::VL1_HELLO => self.handle_incoming_hello(si, ph, node, time_ticks, message_id, source_path, packet_header.hops(), &payload).await, verbs::VL1_HELLO => {
self.handle_incoming_hello(si, ph, node, time_ticks, message_id, source_path, packet_header.hops(), &payload)
.await
}
verbs::VL1_ERROR => self.handle_incoming_error(si, ph, node, time_ticks, source_path, &payload).await, verbs::VL1_ERROR => self.handle_incoming_error(si, ph, node, time_ticks, source_path, &payload).await,
verbs::VL1_OK => self.handle_incoming_ok(si, ph, node, time_ticks, source_path, packet_header.hops(), path_is_known, &payload).await, verbs::VL1_OK => {
self.handle_incoming_ok(si, ph, node, time_ticks, source_path, packet_header.hops(), path_is_known, &payload)
.await
}
verbs::VL1_WHOIS => self.handle_incoming_whois(si, ph, node, time_ticks, message_id, &payload).await, verbs::VL1_WHOIS => self.handle_incoming_whois(si, ph, node, time_ticks, message_id, &payload).await,
verbs::VL1_RENDEZVOUS => self.handle_incoming_rendezvous(si, node, time_ticks, message_id, source_path, &payload).await, verbs::VL1_RENDEZVOUS => {
self.handle_incoming_rendezvous(si, node, time_ticks, message_id, source_path, &payload)
.await
}
verbs::VL1_ECHO => self.handle_incoming_echo(si, ph, node, time_ticks, message_id, &payload).await, verbs::VL1_ECHO => self.handle_incoming_echo(si, ph, node, time_ticks, message_id, &payload).await,
verbs::VL1_PUSH_DIRECT_PATHS => self.handle_incoming_push_direct_paths(si, node, time_ticks, source_path, &payload).await, verbs::VL1_PUSH_DIRECT_PATHS => {
self.handle_incoming_push_direct_paths(si, node, time_ticks, source_path, &payload)
.await
}
verbs::VL1_USER_MESSAGE => self.handle_incoming_user_message(si, node, time_ticks, source_path, &payload).await, verbs::VL1_USER_MESSAGE => self.handle_incoming_user_message(si, node, time_ticks, source_path, &payload).await,
_ => ph.handle_packet(self, &source_path, verb, &payload).await, _ => ph.handle_packet(self, &source_path, verb, &payload).await,
} { } {
@ -614,7 +706,11 @@ impl<SI: SystemInterface> Peer<SI> {
payload: &PacketBuffer, payload: &PacketBuffer,
) -> bool { ) -> bool {
if !(ph.should_communicate_with(&self.identity) || node.this_node_is_root() || node.is_peer_root(self)) { if !(ph.should_communicate_with(&self.identity) || node.this_node_is_root() || node.is_peer_root(self)) {
debug_event!(si, "[vl1] dropping HELLO from {} due to lack of trust relationship", self.identity.address.to_string()); debug_event!(
si,
"[vl1] dropping HELLO from {} due to lack of trust relationship",
self.identity.address.to_string()
);
return true; // packet wasn't invalid, just ignored return true; // packet wasn't invalid, just ignored
} }
@ -633,7 +729,10 @@ impl<SI: SystemInterface> Peer<SI> {
let mut packet = PacketBuffer::new(); let mut packet = PacketBuffer::new();
packet.set_size(v1::HEADER_SIZE); packet.set_size(v1::HEADER_SIZE);
{ {
let f: &mut (v1::message_component_structs::OkHeader, v1::message_component_structs::OkHelloFixedHeaderFields) = packet.append_struct_get_mut().unwrap(); let f: &mut (
v1::message_component_structs::OkHeader,
v1::message_component_structs::OkHelloFixedHeaderFields,
) = packet.append_struct_get_mut().unwrap();
f.0.verb = verbs::VL1_OK; f.0.verb = verbs::VL1_OK;
f.0.in_re_verb = verbs::VL1_HELLO; f.0.in_re_verb = verbs::VL1_HELLO;
f.0.in_re_message_id = message_id.to_ne_bytes(); f.0.in_re_message_id = message_id.to_ne_bytes();
@ -651,7 +750,15 @@ impl<SI: SystemInterface> Peer<SI> {
return false; return false;
} }
async fn handle_incoming_error<PH: InnerProtocolInterface>(&self, _si: &SI, ph: &PH, _node: &Node<SI>, _time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) -> bool { async fn handle_incoming_error<PH: InnerProtocolInterface>(
&self,
_si: &SI,
ph: &PH,
_node: &Node<SI>,
_time_ticks: i64,
source_path: &Arc<Path<SI>>,
payload: &PacketBuffer,
) -> bool {
let mut cursor = 0; let mut cursor = 0;
if let Ok(error_header) = payload.read_struct::<v1::message_component_structs::ErrorHeader>(&mut cursor) { if let Ok(error_header) = payload.read_struct::<v1::message_component_structs::ErrorHeader>(&mut cursor) {
let in_re_message_id: MessageId = u64::from_ne_bytes(error_header.in_re_message_id); let in_re_message_id: MessageId = u64::from_ne_bytes(error_header.in_re_message_id);
@ -659,7 +766,15 @@ impl<SI: SystemInterface> Peer<SI> {
match error_header.in_re_verb { match error_header.in_re_verb {
_ => { _ => {
return ph return ph
.handle_error(self, &source_path, error_header.in_re_verb, in_re_message_id, error_header.error_code, payload, &mut cursor) .handle_error(
self,
&source_path,
error_header.in_re_verb,
in_re_message_id,
error_header.error_code,
payload,
&mut cursor,
)
.await; .await;
} }
} }
@ -685,12 +800,20 @@ impl<SI: SystemInterface> Peer<SI> {
if self.message_id_counter.load(Ordering::Relaxed).wrapping_sub(in_re_message_id) <= PACKET_RESPONSE_COUNTER_DELTA_MAX { if self.message_id_counter.load(Ordering::Relaxed).wrapping_sub(in_re_message_id) <= PACKET_RESPONSE_COUNTER_DELTA_MAX {
match ok_header.in_re_verb { match ok_header.in_re_verb {
verbs::VL1_HELLO => { verbs::VL1_HELLO => {
if let Ok(ok_hello_fixed_header_fields) = payload.read_struct::<v1::message_component_structs::OkHelloFixedHeaderFields>(&mut cursor) { if let Ok(ok_hello_fixed_header_fields) =
payload.read_struct::<v1::message_component_structs::OkHelloFixedHeaderFields>(&mut cursor)
{
if hops == 0 { if hops == 0 {
if let Ok(reported_endpoint) = Endpoint::unmarshal(&payload, &mut cursor) { if let Ok(reported_endpoint) = Endpoint::unmarshal(&payload, &mut cursor) {
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
let reported_endpoint2 = reported_endpoint.clone(); let reported_endpoint2 = reported_endpoint.clone();
if self.remote_node_info.write().reported_local_endpoints.insert(reported_endpoint, time_ticks).is_none() { if self
.remote_node_info
.write()
.reported_local_endpoints
.insert(reported_endpoint, time_ticks)
.is_none()
{
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
debug_event!( debug_event!(
si, si,
@ -725,7 +848,9 @@ impl<SI: SystemInterface> Peer<SI> {
} }
_ => { _ => {
return ph.handle_ok(self, &source_path, ok_header.in_re_verb, in_re_message_id, payload, &mut cursor).await; return ph
.handle_ok(self, &source_path, ok_header.in_re_verb, in_re_message_id, payload, &mut cursor)
.await;
} }
} }
} }
@ -733,7 +858,15 @@ impl<SI: SystemInterface> Peer<SI> {
return false; return false;
} }
async fn handle_incoming_whois<PH: InnerProtocolInterface>(&self, si: &SI, ph: &PH, node: &Node<SI>, time_ticks: i64, message_id: MessageId, payload: &PacketBuffer) -> bool { async fn handle_incoming_whois<PH: InnerProtocolInterface>(
&self,
si: &SI,
ph: &PH,
node: &Node<SI>,
time_ticks: i64,
message_id: MessageId,
payload: &PacketBuffer,
) -> bool {
if node.this_node_is_root() || ph.should_communicate_with(&self.identity) { if node.this_node_is_root() || ph.should_communicate_with(&self.identity) {
let mut packet = PacketBuffer::new(); let mut packet = PacketBuffer::new();
packet.set_size(v1::HEADER_SIZE); packet.set_size(v1::HEADER_SIZE);
@ -763,12 +896,28 @@ impl<SI: SystemInterface> Peer<SI> {
} }
#[allow(unused)] #[allow(unused)]
async fn handle_incoming_rendezvous(&self, si: &SI, node: &Node<SI>, time_ticks: i64, message_id: MessageId, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) -> bool { async fn handle_incoming_rendezvous(
&self,
si: &SI,
node: &Node<SI>,
time_ticks: i64,
message_id: MessageId,
source_path: &Arc<Path<SI>>,
payload: &PacketBuffer,
) -> bool {
if node.is_peer_root(self) {} if node.is_peer_root(self) {}
return true; return true;
} }
async fn handle_incoming_echo<PH: InnerProtocolInterface>(&self, si: &SI, ph: &PH, node: &Node<SI>, time_ticks: i64, message_id: MessageId, payload: &PacketBuffer) -> bool { async fn handle_incoming_echo<PH: InnerProtocolInterface>(
&self,
si: &SI,
ph: &PH,
node: &Node<SI>,
time_ticks: i64,
message_id: MessageId,
payload: &PacketBuffer,
) -> bool {
if ph.should_communicate_with(&self.identity) || node.is_peer_root(self) { if ph.should_communicate_with(&self.identity) || node.is_peer_root(self) {
let mut packet = PacketBuffer::new(); let mut packet = PacketBuffer::new();
packet.set_size(v1::HEADER_SIZE); packet.set_size(v1::HEADER_SIZE);
@ -784,18 +933,36 @@ impl<SI: SystemInterface> Peer<SI> {
false false
} }
} else { } else {
debug_event!(si, "[vl1] dropping ECHO from {} due to lack of trust relationship", self.identity.address.to_string()); debug_event!(
si,
"[vl1] dropping ECHO from {} due to lack of trust relationship",
self.identity.address.to_string()
);
true // packet wasn't invalid, just ignored true // packet wasn't invalid, just ignored
} }
} }
#[allow(unused)] #[allow(unused)]
async fn handle_incoming_push_direct_paths(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) -> bool { async fn handle_incoming_push_direct_paths(
&self,
si: &SI,
node: &Node<SI>,
time_ticks: i64,
source_path: &Arc<Path<SI>>,
payload: &PacketBuffer,
) -> bool {
false false
} }
#[allow(unused)] #[allow(unused)]
async fn handle_incoming_user_message(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) -> bool { async fn handle_incoming_user_message(
&self,
si: &SI,
node: &Node<SI>,
time_ticks: i64,
source_path: &Arc<Path<SI>>,
payload: &PacketBuffer,
) -> bool {
false false
} }
} }

22
network-hypervisor/src/vl1/rootset.rs
View file

@ -90,7 +90,11 @@ impl RootSet {
/// Get the ZeroTier default root set, which contains roots run by ZeroTier Inc. /// Get the ZeroTier default root set, which contains roots run by ZeroTier Inc.
pub fn zerotier_default() -> Self { pub fn zerotier_default() -> Self {
let mut cursor = 0; let mut cursor = 0;
let rs = Self::unmarshal(&Buffer::from(include_bytes!("../../default-rootset/root.zerotier.com.bin")), &mut cursor).unwrap(); let rs = Self::unmarshal(
&Buffer::from(include_bytes!("../../default-rootset/root.zerotier.com.bin")),
&mut cursor,
)
.unwrap();
assert!(rs.verify()); assert!(rs.verify());
rs rs
} }
@ -156,7 +160,13 @@ impl RootSet {
} }
/// Add a member to this definition, replacing any current entry with this address. /// Add a member to this definition, replacing any current entry with this address.
pub fn add<'a, I: Iterator<Item = &'a Endpoint>>(&mut self, member_identity: &Identity, endpoints: Option<I>, priority: u8, protocol_version: u8) { pub fn add<'a, I: Iterator<Item = &'a Endpoint>>(
&mut self,
member_identity: &Identity,
endpoints: Option<I>,
priority: u8,
protocol_version: u8,
) {
self.members.retain(|m| m.identity.address != member_identity.address); self.members.retain(|m| m.identity.address != member_identity.address);
let _ = self.members.push(Root { let _ = self.members.push(Root {
identity: member_identity.clone_without_secret(), identity: member_identity.clone_without_secret(),
@ -263,11 +273,15 @@ impl Marshalable for RootSet {
} }
let name_len = buf.read_varint(cursor)?; let name_len = buf.read_varint(cursor)?;
rc.name = String::from_utf8(buf.read_bytes(name_len as usize, cursor)?.to_vec()).map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid UTF8"))?; rc.name = String::from_utf8(buf.read_bytes(name_len as usize, cursor)?.to_vec())
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid UTF8"))?;
let url_len = buf.read_varint(cursor)?; let url_len = buf.read_varint(cursor)?;
if url_len > 0 { if url_len > 0 {
rc.url = Some(String::from_utf8(buf.read_bytes(url_len as usize, cursor)?.to_vec()).map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid UTF8"))?); rc.url = Some(
String::from_utf8(buf.read_bytes(url_len as usize, cursor)?.to_vec())
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid UTF8"))?,
);
} }
rc.revision = buf.read_varint(cursor)?; rc.revision = buf.read_varint(cursor)?;

5
network-hypervisor/src/vl2/networkid.rs
View file

@ -67,7 +67,10 @@ impl Marshalable for NetworkId {
#[inline(always)] #[inline(always)]
fn unmarshal<const BL: usize>(buf: &Buffer<BL>, cursor: &mut usize) -> std::io::Result<Self> { fn unmarshal<const BL: usize>(buf: &Buffer<BL>, cursor: &mut usize) -> std::io::Result<Self> {
Self::from_u64(buf.read_u64(cursor)?).map_or_else(|| Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "cannot be zero")), |a| Ok(a)) Self::from_u64(buf.read_u64(cursor)?).map_or_else(
|| Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "cannot be zero")),
|a| Ok(a),
)
} }
} }

21
network-hypervisor/src/vl2/switch.rs
View file

@ -18,12 +18,29 @@ impl InnerProtocolInterface for Switch {
} }
#[allow(unused)] #[allow(unused)]
async fn handle_error<SI: SystemInterface>(&self, peer: &Peer<SI>, source_path: &Path<SI>, in_re_verb: u8, in_re_message_id: u64, error_code: u8, payload: &PacketBuffer, cursor: &mut usize) -> bool { async fn handle_error<SI: SystemInterface>(
&self,
peer: &Peer<SI>,
source_path: &Path<SI>,
in_re_verb: u8,
in_re_message_id: u64,
error_code: u8,
payload: &PacketBuffer,
cursor: &mut usize,
) -> bool {
false false
} }
#[allow(unused)] #[allow(unused)]
async fn handle_ok<SI: SystemInterface>(&self, peer: &Peer<SI>, source_path: &Path<SI>, in_re_verb: u8, in_re_message_id: u64, payload: &PacketBuffer, cursor: &mut usize) -> bool { async fn handle_ok<SI: SystemInterface>(
&self,
peer: &Peer<SI>,
source_path: &Path<SI>,
in_re_verb: u8,
in_re_message_id: u64,
payload: &PacketBuffer,
cursor: &mut usize,
) -> bool {
false false
} }

4
rustfmt.toml
View file

@ -1,8 +1,8 @@
#unstable_features = true #unstable_features = true
max_width = 200 max_width = 140
#use_small_heuristics = "Max" #use_small_heuristics = "Max"
edition = "2021" edition = "2021"
empty_item_single_line = true #empty_item_single_line = true
newline_style = "Unix" newline_style = "Unix"
struct_lit_width = 60 struct_lit_width = 60
tab_spaces = 4 tab_spaces = 4

9
system-service/src/datadir.rs
View file

@ -32,7 +32,10 @@ impl DataDir {
if !base_path.is_dir() { if !base_path.is_dir() {
let _ = std::fs::create_dir_all(&base_path); let _ = std::fs::create_dir_all(&base_path);
if !base_path.is_dir() { if !base_path.is_dir() {
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "base path not found and cannot be created")); return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"base path not found and cannot be created",
));
} }
} }
@ -57,7 +60,9 @@ impl DataDir {
/// Load identity.secret from data directory. /// Load identity.secret from data directory.
pub async fn load_identity(&self) -> std::io::Result<Identity> { pub async fn load_identity(&self) -> std::io::Result<Identity> {
let id_data = Identity::from_str(String::from_utf8_lossy(read_limit(self.base_path.join(IDENTITY_SECRET_FILENAME), 4096).await?.as_slice()).as_ref()); let id_data = Identity::from_str(
String::from_utf8_lossy(read_limit(self.base_path.join(IDENTITY_SECRET_FILENAME), 4096).await?.as_slice()).as_ref(),
);
if id_data.is_err() { if id_data.is_err() {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, id_data.err().unwrap())); return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, id_data.err().unwrap()));
} }

16
system-service/src/getifaddrs.rs
View file

@ -27,9 +27,17 @@ pub fn for_each_address<F: FnMut(&InetAddress, &LocalInterface)>(mut f: F) {
let sa_family = (*(*i).ifa_addr).sa_family as u8; let sa_family = (*(*i).ifa_addr).sa_family as u8;
if sa_family == libc::AF_INET as u8 { if sa_family == libc::AF_INET as u8 {
copy_nonoverlapping((*i).ifa_addr.cast::<u8>(), (&mut a as *mut InetAddress).cast::<u8>(), size_of::<libc::sockaddr_in>()); copy_nonoverlapping(
(*i).ifa_addr.cast::<u8>(),
(&mut a as *mut InetAddress).cast::<u8>(),
size_of::<libc::sockaddr_in>(),
);
} else if sa_family == libc::AF_INET6 as u8 { } else if sa_family == libc::AF_INET6 as u8 {
copy_nonoverlapping((*i).ifa_addr.cast::<u8>(), (&mut a as *mut InetAddress).cast::<u8>(), size_of::<libc::sockaddr_in6>()); copy_nonoverlapping(
(*i).ifa_addr.cast::<u8>(),
(&mut a as *mut InetAddress).cast::<u8>(),
size_of::<libc::sockaddr_in6>(),
);
} else { } else {
i = (*i).ifa_next; i = (*i).ifa_next;
continue; continue;
@ -87,7 +95,9 @@ mod tests {
#[test] #[test]
fn test_getifaddrs() { fn test_getifaddrs() {
println!("starting getifaddrs..."); println!("starting getifaddrs...");
crate::getifaddrs::for_each_address(|a: &InetAddress, interface: &LocalInterface| println!(" {} {}", interface.to_string(), a.to_string())); crate::getifaddrs::for_each_address(|a: &InetAddress, interface: &LocalInterface| {
println!(" {} {}", interface.to_string(), a.to_string())
});
println!("done.") println!("done.")
} }
} }

6
system-service/src/ipv6.rs
View file

@ -56,7 +56,11 @@ mod freebsd_like {
let device_name_bytes = device_name.as_bytes(); let device_name_bytes = device_name.as_bytes();
assert!(device_name_bytes.len() <= 15); assert!(device_name_bytes.len() <= 15);
ifr6.ifr_name[..device_name_bytes.len()].copy_from_slice(device_name_bytes); ifr6.ifr_name[..device_name_bytes.len()].copy_from_slice(device_name_bytes);
std::ptr::copy_nonoverlapping((address as *const InetAddress).cast(), &mut ifr6.ifr_ifru.ifru_addr, size_of::<libc::sockaddr_in6>()); std::ptr::copy_nonoverlapping(
(address as *const InetAddress).cast(),
&mut ifr6.ifr_ifru.ifru_addr,
size_of::<libc::sockaddr_in6>(),
);
if libc::ioctl((*info_socket).as_(), SIOCGIFAFLAG_IN6, &mut ifr6 as *mut in6_ifreq) != -1 { if libc::ioctl((*info_socket).as_(), SIOCGIFAFLAG_IN6, &mut ifr6 as *mut in6_ifreq) != -1 {
if (ifr6.ifr_ifru.ifru_flags6 & IN6_IFF_TEMPORARY) != 0 { if (ifr6.ifr_ifru.ifru_flags6 & IN6_IFF_TEMPORARY) != 0 {
return true; return true;

17
system-service/src/localconfig.rs
View file

@ -9,12 +9,17 @@ use zerotier_network_hypervisor::vl2::NetworkId;
/// A list of unassigned or obsolete ports under 1024 that could possibly be squatted. /// A list of unassigned or obsolete ports under 1024 that could possibly be squatted.
pub const UNASSIGNED_PRIVILEGED_PORTS: [u16; 299] = [ pub const UNASSIGNED_PRIVILEGED_PORTS: [u16; 299] = [
4, 6, 8, 10, 12, 14, 15, 16, 26, 28, 30, 32, 34, 36, 40, 60, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 285, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 323, 324, 325, 326, 327, 328, 4, 6, 8, 10, 12, 14, 15, 16, 26, 28, 30, 32, 34, 36, 40, 60, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 285, 288, 289, 290,
329, 330, 331, 332, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 703, 708, 713, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728, 732, 733, 734, 735, 736, 737, 738, 739, 740, 743, 745, 746, 755, 756, 766, 768, 778, 779, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332,
781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, 802, 803, 804, 805, 806, 807, 808, 809, 811, 812, 813, 814, 815, 816, 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 834, 835, 836, 837, 838, 839, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 703, 708, 713, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727,
840, 841, 842, 843, 844, 845, 846, 849, 850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, 884, 885, 889, 890, 891, 892, 893, 894, 895, 896, 897, 728, 732, 733, 734, 735, 736, 737, 738, 739, 740, 743, 745, 746, 755, 756, 766, 768, 778, 779, 781, 782, 783, 784, 785, 786, 787, 788,
898, 899, 904, 905, 906, 907, 908, 909, 910, 911, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 952, 953, 789, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, 802, 803, 804, 805, 806, 807, 808, 809, 811, 812, 813, 814, 815, 816, 817, 818,
954, 955, 956, 957, 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1023, 819, 820, 821, 822, 823, 824, 825, 826, 827, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 849, 850, 851, 852, 853,
854, 855, 856, 857, 858, 859, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883,
884, 885, 889, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 904, 905, 906, 907, 908, 909, 910, 911, 914, 915, 916, 917, 918, 919,
920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, 946,
947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973,
974, 975, 976, 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1023,
]; ];
/// Default primary ZeroTier port. /// Default primary ZeroTier port.

63
system-service/src/main.rs
View file

@ -99,17 +99,36 @@ fn main() {
.subcommand(Command::new("secondaryport").arg(Arg::new("port#").index(1).validator(utils::is_valid_port))) .subcommand(Command::new("secondaryport").arg(Arg::new("port#").index(1).validator(utils::is_valid_port)))
.subcommand( .subcommand(
Command::new("blacklist") Command::new("blacklist")
.subcommand(Command::new("cidr").arg(Arg::new("ip_bits").index(1)).arg(Arg::new("boolean").index(2).validator(utils::is_valid_bool))) .subcommand(
.subcommand(Command::new("if").arg(Arg::new("prefix").index(1)).arg(Arg::new("boolean").index(2).validator(utils::is_valid_bool))), Command::new("cidr")
.arg(Arg::new("ip_bits").index(1))
.arg(Arg::new("boolean").index(2).validator(utils::is_valid_bool)),
)
.subcommand(
Command::new("if")
.arg(Arg::new("prefix").index(1))
.arg(Arg::new("boolean").index(2).validator(utils::is_valid_bool)),
),
) )
.subcommand(Command::new("portmap").arg(Arg::new("boolean").index(1).validator(utils::is_valid_bool))), .subcommand(Command::new("portmap").arg(Arg::new("boolean").index(1).validator(utils::is_valid_bool))),
) )
.subcommand(Command::new("peer").subcommand(Command::new("show").arg(Arg::new("address").index(1).required(true))).subcommand(Command::new("list")).subcommand(Command::new("listroots")).subcommand(Command::new("try"))) .subcommand(
Command::new("peer")
.subcommand(Command::new("show").arg(Arg::new("address").index(1).required(true)))
.subcommand(Command::new("list"))
.subcommand(Command::new("listroots"))
.subcommand(Command::new("try")),
)
.subcommand( .subcommand(
Command::new("network") Command::new("network")
.subcommand(Command::new("show").arg(Arg::new("nwid").index(1).required(true))) .subcommand(Command::new("show").arg(Arg::new("nwid").index(1).required(true)))
.subcommand(Command::new("list")) .subcommand(Command::new("list"))
.subcommand(Command::new("set").arg(Arg::new("nwid").index(1).required(true)).arg(Arg::new("setting").index(2).required(false)).arg(Arg::new("value").index(3).required(false))), .subcommand(
Command::new("set")
.arg(Arg::new("nwid").index(1).required(true))
.arg(Arg::new("setting").index(2).required(false))
.arg(Arg::new("value").index(3).required(false)),
),
) )
.subcommand(Command::new("join").arg(Arg::new("nwid").index(1).required(true))) .subcommand(Command::new("join").arg(Arg::new("nwid").index(1).required(true)))
.subcommand(Command::new("leave").arg(Arg::new("nwid").index(1).required(true))) .subcommand(Command::new("leave").arg(Arg::new("nwid").index(1).required(true)))
@ -120,15 +139,28 @@ fn main() {
.subcommand(Command::new("getpublic").arg(Arg::new("identity").index(1).required(true))) .subcommand(Command::new("getpublic").arg(Arg::new("identity").index(1).required(true)))
.subcommand(Command::new("fingerprint").arg(Arg::new("identity").index(1).required(true))) .subcommand(Command::new("fingerprint").arg(Arg::new("identity").index(1).required(true)))
.subcommand(Command::new("validate").arg(Arg::new("identity").index(1).required(true))) .subcommand(Command::new("validate").arg(Arg::new("identity").index(1).required(true)))
.subcommand(Command::new("sign").arg(Arg::new("identity").index(1).required(true)).arg(Arg::new("path").index(2).required(true))) .subcommand(
.subcommand(Command::new("verify").arg(Arg::new("identity").index(1).required(true)).arg(Arg::new("path").index(2).required(true)).arg(Arg::new("signature").index(3).required(true))), Command::new("sign")
.arg(Arg::new("identity").index(1).required(true))
.arg(Arg::new("path").index(2).required(true)),
)
.subcommand(
Command::new("verify")
.arg(Arg::new("identity").index(1).required(true))
.arg(Arg::new("path").index(2).required(true))
.arg(Arg::new("signature").index(3).required(true)),
),
) )
.subcommand( .subcommand(
Command::new("rootset") Command::new("rootset")
.subcommand(Command::new("add").arg(Arg::new("path").index(1).required(true))) .subcommand(Command::new("add").arg(Arg::new("path").index(1).required(true)))
.subcommand(Command::new("remove").arg(Arg::new("name").index(1).required(true))) .subcommand(Command::new("remove").arg(Arg::new("name").index(1).required(true)))
.subcommand(Command::new("list")) .subcommand(Command::new("list"))
.subcommand(Command::new("sign").arg(Arg::new("path").index(1).required(true)).arg(Arg::new("secret").index(2).required(true))) .subcommand(
Command::new("sign")
.arg(Arg::new("path").index(1).required(true))
.arg(Arg::new("secret").index(2).required(true)),
)
.subcommand(Command::new("verify").arg(Arg::new("path").index(1).required(true))) .subcommand(Command::new("verify").arg(Arg::new("path").index(1).required(true)))
.subcommand(Command::new("marshal").arg(Arg::new("path").index(1).required(true))) .subcommand(Command::new("marshal").arg(Arg::new("path").index(1).required(true)))
.subcommand(Command::new("restoredefault")), .subcommand(Command::new("restoredefault")),
@ -163,7 +195,10 @@ fn main() {
if suggested.is_empty() { if suggested.is_empty() {
eprintln!("Unrecognized option '{}'. Use 'help' for help.", invalid); eprintln!("Unrecognized option '{}'. Use 'help' for help.", invalid);
} else { } else {
eprintln!("Unrecognized option '{}', did you mean {}? Use 'help' for help.", invalid, suggested); eprintln!(
"Unrecognized option '{}', did you mean {}? Use 'help' for help.",
invalid, suggested
);
} }
} }
std::process::exit(exitcode::ERR_USAGE); std::process::exit(exitcode::ERR_USAGE);
@ -173,10 +208,18 @@ fn main() {
let flags = Flags { let flags = Flags {
json_output: global_args.is_present("json"), json_output: global_args.is_present("json"),
base_path: global_args.value_of("path").map_or_else(platform_default_home_path, |p| p.to_string()), base_path: global_args
.value_of("path")
.map_or_else(platform_default_home_path, |p| p.to_string()),
auth_token_path_override: global_args.value_of("token_path").map(|p| p.to_string()), auth_token_path_override: global_args.value_of("token_path").map(|p| p.to_string()),
auth_token_override: global_args.value_of("token").map(|t| t.to_string()), auth_token_override: global_args.value_of("token").map(|t| t.to_string()),
}; };
std::process::exit(tokio::runtime::Builder::new_multi_thread().enable_all().build().unwrap().block_on(async_main(flags, global_args))); std::process::exit(
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap()
.block_on(async_main(flags, global_args)),
);
} }

39
system-service/src/service.rs
View file

@ -63,7 +63,11 @@ impl Service {
/// ///
/// This launches a number of background tasks in the async runtime that will run as long as this object exists. /// This launches a number of background tasks in the async runtime that will run as long as this object exists.
/// When this is dropped these tasks are killed. /// When this is dropped these tasks are killed.
pub async fn new<P: AsRef<Path>>(rt: tokio::runtime::Handle, base_path: P, auto_upgrade_identity: bool) -> Result<Self, Box<dyn Error>> { pub async fn new<P: AsRef<Path>>(
rt: tokio::runtime::Handle,
base_path: P,
auto_upgrade_identity: bool,
) -> Result<Self, Box<dyn Error>> {
let mut si = ServiceImpl { let mut si = ServiceImpl {
rt, rt,
data: DataDir::open(base_path).await.map_err(|e| Box::new(e))?, data: DataDir::open(base_path).await.map_err(|e| Box::new(e))?,
@ -126,8 +130,14 @@ impl ServiceImpl {
let mut buf = core.get_packet_buffer(); let mut buf = core.get_packet_buffer();
if let Ok((bytes, source)) = socket.recv_from(unsafe { buf.entire_buffer_mut() }).await { if let Ok((bytes, source)) = socket.recv_from(unsafe { buf.entire_buffer_mut() }).await {
unsafe { buf.set_size_unchecked(bytes) }; unsafe { buf.set_size_unchecked(bytes) };
core.handle_incoming_physical_packet(&self2, &Endpoint::IpUdp(InetAddress::from(source)), &local_socket, &interface, buf) core.handle_incoming_physical_packet(
.await; &self2,
&Endpoint::IpUdp(InetAddress::from(source)),
&local_socket,
&interface,
buf,
)
.await;
} else { } else {
break; break;
} }
@ -144,7 +154,11 @@ impl ServiceImpl {
let config = self.data.config().await; let config = self.data.config().await;
if let Some(errors) = self if let Some(errors) = self
.update_udp_bindings_for_port(config.settings.primary_port, &config.settings.interface_prefix_blacklist, &config.settings.cidr_blacklist) .update_udp_bindings_for_port(
config.settings.primary_port,
&config.settings.interface_prefix_blacklist,
&config.settings.cidr_blacklist,
)
.await .await
{ {
for e in errors.iter() { for e in errors.iter() {
@ -193,7 +207,14 @@ impl SystemInterface for ServiceImpl {
assert!(self.data.save_identity(id).await.is_ok()) assert!(self.data.save_identity(id).await.is_ok())
} }
async fn wire_send(&self, endpoint: &Endpoint, local_socket: Option<&Self::LocalSocket>, local_interface: Option<&Self::LocalInterface>, data: &[&[u8]], packet_ttl: u8) -> bool { async fn wire_send(
&self,
endpoint: &Endpoint,
local_socket: Option<&Self::LocalSocket>,
local_interface: Option<&Self::LocalInterface>,
data: &[&[u8]],
packet_ttl: u8,
) -> bool {
match endpoint { match endpoint {
Endpoint::IpUdp(address) => { Endpoint::IpUdp(address) => {
// This is the fast path -- the socket is known to the core so just send it. // This is the fast path -- the socket is known to the core so just send it.
@ -251,7 +272,13 @@ impl SystemInterface for ServiceImpl {
return false; return false;
} }
async fn check_path(&self, _id: &Identity, endpoint: &Endpoint, _local_socket: Option<&Self::LocalSocket>, _local_interface: Option<&Self::LocalInterface>) -> bool { async fn check_path(
&self,
_id: &Identity,
endpoint: &Endpoint,
_local_socket: Option<&Self::LocalSocket>,
_local_interface: Option<&Self::LocalInterface>,
) -> bool {
let config = self.data.config().await; let config = self.data.config().await;
if let Some(pps) = config.physical.get(endpoint) { if let Some(pps) = config.physical.get(endpoint) {
!pps.blacklist !pps.blacklist

115
system-service/src/udp.rs
View file

@ -52,7 +52,15 @@ impl BoundUdpSocket {
#[inline(always)] #[inline(always)]
fn set_ttl(&self, packet_ttl: u8) { fn set_ttl(&self, packet_ttl: u8) {
let ttl = packet_ttl as libc::c_int; let ttl = packet_ttl as libc::c_int;
unsafe { libc::setsockopt(self.fd.as_(), libc::IPPROTO_IP.as_(), libc::IP_TOS.as_(), (&ttl as *const libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_()) }; unsafe {
libc::setsockopt(
self.fd.as_(),
libc::IPPROTO_IP.as_(),
libc::IP_TOS.as_(),
(&ttl as *const libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
)
};
} }
#[cfg(any(target_os = "macos", target_os = "freebsd"))] #[cfg(any(target_os = "macos", target_os = "freebsd"))]
@ -65,7 +73,14 @@ impl BoundUdpSocket {
unsafe { unsafe {
if b.len() == 1 { if b.len() == 1 {
let bb = *b.get_unchecked(0); let bb = *b.get_unchecked(0);
ok = libc::sendto(self.fd.as_(), bb.as_ptr().cast(), bb.len().as_(), 0, (dest as *const InetAddress).cast(), size_of::<InetAddress>().as_()) > 0; ok = libc::sendto(
self.fd.as_(),
bb.as_ptr().cast(),
bb.len().as_(),
0,
(dest as *const InetAddress).cast(),
size_of::<InetAddress>().as_(),
) > 0;
} else { } else {
let mut iov: [libc::iovec; 16] = MaybeUninit::uninit().assume_init(); let mut iov: [libc::iovec; 16] = MaybeUninit::uninit().assume_init();
assert!(b.len() <= iov.len()); assert!(b.len() <= iov.len());
@ -143,10 +158,17 @@ impl BoundUdpPort {
/// The caller can check the 'sockets' member variable after calling to determine which if any bindings were /// The caller can check the 'sockets' member variable after calling to determine which if any bindings were
/// successful. Any errors that occurred are returned as tuples of (interface, address, error). The second vector /// successful. Any errors that occurred are returned as tuples of (interface, address, error). The second vector
/// returned contains newly bound sockets. /// returned contains newly bound sockets.
pub fn update_bindings(&mut self, interface_prefix_blacklist: &Vec<String>, cidr_blacklist: &Vec<InetAddress>) -> (Vec<(LocalInterface, InetAddress, std::io::Error)>, Vec<Arc<BoundUdpSocket>>) { pub fn update_bindings(
&mut self,
interface_prefix_blacklist: &Vec<String>,
cidr_blacklist: &Vec<InetAddress>,
) -> (Vec<(LocalInterface, InetAddress, std::io::Error)>, Vec<Arc<BoundUdpSocket>>) {
let mut existing_bindings: HashMap<LocalInterface, HashMap<InetAddress, Arc<BoundUdpSocket>>> = HashMap::with_capacity(4); let mut existing_bindings: HashMap<LocalInterface, HashMap<InetAddress, Arc<BoundUdpSocket>>> = HashMap::with_capacity(4);
for s in self.sockets.drain(..) { for s in self.sockets.drain(..) {
existing_bindings.entry(s.interface).or_insert_with(|| HashMap::with_capacity(4)).insert(s.address.clone(), s); existing_bindings
.entry(s.interface)
.or_insert_with(|| HashMap::with_capacity(4))
.insert(s.address.clone(), s);
} }
let mut errors = Vec::new(); let mut errors = Vec::new();
@ -156,7 +178,10 @@ impl BoundUdpPort {
let mut addr_with_port = address.clone(); let mut addr_with_port = address.clone();
addr_with_port.set_port(self.port); addr_with_port.set_port(self.port);
if address.is_ip() if address.is_ip()
&& matches!(address.scope(), IpScope::Global | IpScope::PseudoPrivate | IpScope::Private | IpScope::Shared) && matches!(
address.scope(),
IpScope::Global | IpScope::PseudoPrivate | IpScope::Private | IpScope::Shared
)
&& !interface_prefix_blacklist.iter().any(|pfx| interface_str.starts_with(pfx.as_str())) && !interface_prefix_blacklist.iter().any(|pfx| interface_str.starts_with(pfx.as_str()))
&& !cidr_blacklist.iter().any(|r| address.is_within(r)) && !cidr_blacklist.iter().any(|r| address.is_within(r))
&& !ipv6::is_ipv6_temporary(interface_str.as_str(), address) && !ipv6::is_ipv6_temporary(interface_str.as_str(), address)
@ -187,7 +212,11 @@ impl BoundUdpPort {
errors.push((interface.clone(), addr_with_port, s.err().unwrap())); errors.push((interface.clone(), addr_with_port, s.err().unwrap()));
} }
} else { } else {
errors.push((interface.clone(), addr_with_port, std::io::Error::new(std::io::ErrorKind::Other, s.err().unwrap()))); errors.push((
interface.clone(),
addr_with_port,
std::io::Error::new(std::io::ErrorKind::Other, s.err().unwrap()),
));
} }
} }
} }
@ -220,15 +249,33 @@ unsafe fn bind_udp_to_device(device_name: &str, address: &InetAddress) -> Result
let mut fl: libc::c_int; let mut fl: libc::c_int;
fl = 1; fl = 1;
setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_REUSEPORT.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_()); setsockopt_results |= libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_REUSEPORT.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
);
debug_assert!(setsockopt_results == 0); debug_assert!(setsockopt_results == 0);
fl = 1; fl = 1;
setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_BROADCAST.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_()); setsockopt_results |= libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_BROADCAST.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
);
debug_assert!(setsockopt_results == 0); debug_assert!(setsockopt_results == 0);
if af == AF_INET6 { if af == AF_INET6 {
fl = 1; fl = 1;
setsockopt_results |= libc::setsockopt(s, libc::IPPROTO_IPV6.as_(), libc::IPV6_V6ONLY.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_()); setsockopt_results |= libc::setsockopt(
s,
libc::IPPROTO_IPV6.as_(),
libc::IPV6_V6ONLY.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
);
debug_assert!(setsockopt_results == 0); debug_assert!(setsockopt_results == 0);
} }
@ -237,7 +284,13 @@ unsafe fn bind_udp_to_device(device_name: &str, address: &InetAddress) -> Result
if !device_name.is_empty() { if !device_name.is_empty() {
let _ = std::ffi::CString::new(device_name).map(|dn| { let _ = std::ffi::CString::new(device_name).map(|dn| {
let dnb = dn.as_bytes_with_nul(); let dnb = dn.as_bytes_with_nul();
let _ = libc::setsockopt(s.as_(), libc::SOL_SOCKET.as_(), libc::SO_BINDTODEVICE.as_(), dnb.as_ptr().cast(), (dnb.len() - 1).as_()); let _ = libc::setsockopt(
s.as_(),
libc::SOL_SOCKET.as_(),
libc::SO_BINDTODEVICE.as_(),
dnb.as_ptr().cast(),
(dnb.len() - 1).as_(),
);
}); });
} }
} }
@ -251,30 +304,62 @@ unsafe fn bind_udp_to_device(device_name: &str, address: &InetAddress) -> Result
#[cfg(not(target_os = "linux"))] #[cfg(not(target_os = "linux"))]
{ {
fl = 0; fl = 0;
libc::setsockopt(s, libc::IPPROTO_IP.as_(), libc::IP_DONTFRAG.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_()); libc::setsockopt(
s,
libc::IPPROTO_IP.as_(),
libc::IP_DONTFRAG.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
);
} }
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
{ {
fl = libc::IP_PMTUDISC_DONT as libc::c_int; fl = libc::IP_PMTUDISC_DONT as libc::c_int;
libc::setsockopt(s, libc::IPPROTO_IP.as_(), libc::IP_MTU_DISCOVER.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_()); libc::setsockopt(
s,
libc::IPPROTO_IP.as_(),
libc::IP_MTU_DISCOVER.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
);
} }
} }
if af == AF_INET6 { if af == AF_INET6 {
fl = 0; fl = 0;
libc::setsockopt(s, libc::IPPROTO_IPV6.as_(), libc::IPV6_DONTFRAG.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_()); libc::setsockopt(
s,
libc::IPPROTO_IPV6.as_(),
libc::IPV6_DONTFRAG.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
);
} }
fl = 1048576; fl = 1048576;
while fl >= 65536 { while fl >= 65536 {
if libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_RCVBUF.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_()) == 0 { if libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_RCVBUF.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
) == 0
{
break; break;
} }
fl -= 65536; fl -= 65536;
} }
fl = 1048576; fl = 1048576;
while fl >= 65536 { while fl >= 65536 {
if libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_SNDBUF.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_()) == 0 { if libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_SNDBUF.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
) == 0
{
break; break;
} }
fl -= 65536; fl -= 65536;

20
system-service/src/utils.rs
View file

@ -99,7 +99,11 @@ pub fn json_patch(target: &mut serde_json::value::Value, source: &serde_json::va
/// ///
/// If there are no changes, None is returned. The depth limit is passed through to json_patch and /// If there are no changes, None is returned. The depth limit is passed through to json_patch and
/// should be set to a sanity check value to prevent overflows. /// should be set to a sanity check value to prevent overflows.
pub fn json_patch_object<O: Serialize + DeserializeOwned + Eq>(obj: O, patch: &str, depth_limit: usize) -> Result<Option<O>, serde_json::Error> { pub fn json_patch_object<O: Serialize + DeserializeOwned + Eq>(
obj: O,
patch: &str,
depth_limit: usize,
) -> Result<Option<O>, serde_json::Error> {
serde_json::from_str::<serde_json::value::Value>(patch).map_or_else( serde_json::from_str::<serde_json::value::Value>(patch).map_or_else(
|e| Err(e), |e| Err(e),
|patch| { |patch| {
@ -156,7 +160,14 @@ pub async fn read_limit<P: AsRef<Path>>(path: P, limit: usize) -> std::io::Resul
pub fn fs_restrict_permissions<P: AsRef<Path>>(path: P) -> bool { pub fn fs_restrict_permissions<P: AsRef<Path>>(path: P) -> bool {
unsafe { unsafe {
let c_path = std::ffi::CString::new(path.as_ref().to_str().unwrap()).unwrap(); let c_path = std::ffi::CString::new(path.as_ref().to_str().unwrap()).unwrap();
libc::chmod(c_path.as_ptr(), if path.as_ref().is_dir() { 0o700 } else { 0o600 }) == 0 libc::chmod(
c_path.as_ptr(),
if path.as_ref().is_dir() {
0o700
} else {
0o600
},
) == 0
} }
} }
@ -177,7 +188,10 @@ pub async fn parse_cli_identity(input: &str, validate: bool) -> Result<Identity,
let input_p = Path::new(input); let input_p = Path::new(input);
if input_p.is_file() { if input_p.is_file() {
read_limit(input_p, 16384).await.map_or_else(|e| Err(e.to_string()), |v| String::from_utf8(v).map_or_else(|e| Err(e.to_string()), |s| parse_func(s.as_str()))) read_limit(input_p, 16384).await.map_or_else(
|e| Err(e.to_string()),
|v| String::from_utf8(v).map_or_else(|e| Err(e.to_string()), |s| parse_func(s.as_str())),
)
} else { } else {
parse_func(input) parse_func(input)
} }

46
system-service/src/vnic/common.rs
View file

@ -9,7 +9,15 @@ use num_traits::AsPrimitive;
#[allow(unused_imports)] #[allow(unused_imports)]
use zerotier_network_hypervisor::vl1::MAC; use zerotier_network_hypervisor::vl1::MAC;
#[cfg(any(target_os = "macos", target_os = "ios", target_os = "netbsd", target_os = "openbsd", target_os = "dragonfly", target_os = "freebsd", target_os = "darwin"))] #[cfg(any(
target_os = "macos",
target_os = "ios",
target_os = "netbsd",
target_os = "openbsd",
target_os = "dragonfly",
target_os = "freebsd",
target_os = "darwin"
))]
#[allow(non_camel_case_types)] #[allow(non_camel_case_types)]
#[repr(C)] #[repr(C)]
struct ifmaddrs { struct ifmaddrs {
@ -19,13 +27,29 @@ struct ifmaddrs {
ifma_lladdr: *mut libc::sockaddr, ifma_lladdr: *mut libc::sockaddr,
} }
#[cfg(any(target_os = "macos", target_os = "ios", target_os = "netbsd", target_os = "openbsd", target_os = "dragonfly", target_os = "freebsd", target_os = "darwin"))] #[cfg(any(
target_os = "macos",
target_os = "ios",
target_os = "netbsd",
target_os = "openbsd",
target_os = "dragonfly",
target_os = "freebsd",
target_os = "darwin"
))]
extern "C" { extern "C" {
fn getifmaddrs(ifmap: *mut *mut ifmaddrs) -> c_int; fn getifmaddrs(ifmap: *mut *mut ifmaddrs) -> c_int;
fn freeifmaddrs(ifmp: *mut ifmaddrs); fn freeifmaddrs(ifmp: *mut ifmaddrs);
} }
#[cfg(any(target_os = "macos", target_os = "ios", target_os = "netbsd", target_os = "openbsd", target_os = "dragonfly", target_os = "freebsd", target_os = "darwin"))] #[cfg(any(
target_os = "macos",
target_os = "ios",
target_os = "netbsd",
target_os = "openbsd",
target_os = "dragonfly",
target_os = "freebsd",
target_os = "darwin"
))]
pub fn get_l2_multicast_subscriptions(dev: &str) -> HashSet<MAC> { pub fn get_l2_multicast_subscriptions(dev: &str) -> HashSet<MAC> {
let mut groups: HashSet<MAC> = HashSet::new(); let mut groups: HashSet<MAC> = HashSet::new();
let dev = dev.as_bytes(); let dev = dev.as_bytes();
@ -37,10 +61,20 @@ pub fn get_l2_multicast_subscriptions(dev: &str) -> HashSet<MAC> {
if !(*i).ifma_name.is_null() && !(*i).ifma_addr.is_null() && (*(*i).ifma_addr).sa_family as i32 == libc::AF_LINK as i32 { if !(*i).ifma_name.is_null() && !(*i).ifma_addr.is_null() && (*(*i).ifma_addr).sa_family as i32 == libc::AF_LINK as i32 {
let in_: &libc::sockaddr_dl = &*((*i).ifma_name.cast()); let in_: &libc::sockaddr_dl = &*((*i).ifma_name.cast());
let la: &libc::sockaddr_dl = &*((*i).ifma_addr.cast()); let la: &libc::sockaddr_dl = &*((*i).ifma_addr.cast());
if la.sdl_alen == 6 && in_.sdl_nlen <= dev.len().as_() && libc::memcmp(dev.as_ptr().cast(), in_.sdl_data.as_ptr().cast(), in_.sdl_nlen.as_()) == 0 { if la.sdl_alen == 6
&& in_.sdl_nlen <= dev.len().as_()
&& libc::memcmp(dev.as_ptr().cast(), in_.sdl_data.as_ptr().cast(), in_.sdl_nlen.as_()) == 0
{
let mi = la.sdl_nlen as usize; let mi = la.sdl_nlen as usize;
MAC::from_u64((la.sdl_data[mi] as u64) << 40 | (la.sdl_data[mi + 1] as u64) << 32 | (la.sdl_data[mi + 2] as u64) << 24 | (la.sdl_data[mi + 3] as u64) << 16 | (la.sdl_data[mi + 4] as u64) << 8 | la.sdl_data[mi + 5] as u64) MAC::from_u64(
.map(|mac| groups.insert(mac)); (la.sdl_data[mi] as u64) << 40
| (la.sdl_data[mi + 1] as u64) << 32
| (la.sdl_data[mi + 2] as u64) << 24
| (la.sdl_data[mi + 3] as u64) << 16
| (la.sdl_data[mi + 4] as u64) << 8
| la.sdl_data[mi + 5] as u64,
)
.map(|mac| groups.insert(mac));
} }
} }
i = (*i).ifma_next; i = (*i).ifma_next;

212
system-service/src/vnic/mac_feth_tap.rs
View file

@ -225,7 +225,15 @@ fn device_ipv6_set_params(device: &String, perform_nud: bool, accept_ra: bool) -
} }
let mut nd: in6_ndireq = zeroed(); let mut nd: in6_ndireq = zeroed();
copy_nonoverlapping(dev.as_ptr(), nd.ifname.as_mut_ptr().cast::<u8>(), if dev.len() > (nd.ifname.len() - 1) { nd.ifname.len() - 1 } else { dev.len() }); copy_nonoverlapping(
dev.as_ptr(),
nd.ifname.as_mut_ptr().cast::<u8>(),
if dev.len() > (nd.ifname.len() - 1) {
nd.ifname.len() - 1
} else {
dev.len()
},
);
if libc::ioctl(s, 76 /* SIOCGIFINFO_IN6 */, (&mut nd as *mut in6_ndireq).cast::<c_void>()) == 0 { if libc::ioctl(s, 76 /* SIOCGIFINFO_IN6 */, (&mut nd as *mut in6_ndireq).cast::<c_void>()) == 0 {
let oldflags = nd.ndi.flags; let oldflags = nd.ndi.flags;
if perform_nud { if perform_nud {
@ -243,7 +251,15 @@ fn device_ipv6_set_params(device: &String, perform_nud: bool, accept_ra: bool) -
} }
let mut ifr: in6_ifreq = zeroed(); let mut ifr: in6_ifreq = zeroed();
copy_nonoverlapping(dev.as_ptr(), ifr.ifr_name.as_mut_ptr().cast::<u8>(), if dev.len() > (ifr.ifr_name.len() - 1) { ifr.ifr_name.len() - 1 } else { dev.len() }); copy_nonoverlapping(
dev.as_ptr(),
ifr.ifr_name.as_mut_ptr().cast::<u8>(),
if dev.len() > (ifr.ifr_name.len() - 1) {
ifr.ifr_name.len() - 1
} else {
dev.len()
},
);
if libc::ioctl( if libc::ioctl(
s, s,
if accept_ra { if accept_ra {
@ -323,7 +339,13 @@ impl MacFethTap {
/// given will not remain valid after it returns. Also note that F will be called /// given will not remain valid after it returns. Also note that F will be called
/// from another thread that is spawned here, so all its bound references must /// from another thread that is spawned here, so all its bound references must
/// be "Send" and "Sync" e.g. Arc<>. /// be "Send" and "Sync" e.g. Arc<>.
pub fn new<F: Fn(&[u8]) + Send + Sync + 'static>(nwid: u64, mac: &MAC, mtu: i32, metric: i32, eth_frame_func: F) -> Result<MacFethTap, String> { pub fn new<F: Fn(&[u8]) + Send + Sync + 'static>(
nwid: u64,
mac: &MAC,
mtu: i32,
metric: i32,
eth_frame_func: F,
) -> Result<MacFethTap, String> {
// This tracks BPF devices we are using so we don't try to reopen them, and also // This tracks BPF devices we are using so we don't try to reopen them, and also
// doubles as a global lock to ensure that only one feth tap is created at once per // doubles as a global lock to ensure that only one feth tap is created at once per
// ZeroTier process per system. // ZeroTier process per system.
@ -369,12 +391,20 @@ impl MacFethTap {
// Create pair of feth interfaces and create MacFethDevice struct. // Create pair of feth interfaces and create MacFethDevice struct.
let cmd = Command::new(IFCONFIG).arg(&device_name).arg("create").spawn(); let cmd = Command::new(IFCONFIG).arg(&device_name).arg("create").spawn();
if cmd.is_err() { if cmd.is_err() {
return Err(format!("unable to create device '{}': {}", device_name.as_str(), cmd.err().unwrap().to_string())); return Err(format!(
"unable to create device '{}': {}",
device_name.as_str(),
cmd.err().unwrap().to_string()
));
} }
let _ = cmd.unwrap().wait(); let _ = cmd.unwrap().wait();
let cmd = Command::new(IFCONFIG).arg(&peer_device_name).arg("create").spawn(); let cmd = Command::new(IFCONFIG).arg(&peer_device_name).arg("create").spawn();
if cmd.is_err() { if cmd.is_err() {
return Err(format!("unable to create device '{}': {}", peer_device_name.as_str(), cmd.err().unwrap().to_string())); return Err(format!(
"unable to create device '{}': {}",
peer_device_name.as_str(),
cmd.err().unwrap().to_string()
));
} }
let _ = cmd.unwrap().wait(); let _ = cmd.unwrap().wait();
let device = MacFethDevice { name: device_name, peer_name: peer_device_name }; let device = MacFethDevice { name: device_name, peer_name: peer_device_name };
@ -382,28 +412,60 @@ impl MacFethTap {
// Set link-layer (MAC) address of primary interface. // Set link-layer (MAC) address of primary interface.
let cmd = Command::new(IFCONFIG).arg(&device.name).arg("lladdr").arg(mac.to_string()).spawn(); let cmd = Command::new(IFCONFIG).arg(&device.name).arg("lladdr").arg(mac.to_string()).spawn();
if cmd.is_err() { if cmd.is_err() {
return Err(format!("unable to configure device '{}': {}", &device.name, cmd.err().unwrap().to_string())); return Err(format!(
"unable to configure device '{}': {}",
&device.name,
cmd.err().unwrap().to_string()
));
} }
let _ = cmd.unwrap().wait(); let _ = cmd.unwrap().wait();
// Bind peer interfaces together. // Bind peer interfaces together.
let cmd = Command::new(IFCONFIG).arg(&device.peer_name).arg("peer").arg(device.name.as_str()).spawn(); let cmd = Command::new(IFCONFIG)
.arg(&device.peer_name)
.arg("peer")
.arg(device.name.as_str())
.spawn();
if cmd.is_err() { if cmd.is_err() {
return Err(format!("unable to configure device '{}': {}", &device.peer_name, cmd.err().unwrap().to_string())); return Err(format!(
"unable to configure device '{}': {}",
&device.peer_name,
cmd.err().unwrap().to_string()
));
} }
let _ = cmd.unwrap().wait(); let _ = cmd.unwrap().wait();
// Set MTU of secondary peer interface, bring up. // Set MTU of secondary peer interface, bring up.
let cmd = Command::new(IFCONFIG).arg(&device.peer_name).arg("mtu").arg(mtu.to_string()).arg("up").spawn(); let cmd = Command::new(IFCONFIG)
.arg(&device.peer_name)
.arg("mtu")
.arg(mtu.to_string())
.arg("up")
.spawn();
if cmd.is_err() { if cmd.is_err() {
return Err(format!("unable to configure device '{}': {}", &device.peer_name, cmd.err().unwrap().to_string())); return Err(format!(
"unable to configure device '{}': {}",
&device.peer_name,
cmd.err().unwrap().to_string()
));
} }
let _ = cmd.unwrap().wait(); let _ = cmd.unwrap().wait();
// Set MTU and metric of primary interface, bring up. // Set MTU and metric of primary interface, bring up.
let cmd = Command::new(IFCONFIG).arg(&device.name).arg("mtu").arg(mtu.to_string()).arg("metric").arg(metric.to_string()).arg("up").spawn(); let cmd = Command::new(IFCONFIG)
.arg(&device.name)
.arg("mtu")
.arg(mtu.to_string())
.arg("metric")
.arg(metric.to_string())
.arg("up")
.spawn();
if cmd.is_err() { if cmd.is_err() {
return Err(format!("unable to configure device '{}': {}", &device.name.as_str(), cmd.err().unwrap().to_string())); return Err(format!(
"unable to configure device '{}': {}",
&device.name.as_str(),
cmd.err().unwrap().to_string()
));
} }
let _ = cmd.unwrap().wait(); let _ = cmd.unwrap().wait();
@ -449,7 +511,11 @@ impl MacFethTap {
// Set immediate mode for "live" capture. // Set immediate mode for "live" capture.
fl = 1; fl = 1;
if unsafe { if unsafe {
libc::ioctl(bpf_fd as c_int, 112 /* BIOCIMMEDIATE */, (&mut fl as *mut c_int).cast::<c_void>()) libc::ioctl(
bpf_fd as c_int,
112, /* BIOCIMMEDIATE */
(&mut fl as *mut c_int).cast::<c_void>(),
)
} != 0 } != 0
{ {
unsafe { unsafe {
@ -461,7 +527,11 @@ impl MacFethTap {
// Do not send us back packets we inject or send. // Do not send us back packets we inject or send.
fl = 0; fl = 0;
if unsafe { if unsafe {
libc::ioctl(bpf_fd as c_int, 119 /* BIOCSSEESENT */, (&mut fl as *mut c_int).cast::<c_void>()) libc::ioctl(
bpf_fd as c_int,
119, /* BIOCSSEESENT */
(&mut fl as *mut c_int).cast::<c_void>(),
)
} != 0 } != 0
{ {
unsafe { unsafe {
@ -474,10 +544,22 @@ impl MacFethTap {
let mut bpf_ifr: ifreq = unsafe { std::mem::zeroed() }; let mut bpf_ifr: ifreq = unsafe { std::mem::zeroed() };
let peer_dev_name_bytes = device.peer_name.as_bytes(); let peer_dev_name_bytes = device.peer_name.as_bytes();
unsafe { unsafe {
copy_nonoverlapping(peer_dev_name_bytes.as_ptr(), bpf_ifr.ifr_name.as_mut_ptr().cast::<u8>(), if peer_dev_name_bytes.len() > (bpf_ifr.ifr_name.len() - 1) { bpf_ifr.ifr_name.len() - 1 } else { peer_dev_name_bytes.len() }); copy_nonoverlapping(
peer_dev_name_bytes.as_ptr(),
bpf_ifr.ifr_name.as_mut_ptr().cast::<u8>(),
if peer_dev_name_bytes.len() > (bpf_ifr.ifr_name.len() - 1) {
bpf_ifr.ifr_name.len() - 1
} else {
peer_dev_name_bytes.len()
},
);
} }
if unsafe { if unsafe {
libc::ioctl(bpf_fd as c_int, 108 /* BIOCSETIF */, (&mut bpf_ifr as *mut ifreq).cast::<c_void>()) libc::ioctl(
bpf_fd as c_int,
108, /* BIOCSETIF */
(&mut bpf_ifr as *mut ifreq).cast::<c_void>(),
)
} != 0 } != 0
{ {
unsafe { unsafe {
@ -489,7 +571,11 @@ impl MacFethTap {
// Include Ethernet header in BPF captures. // Include Ethernet header in BPF captures.
fl = 1; fl = 1;
if unsafe { if unsafe {
libc::ioctl(bpf_fd as c_int, 117 /* BIOCSHDRCMPLT */, (&mut fl as *mut c_int).cast::<c_void>()) libc::ioctl(
bpf_fd as c_int,
117, /* BIOCSHDRCMPLT */
(&mut fl as *mut c_int).cast::<c_void>(),
)
} != 0 } != 0
{ {
unsafe { unsafe {
@ -501,7 +587,11 @@ impl MacFethTap {
// Set promiscuous mode so bridging can work. // Set promiscuous mode so bridging can work.
fl = 1; fl = 1;
if unsafe { if unsafe {
libc::ioctl(bpf_fd as c_int, 105 /* BIOCPROMISC */, (&mut fl as *mut c_int).cast::<c_void>()) libc::ioctl(
bpf_fd as c_int,
105, /* BIOCPROMISC */
(&mut fl as *mut c_int).cast::<c_void>(),
)
} != 0 } != 0
{ {
unsafe { unsafe {
@ -558,9 +648,24 @@ impl MacFethTap {
ndrv_sa.snd_len = std::mem::size_of::<sockaddr_ndrv>() as c_uchar; ndrv_sa.snd_len = std::mem::size_of::<sockaddr_ndrv>() as c_uchar;
ndrv_sa.snd_family = 27 /* AF_NDRV */; ndrv_sa.snd_family = 27 /* AF_NDRV */;
unsafe { unsafe {
copy_nonoverlapping(peer_dev_name_bytes.as_ptr(), ndrv_sa.snd_name.as_mut_ptr().cast::<u8>(), if peer_dev_name_bytes.len() > (bpf_ifr.ifr_name.len() - 1) { bpf_ifr.ifr_name.len() - 1 } else { peer_dev_name_bytes.len() }); copy_nonoverlapping(
peer_dev_name_bytes.as_ptr(),
ndrv_sa.snd_name.as_mut_ptr().cast::<u8>(),
if peer_dev_name_bytes.len() > (bpf_ifr.ifr_name.len() - 1) {
bpf_ifr.ifr_name.len() - 1
} else {
peer_dev_name_bytes.len()
},
);
} }
if unsafe { libc::bind(ndrv_fd, (&ndrv_sa as *const sockaddr_ndrv).cast(), std::mem::size_of::<sockaddr_ndrv>() as libc::socklen_t) } != 0 { if unsafe {
libc::bind(
ndrv_fd,
(&ndrv_sa as *const sockaddr_ndrv).cast(),
std::mem::size_of::<sockaddr_ndrv>() as libc::socklen_t,
)
} != 0
{
unsafe { unsafe {
libc::close(bpf_fd); libc::close(bpf_fd);
} }
@ -569,7 +674,14 @@ impl MacFethTap {
} }
return Err(String::from("unable to bind AF_NDRV socket")); return Err(String::from("unable to bind AF_NDRV socket"));
} }
if unsafe { libc::connect(ndrv_fd, (&ndrv_sa as *const sockaddr_ndrv).cast(), std::mem::size_of::<sockaddr_ndrv>() as libc::socklen_t) } != 0 { if unsafe {
libc::connect(
ndrv_fd,
(&ndrv_sa as *const sockaddr_ndrv).cast(),
std::mem::size_of::<sockaddr_ndrv>() as libc::socklen_t,
)
} != 0
{
unsafe { unsafe {
libc::close(bpf_fd); libc::close(bpf_fd);
} }
@ -605,7 +717,16 @@ impl MacFethTap {
impl VNIC for MacFethTap { impl VNIC for MacFethTap {
fn add_ip(&self, ip: &InetAddress) -> bool { fn add_ip(&self, ip: &InetAddress) -> bool {
if !self.have_ip(ip) { if !self.have_ip(ip) {
let cmd = Command::new(IFCONFIG).arg(&self.device.name).arg(if ip.is_v6() { "inet6" } else { "inet" }).arg(ip.to_string()).arg("alias").spawn(); let cmd = Command::new(IFCONFIG)
.arg(&self.device.name)
.arg(if ip.is_v6() {
"inet6"
} else {
"inet"
})
.arg(ip.to_string())
.arg("alias")
.spawn();
if cmd.is_ok() { if cmd.is_ok() {
let _ = cmd.unwrap().wait(); let _ = cmd.unwrap().wait();
} }
@ -616,7 +737,16 @@ impl VNIC for MacFethTap {
fn remove_ip(&self, ip: &InetAddress) -> bool { fn remove_ip(&self, ip: &InetAddress) -> bool {
if self.have_ip(ip) { if self.have_ip(ip) {
let cmd = Command::new(IFCONFIG).arg(&self.device.name).arg(if ip.is_v6() { "inet6" } else { "inet" }).arg(ip.to_string()).arg("-alias").spawn(); let cmd = Command::new(IFCONFIG)
.arg(&self.device.name)
.arg(if ip.is_v6() {
"inet6"
} else {
"inet"
})
.arg(ip.to_string())
.arg("-alias")
.spawn();
if cmd.is_ok() { if cmd.is_ok() {
let _ = cmd.unwrap().wait(); let _ = cmd.unwrap().wait();
} }
@ -644,18 +774,42 @@ impl VNIC for MacFethTap {
fn get_multicast_groups(&self) -> HashSet<MulticastGroup> { fn get_multicast_groups(&self) -> HashSet<MulticastGroup> {
let mut all_groups: HashSet<MulticastGroup> = HashSet::new(); let mut all_groups: HashSet<MulticastGroup> = HashSet::new();
crate::vnic::common::get_l2_multicast_subscriptions(self.device.name.as_str()).into_iter().for_each(|mac| { crate::vnic::common::get_l2_multicast_subscriptions(self.device.name.as_str())
all_groups.insert(MulticastGroup::from(&mac)); .into_iter()
}); .for_each(|mac| {
all_groups.insert(MulticastGroup::from(&mac));
});
all_groups all_groups
} }
#[inline(always)] #[inline(always)]
fn put(&self, source_mac: &zerotier_network_hypervisor::vl1::MAC, dest_mac: &zerotier_network_hypervisor::vl1::MAC, ethertype: u16, _vlan_id: u16, data: *const u8, len: usize) -> bool { fn put(
&self,
source_mac: &zerotier_network_hypervisor::vl1::MAC,
dest_mac: &zerotier_network_hypervisor::vl1::MAC,
ethertype: u16,
_vlan_id: u16,
data: *const u8,
len: usize,
) -> bool {
let dm = dest_mac.0; let dm = dest_mac.0;
let sm = source_mac.0; let sm = source_mac.0;
let mut hdr: [u8; 14] = let mut hdr: [u8; 14] = [
[(dm >> 40) as u8, (dm >> 32) as u8, (dm >> 24) as u8, (dm >> 16) as u8, (dm >> 8) as u8, dm as u8, (sm >> 40) as u8, (sm >> 32) as u8, (sm >> 24) as u8, (sm >> 16) as u8, (sm >> 8) as u8, sm as u8, (ethertype >> 8) as u8, ethertype as u8]; (dm >> 40) as u8,
(dm >> 32) as u8,
(dm >> 24) as u8,
(dm >> 16) as u8,
(dm >> 8) as u8,
dm as u8,
(sm >> 40) as u8,
(sm >> 32) as u8,
(sm >> 24) as u8,
(sm >> 16) as u8,
(sm >> 8) as u8,
sm as u8,
(ethertype >> 8) as u8,
ethertype as u8,
];
unsafe { unsafe {
let iov: [libc::iovec; 2] = [ let iov: [libc::iovec; 2] = [
libc::iovec { iov_base: hdr.as_mut_ptr().cast(), iov_len: 14 }, libc::iovec { iov_base: hdr.as_mut_ptr().cast(), iov_len: 14 },

76
third_party/kyber/benches/api.rs vendored
View file

@ -9,57 +9,45 @@ const CT_HEX: &str = "EADD5ADA14DA57F0AEF3505F1CAA6485D4238D999A3EF4B0A59A1CDBE0
// Benchmarking key generation // Benchmarking key generation
fn keypair_bench(c: &mut Criterion) { fn keypair_bench(c: &mut Criterion) {
let mut rng = rand::thread_rng(); let mut rng = rand::thread_rng();
c.bench_function( c.bench_function("Keypair Generation", |b| {
"Keypair Generation", b.iter(|| {
|b| b.iter( let _keys = keypair(&mut rng);
|| { })
let _keys = keypair(&mut rng); });
}
)
);
} }
// Encapsulating a single public key // Encapsulating a single public key
fn encap_bench(c: &mut Criterion) { fn encap_bench(c: &mut Criterion) {
let pk = crate::decode_hex(PK_HEX); let pk = crate::decode_hex(PK_HEX);
let mut rng = rand::thread_rng(); let mut rng = rand::thread_rng();
c.bench_function( c.bench_function("Encapsulate", |b| {
"Encapsulate", b.iter(|| {
|b| b.iter( let _enc = encapsulate(&pk, &mut rng);
|| { })
let _enc = encapsulate(&pk, &mut rng); });
}
)
);
} }
// Decapsulating a single correct ciphertext // Decapsulating a single correct ciphertext
fn decap_bench(c: &mut Criterion) { fn decap_bench(c: &mut Criterion) {
let sk = decode_hex(SK_HEX); let sk = decode_hex(SK_HEX);
let ct = decode_hex(CT_HEX); let ct = decode_hex(CT_HEX);
c.bench_function( c.bench_function("Decapsulate", |b| {
"Decapsulate", b.iter(|| {
|b| b.iter( let _dec = decapsulate(&ct, &sk);
|| { })
let _dec = decapsulate(&ct, &sk); });
}
)
);
} }
// Decapsulating a single incorrect ciphertext // Decapsulating a single incorrect ciphertext
fn decap_fail_bench(c: &mut Criterion) { fn decap_fail_bench(c: &mut Criterion) {
let sk = decode_hex(BAD_SK); let sk = decode_hex(BAD_SK);
let ct = decode_hex(CT_HEX); let ct = decode_hex(CT_HEX);
c.bench_function( c.bench_function("Decapsulate Failure", |b| {
"Decapsulate Failure", b.iter(|| {
|b| b.iter( let _dec = decapsulate(&ct, &sk);
|| { })
let _dec = decapsulate(&ct, &sk); });
}
)
);
} }
criterion_group!(benches, keypair_bench, encap_bench, decap_bench, decap_fail_bench); criterion_group!(benches, keypair_bench, encap_bench, decap_bench, decap_fail_bench);
@ -67,8 +55,8 @@ criterion_main!(benches);
// Decodes a hex string into a vector of bytes // Decodes a hex string into a vector of bytes
pub fn decode_hex(s: &str) -> Vec<u8> { pub fn decode_hex(s: &str) -> Vec<u8> {
(0..s.len()) (0..s.len())
.step_by(2) .step_by(2)
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("Hex string decoding")) .map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("Hex string decoding"))
.collect::<Vec<u8>>() .collect::<Vec<u8>>()
} }

36
third_party/kyber/build.rs vendored
View file

@ -1,19 +1,19 @@
fn main() { fn main() {
#[cfg(all(target_arch = "x86_64", not(feature = "reference")))] #[cfg(all(target_arch = "x86_64", not(feature = "reference")))]
cc::Build::new() cc::Build::new()
.include("src/avx2/") .include("src/avx2/")
.file("src/avx2/basemul.S") .file("src/avx2/basemul.S")
.file("src/avx2/fq.S") .file("src/avx2/fq.S")
.file("src/avx2/invntt.S") .file("src/avx2/invntt.S")
.file("src/avx2/ntt.S") .file("src/avx2/ntt.S")
.file("src/avx2/shuffle.S") .file("src/avx2/shuffle.S")
.compile("pqc_kyber"); .compile("pqc_kyber");
// #[cfg( // #[cfg(
// all( // all(
// any(target_arch = "arm", target_arch="aarch64"), // any(target_arch = "arm", target_arch="aarch64"),
// not(feature = "reference") // not(feature = "reference")
// ) // )
// )] // )]
// cc::Build::new() // cc::Build::new()
// .include("src/neon/"); // .include("src/neon/");
} }

36
third_party/kyber/examples/ake.rs vendored
View file

@ -1,27 +1,25 @@
use pqc_kyber::*; use pqc_kyber::*;
fn main() -> Result<(), KyberError> { fn main() -> Result<(), KyberError> {
let mut rng = rand::thread_rng(); let mut rng = rand::thread_rng();
let mut alice = Ake::new(); let mut alice = Ake::new();
let mut bob = Ake::new(); let mut bob = Ake::new();
let alice_keys = keypair(&mut rng); let alice_keys = keypair(&mut rng);
let bob_keys = keypair(&mut rng); let bob_keys = keypair(&mut rng);
// Alice initiates key exchange with bob // Alice initiates key exchange with bob
let client_send = alice.client_init(&bob_keys.public, &mut rng); let client_send = alice.client_init(&bob_keys.public, &mut rng);
// Bob receives the request and authenticates Alice, sends
// encapsulated shared secret back
let server_send = bob.server_receive(
client_send, &alice_keys.public, &bob_keys.secret, &mut rng
)?;
// Alice autheticates and decapsulates // Bob receives the request and authenticates Alice, sends
alice.client_confirm(server_send, &alice_keys.secret)?; // encapsulated shared secret back
let server_send = bob.server_receive(client_send, &alice_keys.public, &bob_keys.secret, &mut rng)?;
// Both structs now have the shared secret // Alice autheticates and decapsulates
assert_eq!(alice.shared_secret, bob.shared_secret); alice.client_confirm(server_send, &alice_keys.secret)?;
Ok(()) // Both structs now have the shared secret
} assert_eq!(alice.shared_secret, bob.shared_secret);
Ok(())
}

28
third_party/kyber/examples/kem.rs vendored
View file

@ -1,18 +1,18 @@
use pqc_kyber::*; use pqc_kyber::*;
fn main () -> Result<(), KyberError> { fn main() -> Result<(), KyberError> {
let mut rng = rand::thread_rng(); let mut rng = rand::thread_rng();
// Alice generates a keypair // Alice generates a keypair
let alice_keys = keypair(&mut rng); let alice_keys = keypair(&mut rng);
// Bob encapsulates a shared secret // Bob encapsulates a shared secret
let (ciphertext, shared_secret_bob) = encapsulate(&alice_keys.public, &mut rng)?; let (ciphertext, shared_secret_bob) = encapsulate(&alice_keys.public, &mut rng)?;
// Alice decapsulates the shared secret // Alice decapsulates the shared secret
let shared_secret_alice = decapsulate(&ciphertext, &alice_keys.secret)?; let shared_secret_alice = decapsulate(&ciphertext, &alice_keys.secret)?;
// Both can now communicate symetrically // Both can now communicate symetrically
assert_eq!(shared_secret_alice, shared_secret_bob); assert_eq!(shared_secret_alice, shared_secret_bob);
Ok(()) Ok(())
} }

34
third_party/kyber/examples/uake.rs vendored
View file

@ -1,27 +1,25 @@
use pqc_kyber::*; use pqc_kyber::*;
fn main() -> Result<(), KyberError> { fn main() -> Result<(), KyberError> {
let mut rng = rand::thread_rng(); let mut rng = rand::thread_rng();
let mut alice = Uake::new(); let mut alice = Uake::new();
let mut bob = Uake::new(); let mut bob = Uake::new();
let bob_keys = keypair(&mut rng); let bob_keys = keypair(&mut rng);
// Alice initiates key exchange with bob // Alice initiates key exchange with bob
let client_send = alice.client_init(&bob_keys.public, &mut rng); let client_send = alice.client_init(&bob_keys.public, &mut rng);
// Bob receives the request and authenticates Alice, sends
// encapsulated shared secret back
let server_send = bob.server_receive(
client_send, &bob_keys.secret, &mut rng
)?;
// Alice autheticates and decapsulates // Bob receives the request and authenticates Alice, sends
alice.client_confirm(server_send)?; // encapsulated shared secret back
let server_send = bob.server_receive(client_send, &bob_keys.secret, &mut rng)?;
// Both structs now have the shared secret // Alice autheticates and decapsulates
assert_eq!(alice.shared_secret, bob.shared_secret); alice.client_confirm(server_send)?;
Ok(()) // Both structs now have the shared secret
} assert_eq!(alice.shared_secret, bob.shared_secret);
Ok(())
}

28
third_party/kyber/fuzz/src/bin/decap.rs vendored
View file

@ -2,16 +2,18 @@ use honggfuzz::fuzz;
use pqc_kyber::*; use pqc_kyber::*;
fn main() -> Result<(), KyberError> { fn main() -> Result<(), KyberError> {
let mut ss = [0u8; KYBER_SSBYTES]; let mut ss = [0u8; KYBER_SSBYTES];
const CTBYTES: usize = KYBER_CIPHERTEXTBYTES; const CTBYTES: usize = KYBER_CIPHERTEXTBYTES;
const SKBYTES: usize = KYBER_SECRETKEYBYTES; const SKBYTES: usize = KYBER_SECRETKEYBYTES;
loop { loop {
fuzz!(|data: &[u8] | { fuzz!(|data: &[u8]| {
if data.len() != CTBYTES + SKBYTES {return}; if data.len() != CTBYTES + SKBYTES {
match crypto_kem_dec(&mut ss, &data[..CTBYTES], &data[CTBYTES..SKBYTES]) { return;
Ok(_) => (), };
Err(_) => () match crypto_kem_dec(&mut ss, &data[..CTBYTES], &data[CTBYTES..SKBYTES]) {
} Ok(_) => (),
}); Err(_) => (),
}; }
} });
}
}

30
third_party/kyber/fuzz/src/bin/encap.rs vendored
View file

@ -1,19 +1,21 @@
use honggfuzz::fuzz; use honggfuzz::fuzz;
use pqc_kyber::*; use pqc_kyber::*;
use rand_xoshiro::rand_core::{SeedableRng, RngCore}; use rand_xoshiro::rand_core::{RngCore, SeedableRng};
use rand_xoshiro::Xoshiro256Plus; use rand_xoshiro::Xoshiro256Plus;
fn main() -> Result<(), KyberError> { fn main() -> Result<(), KyberError> {
let mut _rng = rand::thread_rng(); //placeholder let mut _rng = rand::thread_rng(); //placeholder
let mut rng = Xoshiro256Plus::seed_from_u64(0); let mut rng = Xoshiro256Plus::seed_from_u64(0);
let mut ct = [0u8; KYBER_CIPHERTEXTBYTES]; let mut ct = [0u8; KYBER_CIPHERTEXTBYTES];
let mut ss = [0u8; KYBER_SSBYTES]; let mut ss = [0u8; KYBER_SSBYTES];
let mut s1 = [0u8; 32]; let mut s1 = [0u8; 32];
loop { loop {
rng.fill_bytes(&mut s1); rng.fill_bytes(&mut s1);
fuzz!(|data: &[u8] | { fuzz!(|data: &[u8]| {
if data.len() != KYBER_PUBLICKEYBYTES {return}; if data.len() != KYBER_PUBLICKEYBYTES {
crypto_kem_enc(&mut ct, &mut ss, data, &mut _rng, Some(&s1)); return;
}); };
}; crypto_kem_enc(&mut ct, &mut ss, data, &mut _rng, Some(&s1));
} });
}
}

32
third_party/kyber/fuzz/src/bin/keypair.rs vendored
View file

@ -1,20 +1,20 @@
use honggfuzz::fuzz; use honggfuzz::fuzz;
use pqc_kyber::*; use pqc_kyber::*;
use rand_xoshiro::rand_core::{SeedableRng, RngCore}; use rand_xoshiro::rand_core::{RngCore, SeedableRng};
use rand_xoshiro::Xoshiro256Plus; use rand_xoshiro::Xoshiro256Plus;
fn main() -> Result<(), KyberError> { fn main() -> Result<(), KyberError> {
let mut _rng = rand::thread_rng(); //placeholder let mut _rng = rand::thread_rng(); //placeholder
let mut rng = Xoshiro256Plus::seed_from_u64(0); let mut rng = Xoshiro256Plus::seed_from_u64(0);
let mut public = [0u8; KYBER_PUBLICKEYBYTES]; let mut public = [0u8; KYBER_PUBLICKEYBYTES];
let mut secret = [0u8; KYBER_SECRETKEYBYTES]; let mut secret = [0u8; KYBER_SECRETKEYBYTES];
let mut s1 = [0u8; 32]; let mut s1 = [0u8; 32];
let mut s2 = [0u8; 32]; let mut s2 = [0u8; 32];
loop { loop {
rng.fill_bytes(&mut s1); rng.fill_bytes(&mut s1);
rng.fill_bytes(&mut s2); rng.fill_bytes(&mut s2);
fuzz!(|data: ()| { fuzz!(|data: ()| {
crypto_kem_keypair(&mut public, &mut secret, &mut _rng, Some((&s1, &s2))); crypto_kem_keypair(&mut public, &mut secret, &mut _rng, Some((&s1, &s2)));
}); });
}; }
} }

103
third_party/kyber/src/api.rs vendored
View file

@ -1,13 +1,13 @@
use crate::{ use crate::{
params::*, error::KyberError,
error::KyberError, kem::*,
RngCore, CryptoRng, kex::{Decapsulated, Encapsulated, PublicKey, SecretKey},
kem::*, params::*,
kex::{PublicKey, SecretKey, Encapsulated, Decapsulated} CryptoRng, RngCore,
}; };
/// Keypair generation with a provided RNG. /// Keypair generation with a provided RNG.
/// ///
/// ### Example /// ### Example
/// ``` /// ```
/// # use pqc_kyber::*; /// # use pqc_kyber::*;
@ -16,13 +16,14 @@ use crate::{
/// let keys = keypair(&mut rng); /// let keys = keypair(&mut rng);
/// # Ok(())} /// # Ok(())}
/// ``` /// ```
pub fn keypair<R>(rng: &mut R) -> Keypair pub fn keypair<R>(rng: &mut R) -> Keypair
where R: RngCore + CryptoRng where
R: RngCore + CryptoRng,
{ {
let mut public = [0u8; KYBER_PUBLICKEYBYTES]; let mut public = [0u8; KYBER_PUBLICKEYBYTES];
let mut secret = [0u8; KYBER_SECRETKEYBYTES]; let mut secret = [0u8; KYBER_SECRETKEYBYTES];
crypto_kem_keypair(&mut public, &mut secret, rng, None); crypto_kem_keypair(&mut public, &mut secret, rng, None);
Keypair { public, secret } Keypair { public, secret }
} }
/// Encapsulates a public key returning the ciphertext to send /// Encapsulates a public key returning the ciphertext to send
@ -30,23 +31,24 @@ pub fn keypair<R>(rng: &mut R) -> Keypair
/// ///
/// ### Example /// ### Example
/// ``` /// ```
/// # use pqc_kyber::*; /// # use pqc_kyber::*;
/// # fn main() -> Result<(), KyberError> { /// # fn main() -> Result<(), KyberError> {
/// let mut rng = rand::thread_rng(); /// let mut rng = rand::thread_rng();
/// let keys = keypair(&mut rng); /// let keys = keypair(&mut rng);
/// let (ciphertext, shared_secret) = encapsulate(&keys.public, &mut rng)?; /// let (ciphertext, shared_secret) = encapsulate(&keys.public, &mut rng)?;
/// # Ok(())} /// # Ok(())}
/// ``` /// ```
pub fn encapsulate<R>(pk: &[u8], rng: &mut R) -> Encapsulated pub fn encapsulate<R>(pk: &[u8], rng: &mut R) -> Encapsulated
where R: CryptoRng + RngCore where
R: CryptoRng + RngCore,
{ {
if pk.len() != KYBER_PUBLICKEYBYTES { if pk.len() != KYBER_PUBLICKEYBYTES {
return Err(KyberError::InvalidInput) return Err(KyberError::InvalidInput);
} }
let mut ct = [0u8; KYBER_CIPHERTEXTBYTES]; let mut ct = [0u8; KYBER_CIPHERTEXTBYTES];
let mut ss = [0u8; KYBER_SSBYTES]; let mut ss = [0u8; KYBER_SSBYTES];
crypto_kem_enc(&mut ct, &mut ss, pk, rng, None); crypto_kem_enc(&mut ct, &mut ss, pk, rng, None);
Ok((ct, ss)) Ok((ct, ss))
} }
/// Decapsulates ciphertext with a secret key, the result will contain /// Decapsulates ciphertext with a secret key, the result will contain
@ -63,41 +65,40 @@ pub fn encapsulate<R>(pk: &[u8], rng: &mut R) -> Encapsulated
/// assert_eq!(ss1, ss2); /// assert_eq!(ss1, ss2);
/// # Ok(())} /// # Ok(())}
/// ``` /// ```
pub fn decapsulate(ct: &[u8], sk: &[u8]) -> Decapsulated pub fn decapsulate(ct: &[u8], sk: &[u8]) -> Decapsulated {
{ if ct.len() != KYBER_CIPHERTEXTBYTES || sk.len() != KYBER_SECRETKEYBYTES {
if ct.len() != KYBER_CIPHERTEXTBYTES || sk.len() != KYBER_SECRETKEYBYTES { return Err(KyberError::InvalidInput);
return Err(KyberError::InvalidInput) }
} let mut ss = [0u8; KYBER_SSBYTES];
let mut ss = [0u8; KYBER_SSBYTES]; match crypto_kem_dec(&mut ss, ct, sk) {
match crypto_kem_dec(&mut ss, ct, sk) { Ok(_) => Ok(ss),
Ok(_) => Ok(ss), Err(e) => Err(e),
Err(e) => Err(e) }
}
} }
/// A public/secret keypair for use with Kyber. /// A public/secret keypair for use with Kyber.
/// ///
/// Byte lengths of the keys are determined by the security level chosen. /// Byte lengths of the keys are determined by the security level chosen.
#[derive(Copy, Clone, Debug, Eq, PartialEq)] #[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub struct Keypair { pub struct Keypair {
pub public: PublicKey, pub public: PublicKey,
pub secret: SecretKey pub secret: SecretKey,
} }
impl Keypair { impl Keypair {
/// Securely generates a new keypair` /// Securely generates a new keypair`
/// ``` /// ```
/// # use pqc_kyber::*; /// # use pqc_kyber::*;
/// # fn main() -> Result<(), KyberError> { /// # fn main() -> Result<(), KyberError> {
/// let mut rng = rand::thread_rng(); /// let mut rng = rand::thread_rng();
/// let keys = Keypair::generate(&mut rng); /// let keys = Keypair::generate(&mut rng);
/// # let empty_keys = Keypair{ /// # let empty_keys = Keypair{
/// public: [0u8; KYBER_PUBLICKEYBYTES], secret: [0u8; KYBER_SECRETKEYBYTES] /// public: [0u8; KYBER_PUBLICKEYBYTES], secret: [0u8; KYBER_SECRETKEYBYTES]
/// }; /// };
/// # assert!(empty_keys != keys); /// # assert!(empty_keys != keys);
/// # Ok(()) } /// # Ok(()) }
/// ``` /// ```
pub fn generate<R: CryptoRng + RngCore>(rng: &mut R) -> Keypair { pub fn generate<R: CryptoRng + RngCore>(rng: &mut R) -> Keypair {
keypair(rng) keypair(rng)
} }
} }

279
third_party/kyber/src/avx2/aes256ctr.rs vendored
View file

@ -1,177 +1,170 @@
// Based heavily on public-domain code by Romain Dolbeau // Based heavily on public-domain code by Romain Dolbeau
// Different handling of nonce+counter than original version using // Different handling of nonce+counter than original version using
// separated 64-bit nonce and internal 64-bit counter, starting from zero // separated 64-bit nonce and internal 64-bit counter, starting from zero
// Public Domain // Public Domain
#![cfg(feature="90s")] #![cfg(feature = "90s")]
use core::arch::x86_64::*; use core::arch::x86_64::*;
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
#[repr(C)] #[repr(C)]
pub(crate) struct Aes256CtrCtx { pub(crate) struct Aes256CtrCtx {
pub rkeys: [__m128i; 16], pub rkeys: [__m128i; 16],
pub n: __m128i pub n: __m128i,
} }
impl Aes256CtrCtx { impl Aes256CtrCtx {
pub fn new() -> Self { pub fn new() -> Self {
unsafe { unsafe { Self { rkeys: [_mm_setzero_si128(); 16], n: _mm_setzero_si128() } }
Self {
rkeys: [_mm_setzero_si128(); 16],
n: _mm_setzero_si128()
}
} }
}
} }
unsafe fn aesni_encrypt4(out: &mut[u8], n :&mut __m128i, rkeys: &[__m128i; 16]) unsafe fn aesni_encrypt4(out: &mut [u8], n: &mut __m128i, rkeys: &[__m128i; 16]) {
{ let idx: __m128i = _mm_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 7, 6, 5, 4, 3, 2, 1, 0);
let idx: __m128i = _mm_set_epi8(8,9,10,11,12,13,14,15,7,6,5,4,3,2,1,0);
// Load current counter value // Load current counter value
let mut f = _mm_load_si128(n); let mut f = _mm_load_si128(n);
// Increase counter in 4 consecutive blocks // Increase counter in 4 consecutive blocks
let mut f0 = _mm_shuffle_epi8(_mm_add_epi64(f,_mm_set_epi64x(0,0)),idx); let mut f0 = _mm_shuffle_epi8(_mm_add_epi64(f, _mm_set_epi64x(0, 0)), idx);
let mut f1 = _mm_shuffle_epi8(_mm_add_epi64(f,_mm_set_epi64x(1,0)),idx); let mut f1 = _mm_shuffle_epi8(_mm_add_epi64(f, _mm_set_epi64x(1, 0)), idx);
let mut f2 = _mm_shuffle_epi8(_mm_add_epi64(f,_mm_set_epi64x(2,0)),idx); let mut f2 = _mm_shuffle_epi8(_mm_add_epi64(f, _mm_set_epi64x(2, 0)), idx);
let mut f3 = _mm_shuffle_epi8(_mm_add_epi64(f,_mm_set_epi64x(3,0)),idx); let mut f3 = _mm_shuffle_epi8(_mm_add_epi64(f, _mm_set_epi64x(3, 0)), idx);
// Write counter for next iteration, increased by 4 // Write counter for next iteration, increased by 4
_mm_store_si128(n as *mut __m128i,_mm_add_epi64(f,_mm_set_epi64x(4,0))); _mm_store_si128(n as *mut __m128i, _mm_add_epi64(f, _mm_set_epi64x(4, 0)));
// Actual AES encryption, 4x interleaved4 // Actual AES encryption, 4x interleaved4
f = _mm_load_si128(&rkeys[0]); f = _mm_load_si128(&rkeys[0]);
f0 = _mm_xor_si128(f0,f); f0 = _mm_xor_si128(f0, f);
f1 = _mm_xor_si128(f1,f); f1 = _mm_xor_si128(f1, f);
f2 = _mm_xor_si128(f2,f); f2 = _mm_xor_si128(f2, f);
f3 = _mm_xor_si128(f3,f); f3 = _mm_xor_si128(f3, f);
for i in 1..14 { for i in 1..14 {
f = _mm_load_si128(&rkeys[i]); f = _mm_load_si128(&rkeys[i]);
f0 = _mm_aesenc_si128(f0,f); f0 = _mm_aesenc_si128(f0, f);
f1 = _mm_aesenc_si128(f1,f); f1 = _mm_aesenc_si128(f1, f);
f2 = _mm_aesenc_si128(f2,f); f2 = _mm_aesenc_si128(f2, f);
f3 = _mm_aesenc_si128(f3,f); f3 = _mm_aesenc_si128(f3, f);
} }
f = _mm_load_si128(&rkeys[14]); f = _mm_load_si128(&rkeys[14]);
f0 = _mm_aesenclast_si128(f0,f); f0 = _mm_aesenclast_si128(f0, f);
f1 = _mm_aesenclast_si128(f1,f); f1 = _mm_aesenclast_si128(f1, f);
f2 = _mm_aesenclast_si128(f2,f); f2 = _mm_aesenclast_si128(f2, f);
f3 = _mm_aesenclast_si128(f3,f); f3 = _mm_aesenclast_si128(f3, f);
// Write results // Write results
_mm_storeu_si128(out[..].as_mut_ptr() as *mut __m128i, f0); _mm_storeu_si128(out[..].as_mut_ptr() as *mut __m128i, f0);
_mm_storeu_si128(out[16..].as_mut_ptr() as *mut __m128i, f1); _mm_storeu_si128(out[16..].as_mut_ptr() as *mut __m128i, f1);
_mm_storeu_si128(out[32..].as_mut_ptr() as *mut __m128i, f2); _mm_storeu_si128(out[32..].as_mut_ptr() as *mut __m128i, f2);
_mm_storeu_si128(out[48..].as_mut_ptr() as *mut __m128i, f3); _mm_storeu_si128(out[48..].as_mut_ptr() as *mut __m128i, f3);
} }
// Casting aliases // Casting aliases
unsafe fn cast_128i(x: __m128) -> __m128i unsafe fn cast_128i(x: __m128) -> __m128i {
{ _mm_castps_si128(x)
_mm_castps_si128(x)
} }
unsafe fn cast_128(x: __m128i) -> __m128 unsafe fn cast_128(x: __m128i) -> __m128 {
{ _mm_castsi128_ps(x)
_mm_castsi128_ps(x)
} }
pub(crate) fn aes256ctr_init(state: &mut Aes256CtrCtx, key: &[u8], nonce: [u8; 12]) pub(crate) fn aes256ctr_init(state: &mut Aes256CtrCtx, key: &[u8], nonce: [u8; 12]) {
{ unsafe {
unsafe { let mut idx = 0;
let mut idx = 0; let key0 = _mm_loadu_si128(key.as_ptr() as *const __m128i);
let key0 = _mm_loadu_si128(key.as_ptr() as *const __m128i); let key1 = _mm_loadu_si128(key[16..].as_ptr() as *const __m128i);
let key1 = _mm_loadu_si128(key[16..].as_ptr() as *const __m128i);
state.n = _mm_loadl_epi64(nonce[..].as_ptr() as *const __m128i); state.n = _mm_loadl_epi64(nonce[..].as_ptr() as *const __m128i);
state.rkeys[idx] = key0; state.rkeys[idx] = key0;
idx += 1;
let mut temp0 = key0;
let mut temp1;
let mut temp2 = key1;
let mut temp4 = _mm_setzero_si128();
macro_rules! block1 {
($imm:expr) => {
temp1 = _mm_aeskeygenassist_si128(temp2, $imm);
state.rkeys[idx] = temp2;
idx += 1; idx += 1;
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp0), 0x10)); let mut temp0 = key0;
temp0 = _mm_xor_si128(temp0, temp4); let mut temp1;
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp0), 0x8c)); let mut temp2 = key1;
temp0 = _mm_xor_si128(temp0, temp4); let mut temp4 = _mm_setzero_si128();
temp1 = cast_128i(_mm_shuffle_ps(cast_128(temp1), cast_128(temp1), 0xff));
temp0 = _mm_xor_si128(temp0, temp1) macro_rules! block1 {
}; ($imm:expr) => {
temp1 = _mm_aeskeygenassist_si128(temp2, $imm);
state.rkeys[idx] = temp2;
idx += 1;
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp0), 0x10));
temp0 = _mm_xor_si128(temp0, temp4);
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp0), 0x8c));
temp0 = _mm_xor_si128(temp0, temp4);
temp1 = cast_128i(_mm_shuffle_ps(cast_128(temp1), cast_128(temp1), 0xff));
temp0 = _mm_xor_si128(temp0, temp1)
};
}
macro_rules! block2 {
($imm:expr) => {
temp1 = _mm_aeskeygenassist_si128(temp0, $imm);
state.rkeys[idx] = temp0;
idx += 1;
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp2), 0x10));
temp2 = _mm_xor_si128(temp2, temp4);
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp2), 0x8c));
temp2 = _mm_xor_si128(temp2, temp4);
temp1 = cast_128i(_mm_shuffle_ps(cast_128(temp1), cast_128(temp1), 0xaa));
temp2 = _mm_xor_si128(temp2, temp1)
};
}
block1!(0x01);
block2!(0x01);
block1!(0x02);
block2!(0x02);
block1!(0x04);
block2!(0x04);
block1!(0x08);
block2!(0x08);
block1!(0x10);
block2!(0x10);
block1!(0x20);
block2!(0x20);
block1!(0x40);
state.rkeys[idx] = temp0;
}
}
pub(crate) fn aes256ctr_squeezeblocks(out: &mut [u8], nblocks: usize, state: &mut Aes256CtrCtx) {
let mut idx = 0;
for _ in 0..nblocks {
unsafe {
aesni_encrypt4(&mut out[idx..], &mut state.n, &state.rkeys);
}
idx += 64
}
}
#[cfg(feature = "90s")]
pub(crate) fn aes256ctr_prf(out: &mut [u8], mut outlen: usize, seed: &[u8], nonce: u8) {
let mut buf = [0u8; 64];
let mut idx = 0;
let mut pad_nonce = [0u8; 12];
let mut state = unsafe { Aes256CtrCtx { rkeys: [_mm_setzero_si128(); 16], n: _mm_setzero_si128() } };
pad_nonce[0] = nonce;
aes256ctr_init(&mut state, seed, pad_nonce);
while outlen >= 64 {
unsafe {
aesni_encrypt4(&mut out[idx..], &mut state.n, &state.rkeys);
}
outlen -= 64;
idx += 64;
} }
macro_rules! block2 { if outlen != 0 {
($imm:expr) => { unsafe {
temp1 = _mm_aeskeygenassist_si128(temp0, $imm); aesni_encrypt4(&mut buf, &mut state.n, &state.rkeys);
state.rkeys[idx] = temp0; }
idx += 1; out[idx..][..outlen].copy_from_slice(&buf[..outlen]);
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp2), 0x10)); }
temp2 = _mm_xor_si128(temp2, temp4);
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp2), 0x8c));
temp2 = _mm_xor_si128(temp2, temp4);
temp1 = cast_128i(_mm_shuffle_ps(cast_128(temp1), cast_128(temp1), 0xaa));
temp2 = _mm_xor_si128(temp2, temp1)
};
}
block1!(0x01);
block2!(0x01);
block1!(0x02);
block2!(0x02);
block1!(0x04);
block2!(0x04);
block1!(0x08);
block2!(0x08);
block1!(0x10);
block2!(0x10);
block1!(0x20);
block2!(0x20);
block1!(0x40);
state.rkeys[idx] = temp0;
}
} }
pub(crate) fn aes256ctr_squeezeblocks(out: &mut[u8], nblocks: usize, state: &mut Aes256CtrCtx)
{
let mut idx = 0;
for _ in 0..nblocks {
unsafe { aesni_encrypt4(&mut out[idx..], &mut state.n, &state.rkeys); }
idx += 64
}
}
#[cfg(feature="90s")]
pub(crate) fn aes256ctr_prf(out: &mut[u8], mut outlen: usize, seed: &[u8], nonce: u8)
{
let mut buf = [0u8; 64];
let mut idx = 0;
let mut pad_nonce = [0u8; 12];
let mut state = unsafe{
Aes256CtrCtx{rkeys: [ _mm_setzero_si128(); 16], n: _mm_setzero_si128()}
};
pad_nonce[0] = nonce;
aes256ctr_init(&mut state, seed, pad_nonce);
while outlen >= 64 {
unsafe { aesni_encrypt4(&mut out[idx..], &mut state.n, &state.rkeys); }
outlen -= 64;
idx += 64;
}
if outlen != 0 {
unsafe { aesni_encrypt4(&mut buf, &mut state.n, &state.rkeys); }
out[idx..][..outlen].copy_from_slice(&buf[..outlen]);
}
}

87
third_party/kyber/src/avx2/align.rs vendored
View file

@ -1,11 +1,11 @@
#![allow(dead_code)] #![allow(dead_code)]
use core::arch::x86_64::*; use crate::avx2::rejsample::REJ_UNIFORM_AVX_NBLOCKS;
use crate::fips202::{SHAKE128_RATE, SHAKE256_RATE};
use crate::params::*; use crate::params::*;
use crate::poly::NOISE_NBLOCKS; use crate::poly::NOISE_NBLOCKS;
use crate::fips202::{SHAKE128_RATE, SHAKE256_RATE};
use crate::symmetric::*; use crate::symmetric::*;
use crate::avx2::rejsample::REJ_UNIFORM_AVX_NBLOCKS; use core::arch::x86_64::*;
// Buffer unions // Buffer unions
// #[derive(Copy, Clone)] // #[derive(Copy, Clone)]
@ -26,85 +26,76 @@ use crate::avx2::rejsample::REJ_UNIFORM_AVX_NBLOCKS;
#[derive(Copy, Clone)] #[derive(Copy, Clone)]
#[repr(C, align(32))] #[repr(C, align(32))]
pub union GenMatrixBuf { pub union GenMatrixBuf {
pub coeffs: [u8; REJ_UNIFORM_AVX_NBLOCKS*SHAKE128_RATE], pub coeffs: [u8; REJ_UNIFORM_AVX_NBLOCKS * SHAKE128_RATE],
pub vec: [__m256i; (REJ_UNIFORM_AVX_NBLOCKS*SHAKE128_RATE+31)/32] pub vec: [__m256i; (REJ_UNIFORM_AVX_NBLOCKS * SHAKE128_RATE + 31) / 32],
} }
impl GenMatrixBuf { impl GenMatrixBuf {
pub fn new() -> Self { pub fn new() -> Self {
Self { coeffs: [0u8; REJ_UNIFORM_AVX_NBLOCKS*SHAKE128_RATE]} Self { coeffs: [0u8; REJ_UNIFORM_AVX_NBLOCKS * SHAKE128_RATE] }
} }
} }
#[cfg(feature="90s")] #[cfg(feature = "90s")]
#[repr(C)] #[repr(C)]
pub union GenMatrixBuf90s { pub union GenMatrixBuf90s {
pub coeffs: pub coeffs: [u8; REJ_UNIFORM_AVX_NBLOCKS * XOF_BLOCKBYTES],
[u8; REJ_UNIFORM_AVX_NBLOCKS*XOF_BLOCKBYTES], pub vec: [__m256i; (REJ_UNIFORM_AVX_NBLOCKS * XOF_BLOCKBYTES + 31) / 32],
pub vec:
[__m256i; (REJ_UNIFORM_AVX_NBLOCKS*XOF_BLOCKBYTES+31)/32]
} }
#[cfg(feature="90s")] #[cfg(feature = "90s")]
impl GenMatrixBuf90s { impl GenMatrixBuf90s {
pub fn new() -> Self { pub fn new() -> Self {
Self { Self { coeffs: [0u8; REJ_UNIFORM_AVX_NBLOCKS * XOF_BLOCKBYTES] }
coeffs: [0u8; REJ_UNIFORM_AVX_NBLOCKS*XOF_BLOCKBYTES]
} }
}
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
pub fn checksum(&self) -> i16 { pub fn checksum(&self) -> i16 {
let mut out = 0; let mut out = 0;
for i in 0..REJ_UNIFORM_AVX_NBLOCKS*XOF_BLOCKBYTES { for i in 0..REJ_UNIFORM_AVX_NBLOCKS * XOF_BLOCKBYTES {
unsafe { out ^= self.coeffs[i] as i16; } unsafe {
out ^= self.coeffs[i] as i16;
}
}
out
} }
out
}
} }
#[repr(C)] #[repr(C)]
pub union IndcpaBuf { pub union IndcpaBuf {
pub coeffs: [u8; pub coeffs: [u8; (KYBER_ETA1 * KYBER_N / 4) / XOF_BLOCKBYTES * XOF_BLOCKBYTES + 32],
(KYBER_ETA1*KYBER_N/4) pub vec: [__m256i; ((KYBER_ETA1 * KYBER_N / 4) / XOF_BLOCKBYTES * XOF_BLOCKBYTES + 32 + 31) / 32],
/XOF_BLOCKBYTES*XOF_BLOCKBYTES+32],
pub vec: [__m256i;
((KYBER_ETA1*KYBER_N/4)
/XOF_BLOCKBYTES*XOF_BLOCKBYTES+32+31)/32]
} }
impl IndcpaBuf { impl IndcpaBuf {
pub fn new() -> Self { pub fn new() -> Self {
Self { Self {
coeffs: [0u8; coeffs: [0u8; (KYBER_ETA1 * KYBER_N / 4) / XOF_BLOCKBYTES * XOF_BLOCKBYTES + 32],
(KYBER_ETA1*KYBER_N/4) }
/XOF_BLOCKBYTES*XOF_BLOCKBYTES+32]
} }
}
} }
#[repr(C, align(8))] #[repr(C, align(8))]
pub union Eta2Buf { pub union Eta2Buf {
pub coeffs: [u8; KYBER_ETA2*KYBER_N/4], pub coeffs: [u8; KYBER_ETA2 * KYBER_N / 4],
pub vec: [__m256i; (KYBER_ETA2*KYBER_N/4+31)/32] pub vec: [__m256i; (KYBER_ETA2 * KYBER_N / 4 + 31) / 32],
} }
impl Eta2Buf { impl Eta2Buf {
pub fn new() -> Self { pub fn new() -> Self {
Self { coeffs: [0u8; KYBER_ETA2*KYBER_N/4] } Self { coeffs: [0u8; KYBER_ETA2 * KYBER_N / 4] }
} }
} }
#[derive(Copy, Clone)] #[derive(Copy, Clone)]
#[repr(C, align(8))] #[repr(C, align(8))]
pub union Eta4xBuf { pub union Eta4xBuf {
pub coeffs: [u8; NOISE_NBLOCKS*SHAKE256_RATE], pub coeffs: [u8; NOISE_NBLOCKS * SHAKE256_RATE],
pub vec: [__m256i; (NOISE_NBLOCKS*SHAKE256_RATE+31)/32] pub vec: [__m256i; (NOISE_NBLOCKS * SHAKE256_RATE + 31) / 32],
} }
impl Eta4xBuf { impl Eta4xBuf {
pub fn new() -> Self { pub fn new() -> Self {
Self { coeffs: [0u8; NOISE_NBLOCKS*SHAKE256_RATE] } Self { coeffs: [0u8; NOISE_NBLOCKS * SHAKE256_RATE] }
} }
} }

202
third_party/kyber/src/avx2/cbd.rs vendored
View file

@ -1,136 +1,128 @@
#![allow(non_snake_case, dead_code)] #![allow(non_snake_case, dead_code)]
use core::arch::x86_64::*; use crate::align::Eta4xBuf;
#[cfg(feature = "90s")]
use crate::align::IndcpaBuf;
use crate::params::KYBER_N; use crate::params::KYBER_N;
use crate::poly::*; use crate::poly::*;
use crate::align::Eta4xBuf; use core::arch::x86_64::*;
#[cfg(feature="90s")]
use crate::align::IndcpaBuf;
fn cbd2(r: &mut Poly, buf: &[__m256i]) { fn cbd2(r: &mut Poly, buf: &[__m256i]) {
unsafe { unsafe {
let mask55: __m256i = _mm256_set1_epi32(0x55555555); let mask55: __m256i = _mm256_set1_epi32(0x55555555);
let mask33: __m256i = _mm256_set1_epi32(0x33333333); let mask33: __m256i = _mm256_set1_epi32(0x33333333);
let mask03: __m256i = _mm256_set1_epi32(0x03030303); let mask03: __m256i = _mm256_set1_epi32(0x03030303);
let mask0F: __m256i = _mm256_set1_epi32(0x0F0F0F0F); let mask0F: __m256i = _mm256_set1_epi32(0x0F0F0F0F);
let (mut f0, mut f1, mut f2, mut f3); let (mut f0, mut f1, mut f2, mut f3);
for i in 0..(KYBER_N/64) { for i in 0..(KYBER_N / 64) {
f0 = _mm256_load_si256(&buf[i]); f0 = _mm256_load_si256(&buf[i]);
f1 = _mm256_srli_epi16(f0, 1); f1 = _mm256_srli_epi16(f0, 1);
f0 = _mm256_and_si256(mask55, f0); f0 = _mm256_and_si256(mask55, f0);
f1 = _mm256_and_si256(mask55, f1); f1 = _mm256_and_si256(mask55, f1);
f0 = _mm256_add_epi8(f0, f1); f0 = _mm256_add_epi8(f0, f1);
f1 = _mm256_srli_epi16(f0, 2); f1 = _mm256_srli_epi16(f0, 2);
f0 = _mm256_and_si256(mask33, f0); f0 = _mm256_and_si256(mask33, f0);
f1 = _mm256_and_si256(mask33, f1); f1 = _mm256_and_si256(mask33, f1);
f0 = _mm256_add_epi8(f0, mask33); f0 = _mm256_add_epi8(f0, mask33);
f0 = _mm256_sub_epi8(f0, f1); f0 = _mm256_sub_epi8(f0, f1);
f1 = _mm256_srli_epi16(f0, 4); f1 = _mm256_srli_epi16(f0, 4);
f0 = _mm256_and_si256(mask0F, f0); f0 = _mm256_and_si256(mask0F, f0);
f1 = _mm256_and_si256(mask0F, f1); f1 = _mm256_and_si256(mask0F, f1);
f0 = _mm256_sub_epi8(f0, mask03); f0 = _mm256_sub_epi8(f0, mask03);
f1 = _mm256_sub_epi8(f1, mask03); f1 = _mm256_sub_epi8(f1, mask03);
f2 = _mm256_unpacklo_epi8(f0, f1); f2 = _mm256_unpacklo_epi8(f0, f1);
f3 = _mm256_unpackhi_epi8(f0, f1); f3 = _mm256_unpackhi_epi8(f0, f1);
f0 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(f2)); f0 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(f2));
f1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(f2,1)); f1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(f2, 1));
f2 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(f3)); f2 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(f3));
f3 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(f3,1)); f3 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(f3, 1));
_mm256_store_si256(&mut r.vec[4*i+0], f0); _mm256_store_si256(&mut r.vec[4 * i + 0], f0);
_mm256_store_si256(&mut r.vec[4*i+1], f2); _mm256_store_si256(&mut r.vec[4 * i + 1], f2);
_mm256_store_si256(&mut r.vec[4*i+2], f1); _mm256_store_si256(&mut r.vec[4 * i + 2], f1);
_mm256_store_si256(&mut r.vec[4*i+3], f3); _mm256_store_si256(&mut r.vec[4 * i + 3], f3);
}
} }
}
} }
fn cbd3(r: &mut Poly, buf: &[u8]) { fn cbd3(r: &mut Poly, buf: &[u8]) {
unsafe { unsafe {
let (mut f0, mut f1, mut f2, mut f3); let (mut f0, mut f1, mut f2, mut f3);
let mask249: __m256i = _mm256_set1_epi32(0x249249); let mask249: __m256i = _mm256_set1_epi32(0x249249);
let mask6DB: __m256i = _mm256_set1_epi32(0x6DB6DB); let mask6DB: __m256i = _mm256_set1_epi32(0x6DB6DB);
let mask07: __m256i = _mm256_set1_epi32(7); let mask07: __m256i = _mm256_set1_epi32(7);
let mask70: __m256i = _mm256_set1_epi32(7 << 16); let mask70: __m256i = _mm256_set1_epi32(7 << 16);
let mask: __m256i = _mm256_set1_epi16(3); let mask: __m256i = _mm256_set1_epi16(3);
let shufbidx: __m256i = _mm256_set_epi8( let shufbidx: __m256i = _mm256_set_epi8(
-1,15,14,13,-1,12,11,10,-1, 9, 8, 7,-1, 6, 5, 4, -1, 15, 14, 13, -1, 12, 11, 10, -1, 9, 8, 7, -1, 6, 5, 4, -1, 11, 10, 9, -1, 8, 7, 6, -1, 5, 4, 3, -1, 2, 1, 0,
-1,11,10, 9,-1, 8, 7, 6,-1, 5, 4, 3,-1, 2, 1, 0 );
);
for i in 0..(KYBER_N/32) { for i in 0..(KYBER_N / 32) {
f0 = _mm256_loadu_si256(buf[24*i..].as_ptr() as *const __m256i); f0 = _mm256_loadu_si256(buf[24 * i..].as_ptr() as *const __m256i);
f0 = _mm256_permute4x64_epi64(f0,0x94); f0 = _mm256_permute4x64_epi64(f0, 0x94);
f0 = _mm256_shuffle_epi8(f0,shufbidx); f0 = _mm256_shuffle_epi8(f0, shufbidx);
f1 = _mm256_srli_epi32(f0,1); f1 = _mm256_srli_epi32(f0, 1);
f2 = _mm256_srli_epi32(f0,2); f2 = _mm256_srli_epi32(f0, 2);
f0 = _mm256_and_si256(mask249,f0); f0 = _mm256_and_si256(mask249, f0);
f1 = _mm256_and_si256(mask249,f1); f1 = _mm256_and_si256(mask249, f1);
f2 = _mm256_and_si256(mask249,f2); f2 = _mm256_and_si256(mask249, f2);
f0 = _mm256_add_epi32(f0,f1); f0 = _mm256_add_epi32(f0, f1);
f0 = _mm256_add_epi32(f0,f2); f0 = _mm256_add_epi32(f0, f2);
f1 = _mm256_srli_epi32(f0,3); f1 = _mm256_srli_epi32(f0, 3);
f0 = _mm256_add_epi32(f0,mask6DB); f0 = _mm256_add_epi32(f0, mask6DB);
f0 = _mm256_sub_epi32(f0,f1); f0 = _mm256_sub_epi32(f0, f1);
f1 = _mm256_slli_epi32(f0,10); f1 = _mm256_slli_epi32(f0, 10);
f2 = _mm256_srli_epi32(f0,12); f2 = _mm256_srli_epi32(f0, 12);
f3 = _mm256_srli_epi32(f0, 2); f3 = _mm256_srli_epi32(f0, 2);
f0 = _mm256_and_si256(f0,mask07); f0 = _mm256_and_si256(f0, mask07);
f1 = _mm256_and_si256(f1,mask70); f1 = _mm256_and_si256(f1, mask70);
f2 = _mm256_and_si256(f2,mask07); f2 = _mm256_and_si256(f2, mask07);
f3 = _mm256_and_si256(f3,mask70); f3 = _mm256_and_si256(f3, mask70);
f0 = _mm256_add_epi16(f0,f1); f0 = _mm256_add_epi16(f0, f1);
f1 = _mm256_add_epi16(f2,f3); f1 = _mm256_add_epi16(f2, f3);
f0 = _mm256_sub_epi16(f0,mask); f0 = _mm256_sub_epi16(f0, mask);
f1 = _mm256_sub_epi16(f1,mask); f1 = _mm256_sub_epi16(f1, mask);
f2 = _mm256_unpacklo_epi32(f0,f1); f2 = _mm256_unpacklo_epi32(f0, f1);
f3 = _mm256_unpackhi_epi32(f0,f1); f3 = _mm256_unpackhi_epi32(f0, f1);
f0 = _mm256_permute2x128_si256(f2,f3,0x20); f0 = _mm256_permute2x128_si256(f2, f3, 0x20);
f1 = _mm256_permute2x128_si256(f2,f3,0x31); f1 = _mm256_permute2x128_si256(f2, f3, 0x31);
_mm256_store_si256(&mut r.vec[2*i+0], f0); _mm256_store_si256(&mut r.vec[2 * i + 0], f0);
_mm256_store_si256(&mut r.vec[2*i+1], f1); _mm256_store_si256(&mut r.vec[2 * i + 1], f1);
}
} }
}
} }
pub fn poly_cbd_eta1(r: &mut Poly, buf: &Eta4xBuf) pub fn poly_cbd_eta1(r: &mut Poly, buf: &Eta4xBuf) {
{ unsafe {
unsafe { if cfg!(feature = "kyber512") {
if cfg!(feature="kyber512") { cbd3(r, &buf.coeffs)
cbd3(r, &buf.coeffs) } else {
} cbd2(r, &buf.vec)
else { }
cbd2(r, &buf.vec)
} }
}
} }
#[cfg(feature="90s")] #[cfg(feature = "90s")]
pub fn poly_cbd_eta1_90s(r: &mut Poly, buf: &IndcpaBuf) pub fn poly_cbd_eta1_90s(r: &mut Poly, buf: &IndcpaBuf) {
{ unsafe {
unsafe { if cfg!(feature = "kyber512") {
if cfg!(feature="kyber512") { cbd3(r, &buf.coeffs)
cbd3(r, &buf.coeffs) } else {
} cbd2(r, &buf.vec)
else { }
cbd2(r, &buf.vec)
} }
}
} }
pub fn poly_cbd_eta2(r: &mut Poly, buf: &[__m256i]) {
pub fn poly_cbd_eta2(r: &mut Poly, buf: &[__m256i]) cbd2(r, &buf)
{ }
cbd2(r, &buf)
}

159
third_party/kyber/src/avx2/consts.rs vendored
View file

@ -1,116 +1,69 @@
use core::arch::x86_64::*;
use crate::params::KYBER_Q; use crate::params::KYBER_Q;
use core::arch::x86_64::*;
pub(crate) const Q: i16 = KYBER_Q as i16; pub(crate) const Q: i16 = KYBER_Q as i16;
// pub(crate) const MONT: i16 = -1044; // 2^16 mod q // pub(crate) const MONT: i16 = -1044; // 2^16 mod q
pub(crate) const QINV: i16 = -3327; // q^-1 mod 2^16 pub(crate) const QINV: i16 = -3327; // q^-1 mod 2^16
pub(crate) const V: i16 = 20159; // floor(2^26/q + 0.5) pub(crate) const V: i16 = 20159; // floor(2^26/q + 0.5)
pub(crate) const FHI: i16 = 1441; // mont^2/128 pub(crate) const FHI: i16 = 1441; // mont^2/128
pub(crate) const FLO: i16 = -10079; // qinv*FHI pub(crate) const FLO: i16 = -10079; // qinv*FHI
pub(crate) const MONTSQHI: i16 = 1353; // mont^2 pub(crate) const MONTSQHI: i16 = 1353; // mont^2
pub(crate) const MONTSQLO: i16 = 20553; // qinv*MONTSQHI pub(crate) const MONTSQLO: i16 = 20553; // qinv*MONTSQHI
pub(crate) const MASK: i16 = 4095; pub(crate) const MASK: i16 = 4095;
pub(crate) const SHIFT: i16 = 32; pub(crate) const SHIFT: i16 = 32;
pub(crate) const _16XQ: usize = 0; pub(crate) const _16XQ: usize = 0;
pub(crate) const _16XQINV: usize = 16; pub(crate) const _16XQINV: usize = 16;
pub(crate) const _16XV: usize = 32; pub(crate) const _16XV: usize = 32;
pub(crate) const _16XFLO: usize = 48; pub(crate) const _16XFLO: usize = 48;
pub(crate) const _16XFHI: usize = 64; pub(crate) const _16XFHI: usize = 64;
pub(crate) const _16XMONTSQLO: usize = 80; pub(crate) const _16XMONTSQLO: usize = 80;
pub(crate) const _16XMONTSQHI: usize = 96; pub(crate) const _16XMONTSQHI: usize = 96;
pub(crate) const _16XMASK: usize = 112; pub(crate) const _16XMASK: usize = 112;
pub(crate) const _REVIDXB: usize = 128; pub(crate) const _REVIDXB: usize = 128;
pub(crate) const _REVIDXD: usize = 144; pub(crate) const _REVIDXD: usize = 144;
pub(crate) const _ZETAS_EXP: usize = 160; pub(crate) const _ZETAS_EXP: usize = 160;
pub(crate) const _16XSHIFT: usize = 624; pub(crate) const _16XSHIFT: usize = 624;
#[repr(C, align(32))] #[repr(C, align(32))]
pub union Qdata { pub union Qdata {
pub coeffs: [i16; 640], pub coeffs: [i16; 640],
pub vec: [__m256i; 40] pub vec: [__m256i; 40],
} }
pub const QDATA: Qdata = Qdata { coeffs: pub const QDATA: Qdata = Qdata {
[ Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, coeffs: [
QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV,
QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO,
V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, FLO, FLO, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO,
FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQHI,
FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI,
FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, MONTSQHI, MONTSQHI, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, 3854, 3340,
FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, 2826, 2312, 1798, 1284, 770, 256, 3854, 3340, 2826, 2312, 1798, 1284, 770, 256, 7, 0, 6, 0, 5, 0, 4, 0, 3, 0, 2, 0, 1, 0, 0, 0,
MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, 31498, 31498, 31498, 31498, -758, -758, -758, -758, 5237, 5237, 5237, 5237, 1397, 1397, 1397, 1397, 14745, 14745, 14745, 14745,
MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, -359, -359, -359, -359, -359, -359, -359, -359,
MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, -359, -359, -359, -359, -359, -359, -359, -359, 13525, 13525, 13525, 13525, 13525, 13525, 13525, 13525, -12402, -12402, -12402,
MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, -12402, -12402, -12402, -12402, -12402, 1493, 1493, 1493, 1493, 1493, 1493, 1493, 1493, 1422, 1422, 1422, 1422, 1422, 1422, 1422,
MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, 1422, -20907, -20907, -20907, -20907, 27758, 27758, 27758, 27758, -3799, -3799, -3799, -3799, -15690, -15690, -15690, -15690, -171,
MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, -171, -171, -171, 622, 622, 622, 622, 1577, 1577, 1577, 1577, 182, 182, 182, 182, -5827, -5827, 17363, 17363, -26360, -26360,
MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, -29057, -29057, 5571, 5571, -1102, -1102, 21438, 21438, -26242, -26242, 573, 573, -1325, -1325, 264, 264, 383, 383, -829, -829,
MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, 1458, 1458, -1602, -1602, -130, -130, -5689, -6516, 1496, 30967, -23565, 20179, 20710, 25080, -12796, 26616, 16064, -12442, 9134,
MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, -650, -25986, 27837, 1223, 652, -552, 1015, -1293, 1491, -282, -1544, 516, -8, -320, -666, -1618, -1162, 126, 1469, -335, -11477,
MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, -32227, 20494, -27738, 945, -14883, 6182, 32010, 10631, 29175, -28762, -18486, 17560, -14430, -5276, -1103, 555, -1251, 1550, 422,
3854, 3340, 2826, 2312, 1798, 1284, 770, 256, 177, -291, 1574, -246, 1159, -777, -602, -1590, -872, 418, -156, 11182, 13387, -14233, -21655, 13131, -4587, 23092, 5493, -32502,
3854, 3340, 2826, 2312, 1798, 1284, 770, 256, 30317, -18741, 12639, 20100, 18525, 19529, -12619, 430, 843, 871, 105, 587, -235, -460, 1653, 778, -147, 1483, 1119, 644, 349, 329,
7, 0, 6, 0, 5, 0, 4, 0, 3, 0, 2, 0, 1, 0, 0, 0, -75, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, -1517, -1517, -1517, -1517, -1517, -1517,
31498, 31498, 31498, 31498, -758, -758, -758, -758, -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, 28191, 28191, 28191, 28191, 28191, 28191, 28191, 28191,
5237, 5237, 5237, 5237, 1397, 1397, 1397, 1397, -16694, -16694, -16694, -16694, -16694, -16694, -16694, -16694, 287, 287, 287, 287, 287, 287, 287, 287, 202, 202, 202, 202, 202,
14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 202, 202, 202, 10690, 10690, 10690, 10690, 1358, 1358, 1358, 1358, -11202, -11202, -11202, -11202, 31164, 31164, 31164, 31164, 962,
14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 962, 962, 962, -1202, -1202, -1202, -1202, -1474, -1474, -1474, -1474, 1468, 1468, 1468, 1468, -28073, -28073, 24313, 24313,
-359, -359, -359, -359, -359, -359, -359, -359, -10532, -10532, 8800, 8800, 18426, 18426, 8859, 8859, 26675, 26675, -16163, -16163, -681, -681, 1017, 1017, 732, 732, 608, 608,
-359, -359, -359, -359, -359, -359, -359, -359, -1542, -1542, 411, 411, -205, -205, -1571, -1571, 19883, -28250, -15887, -8898, -28309, 9075, -30199, 18249, 13426, 14017, -29156,
13525, 13525, 13525, 13525, 13525, 13525, 13525, 13525, -12757, 16832, 4311, -24155, -17915, -853, -90, -271, 830, 107, -1421, -247, -951, -398, 961, -1508, -725, 448, -1065, 677, -1275,
-12402, -12402, -12402, -12402, -12402, -12402, -12402, -12402, -31183, 25435, -7382, 24391, -20927, 10946, 24214, 16989, 10335, -7934, -22502, 10906, 31636, 28644, 23998, -17422, 817, 603, 1322,
1493, 1493, 1493, 1493, 1493, 1493, 1493, 1493, -1465, -1215, 1218, -874, -1187, -1185, -1278, -1510, -870, -108, 996, 958, 1522, 20297, 2146, 15355, -32384, -6280, -14903,
1422, 1422, 1422, 1422, 1422, 1422, 1422, 1422, -11044, 14469, -21498, -20198, 23210, -17442, -23860, -20257, 7756, 23132, 1097, 610, -1285, 384, -136, -1335, 220, -1659, -1530,
-20907, -20907, -20907, -20907, 27758, 27758, 27758, 27758, 794, -854, 478, -308, 991, -1460, 1628, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT,
-3799, -3799, -3799, -3799, -15690, -15690, -15690, -15690, SHIFT, SHIFT, SHIFT,
-171, -171, -171, -171, 622, 622, 622, 622, ],
1577, 1577, 1577, 1577, 182, 182, 182, 182,
-5827, -5827, 17363, 17363, -26360, -26360, -29057, -29057,
5571, 5571, -1102, -1102, 21438, 21438, -26242, -26242,
573, 573, -1325, -1325, 264, 264, 383, 383,
-829, -829, 1458, 1458, -1602, -1602, -130, -130,
-5689, -6516, 1496, 30967, -23565, 20179, 20710, 25080,
-12796, 26616, 16064, -12442, 9134, -650, -25986, 27837,
1223, 652, -552, 1015, -1293, 1491, -282, -1544,
516, -8, -320, -666, -1618, -1162, 126, 1469,
-335, -11477, -32227, 20494, -27738, 945, -14883, 6182,
32010, 10631, 29175, -28762, -18486, 17560, -14430, -5276,
-1103, 555, -1251, 1550, 422, 177, -291, 1574,
-246, 1159, -777, -602, -1590, -872, 418, -156,
11182, 13387, -14233, -21655, 13131, -4587, 23092, 5493,
-32502, 30317, -18741, 12639, 20100, 18525, 19529, -12619,
430, 843, 871, 105, 587, -235, -460, 1653,
778, -147, 1483, 1119, 644, 349, 329, -75,
787, 787, 787, 787, 787, 787, 787, 787,
787, 787, 787, 787, 787, 787, 787, 787,
-1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517,
-1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517,
28191, 28191, 28191, 28191, 28191, 28191, 28191, 28191,
-16694, -16694, -16694, -16694, -16694, -16694, -16694, -16694,
287, 287, 287, 287, 287, 287, 287, 287,
202, 202, 202, 202, 202, 202, 202, 202,
10690, 10690, 10690, 10690, 1358, 1358, 1358, 1358,
-11202, -11202, -11202, -11202, 31164, 31164, 31164, 31164,
962, 962, 962, 962, -1202, -1202, -1202, -1202,
-1474, -1474, -1474, -1474, 1468, 1468, 1468, 1468,
-28073, -28073, 24313, 24313, -10532, -10532, 8800, 8800,
18426, 18426, 8859, 8859, 26675, 26675, -16163, -16163,
-681, -681, 1017, 1017, 732, 732, 608, 608,
-1542, -1542, 411, 411, -205, -205, -1571, -1571,
19883, -28250, -15887, -8898, -28309, 9075, -30199, 18249,
13426, 14017, -29156, -12757, 16832, 4311, -24155, -17915,
-853, -90, -271, 830, 107, -1421, -247, -951,
-398, 961, -1508, -725, 448, -1065, 677, -1275,
-31183, 25435, -7382, 24391, -20927, 10946, 24214, 16989,
10335, -7934, -22502, 10906, 31636, 28644, 23998, -17422,
817, 603, 1322, -1465, -1215, 1218, -874, -1187,
-1185, -1278, -1510, -870, -108, 996, 958, 1522,
20297, 2146, 15355, -32384, -6280, -14903, -11044, 14469,
-21498, -20198, 23210, -17442, -23860, -20257, 7756, 23132,
1097, 610, -1285, 384, -136, -1335, 220, -1659,
-1530, 794, -854, 478, -308, 991, -1460, 1628,
SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT,
SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT ]
}; };

826
third_party/kyber/src/avx2/fips202.rs vendored
View file

@ -4,12 +4,11 @@ use crate::symmetric::KeccakState;
pub(crate) const SHAKE128_RATE: usize = 168; pub(crate) const SHAKE128_RATE: usize = 168;
pub(crate) const SHAKE256_RATE: usize = 136; pub(crate) const SHAKE256_RATE: usize = 136;
const SHA3_256_RATE: usize = 136; const SHA3_256_RATE: usize = 136;
const SHA3_512_RATE: usize = 72; const SHA3_512_RATE: usize = 72;
const NROUNDS: usize = 24; const NROUNDS: usize = 24;
fn rol(a: u64, offset: u64) -> u64 fn rol(a: u64, offset: u64) -> u64 {
{ (a << offset) ^ (a >> (64 - offset))
(a << offset) ^ (a >> (64-offset))
} }
// Name: load64 // Name: load64
@ -19,13 +18,12 @@ fn rol(a: u64, offset: u64) -> u64
// Arguments: - const [u8] x: input byte array // Arguments: - const [u8] x: input byte array
// //
// Returns the loaded 64-bit unsigned integer // Returns the loaded 64-bit unsigned integer
pub fn load64(x: &[u8]) -> u64 pub fn load64(x: &[u8]) -> u64 {
{ let mut r = 0u64;
let mut r = 0u64; for i in 0..8 {
for i in 0..8 { r |= (x[i] as u64) << (8 * i);
r |= (x[i] as u64) << (8 * i); }
} r
r
} }
// Name: store64 // Name: store64
@ -34,40 +32,39 @@ pub fn load64(x: &[u8]) -> u64
// //
// Arguments: - [u8] x: the output byte array // Arguments: - [u8] x: the output byte array
// - u64 u: input 64-bit unsigned integer // - u64 u: input 64-bit unsigned integer
pub fn store64(x: &mut[u8], mut u: u64) pub fn store64(x: &mut [u8], mut u: u64) {
{ for i in x.iter_mut().take(8) {
for i in x.iter_mut().take(8) { *i = u as u8;
*i = u as u8; u >>= 8;
u >>= 8; }
}
} }
// Keccak round constants // Keccak round constants
const KECCAKF_ROUNDCONSTANTS: [u64; NROUNDS] = [ const KECCAKF_ROUNDCONSTANTS: [u64; NROUNDS] = [
0x0000000000000001, 0x0000000000000001,
0x0000000000008082, 0x0000000000008082,
0x800000000000808a, 0x800000000000808a,
0x8000000080008000, 0x8000000080008000,
0x000000000000808b, 0x000000000000808b,
0x0000000080000001, 0x0000000080000001,
0x8000000080008081, 0x8000000080008081,
0x8000000000008009, 0x8000000000008009,
0x000000000000008a, 0x000000000000008a,
0x0000000000000088, 0x0000000000000088,
0x0000000080008009, 0x0000000080008009,
0x000000008000000a, 0x000000008000000a,
0x000000008000808b, 0x000000008000808b,
0x800000000000008b, 0x800000000000008b,
0x8000000000008089, 0x8000000000008089,
0x8000000000008003, 0x8000000000008003,
0x8000000000008002, 0x8000000000008002,
0x8000000000000080, 0x8000000000000080,
0x000000000000800a, 0x000000000000800a,
0x800000008000000a, 0x800000008000000a,
0x8000000080008081, 0x8000000080008081,
0x8000000000008080, 0x8000000000008080,
0x0000000080000001, 0x0000000080000001,
0x8000000080008008 0x8000000080008008,
]; ];
// Name: KeccakF1600_StatePermute // Name: KeccakF1600_StatePermute
@ -75,252 +72,251 @@ const KECCAKF_ROUNDCONSTANTS: [u64; NROUNDS] = [
// Description: The Keccak F1600 Permutation // Description: The Keccak F1600 Permutation
// //
// Arguments: - u64 * state: in/output Keccak state // Arguments: - u64 * state: in/output Keccak state
pub(crate) fn keccakf1600_statepermute(state: &mut[u64]) pub(crate) fn keccakf1600_statepermute(state: &mut [u64]) {
{ //copyFromState(A, state)
//copyFromState(A, state) let mut aba = state[0];
let mut aba = state[ 0]; let mut abe = state[1];
let mut abe = state[ 1]; let mut abi = state[2];
let mut abi = state[ 2]; let mut abo = state[3];
let mut abo = state[ 3]; let mut abu = state[4];
let mut abu = state[ 4]; let mut aga = state[5];
let mut aga = state[ 5]; let mut age = state[6];
let mut age = state[ 6]; let mut agi = state[7];
let mut agi = state[ 7]; let mut ago = state[8];
let mut ago = state[ 8]; let mut agu = state[9];
let mut agu = state[ 9]; let mut aka = state[10];
let mut aka = state[10]; let mut ake = state[11];
let mut ake = state[11]; let mut aki = state[12];
let mut aki = state[12]; let mut ako = state[13];
let mut ako = state[13]; let mut aku = state[14];
let mut aku = state[14]; let mut ama = state[15];
let mut ama = state[15]; let mut ame = state[16];
let mut ame = state[16]; let mut ami = state[17];
let mut ami = state[17]; let mut amo = state[18];
let mut amo = state[18]; let mut amu = state[19];
let mut amu = state[19]; let mut asa = state[20];
let mut asa = state[20]; let mut ase = state[21];
let mut ase = state[21]; let mut asi = state[22];
let mut asi = state[22]; let mut aso = state[23];
let mut aso = state[23]; let mut asu = state[24];
let mut asu = state[24];
for round in (0..NROUNDS).step_by(2) { for round in (0..NROUNDS).step_by(2) {
// prepareTheta // prepareTheta
let mut bca = aba^aga^aka^ama^asa; let mut bca = aba ^ aga ^ aka ^ ama ^ asa;
let mut bce = abe^age^ake^ame^ase; let mut bce = abe ^ age ^ ake ^ ame ^ ase;
let mut bci = abi^agi^aki^ami^asi; let mut bci = abi ^ agi ^ aki ^ ami ^ asi;
let mut bco = abo^ago^ako^amo^aso; let mut bco = abo ^ ago ^ ako ^ amo ^ aso;
let mut bcu = abu^agu^aku^amu^asu; let mut bcu = abu ^ agu ^ aku ^ amu ^ asu;
//thetaRhoPiChiIotaPrepareTheta(round , A, E) //thetaRhoPiChiIotaPrepareTheta(round , A, E)
let mut da = bcu^rol(bce, 1); let mut da = bcu ^ rol(bce, 1);
let mut de = bca^rol(bci, 1); let mut de = bca ^ rol(bci, 1);
let mut di = bce^rol(bco, 1); let mut di = bce ^ rol(bco, 1);
let mut d_o = bci^rol(bcu, 1); let mut d_o = bci ^ rol(bcu, 1);
let mut du = bco^rol(bca, 1); let mut du = bco ^ rol(bca, 1);
aba ^= da; aba ^= da;
bca = aba; bca = aba;
age ^= de; age ^= de;
bce = rol(age, 44); bce = rol(age, 44);
aki ^= di; aki ^= di;
bci = rol(aki, 43); bci = rol(aki, 43);
amo ^= d_o; amo ^= d_o;
bco = rol(amo, 21); bco = rol(amo, 21);
asu ^= du; asu ^= du;
bcu = rol(asu, 14); bcu = rol(asu, 14);
let mut eba = bca ^((!bce)& bci ); let mut eba = bca ^ ((!bce) & bci);
eba ^= KECCAKF_ROUNDCONSTANTS[round]; eba ^= KECCAKF_ROUNDCONSTANTS[round];
let mut ebe = bce ^((!bci)& bco ); let mut ebe = bce ^ ((!bci) & bco);
let mut ebi = bci ^((!bco)& bcu ); let mut ebi = bci ^ ((!bco) & bcu);
let mut ebo = bco ^((!bcu)& bca ); let mut ebo = bco ^ ((!bcu) & bca);
let mut ebu = bcu ^((!bca)& bce ); let mut ebu = bcu ^ ((!bca) & bce);
abo ^= d_o; abo ^= d_o;
bca = rol(abo, 28); bca = rol(abo, 28);
agu ^= du; agu ^= du;
bce = rol(agu, 20); bce = rol(agu, 20);
aka ^= da; aka ^= da;
bci = rol(aka, 3); bci = rol(aka, 3);
ame ^= de; ame ^= de;
bco = rol(ame, 45); bco = rol(ame, 45);
asi ^= di; asi ^= di;
bcu = rol(asi, 61); bcu = rol(asi, 61);
let mut ega = bca ^((!bce)& bci ); let mut ega = bca ^ ((!bce) & bci);
let mut ege = bce ^((!bci)& bco ); let mut ege = bce ^ ((!bci) & bco);
let mut egi = bci ^((!bco)& bcu ); let mut egi = bci ^ ((!bco) & bcu);
let mut ego = bco ^((!bcu)& bca ); let mut ego = bco ^ ((!bcu) & bca);
let mut egu = bcu ^((!bca)& bce ); let mut egu = bcu ^ ((!bca) & bce);
abe ^= de; abe ^= de;
bca = rol(abe, 1); bca = rol(abe, 1);
agi ^= di; agi ^= di;
bce = rol(agi, 6); bce = rol(agi, 6);
ako ^= d_o; ako ^= d_o;
bci = rol(ako, 25); bci = rol(ako, 25);
amu ^= du; amu ^= du;
bco = rol(amu, 8); bco = rol(amu, 8);
asa ^= da; asa ^= da;
bcu = rol(asa, 18); bcu = rol(asa, 18);
let mut eka = bca ^((!bce)& bci ); let mut eka = bca ^ ((!bce) & bci);
let mut eke = bce ^((!bci)& bco ); let mut eke = bce ^ ((!bci) & bco);
let mut eki = bci ^((!bco)& bcu ); let mut eki = bci ^ ((!bco) & bcu);
let mut eko = bco ^((!bcu)& bca ); let mut eko = bco ^ ((!bcu) & bca);
let mut eku = bcu ^((!bca)& bce ); let mut eku = bcu ^ ((!bca) & bce);
abu ^= du; abu ^= du;
bca = rol(abu, 27); bca = rol(abu, 27);
aga ^= da; aga ^= da;
bce = rol(aga, 36); bce = rol(aga, 36);
ake ^= de; ake ^= de;
bci = rol(ake, 10); bci = rol(ake, 10);
ami ^= di; ami ^= di;
bco = rol(ami, 15); bco = rol(ami, 15);
aso ^= d_o; aso ^= d_o;
bcu = rol(aso, 56); bcu = rol(aso, 56);
let mut ema = bca ^((!bce)& bci ); let mut ema = bca ^ ((!bce) & bci);
let mut eme = bce ^((!bci)& bco ); let mut eme = bce ^ ((!bci) & bco);
let mut emi = bci ^((!bco)& bcu ); let mut emi = bci ^ ((!bco) & bcu);
let mut emo = bco ^((!bcu)& bca ); let mut emo = bco ^ ((!bcu) & bca);
let mut emu = bcu ^((!bca)& bce ); let mut emu = bcu ^ ((!bca) & bce);
abi ^= di; abi ^= di;
bca = rol(abi, 62); bca = rol(abi, 62);
ago ^= d_o; ago ^= d_o;
bce = rol(ago, 55); bce = rol(ago, 55);
aku ^= du; aku ^= du;
bci = rol(aku, 39); bci = rol(aku, 39);
ama ^= da; ama ^= da;
bco = rol(ama, 41); bco = rol(ama, 41);
ase ^= de; ase ^= de;
bcu = rol(ase, 2); bcu = rol(ase, 2);
let mut esa = bca ^((!bce)& bci ); let mut esa = bca ^ ((!bce) & bci);
let mut ese = bce ^((!bci)& bco ); let mut ese = bce ^ ((!bci) & bco);
let mut esi = bci ^((!bco)& bcu ); let mut esi = bci ^ ((!bco) & bcu);
let mut eso = bco ^((!bcu)& bca ); let mut eso = bco ^ ((!bcu) & bca);
let mut esu = bcu ^((!bca)& bce ); let mut esu = bcu ^ ((!bca) & bce);
// prepareTheta // prepareTheta
bca = eba^ega^eka^ema^esa; bca = eba ^ ega ^ eka ^ ema ^ esa;
bce = ebe^ege^eke^eme^ese; bce = ebe ^ ege ^ eke ^ eme ^ ese;
bci = ebi^egi^eki^emi^esi; bci = ebi ^ egi ^ eki ^ emi ^ esi;
bco = ebo^ego^eko^emo^eso; bco = ebo ^ ego ^ eko ^ emo ^ eso;
bcu = ebu^egu^eku^emu^esu; bcu = ebu ^ egu ^ eku ^ emu ^ esu;
//thetaRhoPiChiIotaPrepareTheta(round+1, E, A) //thetaRhoPiChiIotaPrepareTheta(round+1, E, A)
da = bcu^rol(bce, 1); da = bcu ^ rol(bce, 1);
de = bca^rol(bci, 1); de = bca ^ rol(bci, 1);
di = bce^rol(bco, 1); di = bce ^ rol(bco, 1);
d_o = bci^rol(bcu, 1); d_o = bci ^ rol(bcu, 1);
du = bco^rol(bca, 1); du = bco ^ rol(bca, 1);
eba ^= da; eba ^= da;
bca = eba; bca = eba;
ege ^= de; ege ^= de;
bce = rol(ege, 44); bce = rol(ege, 44);
eki ^= di; eki ^= di;
bci = rol(eki, 43); bci = rol(eki, 43);
emo ^= d_o; emo ^= d_o;
bco = rol(emo, 21); bco = rol(emo, 21);
esu ^= du; esu ^= du;
bcu = rol(esu, 14); bcu = rol(esu, 14);
aba = bca ^((!bce)& bci ); aba = bca ^ ((!bce) & bci);
aba ^= KECCAKF_ROUNDCONSTANTS[round+1]; aba ^= KECCAKF_ROUNDCONSTANTS[round + 1];
abe = bce ^((!bci)& bco ); abe = bce ^ ((!bci) & bco);
abi = bci ^((!bco)& bcu ); abi = bci ^ ((!bco) & bcu);
abo = bco ^((!bcu)& bca ); abo = bco ^ ((!bcu) & bca);
abu = bcu ^((!bca)& bce ); abu = bcu ^ ((!bca) & bce);
ebo ^= d_o; ebo ^= d_o;
bca = rol(ebo, 28); bca = rol(ebo, 28);
egu ^= du; egu ^= du;
bce = rol(egu, 20); bce = rol(egu, 20);
eka ^= da; eka ^= da;
bci = rol(eka, 3); bci = rol(eka, 3);
eme ^= de; eme ^= de;
bco = rol(eme, 45); bco = rol(eme, 45);
esi ^= di; esi ^= di;
bcu = rol(esi, 61); bcu = rol(esi, 61);
aga = bca ^((!bce)& bci ); aga = bca ^ ((!bce) & bci);
age = bce ^((!bci)& bco ); age = bce ^ ((!bci) & bco);
agi = bci ^((!bco)& bcu ); agi = bci ^ ((!bco) & bcu);
ago = bco ^((!bcu)& bca ); ago = bco ^ ((!bcu) & bca);
agu = bcu ^((!bca)& bce ); agu = bcu ^ ((!bca) & bce);
ebe ^= de; ebe ^= de;
bca = rol(ebe, 1); bca = rol(ebe, 1);
egi ^= di; egi ^= di;
bce = rol(egi, 6); bce = rol(egi, 6);
eko ^= d_o; eko ^= d_o;
bci = rol(eko, 25); bci = rol(eko, 25);
emu ^= du; emu ^= du;
bco = rol(emu, 8); bco = rol(emu, 8);
esa ^= da; esa ^= da;
bcu = rol(esa, 18); bcu = rol(esa, 18);
aka = bca ^((!bce)& bci ); aka = bca ^ ((!bce) & bci);
ake = bce ^((!bci)& bco ); ake = bce ^ ((!bci) & bco);
aki = bci ^((!bco)& bcu ); aki = bci ^ ((!bco) & bcu);
ako = bco ^((!bcu)& bca ); ako = bco ^ ((!bcu) & bca);
aku = bcu ^((!bca)& bce ); aku = bcu ^ ((!bca) & bce);
ebu ^= du; ebu ^= du;
bca = rol(ebu, 27); bca = rol(ebu, 27);
ega ^= da; ega ^= da;
bce = rol(ega, 36); bce = rol(ega, 36);
eke ^= de; eke ^= de;
bci = rol(eke, 10); bci = rol(eke, 10);
emi ^= di; emi ^= di;
bco = rol(emi, 15); bco = rol(emi, 15);
eso ^= d_o; eso ^= d_o;
bcu = rol(eso, 56); bcu = rol(eso, 56);
ama = bca ^((!bce)& bci ); ama = bca ^ ((!bce) & bci);
ame = bce ^((!bci)& bco ); ame = bce ^ ((!bci) & bco);
ami = bci ^((!bco)& bcu ); ami = bci ^ ((!bco) & bcu);
amo = bco ^((!bcu)& bca ); amo = bco ^ ((!bcu) & bca);
amu = bcu ^((!bca)& bce ); amu = bcu ^ ((!bca) & bce);
ebi ^= di; ebi ^= di;
bca = rol(ebi, 62); bca = rol(ebi, 62);
ego ^= d_o; ego ^= d_o;
bce = rol(ego, 55); bce = rol(ego, 55);
eku ^= du; eku ^= du;
bci = rol(eku, 39); bci = rol(eku, 39);
ema ^= da; ema ^= da;
bco = rol(ema, 41); bco = rol(ema, 41);
ese ^= de; ese ^= de;
bcu = rol(ese, 2); bcu = rol(ese, 2);
asa = bca ^((!bce)& bci ); asa = bca ^ ((!bce) & bci);
ase = bce ^((!bci)& bco ); ase = bce ^ ((!bci) & bco);
asi = bci ^((!bco)& bcu ); asi = bci ^ ((!bco) & bcu);
aso = bco ^((!bcu)& bca ); aso = bco ^ ((!bcu) & bca);
asu = bcu ^((!bca)& bce ); asu = bcu ^ ((!bca) & bce);
} }
state[ 0] = aba; state[0] = aba;
state[ 1] = abe; state[1] = abe;
state[ 2] = abi; state[2] = abi;
state[ 3] = abo; state[3] = abo;
state[ 4] = abu; state[4] = abu;
state[ 5] = aga; state[5] = aga;
state[ 6] = age; state[6] = age;
state[ 7] = agi; state[7] = agi;
state[ 8] = ago; state[8] = ago;
state[ 9] = agu; state[9] = agu;
state[10] = aka; state[10] = aka;
state[11] = ake; state[11] = ake;
state[12] = aki; state[12] = aki;
state[13] = ako; state[13] = ako;
state[14] = aku; state[14] = aku;
state[15] = ama; state[15] = ama;
state[16] = ame; state[16] = ame;
state[17] = ami; state[17] = ami;
state[18] = amo; state[18] = amo;
state[19] = amu; state[19] = amu;
state[20] = asa; state[20] = asa;
state[21] = ase; state[21] = ase;
state[22] = asi; state[22] = asi;
state[23] = aso; state[23] = aso;
state[24] = asu; state[24] = asu;
} }
// Name: keccak_absorb // Name: keccak_absorb
@ -333,29 +329,22 @@ pub(crate) fn keccakf1600_statepermute(state: &mut[u64])
// - usize r: rate in bytes (e.g., 168 for SHAKE128) // - usize r: rate in bytes (e.g., 168 for SHAKE128)
// - const [u8] input: pointer to input to be absorbed into s // - const [u8] input: pointer to input to be absorbed into s
// - u64 inlen: length of input in bytes // - u64 inlen: length of input in bytes
pub(crate) fn keccak_absorb( pub(crate) fn keccak_absorb(s: &mut [u64], mut pos: usize, r: usize, input: &[u8], mut inlen: usize) -> usize {
s: &mut[u64], let mut idx = 0usize;
mut pos: usize, while pos + inlen >= r {
r: usize, for i in pos..r {
input: &[u8], s[i / 8] ^= (input[idx] as u64) << 8 * (i % 8);
mut inlen: usize idx += 1;
) -> usize }
{ inlen -= r - pos;
let mut idx = 0usize; keccakf1600_statepermute(s);
while pos+inlen >= r { pos = 0;
for i in pos..r {
s[i/8] ^= (input[idx] as u64) << 8 * (i%8);
idx += 1;
} }
inlen -= r-pos; let new_pos = pos + inlen;
keccakf1600_statepermute(s); for i in pos..new_pos {
pos = 0; s[i / 8] ^= (input[idx] as u64) << 8 * (i % 8);
} }
let new_pos = pos+inlen; new_pos
for i in pos..new_pos {
s[i/8] ^= (input[idx] as u64) << 8 * (i%8);
}
new_pos
} }
// Name: keccak_squeezeblocks // Name: keccak_squeezeblocks
@ -368,17 +357,16 @@ pub(crate) fn keccak_absorb(
// - u64 nblocks: number of blocks to be squeezed (written to h) // - u64 nblocks: number of blocks to be squeezed (written to h)
// - u64 *s: in/output Keccak state // - u64 *s: in/output Keccak state
// - usize r: rate in bytes (e.g., 168 for SHAKE128) // - usize r: rate in bytes (e.g., 168 for SHAKE128)
pub(crate) fn keccak_squeezeblocks(h: &mut[u8], mut nblocks: usize, s: &mut [u64], r: usize) pub(crate) fn keccak_squeezeblocks(h: &mut [u8], mut nblocks: usize, s: &mut [u64], r: usize) {
{ let mut idx = 0usize;
let mut idx = 0usize; while nblocks > 0 {
while nblocks > 0 { keccakf1600_statepermute(s);
keccakf1600_statepermute(s); for i in 0..r / 8 {
for i in 0..r/8 { store64(&mut h[idx + 8 * i..], s[i])
store64(&mut h[idx+8*i..], s[i]) }
idx += r;
nblocks -= 1;
} }
idx += r;
nblocks -= 1;
}
} }
// Name: shake128_absorb // Name: shake128_absorb
@ -389,10 +377,9 @@ pub(crate) fn keccak_squeezeblocks(h: &mut[u8], mut nblocks: usize, s: &mut [u64
// Arguments: - u64 *s: (uninitialized) output Keccak state // Arguments: - u64 *s: (uninitialized) output Keccak state
// - const [u8] input: input to be absorbed into s // - const [u8] input: input to be absorbed into s
// - u64 inputByteLen: length of input in bytes // - u64 inputByteLen: length of input in bytes
pub(crate) fn shake128_absorb(state: &mut KeccakState, input: &[u8], inlen: usize) pub(crate) fn shake128_absorb(state: &mut KeccakState, input: &[u8], inlen: usize) {
{ let pos = state.pos;
let pos = state.pos; state.pos = keccak_absorb(&mut state.s, pos, SHAKE128_RATE, input, inlen);
state.pos =keccak_absorb(&mut state.s, pos, SHAKE128_RATE, input, inlen);
} }
// Name: shake128_squeezeblocks // Name: shake128_squeezeblocks
@ -405,9 +392,8 @@ pub(crate) fn shake128_absorb(state: &mut KeccakState, input: &[u8], inlen: usiz
// Arguments: - [u8] out: pointer to output blocks // Arguments: - [u8] out: pointer to output blocks
// - u64 nblocks: number of blocks to be squeezed (written to output) // - u64 nblocks: number of blocks to be squeezed (written to output)
// - KeccakState state: pointer to input/output Keccak state // - KeccakState state: pointer to input/output Keccak state
pub(crate) fn shake128_squeezeblocks(out: &mut[u8], nblocks: usize, state: &mut KeccakState) pub(crate) fn shake128_squeezeblocks(out: &mut [u8], nblocks: usize, state: &mut KeccakState) {
{ keccak_squeezeblocks(out, nblocks, &mut state.s, SHAKE128_RATE);
keccak_squeezeblocks(out, nblocks, &mut state.s, SHAKE128_RATE);
} }
// Name: shake256 // Name: shake256
@ -418,16 +404,15 @@ pub(crate) fn shake128_squeezeblocks(out: &mut[u8], nblocks: usize, state: &mut
// - usize outlen: requested output length in bytes // - usize outlen: requested output length in bytes
// - [u8] input: input // - [u8] input: input
// - usize inlen: length of input in bytes // - usize inlen: length of input in bytes
pub(crate) fn shake256(out: &mut[u8], mut outlen: usize, input: &[u8], inlen: usize) pub(crate) fn shake256(out: &mut [u8], mut outlen: usize, input: &[u8], inlen: usize) {
{ let mut state = KeccakState::new();
let mut state = KeccakState::new(); let mut idx = 0;
let mut idx = 0; shake256_absorb_once(&mut state, input, inlen);
shake256_absorb_once(&mut state, input, inlen); let nblocks = outlen / SHAKE256_RATE;
let nblocks = outlen/SHAKE256_RATE; shake256_squeezeblocks(&mut out[idx..], nblocks, &mut state);
shake256_squeezeblocks(&mut out[idx..], nblocks, &mut state); outlen -= nblocks * SHAKE256_RATE;
outlen -= nblocks*SHAKE256_RATE; idx += nblocks * SHAKE256_RATE;
idx += nblocks*SHAKE256_RATE; shake256_squeeze(&mut out[idx..], outlen, &mut state);
shake256_squeeze(&mut out[idx..], outlen, &mut state);
} }
// Name: sha3_256 // Name: sha3_256
@ -437,14 +422,13 @@ pub(crate) fn shake256(out: &mut[u8], mut outlen: usize, input: &[u8], inlen: us
// Arguments: - [u8] h: output (32 bytes) // Arguments: - [u8] h: output (32 bytes)
// - const [u8] input: input // - const [u8] input: input
// - usize inlen: length of input in bytes // - usize inlen: length of input in bytes
pub(crate) fn sha3_256(h: &mut[u8], input: &[u8], inlen: usize) pub(crate) fn sha3_256(h: &mut [u8], input: &[u8], inlen: usize) {
{ let mut s = [0u64; 25];
let mut s = [0u64; 25]; keccak_absorb_once(&mut s, SHA3_256_RATE, input, inlen, 0x06);
keccak_absorb_once(&mut s, SHA3_256_RATE, input, inlen, 0x06); keccakf1600_statepermute(&mut s);
keccakf1600_statepermute(&mut s); for i in 0..4 {
for i in 0..4 { store64(&mut h[8 * i..], s[i]);
store64(&mut h[8*i..], s[i]); }
}
} }
// Name: sha3_512 // Name: sha3_512
@ -454,18 +438,15 @@ pub(crate) fn sha3_256(h: &mut[u8], input: &[u8], inlen: usize)
// Arguments: - [u8] h: output (64 bytes) // Arguments: - [u8] h: output (64 bytes)
// - const [u8] input: input // - const [u8] input: input
// - usize inlen: length of input in bytes // - usize inlen: length of input in bytes
pub(crate) fn sha3_512(h: &mut[u8], input: &[u8], inlen: usize) pub(crate) fn sha3_512(h: &mut [u8], input: &[u8], inlen: usize) {
{ let mut s = [0u64; 25];
let mut s = [0u64; 25]; keccak_absorb_once(&mut s, SHA3_512_RATE, input, inlen, 0x06);
keccak_absorb_once(&mut s, SHA3_512_RATE, input, inlen, 0x06); keccakf1600_statepermute(&mut s);
keccakf1600_statepermute(&mut s); for i in 0..8 {
for i in 0..8 { store64(&mut h[8 * i..], s[i]);
store64(&mut h[8*i..], s[i]); }
}
} }
// Name: keccak_finalize // Name: keccak_finalize
// //
// Description: Finalize absorb step. // Description: Finalize absorb step.
@ -474,10 +455,9 @@ pub(crate) fn sha3_512(h: &mut[u8], input: &[u8], inlen: usize)
// - usize pos: position in current block to be absorbed // - usize pos: position in current block to be absorbed
// - usize r: rate in bytes (e.g., 168 for SHAKE128) // - usize r: rate in bytes (e.g., 168 for SHAKE128)
// - u8 p: domain separation byte // - u8 p: domain separation byte
fn keccak_finalize(s: &mut[u64], pos: usize, r: usize, p: u8) fn keccak_finalize(s: &mut [u64], pos: usize, r: usize, p: u8) {
{ s[pos / 8] ^= (p as u64) << 8 * (pos % 8);
s[pos/8] ^= (p as u64) << 8*(pos%8); s[r / 8 - 1] ^= 1u64 << 63;
s[r/8-1] ^= 1u64 << 63;
} }
// Name: keccak_absorb_once // Name: keccak_absorb_once
@ -490,34 +470,27 @@ fn keccak_finalize(s: &mut[u64], pos: usize, r: usize, p: u8)
// - const [u8] input: input to be absorbed into s // - const [u8] input: input to be absorbed into s
// - u64 mlen: length of input in bytes // - u64 mlen: length of input in bytes
// - [u8] p: domain-separation byte for different Keccak-derived functions // - [u8] p: domain-separation byte for different Keccak-derived functions
pub(crate) fn keccak_absorb_once( pub(crate) fn keccak_absorb_once(s: &mut [u64], r: usize, input: &[u8], mut inlen: usize, p: u8) {
s: &mut[u64], // Zero State
r: usize, for i in s.iter_mut() {
input: &[u8], *i = 0;
mut inlen:
usize,
p: u8)
{
// Zero State
for i in s.iter_mut() {
*i = 0;
}
let mut idx = 0usize;
while inlen >= r {
for i in 0..(r/8) {
s[i] ^= load64(&input[idx+8*i..]);
} }
idx += r;
inlen -= r;
keccakf1600_statepermute(s);
}
for i in 0..inlen { let mut idx = 0usize;
s[i/8] ^= (input[idx+i] as u64) << 8*(i%8); while inlen >= r {
} for i in 0..(r / 8) {
s[inlen/8] ^= (p as u64) << 8*(inlen%8); s[i] ^= load64(&input[idx + 8 * i..]);
s[(r-1)/8] ^= 1u64 << 63; }
idx += r;
inlen -= r;
keccakf1600_statepermute(s);
}
for i in 0..inlen {
s[i / 8] ^= (input[idx + i] as u64) << 8 * (i % 8);
}
s[inlen / 8] ^= (p as u64) << 8 * (inlen % 8);
s[(r - 1) / 8] ^= 1u64 << 63;
} }
// Name: keccak_squeeze // Name: keccak_squeeze
@ -532,30 +505,23 @@ pub(crate) fn keccak_absorb_once(
// usize pos: number of bytes in current block already squeezed // usize pos: number of bytes in current block already squeezed
// - usize r: rate in bytes (e.g., 168 for SHAKE128) // - usize r: rate in bytes (e.g., 168 for SHAKE128)
// Returns new position pos in current block // Returns new position pos in current block
pub(crate) fn keccak_squeeze( pub(crate) fn keccak_squeeze(out: &mut [u8], mut outlen: usize, s: &mut [u64], mut pos: usize, r: usize) -> usize {
out: &mut[u8], let mut idx = 0;
mut outlen: usize, while outlen > 0 {
s: &mut [u64], if pos == r {
mut pos: usize, keccakf1600_statepermute(s);
r: usize pos = 0
) -> usize }
{ let mut i = pos;
let mut idx = 0; while i < r && i < pos + outlen {
while outlen > 0 { out[idx] = (s[i / 8] >> 8 * (i % 8)) as u8;
if pos == r { i += 1;
keccakf1600_statepermute(s); idx += 1;
pos = 0 }
outlen -= i - pos;
pos = i;
} }
let mut i = pos; pos
while i < r && i < pos+outlen {
out[idx] = (s[i/8] >> 8*(i%8)) as u8;
i += 1;
idx += 1;
}
outlen -= i-pos;
pos = i;
}
pos
} }
// Name: shake128_init // Name: shake128_init
@ -563,21 +529,18 @@ pub(crate) fn keccak_squeeze(
// Description: Initializes Keccak state for use as SHAKE128 XOF // Description: Initializes Keccak state for use as SHAKE128 XOF
// //
// Arguments: - keccak_state state: (uninitialized) Keccak state // Arguments: - keccak_state state: (uninitialized) Keccak state
fn shake128_init(state: &mut KeccakState) fn shake128_init(state: &mut KeccakState) {
{ state.reset()
state.reset()
} }
// Name: shake128_finalize // Name: shake128_finalize
// //
// Description: Finalize absorb step of the SHAKE128 XOF. // Description: Finalize absorb step of the SHAKE128 XOF.
// //
// Arguments: - keccak_state state: pointer to Keccak state // Arguments: - keccak_state state: pointer to Keccak state
fn shake128_finalize(state: &mut KeccakState) fn shake128_finalize(state: &mut KeccakState) {
{ keccak_finalize(&mut state.s, state.pos, SHAKE128_RATE, 0x1F);
keccak_finalize(&mut state.s, state.pos, SHAKE128_RATE, 0x1F); state.pos = SHAKE128_RATE;
state.pos = SHAKE128_RATE;
} }
// Name: shake128_squeeze // Name: shake128_squeeze
@ -588,9 +551,8 @@ fn shake128_finalize(state: &mut KeccakState)
// Arguments: - [u8] out: pointer to output blocks // Arguments: - [u8] out: pointer to output blocks
// - usize outlen : number of bytes to be squeezed (written to output) // - usize outlen : number of bytes to be squeezed (written to output)
// - keccak_state s: pointer to input/output Keccak state // - keccak_state s: pointer to input/output Keccak state
fn shake128_squeeze(out: &mut[u8], outlen: usize, state: &mut KeccakState) fn shake128_squeeze(out: &mut [u8], outlen: usize, state: &mut KeccakState) {
{ state.pos = keccak_squeeze(out, outlen, &mut state.s, state.pos, SHAKE128_RATE);
state.pos = keccak_squeeze(out, outlen, &mut state.s, state.pos, SHAKE128_RATE);
} }
// Name: shake128_absorb_once // Name: shake128_absorb_once
@ -600,52 +562,44 @@ fn shake128_squeeze(out: &mut[u8], outlen: usize, state: &mut KeccakState)
// Arguments: - keccak_state state: pointer to (uninitialized) output Keccak state // Arguments: - keccak_state state: pointer to (uninitialized) output Keccak state
// - const [u8] in: input to be absorbed into s // - const [u8] in: input to be absorbed into s
// - usize inlen: length of input in bytes // - usize inlen: length of input in bytes
pub(crate) fn shake128_absorb_once(state: &mut KeccakState, input: &[u8], inlen: usize) pub(crate) fn shake128_absorb_once(state: &mut KeccakState, input: &[u8], inlen: usize) {
{ keccak_absorb_once(&mut state.s, SHAKE128_RATE, input, inlen, 0x1F);
keccak_absorb_once(&mut state.s, SHAKE128_RATE, input, inlen, 0x1F); state.pos = SHAKE128_RATE;
state.pos = SHAKE128_RATE;
} }
fn shake256_init(state: &mut KeccakState) { fn shake256_init(state: &mut KeccakState) {
state.reset(); state.reset();
} }
fn shake256_absorb(state: &mut KeccakState, input: &[u8], inlen: usize) fn shake256_absorb(state: &mut KeccakState, input: &[u8], inlen: usize) {
{ state.pos = keccak_absorb(&mut state.s, state.pos, SHAKE256_RATE, input, inlen);
state.pos = keccak_absorb(&mut state.s, state.pos, SHAKE256_RATE, input, inlen);
} }
fn shake256_finalize(state: &mut KeccakState) fn shake256_finalize(state: &mut KeccakState) {
{ keccak_finalize(&mut state.s, state.pos, SHAKE256_RATE, 0x1F);
keccak_finalize(&mut state.s, state.pos, SHAKE256_RATE, 0x1F); state.pos = SHAKE256_RATE;
state.pos = SHAKE256_RATE;
} }
fn shake256_squeeze(out: &mut[u8], outlen: usize, state: &mut KeccakState) fn shake256_squeeze(out: &mut [u8], outlen: usize, state: &mut KeccakState) {
{ state.pos = keccak_squeeze(out, outlen, &mut state.s, state.pos, SHAKE256_RATE);
state.pos = keccak_squeeze(out, outlen, &mut state.s, state.pos, SHAKE256_RATE);
} }
pub(crate) fn shake256_absorb_once(state: &mut KeccakState, input: &[u8], inlen: usize) pub(crate) fn shake256_absorb_once(state: &mut KeccakState, input: &[u8], inlen: usize) {
{ keccak_absorb_once(&mut state.s, SHAKE256_RATE, input, inlen, 0x1F);
keccak_absorb_once(&mut state.s, SHAKE256_RATE, input, inlen, 0x1F); state.pos = SHAKE256_RATE;
state.pos = SHAKE256_RATE;
} }
fn shake256_squeezeblocks(out: &mut[u8], nblocks: usize, state: &mut KeccakState) fn shake256_squeezeblocks(out: &mut [u8], nblocks: usize, state: &mut KeccakState) {
{ keccak_squeezeblocks(out, nblocks, &mut state.s, SHAKE256_RATE);
keccak_squeezeblocks(out, nblocks, &mut state.s, SHAKE256_RATE);
} }
fn shake128(out: &mut[u8], mut outlen: usize, input: &[u8], inlen: usize) fn shake128(out: &mut [u8], mut outlen: usize, input: &[u8], inlen: usize) {
{ let mut state = KeccakState::new();
let mut state = KeccakState::new(); let mut idx = 0;
let mut idx = 0; shake128_absorb_once(&mut state, input, inlen);
shake128_absorb_once(&mut state, input, inlen); let nblocks = outlen / SHAKE128_RATE;
let nblocks = outlen/SHAKE128_RATE; shake128_squeezeblocks(&mut out[idx..], nblocks, &mut state);
shake128_squeezeblocks(&mut out[idx..], nblocks, &mut state); outlen -= nblocks * SHAKE128_RATE;
outlen -= nblocks*SHAKE128_RATE; idx += nblocks * SHAKE128_RATE;
idx += nblocks*SHAKE128_RATE; shake128_squeeze(&mut out[idx..], outlen, &mut state);
shake128_squeeze(&mut out[idx..], outlen, &mut state);
} }

246
third_party/kyber/src/avx2/fips202x4.rs vendored
View file

@ -1,124 +1,106 @@
#![allow(dead_code)] #![allow(dead_code)]
use core::arch::x86_64::*; use crate::align::{Eta4xBuf, GenMatrixBuf};
use crate::fips202::*; use crate::fips202::*;
use crate::keccak4x::f1600_x4; use crate::keccak4x::f1600_x4;
use crate::align::{GenMatrixBuf, Eta4xBuf}; use core::arch::x86_64::*;
#[repr(C)] #[repr(C)]
pub struct Keccakx4State { pub struct Keccakx4State {
s: [__m256i; 25] s: [__m256i; 25],
} }
impl Keccakx4State { impl Keccakx4State {
pub fn new() -> Self { pub fn new() -> Self {
unsafe {Keccakx4State { s: [_mm256_setzero_si256(); 25]}} unsafe { Keccakx4State { s: [_mm256_setzero_si256(); 25] } }
} }
} }
pub unsafe fn keccakx4_absorb_once( pub unsafe fn keccakx4_absorb_once(
s: &mut[__m256i; 25], s: &mut [__m256i; 25],
r: usize, r: usize,
in0: &[u8], in0: &[u8],
in1: &[u8], in1: &[u8],
in2: &[u8], in2: &[u8],
in3: &[u8], in3: &[u8],
mut inlen: usize, mut inlen: usize,
p: u8 p: u8,
) ) {
{ let mut pos = 0i64;
let mut pos = 0i64; let mut t;
let mut t; for i in 0..25 {
for i in 0..25 { s[i] = _mm256_setzero_si256();
s[i] = _mm256_setzero_si256();
}
let mut idx = _mm256_set_epi64x(
in3.as_ptr() as i64,
in2.as_ptr() as i64,
in1.as_ptr() as i64,
in0.as_ptr() as i64,
);
while inlen >= r {
for i in 0..(r/8) {
t = _mm256_i64gather_epi64(pos as *const i64, idx, 1);
s[i] = _mm256_xor_si256(s[i], t);
pos += 8;
} }
inlen -= r; let mut idx = _mm256_set_epi64x(in3.as_ptr() as i64, in2.as_ptr() as i64, in1.as_ptr() as i64, in0.as_ptr() as i64);
f1600_x4(s); while inlen >= r {
} for i in 0..(r / 8) {
let end = inlen/8; t = _mm256_i64gather_epi64(pos as *const i64, idx, 1);
for i in 0..end { s[i] = _mm256_xor_si256(s[i], t);
t = _mm256_i64gather_epi64(pos as *const i64, idx, 1); pos += 8;
s[i] = _mm256_xor_si256(s[i], t); }
pos += 8; inlen -= r;
} f1600_x4(s);
inlen -= 8*end; }
let end = inlen / 8;
for i in 0..end {
t = _mm256_i64gather_epi64(pos as *const i64, idx, 1);
s[i] = _mm256_xor_si256(s[i], t);
pos += 8;
}
inlen -= 8 * end;
if inlen > 0 { if inlen > 0 {
t = _mm256_i64gather_epi64(pos as *const i64, idx, 1); t = _mm256_i64gather_epi64(pos as *const i64, idx, 1);
idx = _mm256_set1_epi64x(((1u64 << (8*inlen)) - 1) as i64); idx = _mm256_set1_epi64x(((1u64 << (8 * inlen)) - 1) as i64);
t = _mm256_and_si256(t, idx); t = _mm256_and_si256(t, idx);
s[end] = _mm256_xor_si256(s[end], t);
}
t = _mm256_set1_epi64x(((p as u64) << 8 * inlen) as i64);
s[end] = _mm256_xor_si256(s[end], t); s[end] = _mm256_xor_si256(s[end], t);
} t = _mm256_set1_epi64x((1u64 << 63) as i64);
s[r / 8 - 1] = _mm256_xor_si256(s[r / 8 - 1], t);
t = _mm256_set1_epi64x(((p as u64) << 8*inlen) as i64);
s[end] = _mm256_xor_si256(s[end], t);
t = _mm256_set1_epi64x((1u64 << 63) as i64);
s[r/8 - 1] = _mm256_xor_si256(s[r/8 - 1], t);
} }
pub unsafe fn keccakx4_squeezeblocks128( pub unsafe fn keccakx4_squeezeblocks128(out: &mut [GenMatrixBuf; 4], mut nblocks: usize, r: usize, s: &mut [__m256i; 25]) {
out: &mut [GenMatrixBuf; 4], let mut t;
mut nblocks: usize, let mut idx = 0usize;
r: usize, while nblocks > 0 {
s: &mut [__m256i; 25] f1600_x4(s);
) for i in 0..(r / 8) {
{ t = _mm_castsi128_pd(_mm256_castsi256_si128(s[i]));
let mut t; let out0_ptr = out[0].coeffs[idx + 8 * i..].as_mut_ptr();
let mut idx = 0usize; let out1_ptr = out[1].coeffs[idx + 8 * i..].as_mut_ptr();
while nblocks > 0 { _mm_storel_pd(out0_ptr as *mut f64, t);
f1600_x4(s); _mm_storeh_pd(out1_ptr as *mut f64, t);
for i in 0..(r/8) {
t = _mm_castsi128_pd(_mm256_castsi256_si128(s[i])); t = _mm_castsi128_pd(_mm256_extracti128_si256(s[i], 1));
let out0_ptr = out[0].coeffs[idx+8*i..].as_mut_ptr(); let out2_ptr = out[2].coeffs[idx + 8 * i..].as_mut_ptr();
let out1_ptr = out[1].coeffs[idx+8*i..].as_mut_ptr(); let out3_ptr = out[3].coeffs[idx + 8 * i..].as_mut_ptr();
_mm_storel_pd(out0_ptr as *mut f64, t); _mm_storel_pd(out2_ptr as *mut f64, t);
_mm_storeh_pd(out1_ptr as *mut f64, t); _mm_storeh_pd(out3_ptr as *mut f64, t);
}
t = _mm_castsi128_pd(_mm256_extracti128_si256(s[i],1)); idx += r;
let out2_ptr = out[2].coeffs[idx+8*i..].as_mut_ptr(); nblocks -= 1;
let out3_ptr = out[3].coeffs[idx+8*i..].as_mut_ptr();
_mm_storel_pd(out2_ptr as *mut f64, t);
_mm_storeh_pd(out3_ptr as *mut f64, t);
} }
idx += r;
nblocks -= 1;
}
} }
pub unsafe fn keccakx4_squeezeblocks256( pub unsafe fn keccakx4_squeezeblocks256(out: &mut [Eta4xBuf; 4], mut nblocks: usize, r: usize, s: &mut [__m256i; 25]) {
out: &mut [Eta4xBuf; 4], let mut t;
mut nblocks: usize, let mut idx = 0usize;
r: usize, while nblocks > 0 {
s: &mut [__m256i; 25] f1600_x4(s);
) for i in 0..(r / 8) {
{ t = _mm_castsi128_pd(_mm256_castsi256_si128(s[i]));
let mut t; _mm_storel_pd(out[0].coeffs[idx + 8 * i..].as_mut_ptr() as *mut f64, t);
let mut idx = 0usize; _mm_storeh_pd(out[1].coeffs[idx + 8 * i..].as_mut_ptr() as *mut f64, t);
while nblocks > 0 { t = _mm_castsi128_pd(_mm256_extracti128_si256(s[i], 1));
f1600_x4(s); _mm_storel_pd(out[2].coeffs[idx + 8 * i..].as_mut_ptr() as *mut f64, t);
for i in 0..(r/8) { _mm_storeh_pd(out[3].coeffs[idx + 8 * i..].as_mut_ptr() as *mut f64, t);
t = _mm_castsi128_pd(_mm256_castsi256_si128(s[i])); }
_mm_storel_pd(out[0].coeffs[idx+8*i..].as_mut_ptr() as *mut f64, t); idx += r;
_mm_storeh_pd(out[1].coeffs[idx+8*i..].as_mut_ptr() as *mut f64, t); nblocks -= 1;
t = _mm_castsi128_pd(_mm256_extracti128_si256(s[i],1));
_mm_storel_pd(out[2].coeffs[idx+8*i..].as_mut_ptr() as *mut f64, t);
_mm_storeh_pd(out[3].coeffs[idx+8*i..].as_mut_ptr() as *mut f64, t);
} }
idx += r;
nblocks -= 1;
}
} }
// pub unsafe fn keccakx4_squeezeonce128( // pub unsafe fn keccakx4_squeezeonce128(
@ -156,66 +138,20 @@ pub unsafe fn keccakx4_squeezeblocks256(
// } // }
// } // }
pub unsafe fn shake128x4_absorb_once( pub unsafe fn shake128x4_absorb_once(state: &mut Keccakx4State, in0: &[u8], in1: &[u8], in2: &[u8], in3: &[u8], inlen: usize) {
state: &mut Keccakx4State, keccakx4_absorb_once(&mut state.s, SHAKE128_RATE, in0, in1, in2, in3, inlen, 0x1F)
in0: &[u8],
in1: &[u8],
in2: &[u8],
in3: &[u8],
inlen: usize,
)
{
keccakx4_absorb_once(
&mut state.s,
SHAKE128_RATE,
in0, in1, in2, in3, inlen,
0x1F
)
} }
pub unsafe fn shake128x4_squeezeblocks( pub unsafe fn shake128x4_squeezeblocks(out: &mut [GenMatrixBuf; 4], nblocks: usize, state: &mut Keccakx4State) {
out: &mut[GenMatrixBuf; 4], keccakx4_squeezeblocks128(out, nblocks, SHAKE128_RATE, &mut state.s);
nblocks: usize,
state: &mut Keccakx4State
)
{
keccakx4_squeezeblocks128(
out,
nblocks,
SHAKE128_RATE,
&mut state.s
);
} }
pub unsafe fn shake256x4_absorb_once( pub unsafe fn shake256x4_absorb_once(state: &mut Keccakx4State, in0: &[u8], in1: &[u8], in2: &[u8], in3: &[u8], inlen: usize) {
state: &mut Keccakx4State, keccakx4_absorb_once(&mut state.s, SHAKE256_RATE, in0, in1, in2, in3, inlen, 0x1F)
in0: &[u8],
in1: &[u8],
in2: &[u8],
in3: &[u8],
inlen: usize,
)
{
keccakx4_absorb_once(
&mut state.s,
SHAKE256_RATE,
in0, in1, in2, in3, inlen,
0x1F
)
} }
pub unsafe fn shake256x4_squeezeblocks( pub unsafe fn shake256x4_squeezeblocks(out: &mut [Eta4xBuf; 4], nblocks: usize, state: &mut Keccakx4State) {
out: &mut[Eta4xBuf; 4], keccakx4_squeezeblocks256(out, nblocks, SHAKE256_RATE, &mut state.s);
nblocks: usize,
state: &mut Keccakx4State
)
{
keccakx4_squeezeblocks256(
out,
nblocks,
SHAKE256_RATE,
&mut state.s
);
} }
// pub unsafe fn shake128x4( // pub unsafe fn shake128x4(
@ -234,7 +170,7 @@ pub unsafe fn shake256x4_squeezeblocks(
// shake128x4_absorb_once(&mut state, in0, in1, in2, in3, inlen); // shake128x4_absorb_once(&mut state, in0, in1, in2, in3, inlen);
// shake128x4_squeezeblocks(out, nblocks, &mut state); // shake128x4_squeezeblocks(out, nblocks, &mut state);
// let idx = nblocks*SHAKE128_RATE; // let idx = nblocks*SHAKE128_RATE;
// outlen -= idx; // outlen -= idx;
// if outlen > 0 { // if outlen > 0 {
@ -277,4 +213,4 @@ pub unsafe fn shake256x4_squeezeblocks(
// out[3].coeffs[idx+i] = t[3][i]; // out[3].coeffs[idx+i] = t[3][i];
// } // }
// } // }
// } // }

1031
third_party/kyber/src/avx2/indcpa.rs vendored
View file

File diff suppressed because it is too large Load diff

523
third_party/kyber/src/avx2/keccak4x.rs vendored
View file

@ -1,4 +1,4 @@
// Macro and function code structure is the work of Marek Kotewicz // Macro and function code structure is the work of Marek Kotewicz
// plus contributors to the tiny-keccak crate licensed under // plus contributors to the tiny-keccak crate licensed under
// Creative Commons CC0 1.0 Universal. Thankyou. // Creative Commons CC0 1.0 Universal. Thankyou.
// https://github.com/debris/tiny-keccak // https://github.com/debris/tiny-keccak
@ -16,205 +16,392 @@ use core::arch::x86_64::*;
#[repr(C, align(32))] #[repr(C, align(32))]
union RC_Data { union RC_Data {
vecs: [__m256i; 24], vecs: [__m256i; 24],
u: [u64; 96] u: [u64; 96],
} }
#[repr(C, align(32))] #[repr(C, align(32))]
union Temp { union Temp {
pub vec: __m256i, pub vec: __m256i,
pub u: [u64; 4] pub u: [u64; 4],
} }
const RHO: [u32; 24] = [ const RHO: [u32; 24] = [
1, 3, 6, 10, 15, 21, 28, 36, 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44,
45, 55, 2, 14, 27, 41, 56, 8,
25, 43, 62, 18, 39, 61, 20, 44,
]; ];
const PI: [usize; 24] = [ const PI: [usize; 24] = [
10, 7, 11, 17, 18, 3, 5, 16, 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1,
8, 21, 24, 4, 15, 23, 19, 13,
12, 2, 20, 14, 22, 9, 6, 1,
]; ];
// Set __mm256i constants with a union // Set __mm256i constants with a union
const RC_X4: RC_Data = RC_Data { u: [ const RC_X4: RC_Data = RC_Data {
0x0000000000000001, 0x0000000000000001, 0x0000000000000001, 0x0000000000000001, u: [
0x0000000000008082, 0x0000000000008082, 0x0000000000008082, 0x0000000000008082, 0x0000000000000001,
0x800000000000808a, 0x800000000000808a, 0x800000000000808a, 0x800000000000808a, 0x0000000000000001,
0x8000000080008000, 0x8000000080008000, 0x8000000080008000, 0x8000000080008000, 0x0000000000000001,
0x000000000000808b, 0x000000000000808b, 0x000000000000808b, 0x000000000000808b, 0x0000000000000001,
0x0000000080000001, 0x0000000080000001, 0x0000000080000001, 0x0000000080000001, 0x0000000000008082,
0x8000000080008081, 0x8000000080008081, 0x8000000080008081, 0x8000000080008081, 0x0000000000008082,
0x8000000000008009, 0x8000000000008009, 0x8000000000008009, 0x8000000000008009, 0x0000000000008082,
0x000000000000008a, 0x000000000000008a, 0x000000000000008a, 0x000000000000008a, 0x0000000000008082,
0x0000000000000088, 0x0000000000000088, 0x0000000000000088, 0x0000000000000088, 0x800000000000808a,
0x0000000080008009, 0x0000000080008009, 0x0000000080008009, 0x0000000080008009, 0x800000000000808a,
0x000000008000000a, 0x000000008000000a, 0x000000008000000a, 0x000000008000000a, 0x800000000000808a,
0x000000008000808b, 0x000000008000808b, 0x000000008000808b, 0x000000008000808b, 0x800000000000808a,
0x800000000000008b, 0x800000000000008b, 0x800000000000008b, 0x800000000000008b, 0x8000000080008000,
0x8000000000008089, 0x8000000000008089, 0x8000000000008089, 0x8000000000008089, 0x8000000080008000,
0x8000000000008003, 0x8000000000008003, 0x8000000000008003, 0x8000000000008003, 0x8000000080008000,
0x8000000000008002, 0x8000000000008002, 0x8000000000008002, 0x8000000000008002, 0x8000000080008000,
0x8000000000000080, 0x8000000000000080, 0x8000000000000080, 0x8000000000000080, 0x000000000000808b,
0x000000000000800a, 0x000000000000800a, 0x000000000000800a, 0x000000000000800a, 0x000000000000808b,
0x800000008000000a, 0x800000008000000a, 0x800000008000000a, 0x800000008000000a, 0x000000000000808b,
0x8000000080008081, 0x8000000080008081, 0x8000000080008081, 0x8000000080008081, 0x000000000000808b,
0x8000000000008080, 0x8000000000008080, 0x8000000000008080, 0x8000000000008080, 0x0000000080000001,
0x0000000080000001, 0x0000000080000001, 0x0000000080000001, 0x0000000080000001, 0x0000000080000001,
0x8000000080008008, 0x8000000080008008, 0x8000000080008008, 0x8000000080008008 0x0000000080000001,
]}; 0x0000000080000001,
0x8000000080008081,
0x8000000080008081,
0x8000000080008081,
0x8000000080008081,
0x8000000000008009,
0x8000000000008009,
0x8000000000008009,
0x8000000000008009,
0x000000000000008a,
0x000000000000008a,
0x000000000000008a,
0x000000000000008a,
0x0000000000000088,
0x0000000000000088,
0x0000000000000088,
0x0000000000000088,
0x0000000080008009,
0x0000000080008009,
0x0000000080008009,
0x0000000080008009,
0x000000008000000a,
0x000000008000000a,
0x000000008000000a,
0x000000008000000a,
0x000000008000808b,
0x000000008000808b,
0x000000008000808b,
0x000000008000808b,
0x800000000000008b,
0x800000000000008b,
0x800000000000008b,
0x800000000000008b,
0x8000000000008089,
0x8000000000008089,
0x8000000000008089,
0x8000000000008089,
0x8000000000008003,
0x8000000000008003,
0x8000000000008003,
0x8000000000008003,
0x8000000000008002,
0x8000000000008002,
0x8000000000008002,
0x8000000000008002,
0x8000000000000080,
0x8000000000000080,
0x8000000000000080,
0x8000000000000080,
0x000000000000800a,
0x000000000000800a,
0x000000000000800a,
0x000000000000800a,
0x800000008000000a,
0x800000008000000a,
0x800000008000000a,
0x800000008000000a,
0x8000000080008081,
0x8000000080008081,
0x8000000080008081,
0x8000000080008081,
0x8000000000008080,
0x8000000000008080,
0x8000000000008080,
0x8000000000008080,
0x0000000080000001,
0x0000000080000001,
0x0000000080000001,
0x0000000080000001,
0x8000000080008008,
0x8000000080008008,
0x8000000080008008,
0x8000000080008008,
],
};
macro_rules! unroll5 { macro_rules! unroll5 {
($var:ident, $body:block) => { ($var:ident, $body:block) => {{
{ const $var: usize = 0; $body; } const $var: usize = 0;
{ const $var: usize = 1; $body; } $body;
{ const $var: usize = 2; $body; } }
{ const $var: usize = 3; $body; } {
{ const $var: usize = 4; $body; } const $var: usize = 1;
}; $body;
}
{
const $var: usize = 2;
$body;
}
{
const $var: usize = 3;
$body;
}
{
const $var: usize = 4;
$body;
}};
} }
macro_rules! unroll24 { macro_rules! unroll24 {
($var: ident, $body: block) => { ($var: ident, $body: block) => {{
{ const $var: usize = 0; $body; } const $var: usize = 0;
{ const $var: usize = 1; $body; } $body;
{ const $var: usize = 2; $body; } }
{ const $var: usize = 3; $body; } {
{ const $var: usize = 4; $body; } const $var: usize = 1;
{ const $var: usize = 5; $body; } $body;
{ const $var: usize = 6; $body; } }
{ const $var: usize = 7; $body; } {
{ const $var: usize = 8; $body; } const $var: usize = 2;
{ const $var: usize = 9; $body; } $body;
{ const $var: usize = 10; $body; } }
{ const $var: usize = 11; $body; } {
{ const $var: usize = 12; $body; } const $var: usize = 3;
{ const $var: usize = 13; $body; } $body;
{ const $var: usize = 14; $body; } }
{ const $var: usize = 15; $body; } {
{ const $var: usize = 16; $body; } const $var: usize = 4;
{ const $var: usize = 17; $body; } $body;
{ const $var: usize = 18; $body; } }
{ const $var: usize = 19; $body; } {
{ const $var: usize = 20; $body; } const $var: usize = 5;
{ const $var: usize = 21; $body; } $body;
{ const $var: usize = 22; $body; } }
{ const $var: usize = 23; $body; } {
}; const $var: usize = 6;
$body;
}
{
const $var: usize = 7;
$body;
}
{
const $var: usize = 8;
$body;
}
{
const $var: usize = 9;
$body;
}
{
const $var: usize = 10;
$body;
}
{
const $var: usize = 11;
$body;
}
{
const $var: usize = 12;
$body;
}
{
const $var: usize = 13;
$body;
}
{
const $var: usize = 14;
$body;
}
{
const $var: usize = 15;
$body;
}
{
const $var: usize = 16;
$body;
}
{
const $var: usize = 17;
$body;
}
{
const $var: usize = 18;
$body;
}
{
const $var: usize = 19;
$body;
}
{
const $var: usize = 20;
$body;
}
{
const $var: usize = 21;
$body;
}
{
const $var: usize = 22;
$body;
}
{
const $var: usize = 23;
$body;
}};
} }
#[allow(unused_assignments, non_upper_case_globals)] #[allow(unused_assignments, non_upper_case_globals)]
pub fn f1600_x4(a: &mut [__m256i]) { pub fn f1600_x4(a: &mut [__m256i]) {
unsafe { unsafe {
for i in 0..24 {
for i in 0..24 { let mut array = [_mm256_setzero_si256(); 5];
let mut array = [_mm256_setzero_si256(); 5];
// Theta
unroll5!(x, {
unroll5!(y, {
array[x] = _mm256_xor_si256(array[x], a[5 * y + x]);
});
});
unroll5!(x, {
unroll5!(y, {
let t1 = array[(x + 4) % 5];
let mut t2 = Temp {vec: array[(x + 1) % 5]};
for i in 0..4 {
t2.u[i] = t2.u[i].rotate_left(1);
}
a[5 * y + x] = _mm256_xor_si256(a[5 * y + x], _mm256_xor_si256(t1, t2.vec));
});
});
// Rho and pi // Theta
let mut last = a[1]; unroll5!(x, {
unroll24!(x, { unroll5!(y, {
array[0] = a[PI[x]]; array[x] = _mm256_xor_si256(array[x], a[5 * y + x]);
let mut temp_last = Temp {vec: last}; });
for i in 0..4 { });
temp_last.u[i] = temp_last.u[i].rotate_left(RHO[x]);
unroll5!(x, {
unroll5!(y, {
let t1 = array[(x + 4) % 5];
let mut t2 = Temp { vec: array[(x + 1) % 5] };
for i in 0..4 {
t2.u[i] = t2.u[i].rotate_left(1);
}
a[5 * y + x] = _mm256_xor_si256(a[5 * y + x], _mm256_xor_si256(t1, t2.vec));
});
});
// Rho and pi
let mut last = a[1];
unroll24!(x, {
array[0] = a[PI[x]];
let mut temp_last = Temp { vec: last };
for i in 0..4 {
temp_last.u[i] = temp_last.u[i].rotate_left(RHO[x]);
}
a[PI[x]] = temp_last.vec;
last = array[0];
});
// Chi
unroll5!(y_step, {
let y = 5 * y_step;
unroll5!(x, {
array[x] = a[y + x];
});
unroll5!(x, {
let t1 = array[(x + 1) % 5];
let t2 = array[(x + 2) % 5];
let tmp = _mm256_xor_si256(array[x], _mm256_andnot_si256(t1, t2));
a[y + x] = tmp;
});
});
a[0] = _mm256_xor_si256(a[0], RC_X4.vecs[i]);
} }
a[PI[x]] = temp_last.vec;
last = array[0];
});
// Chi
unroll5!(y_step, {
let y = 5 * y_step;
unroll5!(x, {
array[x] = a[y + x];
});
unroll5!(x, {
let t1 = array[(x + 1) % 5];
let t2 = array[(x + 2) % 5];
let tmp = _mm256_xor_si256(array[x], _mm256_andnot_si256(t1, t2));
a[y+x] = tmp;
});
});
a[0] = _mm256_xor_si256(a[0], RC_X4.vecs[i]);
} }
}
} }
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
const PLEN: usize = 25; const PLEN: usize = 25;
// Test vectors from XKCP // Test vectors from XKCP
// https://github.com/XKCP/XKCP/blob/master/tests/TestVectors/KeccakF-1600-IntermediateValues.txt // https://github.com/XKCP/XKCP/blob/master/tests/TestVectors/KeccakF-1600-IntermediateValues.txt
#[test] #[test]
fn known_vectors() { fn known_vectors() {
let vec1: [u64; 25] = [ let vec1: [u64; 25] = [
0xF1258F7940E1DDE7, 0x84D5CCF933C0478A, 0xD598261EA65AA9EE, 0xBD1547306F80494D, 0xF1258F7940E1DDE7,
0x8B284E056253D057, 0xFF97A42D7F8E6FD4, 0x90FEE5A0A44647C4, 0x8C5BDA0CD6192E76, 0x84D5CCF933C0478A,
0xAD30A6F71B19059C, 0x30935AB7D08FFC64, 0xEB5AA93F2317D635, 0xA9A6E6260D712103, 0xD598261EA65AA9EE,
0x81A57C16DBCF555F, 0x43B831CD0347C826, 0x01F22F1A11A5569F, 0x05E5635A21D9AE61, 0xBD1547306F80494D,
0x64BEFEF28CC970F2, 0x613670957BC46611, 0xB87C5A554FD00ECB, 0x8C3EE88A1CCF32C8, 0x8B284E056253D057,
0x940C7922AE3A2614, 0x1841F924A2C509E4, 0x16F53526E70465C2, 0x75F644E97F30A13B, 0xFF97A42D7F8E6FD4,
0xEAF1FF7B5CECA249 0x90FEE5A0A44647C4,
]; 0x8C5BDA0CD6192E76,
0xAD30A6F71B19059C,
let vec2: [u64; 25] = [ 0x30935AB7D08FFC64,
0x2D5C954DF96ECB3C, 0x6A332CD07057B56D, 0x093D8D1270D76B6C, 0x8A20D9B25569D094, 0xEB5AA93F2317D635,
0x4F9C4F99E5E7F156, 0xF957B9A2DA65FB38, 0x85773DAE1275AF0D, 0xFAF4F247C3D810F7, 0xA9A6E6260D712103,
0x1F1B9EE6F79A8759, 0xE4FECC0FEE98B425, 0x68CE61B6B9CE68A1, 0xDEEA66C4BA8F974F, 0x81A57C16DBCF555F,
0x33C43D836EAFB1F5, 0xE00654042719DBD9, 0x7CF8A9F009831265, 0xFD5449A6BF174743, 0x43B831CD0347C826,
0x97DDAD33D8994B40, 0x48EAD5FC5D0BE774, 0xE3B8C8EE55B7B03C, 0x91A0226E649E42E9, 0x01F22F1A11A5569F,
0x900E3129E7BADD7B, 0x202A9EC5FAA3CCE8, 0x5B3402464E1C3DB6, 0x609F4E62A44C1059, 0x05E5635A21D9AE61,
0x20D06CD26A8FBF5C 0x64BEFEF28CC970F2,
]; 0x613670957BC46611,
0xB87C5A554FD00ECB,
0x8C3EE88A1CCF32C8,
0x940C7922AE3A2614,
0x1841F924A2C509E4,
0x16F53526E70465C2,
0x75F644E97F30A13B,
0xEAF1FF7B5CECA249,
];
// repeat values to check all lanes let vec2: [u64; 25] = [
let tvec1 = expand(vec1); 0x2D5C954DF96ECB3C,
let tvec2 = expand(vec2); 0x6A332CD07057B56D,
0x093D8D1270D76B6C,
0x8A20D9B25569D094,
0x4F9C4F99E5E7F156,
0xF957B9A2DA65FB38,
0x85773DAE1275AF0D,
0xFAF4F247C3D810F7,
0x1F1B9EE6F79A8759,
0xE4FECC0FEE98B425,
0x68CE61B6B9CE68A1,
0xDEEA66C4BA8F974F,
0x33C43D836EAFB1F5,
0xE00654042719DBD9,
0x7CF8A9F009831265,
0xFD5449A6BF174743,
0x97DDAD33D8994B40,
0x48EAD5FC5D0BE774,
0xE3B8C8EE55B7B03C,
0x91A0226E649E42E9,
0x900E3129E7BADD7B,
0x202A9EC5FAA3CCE8,
0x5B3402464E1C3DB6,
0x609F4E62A44C1059,
0x20D06CD26A8FBF5C,
];
unsafe { // repeat values to check all lanes
let mut data = Data { u: [0u64;100] }; let tvec1 = expand(vec1);
f1600_x4(&mut data.lanes); let tvec2 = expand(vec2);
assert_eq!(&data.u , &tvec1);
f1600_x4(&mut data.lanes); unsafe {
assert_eq!(data.u, tvec2); let mut data = Data { u: [0u64; 100] };
f1600_x4(&mut data.lanes);
assert_eq!(&data.u, &tvec1);
f1600_x4(&mut data.lanes);
assert_eq!(data.u, tvec2);
}
} }
#[repr(C)]
} pub union Data {
#[repr(C)] pub lanes: [__m256i; PLEN],
pub union Data { pub u: [u64; PLEN * 4],
pub lanes: [__m256i; PLEN], }
pub u: [u64; PLEN * 4]
} // [0,1...] expands to [0,0,0,0,1,1,1,1...]
fn expand(vec: [u64; PLEN]) -> [u64; 100] {
// [0,1...] expands to [0,0,0,0,1,1,1,1...] let mut out = [0u64; 100];
fn expand(vec: [u64; PLEN]) -> [u64; 100] { for (i, u) in vec.iter().enumerate() {
let mut out = [0u64; 100]; out[i * 4..][..4].copy_from_slice(&[*u; 4]);
for (i,u) in vec.iter().enumerate() { }
out[i*4..][..4].copy_from_slice(&[*u; 4]); out
} }
out
}
} }

3
third_party/kyber/src/avx2/mod.rs vendored
View file

@ -1,4 +1,3 @@
pub mod aes256ctr; pub mod aes256ctr;
pub mod align; pub mod align;
pub mod cbd; pub mod cbd;
@ -10,4 +9,4 @@ pub mod keccak4x;
pub mod poly; pub mod poly;
pub mod polyvec; pub mod polyvec;
pub mod rejsample; pub mod rejsample;
pub mod verify; pub mod verify;

733
third_party/kyber/src/avx2/poly.rs vendored
View file

@ -1,417 +1,388 @@
#![allow(unused_imports)] #![allow(unused_imports)]
use crate::{align::*, cbd::*, consts::*, fips202::*, fips202x4::*, params::*, symmetric::*};
use core::arch::x86_64::*; use core::arch::x86_64::*;
use crate::{
align::*,
cbd::*,
consts::*,
fips202::*,
fips202x4::*,
params::*,
symmetric::*,
};
pub(crate) const NOISE_NBLOCKS: usize = pub(crate) const NOISE_NBLOCKS: usize = (KYBER_ETA1 * KYBER_N / 4 + SHAKE256_RATE - 1) / SHAKE256_RATE;
(KYBER_ETA1*KYBER_N/4+SHAKE256_RATE-1)/SHAKE256_RATE;
#[derive(Clone)] #[derive(Clone)]
#[repr(C)] #[repr(C)]
pub union Poly { pub union Poly {
pub coeffs: [i16; KYBER_N], pub coeffs: [i16; KYBER_N],
pub vec: [__m256i; (KYBER_N+15)/16] pub vec: [__m256i; (KYBER_N + 15) / 16],
} }
impl Copy for Poly {} impl Copy for Poly {}
impl Poly { impl Poly {
pub fn new() -> Self { pub fn new() -> Self {
Poly { Poly { coeffs: [0i16; KYBER_N] }
coeffs: [0i16; KYBER_N]
} }
} // Basic polynomial value checking for development
// Basic polynomial value checking for development // #[cfg(debug_assertions)]
// #[cfg(debug_assertions)] // fn checksum(&self) -> i16 {
// fn checksum(&self) -> i16 { // unsafe{
// unsafe{ // let mut out = 0;
// let mut out = 0; // for x in &self.coeffs {
// for x in &self.coeffs { // out ^= x;
// out ^= x; // }
// } // out
// out // }
// } // }
// }
} }
extern { extern "C" {
fn ntt_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]); fn ntt_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn invntt_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]); fn invntt_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn nttunpack_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]); fn nttunpack_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn basemul_avx( fn basemul_avx(r: &mut [i16; KYBER_N], a: &[i16; KYBER_N], b: &[i16; KYBER_N], q_data: &[i16; 640]);
r: &mut[i16; KYBER_N], fn tomont_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
a: &[i16; KYBER_N], fn reduce_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
b: &[i16; KYBER_N], fn ntttobytes_avx(r: *mut u8, a: &[i16; KYBER_N], q_data: &[i16; 640]);
q_data: &[i16; 640] fn nttfrombytes_avx(r: *mut i16, a: *const u8, q_data: &[i16; 640]);
);
fn tomont_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn reduce_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn ntttobytes_avx(r: *mut u8 , a: &[i16; KYBER_N] , q_data: &[i16; 640]);
fn nttfrombytes_avx(r: *mut i16, a: *const u8, q_data: &[i16; 640]);
} }
// #[target_feature(enable = "avx2")] // #[target_feature(enable = "avx2")]
#[cfg(any(feature="kyber512", not(feature="kyber1024")))] #[cfg(any(feature = "kyber512", not(feature = "kyber1024")))]
pub unsafe fn poly_compress(r: &mut[u8], a: Poly) pub unsafe fn poly_compress(r: &mut [u8], a: Poly) {
{ let (mut f0, mut f1, mut f2, mut f3);
let (mut f0, mut f1, mut f2, mut f3); let v: __m256i = _mm256_load_si256(QDATA.vec[_16XV / 16..].as_ptr());
let v: __m256i = _mm256_load_si256(QDATA.vec[_16XV/16..].as_ptr()); let shift1: __m256i = _mm256_set1_epi16(1 << 9);
let shift1: __m256i = _mm256_set1_epi16(1 << 9); let mask: __m256i = _mm256_set1_epi16(15);
let mask: __m256i = _mm256_set1_epi16(15); let shift2: __m256i = _mm256_set1_epi16((16 << 8) + 1);
let shift2: __m256i = _mm256_set1_epi16((16 << 8) + 1); let permdidx: __m256i = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
let permdidx: __m256i = _mm256_set_epi32(7,3,6,2,5,1,4,0);
for i in 0..KYBER_N/64 { for i in 0..KYBER_N / 64 {
f0 = _mm256_load_si256(&a.vec[4*i+0]); f0 = _mm256_load_si256(&a.vec[4 * i + 0]);
f1 = _mm256_load_si256(&a.vec[4*i+1]); f1 = _mm256_load_si256(&a.vec[4 * i + 1]);
f2 = _mm256_load_si256(&a.vec[4*i+2]); f2 = _mm256_load_si256(&a.vec[4 * i + 2]);
f3 = _mm256_load_si256(&a.vec[4*i+3]); f3 = _mm256_load_si256(&a.vec[4 * i + 3]);
f0 = _mm256_mulhi_epi16(f0,v); f0 = _mm256_mulhi_epi16(f0, v);
f1 = _mm256_mulhi_epi16(f1,v); f1 = _mm256_mulhi_epi16(f1, v);
f2 = _mm256_mulhi_epi16(f2,v); f2 = _mm256_mulhi_epi16(f2, v);
f3 = _mm256_mulhi_epi16(f3,v); f3 = _mm256_mulhi_epi16(f3, v);
f0 = _mm256_mulhrs_epi16(f0,shift1); f0 = _mm256_mulhrs_epi16(f0, shift1);
f1 = _mm256_mulhrs_epi16(f1,shift1); f1 = _mm256_mulhrs_epi16(f1, shift1);
f2 = _mm256_mulhrs_epi16(f2,shift1); f2 = _mm256_mulhrs_epi16(f2, shift1);
f3 = _mm256_mulhrs_epi16(f3,shift1); f3 = _mm256_mulhrs_epi16(f3, shift1);
f0 = _mm256_and_si256(f0,mask); f0 = _mm256_and_si256(f0, mask);
f1 = _mm256_and_si256(f1,mask); f1 = _mm256_and_si256(f1, mask);
f2 = _mm256_and_si256(f2,mask); f2 = _mm256_and_si256(f2, mask);
f3 = _mm256_and_si256(f3,mask); f3 = _mm256_and_si256(f3, mask);
f0 = _mm256_packus_epi16(f0,f1); f0 = _mm256_packus_epi16(f0, f1);
f2 = _mm256_packus_epi16(f2,f3); f2 = _mm256_packus_epi16(f2, f3);
f0 = _mm256_maddubs_epi16(f0,shift2); f0 = _mm256_maddubs_epi16(f0, shift2);
f2 = _mm256_maddubs_epi16(f2,shift2); f2 = _mm256_maddubs_epi16(f2, shift2);
f0 = _mm256_packus_epi16(f0,f2); f0 = _mm256_packus_epi16(f0, f2);
f0 = _mm256_permutevar8x32_epi32(f0,permdidx); f0 = _mm256_permutevar8x32_epi32(f0, permdidx);
_mm256_storeu_si256(r[32*i..].as_mut_ptr() as *mut __m256i,f0); _mm256_storeu_si256(r[32 * i..].as_mut_ptr() as *mut __m256i, f0);
} }
} }
// #[target_feature(enable = "avx2")] // #[target_feature(enable = "avx2")]
#[cfg(any(feature="kyber512", not(feature="kyber1024")))] #[cfg(any(feature = "kyber512", not(feature = "kyber1024")))]
pub unsafe fn poly_decompress(r: &mut Poly, a: &[u8]) pub unsafe fn poly_decompress(r: &mut Poly, a: &[u8]) {
{ let (mut t, mut f);
let (mut t, mut f); let q: __m256i = _mm256_load_si256(QDATA.vec[_16XQ / 16..].as_ptr());
let q: __m256i = _mm256_load_si256(QDATA.vec[_16XQ/16..].as_ptr()); let shufbidx: __m256i = _mm256_set_epi8(
let shufbidx: __m256i = _mm256_set_epi8( 7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0,
7,7,7,7,6,6,6,6,5,5,5,5,4,4,4,4, );
3,3,3,3,2,2,2,2,1,1,1,1,0,0,0,0 let mask: __m256i = _mm256_set1_epi32(0x00F0000F);
); let shift: __m256i = _mm256_set1_epi32((128 << 16) + 2048);
let mask: __m256i = _mm256_set1_epi32(0x00F0000F);
let shift: __m256i = _mm256_set1_epi32((128 << 16) + 2048); for i in 0..KYBER_N / 16 {
t = _mm_loadl_epi64(a[8 * i..].as_ptr() as *const __m128i);
for i in 0..KYBER_N/16 { f = _mm256_broadcastsi128_si256(t);
t = _mm_loadl_epi64(a[8*i..].as_ptr() as *const __m128i); f = _mm256_shuffle_epi8(f, shufbidx);
f = _mm256_broadcastsi128_si256(t); f = _mm256_and_si256(f, mask);
f = _mm256_shuffle_epi8(f,shufbidx); f = _mm256_mullo_epi16(f, shift);
f = _mm256_and_si256(f,mask); f = _mm256_mulhrs_epi16(f, q);
f = _mm256_mullo_epi16(f,shift); _mm256_store_si256(&mut r.vec[i], f);
f = _mm256_mulhrs_epi16(f,q);
_mm256_store_si256(&mut r.vec[i],f);
}
}
// #[target_feature(enable = "avx2")]
#[cfg(feature="kyber1024")]
pub unsafe fn poly_compress(r: &mut[u8], a: Poly)
{
let (mut f0, mut f1);
let (mut t0, mut t1);
let mut tmp;
let v: __m256i = _mm256_load_si256(&QDATA.vec[_16XV/16]);
let shift1: __m256i = _mm256_set1_epi16(1 << 10);
let mask: __m256i = _mm256_set1_epi16(31);
let shift2: __m256i = _mm256_set1_epi16((32 << 8) + 1);
let shift3: __m256i = _mm256_set1_epi32((1024 << 16) + 1);
let sllvdidx: __m256i = _mm256_set1_epi64x(12);
let shufbidx: __m256i = _mm256_set_epi8(
8,-1,-1,-1,-1,-1, 4, 3, 2, 1, 0,-1,12,11,10, 9,
-1,12,11,10, 9, 8,-1,-1,-1,-1,-1 ,4, 3, 2, 1, 0
);
for i in 0..(KYBER_N/32) {
f0 = _mm256_load_si256(&a.vec[2*i+0]);
f1 = _mm256_load_si256(&a.vec[2*i+1]);
f0 = _mm256_mulhi_epi16(f0,v);
f1 = _mm256_mulhi_epi16(f1,v);
f0 = _mm256_mulhrs_epi16(f0,shift1);
f1 = _mm256_mulhrs_epi16(f1,shift1);
f0 = _mm256_and_si256(f0,mask);
f1 = _mm256_and_si256(f1,mask);
f0 = _mm256_packus_epi16(f0,f1);
f0 = _mm256_maddubs_epi16(f0,shift2);
f0 = _mm256_madd_epi16(f0,shift3);
f0 = _mm256_sllv_epi32(f0,sllvdidx);
f0 = _mm256_srlv_epi64(f0,sllvdidx);
f0 = _mm256_shuffle_epi8(f0,shufbidx);
t0 = _mm256_castsi256_si128(f0);
t1 = _mm256_extracti128_si256(f0,1);
t0 = _mm_blendv_epi8(t0,t1,_mm256_castsi256_si128(shufbidx));
_mm_storeu_si128(r[20*i+ 0..].as_mut_ptr() as *mut __m128i,t0);
tmp = _mm_cvtsi128_si32(t1);
r[20*i+16..20*i+20].copy_from_slice(&tmp.to_le_bytes());
}
}
// #[target_feature(enable = "avx2")]
#[cfg(feature="kyber1024")]
pub unsafe fn poly_decompress(r: &mut Poly, a: &[u8])
{
let (mut t, mut f, mut ti);
let q = _mm256_load_si256(&QDATA.vec[_16XQ/16]);
let shufbidx = _mm256_set_epi8(
9,9,9,8,8,8,8,7,7,6,6,6,6,5,5,5,
4,4,4,3,3,3,3,2,2,1,1,1,1,0,0,0
);
let mask = _mm256_set_epi16(
248,1984,62,496,3968,124,992,31,
248,1984,62,496,3968,124,992,31
);
let shift = _mm256_set_epi16(
128,16,512,64,8,256,32,1024,
128,16,512,64,8,256,32,1024
);
for i in 0..KYBER_N/16 {
t = _mm_loadl_epi64(a[10*i+0..].as_ptr() as *const __m128i);
ti = i32::from_le_bytes([a[10*i+8], a[10*i+9], 0, 0]);
t = _mm_insert_epi16(t, ti, 4);
f = _mm256_broadcastsi128_si256(t);
f = _mm256_shuffle_epi8(f,shufbidx);
f = _mm256_and_si256(f,mask);
f = _mm256_mullo_epi16(f,shift);
f = _mm256_mulhrs_epi16(f,q);
_mm256_store_si256(r.vec[i..].as_mut_ptr() as *mut __m256i,f);
}
}
pub fn poly_frombytes(r: &mut Poly, a: &[u8])
{
unsafe {
nttfrombytes_avx(r.coeffs.as_mut_ptr(), a.as_ptr(), &QDATA.coeffs);
}
}
pub fn poly_tobytes(r: &mut[u8], a: Poly)
{
let mut buf = [0u8; KYBER_POLYBYTES];
unsafe { ntttobytes_avx(buf.as_mut_ptr(), &a.coeffs, &QDATA.coeffs); }
r[..KYBER_POLYBYTES].copy_from_slice(&buf[..]);
}
// #[target_feature(enable = "avx2")]
pub unsafe fn poly_frommsg(r: &mut Poly, msg: &[u8])
{
let shift = _mm256_broadcastsi128_si256(_mm_set_epi32(0,1,2,3));
let idx = _mm256_broadcastsi128_si256(
_mm_set_epi8(15,14,11,10,7,6,3,2,13,12,9,8,5,4,1,0)
);
let hqs: __m256i = _mm256_set1_epi16((KYBER_Q+1) as i16/2);
let f = _mm256_loadu_si256(msg.as_ptr() as *const __m256i);
let mut frommsg64 = |i: usize, mut g3: __m256i| {
g3 = _mm256_sllv_epi32(g3,shift);
g3 = _mm256_shuffle_epi8(g3,idx);
let mut g0 = _mm256_slli_epi16(g3,12);
let mut g1 = _mm256_slli_epi16(g3,8);
let mut g2 = _mm256_slli_epi16(g3,4);
g0 = _mm256_srai_epi16(g0,15);
g1 = _mm256_srai_epi16(g1,15);
g2 = _mm256_srai_epi16(g2,15);
g3 = _mm256_srai_epi16(g3,15);
g0 = _mm256_and_si256(g0,hqs); // 19 18 17 16 3 2 1 0
g1 = _mm256_and_si256(g1,hqs); // 23 22 21 20 7 6 5 4
g2 = _mm256_and_si256(g2,hqs); // 27 26 25 24 11 10 9 8
g3 = _mm256_and_si256(g3,hqs); // 31 30 29 28 15 14 13 12
let h0 = _mm256_unpacklo_epi64(g0,g1);
let h2 = _mm256_unpackhi_epi64(g0,g1);
let h1 = _mm256_unpacklo_epi64(g2,g3);
let h3 = _mm256_unpackhi_epi64(g2,g3);
g0 = _mm256_permute2x128_si256(h0,h1,0x20);
g2 = _mm256_permute2x128_si256(h0,h1,0x31);
g1 = _mm256_permute2x128_si256(h2,h3,0x20);
g3 = _mm256_permute2x128_si256(h2,h3,0x31);
_mm256_store_si256(&mut r.vec[0+2*i+0],g0);
_mm256_store_si256(&mut r.vec[0+2*i+1],g1);
_mm256_store_si256(&mut r.vec[8+2*i+0],g2);
_mm256_store_si256(&mut r.vec[8+2*i+1],g3);
};
frommsg64(0, _mm256_shuffle_epi32(f, 0));
frommsg64(1, _mm256_shuffle_epi32(f, 85));
frommsg64(2, _mm256_shuffle_epi32(f, 170));
frommsg64(3, _mm256_shuffle_epi32(f, 255));
}
// #[target_feature(enable = "avx2")]
pub fn poly_tomsg(msg: &mut[u8], a: Poly)
{
unsafe {
let (mut f0, mut f1, mut g0, mut g1);
let hq: __m256i = _mm256_set1_epi16((KYBER_Q - 1) as i16/2);
let hhq: __m256i = _mm256_set1_epi16((KYBER_Q - 1) as i16/4);
for i in 0..KYBER_N/32 {
f0 = _mm256_load_si256(&a.vec[2*i+0]);
f1 = _mm256_load_si256(&a.vec[2*i+1]);
f0 = _mm256_sub_epi16(hq, f0);
f1 = _mm256_sub_epi16(hq, f1);
g0 = _mm256_srai_epi16(f0, 15);
g1 = _mm256_srai_epi16(f1, 15);
f0 = _mm256_xor_si256(f0, g0);
f1 = _mm256_xor_si256(f1, g1);
f0 = _mm256_sub_epi16(f0, hhq);
f1 = _mm256_sub_epi16(f1, hhq);
f0 = _mm256_packs_epi16(f0, f1);
f0 = _mm256_permute4x64_epi64(f0, 0xD8);
let small = _mm256_movemask_epi8(f0);
msg[4*i..][..4].copy_from_slice(&small.to_ne_bytes());
} }
}
} }
#[cfg(all(any(feature="kyber1024", feature="kyber512"), not(feature="90s")))] // #[target_feature(enable = "avx2")]
pub fn poly_getnoise_eta2(r: &mut Poly, seed: &[u8], nonce: u8) #[cfg(feature = "kyber1024")]
{ pub unsafe fn poly_compress(r: &mut [u8], a: Poly) {
let mut buf = Eta2Buf::new(); let (mut f0, mut f1);
unsafe { let (mut t0, mut t1);
prf(&mut buf.coeffs, KYBER_ETA2*KYBER_N/4, seed, nonce); let mut tmp;
poly_cbd_eta2(r, &buf.vec); let v: __m256i = _mm256_load_si256(&QDATA.vec[_16XV / 16]);
} let shift1: __m256i = _mm256_set1_epi16(1 << 10);
let mask: __m256i = _mm256_set1_epi16(31);
let shift2: __m256i = _mm256_set1_epi16((32 << 8) + 1);
let shift3: __m256i = _mm256_set1_epi32((1024 << 16) + 1);
let sllvdidx: __m256i = _mm256_set1_epi64x(12);
let shufbidx: __m256i = _mm256_set_epi8(
8, -1, -1, -1, -1, -1, 4, 3, 2, 1, 0, -1, 12, 11, 10, 9, -1, 12, 11, 10, 9, 8, -1, -1, -1, -1, -1, 4, 3, 2, 1, 0,
);
for i in 0..(KYBER_N / 32) {
f0 = _mm256_load_si256(&a.vec[2 * i + 0]);
f1 = _mm256_load_si256(&a.vec[2 * i + 1]);
f0 = _mm256_mulhi_epi16(f0, v);
f1 = _mm256_mulhi_epi16(f1, v);
f0 = _mm256_mulhrs_epi16(f0, shift1);
f1 = _mm256_mulhrs_epi16(f1, shift1);
f0 = _mm256_and_si256(f0, mask);
f1 = _mm256_and_si256(f1, mask);
f0 = _mm256_packus_epi16(f0, f1);
f0 = _mm256_maddubs_epi16(f0, shift2);
f0 = _mm256_madd_epi16(f0, shift3);
f0 = _mm256_sllv_epi32(f0, sllvdidx);
f0 = _mm256_srlv_epi64(f0, sllvdidx);
f0 = _mm256_shuffle_epi8(f0, shufbidx);
t0 = _mm256_castsi256_si128(f0);
t1 = _mm256_extracti128_si256(f0, 1);
t0 = _mm_blendv_epi8(t0, t1, _mm256_castsi256_si128(shufbidx));
_mm_storeu_si128(r[20 * i + 0..].as_mut_ptr() as *mut __m128i, t0);
tmp = _mm_cvtsi128_si32(t1);
r[20 * i + 16..20 * i + 20].copy_from_slice(&tmp.to_le_bytes());
}
} }
#[cfg(not(feature="90s"))] // #[target_feature(enable = "avx2")]
#[cfg(feature = "kyber1024")]
pub unsafe fn poly_decompress(r: &mut Poly, a: &[u8]) {
let (mut t, mut f, mut ti);
let q = _mm256_load_si256(&QDATA.vec[_16XQ / 16]);
let shufbidx = _mm256_set_epi8(
9, 9, 9, 8, 8, 8, 8, 7, 7, 6, 6, 6, 6, 5, 5, 5, 4, 4, 4, 3, 3, 3, 3, 2, 2, 1, 1, 1, 1, 0, 0, 0,
);
let mask = _mm256_set_epi16(248, 1984, 62, 496, 3968, 124, 992, 31, 248, 1984, 62, 496, 3968, 124, 992, 31);
let shift = _mm256_set_epi16(128, 16, 512, 64, 8, 256, 32, 1024, 128, 16, 512, 64, 8, 256, 32, 1024);
for i in 0..KYBER_N / 16 {
t = _mm_loadl_epi64(a[10 * i + 0..].as_ptr() as *const __m128i);
ti = i32::from_le_bytes([a[10 * i + 8], a[10 * i + 9], 0, 0]);
t = _mm_insert_epi16(t, ti, 4);
f = _mm256_broadcastsi128_si256(t);
f = _mm256_shuffle_epi8(f, shufbidx);
f = _mm256_and_si256(f, mask);
f = _mm256_mullo_epi16(f, shift);
f = _mm256_mulhrs_epi16(f, q);
_mm256_store_si256(r.vec[i..].as_mut_ptr() as *mut __m256i, f);
}
}
pub fn poly_frombytes(r: &mut Poly, a: &[u8]) {
unsafe {
nttfrombytes_avx(r.coeffs.as_mut_ptr(), a.as_ptr(), &QDATA.coeffs);
}
}
pub fn poly_tobytes(r: &mut [u8], a: Poly) {
let mut buf = [0u8; KYBER_POLYBYTES];
unsafe {
ntttobytes_avx(buf.as_mut_ptr(), &a.coeffs, &QDATA.coeffs);
}
r[..KYBER_POLYBYTES].copy_from_slice(&buf[..]);
}
// #[target_feature(enable = "avx2")]
pub unsafe fn poly_frommsg(r: &mut Poly, msg: &[u8]) {
let shift = _mm256_broadcastsi128_si256(_mm_set_epi32(0, 1, 2, 3));
let idx = _mm256_broadcastsi128_si256(_mm_set_epi8(15, 14, 11, 10, 7, 6, 3, 2, 13, 12, 9, 8, 5, 4, 1, 0));
let hqs: __m256i = _mm256_set1_epi16((KYBER_Q + 1) as i16 / 2);
let f = _mm256_loadu_si256(msg.as_ptr() as *const __m256i);
let mut frommsg64 = |i: usize, mut g3: __m256i| {
g3 = _mm256_sllv_epi32(g3, shift);
g3 = _mm256_shuffle_epi8(g3, idx);
let mut g0 = _mm256_slli_epi16(g3, 12);
let mut g1 = _mm256_slli_epi16(g3, 8);
let mut g2 = _mm256_slli_epi16(g3, 4);
g0 = _mm256_srai_epi16(g0, 15);
g1 = _mm256_srai_epi16(g1, 15);
g2 = _mm256_srai_epi16(g2, 15);
g3 = _mm256_srai_epi16(g3, 15);
g0 = _mm256_and_si256(g0, hqs); // 19 18 17 16 3 2 1 0
g1 = _mm256_and_si256(g1, hqs); // 23 22 21 20 7 6 5 4
g2 = _mm256_and_si256(g2, hqs); // 27 26 25 24 11 10 9 8
g3 = _mm256_and_si256(g3, hqs); // 31 30 29 28 15 14 13 12
let h0 = _mm256_unpacklo_epi64(g0, g1);
let h2 = _mm256_unpackhi_epi64(g0, g1);
let h1 = _mm256_unpacklo_epi64(g2, g3);
let h3 = _mm256_unpackhi_epi64(g2, g3);
g0 = _mm256_permute2x128_si256(h0, h1, 0x20);
g2 = _mm256_permute2x128_si256(h0, h1, 0x31);
g1 = _mm256_permute2x128_si256(h2, h3, 0x20);
g3 = _mm256_permute2x128_si256(h2, h3, 0x31);
_mm256_store_si256(&mut r.vec[0 + 2 * i + 0], g0);
_mm256_store_si256(&mut r.vec[0 + 2 * i + 1], g1);
_mm256_store_si256(&mut r.vec[8 + 2 * i + 0], g2);
_mm256_store_si256(&mut r.vec[8 + 2 * i + 1], g3);
};
frommsg64(0, _mm256_shuffle_epi32(f, 0));
frommsg64(1, _mm256_shuffle_epi32(f, 85));
frommsg64(2, _mm256_shuffle_epi32(f, 170));
frommsg64(3, _mm256_shuffle_epi32(f, 255));
}
// #[target_feature(enable = "avx2")]
pub fn poly_tomsg(msg: &mut [u8], a: Poly) {
unsafe {
let (mut f0, mut f1, mut g0, mut g1);
let hq: __m256i = _mm256_set1_epi16((KYBER_Q - 1) as i16 / 2);
let hhq: __m256i = _mm256_set1_epi16((KYBER_Q - 1) as i16 / 4);
for i in 0..KYBER_N / 32 {
f0 = _mm256_load_si256(&a.vec[2 * i + 0]);
f1 = _mm256_load_si256(&a.vec[2 * i + 1]);
f0 = _mm256_sub_epi16(hq, f0);
f1 = _mm256_sub_epi16(hq, f1);
g0 = _mm256_srai_epi16(f0, 15);
g1 = _mm256_srai_epi16(f1, 15);
f0 = _mm256_xor_si256(f0, g0);
f1 = _mm256_xor_si256(f1, g1);
f0 = _mm256_sub_epi16(f0, hhq);
f1 = _mm256_sub_epi16(f1, hhq);
f0 = _mm256_packs_epi16(f0, f1);
f0 = _mm256_permute4x64_epi64(f0, 0xD8);
let small = _mm256_movemask_epi8(f0);
msg[4 * i..][..4].copy_from_slice(&small.to_ne_bytes());
}
}
}
#[cfg(all(any(feature = "kyber1024", feature = "kyber512"), not(feature = "90s")))]
pub fn poly_getnoise_eta2(r: &mut Poly, seed: &[u8], nonce: u8) {
let mut buf = Eta2Buf::new();
unsafe {
prf(&mut buf.coeffs, KYBER_ETA2 * KYBER_N / 4, seed, nonce);
poly_cbd_eta2(r, &buf.vec);
}
}
#[cfg(not(feature = "90s"))]
pub fn poly_getnoise_eta1_4x( pub fn poly_getnoise_eta1_4x(
r0: &mut Poly, r1: &mut Poly, r2: &mut Poly, r3: &mut Poly, seed: &[u8], r0: &mut Poly,
nonce0: u8, nonce1: u8, nonce2: u8, nonce3: u8 r1: &mut Poly,
) r2: &mut Poly,
{ r3: &mut Poly,
unsafe { seed: &[u8],
nonce0: u8,
nonce1: u8,
nonce2: u8,
nonce3: u8,
) {
unsafe {
let mut buf = [Eta4xBuf::new(); 4];
let mut state = Keccakx4State::new();
let f = _mm256_loadu_si256(seed.as_ptr() as *const __m256i);
_mm256_store_si256(buf[0].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[1].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[2].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[3].vec.as_mut_ptr(), f);
buf[0].coeffs[32] = nonce0;
buf[1].coeffs[32] = nonce1;
buf[2].coeffs[32] = nonce2;
buf[3].coeffs[32] = nonce3;
shake256x4_absorb_once(&mut state, &buf[0].coeffs, &buf[1].coeffs, &buf[2].coeffs, &buf[3].coeffs, 33);
shake256x4_squeezeblocks(&mut buf, NOISE_NBLOCKS, &mut state);
poly_cbd_eta1(r0, &buf[0]);
poly_cbd_eta1(r1, &buf[1]);
poly_cbd_eta1(r2, &buf[2]);
poly_cbd_eta1(r3, &buf[3]);
}
}
#[cfg(all(feature = "kyber512", not(feature = "90s")))]
pub fn poly_getnoise_eta1122_4x(
r0: &mut Poly,
r1: &mut Poly,
r2: &mut Poly,
r3: &mut Poly,
seed: &[u8],
nonce0: u8,
nonce1: u8,
nonce2: u8,
nonce3: u8,
) {
let mut buf = [Eta4xBuf::new(); 4]; let mut buf = [Eta4xBuf::new(); 4];
let mut state = Keccakx4State::new(); let mut state = Keccakx4State::new();
let f = _mm256_loadu_si256(seed.as_ptr() as *const __m256i);
_mm256_store_si256(buf[0].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[1].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[2].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[3].vec.as_mut_ptr(), f);
buf[0].coeffs[32] = nonce0;
buf[1].coeffs[32] = nonce1;
buf[2].coeffs[32] = nonce2;
buf[3].coeffs[32] = nonce3;
shake256x4_absorb_once(
&mut state,
&buf[0].coeffs, &buf[1].coeffs,
&buf[2].coeffs, &buf[3].coeffs,
33
);
shake256x4_squeezeblocks(&mut buf, NOISE_NBLOCKS, &mut state);
poly_cbd_eta1(r0, &buf[0]);
poly_cbd_eta1(r1, &buf[1]);
poly_cbd_eta1(r2, &buf[2]);
poly_cbd_eta1(r3, &buf[3]);
}
}
#[cfg(all(feature="kyber512", not(feature="90s")))]
pub fn poly_getnoise_eta1122_4x(
r0: &mut Poly, r1: &mut Poly, r2: &mut Poly, r3: &mut Poly, seed: &[u8],
nonce0: u8, nonce1: u8, nonce2: u8, nonce3: u8,
)
{
let mut buf = [Eta4xBuf::new(); 4];
let mut state = Keccakx4State::new();
unsafe {
let f = _mm256_loadu_si256(seed.as_ptr() as *const __m256i);
_mm256_store_si256(buf[0].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[1].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[2].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[3].vec.as_mut_ptr(), f);
buf[0].coeffs[32] = nonce0;
buf[1].coeffs[32] = nonce1;
buf[2].coeffs[32] = nonce2;
buf[3].coeffs[32] = nonce3;
shake256x4_absorb_once(
&mut state,
&buf[0].coeffs, &buf[1].coeffs,
&buf[2].coeffs, &buf[3].coeffs,
33
);
shake256x4_squeezeblocks(&mut buf, NOISE_NBLOCKS, &mut state);
poly_cbd_eta1(r0, &buf[0]);
poly_cbd_eta1(r1, &buf[1]);
poly_cbd_eta2(r2, &buf[2].vec);
poly_cbd_eta2(r3, &buf[3].vec);
}
}
pub fn poly_ntt(r: &mut Poly)
{
unsafe { ntt_avx(&mut r.coeffs, &QDATA.coeffs); }
}
pub fn poly_invntt_tomont(r: &mut Poly)
{
unsafe { invntt_avx(&mut r.coeffs, &QDATA.coeffs); }
}
pub fn poly_nttunpack(r: &mut Poly)
{
unsafe { nttunpack_avx(&mut r.coeffs, &QDATA.coeffs); }
}
pub fn poly_basemul(r: &mut Poly, a: &Poly, b: &Poly)
{
unsafe { basemul_avx(&mut r.coeffs, &a.coeffs, &b.coeffs, &QDATA.coeffs); }
}
pub fn poly_tomont(r: &mut Poly)
{
unsafe { tomont_avx(&mut r.coeffs, &QDATA.coeffs); }
}
pub fn poly_reduce(r: &mut Poly)
{
unsafe { reduce_avx(&mut r.coeffs, &QDATA.coeffs); }
}
pub fn poly_add(r: &mut Poly, b: &Poly)
{
let (mut f0, mut f1);
for i in 0..(KYBER_N/16) {
unsafe { unsafe {
f0 = _mm256_load_si256(&r.vec[i]); let f = _mm256_loadu_si256(seed.as_ptr() as *const __m256i);
f1 = _mm256_load_si256(&b.vec[i]); _mm256_store_si256(buf[0].vec.as_mut_ptr(), f);
f0 = _mm256_add_epi16(f0, f1); _mm256_store_si256(buf[1].vec.as_mut_ptr(), f);
_mm256_store_si256(&mut r.vec[i] , f0); _mm256_store_si256(buf[2].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[3].vec.as_mut_ptr(), f);
buf[0].coeffs[32] = nonce0;
buf[1].coeffs[32] = nonce1;
buf[2].coeffs[32] = nonce2;
buf[3].coeffs[32] = nonce3;
shake256x4_absorb_once(&mut state, &buf[0].coeffs, &buf[1].coeffs, &buf[2].coeffs, &buf[3].coeffs, 33);
shake256x4_squeezeblocks(&mut buf, NOISE_NBLOCKS, &mut state);
poly_cbd_eta1(r0, &buf[0]);
poly_cbd_eta1(r1, &buf[1]);
poly_cbd_eta2(r2, &buf[2].vec);
poly_cbd_eta2(r3, &buf[3].vec);
} }
}
} }
pub fn poly_sub(r: &mut Poly, a: &Poly) pub fn poly_ntt(r: &mut Poly) {
{
let (mut f0, mut f1);
for i in 0..(KYBER_N/16) {
unsafe { unsafe {
f0 = _mm256_load_si256(&a.vec[i]); ntt_avx(&mut r.coeffs, &QDATA.coeffs);
f1 = _mm256_load_si256(&r.vec[i]);
f0 = _mm256_sub_epi16(f0, f1);
_mm256_store_si256(&mut r.vec[i], f0);
} }
}
} }
pub fn poly_invntt_tomont(r: &mut Poly) {
unsafe {
invntt_avx(&mut r.coeffs, &QDATA.coeffs);
}
}
pub fn poly_nttunpack(r: &mut Poly) {
unsafe {
nttunpack_avx(&mut r.coeffs, &QDATA.coeffs);
}
}
pub fn poly_basemul(r: &mut Poly, a: &Poly, b: &Poly) {
unsafe {
basemul_avx(&mut r.coeffs, &a.coeffs, &b.coeffs, &QDATA.coeffs);
}
}
pub fn poly_tomont(r: &mut Poly) {
unsafe {
tomont_avx(&mut r.coeffs, &QDATA.coeffs);
}
}
pub fn poly_reduce(r: &mut Poly) {
unsafe {
reduce_avx(&mut r.coeffs, &QDATA.coeffs);
}
}
pub fn poly_add(r: &mut Poly, b: &Poly) {
let (mut f0, mut f1);
for i in 0..(KYBER_N / 16) {
unsafe {
f0 = _mm256_load_si256(&r.vec[i]);
f1 = _mm256_load_si256(&b.vec[i]);
f0 = _mm256_add_epi16(f0, f1);
_mm256_store_si256(&mut r.vec[i], f0);
}
}
}
pub fn poly_sub(r: &mut Poly, a: &Poly) {
let (mut f0, mut f1);
for i in 0..(KYBER_N / 16) {
unsafe {
f0 = _mm256_load_si256(&a.vec[i]);
f1 = _mm256_load_si256(&r.vec[i]);
f0 = _mm256_sub_epi16(f0, f1);
_mm256_store_si256(&mut r.vec[i], f0);
}
}
}

421
third_party/kyber/src/avx2/polyvec.rs vendored
View file

@ -1,222 +1,197 @@
use crate::{consts::*, params::*, poly::*};
use core::arch::x86_64::*; use core::arch::x86_64::*;
use crate::{
poly::*,
params::*,
consts::*
};
#[derive(Clone)] #[derive(Clone)]
pub struct Polyvec { pub struct Polyvec {
pub vec: [Poly; KYBER_K] pub vec: [Poly; KYBER_K],
} }
impl Copy for Polyvec {} impl Copy for Polyvec {}
impl Polyvec { impl Polyvec {
pub fn new() -> Self { pub fn new() -> Self {
Polyvec { Polyvec { vec: [Poly::new(); KYBER_K] }
vec: [Poly::new(); KYBER_K]
} }
} // Basic polynomial value check for development
// Basic polynomial value check for development // #[cfg(debug_assertions)]
// #[cfg(debug_assertions)] // pub fn checksum(&self) -> i16 {
// pub fn checksum(&self) -> i16 { // unsafe {
// unsafe { // let mut out = 0i16;
// let mut out = 0i16; // for i in 0..KYBER_K {
// for i in 0..KYBER_K { // for j in 0..KYBER_N {
// for j in 0..KYBER_N { // out ^= &self.vec[i].coeffs[j];
// out ^= &self.vec[i].coeffs[j]; // }
// } // }
// } // out
// out // }
// } // }
// }
} }
// #[target_feature(enable = "avx")] // #[target_feature(enable = "avx")]
pub unsafe fn poly_compress10(r: &mut[u8], a: &Poly) pub unsafe fn poly_compress10(r: &mut [u8], a: &Poly) {
{ let (mut f0, mut f1, mut f2);
let (mut f0, mut f1, mut f2); let (mut t0, mut t1);
let (mut t0, mut t1);
let v = _mm256_load_si256(QDATA.vec[_16XV/16..].as_ptr()); let v = _mm256_load_si256(QDATA.vec[_16XV / 16..].as_ptr());
let v8 = _mm256_slli_epi16(v,3); let v8 = _mm256_slli_epi16(v, 3);
let off = _mm256_set1_epi16(15); let off = _mm256_set1_epi16(15);
let shift1 = _mm256_set1_epi16(1 << 12); let shift1 = _mm256_set1_epi16(1 << 12);
let mask = _mm256_set1_epi16(1023); let mask = _mm256_set1_epi16(1023);
let shift2 = _mm256_set1_epi64x( let shift2 = _mm256_set1_epi64x(((1024u64 << 48) + (1u64 << 32) + (1024 << 16) + 1) as i64);
((1024u64 << 48) + (1u64 << 32) + (1024 << 16) + 1) as i64 let sllvdidx = _mm256_set1_epi64x(12);
); let shufbidx = _mm256_set_epi8(
let sllvdidx = _mm256_set1_epi64x(12); 8, 4, 3, 2, 1, 0, -1, -1, -1, -1, -1, -1, 12, 11, 10, 9, -1, -1, -1, -1, -1, -1, 12, 11, 10, 9, 8, 4, 3, 2, 1, 0,
let shufbidx = _mm256_set_epi8( );
8, 4, 3, 2, 1, 0,-1,-1,-1,-1,-1,-1,12,11,10, 9,
-1,-1,-1,-1,-1,-1,12,11,10, 9, 8, 4, 3, 2, 1, 0
);
for i in 0..(KYBER_N/16) { for i in 0..(KYBER_N / 16) {
f0 = _mm256_load_si256(&a.vec[i]); f0 = _mm256_load_si256(&a.vec[i]);
f1 = _mm256_mullo_epi16(f0,v8); f1 = _mm256_mullo_epi16(f0, v8);
f2 = _mm256_add_epi16(f0,off); f2 = _mm256_add_epi16(f0, off);
f0 = _mm256_slli_epi16(f0,3); f0 = _mm256_slli_epi16(f0, 3);
f0 = _mm256_mulhi_epi16(f0,v); f0 = _mm256_mulhi_epi16(f0, v);
f2 = _mm256_sub_epi16(f1,f2); f2 = _mm256_sub_epi16(f1, f2);
f1 = _mm256_andnot_si256(f1,f2); f1 = _mm256_andnot_si256(f1, f2);
f1 = _mm256_srli_epi16(f1,15); f1 = _mm256_srli_epi16(f1, 15);
f0 = _mm256_sub_epi16(f0,f1); f0 = _mm256_sub_epi16(f0, f1);
f0 = _mm256_mulhrs_epi16(f0,shift1); f0 = _mm256_mulhrs_epi16(f0, shift1);
f0 = _mm256_and_si256(f0,mask); f0 = _mm256_and_si256(f0, mask);
f0 = _mm256_madd_epi16(f0,shift2); f0 = _mm256_madd_epi16(f0, shift2);
f0 = _mm256_sllv_epi32(f0,sllvdidx); f0 = _mm256_sllv_epi32(f0, sllvdidx);
f0 = _mm256_srli_epi64(f0,12); f0 = _mm256_srli_epi64(f0, 12);
f0 = _mm256_shuffle_epi8(f0,shufbidx); f0 = _mm256_shuffle_epi8(f0, shufbidx);
t0 = _mm256_castsi256_si128(f0); t0 = _mm256_castsi256_si128(f0);
t1 = _mm256_extracti128_si256(f0,1); t1 = _mm256_extracti128_si256(f0, 1);
t0 = _mm_blend_epi16(t0,t1,0xE0); t0 = _mm_blend_epi16(t0, t1, 0xE0);
_mm_storeu_si128(r[20*i..].as_mut_ptr() as *mut __m128i,t0); _mm_storeu_si128(r[20 * i..].as_mut_ptr() as *mut __m128i, t0);
_mm_storeu_si128(r[20*i+16..].as_mut_ptr() as *mut __m128i, t1); _mm_storeu_si128(r[20 * i + 16..].as_mut_ptr() as *mut __m128i, t1);
}
}
// #[target_feature(enable = "avx")]
pub unsafe fn poly_decompress10(r: &mut Poly, a: &[u8])
{
let mut f;
let q = _mm256_set1_epi32(((KYBER_Q as i32) << 16) + 4*KYBER_Q as i32);
let shufbidx = _mm256_set_epi8(11,10,10, 9, 9, 8, 8, 7,
6, 5, 5, 4, 4, 3, 3, 2,
9, 8, 8, 7, 7, 6, 6, 5,
4, 3, 3, 2, 2, 1, 1, 0);
let sllvdidx = _mm256_set1_epi64x(4);
let mask = _mm256_set1_epi32((32736 << 16) + 8184);
for i in 0..KYBER_N/16 {
f = _mm256_loadu_si256(a[20*i..].as_ptr() as *const __m256i);
f = _mm256_permute4x64_epi64(f,0x94);
f = _mm256_shuffle_epi8(f,shufbidx);
f = _mm256_sllv_epi32(f,sllvdidx);
f = _mm256_srli_epi16(f,1);
f = _mm256_and_si256(f,mask);
f = _mm256_mulhrs_epi16(f,q);
_mm256_store_si256(&mut r.vec[i],f);
}
}
// #[target_feature(enable = "avx")]
pub unsafe fn poly_compress11(r: &mut[u8], a: &Poly)
{
let (mut f0, mut f1, mut f2);
let (mut t0, mut t1);
let v = _mm256_load_si256(QDATA.vec[_16XV/16..].as_ptr());
let v8 = _mm256_slli_epi16(v,3);
let off = _mm256_set1_epi16(36);
let shift1 = _mm256_set1_epi16(1 << 13);
let mask = _mm256_set1_epi16(2047);
let shift2 = _mm256_set1_epi64x(
((2048u64 << 48) + (1u64 << 32) + (2048 << 16) + 1) as i64
);
let sllvdidx = _mm256_set1_epi64x(10);
let srlvqidx = _mm256_set_epi64x(30,10,30,10);
let shufbidx = _mm256_set_epi8(
4, 3, 2, 1, 0, 0,-1,-1,-1,-1,10, 9, 8, 7, 6, 5,
-1,-1,-1,-1,-1,10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
);
for i in 0..KYBER_N/16 {
f0 = _mm256_load_si256(&a.vec[i]);
f1 = _mm256_mullo_epi16(f0,v8);
f2 = _mm256_add_epi16(f0,off);
f0 = _mm256_slli_epi16(f0,3);
f0 = _mm256_mulhi_epi16(f0,v);
f2 = _mm256_sub_epi16(f1,f2);
f1 = _mm256_andnot_si256(f1,f2);
f1 = _mm256_srli_epi16(f1,15);
f0 = _mm256_sub_epi16(f0,f1);
f0 = _mm256_mulhrs_epi16(f0,shift1);
f0 = _mm256_and_si256(f0,mask);
f0 = _mm256_madd_epi16(f0,shift2);
f0 = _mm256_sllv_epi32(f0,sllvdidx);
f1 = _mm256_bsrli_epi128(f0,8);
f0 = _mm256_srlv_epi64(f0,srlvqidx);
f1 = _mm256_slli_epi64(f1,34);
f0 = _mm256_add_epi64(f0,f1);
f0 = _mm256_shuffle_epi8(f0,shufbidx);
t0 = _mm256_castsi256_si128(f0);
t1 = _mm256_extracti128_si256(f0,1);
t0 = _mm_blendv_epi8(t0,t1,_mm256_castsi256_si128(shufbidx));
_mm_storeu_si128(r[22*i+ 0..].as_mut_ptr() as *mut __m128i,t0);
_mm_storel_epi64(r[22*i+16..].as_mut_ptr() as *mut __m128i,t1);
}
}
// #[target_feature(enable = "avx")]
pub unsafe fn poly_decompress11(r: &mut Poly, a: &[u8])
{
let mut f;
let q = _mm256_load_si256(QDATA.vec[_16XQ/16..].as_ptr());
let shufbidx = _mm256_set_epi8(
13,12,12,11,10, 9, 9, 8,
8, 7, 6, 5, 5, 4, 4, 3,
10, 9, 9, 8, 7, 6, 6, 5,
5, 4, 3, 2, 2, 1, 1, 0
);
let srlvdidx = _mm256_set_epi32(0,0,1,0,0,0,1,0);
let srlvqidx = _mm256_set_epi64x(2,0,2,0);
let shift = _mm256_set_epi16(4,32,1,8,32,1,4,32,4,32,1,8,32,1,4,32);
let mask = _mm256_set1_epi16(32752);
for i in 0..(KYBER_N/16) {
f = _mm256_loadu_si256(a[22*i..].as_ptr() as *const __m256i);
f = _mm256_permute4x64_epi64(f,0x94);
f = _mm256_shuffle_epi8(f,shufbidx);
f = _mm256_srlv_epi32(f,srlvdidx);
f = _mm256_srlv_epi64(f,srlvqidx);
f = _mm256_mullo_epi16(f,shift);
f = _mm256_srli_epi16(f,1);
f = _mm256_and_si256(f,mask);
f = _mm256_mulhrs_epi16(f,q);
_mm256_store_si256(&mut r.vec[i],f);
}
}
pub unsafe fn polyvec_compress(r: &mut[u8], a: &Polyvec)
{
if cfg!(feature="kyber1024") {
for i in 0..KYBER_K {
poly_compress11(&mut r[352*i..], &a.vec[i]);
}
} else {
for i in 0..KYBER_K {
poly_compress10(&mut r[320*i..], &a.vec[i]);
} }
}
} }
pub unsafe fn polyvec_decompress(r: &mut Polyvec, a: &[u8]) // #[target_feature(enable = "avx")]
{ pub unsafe fn poly_decompress10(r: &mut Poly, a: &[u8]) {
if cfg!(feature="kyber1024") { let mut f;
for i in 0..KYBER_K { let q = _mm256_set1_epi32(((KYBER_Q as i32) << 16) + 4 * KYBER_Q as i32);
poly_decompress11(&mut r.vec[i], &a[352*i..]); let shufbidx = _mm256_set_epi8(
} 11, 10, 10, 9, 9, 8, 8, 7, 6, 5, 5, 4, 4, 3, 3, 2, 9, 8, 8, 7, 7, 6, 6, 5, 4, 3, 3, 2, 2, 1, 1, 0,
} else { );
for i in 0..KYBER_K { let sllvdidx = _mm256_set1_epi64x(4);
poly_decompress10(&mut r.vec[i], &a[320*i..]); let mask = _mm256_set1_epi32((32736 << 16) + 8184);
for i in 0..KYBER_N / 16 {
f = _mm256_loadu_si256(a[20 * i..].as_ptr() as *const __m256i);
f = _mm256_permute4x64_epi64(f, 0x94);
f = _mm256_shuffle_epi8(f, shufbidx);
f = _mm256_sllv_epi32(f, sllvdidx);
f = _mm256_srli_epi16(f, 1);
f = _mm256_and_si256(f, mask);
f = _mm256_mulhrs_epi16(f, q);
_mm256_store_si256(&mut r.vec[i], f);
} }
}
} }
pub fn polyvec_tobytes(r: &mut[u8], a: &Polyvec) // #[target_feature(enable = "avx")]
{ pub unsafe fn poly_compress11(r: &mut [u8], a: &Poly) {
for i in 0..KYBER_K { let (mut f0, mut f1, mut f2);
poly_tobytes(&mut r[i*KYBER_POLYBYTES..], a.vec[i]); let (mut t0, mut t1);
} let v = _mm256_load_si256(QDATA.vec[_16XV / 16..].as_ptr());
let v8 = _mm256_slli_epi16(v, 3);
let off = _mm256_set1_epi16(36);
let shift1 = _mm256_set1_epi16(1 << 13);
let mask = _mm256_set1_epi16(2047);
let shift2 = _mm256_set1_epi64x(((2048u64 << 48) + (1u64 << 32) + (2048 << 16) + 1) as i64);
let sllvdidx = _mm256_set1_epi64x(10);
let srlvqidx = _mm256_set_epi64x(30, 10, 30, 10);
let shufbidx = _mm256_set_epi8(
4, 3, 2, 1, 0, 0, -1, -1, -1, -1, 10, 9, 8, 7, 6, 5, -1, -1, -1, -1, -1, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
);
for i in 0..KYBER_N / 16 {
f0 = _mm256_load_si256(&a.vec[i]);
f1 = _mm256_mullo_epi16(f0, v8);
f2 = _mm256_add_epi16(f0, off);
f0 = _mm256_slli_epi16(f0, 3);
f0 = _mm256_mulhi_epi16(f0, v);
f2 = _mm256_sub_epi16(f1, f2);
f1 = _mm256_andnot_si256(f1, f2);
f1 = _mm256_srli_epi16(f1, 15);
f0 = _mm256_sub_epi16(f0, f1);
f0 = _mm256_mulhrs_epi16(f0, shift1);
f0 = _mm256_and_si256(f0, mask);
f0 = _mm256_madd_epi16(f0, shift2);
f0 = _mm256_sllv_epi32(f0, sllvdidx);
f1 = _mm256_bsrli_epi128(f0, 8);
f0 = _mm256_srlv_epi64(f0, srlvqidx);
f1 = _mm256_slli_epi64(f1, 34);
f0 = _mm256_add_epi64(f0, f1);
f0 = _mm256_shuffle_epi8(f0, shufbidx);
t0 = _mm256_castsi256_si128(f0);
t1 = _mm256_extracti128_si256(f0, 1);
t0 = _mm_blendv_epi8(t0, t1, _mm256_castsi256_si128(shufbidx));
_mm_storeu_si128(r[22 * i + 0..].as_mut_ptr() as *mut __m128i, t0);
_mm_storel_epi64(r[22 * i + 16..].as_mut_ptr() as *mut __m128i, t1);
}
} }
pub unsafe fn polyvec_frombytes(r: &mut Polyvec, a: &[u8]) // #[target_feature(enable = "avx")]
{ pub unsafe fn poly_decompress11(r: &mut Poly, a: &[u8]) {
for i in 0..KYBER_K { let mut f;
poly_frombytes(&mut r.vec[i], &a[i*KYBER_POLYBYTES..]);
} let q = _mm256_load_si256(QDATA.vec[_16XQ / 16..].as_ptr());
let shufbidx = _mm256_set_epi8(
13, 12, 12, 11, 10, 9, 9, 8, 8, 7, 6, 5, 5, 4, 4, 3, 10, 9, 9, 8, 7, 6, 6, 5, 5, 4, 3, 2, 2, 1, 1, 0,
);
let srlvdidx = _mm256_set_epi32(0, 0, 1, 0, 0, 0, 1, 0);
let srlvqidx = _mm256_set_epi64x(2, 0, 2, 0);
let shift = _mm256_set_epi16(4, 32, 1, 8, 32, 1, 4, 32, 4, 32, 1, 8, 32, 1, 4, 32);
let mask = _mm256_set1_epi16(32752);
for i in 0..(KYBER_N / 16) {
f = _mm256_loadu_si256(a[22 * i..].as_ptr() as *const __m256i);
f = _mm256_permute4x64_epi64(f, 0x94);
f = _mm256_shuffle_epi8(f, shufbidx);
f = _mm256_srlv_epi32(f, srlvdidx);
f = _mm256_srlv_epi64(f, srlvqidx);
f = _mm256_mullo_epi16(f, shift);
f = _mm256_srli_epi16(f, 1);
f = _mm256_and_si256(f, mask);
f = _mm256_mulhrs_epi16(f, q);
_mm256_store_si256(&mut r.vec[i], f);
}
}
pub unsafe fn polyvec_compress(r: &mut [u8], a: &Polyvec) {
if cfg!(feature = "kyber1024") {
for i in 0..KYBER_K {
poly_compress11(&mut r[352 * i..], &a.vec[i]);
}
} else {
for i in 0..KYBER_K {
poly_compress10(&mut r[320 * i..], &a.vec[i]);
}
}
}
pub unsafe fn polyvec_decompress(r: &mut Polyvec, a: &[u8]) {
if cfg!(feature = "kyber1024") {
for i in 0..KYBER_K {
poly_decompress11(&mut r.vec[i], &a[352 * i..]);
}
} else {
for i in 0..KYBER_K {
poly_decompress10(&mut r.vec[i], &a[320 * i..]);
}
}
}
pub fn polyvec_tobytes(r: &mut [u8], a: &Polyvec) {
for i in 0..KYBER_K {
poly_tobytes(&mut r[i * KYBER_POLYBYTES..], a.vec[i]);
}
}
pub unsafe fn polyvec_frombytes(r: &mut Polyvec, a: &[u8]) {
for i in 0..KYBER_K {
poly_frombytes(&mut r.vec[i], &a[i * KYBER_POLYBYTES..]);
}
} }
// Name: polyvec_ntt // Name: polyvec_ntt
@ -224,11 +199,10 @@ pub unsafe fn polyvec_frombytes(r: &mut Polyvec, a: &[u8])
// Description: Apply forward NTT to all elements of a vector of polynomials // Description: Apply forward NTT to all elements of a vector of polynomials
// //
// Arguments: - Polyvec r: in/output vector of polynomials // Arguments: - Polyvec r: in/output vector of polynomials
pub fn polyvec_ntt(r: &mut Polyvec) pub fn polyvec_ntt(r: &mut Polyvec) {
{ for i in 0..KYBER_K {
for i in 0..KYBER_K { poly_ntt(&mut r.vec[i]);
poly_ntt(&mut r.vec[i]); }
}
} }
// Name: polyvec_invntt // Name: polyvec_invntt
@ -236,11 +210,10 @@ pub fn polyvec_ntt(r: &mut Polyvec)
// Description: Apply inverse NTT to all elements of a vector of polynomials // Description: Apply inverse NTT to all elements of a vector of polynomials
// //
// Arguments: - Polyvec r: in/output vector of polynomials // Arguments: - Polyvec r: in/output vector of polynomials
pub fn polyvec_invntt_tomont(r: &mut Polyvec) pub fn polyvec_invntt_tomont(r: &mut Polyvec) {
{ for i in 0..KYBER_K {
for i in 0..KYBER_K { poly_invntt_tomont(&mut r.vec[i]);
poly_invntt_tomont(&mut r.vec[i]); }
}
} }
// Name: polyvec_basemul_acc_montgomery // Name: polyvec_basemul_acc_montgomery
@ -250,31 +223,28 @@ pub fn polyvec_invntt_tomont(r: &mut Polyvec)
// Arguments: - poly *r: output polynomial // Arguments: - poly *r: output polynomial
// - const Polyvec a: first input vector of polynomials // - const Polyvec a: first input vector of polynomials
// - const Polyvec b: second input vector of polynomials // - const Polyvec b: second input vector of polynomials
pub fn polyvec_basemul_acc_montgomery(r: &mut Poly, a: &Polyvec, b: &Polyvec) pub fn polyvec_basemul_acc_montgomery(r: &mut Poly, a: &Polyvec, b: &Polyvec) {
{ let mut t = Poly::new();
let mut t = Poly::new(); poly_basemul(r, &a.vec[0], &b.vec[0]);
poly_basemul(r, &a.vec[0], &b.vec[0]); for i in 1..KYBER_K {
for i in 1..KYBER_K { poly_basemul(&mut t, &a.vec[i], &b.vec[i]);
poly_basemul(&mut t, &a.vec[i], &b.vec[i]); poly_add(r, &t);
poly_add(r, &t); }
}
} }
// Name: polyvec_reduce // Name: polyvec_reduce
// //
// Description: Applies Barrett reduction to each coefficient // Description: Applies Barrett reduction to each coefficient
// of each element of a vector of polynomials // of each element of a vector of polynomials
// for details of the Barrett reduction see comments in reduce.c // for details of the Barrett reduction see comments in reduce.c
// //
// Arguments: - poly *r: input/output polynomial // Arguments: - poly *r: input/output polynomial
pub fn polyvec_reduce(r: &mut Polyvec) pub fn polyvec_reduce(r: &mut Polyvec) {
{ for i in 0..KYBER_K {
for i in 0..KYBER_K { poly_reduce(&mut r.vec[i]);
poly_reduce(&mut r.vec[i]); }
}
} }
// Name: polyvec_add // Name: polyvec_add
// //
// Description: Add vectors of polynomials // Description: Add vectors of polynomials
@ -282,9 +252,8 @@ pub fn polyvec_reduce(r: &mut Polyvec)
// Arguments: - Polyvec r: output vector of polynomials // Arguments: - Polyvec r: output vector of polynomials
// - const Polyvec a: first input vector of polynomials // - const Polyvec a: first input vector of polynomials
// - const Polyvec b: second input vector of polynomials // - const Polyvec b: second input vector of polynomials
pub fn polyvec_add(r: &mut Polyvec, b: &Polyvec) pub fn polyvec_add(r: &mut Polyvec, b: &Polyvec) {
{ for i in 0..KYBER_K {
for i in 0..KYBER_K { poly_add(&mut r.vec[i], &b.vec[i]);
poly_add(&mut r.vec[i], &b.vec[i]); }
}
} }

710
third_party/kyber/src/avx2/rejsample.rs vendored
View file

@ -1,380 +1,372 @@
use crate::{consts::*, params::*, symmetric::*};
use core::arch::x86_64::*; use core::arch::x86_64::*;
use crate::{
params::*,
consts::*,
symmetric::*
};
pub(crate) const REJ_UNIFORM_AVX_NBLOCKS: usize = pub(crate) const REJ_UNIFORM_AVX_NBLOCKS: usize = (12 * KYBER_N / 8 * (1 << 12) / KYBER_Q + XOF_BLOCKBYTES) / XOF_BLOCKBYTES;
(12*KYBER_N/8*(1 << 12)/KYBER_Q + XOF_BLOCKBYTES)/XOF_BLOCKBYTES; const REJ_UNIFORM_AVX_BUFLEN: usize = REJ_UNIFORM_AVX_NBLOCKS * XOF_BLOCKBYTES;
const REJ_UNIFORM_AVX_BUFLEN: usize = REJ_UNIFORM_AVX_NBLOCKS*XOF_BLOCKBYTES;
pub unsafe fn _mm256_cmpge_epu16(a: __m256i, b: __m256i) -> __m256i { pub unsafe fn _mm256_cmpge_epu16(a: __m256i, b: __m256i) -> __m256i {
_mm256_cmpeq_epi16(_mm256_max_epu16(a, b), a) _mm256_cmpeq_epi16(_mm256_max_epu16(a, b), a)
} }
pub unsafe fn _mm_cmpge_epu16(a: __m128i, b: __m128i) -> __m128i { pub unsafe fn _mm_cmpge_epu16(a: __m128i, b: __m128i) -> __m128i {
_mm_cmpeq_epi16(_mm_max_epu16(a, b), a) _mm_cmpeq_epi16(_mm_max_epu16(a, b), a)
} }
pub unsafe fn rej_uniform_avx(r: &mut[i16], buf: &[u8]) -> usize { pub unsafe fn rej_uniform_avx(r: &mut [i16], buf: &[u8]) -> usize {
let mut ctr = 0; let mut ctr = 0;
let mut pos = 0; let mut pos = 0;
let mut good: usize; let mut good: usize;
let (mut val0, mut val1); let (mut val0, mut val1);
let (mut f0, mut f1, mut g0, mut g1, mut g2, mut g3); let (mut f0, mut f1, mut g0, mut g1, mut g2, mut g3);
let (mut f, mut t, mut pilo, mut pihi); let (mut f, mut t, mut pilo, mut pihi);
let qdata_ptr = QDATA.coeffs[_16XQ..].as_ptr(); let qdata_ptr = QDATA.coeffs[_16XQ..].as_ptr();
let bound = _mm256_load_si256(qdata_ptr as *const __m256i); let bound = _mm256_load_si256(qdata_ptr as *const __m256i);
let ones = _mm256_set1_epi8(1); let ones = _mm256_set1_epi8(1);
let mask = _mm256_set1_epi16(0xFFF); let mask = _mm256_set1_epi16(0xFFF);
let idx8 = _mm256_set_epi8( let idx8 = _mm256_set_epi8(
15,14,14,13,12,11,11,10, 15, 14, 14, 13, 12, 11, 11, 10, 9, 8, 8, 7, 6, 5, 5, 4, 11, 10, 10, 9, 8, 7, 7, 6, 5, 4, 4, 3, 2, 1, 1, 0,
9, 8, 8, 7, 6, 5, 5, 4, );
11,10,10, 9, 8, 7, 7, 6, while ctr <= KYBER_N - 32 && pos <= REJ_UNIFORM_AVX_BUFLEN - 48 {
5, 4, 4, 3, 2, 1, 1, 0 f0 = _mm256_loadu_si256(buf[pos..].as_ptr() as *const __m256i);
); f1 = _mm256_loadu_si256(buf[pos + 24..].as_ptr() as *const __m256i);
while ctr <= KYBER_N - 32 && pos <= REJ_UNIFORM_AVX_BUFLEN - 48 { f0 = _mm256_permute4x64_epi64(f0, 0x94);
f0 = _mm256_loadu_si256(buf[pos..].as_ptr() as *const __m256i); f1 = _mm256_permute4x64_epi64(f1, 0x94);
f1 = _mm256_loadu_si256(buf[pos+24..].as_ptr() as *const __m256i); f0 = _mm256_shuffle_epi8(f0, idx8);
f0 = _mm256_permute4x64_epi64(f0, 0x94); f1 = _mm256_shuffle_epi8(f1, idx8);
f1 = _mm256_permute4x64_epi64(f1, 0x94); g0 = _mm256_srli_epi16(f0, 4);
f0 = _mm256_shuffle_epi8(f0, idx8); g1 = _mm256_srli_epi16(f1, 4);
f1 = _mm256_shuffle_epi8(f1, idx8); f0 = _mm256_blend_epi16(f0, g0, 0xAA);
g0 = _mm256_srli_epi16(f0, 4); f1 = _mm256_blend_epi16(f1, g1, 0xAA);
g1 = _mm256_srli_epi16(f1, 4); f0 = _mm256_and_si256(f0, mask);
f0 = _mm256_blend_epi16(f0, g0, 0xAA); f1 = _mm256_and_si256(f1, mask);
f1 = _mm256_blend_epi16(f1, g1, 0xAA); pos += 48;
f0 = _mm256_and_si256(f0, mask);
f1 = _mm256_and_si256(f1, mask);
pos += 48;
g0 = _mm256_cmpgt_epi16(bound, f0); g0 = _mm256_cmpgt_epi16(bound, f0);
g1 = _mm256_cmpgt_epi16(bound, f1); g1 = _mm256_cmpgt_epi16(bound, f1);
g0 = _mm256_packs_epi16(g0, g1); g0 = _mm256_packs_epi16(g0, g1);
good = _mm256_movemask_epi8(g0) as usize; good = _mm256_movemask_epi8(g0) as usize;
let mut l0 = _mm_loadl_epi64(IDX[(good >> 0) & 0xFF].as_ptr() as * const __m128i); let mut l0 = _mm_loadl_epi64(IDX[(good >> 0) & 0xFF].as_ptr() as *const __m128i);
g0 = _mm256_castsi128_si256(l0); g0 = _mm256_castsi128_si256(l0);
let mut l1 = _mm_loadl_epi64(IDX[(good >> 8) & 0xFF].as_ptr() as *const __m128i); let mut l1 = _mm_loadl_epi64(IDX[(good >> 8) & 0xFF].as_ptr() as *const __m128i);
g1 = _mm256_castsi128_si256(l1); g1 = _mm256_castsi128_si256(l1);
l0 = _mm_loadl_epi64(IDX[(good >> 16) & 0xFF].as_ptr() as *const __m128i); l0 = _mm_loadl_epi64(IDX[(good >> 16) & 0xFF].as_ptr() as *const __m128i);
g0 = _mm256_inserti128_si256(g0, l0, 1); g0 = _mm256_inserti128_si256(g0, l0, 1);
l1 = _mm_loadl_epi64(IDX[(good >> 24) & 0xFF].as_ptr() as *const __m128i); l1 = _mm_loadl_epi64(IDX[(good >> 24) & 0xFF].as_ptr() as *const __m128i);
g1 = _mm256_inserti128_si256(g1, l1, 1); g1 = _mm256_inserti128_si256(g1, l1, 1);
g2 = _mm256_add_epi8(g0, ones);
g3 = _mm256_add_epi8(g1, ones);
g0 = _mm256_unpacklo_epi8(g0, g2);
g1 = _mm256_unpacklo_epi8(g1, g3);
f0 = _mm256_shuffle_epi8(f0, g0); g2 = _mm256_add_epi8(g0, ones);
f1 = _mm256_shuffle_epi8(f1, g1); g3 = _mm256_add_epi8(g1, ones);
g0 = _mm256_unpacklo_epi8(g0, g2);
g1 = _mm256_unpacklo_epi8(g1, g3);
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(f0)); f0 = _mm256_shuffle_epi8(f0, g0);
ctr += _popcnt32(((good >> 0) & 0xFF) as i32) as usize; f1 = _mm256_shuffle_epi8(f1, g1);
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_extracti128_si256(f0, 1));
ctr += _popcnt32(((good >> 16) & 0xFF) as i32) as usize;
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(f1));
ctr += _popcnt32(((good >> 8) & 0xFF) as i32) as usize;
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_extracti128_si256(f1, 1));
ctr += _popcnt32(((good >> 24) & 0xFF) as i32) as usize;
}
while ctr <= KYBER_N - 8 && pos <= REJ_UNIFORM_AVX_BUFLEN - 12 { _mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(f0));
f = _mm_loadu_si128(buf[pos..].as_ptr() as *const __m128i); ctr += _popcnt32(((good >> 0) & 0xFF) as i32) as usize;
f = _mm_shuffle_epi8(f, _mm256_castsi256_si128(idx8)); _mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_extracti128_si256(f0, 1));
t = _mm_srli_epi16(f, 4); ctr += _popcnt32(((good >> 16) & 0xFF) as i32) as usize;
f = _mm_blend_epi16(f, t, 0xAA); _mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(f1));
f = _mm_and_si128(f, _mm256_castsi256_si128(mask)); ctr += _popcnt32(((good >> 8) & 0xFF) as i32) as usize;
pos += 12; _mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_extracti128_si256(f1, 1));
ctr += _popcnt32(((good >> 24) & 0xFF) as i32) as usize;
t = _mm_cmpgt_epi16(_mm256_castsi256_si128(bound), f);
good = _mm_movemask_epi8(t) as usize;
let good = _pext_u32(good as u32, 0x5555) as usize;
pilo = _mm_loadl_epi64(IDX[good][..].as_ptr() as *const __m128i);
pihi = _mm_add_epi8(pilo, _mm256_castsi256_si128(ones));
pilo = _mm_unpacklo_epi8(pilo, pihi);
f = _mm_shuffle_epi8(f, pilo);
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, f);
ctr += _popcnt32(good as i32) as usize;
}
while ctr < KYBER_N && pos <= REJ_UNIFORM_AVX_BUFLEN - 3 {
val0 = (buf[pos+0] >> 0) as u16 | ((buf[pos+1] as u16) << 8) & 0xFFF;
val1 = (buf[pos+1] >> 4) as u16 | ((buf[pos+2] as u16) << 4);
pos += 3;
if (val0 as usize) < KYBER_Q {
r[ctr] = val0 as i16;
ctr += 1;
} }
if (val1 as usize) < KYBER_Q && ctr < KYBER_N {
r[ctr] = val1 as i16; while ctr <= KYBER_N - 8 && pos <= REJ_UNIFORM_AVX_BUFLEN - 12 {
ctr += 1; f = _mm_loadu_si128(buf[pos..].as_ptr() as *const __m128i);
f = _mm_shuffle_epi8(f, _mm256_castsi256_si128(idx8));
t = _mm_srli_epi16(f, 4);
f = _mm_blend_epi16(f, t, 0xAA);
f = _mm_and_si128(f, _mm256_castsi256_si128(mask));
pos += 12;
t = _mm_cmpgt_epi16(_mm256_castsi256_si128(bound), f);
good = _mm_movemask_epi8(t) as usize;
let good = _pext_u32(good as u32, 0x5555) as usize;
pilo = _mm_loadl_epi64(IDX[good][..].as_ptr() as *const __m128i);
pihi = _mm_add_epi8(pilo, _mm256_castsi256_si128(ones));
pilo = _mm_unpacklo_epi8(pilo, pihi);
f = _mm_shuffle_epi8(f, pilo);
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, f);
ctr += _popcnt32(good as i32) as usize;
} }
}
ctr while ctr < KYBER_N && pos <= REJ_UNIFORM_AVX_BUFLEN - 3 {
val0 = (buf[pos + 0] >> 0) as u16 | ((buf[pos + 1] as u16) << 8) & 0xFFF;
val1 = (buf[pos + 1] >> 4) as u16 | ((buf[pos + 2] as u16) << 4);
pos += 3;
if (val0 as usize) < KYBER_Q {
r[ctr] = val0 as i16;
ctr += 1;
}
if (val1 as usize) < KYBER_Q && ctr < KYBER_N {
r[ctr] = val1 as i16;
ctr += 1;
}
}
ctr
} }
const IDX: [[i8; 8]; 256] = [ const IDX: [[i8; 8]; 256] = [
[-1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1],
[ 0, -1, -1, -1, -1, -1, -1, -1], [0, -1, -1, -1, -1, -1, -1, -1],
[ 2, -1, -1, -1, -1, -1, -1, -1], [2, -1, -1, -1, -1, -1, -1, -1],
[ 0, 2, -1, -1, -1, -1, -1, -1], [0, 2, -1, -1, -1, -1, -1, -1],
[ 4, -1, -1, -1, -1, -1, -1, -1], [4, -1, -1, -1, -1, -1, -1, -1],
[ 0, 4, -1, -1, -1, -1, -1, -1], [0, 4, -1, -1, -1, -1, -1, -1],
[ 2, 4, -1, -1, -1, -1, -1, -1], [2, 4, -1, -1, -1, -1, -1, -1],
[ 0, 2, 4, -1, -1, -1, -1, -1], [0, 2, 4, -1, -1, -1, -1, -1],
[ 6, -1, -1, -1, -1, -1, -1, -1], [6, -1, -1, -1, -1, -1, -1, -1],
[ 0, 6, -1, -1, -1, -1, -1, -1], [0, 6, -1, -1, -1, -1, -1, -1],
[ 2, 6, -1, -1, -1, -1, -1, -1], [2, 6, -1, -1, -1, -1, -1, -1],
[ 0, 2, 6, -1, -1, -1, -1, -1], [0, 2, 6, -1, -1, -1, -1, -1],
[ 4, 6, -1, -1, -1, -1, -1, -1], [4, 6, -1, -1, -1, -1, -1, -1],
[ 0, 4, 6, -1, -1, -1, -1, -1], [0, 4, 6, -1, -1, -1, -1, -1],
[ 2, 4, 6, -1, -1, -1, -1, -1], [2, 4, 6, -1, -1, -1, -1, -1],
[ 0, 2, 4, 6, -1, -1, -1, -1], [0, 2, 4, 6, -1, -1, -1, -1],
[ 8, -1, -1, -1, -1, -1, -1, -1], [8, -1, -1, -1, -1, -1, -1, -1],
[ 0, 8, -1, -1, -1, -1, -1, -1], [0, 8, -1, -1, -1, -1, -1, -1],
[ 2, 8, -1, -1, -1, -1, -1, -1], [2, 8, -1, -1, -1, -1, -1, -1],
[ 0, 2, 8, -1, -1, -1, -1, -1], [0, 2, 8, -1, -1, -1, -1, -1],
[ 4, 8, -1, -1, -1, -1, -1, -1], [4, 8, -1, -1, -1, -1, -1, -1],
[ 0, 4, 8, -1, -1, -1, -1, -1], [0, 4, 8, -1, -1, -1, -1, -1],
[ 2, 4, 8, -1, -1, -1, -1, -1], [2, 4, 8, -1, -1, -1, -1, -1],
[ 0, 2, 4, 8, -1, -1, -1, -1], [0, 2, 4, 8, -1, -1, -1, -1],
[ 6, 8, -1, -1, -1, -1, -1, -1], [6, 8, -1, -1, -1, -1, -1, -1],
[ 0, 6, 8, -1, -1, -1, -1, -1], [0, 6, 8, -1, -1, -1, -1, -1],
[ 2, 6, 8, -1, -1, -1, -1, -1], [2, 6, 8, -1, -1, -1, -1, -1],
[ 0, 2, 6, 8, -1, -1, -1, -1], [0, 2, 6, 8, -1, -1, -1, -1],
[ 4, 6, 8, -1, -1, -1, -1, -1], [4, 6, 8, -1, -1, -1, -1, -1],
[ 0, 4, 6, 8, -1, -1, -1, -1], [0, 4, 6, 8, -1, -1, -1, -1],
[ 2, 4, 6, 8, -1, -1, -1, -1], [2, 4, 6, 8, -1, -1, -1, -1],
[ 0, 2, 4, 6, 8, -1, -1, -1], [0, 2, 4, 6, 8, -1, -1, -1],
[10, -1, -1, -1, -1, -1, -1, -1], [10, -1, -1, -1, -1, -1, -1, -1],
[ 0, 10, -1, -1, -1, -1, -1, -1], [0, 10, -1, -1, -1, -1, -1, -1],
[ 2, 10, -1, -1, -1, -1, -1, -1], [2, 10, -1, -1, -1, -1, -1, -1],
[ 0, 2, 10, -1, -1, -1, -1, -1], [0, 2, 10, -1, -1, -1, -1, -1],
[ 4, 10, -1, -1, -1, -1, -1, -1], [4, 10, -1, -1, -1, -1, -1, -1],
[ 0, 4, 10, -1, -1, -1, -1, -1], [0, 4, 10, -1, -1, -1, -1, -1],
[ 2, 4, 10, -1, -1, -1, -1, -1], [2, 4, 10, -1, -1, -1, -1, -1],
[ 0, 2, 4, 10, -1, -1, -1, -1], [0, 2, 4, 10, -1, -1, -1, -1],
[ 6, 10, -1, -1, -1, -1, -1, -1], [6, 10, -1, -1, -1, -1, -1, -1],
[ 0, 6, 10, -1, -1, -1, -1, -1], [0, 6, 10, -1, -1, -1, -1, -1],
[ 2, 6, 10, -1, -1, -1, -1, -1], [2, 6, 10, -1, -1, -1, -1, -1],
[ 0, 2, 6, 10, -1, -1, -1, -1], [0, 2, 6, 10, -1, -1, -1, -1],
[ 4, 6, 10, -1, -1, -1, -1, -1], [4, 6, 10, -1, -1, -1, -1, -1],
[ 0, 4, 6, 10, -1, -1, -1, -1], [0, 4, 6, 10, -1, -1, -1, -1],
[ 2, 4, 6, 10, -1, -1, -1, -1], [2, 4, 6, 10, -1, -1, -1, -1],
[ 0, 2, 4, 6, 10, -1, -1, -1], [0, 2, 4, 6, 10, -1, -1, -1],
[ 8, 10, -1, -1, -1, -1, -1, -1], [8, 10, -1, -1, -1, -1, -1, -1],
[ 0, 8, 10, -1, -1, -1, -1, -1], [0, 8, 10, -1, -1, -1, -1, -1],
[ 2, 8, 10, -1, -1, -1, -1, -1], [2, 8, 10, -1, -1, -1, -1, -1],
[ 0, 2, 8, 10, -1, -1, -1, -1], [0, 2, 8, 10, -1, -1, -1, -1],
[ 4, 8, 10, -1, -1, -1, -1, -1], [4, 8, 10, -1, -1, -1, -1, -1],
[ 0, 4, 8, 10, -1, -1, -1, -1], [0, 4, 8, 10, -1, -1, -1, -1],
[ 2, 4, 8, 10, -1, -1, -1, -1], [2, 4, 8, 10, -1, -1, -1, -1],
[ 0, 2, 4, 8, 10, -1, -1, -1], [0, 2, 4, 8, 10, -1, -1, -1],
[ 6, 8, 10, -1, -1, -1, -1, -1], [6, 8, 10, -1, -1, -1, -1, -1],
[ 0, 6, 8, 10, -1, -1, -1, -1], [0, 6, 8, 10, -1, -1, -1, -1],
[ 2, 6, 8, 10, -1, -1, -1, -1], [2, 6, 8, 10, -1, -1, -1, -1],
[ 0, 2, 6, 8, 10, -1, -1, -1], [0, 2, 6, 8, 10, -1, -1, -1],
[ 4, 6, 8, 10, -1, -1, -1, -1], [4, 6, 8, 10, -1, -1, -1, -1],
[ 0, 4, 6, 8, 10, -1, -1, -1], [0, 4, 6, 8, 10, -1, -1, -1],
[ 2, 4, 6, 8, 10, -1, -1, -1], [2, 4, 6, 8, 10, -1, -1, -1],
[ 0, 2, 4, 6, 8, 10, -1, -1], [0, 2, 4, 6, 8, 10, -1, -1],
[12, -1, -1, -1, -1, -1, -1, -1], [12, -1, -1, -1, -1, -1, -1, -1],
[ 0, 12, -1, -1, -1, -1, -1, -1], [0, 12, -1, -1, -1, -1, -1, -1],
[ 2, 12, -1, -1, -1, -1, -1, -1], [2, 12, -1, -1, -1, -1, -1, -1],
[ 0, 2, 12, -1, -1, -1, -1, -1], [0, 2, 12, -1, -1, -1, -1, -1],
[ 4, 12, -1, -1, -1, -1, -1, -1], [4, 12, -1, -1, -1, -1, -1, -1],
[ 0, 4, 12, -1, -1, -1, -1, -1], [0, 4, 12, -1, -1, -1, -1, -1],
[ 2, 4, 12, -1, -1, -1, -1, -1], [2, 4, 12, -1, -1, -1, -1, -1],
[ 0, 2, 4, 12, -1, -1, -1, -1], [0, 2, 4, 12, -1, -1, -1, -1],
[ 6, 12, -1, -1, -1, -1, -1, -1], [6, 12, -1, -1, -1, -1, -1, -1],
[ 0, 6, 12, -1, -1, -1, -1, -1], [0, 6, 12, -1, -1, -1, -1, -1],
[ 2, 6, 12, -1, -1, -1, -1, -1], [2, 6, 12, -1, -1, -1, -1, -1],
[ 0, 2, 6, 12, -1, -1, -1, -1], [0, 2, 6, 12, -1, -1, -1, -1],
[ 4, 6, 12, -1, -1, -1, -1, -1], [4, 6, 12, -1, -1, -1, -1, -1],
[ 0, 4, 6, 12, -1, -1, -1, -1], [0, 4, 6, 12, -1, -1, -1, -1],
[ 2, 4, 6, 12, -1, -1, -1, -1], [2, 4, 6, 12, -1, -1, -1, -1],
[ 0, 2, 4, 6, 12, -1, -1, -1], [0, 2, 4, 6, 12, -1, -1, -1],
[ 8, 12, -1, -1, -1, -1, -1, -1], [8, 12, -1, -1, -1, -1, -1, -1],
[ 0, 8, 12, -1, -1, -1, -1, -1], [0, 8, 12, -1, -1, -1, -1, -1],
[ 2, 8, 12, -1, -1, -1, -1, -1], [2, 8, 12, -1, -1, -1, -1, -1],
[ 0, 2, 8, 12, -1, -1, -1, -1], [0, 2, 8, 12, -1, -1, -1, -1],
[ 4, 8, 12, -1, -1, -1, -1, -1], [4, 8, 12, -1, -1, -1, -1, -1],
[ 0, 4, 8, 12, -1, -1, -1, -1], [0, 4, 8, 12, -1, -1, -1, -1],
[ 2, 4, 8, 12, -1, -1, -1, -1], [2, 4, 8, 12, -1, -1, -1, -1],
[ 0, 2, 4, 8, 12, -1, -1, -1], [0, 2, 4, 8, 12, -1, -1, -1],
[ 6, 8, 12, -1, -1, -1, -1, -1], [6, 8, 12, -1, -1, -1, -1, -1],
[ 0, 6, 8, 12, -1, -1, -1, -1], [0, 6, 8, 12, -1, -1, -1, -1],
[ 2, 6, 8, 12, -1, -1, -1, -1], [2, 6, 8, 12, -1, -1, -1, -1],
[ 0, 2, 6, 8, 12, -1, -1, -1], [0, 2, 6, 8, 12, -1, -1, -1],
[ 4, 6, 8, 12, -1, -1, -1, -1], [4, 6, 8, 12, -1, -1, -1, -1],
[ 0, 4, 6, 8, 12, -1, -1, -1], [0, 4, 6, 8, 12, -1, -1, -1],
[ 2, 4, 6, 8, 12, -1, -1, -1], [2, 4, 6, 8, 12, -1, -1, -1],
[ 0, 2, 4, 6, 8, 12, -1, -1], [0, 2, 4, 6, 8, 12, -1, -1],
[10, 12, -1, -1, -1, -1, -1, -1], [10, 12, -1, -1, -1, -1, -1, -1],
[ 0, 10, 12, -1, -1, -1, -1, -1], [0, 10, 12, -1, -1, -1, -1, -1],
[ 2, 10, 12, -1, -1, -1, -1, -1], [2, 10, 12, -1, -1, -1, -1, -1],
[ 0, 2, 10, 12, -1, -1, -1, -1], [0, 2, 10, 12, -1, -1, -1, -1],
[ 4, 10, 12, -1, -1, -1, -1, -1], [4, 10, 12, -1, -1, -1, -1, -1],
[ 0, 4, 10, 12, -1, -1, -1, -1], [0, 4, 10, 12, -1, -1, -1, -1],
[ 2, 4, 10, 12, -1, -1, -1, -1], [2, 4, 10, 12, -1, -1, -1, -1],
[ 0, 2, 4, 10, 12, -1, -1, -1], [0, 2, 4, 10, 12, -1, -1, -1],
[ 6, 10, 12, -1, -1, -1, -1, -1], [6, 10, 12, -1, -1, -1, -1, -1],
[ 0, 6, 10, 12, -1, -1, -1, -1], [0, 6, 10, 12, -1, -1, -1, -1],
[ 2, 6, 10, 12, -1, -1, -1, -1], [2, 6, 10, 12, -1, -1, -1, -1],
[ 0, 2, 6, 10, 12, -1, -1, -1], [0, 2, 6, 10, 12, -1, -1, -1],
[ 4, 6, 10, 12, -1, -1, -1, -1], [4, 6, 10, 12, -1, -1, -1, -1],
[ 0, 4, 6, 10, 12, -1, -1, -1], [0, 4, 6, 10, 12, -1, -1, -1],
[ 2, 4, 6, 10, 12, -1, -1, -1], [2, 4, 6, 10, 12, -1, -1, -1],
[ 0, 2, 4, 6, 10, 12, -1, -1], [0, 2, 4, 6, 10, 12, -1, -1],
[ 8, 10, 12, -1, -1, -1, -1, -1], [8, 10, 12, -1, -1, -1, -1, -1],
[ 0, 8, 10, 12, -1, -1, -1, -1], [0, 8, 10, 12, -1, -1, -1, -1],
[ 2, 8, 10, 12, -1, -1, -1, -1], [2, 8, 10, 12, -1, -1, -1, -1],
[ 0, 2, 8, 10, 12, -1, -1, -1], [0, 2, 8, 10, 12, -1, -1, -1],
[ 4, 8, 10, 12, -1, -1, -1, -1], [4, 8, 10, 12, -1, -1, -1, -1],
[ 0, 4, 8, 10, 12, -1, -1, -1], [0, 4, 8, 10, 12, -1, -1, -1],
[ 2, 4, 8, 10, 12, -1, -1, -1], [2, 4, 8, 10, 12, -1, -1, -1],
[ 0, 2, 4, 8, 10, 12, -1, -1], [0, 2, 4, 8, 10, 12, -1, -1],
[ 6, 8, 10, 12, -1, -1, -1, -1], [6, 8, 10, 12, -1, -1, -1, -1],
[ 0, 6, 8, 10, 12, -1, -1, -1], [0, 6, 8, 10, 12, -1, -1, -1],
[ 2, 6, 8, 10, 12, -1, -1, -1], [2, 6, 8, 10, 12, -1, -1, -1],
[ 0, 2, 6, 8, 10, 12, -1, -1], [0, 2, 6, 8, 10, 12, -1, -1],
[ 4, 6, 8, 10, 12, -1, -1, -1], [4, 6, 8, 10, 12, -1, -1, -1],
[ 0, 4, 6, 8, 10, 12, -1, -1], [0, 4, 6, 8, 10, 12, -1, -1],
[ 2, 4, 6, 8, 10, 12, -1, -1], [2, 4, 6, 8, 10, 12, -1, -1],
[ 0, 2, 4, 6, 8, 10, 12, -1], [0, 2, 4, 6, 8, 10, 12, -1],
[14, -1, -1, -1, -1, -1, -1, -1], [14, -1, -1, -1, -1, -1, -1, -1],
[ 0, 14, -1, -1, -1, -1, -1, -1], [0, 14, -1, -1, -1, -1, -1, -1],
[ 2, 14, -1, -1, -1, -1, -1, -1], [2, 14, -1, -1, -1, -1, -1, -1],
[ 0, 2, 14, -1, -1, -1, -1, -1], [0, 2, 14, -1, -1, -1, -1, -1],
[ 4, 14, -1, -1, -1, -1, -1, -1], [4, 14, -1, -1, -1, -1, -1, -1],
[ 0, 4, 14, -1, -1, -1, -1, -1], [0, 4, 14, -1, -1, -1, -1, -1],
[ 2, 4, 14, -1, -1, -1, -1, -1], [2, 4, 14, -1, -1, -1, -1, -1],
[ 0, 2, 4, 14, -1, -1, -1, -1], [0, 2, 4, 14, -1, -1, -1, -1],
[ 6, 14, -1, -1, -1, -1, -1, -1], [6, 14, -1, -1, -1, -1, -1, -1],
[ 0, 6, 14, -1, -1, -1, -1, -1], [0, 6, 14, -1, -1, -1, -1, -1],
[ 2, 6, 14, -1, -1, -1, -1, -1], [2, 6, 14, -1, -1, -1, -1, -1],
[ 0, 2, 6, 14, -1, -1, -1, -1], [0, 2, 6, 14, -1, -1, -1, -1],
[ 4, 6, 14, -1, -1, -1, -1, -1], [4, 6, 14, -1, -1, -1, -1, -1],
[ 0, 4, 6, 14, -1, -1, -1, -1], [0, 4, 6, 14, -1, -1, -1, -1],
[ 2, 4, 6, 14, -1, -1, -1, -1], [2, 4, 6, 14, -1, -1, -1, -1],
[ 0, 2, 4, 6, 14, -1, -1, -1], [0, 2, 4, 6, 14, -1, -1, -1],
[ 8, 14, -1, -1, -1, -1, -1, -1], [8, 14, -1, -1, -1, -1, -1, -1],
[ 0, 8, 14, -1, -1, -1, -1, -1], [0, 8, 14, -1, -1, -1, -1, -1],
[ 2, 8, 14, -1, -1, -1, -1, -1], [2, 8, 14, -1, -1, -1, -1, -1],
[ 0, 2, 8, 14, -1, -1, -1, -1], [0, 2, 8, 14, -1, -1, -1, -1],
[ 4, 8, 14, -1, -1, -1, -1, -1], [4, 8, 14, -1, -1, -1, -1, -1],
[ 0, 4, 8, 14, -1, -1, -1, -1], [0, 4, 8, 14, -1, -1, -1, -1],
[ 2, 4, 8, 14, -1, -1, -1, -1], [2, 4, 8, 14, -1, -1, -1, -1],
[ 0, 2, 4, 8, 14, -1, -1, -1], [0, 2, 4, 8, 14, -1, -1, -1],
[ 6, 8, 14, -1, -1, -1, -1, -1], [6, 8, 14, -1, -1, -1, -1, -1],
[ 0, 6, 8, 14, -1, -1, -1, -1], [0, 6, 8, 14, -1, -1, -1, -1],
[ 2, 6, 8, 14, -1, -1, -1, -1], [2, 6, 8, 14, -1, -1, -1, -1],
[ 0, 2, 6, 8, 14, -1, -1, -1], [0, 2, 6, 8, 14, -1, -1, -1],
[ 4, 6, 8, 14, -1, -1, -1, -1], [4, 6, 8, 14, -1, -1, -1, -1],
[ 0, 4, 6, 8, 14, -1, -1, -1], [0, 4, 6, 8, 14, -1, -1, -1],
[ 2, 4, 6, 8, 14, -1, -1, -1], [2, 4, 6, 8, 14, -1, -1, -1],
[ 0, 2, 4, 6, 8, 14, -1, -1], [0, 2, 4, 6, 8, 14, -1, -1],
[10, 14, -1, -1, -1, -1, -1, -1], [10, 14, -1, -1, -1, -1, -1, -1],
[ 0, 10, 14, -1, -1, -1, -1, -1], [0, 10, 14, -1, -1, -1, -1, -1],
[ 2, 10, 14, -1, -1, -1, -1, -1], [2, 10, 14, -1, -1, -1, -1, -1],
[ 0, 2, 10, 14, -1, -1, -1, -1], [0, 2, 10, 14, -1, -1, -1, -1],
[ 4, 10, 14, -1, -1, -1, -1, -1], [4, 10, 14, -1, -1, -1, -1, -1],
[ 0, 4, 10, 14, -1, -1, -1, -1], [0, 4, 10, 14, -1, -1, -1, -1],
[ 2, 4, 10, 14, -1, -1, -1, -1], [2, 4, 10, 14, -1, -1, -1, -1],
[ 0, 2, 4, 10, 14, -1, -1, -1], [0, 2, 4, 10, 14, -1, -1, -1],
[ 6, 10, 14, -1, -1, -1, -1, -1], [6, 10, 14, -1, -1, -1, -1, -1],
[ 0, 6, 10, 14, -1, -1, -1, -1], [0, 6, 10, 14, -1, -1, -1, -1],
[ 2, 6, 10, 14, -1, -1, -1, -1], [2, 6, 10, 14, -1, -1, -1, -1],
[ 0, 2, 6, 10, 14, -1, -1, -1], [0, 2, 6, 10, 14, -1, -1, -1],
[ 4, 6, 10, 14, -1, -1, -1, -1], [4, 6, 10, 14, -1, -1, -1, -1],
[ 0, 4, 6, 10, 14, -1, -1, -1], [0, 4, 6, 10, 14, -1, -1, -1],
[ 2, 4, 6, 10, 14, -1, -1, -1], [2, 4, 6, 10, 14, -1, -1, -1],
[ 0, 2, 4, 6, 10, 14, -1, -1], [0, 2, 4, 6, 10, 14, -1, -1],
[ 8, 10, 14, -1, -1, -1, -1, -1], [8, 10, 14, -1, -1, -1, -1, -1],
[ 0, 8, 10, 14, -1, -1, -1, -1], [0, 8, 10, 14, -1, -1, -1, -1],
[ 2, 8, 10, 14, -1, -1, -1, -1], [2, 8, 10, 14, -1, -1, -1, -1],
[ 0, 2, 8, 10, 14, -1, -1, -1], [0, 2, 8, 10, 14, -1, -1, -1],
[ 4, 8, 10, 14, -1, -1, -1, -1], [4, 8, 10, 14, -1, -1, -1, -1],
[ 0, 4, 8, 10, 14, -1, -1, -1], [0, 4, 8, 10, 14, -1, -1, -1],
[ 2, 4, 8, 10, 14, -1, -1, -1], [2, 4, 8, 10, 14, -1, -1, -1],
[ 0, 2, 4, 8, 10, 14, -1, -1], [0, 2, 4, 8, 10, 14, -1, -1],
[ 6, 8, 10, 14, -1, -1, -1, -1], [6, 8, 10, 14, -1, -1, -1, -1],
[ 0, 6, 8, 10, 14, -1, -1, -1], [0, 6, 8, 10, 14, -1, -1, -1],
[ 2, 6, 8, 10, 14, -1, -1, -1], [2, 6, 8, 10, 14, -1, -1, -1],
[ 0, 2, 6, 8, 10, 14, -1, -1], [0, 2, 6, 8, 10, 14, -1, -1],
[ 4, 6, 8, 10, 14, -1, -1, -1], [4, 6, 8, 10, 14, -1, -1, -1],
[ 0, 4, 6, 8, 10, 14, -1, -1], [0, 4, 6, 8, 10, 14, -1, -1],
[ 2, 4, 6, 8, 10, 14, -1, -1], [2, 4, 6, 8, 10, 14, -1, -1],
[ 0, 2, 4, 6, 8, 10, 14, -1], [0, 2, 4, 6, 8, 10, 14, -1],
[12, 14, -1, -1, -1, -1, -1, -1], [12, 14, -1, -1, -1, -1, -1, -1],
[ 0, 12, 14, -1, -1, -1, -1, -1], [0, 12, 14, -1, -1, -1, -1, -1],
[ 2, 12, 14, -1, -1, -1, -1, -1], [2, 12, 14, -1, -1, -1, -1, -1],
[ 0, 2, 12, 14, -1, -1, -1, -1], [0, 2, 12, 14, -1, -1, -1, -1],
[ 4, 12, 14, -1, -1, -1, -1, -1], [4, 12, 14, -1, -1, -1, -1, -1],
[ 0, 4, 12, 14, -1, -1, -1, -1], [0, 4, 12, 14, -1, -1, -1, -1],
[ 2, 4, 12, 14, -1, -1, -1, -1], [2, 4, 12, 14, -1, -1, -1, -1],
[ 0, 2, 4, 12, 14, -1, -1, -1], [0, 2, 4, 12, 14, -1, -1, -1],
[ 6, 12, 14, -1, -1, -1, -1, -1], [6, 12, 14, -1, -1, -1, -1, -1],
[ 0, 6, 12, 14, -1, -1, -1, -1], [0, 6, 12, 14, -1, -1, -1, -1],
[ 2, 6, 12, 14, -1, -1, -1, -1], [2, 6, 12, 14, -1, -1, -1, -1],
[ 0, 2, 6, 12, 14, -1, -1, -1], [0, 2, 6, 12, 14, -1, -1, -1],
[ 4, 6, 12, 14, -1, -1, -1, -1], [4, 6, 12, 14, -1, -1, -1, -1],
[ 0, 4, 6, 12, 14, -1, -1, -1], [0, 4, 6, 12, 14, -1, -1, -1],
[ 2, 4, 6, 12, 14, -1, -1, -1], [2, 4, 6, 12, 14, -1, -1, -1],
[ 0, 2, 4, 6, 12, 14, -1, -1], [0, 2, 4, 6, 12, 14, -1, -1],
[ 8, 12, 14, -1, -1, -1, -1, -1], [8, 12, 14, -1, -1, -1, -1, -1],
[ 0, 8, 12, 14, -1, -1, -1, -1], [0, 8, 12, 14, -1, -1, -1, -1],
[ 2, 8, 12, 14, -1, -1, -1, -1], [2, 8, 12, 14, -1, -1, -1, -1],
[ 0, 2, 8, 12, 14, -1, -1, -1], [0, 2, 8, 12, 14, -1, -1, -1],
[ 4, 8, 12, 14, -1, -1, -1, -1], [4, 8, 12, 14, -1, -1, -1, -1],
[ 0, 4, 8, 12, 14, -1, -1, -1], [0, 4, 8, 12, 14, -1, -1, -1],
[ 2, 4, 8, 12, 14, -1, -1, -1], [2, 4, 8, 12, 14, -1, -1, -1],
[ 0, 2, 4, 8, 12, 14, -1, -1], [0, 2, 4, 8, 12, 14, -1, -1],
[ 6, 8, 12, 14, -1, -1, -1, -1], [6, 8, 12, 14, -1, -1, -1, -1],
[ 0, 6, 8, 12, 14, -1, -1, -1], [0, 6, 8, 12, 14, -1, -1, -1],
[ 2, 6, 8, 12, 14, -1, -1, -1], [2, 6, 8, 12, 14, -1, -1, -1],
[ 0, 2, 6, 8, 12, 14, -1, -1], [0, 2, 6, 8, 12, 14, -1, -1],
[ 4, 6, 8, 12, 14, -1, -1, -1], [4, 6, 8, 12, 14, -1, -1, -1],
[ 0, 4, 6, 8, 12, 14, -1, -1], [0, 4, 6, 8, 12, 14, -1, -1],
[ 2, 4, 6, 8, 12, 14, -1, -1], [2, 4, 6, 8, 12, 14, -1, -1],
[ 0, 2, 4, 6, 8, 12, 14, -1], [0, 2, 4, 6, 8, 12, 14, -1],
[10, 12, 14, -1, -1, -1, -1, -1], [10, 12, 14, -1, -1, -1, -1, -1],
[ 0, 10, 12, 14, -1, -1, -1, -1], [0, 10, 12, 14, -1, -1, -1, -1],
[ 2, 10, 12, 14, -1, -1, -1, -1], [2, 10, 12, 14, -1, -1, -1, -1],
[ 0, 2, 10, 12, 14, -1, -1, -1], [0, 2, 10, 12, 14, -1, -1, -1],
[ 4, 10, 12, 14, -1, -1, -1, -1], [4, 10, 12, 14, -1, -1, -1, -1],
[ 0, 4, 10, 12, 14, -1, -1, -1], [0, 4, 10, 12, 14, -1, -1, -1],
[ 2, 4, 10, 12, 14, -1, -1, -1], [2, 4, 10, 12, 14, -1, -1, -1],
[ 0, 2, 4, 10, 12, 14, -1, -1], [0, 2, 4, 10, 12, 14, -1, -1],
[ 6, 10, 12, 14, -1, -1, -1, -1], [6, 10, 12, 14, -1, -1, -1, -1],
[ 0, 6, 10, 12, 14, -1, -1, -1], [0, 6, 10, 12, 14, -1, -1, -1],
[ 2, 6, 10, 12, 14, -1, -1, -1], [2, 6, 10, 12, 14, -1, -1, -1],
[ 0, 2, 6, 10, 12, 14, -1, -1], [0, 2, 6, 10, 12, 14, -1, -1],
[ 4, 6, 10, 12, 14, -1, -1, -1], [4, 6, 10, 12, 14, -1, -1, -1],
[ 0, 4, 6, 10, 12, 14, -1, -1], [0, 4, 6, 10, 12, 14, -1, -1],
[ 2, 4, 6, 10, 12, 14, -1, -1], [2, 4, 6, 10, 12, 14, -1, -1],
[ 0, 2, 4, 6, 10, 12, 14, -1], [0, 2, 4, 6, 10, 12, 14, -1],
[ 8, 10, 12, 14, -1, -1, -1, -1], [8, 10, 12, 14, -1, -1, -1, -1],
[ 0, 8, 10, 12, 14, -1, -1, -1], [0, 8, 10, 12, 14, -1, -1, -1],
[ 2, 8, 10, 12, 14, -1, -1, -1], [2, 8, 10, 12, 14, -1, -1, -1],
[ 0, 2, 8, 10, 12, 14, -1, -1], [0, 2, 8, 10, 12, 14, -1, -1],
[ 4, 8, 10, 12, 14, -1, -1, -1], [4, 8, 10, 12, 14, -1, -1, -1],
[ 0, 4, 8, 10, 12, 14, -1, -1], [0, 4, 8, 10, 12, 14, -1, -1],
[ 2, 4, 8, 10, 12, 14, -1, -1], [2, 4, 8, 10, 12, 14, -1, -1],
[ 0, 2, 4, 8, 10, 12, 14, -1], [0, 2, 4, 8, 10, 12, 14, -1],
[ 6, 8, 10, 12, 14, -1, -1, -1], [6, 8, 10, 12, 14, -1, -1, -1],
[ 0, 6, 8, 10, 12, 14, -1, -1], [0, 6, 8, 10, 12, 14, -1, -1],
[ 2, 6, 8, 10, 12, 14, -1, -1], [2, 6, 8, 10, 12, 14, -1, -1],
[ 0, 2, 6, 8, 10, 12, 14, -1], [0, 2, 6, 8, 10, 12, 14, -1],
[ 4, 6, 8, 10, 12, 14, -1, -1], [4, 6, 8, 10, 12, 14, -1, -1],
[ 0, 4, 6, 8, 10, 12, 14, -1], [0, 4, 6, 8, 10, 12, 14, -1],
[ 2, 4, 6, 8, 10, 12, 14, -1], [2, 4, 6, 8, 10, 12, 14, -1],
[ 0, 2, 4, 6, 8, 10, 12, 14] [0, 2, 4, 6, 8, 10, 12, 14],
]; ];

72
third_party/kyber/src/avx2/verify.rs vendored
View file

@ -1,43 +1,41 @@
use core::arch::x86_64::*; use core::arch::x86_64::*;
pub fn verify(a: &[u8], b: &[u8], mut len: usize) -> u8 pub fn verify(a: &[u8], b: &[u8], mut len: usize) -> u8 {
{ let (mut f, mut g);
let (mut f, mut g); let mut r: u64;
let mut r: u64; unsafe {
unsafe { let mut h = _mm256_setzero_si256();
let mut h = _mm256_setzero_si256(); for i in 0..(len / 32) {
for i in 0..(len/32) { f = _mm256_loadu_si256(a[32 * i..].as_ptr() as *const __m256i);
f = _mm256_loadu_si256(a[32*i..].as_ptr() as *const __m256i); g = _mm256_loadu_si256(b[32 * i..].as_ptr() as *const __m256i);
g = _mm256_loadu_si256(b[32*i..].as_ptr() as *const __m256i); f = _mm256_xor_si256(f, g);
f = _mm256_xor_si256(f,g); h = _mm256_or_si256(h, f);
h = _mm256_or_si256(h,f); }
r = 1 - _mm256_testz_si256(h, h) as u64;
} }
r = 1 - _mm256_testz_si256(h,h) as u64; let idx = 32 * (len / 32);
} len -= idx;
let idx = 32*(len/32); for i in 0..len {
len -= idx; r |= (a[idx + i] ^ b[idx + i]) as u64;
for i in 0..len { }
r |= (a[idx+i] ^ b[idx+i]) as u64; (r.wrapping_neg() >> 63) as u8
}
(r.wrapping_neg() >> 63) as u8
} }
pub fn cmov(r: &mut[u8], x: &[u8], mut len: usize, mut b: u8) pub fn cmov(r: &mut [u8], x: &[u8], mut len: usize, mut b: u8) {
{ let (mut xvec, mut rvec);
let (mut xvec, mut rvec); unsafe {
unsafe { let bvec = _mm256_set1_epi64x(b as i64);
let bvec = _mm256_set1_epi64x(b as i64); for i in 0..(len / 32) {
for i in 0..(len/32) { rvec = _mm256_loadu_si256(r[32 * i..].as_ptr() as *const __m256i);
rvec = _mm256_loadu_si256(r[32*i..].as_ptr() as *const __m256i); xvec = _mm256_loadu_si256(x[32 * i..].as_ptr() as *const __m256i);
xvec = _mm256_loadu_si256(x[32*i..].as_ptr() as *const __m256i); rvec = _mm256_blendv_epi8(rvec, xvec, bvec);
rvec = _mm256_blendv_epi8(rvec,xvec,bvec); _mm256_storeu_si256(r[32 * i..].as_mut_ptr() as *mut __m256i, rvec);
_mm256_storeu_si256(r[32*i..].as_mut_ptr() as *mut __m256i,rvec); }
} }
} let idx = 32 * (len / 32);
let idx = 32*(len/32); len -= idx;
len -= idx; b = b.wrapping_neg();
b = b.wrapping_neg(); for i in 0..len {
for i in 0..len { r[idx + i] ^= b & (x[idx + i] ^ r[idx + i]);
r[idx+i] ^= b & (x[idx+i] ^ r[idx+i]); }
} }
}

22
third_party/kyber/src/error.rs vendored
View file

@ -1,19 +1,19 @@
#[derive(Debug, PartialEq)] #[derive(Debug, PartialEq)]
/// Error types for the failure modes /// Error types for the failure modes
pub enum KyberError { pub enum KyberError {
/// One or more inputs to a function are incorrectly sized. A likely cause of this is two parties using different security /// One or more inputs to a function are incorrectly sized. A likely cause of this is two parties using different security
/// levels while trying to negotiate a key exchange. /// levels while trying to negotiate a key exchange.
InvalidInput, InvalidInput,
/// The ciphertext was unable to be authenticated. /// The ciphertext was unable to be authenticated.
/// The shared secret was not decapsulated. /// The shared secret was not decapsulated.
Decapsulation, Decapsulation,
} }
impl core::fmt::Display for KyberError { impl core::fmt::Display for KyberError {
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
match *self { match *self {
KyberError::InvalidInput => write!(f, "Function input is of incorrect length"), KyberError::InvalidInput => write!(f, "Function input is of incorrect length"),
KyberError::Decapsulation => write!(f, "Decapsulation Failure, unable to obtain shared secret from ciphertext"), KyberError::Decapsulation => write!(f, "Decapsulation Failure, unable to obtain shared secret from ciphertext"),
}
} }
}
} }

241
third_party/kyber/src/kem.rs vendored
View file

@ -1,128 +1,113 @@
#[cfg(not(feature="KATs"))] #[cfg(not(feature = "KATs"))]
use crate::rng::randombytes; use crate::rng::randombytes;
use rand_core::{RngCore, CryptoRng}; use crate::{error::KyberError, indcpa::*, params::*, symmetric::*, verify::*};
use crate::{ use rand_core::{CryptoRng, RngCore};
params::*,
indcpa::*, // Name: crypto_kem_keypair
symmetric::*, //
error::KyberError, // Description: Generates public and private key
verify::* // for CCA-secure Kyber key encapsulation mechanism
}; //
// Arguments: - [u8] pk: output public key (an already allocated array of CRYPTO_PUBLICKEYBYTES bytes)
// Name: crypto_kem_keypair // - [u8] sk: output private key (an already allocated array of CRYPTO_SECRETKEYBYTES bytes)
// pub fn crypto_kem_keypair<R>(pk: &mut [u8], sk: &mut [u8], _rng: &mut R, _seed: Option<(&[u8], &[u8])>)
// Description: Generates public and private key where
// for CCA-secure Kyber key encapsulation mechanism R: RngCore + CryptoRng,
// {
// Arguments: - [u8] pk: output public key (an already allocated array of CRYPTO_PUBLICKEYBYTES bytes) const PK_START: usize = KYBER_SECRETKEYBYTES - (2 * KYBER_SYMBYTES);
// - [u8] sk: output private key (an already allocated array of CRYPTO_SECRETKEYBYTES bytes) const SK_START: usize = KYBER_SECRETKEYBYTES - KYBER_SYMBYTES;
pub fn crypto_kem_keypair<R>( const END: usize = KYBER_INDCPA_PUBLICKEYBYTES + KYBER_INDCPA_SECRETKEYBYTES;
pk: &mut[u8], sk: &mut[u8], _rng: &mut R, _seed: Option<(&[u8], &[u8])>
) indcpa_keypair(pk, sk, _seed, _rng);
where R: RngCore + CryptoRng
{ sk[KYBER_INDCPA_SECRETKEYBYTES..END].copy_from_slice(&pk[..KYBER_INDCPA_PUBLICKEYBYTES]);
const PK_START: usize = KYBER_SECRETKEYBYTES - (2 * KYBER_SYMBYTES); hash_h(&mut sk[PK_START..], pk, KYBER_PUBLICKEYBYTES);
const SK_START: usize = KYBER_SECRETKEYBYTES-KYBER_SYMBYTES;
const END: usize = KYBER_INDCPA_PUBLICKEYBYTES + KYBER_INDCPA_SECRETKEYBYTES; #[cfg(feature = "KATs")]
sk[SK_START..].copy_from_slice(&_seed.expect("KATs feature only for testing").1);
indcpa_keypair(pk, sk, _seed, _rng);
#[cfg(not(feature = "KATs"))]
sk[KYBER_INDCPA_SECRETKEYBYTES..END] randombytes(&mut sk[SK_START..], KYBER_SYMBYTES, _rng);
.copy_from_slice(&pk[..KYBER_INDCPA_PUBLICKEYBYTES]); }
hash_h(&mut sk[PK_START..], pk, KYBER_PUBLICKEYBYTES);
// Name: crypto_kem_enc
#[cfg(feature="KATs")] //
sk[SK_START..].copy_from_slice(&_seed.expect("KATs feature only for testing").1); // Description: Generates cipher text and shared
// secret for given public key
#[cfg(not(feature="KATs"))] //
randombytes(&mut sk[SK_START..],KYBER_SYMBYTES, _rng); // Arguments: - [u8] ct: output cipher text (an already allocated array of CRYPTO_CIPHERTEXTBYTES bytes)
// - [u8] ss: output shared secret (an already allocated array of CRYPTO_BYTES bytes)
} // - const [u8] pk: input public key (an already allocated array of CRYPTO_PUBLICKEYBYTES bytes)
pub fn crypto_kem_enc<R>(ct: &mut [u8], ss: &mut [u8], pk: &[u8], _rng: &mut R, _seed: Option<&[u8]>)
// Name: crypto_kem_enc where
// R: RngCore + CryptoRng,
// Description: Generates cipher text and shared {
// secret for given public key let mut kr = [0u8; 2 * KYBER_SYMBYTES];
// let mut buf = [0u8; 2 * KYBER_SYMBYTES];
// Arguments: - [u8] ct: output cipher text (an already allocated array of CRYPTO_CIPHERTEXTBYTES bytes) let mut randbuf = [0u8; 2 * KYBER_SYMBYTES];
// - [u8] ss: output shared secret (an already allocated array of CRYPTO_BYTES bytes)
// - const [u8] pk: input public key (an already allocated array of CRYPTO_PUBLICKEYBYTES bytes) #[cfg(not(feature = "KATs"))]
pub fn crypto_kem_enc<R>( randombytes(&mut randbuf, KYBER_SYMBYTES, _rng);
ct: &mut[u8], ss: &mut[u8], pk: &[u8], _rng: &mut R,_seed: Option<&[u8]>
) // Deterministic randbuf for KAT's
where R: RngCore + CryptoRng #[cfg(feature = "KATs")]
{ randbuf[..KYBER_SYMBYTES].copy_from_slice(&_seed.expect("KATs feature only works with `cargo test`"));
let mut kr = [0u8; 2*KYBER_SYMBYTES];
let mut buf = [0u8; 2*KYBER_SYMBYTES]; // Don't release system RNG output
let mut randbuf = [0u8; 2*KYBER_SYMBYTES]; hash_h(&mut buf, &randbuf, KYBER_SYMBYTES);
#[cfg(not(feature="KATs"))] // Multitarget countermeasure for coins + contributory KEM
randombytes(&mut randbuf, KYBER_SYMBYTES, _rng); hash_h(&mut buf[KYBER_SYMBYTES..], pk, KYBER_PUBLICKEYBYTES);
hash_g(&mut kr, &buf, 2 * KYBER_SYMBYTES);
// Deterministic randbuf for KAT's
#[cfg(feature="KATs")] // coins are in kr[KYBER_SYMBYTES..]
randbuf[..KYBER_SYMBYTES] indcpa_enc(ct, &buf, pk, &kr[KYBER_SYMBYTES..]);
.copy_from_slice(&_seed.expect("KATs feature only works with `cargo test`"));
// overwrite coins in kr with H(c)
// Don't release system RNG output hash_h(&mut kr[KYBER_SYMBYTES..], ct, KYBER_CIPHERTEXTBYTES);
hash_h(&mut buf, &randbuf, KYBER_SYMBYTES);
// hash concatenation of pre-k and H(c) to k
// Multitarget countermeasure for coins + contributory KEM kdf(ss, &kr, 2 * KYBER_SYMBYTES);
hash_h(&mut buf[KYBER_SYMBYTES..], pk, KYBER_PUBLICKEYBYTES); }
hash_g(&mut kr, &buf, 2*KYBER_SYMBYTES);
// Name: crypto_kem_dec
// coins are in kr[KYBER_SYMBYTES..] //
indcpa_enc(ct, &buf, pk, &kr[KYBER_SYMBYTES..]); // Description: Generates shared secret for given
// cipher text and private key
// overwrite coins in kr with H(c) //
hash_h(&mut kr[KYBER_SYMBYTES..], ct, KYBER_CIPHERTEXTBYTES); // Arguments: - [u8] ss: output shared secret (an already allocated array of CRYPTO_BYTES bytes)
// - const [u8] ct: input cipher text (an already allocated array of CRYPTO_CIPHERTEXTBYTES bytes)
// hash concatenation of pre-k and H(c) to k // - const [u8] sk: input private key (an already allocated array of CRYPTO_SECRETKEYBYTES bytes)
kdf(ss, &kr, 2*KYBER_SYMBYTES); //
} // On failure, ss will contain a pseudo-random value.
pub fn crypto_kem_dec(ss: &mut [u8], ct: &[u8], sk: &[u8]) -> Result<(), KyberError> {
// Name: crypto_kem_dec let mut buf = [0u8; 2 * KYBER_SYMBYTES];
// let mut kr = [0u8; 2 * KYBER_SYMBYTES];
// Description: Generates shared secret for given let mut cmp = [0u8; KYBER_CIPHERTEXTBYTES];
// cipher text and private key let mut pk = [0u8; KYBER_INDCPA_PUBLICKEYBYTES + 2 * KYBER_SYMBYTES];
//
// Arguments: - [u8] ss: output shared secret (an already allocated array of CRYPTO_BYTES bytes) pk.copy_from_slice(&sk[KYBER_INDCPA_SECRETKEYBYTES..]);
// - const [u8] ct: input cipher text (an already allocated array of CRYPTO_CIPHERTEXTBYTES bytes)
// - const [u8] sk: input private key (an already allocated array of CRYPTO_SECRETKEYBYTES bytes) indcpa_dec(&mut buf, ct, sk);
//
// On failure, ss will contain a pseudo-random value. // Multitarget countermeasure for coins + contributory KEM
pub fn crypto_kem_dec( const START: usize = KYBER_SECRETKEYBYTES - 2 * KYBER_SYMBYTES;
ss: &mut[u8], ct: &[u8], sk: &[u8] const END: usize = KYBER_SECRETKEYBYTES - KYBER_SYMBYTES;
) buf[KYBER_SYMBYTES..].copy_from_slice(&sk[START..END]);
-> Result<(), KyberError> hash_g(&mut kr, &buf, 2 * KYBER_SYMBYTES);
{
let mut buf = [0u8; 2*KYBER_SYMBYTES]; // coins are in kr[KYBER_SYMBYTES..]
let mut kr = [0u8; 2*KYBER_SYMBYTES]; indcpa_enc(&mut cmp, &buf, &pk, &kr[KYBER_SYMBYTES..]);
let mut cmp = [0u8; KYBER_CIPHERTEXTBYTES]; let fail = verify(ct, &cmp, KYBER_CIPHERTEXTBYTES);
let mut pk = [0u8; KYBER_INDCPA_PUBLICKEYBYTES + 2*KYBER_SYMBYTES]; // overwrite coins in kr with H(c)
hash_h(&mut kr[KYBER_SYMBYTES..], ct, KYBER_CIPHERTEXTBYTES);
pk.copy_from_slice(&sk[KYBER_INDCPA_SECRETKEYBYTES..]); // Overwrite pre-k with z on re-encryption failure
cmov(&mut kr, &sk[END..], KYBER_SYMBYTES, fail);
indcpa_dec(&mut buf, ct, sk); // hash concatenation of pre-k and H(c) to k
kdf(ss, &kr, 2 * KYBER_SYMBYTES);
// Multitarget countermeasure for coins + contributory KEM
const START: usize = KYBER_SECRETKEYBYTES-2*KYBER_SYMBYTES; match fail {
const END: usize = KYBER_SECRETKEYBYTES-KYBER_SYMBYTES; 0 => Ok(()),
buf[KYBER_SYMBYTES..].copy_from_slice(&sk[START..END]); _ => Err(KyberError::Decapsulation),
hash_g(&mut kr, &buf, 2*KYBER_SYMBYTES); }
}
// coins are in kr[KYBER_SYMBYTES..]
indcpa_enc(&mut cmp, &buf, &pk, &kr[KYBER_SYMBYTES..]);
let fail = verify(ct, &cmp, KYBER_CIPHERTEXTBYTES);
// overwrite coins in kr with H(c)
hash_h(&mut kr[KYBER_SYMBYTES..], ct, KYBER_CIPHERTEXTBYTES);
// Overwrite pre-k with z on re-encryption failure
cmov(&mut kr, &sk[END..], KYBER_SYMBYTES, fail);
// hash concatenation of pre-k and H(c) to k
kdf(ss, &kr, 2*KYBER_SYMBYTES);
match fail {
0 => Ok(()),
_ => Err(KyberError::Decapsulation)
}
}

513
third_party/kyber/src/kex.rs vendored
View file

@ -1,22 +1,17 @@
use rand_core::{RngCore, CryptoRng}; use crate::{kem::*, params::*, symmetric::kdf, KyberError};
use crate::{ use rand_core::{CryptoRng, RngCore};
kem::*,
symmetric::kdf,
params::*,
KyberError
};
/// Unilateral Key Exchange Initiation Byte Length /// Unilateral Key Exchange Initiation Byte Length
pub const UAKE_INIT_BYTES: usize = KYBER_PUBLICKEYBYTES + KYBER_CIPHERTEXTBYTES; pub const UAKE_INIT_BYTES: usize = KYBER_PUBLICKEYBYTES + KYBER_CIPHERTEXTBYTES;
/// Unilateral Key Exchange Response Byte Length /// Unilateral Key Exchange Response Byte Length
pub const UAKE_RESPONSE_BYTES: usize = KYBER_CIPHERTEXTBYTES; pub const UAKE_RESPONSE_BYTES: usize = KYBER_CIPHERTEXTBYTES;
/// Mutual Key Exchange Initiation Byte Length /// Mutual Key Exchange Initiation Byte Length
pub const AKE_INIT_BYTES: usize = KYBER_PUBLICKEYBYTES + KYBER_CIPHERTEXTBYTES; pub const AKE_INIT_BYTES: usize = KYBER_PUBLICKEYBYTES + KYBER_CIPHERTEXTBYTES;
/// Mutual Key Exchange Response Byte Length /// Mutual Key Exchange Response Byte Length
pub const AKE_RESPONSE_BYTES: usize = 2 * KYBER_CIPHERTEXTBYTES; pub const AKE_RESPONSE_BYTES: usize = 2 * KYBER_CIPHERTEXTBYTES;
/// Result of encapsulating a public key which includes the ciphertext and shared secret /// Result of encapsulating a public key which includes the ciphertext and shared secret
pub type Encapsulated = Result<([u8; KYBER_CIPHERTEXTBYTES], [u8; KYBER_SSBYTES]), KyberError>; pub type Encapsulated = Result<([u8; KYBER_CIPHERTEXTBYTES], [u8; KYBER_SSBYTES]), KyberError>;
/// The result of decapsulating a ciphertext which produces a shared secret when confirmed /// The result of decapsulating a ciphertext which produces a shared secret when confirmed
pub type Decapsulated = Result<[u8; KYBER_SSBYTES], KyberError>; pub type Decapsulated = Result<[u8; KYBER_SSBYTES], KyberError>;
/// Kyber public key /// Kyber public key
@ -24,15 +19,15 @@ pub type PublicKey = [u8; KYBER_PUBLICKEYBYTES];
/// Kyber secret key /// Kyber secret key
pub type SecretKey = [u8; KYBER_SECRETKEYBYTES]; pub type SecretKey = [u8; KYBER_SECRETKEYBYTES];
/// Kyber Shared Secret /// Kyber Shared Secret
pub type SharedSecret = [u8; KYBER_SSBYTES]; pub type SharedSecret = [u8; KYBER_SSBYTES];
/// Bytes to send when initiating a unilateral key exchange /// Bytes to send when initiating a unilateral key exchange
pub type UakeSendInit = [u8; UAKE_INIT_BYTES]; pub type UakeSendInit = [u8; UAKE_INIT_BYTES];
/// Bytes to send when responding to a unilateral key exchange /// Bytes to send when responding to a unilateral key exchange
pub type UakeSendResponse = [u8; UAKE_RESPONSE_BYTES]; pub type UakeSendResponse = [u8; UAKE_RESPONSE_BYTES];
/// Bytes to send when initiating a mutual key exchange /// Bytes to send when initiating a mutual key exchange
pub type AkeSendInit = [u8; AKE_INIT_BYTES]; pub type AkeSendInit = [u8; AKE_INIT_BYTES];
/// Bytes to send when responding to a mutual key exchange /// Bytes to send when responding to a mutual key exchange
pub type AkeSendResponse = [u8; AKE_RESPONSE_BYTES]; pub type AkeSendResponse = [u8; AKE_RESPONSE_BYTES];
// Ephemeral keys // Ephemeral keys
type TempKey = [u8; KYBER_SSBYTES]; type TempKey = [u8; KYBER_SSBYTES];
@ -40,349 +35,293 @@ type Eska = [u8; KYBER_SECRETKEYBYTES];
// TODO: implement zeroise feature // TODO: implement zeroise feature
/// Used for unilaterally authenticated key exchange between two parties. /// Used for unilaterally authenticated key exchange between two parties.
/// ///
/// ``` /// ```
/// # use pqc_kyber::*; /// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> { /// # fn main() -> Result<(),KyberError> {
/// let mut rng = rand::thread_rng(); /// let mut rng = rand::thread_rng();
/// ///
/// let mut alice = Uake::new(); /// let mut alice = Uake::new();
/// let mut bob = Uake::new(); /// let mut bob = Uake::new();
/// let bob_keys = keypair(&mut rng); /// let bob_keys = keypair(&mut rng);
/// ///
/// let client_init = alice.client_init(&bob_keys.public, &mut rng); /// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng)?; /// let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng)?;
/// let client_confirm = alice.client_confirm(server_send); /// let client_confirm = alice.client_confirm(server_send);
/// ///
/// assert_eq!(alice.shared_secret, bob.shared_secret); /// assert_eq!(alice.shared_secret, bob.shared_secret);
/// # Ok(()) } /// # Ok(()) }
#[derive(Copy, Clone, Debug, Eq, PartialEq)] #[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub struct Uake { pub struct Uake {
/// The resulting shared secret from a key exchange /// The resulting shared secret from a key exchange
pub shared_secret: SharedSecret, pub shared_secret: SharedSecret,
/// Sent when initiating a key exchange /// Sent when initiating a key exchange
send_a: UakeSendInit, send_a: UakeSendInit,
/// Response to a key exchange initiation /// Response to a key exchange initiation
send_b: UakeSendResponse, send_b: UakeSendResponse,
// Epheremal keys // Epheremal keys
temp_key: TempKey, temp_key: TempKey,
eska: Eska eska: Eska,
} }
impl Default for Uake { impl Default for Uake {
fn default() -> Self { fn default() -> Self {
Uake { Uake {
shared_secret: [0u8; KYBER_SSBYTES], shared_secret: [0u8; KYBER_SSBYTES],
send_a: [0u8; UAKE_INIT_BYTES], send_a: [0u8; UAKE_INIT_BYTES],
send_b: [0u8; UAKE_RESPONSE_BYTES], send_b: [0u8; UAKE_RESPONSE_BYTES],
temp_key: [0u8; KYBER_SSBYTES], temp_key: [0u8; KYBER_SSBYTES],
eska: [0u8; KYBER_SECRETKEYBYTES], eska: [0u8; KYBER_SECRETKEYBYTES],
}
} }
}
} }
impl Uake { impl Uake {
/// Builds new UAKE struct /// Builds new UAKE struct
/// ``` /// ```
/// # use pqc_kyber::Uake; /// # use pqc_kyber::Uake;
/// let mut kex = Uake::new(); /// let mut kex = Uake::new();
/// ``` /// ```
pub fn new() -> Self { pub fn new() -> Self {
Self::default() Self::default()
} }
/// Initiates a Unilaterally Authenticated Key Exchange. /// Initiates a Unilaterally Authenticated Key Exchange.
/// ``` /// ```
/// # use pqc_kyber::*; /// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> { /// # fn main() -> Result<(),KyberError> {
/// let mut rng = rand::thread_rng(); /// let mut rng = rand::thread_rng();
/// let mut alice = Uake::new(); /// let mut alice = Uake::new();
/// let bob_keys = keypair(&mut rng); /// let bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng); /// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// # Ok(()) } /// # Ok(()) }
/// ``` /// ```
pub fn client_init<R>(&mut self, pubkey: &PublicKey, rng: &mut R) pub fn client_init<R>(&mut self, pubkey: &PublicKey, rng: &mut R) -> UakeSendInit
-> UakeSendInit where
where R: CryptoRng + RngCore R: CryptoRng + RngCore,
{ {
uake_init_a( uake_init_a(&mut self.send_a, &mut self.temp_key, &mut self.eska, pubkey, rng);
&mut self.send_a, &mut self.temp_key, self.send_a
&mut self.eska, pubkey, rng }
);
self.send_a
}
/// Handles the output of a `client_init()` request /// Handles the output of a `client_init()` request
/// ``` /// ```
/// # use pqc_kyber::*; /// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> { /// # fn main() -> Result<(),KyberError> {
/// # let mut rng = rand::thread_rng(); /// # let mut rng = rand::thread_rng();
/// let mut alice = Uake::new(); /// let mut alice = Uake::new();
/// let mut bob = Uake::new(); /// let mut bob = Uake::new();
/// let mut bob_keys = keypair(&mut rng); /// let mut bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng); /// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng)?; /// let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng)?;
/// # Ok(()) } /// # Ok(()) }
pub fn server_receive<R>( pub fn server_receive<R>(&mut self, send_a: UakeSendInit, secretkey: &SecretKey, rng: &mut R) -> Result<UakeSendResponse, KyberError>
&mut self, send_a: UakeSendInit, secretkey: &SecretKey, rng: &mut R where
) R: CryptoRng + RngCore,
-> Result<UakeSendResponse, KyberError> {
where R: CryptoRng + RngCore uake_shared_b(&mut self.send_b, &mut self.shared_secret, &send_a, secretkey, rng)?;
{ Ok(self.send_b)
uake_shared_b( }
&mut self.send_b, &mut self.shared_secret,
&send_a, secretkey, rng
)?;
Ok(self.send_b)
}
/// Decapsulates and authenticates the shared secret from the output of /// Decapsulates and authenticates the shared secret from the output of
/// `server_receive()` /// `server_receive()`
/// ``` /// ```
/// # use pqc_kyber::*; /// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> { /// # fn main() -> Result<(),KyberError> {
/// # let mut rng = rand::thread_rng(); /// # let mut rng = rand::thread_rng();
/// # let mut alice = Uake::new(); /// # let mut alice = Uake::new();
/// # let mut bob = Uake::new(); /// # let mut bob = Uake::new();
/// # let bob_keys = keypair(&mut rng); /// # let bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng); /// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng)?; /// let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng)?;
/// let client_confirm = alice.client_confirm(server_send); /// let client_confirm = alice.client_confirm(server_send);
/// assert_eq!(alice.shared_secret, bob.shared_secret); /// assert_eq!(alice.shared_secret, bob.shared_secret);
/// # Ok(()) } /// # Ok(()) }
pub fn client_confirm(&mut self, send_b: UakeSendResponse) pub fn client_confirm(&mut self, send_b: UakeSendResponse) -> Result<(), KyberError> {
-> Result<(), KyberError> uake_shared_a(&mut self.shared_secret, &send_b, &self.temp_key, &self.eska)?;
{ Ok(())
uake_shared_a( }
&mut self.shared_secret, &send_b,
&self.temp_key, &self.eska
)?;
Ok(())
}
} }
/// Used for mutually authenticated key exchange between two parties. /// Used for mutually authenticated key exchange between two parties.
/// ///
/// # Example: /// # Example:
/// ``` /// ```
/// # use pqc_kyber::*; /// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> { /// # fn main() -> Result<(),KyberError> {
/// let mut rng = rand::thread_rng(); /// let mut rng = rand::thread_rng();
/// ///
/// let mut alice = Ake::new(); /// let mut alice = Ake::new();
/// let mut bob = Ake::new(); /// let mut bob = Ake::new();
/// ///
/// let alice_keys = keypair(&mut rng); /// let alice_keys = keypair(&mut rng);
/// let bob_keys = keypair(&mut rng); /// let bob_keys = keypair(&mut rng);
/// ///
/// let client_init = alice.client_init(&bob_keys.public, &mut rng); /// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng)?; /// let server_send = bob.server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng)?;
/// let client_confirm = alice.client_confirm(server_send, &alice_keys.secret); /// let client_confirm = alice.client_confirm(server_send, &alice_keys.secret);
/// ///
/// assert_eq!(alice.shared_secret, bob.shared_secret); /// assert_eq!(alice.shared_secret, bob.shared_secret);
/// # Ok(()) } /// # Ok(()) }
/// ``` /// ```
#[derive(Copy, Clone, Debug, Eq, PartialEq)] #[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub struct Ake { pub struct Ake {
/// The resulting shared secret from a key exchange /// The resulting shared secret from a key exchange
pub shared_secret: SharedSecret, pub shared_secret: SharedSecret,
/// Sent when initiating a key exchange /// Sent when initiating a key exchange
send_a: AkeSendInit, send_a: AkeSendInit,
/// Sent back responding to a key exchange initiation /// Sent back responding to a key exchange initiation
send_b: AkeSendResponse, send_b: AkeSendResponse,
// Epheremal keys // Epheremal keys
temp_key: TempKey, temp_key: TempKey,
eska: Eska eska: Eska,
} }
impl Default for Ake { impl Default for Ake {
fn default() -> Self { fn default() -> Self {
Ake { Ake {
shared_secret: [0u8; KYBER_SSBYTES], shared_secret: [0u8; KYBER_SSBYTES],
send_a: [0u8; AKE_INIT_BYTES], send_a: [0u8; AKE_INIT_BYTES],
send_b: [0u8; AKE_RESPONSE_BYTES], send_b: [0u8; AKE_RESPONSE_BYTES],
temp_key: [0u8; KYBER_SSBYTES], temp_key: [0u8; KYBER_SSBYTES],
eska: [0u8; KYBER_SECRETKEYBYTES], eska: [0u8; KYBER_SECRETKEYBYTES],
}
} }
}
} }
impl Ake { impl Ake {
/// Builds a new AKE struct /// Builds a new AKE struct
/// ``` /// ```
/// # use pqc_kyber::Ake; /// # use pqc_kyber::Ake;
/// let mut kex = Ake::new(); /// let mut kex = Ake::new();
/// ``` /// ```
pub fn new() -> Self { pub fn new() -> Self {
Self::default() Self::default()
} }
/// Initiates a Mutually Authenticated Key Exchange. /// Initiates a Mutually Authenticated Key Exchange.
/// ``` /// ```
/// # use pqc_kyber::*; /// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> { /// # fn main() -> Result<(),KyberError> {
/// let mut rng = rand::thread_rng(); /// let mut rng = rand::thread_rng();
/// let mut alice = Ake::new(); /// let mut alice = Ake::new();
/// let bob_keys = keypair(&mut rng); /// let bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng); /// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// # Ok(()) } /// # Ok(()) }
/// ``` /// ```
pub fn client_init<R>(&mut self, pubkey: &PublicKey, rng: &mut R) pub fn client_init<R>(&mut self, pubkey: &PublicKey, rng: &mut R) -> AkeSendInit
-> AkeSendInit where
where R: CryptoRng + RngCore R: CryptoRng + RngCore,
{ {
ake_init_a( ake_init_a(&mut self.send_a, &mut self.temp_key, &mut self.eska, pubkey, rng);
&mut self.send_a, &mut self.temp_key, self.send_a
&mut self.eska, pubkey, rng }
);
self.send_a
}
/// Handles and authenticates the output of a `client_init()` request /// Handles and authenticates the output of a `client_init()` request
/// ``` /// ```
/// # use pqc_kyber::*; /// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> { /// # fn main() -> Result<(),KyberError> {
/// # let mut rng = rand::thread_rng(); /// # let mut rng = rand::thread_rng();
/// let mut alice = Ake::new(); /// let mut alice = Ake::new();
/// let mut bob = Ake::new(); /// let mut bob = Ake::new();
/// let alice_keys = keypair(&mut rng); /// let alice_keys = keypair(&mut rng);
/// let bob_keys = keypair(&mut rng); /// let bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng); /// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng)?; /// let server_send = bob.server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng)?;
/// # Ok(()) } /// # Ok(()) }
pub fn server_receive<R>( pub fn server_receive<R>(
&mut self, ake_send_a: AkeSendInit, pubkey: &PublicKey, &mut self,
secretkey: &SecretKey, rng: &mut R ake_send_a: AkeSendInit,
) pubkey: &PublicKey,
-> Result<AkeSendResponse, KyberError> secretkey: &SecretKey,
where R: CryptoRng + RngCore rng: &mut R,
{ ) -> Result<AkeSendResponse, KyberError>
ake_shared_b( where
&mut self.send_b, &mut self.shared_secret, R: CryptoRng + RngCore,
&ake_send_a, secretkey, pubkey, rng {
)?; ake_shared_b(&mut self.send_b, &mut self.shared_secret, &ake_send_a, secretkey, pubkey, rng)?;
Ok(self.send_b) Ok(self.send_b)
} }
/// Decapsulates and authenticates the shared secret from the output of /// Decapsulates and authenticates the shared secret from the output of
/// `server_receive()` /// `server_receive()`
/// ``` /// ```
/// # use pqc_kyber::*; /// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> { /// # fn main() -> Result<(),KyberError> {
/// # let mut rng = rand::thread_rng(); /// # let mut rng = rand::thread_rng();
/// # let mut alice = Ake::new(); /// # let mut alice = Ake::new();
/// # let mut bob = Ake::new(); /// # let mut bob = Ake::new();
/// # let alice_keys = keypair(&mut rng); /// # let alice_keys = keypair(&mut rng);
/// # let bob_keys = keypair(&mut rng); /// # let bob_keys = keypair(&mut rng);
/// # let client_init = alice.client_init(&bob_keys.public, &mut rng); /// # let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng)?; /// let server_send = bob.server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng)?;
/// let client_confirm = alice.client_confirm(server_send, &alice_keys.secret); /// let client_confirm = alice.client_confirm(server_send, &alice_keys.secret);
/// assert_eq!(alice.shared_secret, bob.shared_secret); /// assert_eq!(alice.shared_secret, bob.shared_secret);
/// # Ok(()) } /// # Ok(()) }
pub fn client_confirm(&mut self, send_b: AkeSendResponse, secretkey: &SecretKey) pub fn client_confirm(&mut self, send_b: AkeSendResponse, secretkey: &SecretKey) -> Result<(), KyberError> {
-> Result<(), KyberError> ake_shared_a(&mut self.shared_secret, &send_b, &self.temp_key, &self.eska, secretkey)?;
{ Ok(())
ake_shared_a( }
&mut self.shared_secret, &send_b,
&self.temp_key, &self.eska, secretkey
)?;
Ok(())
}
} }
// Unilaterally Authenticated Key Exchange initiation // Unilaterally Authenticated Key Exchange initiation
fn uake_init_a<R>( fn uake_init_a<R>(send: &mut [u8], tk: &mut [u8], sk: &mut [u8], pkb: &[u8], rng: &mut R)
send: &mut[u8], where
tk: &mut[u8], R: CryptoRng + RngCore,
sk: &mut[u8],
pkb: &[u8],
rng: &mut R
)
where R: CryptoRng + RngCore
{ {
crypto_kem_keypair(send, sk, rng, None); crypto_kem_keypair(send, sk, rng, None);
crypto_kem_enc(&mut send[KYBER_PUBLICKEYBYTES..], tk, pkb, rng, None); crypto_kem_enc(&mut send[KYBER_PUBLICKEYBYTES..], tk, pkb, rng, None);
} }
// Unilaterally authenticated key exchange computation by Bob // Unilaterally authenticated key exchange computation by Bob
fn uake_shared_b<R>( fn uake_shared_b<R>(send: &mut [u8], k: &mut [u8], recv: &[u8], skb: &[u8], rng: &mut R) -> Result<(), KyberError>
send: &mut[u8], where
k: &mut[u8], R: CryptoRng + RngCore,
recv: &[u8],
skb: &[u8],
rng: &mut R
) -> Result<(), KyberError>
where R: CryptoRng + RngCore
{ {
let mut buf = [0u8; 2*KYBER_SYMBYTES]; let mut buf = [0u8; 2 * KYBER_SYMBYTES];
crypto_kem_enc(send, &mut buf, recv, rng, None); crypto_kem_enc(send, &mut buf, recv, rng, None);
crypto_kem_dec(&mut buf[KYBER_SYMBYTES..], &recv[KYBER_PUBLICKEYBYTES..], skb)?; crypto_kem_dec(&mut buf[KYBER_SYMBYTES..], &recv[KYBER_PUBLICKEYBYTES..], skb)?;
kdf(k, &buf, 2*KYBER_SYMBYTES); kdf(k, &buf, 2 * KYBER_SYMBYTES);
Ok(()) Ok(())
} }
// Unilaterally authenticated key exchange computation by Alice // Unilaterally authenticated key exchange computation by Alice
fn uake_shared_a( fn uake_shared_a(k: &mut [u8], recv: &[u8], tk: &[u8], sk: &[u8]) -> Result<(), KyberError> {
k: &mut[u8], let mut buf = [0u8; 2 * KYBER_SYMBYTES];
recv: &[u8], crypto_kem_dec(&mut buf, recv, sk)?;
tk: &[u8], buf[KYBER_SYMBYTES..].copy_from_slice(&tk[..]);
sk: &[u8] kdf(k, &buf, 2 * KYBER_SYMBYTES);
) -> Result<(), KyberError> Ok(())
{
let mut buf = [0u8; 2*KYBER_SYMBYTES];
crypto_kem_dec(&mut buf, recv, sk)?;
buf[KYBER_SYMBYTES..].copy_from_slice(&tk[..]);
kdf(k, &buf, 2*KYBER_SYMBYTES);
Ok(())
} }
// Authenticated key exchange initiation by Alice // Authenticated key exchange initiation by Alice
fn ake_init_a<R>( fn ake_init_a<R>(send: &mut [u8], tk: &mut [u8], sk: &mut [u8], pkb: &[u8], rng: &mut R)
send: &mut[u8], where
tk: &mut[u8], R: CryptoRng + RngCore,
sk: &mut[u8],
pkb: &[u8],
rng: &mut R
)
where R: CryptoRng + RngCore
{ {
crypto_kem_keypair(send, sk, rng, None); crypto_kem_keypair(send, sk, rng, None);
crypto_kem_enc(&mut send[KYBER_PUBLICKEYBYTES..], tk, pkb, rng, None); crypto_kem_enc(&mut send[KYBER_PUBLICKEYBYTES..], tk, pkb, rng, None);
} }
// Mutually authenticated key exchange computation by Bob // Mutually authenticated key exchange computation by Bob
fn ake_shared_b<R>( fn ake_shared_b<R>(send: &mut [u8], k: &mut [u8], recv: &[u8], skb: &[u8], pka: &[u8], rng: &mut R) -> Result<(), KyberError>
send: &mut[u8], where
k: &mut[u8], R: CryptoRng + RngCore,
recv: &[u8],
skb: &[u8],
pka: &[u8],
rng: &mut R
) -> Result<(), KyberError>
where R: CryptoRng + RngCore
{ {
let mut buf = [0u8; 3*KYBER_SYMBYTES]; let mut buf = [0u8; 3 * KYBER_SYMBYTES];
crypto_kem_enc(send, &mut buf, recv, rng, None); crypto_kem_enc(send, &mut buf, recv, rng, None);
crypto_kem_enc(&mut send[KYBER_CIPHERTEXTBYTES..], &mut buf[KYBER_SYMBYTES..], pka, rng, None); crypto_kem_enc(&mut send[KYBER_CIPHERTEXTBYTES..], &mut buf[KYBER_SYMBYTES..], pka, rng, None);
crypto_kem_dec(&mut buf[2*KYBER_SYMBYTES..], &recv[KYBER_PUBLICKEYBYTES..], skb)?; crypto_kem_dec(&mut buf[2 * KYBER_SYMBYTES..], &recv[KYBER_PUBLICKEYBYTES..], skb)?;
kdf(k, &buf, 3*KYBER_SYMBYTES); kdf(k, &buf, 3 * KYBER_SYMBYTES);
Ok(()) Ok(())
} }
// Mutually authenticated key exchange computation by Alice // Mutually authenticated key exchange computation by Alice
fn ake_shared_a( fn ake_shared_a(k: &mut [u8], recv: &[u8], tk: &[u8], sk: &[u8], ska: &[u8]) -> Result<(), KyberError> {
k: &mut[u8], let mut buf = [0u8; 3 * KYBER_SYMBYTES];
recv: &[u8], crypto_kem_dec(&mut buf, recv, sk)?;
tk: &[u8], crypto_kem_dec(&mut buf[KYBER_SYMBYTES..], &recv[KYBER_CIPHERTEXTBYTES..], ska)?;
sk: &[u8], buf[2 * KYBER_SYMBYTES..].copy_from_slice(&tk[..]);
ska: &[u8] kdf(k, &buf, 3 * KYBER_SYMBYTES);
) -> Result<(), KyberError> Ok(())
{ }
let mut buf = [0u8; 3*KYBER_SYMBYTES];
crypto_kem_dec(&mut buf, recv, sk)?;
crypto_kem_dec(&mut buf[KYBER_SYMBYTES..], &recv[KYBER_CIPHERTEXTBYTES..], ska)?;
buf[2*KYBER_SYMBYTES..].copy_from_slice(&tk[..]);
kdf(k, &buf, 3*KYBER_SYMBYTES);
Ok(())
}

134
third_party/kyber/src/params.rs vendored
View file

@ -1,67 +1,67 @@
/// The security level of Kyber /// The security level of Kyber
/// ///
/// Defaults to 3 (kyber768), will be 2 or 4 repsectively when /// Defaults to 3 (kyber768), will be 2 or 4 repsectively when
/// kyber512 or kyber1024 are selected with feature flags. /// kyber512 or kyber1024 are selected with feature flags.
/// ///
/// * Kyber-512 aims at security roughly equivalent to AES-128 /// * Kyber-512 aims at security roughly equivalent to AES-128
/// * Kyber-768 aims at security roughly equivalent to AES-192 /// * Kyber-768 aims at security roughly equivalent to AES-192
/// * Kyber-1024 aims at security roughly equivalent to AES-256 /// * Kyber-1024 aims at security roughly equivalent to AES-256
#[cfg(not(any(feature = "kyber512", feature = "kyber1024")))] #[cfg(not(any(feature = "kyber512", feature = "kyber1024")))]
pub const KYBER_K: usize = 3; pub const KYBER_K: usize = 3;
#[cfg(feature = "kyber512")] #[cfg(feature = "kyber512")]
pub const KYBER_K: usize = 2; pub const KYBER_K: usize = 2;
#[cfg(feature = "kyber1024")] #[cfg(feature = "kyber1024")]
pub const KYBER_K: usize = 4; pub const KYBER_K: usize = 4;
/// A boolean flag for whether 90's mode is activated. /// A boolean flag for whether 90's mode is activated.
/// ///
/// If true AES-CTR and SHA2 will be used as cryptographic primitives instead, /// If true AES-CTR and SHA2 will be used as cryptographic primitives instead,
/// which may have hardware speed-ups on certain platforms. /// which may have hardware speed-ups on certain platforms.
/// ///
/// Defaults to false, set`features = ["90s"]` in Cargo.toml to enable. /// Defaults to false, set`features = ["90s"]` in Cargo.toml to enable.
#[cfg(not(feature = "90s"))] #[cfg(not(feature = "90s"))]
pub const KYBER_90S: bool = false; pub const KYBER_90S: bool = false;
#[cfg(feature = "90s")] #[cfg(feature = "90s")]
pub const KYBER_90S: bool = true; pub const KYBER_90S: bool = true;
pub(crate) const KYBER_N: usize = 256; pub(crate) const KYBER_N: usize = 256;
pub(crate) const KYBER_Q: usize = 3329; pub(crate) const KYBER_Q: usize = 3329;
#[cfg(feature = "kyber512")] #[cfg(feature = "kyber512")]
pub(crate) const KYBER_ETA1: usize = 3; pub(crate) const KYBER_ETA1: usize = 3;
#[cfg(not(feature = "kyber512"))] #[cfg(not(feature = "kyber512"))]
pub(crate) const KYBER_ETA1: usize = 2; pub(crate) const KYBER_ETA1: usize = 2;
pub(crate) const KYBER_ETA2: usize = 2; pub(crate) const KYBER_ETA2: usize = 2;
// Size of the hashes and seeds // Size of the hashes and seeds
pub(crate) const KYBER_SYMBYTES: usize = 32; pub(crate) const KYBER_SYMBYTES: usize = 32;
/// Size of the shared key /// Size of the shared key
pub const KYBER_SSBYTES: usize = 32; pub const KYBER_SSBYTES: usize = 32;
pub(crate) const KYBER_POLYBYTES: usize = 384; pub(crate) const KYBER_POLYBYTES: usize = 384;
pub(crate) const KYBER_POLYVECBYTES: usize = KYBER_K * KYBER_POLYBYTES; pub(crate) const KYBER_POLYVECBYTES: usize = KYBER_K * KYBER_POLYBYTES;
#[cfg(not(feature = "kyber1024"))] #[cfg(not(feature = "kyber1024"))]
pub(crate) const KYBER_POLYCOMPRESSEDBYTES: usize = 128; pub(crate) const KYBER_POLYCOMPRESSEDBYTES: usize = 128;
#[cfg(not(feature = "kyber1024"))] #[cfg(not(feature = "kyber1024"))]
pub(crate) const KYBER_POLYVECCOMPRESSEDBYTES: usize = KYBER_K * 320; pub(crate) const KYBER_POLYVECCOMPRESSEDBYTES: usize = KYBER_K * 320;
#[cfg(feature = "kyber1024")] #[cfg(feature = "kyber1024")]
pub(crate) const KYBER_POLYCOMPRESSEDBYTES: usize = 160; pub(crate) const KYBER_POLYCOMPRESSEDBYTES: usize = 160;
#[cfg(feature = "kyber1024")] #[cfg(feature = "kyber1024")]
pub(crate) const KYBER_POLYVECCOMPRESSEDBYTES: usize = KYBER_K * 352; pub(crate) const KYBER_POLYVECCOMPRESSEDBYTES: usize = KYBER_K * 352;
pub(crate) const KYBER_INDCPA_PUBLICKEYBYTES: usize = KYBER_POLYVECBYTES + KYBER_SYMBYTES; pub(crate) const KYBER_INDCPA_PUBLICKEYBYTES: usize = KYBER_POLYVECBYTES + KYBER_SYMBYTES;
pub(crate) const KYBER_INDCPA_SECRETKEYBYTES: usize = KYBER_POLYVECBYTES; pub(crate) const KYBER_INDCPA_SECRETKEYBYTES: usize = KYBER_POLYVECBYTES;
pub(crate) const KYBER_INDCPA_BYTES: usize = KYBER_POLYVECCOMPRESSEDBYTES + KYBER_POLYCOMPRESSEDBYTES; pub(crate) const KYBER_INDCPA_BYTES: usize = KYBER_POLYVECCOMPRESSEDBYTES + KYBER_POLYCOMPRESSEDBYTES;
/// Size in bytes of the Kyber public key /// Size in bytes of the Kyber public key
pub const KYBER_PUBLICKEYBYTES: usize = KYBER_INDCPA_PUBLICKEYBYTES; pub const KYBER_PUBLICKEYBYTES: usize = KYBER_INDCPA_PUBLICKEYBYTES;
/// Size in bytes of the Kyber secret key /// Size in bytes of the Kyber secret key
pub const KYBER_SECRETKEYBYTES: usize = KYBER_INDCPA_SECRETKEYBYTES + KYBER_INDCPA_PUBLICKEYBYTES + 2*KYBER_SYMBYTES; pub const KYBER_SECRETKEYBYTES: usize = KYBER_INDCPA_SECRETKEYBYTES + KYBER_INDCPA_PUBLICKEYBYTES + 2 * KYBER_SYMBYTES;
/// Size in bytes of the Kyber ciphertext /// Size in bytes of the Kyber ciphertext
pub const KYBER_CIPHERTEXTBYTES: usize = KYBER_INDCPA_BYTES; pub const KYBER_CIPHERTEXTBYTES: usize = KYBER_INDCPA_BYTES;

1225
third_party/kyber/src/reference/aes256ctr.rs vendored
View file

File diff suppressed because it is too large Load diff

189
third_party/kyber/src/reference/cbd.rs vendored
View file

@ -1,98 +1,91 @@
use crate::poly::Poly; use crate::params::KYBER_N;
use crate::params::KYBER_N; use crate::poly::Poly;
// Name: load32_littleendian // Name: load32_littleendian
// //
// Description: load 4 bytes into a 32-bit integer // Description: load 4 bytes into a 32-bit integer
// in little-endian order // in little-endian order
// //
// Arguments: - const [u8] x: input byte array // Arguments: - const [u8] x: input byte array
// //
// Returns 32-bit unsigned integer loaded from x // Returns 32-bit unsigned integer loaded from x
fn load32_littleendian(x: &[u8]) -> u32 fn load32_littleendian(x: &[u8]) -> u32 {
{ let mut r = x[0] as u32;
let mut r = x[0] as u32; r |= (x[1] as u32) << 8;
r |= (x[1] as u32) << 8; r |= (x[2] as u32) << 16;
r |= (x[2] as u32) << 16; r |= (x[3] as u32) << 24;
r |= (x[3] as u32) << 24; r
r }
}
// Name: load32_littleendian
// Name: load32_littleendian //
// // Description: load 3 bytes into a 32-bit integer
// Description: load 3 bytes into a 32-bit integer // in little-endian order
// in little-endian order // This function is only needed for Kyber-512
// This function is only needed for Kyber-512 //
// // Arguments: - const [u8] x: input byte array
// Arguments: - const [u8] x: input byte array //
// // Returns 32-bit unsigned integer loaded from x
// Returns 32-bit unsigned integer loaded from x fn load24_littleendian(x: &[u8]) -> u32 {
fn load24_littleendian(x: &[u8]) -> u32 let mut r = x[0] as u32;
{ r |= (x[1] as u32) << 8;
let mut r = x[0] as u32; r |= (x[2] as u32) << 16;
r |= (x[1] as u32) << 8; r
r |= (x[2] as u32) << 16; }
r
} // Name: cbd2
//
// Name: cbd2 // Description: Given an array of uniformly random bytes, compute
// // polynomial with coefficients distributed according to
// Description: Given an array of uniformly random bytes, compute // a centered binomial distribution with parameter eta=2
// polynomial with coefficients distributed according to //
// a centered binomial distribution with parameter eta=2 // Arguments: - poly *r: output polynomial
// // - const [u8] buf: input byte array
// Arguments: - poly *r: output polynomial pub fn cbd2(r: &mut Poly, buf: &[u8]) {
// - const [u8] buf: input byte array let (mut d, mut t, mut a, mut b);
pub fn cbd2(r: &mut Poly, buf: &[u8]) for i in 0..(KYBER_N / 8) {
{ t = load32_littleendian(&buf[4 * i..]);
let (mut d, mut t, mut a, mut b); d = t & 0x55555555;
for i in 0..(KYBER_N/8) { d += (t >> 1) & 0x55555555;
t = load32_littleendian(&buf[4*i..]); for j in 0..8 {
d = t & 0x55555555; a = ((d >> (4 * j)) & 0x3) as i16;
d += (t>>1) & 0x55555555; b = ((d >> (4 * j + 2)) & 0x3) as i16;
for j in 0..8 { r.coeffs[8 * i + j] = a - b;
a = ((d >> (4*j)) & 0x3) as i16; }
b = ((d >> (4*j+2)) & 0x3) as i16; }
r.coeffs[8*i+j] = a - b; }
}
} // Name: cbd3
} //
// Description: Given an array of uniformly random bytes, compute
// Name: cbd3 // polynomial with coefficients distributed according to
// // a centered binomial distribution with parameter eta=3
// Description: Given an array of uniformly random bytes, compute // This function is only needed for Kyber-512
// polynomial with coefficients distributed according to // Arguments: - poly *r: output polynomial
// a centered binomial distribution with parameter eta=3 // - const [u8] buf: input byte array
// This function is only needed for Kyber-512 pub fn cbd3(r: &mut Poly, buf: &[u8]) {
// Arguments: - poly *r: output polynomial let (mut d, mut t, mut a, mut b);
// - const [u8] buf: input byte array for i in 0..(KYBER_N / 4) {
pub fn cbd3(r: &mut Poly, buf: &[u8]) t = load24_littleendian(&buf[3 * i..]);
{ d = t & 0x00249249;
let (mut d, mut t, mut a, mut b); d += (t >> 1) & 0x00249249;
for i in 0..(KYBER_N/4) { d += (t >> 2) & 0x00249249;
t = load24_littleendian(&buf[3*i..]); for j in 0..4 {
d = t & 0x00249249; a = ((d >> (6 * j)) & 0x7) as i16;
d += (t>>1) & 0x00249249; b = ((d >> (6 * j + 3)) & 0x7) as i16;
d += (t>>2) & 0x00249249; r.coeffs[4 * i + j] = a - b;
for j in 0..4 { }
a = ((d >> (6*j)) & 0x7) as i16; }
b = ((d >> (6*j+3)) & 0x7) as i16; }
r.coeffs[4*i+j] = a - b;
} pub fn poly_cbd_eta1(r: &mut Poly, buf: &[u8]) {
} if cfg!(feature = "kyber512") {
} cbd3(r, buf)
} else {
pub fn poly_cbd_eta1(r: &mut Poly, buf: &[u8]) cbd2(r, buf)
{ }
if cfg!(feature="kyber512") { }
cbd3(r, buf)
} pub fn poly_cbd_eta2(r: &mut Poly, buf: &[u8]) {
else { cbd2(r, buf)
cbd2(r, buf) }
}
}
pub fn poly_cbd_eta2(r: &mut Poly, buf: &[u8])
{
cbd2(r, buf)
}

1258
third_party/kyber/src/reference/fips202.rs vendored
View file

File diff suppressed because it is too large Load diff

605
third_party/kyber/src/reference/indcpa.rs vendored
View file

@ -1,316 +1,289 @@
#[cfg(not(feature="KATs"))] #[cfg(not(feature = "KATs"))]
use crate::rng::randombytes; use crate::rng::randombytes;
use crate::{ use crate::{params::*, poly::*, polyvec::*, symmetric::*, CryptoRng, RngCore};
poly::*,
polyvec::*, // Name: pack_pk
symmetric::*, //
params::*, // Description: Serialize the public key as concatenation of the
RngCore, // serialized vector of polynomials pk
CryptoRng, // and the public seed used to generate the matrix A.
}; //
// Arguments: [u8] r: the output serialized public key
// Name: pack_pk // const poly *pk: the input public-key polynomial
// // const [u8] seed: the input public seed
// Description: Serialize the public key as concatenation of the fn pack_pk(r: &mut [u8], pk: &mut Polyvec, seed: &[u8]) {
// serialized vector of polynomials pk const END: usize = KYBER_SYMBYTES + KYBER_POLYVECBYTES;
// and the public seed used to generate the matrix A. polyvec_tobytes(r, pk);
// r[KYBER_POLYVECBYTES..END].copy_from_slice(&seed[..KYBER_SYMBYTES]);
// Arguments: [u8] r: the output serialized public key }
// const poly *pk: the input public-key polynomial
// const [u8] seed: the input public seed // Name: unpack_pk
fn pack_pk(r: &mut[u8], pk: &mut Polyvec, seed: &[u8]) //
{ // Description: De-serialize public key from a byte array;
const END: usize = KYBER_SYMBYTES + KYBER_POLYVECBYTES; // approximate inverse of pack_pk
polyvec_tobytes(r, pk); //
r[KYBER_POLYVECBYTES..END].copy_from_slice(&seed[..KYBER_SYMBYTES]); // Arguments: - Polyvec pk: output public-key vector of polynomials
} // - [u8] seed: output seed to generate matrix A
// - const [u8] packedpk: input serialized public key
// Name: unpack_pk fn unpack_pk(pk: &mut Polyvec, seed: &mut [u8], packedpk: &[u8]) {
// const END: usize = KYBER_SYMBYTES + KYBER_POLYVECBYTES;
// Description: De-serialize public key from a byte array; polyvec_frombytes(pk, packedpk);
// approximate inverse of pack_pk seed[..KYBER_SYMBYTES].copy_from_slice(&packedpk[KYBER_POLYVECBYTES..END]);
// }
// Arguments: - Polyvec pk: output public-key vector of polynomials
// - [u8] seed: output seed to generate matrix A // Name: pack_sk
// - const [u8] packedpk: input serialized public key //
fn unpack_pk(pk: &mut Polyvec, seed: &mut[u8], packedpk: &[u8]) // Description: Serialize the secret key
{ //
const END: usize = KYBER_SYMBYTES + KYBER_POLYVECBYTES; // Arguments: - [u8] r: output serialized secret key
polyvec_frombytes(pk, packedpk); // - const Polyvec sk: input vector of polynomials (secret key)
seed[..KYBER_SYMBYTES].copy_from_slice(&packedpk[KYBER_POLYVECBYTES..END]); fn pack_sk(r: &mut [u8], sk: &mut Polyvec) {
} polyvec_tobytes(r, sk);
}
// Name: pack_sk
// // Name: unpack_sk
// Description: Serialize the secret key //
// // Description: De-serialize the secret key, inverse of pack_sk
// Arguments: - [u8] r: output serialized secret key //
// - const Polyvec sk: input vector of polynomials (secret key) // Arguments: - Polyvec sk: output vector of polynomials (secret key)
fn pack_sk(r: &mut[u8], sk: &mut Polyvec) // - const [u8] packedsk: input serialized secret key
{ fn unpack_sk(sk: &mut Polyvec, packedsk: &[u8]) {
polyvec_tobytes(r, sk); polyvec_frombytes(sk, packedsk);
} }
// Name: unpack_sk // Name: pack_ciphertext
// //
// Description: De-serialize the secret key, inverse of pack_sk // Description: Serialize the ciphertext as concatenation of the
// // compressed and serialized vector of polynomials b
// Arguments: - Polyvec sk: output vector of polynomials (secret key) // and the compressed and serialized polynomial v
// - const [u8] packedsk: input serialized secret key //
fn unpack_sk(sk: &mut Polyvec, packedsk: &[u8]) // Arguments: [u8] r: the output serialized ciphertext
{ // const poly *pk: the input vector of polynomials b
polyvec_frombytes(sk, packedsk); // const [u8] seed: the input polynomial v
} fn pack_ciphertext(r: &mut [u8], b: &mut Polyvec, v: Poly) {
polyvec_compress(r, *b);
// Name: pack_ciphertext poly_compress(&mut r[KYBER_POLYVECCOMPRESSEDBYTES..], v);
// }
// Description: Serialize the ciphertext as concatenation of the
// compressed and serialized vector of polynomials b // Name: unpack_ciphertext
// and the compressed and serialized polynomial v //
// // Description: De-serialize and decompress ciphertext from a byte array;
// Arguments: [u8] r: the output serialized ciphertext // approximate inverse of pack_ciphertext
// const poly *pk: the input vector of polynomials b //
// const [u8] seed: the input polynomial v // Arguments: - Polyvec b: output vector of polynomials b
fn pack_ciphertext(r: &mut[u8], b: &mut Polyvec, v: Poly) // - poly *v: output polynomial v
{ // - const [u8] c: input serialized ciphertext
polyvec_compress(r, *b); fn unpack_ciphertext(b: &mut Polyvec, v: &mut Poly, c: &[u8]) {
poly_compress(&mut r[KYBER_POLYVECCOMPRESSEDBYTES..], v); polyvec_decompress(b, c);
} poly_decompress(v, &c[KYBER_POLYVECCOMPRESSEDBYTES..]);
}
// Name: unpack_ciphertext
// // Name: rej_uniform
// Description: De-serialize and decompress ciphertext from a byte array; //
// approximate inverse of pack_ciphertext // Description: Run rejection sampling on uniform random bytes to generate
// // uniform random integers mod q
// Arguments: - Polyvec b: output vector of polynomials b //
// - poly *v: output polynomial v // Arguments: - i16 *r: output buffer
// - const [u8] c: input serialized ciphertext // - usize len: requested number of 16-bit integers (uniform mod q)
fn unpack_ciphertext(b: &mut Polyvec, v: &mut Poly, c: &[u8]) // - const [u8] buf: input buffer (assumed to be uniform random bytes)
{ // - usize buflen: length of input buffer in bytes
polyvec_decompress(b, c); //
poly_decompress(v, &c[KYBER_POLYVECCOMPRESSEDBYTES..]); // Returns number of sampled 16-bit integers (at most len)
} fn rej_uniform(r: &mut [i16], len: usize, buf: &[u8], buflen: usize) -> usize {
let (mut ctr, mut pos) = (0usize, 0usize);
// Name: rej_uniform let (mut val0, mut val1);
//
// Description: Run rejection sampling on uniform random bytes to generate while ctr < len && pos + 3 <= buflen {
// uniform random integers mod q val0 = ((buf[pos + 0] >> 0) as u16 | (buf[pos + 1] as u16) << 8) & 0xFFF;
// val1 = ((buf[pos + 1] >> 4) as u16 | (buf[pos + 2] as u16) << 4) & 0xFFF;
// Arguments: - i16 *r: output buffer pos += 3;
// - usize len: requested number of 16-bit integers (uniform mod q)
// - const [u8] buf: input buffer (assumed to be uniform random bytes) if val0 < KYBER_Q as u16 {
// - usize buflen: length of input buffer in bytes r[ctr] = val0 as i16;
// ctr += 1;
// Returns number of sampled 16-bit integers (at most len) }
fn rej_uniform(r: &mut[i16], len: usize, buf: &[u8], buflen: usize) -> usize if ctr < len && val1 < KYBER_Q as u16 {
{ r[ctr] = val1 as i16;
let (mut ctr, mut pos) = (0usize, 0usize); ctr += 1;
let (mut val0, mut val1); }
}
while ctr < len && pos + 3 <= buflen { ctr
val0 = ((buf[pos+0] >> 0) as u16 | (buf[pos+1] as u16) << 8) & 0xFFF; }
val1 = ((buf[pos+1] >> 4) as u16 | (buf[pos+2] as u16) << 4) & 0xFFF;
pos += 3; fn gen_a(a: &mut [Polyvec], b: &[u8]) {
gen_matrix(a, b, false);
if val0 < KYBER_Q as u16 { }
r[ctr] = val0 as i16;
ctr += 1; fn gen_at(a: &mut [Polyvec], b: &[u8]) {
} gen_matrix(a, b, true);
if ctr < len && val1 < KYBER_Q as u16 { }
r[ctr] = val1 as i16;
ctr += 1; // Name: gen_matrix
} //
} // Description: Deterministically generate matrix A (or the transpose of A)
ctr // from a seed. Entries of the matrix are polynomials that look
} // uniformly random. Performs rejection sampling on output of
// a XOF
fn gen_a(a: &mut [Polyvec], b: &[u8]) //
{ // Arguments: - Polyvec a: ouptput matrix A
gen_matrix(a, b, false); // - const [u8] seed: input seed
} // - bool transposed: boolean deciding whether A or A^T is generated
fn gen_matrix(a: &mut [Polyvec], seed: &[u8], transposed: bool) {
fn gen_at(a: &mut [Polyvec], b: &[u8]) let mut ctr;
{ // 530 is expected number of required bytes
gen_matrix(a, b, true); const GEN_MATRIX_NBLOCKS: usize = (12 * KYBER_N / 8 * (1 << 12) / KYBER_Q + XOF_BLOCKBYTES) / XOF_BLOCKBYTES;
} const BUFLEN: usize = GEN_MATRIX_NBLOCKS * XOF_BLOCKBYTES;
let mut buf = [0u8; BUFLEN + 2];
// Name: gen_matrix let mut off: usize;
// let mut state = XofState::new();
// Description: Deterministically generate matrix A (or the transpose of A)
// from a seed. Entries of the matrix are polynomials that look for i in 0..KYBER_K {
// uniformly random. Performs rejection sampling on output of for j in 0..KYBER_K {
// a XOF if transposed {
// xof_absorb(&mut state, seed, i as u8, j as u8);
// Arguments: - Polyvec a: ouptput matrix A } else {
// - const [u8] seed: input seed xof_absorb(&mut state, seed, j as u8, i as u8);
// - bool transposed: boolean deciding whether A or A^T is generated }
fn gen_matrix(a: &mut [Polyvec], seed: &[u8], transposed: bool) xof_squeezeblocks(&mut buf, GEN_MATRIX_NBLOCKS, &mut state);
{ ctr = rej_uniform(&mut a[i].vec[j].coeffs, KYBER_N, &buf, BUFLEN);
let mut ctr;
// 530 is expected number of required bytes while ctr < KYBER_N {
const GEN_MATRIX_NBLOCKS: usize = off = BUFLEN % 3;
(12*KYBER_N/8*(1 << 12)/KYBER_Q + XOF_BLOCKBYTES)/XOF_BLOCKBYTES; for k in 0..off {
const BUFLEN: usize = GEN_MATRIX_NBLOCKS*XOF_BLOCKBYTES; buf[k] = buf[BUFLEN - off + k];
let mut buf = [0u8; BUFLEN+2]; }
let mut off: usize; xof_squeezeblocks(&mut buf[off..], 1, &mut state);
let mut state = XofState::new(); ctr += rej_uniform(&mut a[i].vec[j].coeffs[ctr..], KYBER_N - ctr, &buf, BUFLEN);
}
for i in 0..KYBER_K { }
for j in 0..KYBER_K { }
if transposed { }
xof_absorb(&mut state, seed, i as u8, j as u8);
} // Name: indcpa_keypair
else { //
xof_absorb(&mut state, seed, j as u8, i as u8); // Description: Generates public and private key for the CPA-secure
} // public-key encryption scheme underlying Kyber
xof_squeezeblocks(&mut buf, GEN_MATRIX_NBLOCKS, &mut state); //
ctr = rej_uniform(&mut a[i].vec[j].coeffs, KYBER_N, &buf, BUFLEN); // Arguments: - [u8] pk: output public key (length KYBER_INDCPA_PUBLICKEYBYTES)
// - [u8] sk: output private key (length KYBER_INDCPA_SECRETKEYBYTES)
while ctr < KYBER_N pub fn indcpa_keypair<R>(pk: &mut [u8], sk: &mut [u8], _seed: Option<(&[u8], &[u8])>, _rng: &mut R)
{ where
off = BUFLEN % 3; R: CryptoRng + RngCore,
for k in 0..off { {
buf[k] = buf[BUFLEN - off + k]; let mut a = [Polyvec::new(); KYBER_K];
} let (mut e, mut pkpv, mut skpv) = (Polyvec::new(), Polyvec::new(), Polyvec::new());
xof_squeezeblocks(&mut buf[off..], 1, &mut state); let mut nonce = 0u8;
ctr += rej_uniform(&mut a[i].vec[j].coeffs[ctr..], KYBER_N - ctr, &buf, BUFLEN); let mut buf = [0u8; 2 * KYBER_SYMBYTES];
} let mut randbuf = [0u8; 2 * KYBER_SYMBYTES];
}
} #[cfg(not(feature = "KATs"))]
} randombytes(&mut randbuf, KYBER_SYMBYTES, _rng);
// Name: indcpa_keypair // Use rng seed for test vectors
// #[cfg(feature = "KATs")]
// Description: Generates public and private key for the CPA-secure randbuf[..KYBER_SYMBYTES].copy_from_slice(&_seed.expect("KAT feature only for testing").0);
// public-key encryption scheme underlying Kyber
// hash_g(&mut buf, &randbuf, KYBER_SYMBYTES);
// Arguments: - [u8] pk: output public key (length KYBER_INDCPA_PUBLICKEYBYTES)
// - [u8] sk: output private key (length KYBER_INDCPA_SECRETKEYBYTES) let (publicseed, noiseseed) = buf.split_at(KYBER_SYMBYTES);
pub fn indcpa_keypair<R>( gen_a(&mut a, publicseed);
pk : &mut[u8],
sk: &mut[u8], for i in 0..KYBER_K {
_seed: Option<(&[u8], &[u8])>, poly_getnoise_eta1(&mut skpv.vec[i], noiseseed, nonce);
_rng: &mut R nonce += 1;
) }
where R: CryptoRng + RngCore for i in 0..KYBER_K {
{ poly_getnoise_eta1(&mut e.vec[i], noiseseed, nonce);
let mut a = [Polyvec::new(); KYBER_K]; nonce += 1;
let (mut e, mut pkpv, mut skpv) = (Polyvec::new(), Polyvec::new(), Polyvec::new()); }
let mut nonce = 0u8;
let mut buf = [0u8; 2*KYBER_SYMBYTES]; polyvec_ntt(&mut skpv);
let mut randbuf = [0u8; 2*KYBER_SYMBYTES]; polyvec_ntt(&mut e);
#[cfg(not(feature="KATs"))] // matrix-vector multiplication
randombytes(&mut randbuf, KYBER_SYMBYTES, _rng); for i in 0..KYBER_K {
polyvec_basemul_acc_montgomery(&mut pkpv.vec[i], &a[i], &skpv);
// Use rng seed for test vectors poly_frommont(&mut pkpv.vec[i]);
#[cfg(feature="KATs")] }
randbuf[..KYBER_SYMBYTES].copy_from_slice(&_seed.expect("KAT feature only for testing").0); polyvec_add(&mut pkpv, &e);
polyvec_reduce(&mut pkpv);
hash_g(&mut buf, &randbuf, KYBER_SYMBYTES);
pack_sk(sk, &mut skpv);
let (publicseed, noiseseed) = buf.split_at(KYBER_SYMBYTES); pack_pk(pk, &mut pkpv, publicseed);
gen_a(&mut a, publicseed); }
for i in 0..KYBER_K { // Name: indcpa_enc
poly_getnoise_eta1(&mut skpv.vec[i], noiseseed, nonce); //
nonce += 1; // Description: Encryption function of the CPA-secure
} // public-key encryption scheme underlying Kyber.
for i in 0..KYBER_K { //
poly_getnoise_eta1(&mut e.vec[i], noiseseed, nonce); // Arguments: - [u8] c: output ciphertext (length KYBER_INDCPA_BYTES)
nonce += 1; // - const [u8] m: input message (length KYBER_INDCPA_MSGBYTES)
} // - const [u8] pk: input public key (length KYBER_INDCPA_PUBLICKEYBYTES)
// - const [u8] coin: input random coins used as seed (length KYBER_SYMBYTES)
polyvec_ntt(&mut skpv); // to deterministically generate all randomness
polyvec_ntt(&mut e); pub fn indcpa_enc(c: &mut [u8], m: &[u8], pk: &[u8], coins: &[u8]) {
let mut at = [Polyvec::new(); KYBER_K];
// matrix-vector multiplication let (mut sp, mut pkpv, mut ep, mut b) = (Polyvec::new(), Polyvec::new(), Polyvec::new(), Polyvec::new());
for i in 0..KYBER_K { let (mut v, mut k, mut epp) = (Poly::new(), Poly::new(), Poly::new());
polyvec_basemul_acc_montgomery(&mut pkpv.vec[i], &a[i], &skpv); let mut seed = [0u8; KYBER_SYMBYTES];
poly_frommont(&mut pkpv.vec[i]); let mut nonce = 0u8;
}
polyvec_add(&mut pkpv, &e); unpack_pk(&mut pkpv, &mut seed, pk);
polyvec_reduce(&mut pkpv); poly_frommsg(&mut k, m);
gen_at(&mut at, &seed);
pack_sk(sk, &mut skpv);
pack_pk(pk, &mut pkpv, publicseed); for i in 0..KYBER_K {
} poly_getnoise_eta1(&mut sp.vec[i], coins, nonce);
nonce += 1;
// Name: indcpa_enc }
// for i in 0..KYBER_K {
// Description: Encryption function of the CPA-secure poly_getnoise_eta2(&mut ep.vec[i], coins, nonce);
// public-key encryption scheme underlying Kyber. nonce += 1;
// }
// Arguments: - [u8] c: output ciphertext (length KYBER_INDCPA_BYTES) poly_getnoise_eta2(&mut epp, coins, nonce);
// - const [u8] m: input message (length KYBER_INDCPA_MSGBYTES)
// - const [u8] pk: input public key (length KYBER_INDCPA_PUBLICKEYBYTES) polyvec_ntt(&mut sp);
// - const [u8] coin: input random coins used as seed (length KYBER_SYMBYTES)
// to deterministically generate all randomness // matrix-vector multiplication
pub fn indcpa_enc(c: &mut[u8], m: &[u8], pk: &[u8], coins: &[u8]) for i in 0..KYBER_K {
{ polyvec_basemul_acc_montgomery(&mut b.vec[i], &at[i], &sp);
let mut at = [Polyvec::new(); KYBER_K]; }
let (mut sp, mut pkpv, mut ep, mut b) =
(Polyvec::new(),Polyvec::new(), Polyvec::new(), Polyvec::new()); polyvec_basemul_acc_montgomery(&mut v, &pkpv, &sp);
let (mut v, mut k, mut epp) = (Poly::new(), Poly::new(), Poly::new()); polyvec_invntt_tomont(&mut b);
let mut seed = [0u8; KYBER_SYMBYTES]; poly_invntt_tomont(&mut v);
let mut nonce = 0u8;
polyvec_add(&mut b, &ep);
unpack_pk(&mut pkpv, &mut seed, pk); poly_add(&mut v, &epp);
poly_frommsg(&mut k, m); poly_add(&mut v, &k);
gen_at(&mut at, &seed); polyvec_reduce(&mut b);
poly_reduce(&mut v);
for i in 0..KYBER_K {
poly_getnoise_eta1(&mut sp.vec[i], coins, nonce); pack_ciphertext(c, &mut b, v);
nonce += 1; }
}
for i in 0..KYBER_K { // Name: indcpa_dec
poly_getnoise_eta2(&mut ep.vec[i], coins, nonce); //
nonce += 1; // Description: Decryption function of the CPA-secure
} // public-key encryption scheme underlying Kyber.
poly_getnoise_eta2(&mut epp, coins, nonce); //
// Arguments: - [u8] m: output decrypted message (of length KYBER_INDCPA_MSGBYTES)
polyvec_ntt(&mut sp); // - const [u8] c: input ciphertext (of length KYBER_INDCPA_BYTES)
// - const [u8] sk: input secret key (of length KYBER_INDCPA_SECRETKEYBYTES)
// matrix-vector multiplication pub fn indcpa_dec(m: &mut [u8], c: &[u8], sk: &[u8]) {
for i in 0..KYBER_K { let (mut b, mut skpv) = (Polyvec::new(), Polyvec::new());
polyvec_basemul_acc_montgomery(&mut b.vec[i], &at[i], &sp); let (mut v, mut mp) = (Poly::new(), Poly::new());
}
unpack_ciphertext(&mut b, &mut v, c);
polyvec_basemul_acc_montgomery(&mut v, &pkpv, &sp); unpack_sk(&mut skpv, sk);
polyvec_invntt_tomont(&mut b);
poly_invntt_tomont(&mut v); polyvec_ntt(&mut b);
polyvec_basemul_acc_montgomery(&mut mp, &skpv, &b);
polyvec_add(&mut b, &ep); poly_invntt_tomont(&mut mp);
poly_add(&mut v, &epp);
poly_add(&mut v, &k); poly_sub(&mut mp, &v);
polyvec_reduce(&mut b); poly_reduce(&mut mp);
poly_reduce(&mut v);
poly_tomsg(m, mp);
pack_ciphertext(c, &mut b, v); }
}
// Name: indcpa_dec
//
// Description: Decryption function of the CPA-secure
// public-key encryption scheme underlying Kyber.
//
// Arguments: - [u8] m: output decrypted message (of length KYBER_INDCPA_MSGBYTES)
// - const [u8] c: input ciphertext (of length KYBER_INDCPA_BYTES)
// - const [u8] sk: input secret key (of length KYBER_INDCPA_SECRETKEYBYTES)
pub fn indcpa_dec(m: &mut[u8], c: &[u8], sk: &[u8])
{
let (mut b, mut skpv) = (Polyvec::new(),Polyvec::new());
let (mut v, mut mp) = (Poly::new(),Poly::new());
unpack_ciphertext(&mut b, &mut v, c);
unpack_sk(&mut skpv, sk);
polyvec_ntt(&mut b);
polyvec_basemul_acc_montgomery(&mut mp, &skpv, &b);
poly_invntt_tomont(&mut mp);
poly_sub(&mut mp, &v);
poly_reduce(&mut mp);
poly_tomsg(m, mp);
}

2
third_party/kyber/src/reference/mod.rs vendored
View file

@ -6,4 +6,4 @@ pub mod ntt;
pub mod poly; pub mod poly;
pub mod polyvec; pub mod polyvec;
pub mod reduce; pub mod reduce;
pub mod verify; pub mod verify;

294
third_party/kyber/src/reference/ntt.rs vendored
View file

@ -1,154 +1,140 @@
use crate::reduce::*; use crate::reduce::*;
// Code to generate zetas used in the number-theoretic transform: // Code to generate zetas used in the number-theoretic transform:
// //
//#define KYBER_ROOT_OF_UNITY 17 //#define KYBER_ROOT_OF_UNITY 17
// //
//static const uint8_t tree[128] = { //static const uint8_t tree[128] = {
// 0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120, // 0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120,
// 4, 68, 36, 100, 20, 84, 52, 116, 12, 76, 44, 108, 28, 92, 60, 124, // 4, 68, 36, 100, 20, 84, 52, 116, 12, 76, 44, 108, 28, 92, 60, 124,
// 2, 66, 34, 98, 18, 82, 50, 114, 10, 74, 42, 106, 26, 90, 58, 122, // 2, 66, 34, 98, 18, 82, 50, 114, 10, 74, 42, 106, 26, 90, 58, 122,
// 6, 70, 38, 102, 22, 86, 54, 118, 14, 78, 46, 110, 30, 94, 62, 126, // 6, 70, 38, 102, 22, 86, 54, 118, 14, 78, 46, 110, 30, 94, 62, 126,
// 1, 65, 33, 97, 17, 81, 49, 113, 9, 73, 41, 105, 25, 89, 57, 121, // 1, 65, 33, 97, 17, 81, 49, 113, 9, 73, 41, 105, 25, 89, 57, 121,
// 5, 69, 37, 101, 21, 85, 53, 117, 13, 77, 45, 109, 29, 93, 61, 125, // 5, 69, 37, 101, 21, 85, 53, 117, 13, 77, 45, 109, 29, 93, 61, 125,
// 3, 67, 35, 99, 19, 83, 51, 115, 11, 75, 43, 107, 27, 91, 59, 123, // 3, 67, 35, 99, 19, 83, 51, 115, 11, 75, 43, 107, 27, 91, 59, 123,
// 7, 71, 39, 103, 23, 87, 55, 119, 15, 79, 47, 111, 31, 95, 63, 127}; // 7, 71, 39, 103, 23, 87, 55, 119, 15, 79, 47, 111, 31, 95, 63, 127};
// //
// //
// static int16_t fqmul(int16_t a, int16_t b) { // static int16_t fqmul(int16_t a, int16_t b) {
// return montgomery_reduce((int32_t)a*b); // return montgomery_reduce((int32_t)a*b);
//} //}
// //
// void init_ntt() { // void init_ntt() {
// unsigned int i; // unsigned int i;
// int16_t tmp[128]; // int16_t tmp[128];
// //
// tmp[0] = MONT; // tmp[0] = MONT;
// for(i = 1; i < 128; ++i) // for(i = 1; i < 128; ++i)
// tmp[i] = fqmul(tmp[i-1], MONT*KYBER_ROOT_OF_UNITY % KYBER_Q); // tmp[i] = fqmul(tmp[i-1], MONT*KYBER_ROOT_OF_UNITY % KYBER_Q);
// //
// for(i = 0; i < 128; ++i) // for(i = 0; i < 128; ++i)
// zetas[i] = tmp[tree[i]]; // zetas[i] = tmp[tree[i]];
// //
// if(zetas[i] > KYBER_Q/2) // if(zetas[i] > KYBER_Q/2)
// zetas[i] -= KYBER_Q; // zetas[i] -= KYBER_Q;
// if(zetas[i] < -KYBER_Q/2) // if(zetas[i] < -KYBER_Q/2)
// zetas[i] += KYBER_Q; // zetas[i] += KYBER_Q;
//} //}
pub const ZETAS: [i16; 128] = [ pub const ZETAS: [i16; 128] = [
-1044, -758, -359, -1517, 1493, 1422, 287, 202, -1044, -758, -359, -1517, 1493, 1422, 287, 202, -171, 622, 1577, 182, 962, -1202, -1474, 1468, 573, -1325, 264, 383, -829, 1458, -1602,
-171, 622, 1577, 182, 962, -1202, -1474, 1468, -130, -681, 1017, 732, 608, -1542, 411, -205, -1571, 1223, 652, -552, 1015, -1293, 1491, -282, -1544, 516, -8, -320, -666, -1618,
573, -1325, 264, 383, -829, 1458, -1602, -130, -1162, 126, 1469, -853, -90, -271, 830, 107, -1421, -247, -951, -398, 961, -1508, -725, 448, -1065, 677, -1275, -1103, 430, 555, 843,
-681, 1017, 732, 608, -1542, 411, -205, -1571, -1251, 871, 1550, 105, 422, 587, 177, -235, -291, -460, 1574, 1653, -246, 778, 1159, -147, -777, 1483, -602, 1119, -1590, 644, -872,
1223, 652, -552, 1015, -1293, 1491, -282, -1544, 349, 418, 329, -156, -75, 817, 1097, 603, 610, 1322, -1285, -1465, 384, -1215, -136, 1218, -1335, -874, 220, -1187, -1659, -1185,
516, -8, -320, -666, -1618, -1162, 126, 1469, -1530, -1278, 794, -1510, -854, -870, 478, -108, -308, 996, 991, 958, -1460, 1522, 1628,
-853, -90, -271, 830, 107, -1421, -247, -951, ];
-398, 961, -1508, -725, 448, -1065, 677, -1275,
-1103, 430, 555, 843, -1251, 871, 1550, 105, // Name: fqmul
422, 587, 177, -235, -291, -460, 1574, 1653, //
-246, 778, 1159, -147, -777, 1483, -602, 1119, // Description: Multiplication followed by Montgomery reduction
-1590, 644, -872, 349, 418, 329, -156, -75, //
817, 1097, 603, 610, 1322, -1285, -1465, 384, // Arguments: - i16 a: first factor
-1215, -136, 1218, -1335, -874, 220, -1187, -1659, // - i16 b: second factor
-1185, -1530, -1278, 794, -1510, -854, -870, 478, //
-108, -308, 996, 991, 958, -1460, 1522, 1628 // Returns 16-bit integer congruent to a*b*R^{-1} mod q
]; pub fn fqmul(a: i16, b: i16) -> i16 {
montgomery_reduce(a as i32 * b as i32)
// Name: fqmul }
//
// Description: Multiplication followed by Montgomery reduction // Name: ntt
// //
// Arguments: - i16 a: first factor // Description: Inplace number-theoretic transform (NTT) in Rq
// - i16 b: second factor // input is in standard order, output is in bitreversed order
// //
// Returns 16-bit integer congruent to a*b*R^{-1} mod q // Arguments: - i16 r[256]: input/output vector of elements of Zq
pub fn fqmul(a: i16, b: i16) -> i16 pub fn ntt(r: &mut [i16]) {
{ let mut j;
montgomery_reduce(a as i32 * b as i32) let mut k = 1usize;
} let mut len = 128;
let (mut t, mut zeta);
// Name: ntt
// while len >= 2 {
// Description: Inplace number-theoretic transform (NTT) in Rq let mut start = 0;
// input is in standard order, output is in bitreversed order while start < 256 {
// zeta = ZETAS[k];
// Arguments: - i16 r[256]: input/output vector of elements of Zq k += 1;
pub fn ntt(r: &mut[i16]) j = start;
{ while j < (start + len) {
let mut j; t = fqmul(zeta, r[j + len]);
let mut k = 1usize; r[j + len] = r[j] - t;
let mut len = 128; r[j] += t;
let (mut t, mut zeta); j += 1;
}
while len >= 2 { start = j + len;
let mut start = 0; }
while start < 256 { len >>= 1;
zeta = ZETAS[k]; }
k += 1; }
j = start;
while j < (start + len) { // Name: invntt
t = fqmul(zeta, r[j + len]); //
r[j + len] = r[j] - t; // Description: Inplace inverse number-theoretic transform in Rq
r[j] += t; // input is in bitreversed order, output is in standard order
j += 1; //
} // Arguments: - i16 r[256]: input/output vector of elements of Zq
start = j + len; pub fn invntt(r: &mut [i16]) {
} let mut j;
len >>= 1; let mut k = 127usize;
} let mut len = 2;
} let (mut t, mut zeta);
const F: i16 = 1441; // mont^2/128
// Name: invntt while len <= 128 {
// let mut start = 0;
// Description: Inplace inverse number-theoretic transform in Rq while start < 256 {
// input is in bitreversed order, output is in standard order zeta = ZETAS[k];
// k -= 1;
// Arguments: - i16 r[256]: input/output vector of elements of Zq j = start;
pub fn invntt(r: &mut[i16]) while j < (start + len) {
{ t = r[j];
let mut j; r[j] = barrett_reduce(t + r[j + len]);
let mut k = 127usize; r[j + len] = r[j + len] - t;
let mut len = 2; r[j + len] = fqmul(zeta, r[j + len]);
let (mut t, mut zeta); j += 1
const F: i16 = 1441; // mont^2/128 }
while len <= 128 { start = j + len;
let mut start = 0; }
while start < 256 { len <<= 1;
zeta = ZETAS[k]; }
k -= 1; for j in 0..256 {
j = start; r[j] = fqmul(r[j], F);
while j < (start + len) { }
t = r[j]; }
r[j] = barrett_reduce(t + r[j + len]);
r[j + len] = r[j + len] -t; // Name: basemul
r[j + len] = fqmul(zeta, r[j + len]); //
j += 1 // Description: Multiplication of polynomials in Zq[X]/((X^2-zeta))
} // used for multiplication of elements in Rq in NTT domain
start = j + len; //
} // Arguments: - i16 r[2]: the output polynomial
len <<= 1; // - const i16 a[2]: the first factor
} // - const i16 b[2]: the second factor
for j in 0..256 { // - i16 zeta: integer defining the reduction polynomial
r[j] = fqmul(r[j], F); pub fn basemul(r: &mut [i16], a: &[i16], b: &[i16], zeta: i16) {
} r[0] = fqmul(a[1], b[1]);
} r[0] = fqmul(r[0], zeta);
r[0] += fqmul(a[0], b[0]);
// Name: basemul
// r[1] = fqmul(a[0], b[1]);
// Description: Multiplication of polynomials in Zq[X]/((X^2-zeta)) r[1] += fqmul(a[1], b[0]);
// used for multiplication of elements in Rq in NTT domain }
//
// Arguments: - i16 r[2]: the output polynomial
// - const i16 a[2]: the first factor
// - const i16 b[2]: the second factor
// - i16 zeta: integer defining the reduction polynomial
pub fn basemul(r: &mut[i16], a: &[i16], b: &[i16], zeta: i16)
{
r[0] = fqmul(a[1], b[1]);
r[0] = fqmul(r[0], zeta);
r[0] += fqmul(a[0], b[0]);
r[1] = fqmul(a[0], b[1]);
r[1] += fqmul(a[1], b[0]);
}

639
third_party/kyber/src/reference/poly.rs vendored
View file

@ -1,335 +1,304 @@
use crate::{ use crate::{cbd::*, ntt::*, params::*, reduce::*, symmetric::*};
params::*,
ntt::*, #[derive(Clone)]
reduce::*, pub struct Poly {
cbd::*, pub coeffs: [i16; KYBER_N],
symmetric::* }
};
impl Copy for Poly {}
#[derive(Clone)]
pub struct Poly { impl Default for Poly {
pub coeffs: [i16; KYBER_N] fn default() -> Self {
} Poly { coeffs: [0i16; KYBER_N] }
}
impl Copy for Poly {} }
impl Default for Poly { // new() is nicer
fn default() -> Self { impl Poly {
Poly { pub fn new() -> Self {
coeffs: [0i16; KYBER_N] Self::default()
} }
} }
}
// Name: poly_compress
// new() is nicer //
impl Poly { // Description: Compression and subsequent serialization of a polynomial
pub fn new() -> Self { //
Self::default() // Arguments: - [u8] r: output byte array (needs space for KYBER_POLYCOMPRESSEDBYTES bytes)
} // - const poly *a: input polynomial
} pub fn poly_compress(r: &mut [u8], a: Poly) {
let mut t = [0u8; 8];
// Name: poly_compress let mut k = 0usize;
// let mut u: i16;
// Description: Compression and subsequent serialization of a polynomial
// match KYBER_POLYCOMPRESSEDBYTES {
// Arguments: - [u8] r: output byte array (needs space for KYBER_POLYCOMPRESSEDBYTES bytes) 128 => {
// - const poly *a: input polynomial for i in 0..KYBER_N / 8 {
pub fn poly_compress(r: &mut[u8], a: Poly) for j in 0..8 {
{ // map to positive standard representatives
let mut t = [0u8; 8]; u = a.coeffs[8 * i + j];
let mut k = 0usize; u += (u >> 15) & KYBER_Q as i16;
let mut u: i16; t[j] = (((((u as u16) << 4) + KYBER_Q as u16 / 2) / KYBER_Q as u16) & 15) as u8;
}
match KYBER_POLYCOMPRESSEDBYTES { r[k] = t[0] | (t[1] << 4);
128 => { r[k + 1] = t[2] | (t[3] << 4);
for i in 0..KYBER_N/8 { r[k + 2] = t[4] | (t[5] << 4);
for j in 0..8 { r[k + 3] = t[6] | (t[7] << 4);
// map to positive standard representatives k += 4;
u = a.coeffs[8*i+j]; }
u += (u >> 15) & KYBER_Q as i16; }
t[j] = (((((u as u16) << 4) + KYBER_Q as u16 /2) / KYBER_Q as u16) & 15) as u8; 160 => {
} for i in 0..(KYBER_N / 8) {
r[k] = t[0] | (t[1] << 4); for j in 0..8 {
r[k+1] = t[2] | (t[3] << 4); // map to positive standard representatives
r[k+2] = t[4] | (t[5] << 4); u = a.coeffs[8 * i + j];
r[k+3] = t[6] | (t[7] << 4); u += (u >> 15) & KYBER_Q as i16;
k += 4; t[j] = (((((u as u32) << 5) + KYBER_Q as u32 / 2) / KYBER_Q as u32) & 31) as u8;
} }
}, r[k] = t[0] | (t[1] << 5);
160 => { r[k + 1] = (t[1] >> 3) | (t[2] << 2) | (t[3] << 7);
for i in 0..(KYBER_N/8) { r[k + 2] = (t[3] >> 1) | (t[4] << 4);
for j in 0..8 { r[k + 3] = (t[4] >> 4) | (t[5] << 1) | (t[6] << 6);
// map to positive standard representatives r[k + 4] = (t[6] >> 2) | (t[7] << 3);
u = a.coeffs[8*i+j]; k += 5;
u += (u >> 15) & KYBER_Q as i16; }
t[j] = (((((u as u32) << 5) + KYBER_Q as u32/2) / KYBER_Q as u32) & 31) as u8; }
} _ => panic!("KYBER_POLYCOMPRESSEDBYTES needs to be one of (128, 160)"),
r[k] = t[0] | (t[1] << 5); }
r[k+1] = (t[1] >> 3) | (t[2] << 2) | (t[3] << 7); }
r[k+2] = (t[3] >> 1) | (t[4] << 4);
r[k+3] = (t[4] >> 4) | (t[5] << 1) | (t[6] << 6); // Name: poly_decompress
r[k+4] = (t[6] >> 2) | (t[7] << 3); //
k += 5; // Description: De-serialization and subsequent decompression of a polynomial;
} // approximate inverse of poly_compress
}, //
_ => panic!("KYBER_POLYCOMPRESSEDBYTES needs to be one of (128, 160)") // Arguments: - poly *r: output polynomial
} // - const [u8] a: input byte array (of length KYBER_POLYCOMPRESSEDBYTES bytes)
} pub fn poly_decompress(r: &mut Poly, a: &[u8]) {
match KYBER_POLYCOMPRESSEDBYTES {
128 => {
// Name: poly_decompress let mut idx = 0usize;
// for i in 0..KYBER_N / 2 {
// Description: De-serialization and subsequent decompression of a polynomial; r.coeffs[2 * i + 0] = ((((a[idx] & 15) as usize * KYBER_Q) + 8) >> 4) as i16;
// approximate inverse of poly_compress r.coeffs[2 * i + 1] = ((((a[idx] >> 4) as usize * KYBER_Q) + 8) >> 4) as i16;
// idx += 1;
// Arguments: - poly *r: output polynomial }
// - const [u8] a: input byte array (of length KYBER_POLYCOMPRESSEDBYTES bytes) }
pub fn poly_decompress(r: &mut Poly, a: &[u8]) 160 => {
{ let mut idx = 0usize;
match KYBER_POLYCOMPRESSEDBYTES { let mut t = [0u8; 8];
128 => { for i in 0..KYBER_N / 8 {
let mut idx = 0usize; t[0] = a[idx + 0];
for i in 0..KYBER_N/2 { t[1] = (a[idx + 0] >> 5) | (a[idx + 1] << 3);
r.coeffs[2*i+0] = ((((a[idx] & 15) as usize * KYBER_Q) + 8) >> 4) as i16; t[2] = a[idx + 1] >> 2;
r.coeffs[2*i+1] = ((((a[idx] >> 4) as usize * KYBER_Q) + 8) >> 4) as i16; t[3] = (a[idx + 1] >> 7) | (a[idx + 2] << 1);
idx += 1; t[4] = (a[idx + 2] >> 4) | (a[idx + 3] << 4);
} t[5] = a[idx + 3] >> 1;
}, t[6] = (a[idx + 3] >> 6) | (a[idx + 4] << 2);
160 => { t[7] = a[idx + 4] >> 3;
let mut idx = 0usize; idx += 5;
let mut t = [0u8;8]; for j in 0..8 {
for i in 0..KYBER_N/8 { r.coeffs[8 * i + j] = ((((t[j] as u32) & 31) * KYBER_Q as u32 + 16) >> 5) as i16;
t[0] = a[idx+0]; }
t[1] = (a[idx+0] >> 5) | (a[idx+1] << 3); }
t[2] = a[idx+1] >> 2; }
t[3] = (a[idx+1] >> 7) | (a[idx+2] << 1); _ => panic!("KYBER_POLYCOMPRESSEDBYTES needs to be either (128, 160)"),
t[4] = (a[idx+2] >> 4) | (a[idx+3] << 4); }
t[5] = a[idx+3] >> 1; }
t[6] = (a[idx+3] >> 6) | (a[idx+4] << 2);
t[7] = a[idx+4] >> 3; // Name: poly_tobytes
idx += 5; //
for j in 0..8 { // Description: Serialization of a polynomial
r.coeffs[8*i+j] = ((((t[j] as u32) & 31)*KYBER_Q as u32 + 16) >> 5) as i16; //
} // Arguments: - [u8] r: output byte array (needs space for KYBER_POLYBYTES bytes)
} // - const poly *a: input polynomial
}, pub fn poly_tobytes(r: &mut [u8], a: Poly) {
_ => panic!("KYBER_POLYCOMPRESSEDBYTES needs to be either (128, 160)") let (mut t0, mut t1);
}
} for i in 0..(KYBER_N / 2) {
// map to positive standard representatives
// Name: poly_tobytes t0 = a.coeffs[2 * i];
// t0 += (t0 >> 15) & KYBER_Q as i16;
// Description: Serialization of a polynomial t1 = a.coeffs[2 * i + 1];
// t1 += (t1 >> 15) & KYBER_Q as i16;
// Arguments: - [u8] r: output byte array (needs space for KYBER_POLYBYTES bytes) r[3 * i + 0] = (t0 >> 0) as u8;
// - const poly *a: input polynomial r[3 * i + 1] = ((t0 >> 8) | (t1 << 4)) as u8;
pub fn poly_tobytes(r: &mut[u8], a: Poly) r[3 * i + 2] = (t1 >> 4) as u8;
{ }
let (mut t0, mut t1); }
for i in 0..(KYBER_N/2) { // Name: poly_frombytes
// map to positive standard representatives //
t0 = a.coeffs[2*i]; // Description: De-serialization of a polynomial;
t0 += (t0 >> 15) & KYBER_Q as i16; // inverse of poly_tobytes
t1 = a.coeffs[2*i+1]; //
t1 += (t1 >> 15) & KYBER_Q as i16; // Arguments: - poly *r: output polynomial
r[3*i+0] = (t0 >> 0) as u8; // - const [u8] a: input byte array (of KYBER_POLYBYTES bytes)
r[3*i+1] = ((t0 >> 8) | (t1 << 4)) as u8; pub fn poly_frombytes(r: &mut Poly, a: &[u8]) {
r[3*i+2] = (t1 >> 4) as u8; for i in 0..(KYBER_N / 2) {
} r.coeffs[2 * i + 0] = ((a[3 * i + 0] >> 0) as u16 | ((a[3 * i + 1] as u16) << 8) & 0xFFF) as i16;
} r.coeffs[2 * i + 1] = ((a[3 * i + 1] >> 4) as u16 | ((a[3 * i + 2] as u16) << 4) & 0xFFF) as i16;
}
// Name: poly_frombytes }
//
// Description: De-serialization of a polynomial; // Name: poly_getnoise_eta1
// inverse of poly_tobytes //
// // Description: Sample a polynomial deterministically from a seed and a nonce,
// Arguments: - poly *r: output polynomial // with output polynomial close to centered binomial distribution
// - const [u8] a: input byte array (of KYBER_POLYBYTES bytes) // with parameter KYBER_ETA1
pub fn poly_frombytes(r: &mut Poly, a: &[u8]) //
{ // Arguments: - poly *r: output polynomial
for i in 0..(KYBER_N/2) { // - const [u8] seed: input seed (pointing to array of length KYBER_SYMBYTES bytes)
r.coeffs[2*i+0] = ((a[3*i+0] >> 0) as u16 | ((a[3*i+1] as u16) << 8) & 0xFFF) as i16; // - [u8] nonce: one-byte input nonce
r.coeffs[2*i+1] = ((a[3*i+1] >> 4) as u16 | ((a[3*i+2] as u16) << 4) & 0xFFF) as i16; pub fn poly_getnoise_eta1(r: &mut Poly, seed: &[u8], nonce: u8) {
} const LENGTH: usize = KYBER_ETA1 * KYBER_N / 4;
} let mut buf = [0u8; LENGTH];
prf(&mut buf, LENGTH, seed, nonce);
// Name: poly_getnoise_eta1 poly_cbd_eta1(r, &buf);
// }
// Description: Sample a polynomial deterministically from a seed and a nonce,
// with output polynomial close to centered binomial distribution // Name: poly_getnoise_eta2
// with parameter KYBER_ETA1 //
// // Description: Sample a polynomial deterministically from a seed and a nonce,
// Arguments: - poly *r: output polynomial // with output polynomial close to centered binomial distribution
// - const [u8] seed: input seed (pointing to array of length KYBER_SYMBYTES bytes) // with parameter KYBER_ETA2
// - [u8] nonce: one-byte input nonce //
pub fn poly_getnoise_eta1(r: &mut Poly, seed: &[u8], nonce: u8) // Arguments: - poly *r: output polynomial
{ // - const [u8] seed: input seed (pointing to array of length KYBER_SYMBYTES bytes)
const LENGTH: usize = KYBER_ETA1*KYBER_N/4; // - [u8] nonce: one-byte input nonce
let mut buf = [0u8; LENGTH]; pub fn poly_getnoise_eta2(r: &mut Poly, seed: &[u8], nonce: u8) {
prf(&mut buf, LENGTH, seed, nonce); const LENGTH: usize = KYBER_ETA2 * KYBER_N / 4;
poly_cbd_eta1(r, &buf); let mut buf = [0u8; LENGTH];
} prf(&mut buf, LENGTH, seed, nonce);
poly_cbd_eta2(r, &buf);
// Name: poly_getnoise_eta2 }
//
// Description: Sample a polynomial deterministically from a seed and a nonce, // Name: poly_ntt
// with output polynomial close to centered binomial distribution //
// with parameter KYBER_ETA2 // Description: Computes negacyclic number-theoretic transform (NTT) of
// // a polynomial in place;
// Arguments: - poly *r: output polynomial // inputs assumed to be in normal order, output in bitreversed order
// - const [u8] seed: input seed (pointing to array of length KYBER_SYMBYTES bytes) //
// - [u8] nonce: one-byte input nonce // Arguments: - Poly r: in/output polynomial
pub fn poly_getnoise_eta2(r: &mut Poly, seed: &[u8], nonce: u8) pub fn poly_ntt(r: &mut Poly) {
{ ntt(&mut r.coeffs);
const LENGTH: usize = KYBER_ETA2*KYBER_N/4; poly_reduce(r);
let mut buf = [0u8; LENGTH]; }
prf(&mut buf, LENGTH, seed, nonce);
poly_cbd_eta2(r, &buf); // Name: poly_invntt
} //
// Description: Computes inverse of negacyclic number-theoretic transform (NTT) of
// a polynomial in place;
// inputs assumed to be in bitreversed order, output in normal order
// Name: poly_ntt //
// // Arguments: - Poly a: in/output polynomial
// Description: Computes negacyclic number-theoretic transform (NTT) of pub fn poly_invntt_tomont(r: &mut Poly) {
// a polynomial in place; invntt(&mut r.coeffs);
// inputs assumed to be in normal order, output in bitreversed order }
//
// Arguments: - Poly r: in/output polynomial // Name: poly_basemul
pub fn poly_ntt(r: &mut Poly) //
{ // Description: Multiplication of two polynomials in NTT domain
ntt(&mut r.coeffs); //
poly_reduce(r); // Arguments: - poly *r: output polynomial
} // - const poly *a: first input polynomial
// - const poly *b: second input polynomial
// Name: poly_invntt pub fn poly_basemul(r: &mut Poly, a: &Poly, b: &Poly) {
// for i in 0..(KYBER_N / 4) {
// Description: Computes inverse of negacyclic number-theoretic transform (NTT) of basemul(&mut r.coeffs[4 * i..], &a.coeffs[4 * i..], &b.coeffs[4 * i..], ZETAS[64 + i]);
// a polynomial in place; basemul(
// inputs assumed to be in bitreversed order, output in normal order &mut r.coeffs[4 * i + 2..],
// &a.coeffs[4 * i + 2..],
// Arguments: - Poly a: in/output polynomial &b.coeffs[4 * i + 2..],
pub fn poly_invntt_tomont(r: &mut Poly) -(ZETAS[64 + i]),
{ );
invntt(&mut r.coeffs); }
} }
// Name: poly_basemul // Name: poly_frommont
// //
// Description: Multiplication of two polynomials in NTT domain // Description: Inplace conversion of all coefficients of a polynomial
// // from Montgomery domain to normal domain
// Arguments: - poly *r: output polynomial //
// - const poly *a: first input polynomial // Arguments: - poly *r: input/output polynomial
// - const poly *b: second input polynomial pub fn poly_frommont(r: &mut Poly) {
pub fn poly_basemul(r: &mut Poly, a: &Poly, b: &Poly) let f = ((1u64 << 32) % KYBER_Q as u64) as i16;
{ for i in 0..KYBER_N {
for i in 0..(KYBER_N/4) { let a = r.coeffs[i] as i32 * f as i32;
r.coeffs[i] = montgomery_reduce(a);
basemul( }
&mut r.coeffs[4*i..], }
&a.coeffs[4*i..],
&b.coeffs[4*i..], // Name: poly_reduce
ZETAS[64 + i] //
); // Description: Applies Barrett reduction to all coefficients of a polynomial
basemul( // for details of the Barrett reduction see comments in reduce.c
&mut r.coeffs[4*i+2..], //
&a.coeffs[4*i+2..], // Arguments: - poly *r: input/output polynomial
&b.coeffs[4*i+2..], pub fn poly_reduce(r: &mut Poly) {
-(ZETAS[64 + i])); for i in 0..KYBER_N {
} r.coeffs[i] = barrett_reduce(r.coeffs[i]);
} }
}
// Name: poly_frommont
// // Name: poly_add
// Description: Inplace conversion of all coefficients of a polynomial //
// from Montgomery domain to normal domain // Description: Add two polynomials; no modular reduction is performed
// //
// Arguments: - poly *r: input/output polynomial // Arguments: - poly *r: output polynomial
pub fn poly_frommont(r: &mut Poly) // - const poly *a: first input polynomial
{ // - const poly *b: second input polynomial
let f = ((1u64 << 32) % KYBER_Q as u64) as i16; pub fn poly_add(r: &mut Poly, b: &Poly) {
for i in 0..KYBER_N { for i in 0..KYBER_N {
let a = r.coeffs[i] as i32 * f as i32; r.coeffs[i] += b.coeffs[i];
r.coeffs[i] = montgomery_reduce(a); }
} }
}
// Name: poly_sub
// Name: poly_reduce //
// // Description: Subtract two polynomials; no modular reduction is performed
// Description: Applies Barrett reduction to all coefficients of a polynomial //
// for details of the Barrett reduction see comments in reduce.c // Arguments: - poly *r: output polynomial
// // - const poly *a: first input polynomial
// Arguments: - poly *r: input/output polynomial // - const poly *b: second input polynomial
pub fn poly_reduce(r: &mut Poly) pub fn poly_sub(r: &mut Poly, a: &Poly) {
{ for i in 0..KYBER_N {
for i in 0..KYBER_N { r.coeffs[i] = a.coeffs[i] - r.coeffs[i];
r.coeffs[i] = barrett_reduce(r.coeffs[i]); }
} }
}
// Name: poly_frommsg
// Name: poly_add //
// // Description: Convert 32-byte message to polynomial
// Description: Add two polynomials; no modular reduction is performed //
// // Arguments: - poly *r: output polynomial
// Arguments: - poly *r: output polynomial // - const [u8] msg: input message
// - const poly *a: first input polynomial pub fn poly_frommsg(r: &mut Poly, msg: &[u8]) {
// - const poly *b: second input polynomial let mut mask;
pub fn poly_add(r: &mut Poly, b: &Poly) for i in 0..KYBER_SYMBYTES {
{ for j in 0..8 {
for i in 0..KYBER_N { mask = ((msg[i] as u16 >> j) & 1).wrapping_neg();
r.coeffs[i] += b.coeffs[i]; r.coeffs[8 * i + j] = (mask & ((KYBER_Q + 1) / 2) as u16) as i16;
} }
} }
}
// Name: poly_sub
// // Name: poly_tomsg
// Description: Subtract two polynomials; no modular reduction is performed //
// // Description: Convert polynomial to 32-byte message
// Arguments: - poly *r: output polynomial //
// - const poly *a: first input polynomial // Arguments: - [u8] msg: output message
// - const poly *b: second input polynomial // - const poly *a: input polynomial
pub fn poly_sub(r: &mut Poly, a: &Poly) pub fn poly_tomsg(msg: &mut [u8], a: Poly) {
{ let mut t;
for i in 0..KYBER_N {
r.coeffs[i] = a.coeffs[i] - r.coeffs[i]; for i in 0..KYBER_SYMBYTES {
} msg[i] = 0;
} for j in 0..8 {
t = a.coeffs[8 * i + j];
// Name: poly_frommsg t += (t >> 15) & KYBER_Q as i16;
// t = (((t << 1) + KYBER_Q as i16 / 2) / KYBER_Q as i16) & 1;
// Description: Convert 32-byte message to polynomial msg[i] |= (t << j) as u8;
// }
// Arguments: - poly *r: output polynomial }
// - const [u8] msg: input message }
pub fn poly_frommsg(r: &mut Poly, msg: &[u8])
{
let mut mask;
for i in 0..KYBER_SYMBYTES {
for j in 0..8 {
mask = ((msg[i] as u16 >> j) & 1 ).wrapping_neg();
r.coeffs[8*i+j] = (mask & ((KYBER_Q+1)/2) as u16) as i16;
}
}
}
// Name: poly_tomsg
//
// Description: Convert polynomial to 32-byte message
//
// Arguments: - [u8] msg: output message
// - const poly *a: input polynomial
pub fn poly_tomsg(msg: &mut[u8], a: Poly)
{
let mut t;
for i in 0..KYBER_SYMBYTES {
msg[i] = 0;
for j in 0..8 {
t = a.coeffs[8*i+j];
t += (t >> 15) & KYBER_Q as i16;
t = (((t << 1) + KYBER_Q as i16 /2) / KYBER_Q as i16) & 1;
msg[i] |= (t << j) as u8;
}
}
}

463
third_party/kyber/src/reference/polyvec.rs vendored
View file

@ -1,241 +1,222 @@
#![allow(clippy::precedence)] #![allow(clippy::precedence)]
use crate::{ use crate::{params::*, poly::*};
poly::*,
params::* #[derive(Clone)]
}; pub struct Polyvec {
pub vec: [Poly; KYBER_K],
#[derive(Clone)] }
pub struct Polyvec {
pub vec: [Poly; KYBER_K] impl Copy for Polyvec {}
}
impl Polyvec {
impl Copy for Polyvec {} pub fn new() -> Self {
Polyvec { vec: [Poly::new(); KYBER_K] }
impl Polyvec { }
pub fn new() -> Self {
Polyvec { // #[cfg(debug_assertions)]
vec: [Poly::new(); KYBER_K] // pub fn checksum(&self) -> i16 {
} // let mut out = 0i16;
} // for i in 0..KYBER_K {
// for j in 0..KYBER_N {
// #[cfg(debug_assertions)] // out ^= &self.vec[i].coeffs[j]
// pub fn checksum(&self) -> i16 { // }
// let mut out = 0i16; // }
// for i in 0..KYBER_K { // out
// for j in 0..KYBER_N { // }
// out ^= &self.vec[i].coeffs[j] }
// }
// } // Name: polyvec_compress
// out //
// } // Description: Compress and serialize vector of polynomials
} //
// Arguments: - [u8] r: output byte array (needs space for KYBER_POLYVECCOMPRESSEDBYTES)
// Name: polyvec_compress // - const Polyvec a: input vector of polynomials
// pub fn polyvec_compress(r: &mut [u8], a: Polyvec) {
// Description: Compress and serialize vector of polynomials #[cfg(feature = "kyber1024")]
// {
// Arguments: - [u8] r: output byte array (needs space for KYBER_POLYVECCOMPRESSEDBYTES) let mut t = [0u16; 8];
// - const Polyvec a: input vector of polynomials let mut idx = 0usize;
pub fn polyvec_compress(r: &mut[u8], a: Polyvec) for i in 0..KYBER_K {
{ for j in 0..KYBER_N / 8 {
#[cfg(feature="kyber1024")] for k in 0..8 {
{ t[k] = a.vec[i].coeffs[8 * j + k] as u16;
let mut t = [0u16; 8]; t[k] = t[k].wrapping_add((((t[k] as i16) >> 15) & KYBER_Q as i16) as u16);
let mut idx = 0usize; t[k] = (((((t[k] as u32) << 11) + KYBER_Q as u32 / 2) / KYBER_Q as u32) & 0x7ff) as u16;
for i in 0..KYBER_K { }
for j in 0..KYBER_N/8 { r[idx + 0] = (t[0] >> 0) as u8;
for k in 0..8 { r[idx + 1] = ((t[0] >> 8) | (t[1] << 3)) as u8;
t[k] = a.vec[i].coeffs[8*j+k] as u16; r[idx + 2] = ((t[1] >> 5) | (t[2] << 6)) as u8;
t[k] = t[k].wrapping_add((((t[k] as i16) >> 15) & KYBER_Q as i16) as u16); r[idx + 3] = (t[2] >> 2) as u8;
t[k] = (((((t[k] as u32) << 11) + KYBER_Q as u32/2)/KYBER_Q as u32) & 0x7ff ) as u16; r[idx + 4] = ((t[2] >> 10) | (t[3] << 1)) as u8;
} r[idx + 5] = ((t[3] >> 7) | (t[4] << 4)) as u8;
r[idx+0] = (t[0] >> 0) as u8; r[idx + 6] = ((t[4] >> 4) | (t[5] << 7)) as u8;
r[idx+1] = ((t[0] >> 8) | (t[1] << 3)) as u8; r[idx + 7] = (t[5] >> 1) as u8;
r[idx+2] = ((t[1] >> 5) | (t[2] << 6)) as u8; r[idx + 8] = ((t[5] >> 9) | (t[6] << 2)) as u8;
r[idx+3] = (t[2] >> 2) as u8; r[idx + 9] = ((t[6] >> 6) | (t[7] << 5)) as u8;
r[idx+4] = ((t[2] >> 10) | (t[3] << 1)) as u8; r[idx + 10] = (t[7] >> 3) as u8;
r[idx+5] = ((t[3] >> 7) | (t[4] << 4)) as u8; idx += 11
r[idx+6] = ((t[4] >> 4) | (t[5] << 7)) as u8; }
r[idx+7] = (t[5] >> 1) as u8; }
r[idx+8] = ((t[5] >> 9) | (t[6] << 2)) as u8; }
r[idx+9] = ((t[6] >> 6) | (t[7] << 5)) as u8;
r[idx+10] = (t[7] >> 3) as u8; #[cfg(not(feature = "kyber1024"))]
idx += 11 {
} let mut t = [0u16; 4];
} let mut idx = 0usize;
} for i in 0..KYBER_K {
for j in 0..KYBER_N / 4 {
#[cfg(not(feature="kyber1024"))] for k in 0..4 {
{ t[k] = a.vec[i].coeffs[4 * j + k] as u16;
let mut t = [0u16; 4]; t[k] = t[k].wrapping_add((((t[k] as i16) >> 15) & KYBER_Q as i16) as u16);
let mut idx = 0usize; t[k] = (((((t[k] as u32) << 10) + KYBER_Q as u32 / 2) / KYBER_Q as u32) & 0x3ff) as u16;
for i in 0..KYBER_K { }
for j in 0..KYBER_N/4 { r[idx + 0] = (t[0] >> 0) as u8;
for k in 0..4 { r[idx + 1] = ((t[0] >> 8) | (t[1] << 2)) as u8;
t[k] = a.vec[i].coeffs[4*j+k] as u16; r[idx + 2] = ((t[1] >> 6) | (t[2] << 4)) as u8;
t[k] = t[k].wrapping_add((((t[k] as i16) >> 15) & KYBER_Q as i16) as u16); r[idx + 3] = ((t[2] >> 4) | (t[3] << 6)) as u8;
t[k] = r[idx + 4] = (t[3] >> 2) as u8;
(((((t[k] as u32) << 10) + KYBER_Q as u32/2)/ KYBER_Q as u32) & 0x3ff) as u16; idx += 5;
} }
r[idx+0] = (t[0] >> 0) as u8; }
r[idx+1] = ((t[0] >> 8) | (t[1] << 2)) as u8; }
r[idx+2] = ((t[1] >> 6) | (t[2] << 4)) as u8; }
r[idx+3] = ((t[2] >> 4) | (t[3] << 6)) as u8;
r[idx+4] = (t[3] >> 2) as u8; // Name: polyvec_decompress
idx += 5; //
} // Description: De-serialize and decompress vector of polynomials;
} // approximate inverse of polyvec_compress
} //
} // Arguments: - Polyvec r: output vector of polynomials
// - [u8] a: input byte array (of length KYBER_POLYVECCOMPRESSEDBYTES)
// Name: polyvec_decompress pub fn polyvec_decompress(r: &mut Polyvec, a: &[u8]) {
// #[cfg(feature = "kyber1024")]
// Description: De-serialize and decompress vector of polynomials; {
// approximate inverse of polyvec_compress let mut t = [0u16; 8];
// let mut idx = 0usize;
// Arguments: - Polyvec r: output vector of polynomials for i in 0..KYBER_K {
// - [u8] a: input byte array (of length KYBER_POLYVECCOMPRESSEDBYTES) for j in 0..KYBER_N / 8 {
pub fn polyvec_decompress(r: &mut Polyvec, a: &[u8]) t[0] = (a[idx + 0] >> 0) as u16 | (a[idx + 1] as u16) << 8;
{ t[1] = (a[idx + 1] >> 3) as u16 | (a[idx + 2] as u16) << 5;
t[2] = (a[idx + 2] >> 6) as u16 | (a[idx + 3] as u16) << 2 | (a[idx + 4] as u16) << 10;
#[cfg(feature="kyber1024")] t[3] = (a[idx + 4] >> 1) as u16 | (a[idx + 5] as u16) << 7;
{ t[4] = (a[idx + 5] >> 4) as u16 | (a[idx + 6] as u16) << 4;
let mut t = [0u16; 8]; t[5] = (a[idx + 6] >> 7) as u16 | (a[idx + 7] as u16) << 1 | (a[idx + 8] as u16) << 9;
let mut idx = 0usize; t[6] = (a[idx + 8] >> 2) as u16 | (a[idx + 9] as u16) << 6;
for i in 0..KYBER_K { t[7] = (a[idx + 9] >> 5) as u16 | (a[idx + 10] as u16) << 3;
for j in 0..KYBER_N/8 { idx += 11;
t[0] = (a[idx+0] >> 0) as u16 | (a[idx+ 1] as u16) << 8;
t[1] = (a[idx+1] >> 3) as u16 | (a[idx+ 2] as u16) << 5; for k in 0..8 {
t[2] = (a[idx+2] >> 6) as u16 | (a[idx+ 3] as u16) << 2 | (a[idx+4] as u16) << 10; r.vec[i].coeffs[8 * j + k] = (((t[k] & 0x7FF) as u32 * KYBER_Q as u32 + 1024) >> 11) as i16;
t[3] = (a[idx+4] >> 1) as u16 | (a[idx+ 5] as u16) << 7; }
t[4] = (a[idx+5] >> 4) as u16 | (a[idx+ 6] as u16) << 4; }
t[5] = (a[idx+6] >> 7) as u16 | (a[idx+ 7] as u16) << 1 | (a[idx+8] as u16) << 9; }
t[6] = (a[idx+8] >> 2) as u16 | (a[idx+ 9] as u16) << 6; }
t[7] = (a[idx+9] >> 5) as u16 | (a[idx+10] as u16) << 3;
idx += 11; #[cfg(not(feature = "kyber1024"))]
{
for k in 0..8 { let mut idx = 0usize;
r.vec[i].coeffs[8*j+k] = let mut t = [0u16; 4];
(((t[k] & 0x7FF)as u32 * KYBER_Q as u32 + 1024) >> 11) as i16; for i in 0..KYBER_K {
} for j in 0..KYBER_N / 4 {
} t[0] = (a[idx + 0] >> 0) as u16 | (a[idx + 1] as u16) << 8;
} t[1] = (a[idx + 1] >> 2) as u16 | (a[idx + 2] as u16) << 6;
} t[2] = (a[idx + 2] >> 4) as u16 | (a[idx + 3] as u16) << 4;
t[3] = (a[idx + 3] >> 6) as u16 | (a[idx + 4] as u16) << 2;
#[cfg(not(feature="kyber1024"))] idx += 5;
{
let mut idx = 0usize; for k in 0..4 {
let mut t = [0u16; 4]; r.vec[i].coeffs[4 * j + k] = ((((t[k] as u32) & 0x3FF) * KYBER_Q as u32 + 512) >> 10) as i16;
for i in 0..KYBER_K { }
for j in 0..KYBER_N/4 { }
t[0] = (a[idx+0] >> 0) as u16 | (a[idx+1] as u16) << 8; }
t[1] = (a[idx+1] >> 2) as u16 | (a[idx+2] as u16) << 6; }
t[2] = (a[idx+2] >> 4) as u16 | (a[idx+3] as u16) << 4; }
t[3] = (a[idx+3] >> 6) as u16 | (a[idx+4] as u16) << 2;
idx += 5; // Name: polyvec_tobytes
//
for k in 0..4 { // Description: Serialize vector of polynomials
r.vec[i].coeffs[4*j+k] = //
((((t[k] as u32) & 0x3FF) * KYBER_Q as u32 + 512) >> 10) as i16; // Arguments: - [u8] r: output byte array (needs space for KYBER_POLYVECBYTES)
} // - const Polyvec a: input vector of polynomials
} pub fn polyvec_tobytes(r: &mut [u8], a: &Polyvec) {
} for i in 0..KYBER_K {
} poly_tobytes(&mut r[i * KYBER_POLYBYTES..], a.vec[i]);
} }
}
// Name: polyvec_tobytes
// // Name: polyvec_frombytes
// Description: Serialize vector of polynomials //
// // Description: De-serialize vector of polynomials;
// Arguments: - [u8] r: output byte array (needs space for KYBER_POLYVECBYTES) // inverse of polyvec_tobytes
// - const Polyvec a: input vector of polynomials //
pub fn polyvec_tobytes(r: &mut[u8], a: &Polyvec) // Arguments: - [u8] r: output byte array
{ // - const Polyvec a: input vector of polynomials (of length KYBER_POLYVECBYTES)
for i in 0..KYBER_K { pub fn polyvec_frombytes(r: &mut Polyvec, a: &[u8]) {
poly_tobytes(&mut r[i*KYBER_POLYBYTES..], a.vec[i]); for i in 0..KYBER_K {
} poly_frombytes(&mut r.vec[i], &a[i * KYBER_POLYBYTES..]);
} }
}
// Name: polyvec_frombytes
// // Name: polyvec_ntt
// Description: De-serialize vector of polynomials; //
// inverse of polyvec_tobytes // Description: Apply forward NTT to all elements of a vector of polynomials
// //
// Arguments: - [u8] r: output byte array // Arguments: - Polyvec r: in/output vector of polynomials
// - const Polyvec a: input vector of polynomials (of length KYBER_POLYVECBYTES) pub fn polyvec_ntt(r: &mut Polyvec) {
pub fn polyvec_frombytes(r: &mut Polyvec, a: &[u8]) for i in 0..KYBER_K {
{ poly_ntt(&mut r.vec[i]);
for i in 0..KYBER_K { }
poly_frombytes(&mut r.vec[i], &a[i*KYBER_POLYBYTES..]); }
}
} // Name: polyvec_invntt
//
// Name: polyvec_ntt // Description: Apply inverse NTT to all elements of a vector of polynomials
// //
// Description: Apply forward NTT to all elements of a vector of polynomials // Arguments: - Polyvec r: in/output vector of polynomials
// pub fn polyvec_invntt_tomont(r: &mut Polyvec) {
// Arguments: - Polyvec r: in/output vector of polynomials for i in 0..KYBER_K {
pub fn polyvec_ntt(r: &mut Polyvec) poly_invntt_tomont(&mut r.vec[i]);
{ }
for i in 0..KYBER_K { }
poly_ntt(&mut r.vec[i]);
} // Name: polyvec_basemul_acc_montgomery
} //
// Description: Pointwise multiply elements of a and b and accumulate into r
// Name: polyvec_invntt //
// // Arguments: - poly *r: output polynomial
// Description: Apply inverse NTT to all elements of a vector of polynomials // - const Polyvec a: first input vector of polynomials
// // - const Polyvec b: second input vector of polynomials
// Arguments: - Polyvec r: in/output vector of polynomials pub fn polyvec_basemul_acc_montgomery(r: &mut Poly, a: &Polyvec, b: &Polyvec) {
pub fn polyvec_invntt_tomont(r: &mut Polyvec) let mut t = Poly::new();
{ poly_basemul(r, &a.vec[0], &b.vec[0]);
for i in 0..KYBER_K { for i in 1..KYBER_K {
poly_invntt_tomont(&mut r.vec[i]); poly_basemul(&mut t, &a.vec[i], &b.vec[i]);
} poly_add(r, &t);
} }
poly_reduce(r);
// Name: polyvec_basemul_acc_montgomery }
//
// Description: Pointwise multiply elements of a and b and accumulate into r // Name: polyvec_reduce
// //
// Arguments: - poly *r: output polynomial // Description: Applies Barrett reduction to each coefficient
// - const Polyvec a: first input vector of polynomials // of each element of a vector of polynomials
// - const Polyvec b: second input vector of polynomials // for details of the Barrett reduction see comments in reduce.c
pub fn polyvec_basemul_acc_montgomery(r: &mut Poly, a: &Polyvec, b: &Polyvec) //
{ // Arguments: - poly *r: input/output polynomial
let mut t = Poly::new(); pub fn polyvec_reduce(r: &mut Polyvec) {
poly_basemul(r, &a.vec[0], &b.vec[0]); for i in 0..KYBER_K {
for i in 1..KYBER_K { poly_reduce(&mut r.vec[i]);
poly_basemul(&mut t, &a.vec[i], &b.vec[i]); }
poly_add(r, &t); }
}
poly_reduce(r); // Name: polyvec_add
} //
// Description: Add vectors of polynomials
// Name: polyvec_reduce //
// // Arguments: - Polyvec r: output vector of polynomials
// Description: Applies Barrett reduction to each coefficient // - const Polyvec b: second input vector of polynomials
// of each element of a vector of polynomials pub fn polyvec_add(r: &mut Polyvec, b: &Polyvec) {
// for details of the Barrett reduction see comments in reduce.c for i in 0..KYBER_K {
// poly_add(&mut r.vec[i], &b.vec[i]);
// Arguments: - poly *r: input/output polynomial }
pub fn polyvec_reduce(r: &mut Polyvec) }
{
for i in 0..KYBER_K {
poly_reduce(&mut r.vec[i]);
}
}
// Name: polyvec_add
//
// Description: Add vectors of polynomials
//
// Arguments: - Polyvec r: output vector of polynomials
// - const Polyvec b: second input vector of polynomials
pub fn polyvec_add(r: &mut Polyvec, b: &Polyvec)
{
for i in 0..KYBER_K {
poly_add(&mut r.vec[i], &b.vec[i]);
}
}

78
third_party/kyber/src/reference/reduce.rs vendored
View file

@ -1,41 +1,37 @@
use crate::params::*; use crate::params::*;
const QINV: i32 = 62209; // q^(-1) mod 2^16 const QINV: i32 = 62209; // q^(-1) mod 2^16
// Name: montgomery_reduce // Name: montgomery_reduce
// //
// Description: Montgomery reduction; given a 32-bit integer a, computes // Description: Montgomery reduction; given a 32-bit integer a, computes
// 16-bit integer congruent to a * R^-1 mod q, // 16-bit integer congruent to a * R^-1 mod q,
// where R=2^16 // where R=2^16
// //
// Arguments: - i32 a: input integer to be reduced; has to be in {-q2^15,...,q2^15-1} // Arguments: - i32 a: input integer to be reduced; has to be in {-q2^15,...,q2^15-1}
// //
// Returns: integer in {-q+1,...,q-1} congruent to a * R^-1 modulo q. // Returns: integer in {-q+1,...,q-1} congruent to a * R^-1 modulo q.
pub fn montgomery_reduce(a: i32) -> i16 pub fn montgomery_reduce(a: i32) -> i16 {
{ let ua = a.wrapping_mul(QINV) as i16;
let ua = a.wrapping_mul(QINV) as i16; let u = ua as i32;
let u = ua as i32; let mut t = u * KYBER_Q as i32;
let mut t = u * KYBER_Q as i32; t = a - t;
t = a - t; t >>= 16;
t >>= 16; t as i16
t as i16 }
}
// Name: barrett_reduce
// Name: barrett_reduce //
// // Description: Barrett reduction; given a 16-bit integer a, computes
// Description: Barrett reduction; given a 16-bit integer a, computes // centered representative congruent to a mod q in {-(q-1)/2,...,(q-1)/2}
// centered representative congruent to a mod q in {-(q-1)/2,...,(q-1)/2} //
// // Arguments: - i16 a: input integer to be reduced
// Arguments: - i16 a: input integer to be reduced //
// // Returns: i16 in {-(q-1)/2,...,(q-1)/2} congruent to a modulo q.
// Returns: i16 in {-(q-1)/2,...,(q-1)/2} congruent to a modulo q. pub fn barrett_reduce(a: i16) -> i16 {
pub fn barrett_reduce(a: i16) -> i16 let v = ((1u32 << 26) / KYBER_Q as u32 + 1) as i32;
{ let mut t = v * a as i32 + (1 << 25);
let v = ((1u32 << 26)/KYBER_Q as u32 + 1) as i32; t >>= 26;
let mut t = v * a as i32 + (1 << 25); t *= KYBER_Q as i32;
t >>= 26; a - t as i16
t *= KYBER_Q as i32; }
a - t as i16
}

72
third_party/kyber/src/reference/verify.rs vendored
View file

@ -1,37 +1,35 @@
// Name: verify // Name: verify
// //
// Description: Compare two arrays for equality in constant time. // Description: Compare two arrays for equality in constant time.
// //
// Arguments: const [u8] a: first byte array // Arguments: const [u8] a: first byte array
// const [u8] b: second byte array // const [u8] b: second byte array
// size_t len: length of the byte arrays // size_t len: length of the byte arrays
// //
// Returns 0 if the byte arrays are equal, 1 otherwise // Returns 0 if the byte arrays are equal, 1 otherwise
pub fn verify(a: &[u8], b: &[u8], len: usize) -> u8 pub fn verify(a: &[u8], b: &[u8], len: usize) -> u8 {
{ let mut r = 0u64;
let mut r = 0u64; for i in 0..len {
for i in 0..len { r |= (a[i] ^ b[i]) as u64;
r |= (a[i] ^ b[i]) as u64; }
} r = r.wrapping_neg() >> 63;
r = r.wrapping_neg() >> 63; r as u8
r as u8 }
}
// Name: cmov
// Name: cmov //
// // Description: Copy len bytes from x to r if b is 1;
// Description: Copy len bytes from x to r if b is 1; // don't modify x if b is 0. Requires b to be in {0,1};
// don't modify x if b is 0. Requires b to be in {0,1}; // assumes two's complement representation of negative integers.
// assumes two's complement representation of negative integers. // Runs in constant time.
// Runs in constant time. //
// // Arguments: [u8] r: output byte array
// Arguments: [u8] r: output byte array // const [u8] x: input byte array
// const [u8] x: input byte array // size_t len: Amount of bytes to be copied
// size_t len: Amount of bytes to be copied // [u8] b: Condition bit; has to be in {0,1}
// [u8] b: Condition bit; has to be in {0,1} pub fn cmov(r: &mut [u8], x: &[u8], len: usize, mut b: u8) {
pub fn cmov(r: &mut[u8], x: &[u8], len: usize, mut b: u8) b = b.wrapping_neg();
{ for i in 0..len {
b = b.wrapping_neg(); r[i] ^= b & (x[i] ^ r[i]);
for i in 0..len { }
r[i] ^= b & (x[i] ^ r[i]); }
}
}

21
third_party/kyber/src/rng.rs vendored
View file

@ -1,10 +1,11 @@
#![cfg(not(feature="KATs"))] #![cfg(not(feature = "KATs"))]
use rand_core::*; use rand_core::*;
// Fills buffer x with len bytes, RNG must satisfy the // Fills buffer x with len bytes, RNG must satisfy the
// RngCore trait and CryptoRng marker trait requirements // RngCore trait and CryptoRng marker trait requirements
pub fn randombytes<R>(x: &mut [u8], len: usize, rng: &mut R) pub fn randombytes<R>(x: &mut [u8], len: usize, rng: &mut R)
where R: RngCore + CryptoRng, where
{ R: RngCore + CryptoRng,
rng.fill_bytes(&mut x[..len]); {
} rng.fill_bytes(&mut x[..len]);
}

386
third_party/kyber/src/symmetric.rs vendored
View file

@ -1,205 +1,181 @@
#![allow(dead_code)] #![allow(dead_code)]
#[cfg(not(feature = "90s"))] use crate::{fips202::*, params::*}; #[cfg(feature = "90s")]
#[cfg(feature = "90s")] use crate::aes256ctr::*; use crate::aes256ctr::*;
#[cfg(feature = "90s")] use sha2::{Sha256, Sha512, Digest}; #[cfg(not(feature = "90s"))]
// TODO: Rustrypto AES-CTR feature use crate::{fips202::*, params::*};
// #[cfg(feature = "90s")] use aes_ctr::Aes256Ctr; #[cfg(feature = "90s")]
// #[cfg(feature = "90s")] use aes_ctr::cipher::{ use sha2::{Digest, Sha256, Sha512};
// generic_array::GenericArray, // TODO: Rustrypto AES-CTR feature
// stream::{NewStreamCipher, SyncStreamCipher} // #[cfg(feature = "90s")] use aes_ctr::Aes256Ctr;
// }; // #[cfg(feature = "90s")] use aes_ctr::cipher::{
// generic_array::GenericArray,
#[cfg(feature = "90s")] // stream::{NewStreamCipher, SyncStreamCipher}
pub(crate) const AES256CTR_BLOCKBYTES: usize = 64; // };
#[cfg(feature = "90s")] #[cfg(feature = "90s")]
pub(crate) const XOF_BLOCKBYTES: usize = AES256CTR_BLOCKBYTES; pub(crate) const AES256CTR_BLOCKBYTES: usize = 64;
#[cfg(not(feature = "90s"))]
pub(crate) const XOF_BLOCKBYTES: usize = SHAKE128_RATE; #[cfg(feature = "90s")]
pub(crate) const XOF_BLOCKBYTES: usize = AES256CTR_BLOCKBYTES;
#[cfg(not(feature = "90s"))] #[cfg(not(feature = "90s"))]
pub(crate) type XofState = KeccakState; pub(crate) const XOF_BLOCKBYTES: usize = SHAKE128_RATE;
#[cfg(feature = "90s")] #[cfg(not(feature = "90s"))]
pub(crate) type XofState = Aes256CtrCtx; pub(crate) type XofState = KeccakState;
#[derive(Copy, Clone)] #[cfg(feature = "90s")]
pub(crate) struct KeccakState { pub(crate) type XofState = Aes256CtrCtx;
pub s: [u64; 25],
pub pos: usize #[derive(Copy, Clone)]
} pub(crate) struct KeccakState {
pub s: [u64; 25],
impl KeccakState { pub pos: usize,
pub fn new() -> Self { }
KeccakState {
s: [0u64; 25], impl KeccakState {
pos: 0usize pub fn new() -> Self {
} KeccakState { s: [0u64; 25], pos: 0usize }
} }
pub fn reset(&mut self) { pub fn reset(&mut self) {
self.s = [0u64; 25]; self.s = [0u64; 25];
self.pos = 0; self.pos = 0;
} }
} }
// SHA3-256 // SHA3-256
#[cfg(not(feature = "90s"))] #[cfg(not(feature = "90s"))]
pub(crate) fn hash_h(out: &mut[u8], input: &[u8], inlen: usize) pub(crate) fn hash_h(out: &mut [u8], input: &[u8], inlen: usize) {
{ sha3_256(out, input, inlen);
sha3_256(out, input, inlen); }
}
// 90s mode SHA2-256
// 90s mode SHA2-256 #[cfg(feature = "90s")]
#[cfg(feature = "90s")] pub(crate) fn hash_h(out: &mut [u8], input: &[u8], inlen: usize) {
pub(crate) fn hash_h(out: &mut[u8], input: &[u8], inlen: usize) let mut hasher = Sha256::new();
{ hasher.update(&input[..inlen]);
let mut hasher = Sha256::new(); let digest = hasher.finalize();
hasher.update(&input[..inlen]); out[..digest.len()].copy_from_slice(&digest);
let digest = hasher.finalize(); }
out[..digest.len()].copy_from_slice(&digest);
} #[cfg(not(feature = "90s"))]
pub(crate) fn hash_g(out: &mut [u8], input: &[u8], inlen: usize) {
#[cfg(not(feature = "90s"))] sha3_512(out, input, inlen);
pub(crate) fn hash_g(out: &mut[u8], input: &[u8], inlen: usize) }
{
sha3_512(out, input, inlen); #[cfg(feature = "90s")]
} pub(crate) fn hash_g(out: &mut [u8], input: &[u8], inlen: usize) {
let mut hasher = Sha512::new();
#[cfg(feature = "90s")] hasher.update(&input[..inlen]);
pub(crate) fn hash_g(out: &mut[u8], input: &[u8], inlen: usize) let digest = hasher.finalize();
{ out[..digest.len()].copy_from_slice(&digest);
let mut hasher = Sha512::new(); }
hasher.update(&input[..inlen]);
let digest = hasher.finalize(); #[cfg(not(feature = "90s"))]
out[..digest.len()].copy_from_slice(&digest); pub(crate) fn xof_absorb(state: &mut XofState, input: &[u8], x: u8, y: u8) {
} kyber_shake128_absorb(state, &input, x, y);
}
#[cfg(not(feature = "90s"))]
pub(crate) fn xof_absorb(state: &mut XofState, input: &[u8], x: u8, y: u8) #[cfg(feature = "90s")]
{ pub(crate) fn xof_absorb(state: &mut XofState, input: &[u8], x: u8, y: u8) {
kyber_shake128_absorb(state, &input, x, y); let mut nonce = [0u8; 12];
} nonce[0] = x;
nonce[1] = y;
#[cfg(feature = "90s")] aes256ctr_init(state, &input, nonce);
pub(crate) fn xof_absorb(state: &mut XofState, input: &[u8], x: u8, y: u8) }
{
let mut nonce = [0u8; 12]; #[cfg(not(feature = "90s"))]
nonce[0] = x; pub(crate) fn xof_squeezeblocks(out: &mut [u8], outblocks: usize, state: &mut XofState) {
nonce[1] = y; kyber_shake128_squeezeblocks(out, outblocks, state);
aes256ctr_init(state, &input, nonce); }
}
#[cfg(feature = "90s")]
#[cfg(not(feature = "90s"))] pub(crate) fn xof_squeezeblocks(out: &mut [u8], outblocks: usize, state: &mut XofState) {
pub(crate) fn xof_squeezeblocks(out: &mut[u8], outblocks: usize, state: &mut XofState) aes256ctr_squeezeblocks(out, outblocks, state);
{ }
kyber_shake128_squeezeblocks(out, outblocks, state);
} #[cfg(not(feature = "90s"))]
pub(crate) fn prf(out: &mut [u8], outbytes: usize, key: &[u8], nonce: u8) {
#[cfg(feature = "90s")] shake256_prf(out, outbytes, &key, nonce);
pub(crate) fn xof_squeezeblocks(out: &mut[u8], outblocks: usize, state: &mut XofState) }
{
aes256ctr_squeezeblocks(out, outblocks, state); #[cfg(feature = "90s")]
} pub(crate) fn prf(out: &mut [u8], outbytes: usize, key: &[u8], nonce: u8) {
aes256ctr_prf(out, outbytes, &key, nonce);
#[cfg(not(feature = "90s"))]
pub(crate) fn prf(out: &mut[u8], outbytes: usize, key: &[u8], nonce: u8) // TODO: Add feature to use RustCrypto AES_CTR
{ // implementation with no lookup tables
shake256_prf(out, outbytes, &key, nonce); // Perhaps add an option for ring also.
}
// Working RustCrypto code:
#[cfg(feature = "90s")] // if cfg!(feature = "rustcrypto-aes") {
pub(crate) fn prf(out: &mut[u8], outbytes: usize, key: &[u8], nonce: u8) // let mut expnonce = [0u8; 16];
{ // expnonce[0] = nonce;
aes256ctr_prf(out, outbytes, &key, nonce); // let key = GenericArray::from_slice(key);
// let iv = GenericArray::from_slice(&expnonce);
// TODO: Add feature to use RustCrypto AES_CTR // let mut cipher = Aes256Ctr::new(&key, &iv);
// implementation with no lookup tables // cipher.apply_keystream(out);
// Perhaps add an option for ring also. // }
}
// Working RustCrypto code:
// if cfg!(feature = "rustcrypto-aes") { #[cfg(not(feature = "90s"))]
// let mut expnonce = [0u8; 16]; pub(crate) fn kdf(out: &mut [u8], input: &[u8], inlen: usize) {
// expnonce[0] = nonce; shake256(out, KYBER_SSBYTES, input, inlen);
// let key = GenericArray::from_slice(key); }
// let iv = GenericArray::from_slice(&expnonce);
// let mut cipher = Aes256Ctr::new(&key, &iv); #[cfg(feature = "90s")]
// cipher.apply_keystream(out); pub(crate) fn kdf(out: &mut [u8], input: &[u8], inlen: usize) {
// } let mut hasher = Sha256::new();
} hasher.update(&input[..inlen]);
let digest = hasher.finalize();
#[cfg(not(feature = "90s"))] out[..digest.len()].copy_from_slice(&digest);
pub(crate) fn kdf(out: &mut[u8], input: &[u8], inlen: usize) }
{
shake256(out, KYBER_SSBYTES, input, inlen); // Name: kyber_shake128_absorb
} //
// Description: Absorb step of the SHAKE128 specialized for the Kyber context.
#[cfg(feature = "90s")] //
pub(crate) fn kdf(out: &mut[u8], input: &[u8], inlen: usize) // Arguments: - u64 *s: (uninitialized) output Keccak state
{ // - const [u8] input: KYBER_SYMBYTES input to be absorbed into s
let mut hasher = Sha256::new(); // - u8 x additional byte of input
hasher.update(&input[..inlen]); // - u8 y additional byte of input
let digest = hasher.finalize(); #[cfg(not(feature = "90s"))]
out[..digest.len()].copy_from_slice(&digest); fn kyber_shake128_absorb(s: &mut KeccakState, input: &[u8], x: u8, y: u8) {
} let mut extseed = [0u8; KYBER_SYMBYTES + 2];
extseed[..KYBER_SYMBYTES].copy_from_slice(input);
// Name: kyber_shake128_absorb extseed[KYBER_SYMBYTES] = x;
// extseed[KYBER_SYMBYTES + 1] = y;
// Description: Absorb step of the SHAKE128 specialized for the Kyber context. shake128_absorb_once(s, &extseed, KYBER_SYMBYTES + 2);
// }
// Arguments: - u64 *s: (uninitialized) output Keccak state
// - const [u8] input: KYBER_SYMBYTES input to be absorbed into s // Name: kyber_shake128_squeezeblocks
// - u8 x additional byte of input //
// - u8 y additional byte of input // Description: Squeeze step of SHAKE128 XOF. Squeezes full blocks of SHAKE128_RATE bytes each.
#[cfg(not(feature = "90s"))] // Modifies the state. Can be called multiple times to keep squeezing,
fn kyber_shake128_absorb( // i.e., is incremental.
s: &mut KeccakState, //
input: &[u8], // Arguments: - [u8] output: output blocks
x: u8, // - u64 nblocks: number of blocks to be squeezed (written to output)
y: u8 // - keccak_state *s: in/output Keccak state
) #[cfg(not(feature = "90s"))]
{ fn kyber_shake128_squeezeblocks(output: &mut [u8], nblocks: usize, s: &mut KeccakState) {
let mut extseed = [0u8; KYBER_SYMBYTES + 2]; shake128_squeezeblocks(output, nblocks, s);
extseed[..KYBER_SYMBYTES].copy_from_slice(input); }
extseed[KYBER_SYMBYTES] = x;
extseed[KYBER_SYMBYTES+1] = y; // Name: shake256_prf
shake128_absorb_once(s, &extseed, KYBER_SYMBYTES + 2); //
} // Description: Usage of SHAKE256 as a PRF, concatenates secret and public input
// and then generates outlen bytes of SHAKE256 output
// Name: kyber_shake128_squeezeblocks //
// // Arguments: - [u8] output: output
// Description: Squeeze step of SHAKE128 XOF. Squeezes full blocks of SHAKE128_RATE bytes each. // - u64 outlen: number of requested output bytes
// Modifies the state. Can be called multiple times to keep squeezing, // - const [u8] key: the key (of length KYBER_SYMBYTES)
// i.e., is incremental. // - const [u8] nonce: single-byte nonce (public PRF input)
// #[cfg(not(feature = "90s"))]
// Arguments: - [u8] output: output blocks fn shake256_prf(output: &mut [u8], outlen: usize, key: &[u8], nonce: u8) {
// - u64 nblocks: number of blocks to be squeezed (written to output) let mut extkey = [0u8; KYBER_SYMBYTES + 1];
// - keccak_state *s: in/output Keccak state extkey[..KYBER_SYMBYTES].copy_from_slice(key);
#[cfg(not(feature = "90s"))] extkey[KYBER_SYMBYTES] = nonce;
fn kyber_shake128_squeezeblocks( shake256(output, outlen, &extkey, KYBER_SYMBYTES + 1);
output: &mut[u8], }
nblocks: usize,
s: &mut KeccakState
)
{
shake128_squeezeblocks(output, nblocks, s);
}
// Name: shake256_prf
//
// Description: Usage of SHAKE256 as a PRF, concatenates secret and public input
// and then generates outlen bytes of SHAKE256 output
//
// Arguments: - [u8] output: output
// - u64 outlen: number of requested output bytes
// - const [u8] key: the key (of length KYBER_SYMBYTES)
// - const [u8] nonce: single-byte nonce (public PRF input)
#[cfg(not(feature = "90s"))]
fn shake256_prf(output: &mut[u8], outlen: usize, key: &[u8], nonce: u8)
{
let mut extkey = [0u8; KYBER_SYMBYTES+1];
extkey[..KYBER_SYMBYTES].copy_from_slice(key);
extkey[KYBER_SYMBYTES] = nonce;
shake256(output, outlen, &extkey, KYBER_SYMBYTES + 1);
}

122
third_party/kyber/src/wasm.rs vendored
View file

@ -3,43 +3,43 @@ use wasm_bindgen::prelude::*;
#[wasm_bindgen] #[wasm_bindgen]
pub fn keypair() -> Keys { pub fn keypair() -> Keys {
let mut pk = [0u8; KYBER_PUBLICKEYBYTES]; let mut pk = [0u8; KYBER_PUBLICKEYBYTES];
let mut sk = [0u8; KYBER_SECRETKEYBYTES]; let mut sk = [0u8; KYBER_SECRETKEYBYTES];
kem::crypto_kem_keypair(&mut pk, &mut sk, None); kem::crypto_kem_keypair(&mut pk, &mut sk, None);
Keys{ Keys {
pubkey: Box::new(pk), pubkey: Box::new(pk),
secret: Box::new(sk), secret: Box::new(sk),
..Default::default() ..Default::default()
} }
} }
#[wasm_bindgen] #[wasm_bindgen]
pub fn encapsulate(pk: Box<[u8]>) -> Result<Keys, JsValue> { pub fn encapsulate(pk: Box<[u8]>) -> Result<Keys, JsValue> {
if pk.len() != KYBER_PUBLICKEYBYTES { if pk.len() != KYBER_PUBLICKEYBYTES {
return Err(JsValue::null()) return Err(JsValue::null());
} }
let mut ct = [0u8; KYBER_CIPHERTEXTBYTES]; let mut ct = [0u8; KYBER_CIPHERTEXTBYTES];
let mut ss = [0u8; KYBER_SSBYTES]; let mut ss = [0u8; KYBER_SSBYTES];
kem::crypto_kem_enc(&mut ct, &mut ss, &pk, None); kem::crypto_kem_enc(&mut ct, &mut ss, &pk, None);
Ok(Keys{ Ok(Keys {
ciphertext: Box::new(ct), ciphertext: Box::new(ct),
shared_secret: Box::new(ss), shared_secret: Box::new(ss),
..Default::default() ..Default::default()
}) })
} }
#[wasm_bindgen] #[wasm_bindgen]
pub fn decapsulate(ct: Box<[u8]>, sk: Box<[u8]>) -> Result<Keys, JsValue> { pub fn decapsulate(ct: Box<[u8]>, sk: Box<[u8]>) -> Result<Keys, JsValue> {
let mut ss = [0u8; KYBER_SSBYTES]; let mut ss = [0u8; KYBER_SSBYTES];
match kem::crypto_kem_dec(&mut ss, &ct, &sk) { match kem::crypto_kem_dec(&mut ss, &ct, &sk) {
Ok(_) => Ok(Keys {shared_secret: Box::new(ss), ..Default::default()}), Ok(_) => Ok(Keys { shared_secret: Box::new(ss), ..Default::default() }),
Err(_) => Err(JsValue::null()) Err(_) => Err(JsValue::null()),
} }
} }
#[wasm_bindgen] #[wasm_bindgen]
#[derive(Default, Clone, Debug)] #[derive(Default, Clone, Debug)]
pub struct Keys{ pub struct Keys {
pubkey: Box<[u8]>, pubkey: Box<[u8]>,
secret: Box<[u8]>, secret: Box<[u8]>,
ciphertext: Box<[u8]>, ciphertext: Box<[u8]>,
@ -48,48 +48,48 @@ pub struct Keys{
#[wasm_bindgen] #[wasm_bindgen]
impl Keys { impl Keys {
#[wasm_bindgen(constructor)] #[wasm_bindgen(constructor)]
pub fn new() -> Self { pub fn new() -> Self {
Keys::default() Keys::default()
} }
#[wasm_bindgen(getter)] #[wasm_bindgen(getter)]
pub fn pubkey(&self) -> Box<[u8]> { pub fn pubkey(&self) -> Box<[u8]> {
self.pubkey.clone() self.pubkey.clone()
} }
#[wasm_bindgen(getter)] #[wasm_bindgen(getter)]
pub fn secret(&self) -> Box<[u8]> { pub fn secret(&self) -> Box<[u8]> {
self.secret.clone() self.secret.clone()
} }
#[wasm_bindgen(getter)] #[wasm_bindgen(getter)]
pub fn ciphertext(&self) -> Box<[u8]> { pub fn ciphertext(&self) -> Box<[u8]> {
self.ciphertext.clone() self.ciphertext.clone()
} }
#[wasm_bindgen(getter)] #[wasm_bindgen(getter)]
pub fn shared_secret(&self) -> Box<[u8]> { pub fn shared_secret(&self) -> Box<[u8]> {
self.shared_secret.clone() self.shared_secret.clone()
} }
#[wasm_bindgen(setter)] #[wasm_bindgen(setter)]
pub fn set_pubkey(&mut self, pubkey: Box<[u8]>) { pub fn set_pubkey(&mut self, pubkey: Box<[u8]>) {
self.pubkey = pubkey; self.pubkey = pubkey;
} }
#[wasm_bindgen(setter)] #[wasm_bindgen(setter)]
pub fn set_secret(&mut self, secret: Box<[u8]>) { pub fn set_secret(&mut self, secret: Box<[u8]>) {
self.secret = secret; self.secret = secret;
} }
#[wasm_bindgen(setter)] #[wasm_bindgen(setter)]
pub fn set_ciphertext(&mut self, ciphertext: Box<[u8]>) { pub fn set_ciphertext(&mut self, ciphertext: Box<[u8]>) {
self.ciphertext = ciphertext; self.ciphertext = ciphertext;
} }
#[wasm_bindgen(setter)] #[wasm_bindgen(setter)]
pub fn set_shared_secret(&mut self, shared_secret: Box<[u8]>) { pub fn set_shared_secret(&mut self, shared_secret: Box<[u8]>) {
self.shared_secret = shared_secret; self.shared_secret = shared_secret;
} }
} }

91
third_party/kyber/tests/kat.rs vendored
View file

@ -1,75 +1,74 @@
#![cfg(feature="KATs")] #![cfg(feature = "KATs")]
mod load; mod load;
use pqc_kyber::*;
use load::*; use load::*;
use pqc_kyber::*;
// Generate KAT keypairs from seeds. // Generate KAT keypairs from seeds.
#[test] #[test]
fn keypairs() { fn keypairs() {
let kats = build_kats(); let kats = build_kats();
let mut _rng = rand::thread_rng(); // placeholder let mut _rng = rand::thread_rng(); // placeholder
for kat in kats { for kat in kats {
let known_pk = decode_hex(&kat.pk); let known_pk = decode_hex(&kat.pk);
let known_sk = decode_hex(&kat.sk); let known_sk = decode_hex(&kat.sk);
let buf1 = decode_hex(&kat.keygen_buffer1); let buf1 = decode_hex(&kat.keygen_buffer1);
let buf2 = decode_hex(&kat.keygen_buffer2); let buf2 = decode_hex(&kat.keygen_buffer2);
let bufs = Some((&buf1[..], &buf2[..])); let bufs = Some((&buf1[..], &buf2[..]));
let mut pk = [0u8; KYBER_PUBLICKEYBYTES]; let mut pk = [0u8; KYBER_PUBLICKEYBYTES];
let mut sk = [0u8; KYBER_SECRETKEYBYTES]; let mut sk = [0u8; KYBER_SECRETKEYBYTES];
crypto_kem_keypair(&mut pk, &mut sk, &mut _rng, bufs); crypto_kem_keypair(&mut pk, &mut sk, &mut _rng, bufs);
assert_eq!(&pk[..], &known_pk[..], "Public key KAT mismatch"); assert_eq!(&pk[..], &known_pk[..], "Public key KAT mismatch");
assert_eq!(&sk[..], &known_sk[..], "Secret key KAT mismatch"); assert_eq!(&sk[..], &known_sk[..], "Secret key KAT mismatch");
} }
} }
// Encapsulating KAT's using deterministic rand buffers // Encapsulating KAT's using deterministic rand buffers
#[test] #[test]
fn encaps() { fn encaps() {
let kats = build_kats(); let kats = build_kats();
let mut _rng = rand::thread_rng(); // placeholder let mut _rng = rand::thread_rng(); // placeholder
for kat in kats { for kat in kats {
let known_ss = decode_hex(&kat.ss); let known_ss = decode_hex(&kat.ss);
let pk = decode_hex(&kat.pk); let pk = decode_hex(&kat.pk);
let buf1 = decode_hex(&kat.encap_buffer); let buf1 = decode_hex(&kat.encap_buffer);
let encap_buf = Some(&buf1[..]); let encap_buf = Some(&buf1[..]);
let mut ct = [0u8; KYBER_CIPHERTEXTBYTES]; let mut ct = [0u8; KYBER_CIPHERTEXTBYTES];
let mut ss = [0u8; KYBER_SSBYTES]; let mut ss = [0u8; KYBER_SSBYTES];
crypto_kem_enc(&mut ct, &mut ss, &pk, &mut _rng, encap_buf); crypto_kem_enc(&mut ct, &mut ss, &pk, &mut _rng, encap_buf);
assert_eq!(&ss[..], &known_ss[..], "Shared secret KAT mismatch"); assert_eq!(&ss[..], &known_ss[..], "Shared secret KAT mismatch");
} }
} }
// Decapsulating KAT's // Decapsulating KAT's
#[test] #[test]
fn decaps() { fn decaps() {
let kats = build_kats(); let kats = build_kats();
for kat in kats { for kat in kats {
let sk = decode_hex(&kat.sk); let sk = decode_hex(&kat.sk);
let ct = decode_hex(&kat.ct); let ct = decode_hex(&kat.ct);
let known_ss = decode_hex(&kat.ss); let known_ss = decode_hex(&kat.ss);
let decap_result = decapsulate(&ct, &sk); let decap_result = decapsulate(&ct, &sk);
assert!(decap_result.is_ok(), "KEM decapsulation failure"); assert!(decap_result.is_ok(), "KEM decapsulation failure");
assert_eq!(&decap_result.unwrap()[..], &known_ss[..], "Shared secret KAT doesn't match") assert_eq!(&decap_result.unwrap()[..], &known_ss[..], "Shared secret KAT doesn't match")
} }
} }
// Helper functions // Helper functions
// Encodes byte slice into a hex string // Encodes byte slice into a hex string
pub fn encode_hex(bytes: &[u8]) -> String { pub fn encode_hex(bytes: &[u8]) -> String {
let mut output = String::new(); let mut output = String::new();
for b in bytes { for b in bytes {
output.push_str(&format!("{:02X}", b)); output.push_str(&format!("{:02X}", b));
} }
output output
} }
// Decodes hex string into a vector of bytes // Decodes hex string into a vector of bytes
pub fn decode_hex(s: &str) -> Vec<u8> { pub fn decode_hex(s: &str) -> Vec<u8> {
(0..s.len()) (0..s.len())
.step_by(2) .step_by(2)
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("Hex string decoding")) .map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("Hex string decoding"))
.collect::<Vec<u8>>() .collect::<Vec<u8>>()
} }

13
third_party/kyber/tests/kem.rs vendored
View file

@ -1,12 +1,11 @@
#![cfg(not(feature="KATs"))] #![cfg(not(feature = "KATs"))]
use pqc_kyber::*; use pqc_kyber::*;
#[test] #[test]
fn keypair_encap_decap_kem() { fn keypair_encap_decap_kem() {
let mut rng = rand::thread_rng(); let mut rng = rand::thread_rng();
let keys = keypair(&mut rng); let keys = keypair(&mut rng);
let (ct, ss1) = encapsulate(&keys.public, &mut rng).unwrap(); let (ct, ss1) = encapsulate(&keys.public, &mut rng).unwrap();
let ss2 = decapsulate(&ct, &keys.secret).unwrap(); let ss2 = decapsulate(&ct, &keys.secret).unwrap();
assert_eq!(ss1, ss2); assert_eq!(ss1, ss2);
} }

38
third_party/kyber/tests/kex.rs vendored
View file

@ -1,28 +1,30 @@
#![cfg(not(feature="KATs"))] #![cfg(not(feature = "KATs"))]
use pqc_kyber::*; use pqc_kyber::*;
// Kyber struct uake and ake functions // Kyber struct uake and ake functions
#[test] #[test]
fn kyber_uake() { fn kyber_uake() {
let mut rng = rand::thread_rng(); let mut rng = rand::thread_rng();
let mut alice = Uake::new(); let mut alice = Uake::new();
let mut bob = Uake::new(); let mut bob = Uake::new();
let bob_keys = keypair(&mut rng); let bob_keys = keypair(&mut rng);
let client_init = alice.client_init(&bob_keys.public, &mut rng); let client_init = alice.client_init(&bob_keys.public, &mut rng);
let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng).unwrap(); let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng).unwrap();
let _client_confirm = alice.client_confirm(server_send).unwrap(); let _client_confirm = alice.client_confirm(server_send).unwrap();
assert_eq!(alice.shared_secret, bob.shared_secret); assert_eq!(alice.shared_secret, bob.shared_secret);
} }
#[test] #[test]
fn kyber_ake() { fn kyber_ake() {
let mut rng = rand::thread_rng(); let mut rng = rand::thread_rng();
let mut alice = Ake::new(); let mut alice = Ake::new();
let mut bob = Ake::new(); let mut bob = Ake::new();
let alice_keys = keypair(&mut rng); let alice_keys = keypair(&mut rng);
let bob_keys = keypair(&mut rng); let bob_keys = keypair(&mut rng);
let client_init = alice.client_init(&bob_keys.public, &mut rng); let client_init = alice.client_init(&bob_keys.public, &mut rng);
let server_send = bob.server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng).unwrap(); let server_send = bob
let _client_confirm = alice.client_confirm(server_send, &alice_keys.secret).unwrap(); .server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng)
assert_eq!(alice.shared_secret, bob.shared_secret); .unwrap();
let _client_confirm = alice.client_confirm(server_send, &alice_keys.secret).unwrap();
assert_eq!(alice.shared_secret, bob.shared_secret);
} }

128
third_party/kyber/tests/load/mod.rs vendored
View file

@ -1,100 +1,94 @@
use pqc_kyber::{KYBER_90S, KYBER_K};
use std::fs::File; use std::fs::File;
use std::path::PathBuf;
use std::io::{prelude::*, BufReader}; use std::io::{prelude::*, BufReader};
use pqc_kyber::{KYBER_K, KYBER_90S}; use std::path::PathBuf;
// Known Answer Tests // Known Answer Tests
#[derive(Debug)] #[derive(Debug)]
pub struct Kat { pub struct Kat {
// Deterministic RNG buffers // Deterministic RNG buffers
pub keygen_buffer1: String, pub keygen_buffer1: String,
pub keygen_buffer2: String, pub keygen_buffer2: String,
pub encap_buffer: String, pub encap_buffer: String,
// Keys, Ciphertext and Shared Secret // Keys, Ciphertext and Shared Secret
pub pk: String, pub pk: String,
pub sk: String, pub sk: String,
pub ct: String, pub ct: String,
pub ss: String pub ss: String,
} }
// Converts string octuples from tvec files into Kat structs // Converts string octuples from tvec files into Kat structs
impl From<&[String]> for Kat { impl From<&[String]> for Kat {
fn from(kat: &[String]) -> Self { fn from(kat: &[String]) -> Self {
// Extract values from key:value lines // Extract values from key:value lines
let values: Vec<String> = kat.iter() let values: Vec<String> = kat
.map( .iter()
|katline| { .map(|katline| {
let val: Vec<&str> = katline.split(": ").collect(); let val: Vec<&str> = katline.split(": ").collect();
if val.len() > 1 { if val.len() > 1 {
val[1].into() val[1].into()
} else { } else {
val[0].into() val[0].into()
} }
})
.collect();
// Build KAT from values
Kat {
keygen_buffer1: values[0].clone(),
keygen_buffer2: values[1].clone(),
pk: values[2].clone(),
sk: values[3].clone(),
encap_buffer: values[4].clone(),
ct: values[5].clone(),
ss: values[6].clone(),
} }
)
.collect();
// Build KAT from values
Kat {
keygen_buffer1: values[0].clone(),
keygen_buffer2: values[1].clone(),
pk: values[2].clone(),
sk: values[3].clone(),
encap_buffer: values[4].clone(),
ct: values[5].clone(),
ss: values[6].clone(),
} }
}
} }
// Get KAT filename based on security level and if 90s mode // Get KAT filename based on security level and if 90s mode
fn get_filename() -> String { fn get_filename() -> String {
let mut filename = match KYBER_K { let mut filename = match KYBER_K {
2 => "tvecs512".to_string(), 2 => "tvecs512".to_string(),
3 => "tvecs768".to_string(), 3 => "tvecs768".to_string(),
4 => "tvecs1024".to_string(), 4 => "tvecs1024".to_string(),
_ => panic!("No security level set") _ => panic!("No security level set"),
}; };
if KYBER_90S { if KYBER_90S {
filename.push_str("-90s"); filename.push_str("-90s");
} }
println!("Using KAT file: {}", &filename); println!("Using KAT file: {}", &filename);
filename filename
} }
// Base dir // Base dir
fn get_test_dir() -> PathBuf { fn get_test_dir() -> PathBuf {
let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
path.extend(&["tests"]); path.extend(&["tests"]);
path path
} }
// KATs path // KATs path
fn get_kat_filepath() -> PathBuf { fn get_kat_filepath() -> PathBuf {
let mut path = get_test_dir(); let mut path = get_test_dir();
path.extend(&["KATs"]); path.extend(&["KATs"]);
path.extend(&[get_filename()]); path.extend(&[get_filename()]);
path path
} }
fn load_file(filepath: PathBuf) -> File { fn load_file(filepath: PathBuf) -> File {
File::open(filepath).expect("Error loading KAT file") File::open(filepath).expect("Error loading KAT file")
} }
fn parse_kats() -> Vec<String> { fn parse_kats() -> Vec<String> {
let file = load_file(get_kat_filepath()); let file = load_file(get_kat_filepath());
let buf = BufReader::new(file); let buf = BufReader::new(file);
buf.lines() buf.lines().map(|l| l.expect("Unable to parse line")).collect()
.map(|l| l.expect("Unable to parse line"))
.collect()
} }
// Packs chunks of lines into Kat structs // Packs chunks of lines into Kat structs
pub fn build_kats() -> Vec<Kat> { pub fn build_kats() -> Vec<Kat> {
let lines = parse_kats(); let lines = parse_kats();
let kats = lines.chunks_exact(8); let kats = lines.chunks_exact(8);
// Map String slices into Vec<KAT> // Map String slices into Vec<KAT>
kats.map( kats.map(|c| c.into()).collect::<Vec<Kat>>()
|c| {c.into()} }
)
.collect::<Vec<Kat>>()
}

6
utils/src/gatherarray.rs
View file

@ -47,7 +47,11 @@ impl<T, const C: usize> GatherArray<T, C> {
if (self.have_count as usize) == goal { if (self.have_count as usize) == goal {
debug_assert_eq!(0xffffffffffffffffu64.wrapping_shr(64 - goal as u32), have); debug_assert_eq!(0xffffffffffffffffu64.wrapping_shr(64 - goal as u32), have);
let mut tmp = ArrayVec::new(); let mut tmp = ArrayVec::new();
copy_nonoverlapping(self.a.as_ptr().cast::<u8>(), tmp.a.as_mut_ptr().cast::<u8>(), size_of::<MaybeUninit<T>>() * goal); copy_nonoverlapping(
self.a.as_ptr().cast::<u8>(),
tmp.a.as_mut_ptr().cast::<u8>(),
size_of::<MaybeUninit<T>>() * goal,
);
tmp.s = goal; tmp.s = goal;
self.goal = 0; self.goal = 0;
return Some(tmp); return Some(tmp);

4
utils/src/hex.rs
View file

@ -1,6 +1,8 @@
// (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md. // (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md.
pub const HEX_CHARS: [u8; 16] = [b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f']; pub const HEX_CHARS: [u8; 16] = [
b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f',
];
/// Encode a byte slice to a hexadecimal string. /// Encode a byte slice to a hexadecimal string.
pub fn to_string(b: &[u8]) -> String { pub fn to_string(b: &[u8]) -> String {

5
utils/src/pool.rs
View file

@ -43,7 +43,10 @@ impl<O: Send, F: PoolFactory<O>> Pooled<O, F> {
#[inline(always)] #[inline(always)]
pub unsafe fn into_raw(self) -> *mut O { pub unsafe fn into_raw(self) -> *mut O {
// Verify that the structure is not padded before 'obj'. // Verify that the structure is not padded before 'obj'.
assert_eq!((&self.0.as_ref().obj as *const O).cast::<u8>(), (self.0.as_ref() as *const PoolEntry<O, F>).cast::<u8>()); assert_eq!(
(&self.0.as_ref().obj as *const O).cast::<u8>(),
(self.0.as_ref() as *const PoolEntry<O, F>).cast::<u8>()
);
let ptr = self.0.as_ptr().cast::<O>(); let ptr = self.0.as_ptr().cast::<O>();
std::mem::forget(self); std::mem::forget(self);