void/ZeroTierOne
2
0
Fork 0
mirror of https://github.com/zerotier/ZeroTierOne.git synced 2025-06-06 12:33:44 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

rustfmt

Browse source
This commit is contained in:
Adam Ierymenko 2022-09-13 10:48:36 -04:00
parent 4763404d98
commit 07fc8b2d2b
No known key found for this signature in database
GPG key ID: C8877CF2D7A5D7F3
87 changed files with 8325 additions and 6871 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

153
attic/fastudpsocket.rs
View file

@ -44,25 +44,55 @@ fn bind_udp_socket(_device_name: &str, address: &InetAddress) -> Result<FastUDPR
let mut setsockopt_results: c_int = 0;
fl = 1;
setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_REUSEPORT.as_(), (&mut fl as *mut c_int).cast(), fl_size);
setsockopt_results |= libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_REUSEPORT.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
);
fl = 0;
setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_LINGER.as_(), (&mut fl as *mut c_int).cast(), fl_size);
setsockopt_results |= libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_LINGER.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
);
//fl = 1;
//setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET, libc::SO_REUSEADDR, (&mut fl as *mut c_int).cast(), fl_size);
fl = 1;
setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_BROADCAST.as_(), (&mut fl as *mut c_int).cast(), fl_size);
setsockopt_results |= libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_BROADCAST.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
);
if af == libc::AF_INET6 {
fl = 1;
setsockopt_results |= libc::setsockopt(s, libc::IPPROTO_IPV6.as_(), libc::IPV6_V6ONLY.as_(), (&mut fl as *mut c_int).cast(), fl_size);
setsockopt_results |= libc::setsockopt(
s,
libc::IPPROTO_IPV6.as_(),
libc::IPV6_V6ONLY.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
);
}
#[cfg(any(target_os = "macos", target_os = "ios"))]
{
fl = 1;
setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_NOSIGPIPE.as_(), (&mut fl as *mut c_int).cast(), fl_size)
setsockopt_results |= libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_NOSIGPIPE.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
)
}
#[cfg(target_os = "linux")]
@ -70,7 +100,13 @@ fn bind_udp_socket(_device_name: &str, address: &InetAddress) -> Result<FastUDPR
if !_device_name.is_empty() {
let _ = std::ffi::CString::new(_device_name).map(|dn| {
let dnb = dn.as_bytes_with_nul();
let _ = libc::setsockopt(s.as_(), libc::SOL_SOCKET.as_(), libc::SO_BINDTODEVICE.as_(), dnb.as_ptr().cast(), (dnb.len() - 1).as_());
let _ = libc::setsockopt(
s.as_(),
libc::SOL_SOCKET.as_(),
libc::SO_BINDTODEVICE.as_(),
dnb.as_ptr().cast(),
(dnb.len() - 1).as_(),
);
});
}
}
@ -84,30 +120,62 @@ fn bind_udp_socket(_device_name: &str, address: &InetAddress) -> Result<FastUDPR
#[cfg(not(target_os = "linux"))]
{
fl = 0;
libc::setsockopt(s, libc::IPPROTO_IP.as_(), libc::IP_DF.as_(), (&mut fl as *mut c_int).cast(), fl_size);
libc::setsockopt(
s,
libc::IPPROTO_IP.as_(),
libc::IP_DF.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
);
}
#[cfg(target_os = "linux")]
{
fl = libc::IP_PMTUDISC_DONT as c_int;
libc::setsockopt(s, libc::IPPROTO_IP.as_(), libc::IP_MTU_DISCOVER.as_(), (&mut fl as *mut c_int).cast(), fl_size);
libc::setsockopt(
s,
libc::IPPROTO_IP.as_(),
libc::IP_MTU_DISCOVER.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
);
}
}
if af == libc::AF_INET6 {
fl = 0;
libc::setsockopt(s, libc::IPPROTO_IPV6.as_(), libc::IPV6_DONTFRAG.as_(), (&mut fl as *mut c_int).cast(), fl_size);
libc::setsockopt(
s,
libc::IPPROTO_IPV6.as_(),
libc::IPV6_DONTFRAG.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
);
}
fl = 1048576;
while fl >= 131072 {
if libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_RCVBUF.as_(), (&mut fl as *mut c_int).cast(), fl_size) == 0 {
if libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_RCVBUF.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
) == 0
{
break;
}
fl -= 65536;
}
fl = 1048576;
while fl >= 131072 {
if libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_SNDBUF.as_(), (&mut fl as *mut c_int).cast(), fl_size) == 0 {
if libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_SNDBUF.as_(),
(&mut fl as *mut c_int).cast(),
fl_size,
) == 0
{
break;
}
fl -= 65536;
@ -172,10 +240,22 @@ pub(crate) fn fast_udp_socket_sendto(socket: &FastUDPRawOsSocket, to_address: &I
let _ = libc::sendmsg(socket.get().as_(), transmute(&mhdr as *const libc::msghdr), 0);
} else {
let mut ttl = packet_ttl as c_int;
libc::setsockopt(socket.get().as_(), libc::IPPROTO_IP.as_(), libc::IP_TTL.as_(), (&mut ttl as *mut c_int).cast(), std::mem::size_of::<c_int>().as_());
libc::setsockopt(
socket.get().as_(),
libc::IPPROTO_IP.as_(),
libc::IP_TTL.as_(),
(&mut ttl as *mut c_int).cast(),
std::mem::size_of::<c_int>().as_(),
);
let _ = libc::sendmsg(socket.get().as_(), transmute(&mhdr as *const libc::msghdr), 0);
ttl = 255;
libc::setsockopt(socket.get().as_(), libc::IPPROTO_IP.as_(), libc::IP_TTL.as_(), (&mut ttl as *mut c_int).cast(), std::mem::size_of::<c_int>().as_());
libc::setsockopt(
socket.get().as_(),
libc::IPPROTO_IP.as_(),
libc::IP_TTL.as_(),
(&mut ttl as *mut c_int).cast(),
std::mem::size_of::<c_int>().as_(),
);
}
}
}
@ -185,7 +265,14 @@ pub(crate) fn fast_udp_socket_sendto(socket: &FastUDPRawOsSocket, to_address: &I
fn fast_udp_socket_recvfrom(socket: &FastUDPRawOsSocket, buf: &mut PacketBuffer, from_address: &mut InetAddress) -> isize {
unsafe {
let mut addrlen = std::mem::size_of::<InetAddress>() as libc::socklen_t;
let s = libc::recvfrom(socket.get().as_(), buf.as_mut_ptr().cast(), buf.capacity().as_(), 0, (from_address as *mut InetAddress).cast(), &mut addrlen) as isize;
let s = libc::recvfrom(
socket.get().as_(),
buf.as_mut_ptr().cast(),
buf.capacity().as_(),
0,
(from_address as *mut InetAddress).cast(),
&mut addrlen,
) as isize;
if s > 0 {
buf.set_size_unchecked(s as usize);
}
@ -194,7 +281,12 @@ fn fast_udp_socket_recvfrom(socket: &FastUDPRawOsSocket, buf: &mut PacketBuffer,
}
impl FastUDPSocket {
pub fn new<F: Fn(&FastUDPRawOsSocket, &InetAddress, PacketBuffer) + Send + Sync + Clone + 'static>(device_name: &str, address: &InetAddress, packet_buffer_pool: &Arc<PacketBufferPool>, handler: F) -> Result<Self, String> {
pub fn new<F: Fn(&FastUDPRawOsSocket, &InetAddress, PacketBuffer) + Send + Sync + Clone + 'static>(
device_name: &str,
address: &InetAddress,
packet_buffer_pool: &Arc<PacketBufferPool>,
handler: F,
) -> Result<Self, String> {
let thread_count = num_cpus::get_physical().clamp(1, FAST_UDP_SOCKET_MAX_THREADS);
let mut s = Self {
@ -273,7 +365,14 @@ impl Drop for FastUDPSocket {
self.thread_run.store(false, Ordering::Relaxed);
for s in self.sockets.iter() {
unsafe {
libc::sendto(s.get().as_(), tmp.as_ptr().cast(), 0, 0, (&self.bind_address as *const InetAddress).cast(), std::mem::size_of::<InetAddress>() as libc::socklen_t);
libc::sendto(
s.get().as_(),
tmp.as_ptr().cast(),
0,
0,
(&self.bind_address as *const InetAddress).cast(),
std::mem::size_of::<InetAddress>() as libc::socklen_t,
);
}
}
for s in self.sockets.iter() {
@ -309,9 +408,14 @@ mod tests {
let ba0 = ba0.unwrap();
let cnt0 = Arc::new(AtomicU32::new(0));
let cnt0c = cnt0.clone();
let s0 = FastUDPSocket::new("", &ba0, &pool, move |sock: &FastUDPRawOsSocket, _: &InetAddress, data: PacketBuffer| {
cnt0c.fetch_add(1, Ordering::Relaxed);
});
let s0 = FastUDPSocket::new(
"",
&ba0,
&pool,
move |sock: &FastUDPRawOsSocket, _: &InetAddress, data: PacketBuffer| {
cnt0c.fetch_add(1, Ordering::Relaxed);
},
);
assert!(s0.is_ok());
let s0 = s0.unwrap();
@ -320,9 +424,14 @@ mod tests {
let ba1 = ba1.unwrap();
let cnt1 = Arc::new(AtomicU32::new(0));
let cnt1c = cnt1.clone();
let s1 = FastUDPSocket::new("", &ba1, &pool, move |sock: &FastUDPRawOsSocket, _: &InetAddress, data: PacketBuffer| {
cnt1c.fetch_add(1, Ordering::Relaxed);
});
let s1 = FastUDPSocket::new(
"",
&ba1,
&pool,
move |sock: &FastUDPRawOsSocket, _: &InetAddress, data: PacketBuffer| {
cnt1c.fetch_add(1, Ordering::Relaxed);
},
);
assert!(s1.is_ok());
let s1 = s1.unwrap();

32
attic/iblt/src/lib.rs
View file

@ -55,7 +55,11 @@ impl<const BUCKETS: usize, const ITEM_BYTES: usize, const HASHES: usize> Clone f
// of implementing clone() may not since check_hash[] is u32.
unsafe {
let mut tmp: Self = std::mem::MaybeUninit::uninit().assume_init();
std::ptr::copy_nonoverlapping((self as *const Self).cast::<u8>(), (&mut tmp as *mut Self).cast::<u8>(), Self::SIZE_BYTES);
std::ptr::copy_nonoverlapping(
(self as *const Self).cast::<u8>(),
(&mut tmp as *mut Self).cast::<u8>(),
Self::SIZE_BYTES,
);
tmp
}
}
@ -159,7 +163,10 @@ impl<const BUCKETS: usize, const ITEM_BYTES: usize, const HASHES: usize> IBLT<BU
/// Subtract another IBLT from this one to get a set difference.
pub fn subtract(&mut self, other: &Self) {
self.check_hash.iter_mut().zip(other.check_hash.iter()).for_each(|(a, b)| *a ^= *b);
self.count.iter_mut().zip(other.count.iter()).for_each(|(a, b)| *a = a.wrapping_sub(*b));
self.count
.iter_mut()
.zip(other.count.iter())
.for_each(|(a, b)| *a = a.wrapping_sub(*b));
self.key.iter_mut().zip(other.key.iter()).for_each(|(a, b)| xor_with(a, b));
}
@ -310,7 +317,14 @@ mod tests {
assert!(d);
});
println!("inserted: {}\tlisted: {}\tcapacity: {}\tscore: {:.4}\tfill: {:.4}", count, list_count, CAPACITY, (list_count as f64) / (count as f64), (count as f64) / (CAPACITY as f64));
println!(
"inserted: {}\tlisted: {}\tcapacity: {}\tscore: {:.4}\tfill: {:.4}",
count,
list_count,
CAPACITY,
(list_count as f64) / (count as f64),
(count as f64) / (CAPACITY as f64)
);
count += 32;
}
}
@ -358,7 +372,17 @@ mod tests {
cnt += 1;
});
println!("total: {} missing: {:5} recovered: {:5} size: {:5} score: {:.4} bytes/item: {:.2} extract(fill): {:.4} 100%: {}", REMOTE_SIZE, missing.len(), cnt, bytes, (cnt as f64) / (missing.len() as f64), (bytes as f64) / (cnt as f64), (cnt as f64) / (CAPACITY as f64), all_success);
println!(
"total: {} missing: {:5} recovered: {:5} size: {:5} score: {:.4} bytes/item: {:.2} extract(fill): {:.4} 100%: {}",
REMOTE_SIZE,
missing.len(),
cnt,
bytes,
(cnt as f64) / (missing.len() as f64),
(bytes as f64) / (cnt as f64),
(cnt as f64) / (CAPACITY as f64),
all_success
);
missing_count += STEP;
}

16
attic/syncwhole/src/datastore.rs
View file

@ -84,10 +84,22 @@ pub trait DataStore: Sync + Send {
/// Iterate through keys in a range.
///
/// Keys MUST be output in ascending binary sort order.
async fn keys<F: Send + FnMut(&[u8]) -> bool>(&self, subset: Option<&[u8]>, range_start: &[u8; KEY_SIZE], range_end: &[u8; KEY_SIZE], f: F);
async fn keys<F: Send + FnMut(&[u8]) -> bool>(
&self,
subset: Option<&[u8]>,
range_start: &[u8; KEY_SIZE],
range_end: &[u8; KEY_SIZE],
f: F,
);
/// Iterate through values in a range.
///
/// Entries MUST be output in ascending binary sort order.
async fn values<F: Send + FnMut(&[u8], &[u8]) -> bool>(&self, subset: Option<&[u8]>, range_start: &[u8; KEY_SIZE], range_end: &[u8; KEY_SIZE], f: F);
async fn values<F: Send + FnMut(&[u8], &[u8]) -> bool>(
&self,
subset: Option<&[u8]>,
range_start: &[u8; KEY_SIZE],
range_end: &[u8; KEY_SIZE],
f: F,
);
}

85
attic/syncwhole/src/main.rs
View file

@ -140,46 +140,57 @@ impl DataStore for TestNodeHost {
}
fn main() {
tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap().block_on(async {
println!("Running syncwhole local self-test network with {} nodes starting at 127.0.0.1:{}", TEST_NODE_COUNT, TEST_PORT_RANGE_START);
println!();
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(async {
println!(
"Running syncwhole local self-test network with {} nodes starting at 127.0.0.1:{}",
TEST_NODE_COUNT, TEST_PORT_RANGE_START
);
println!();
println!("Starting nodes on 127.0.0.1...");
let mut nodes: Vec<Node<TestNodeHost, TestNodeHost>> = Vec::with_capacity(TEST_NODE_COUNT);
for port in TEST_PORT_RANGE_START..(TEST_PORT_RANGE_START + (TEST_NODE_COUNT as u16)) {
let mut peers: Vec<SocketAddr> = Vec::with_capacity(TEST_NODE_COUNT);
for port2 in TEST_PORT_RANGE_START..(TEST_PORT_RANGE_START + (TEST_NODE_COUNT as u16)) {
if port != port2 {
peers.push(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port2)));
println!("Starting nodes on 127.0.0.1...");
let mut nodes: Vec<Node<TestNodeHost, TestNodeHost>> = Vec::with_capacity(TEST_NODE_COUNT);
for port in TEST_PORT_RANGE_START..(TEST_PORT_RANGE_START + (TEST_NODE_COUNT as u16)) {
let mut peers: Vec<SocketAddr> = Vec::with_capacity(TEST_NODE_COUNT);
for port2 in TEST_PORT_RANGE_START..(TEST_PORT_RANGE_START + (TEST_NODE_COUNT as u16)) {
if port != port2 {
peers.push(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port2)));
}
}
let mut th = TestNodeHost::new_random(port as usize);
th.config.anchors = peers;
th.config.name = port.to_string();
let nh = Arc::new(th);
//println!("Starting node on 127.0.0.1:{}...", port, nh.db.lock().unwrap().len());
nodes.push(
Node::new(nh.clone(), nh.clone(), SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port)))
.await
.unwrap(),
);
}
print!("Waiting for all connections to be established...");
let _ = stdout().flush();
loop {
tokio::time::sleep(Duration::from_secs(1)).await;
let mut count = 0;
for n in nodes.iter() {
count += n.connection_count().await;
}
if count == (TEST_NODE_COUNT * (TEST_NODE_COUNT - 1)) {
println!(" {} connections up.", count);
break;
} else {
print!(".");
let _ = stdout().flush();
}
}
let mut th = TestNodeHost::new_random(port as usize);
th.config.anchors = peers;
th.config.name = port.to_string();
let nh = Arc::new(th);
//println!("Starting node on 127.0.0.1:{}...", port, nh.db.lock().unwrap().len());
nodes.push(Node::new(nh.clone(), nh.clone(), SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port))).await.unwrap());
}
print!("Waiting for all connections to be established...");
let _ = stdout().flush();
loop {
tokio::time::sleep(Duration::from_secs(1)).await;
let mut count = 0;
for n in nodes.iter() {
count += n.connection_count().await;
loop {
tokio::time::sleep(Duration::from_secs(1)).await;
}
if count == (TEST_NODE_COUNT * (TEST_NODE_COUNT - 1)) {
println!(" {} connections up.", count);
break;
} else {
print!(".");
let _ = stdout().flush();
}
}
loop {
tokio::time::sleep(Duration::from_secs(1)).await;
}
});
});
}

105
attic/syncwhole/src/node.rs
View file

@ -75,7 +75,11 @@ pub struct Node<D: DataStore + 'static, H: Host + 'static> {
impl<D: DataStore + 'static, H: Host + 'static> Node<D, H> {
pub async fn new(db: Arc<D>, host: Arc<H>, bind_address: SocketAddr) -> std::io::Result<Self> {
let listener = if bind_address.is_ipv4() { TcpSocket::new_v4() } else { TcpSocket::new_v6() }?;
let listener = if bind_address.is_ipv4() {
TcpSocket::new_v4()
} else {
TcpSocket::new_v6()
}?;
configure_tcp_socket(&listener)?;
listener.bind(bind_address.clone())?;
let listener = listener.listen(1024)?;
@ -119,7 +123,12 @@ impl<D: DataStore + 'static, H: Host + 'static> Node<D, H> {
/// to the endpoint. An error is returned if the connection fails.
pub async fn connect(&self, address: &SocketAddr) -> std::io::Result<bool> {
if self.internal.connecting_to.lock().await.insert(address.clone()) {
self.internal.connect(address, Instant::now().add(Duration::from_millis(self.internal.host.node_config().connection_timeout))).await
self.internal
.connect(
address,
Instant::now().add(Duration::from_millis(self.internal.host.node_config().connection_timeout)),
)
.await
} else {
Ok(false)
}
@ -161,7 +170,12 @@ fn configure_tcp_socket(socket: &TcpSocket) -> std::io::Result<()> {
}
fn decode_msgpack<'a, T: Deserialize<'a>>(b: &'a [u8]) -> std::io::Result<T> {
rmp_serde::from_slice(b).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, format!("invalid msgpack object: {}", e.to_string())))
rmp_serde::from_slice(b).map_err(|e| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("invalid msgpack object: {}", e.to_string()),
)
})
}
pub struct NodeInternal<D: DataStore + 'static, H: Host + 'static> {
@ -228,7 +242,10 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
let e = j.unwrap().await;
if e.is_ok() {
let e = e.unwrap();
host.on_connection_closed(&*cc.info.lock().unwrap(), e.map_or_else(|e| e.to_string(), |_| "unknown error".to_string()));
host.on_connection_closed(
&*cc.info.lock().unwrap(),
e.map_or_else(|e| e.to_string(), |_| "unknown error".to_string()),
);
} else {
host.on_connection_closed(&*cc.info.lock().unwrap(), "remote host closed connection".to_string());
}
@ -299,7 +316,10 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
background_tasks.spawn(async move {
// If the connection dies this will either fail or time out in 1s. Usually these execute instantly due to
// write buffering but a short timeout prevents them from building up too much.
let _ = tokio::time::timeout(announce_timeout, c2.send_msg(MessageType::HaveRecords, have_records.as_slice(), now));
let _ = tokio::time::timeout(
announce_timeout,
c2.send_msg(MessageType::HaveRecords, have_records.as_slice(), now),
);
});
have_records = Vec::with_capacity(have_records_est_size);
}
@ -332,7 +352,11 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
/// either succeeds or fails.
async fn connect(self: &Arc<Self>, address: &SocketAddr, deadline: Instant) -> std::io::Result<bool> {
let f = async {
let stream = if address.is_ipv4() { TcpSocket::new_v4() } else { TcpSocket::new_v6() }?;
let stream = if address.is_ipv4() {
TcpSocket::new_v4()
} else {
TcpSocket::new_v6()
}?;
configure_tcp_socket(&stream)?;
stream.bind(self.bind_address.clone())?;
@ -389,7 +413,12 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
ok
}
async fn connection_io_task_main(self: Arc<Self>, connection: &Arc<Connection>, remote_address: SocketAddr, mut reader: OwnedReadHalf) -> std::io::Result<()> {
async fn connection_io_task_main(
self: Arc<Self>,
connection: &Arc<Connection>,
remote_address: SocketAddr,
mut reader: OwnedReadHalf,
) -> std::io::Result<()> {
const BUF_CHUNK_SIZE: usize = 4096;
const READ_BUF_INITIAL_SIZE: usize = 65536; // should be a multiple of BUF_CHUNK_SIZE
@ -479,20 +508,29 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
info.name = msg.node_name.to_string();
info.contact = msg.node_contact.to_string();
let _ = msg.explicit_ipv4.map(|pv4| {
info.explicit_addresses.push(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::from(pv4.ip), pv4.port)));
info.explicit_addresses
.push(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::from(pv4.ip), pv4.port)));
});
let _ = msg.explicit_ipv6.map(|pv6| {
info.explicit_addresses.push(SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::from(pv6.ip), pv6.port, 0, 0)));
info.explicit_addresses
.push(SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::from(pv6.ip), pv6.port, 0, 0)));
});
let info = info.clone();
let auth_challenge_response = self.host.authenticate(&info, msg.auth_challenge);
if auth_challenge_response.is_none() {
return Err(std::io::Error::new(std::io::ErrorKind::Other, "authenticate() returned None, connection dropped"));
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"authenticate() returned None, connection dropped",
));
}
let auth_challenge_response = auth_challenge_response.unwrap();
(H::hmac_sha512(&self.anti_loopback_secret, msg.anti_loopback_challenge), H::hmac_sha512(&H::sha512(&[self.host.node_config().domain.as_bytes()]), msg.domain_challenge), auth_challenge_response)
(
H::hmac_sha512(&self.anti_loopback_secret, msg.anti_loopback_challenge),
H::hmac_sha512(&H::sha512(&[self.host.node_config().domain.as_bytes()]), msg.domain_challenge),
auth_challenge_response,
)
};
connection
@ -517,14 +555,27 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
return Err(std::io::Error::new(std::io::ErrorKind::Other, "duplicate init response"));
}
if msg.anti_loopback_response.eq(&H::hmac_sha512(&self.anti_loopback_secret, &anti_loopback_challenge_sent)) {
if msg
.anti_loopback_response
.eq(&H::hmac_sha512(&self.anti_loopback_secret, &anti_loopback_challenge_sent))
{
return Err(std::io::Error::new(std::io::ErrorKind::Other, "rejected connection to self"));
}
if !msg.domain_response.eq(&H::hmac_sha512(&H::sha512(&[self.host.node_config().domain.as_bytes()]), &domain_challenge_sent)) {
if !msg.domain_response.eq(&H::hmac_sha512(
&H::sha512(&[self.host.node_config().domain.as_bytes()]),
&domain_challenge_sent,
)) {
return Err(std::io::Error::new(std::io::ErrorKind::Other, "domain mismatch"));
}
if !self.host.authenticate(&info, &auth_challenge_sent).map_or(false, |cr| msg.auth_response.eq(&cr)) {
return Err(std::io::Error::new(std::io::ErrorKind::Other, "challenge/response authentication failed"));
if !self
.host
.authenticate(&info, &auth_challenge_sent)
.map_or(false, |cr| msg.auth_response.eq(&cr))
{
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"challenge/response authentication failed",
));
}
initialized = true;
@ -537,7 +588,10 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
// Handle messages other than INIT and INIT_RESPONSE after checking 'initialized' flag.
_ => {
if !initialized {
return Err(std::io::Error::new(std::io::ErrorKind::Other, "init exchange must be completed before other messages are sent"));
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"init exchange must be completed before other messages are sent",
));
}
match message_type {
@ -556,7 +610,10 @@ impl<D: DataStore + 'static, H: Host + 'static> NodeInternal<D, H> {
}
}
StoreResult::Rejected => {
return Err(std::io::Error::new(std::io::ErrorKind::Other, format!("record rejected by data store: {}", to_hex_string(&key))));
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!("record rejected by data store: {}", to_hex_string(&key)),
));
}
_ => {}
}
@ -615,7 +672,14 @@ impl Connection {
let mut header: [u8; 16] = unsafe { MaybeUninit::uninit().assume_init() };
header[0] = message_type as u8;
let header_size = 1 + varint::encode(&mut header[1..], data.len() as u64);
if self.writer.lock().await.write_vectored(&[IoSlice::new(&header[0..header_size]), IoSlice::new(data)]).await? == (data.len() + header_size) {
if self
.writer
.lock()
.await
.write_vectored(&[IoSlice::new(&header[0..header_size]), IoSlice::new(data)])
.await?
== (data.len() + header_size)
{
self.last_send_time.store(now, Ordering::Relaxed);
Ok(())
} else {
@ -628,7 +692,10 @@ impl Connection {
if rmp_serde::encode::write_named(write_buf, obj).is_ok() {
self.send_msg(message_type, write_buf.as_slice(), now).await
} else {
Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "serialize failure (internal error)"))
Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"serialize failure (internal error)",
))
}
}
}

9
attic/syncwhole/src/utils.rs
View file

@ -16,12 +16,17 @@ use tokio::task::JoinHandle;
/// Get the real time clock in milliseconds since Unix epoch.
pub fn ms_since_epoch() -> i64 {
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis() as i64
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as i64
}
/// Encode a byte slice to a hexadecimal string.
pub fn to_hex_string(b: &[u8]) -> String {
const HEX_CHARS: [u8; 16] = [b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f'];
const HEX_CHARS: [u8; 16] = [
b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f',
];
let mut s = String::new();
s.reserve(b.len() * 2);
for c in b {

4
crypto/benches/benchmark_crypto.rs
View file

@ -19,7 +19,9 @@ pub fn criterion_benchmark(c: &mut Criterion) {
let mut group = c.benchmark_group("cryptography");
group.measurement_time(Duration::new(10, 0));
group.bench_function("ecdhp384", |b| b.iter(|| p384_a.agree(p384_b.public_key()).expect("ecdhp384 failed")));
group.bench_function("ecdhp384", |b| {
b.iter(|| p384_a.agree(p384_b.public_key()).expect("ecdhp384 failed"))
});
group.bench_function("ecdhx25519", |b| b.iter(|| x25519_a.agree(&x25519_b_pub)));
group.bench_function("kyber_encapsulate", |b| {
b.iter(|| pqc_kyber::encapsulate(&kyber_a.public, &mut random::SecureRandom::default()).expect("kyber encapsulate failed"))

72
crypto/src/aes.rs
View file

@ -36,7 +36,14 @@ mod fruit_flavored {
options: i32,
cryyptor_ref: *mut *mut c_void,
) -> i32;
fn CCCryptorUpdate(cryptor_ref: *mut c_void, data_in: *const c_void, data_in_len: usize, data_out: *mut c_void, data_out_len: usize, data_out_written: *mut usize) -> i32;
fn CCCryptorUpdate(
cryptor_ref: *mut c_void,
data_in: *const c_void,
data_in_len: usize,
data_out: *mut c_void,
data_out_len: usize,
data_out_written: *mut usize,
) -> i32;
//fn CCCryptorReset(cryptor_ref: *mut c_void, iv: *const c_void) -> i32;
fn CCCryptorRelease(cryptor_ref: *mut c_void) -> i32;
fn CCCryptorGCMSetIV(cryptor_ref: *mut c_void, iv: *const c_void, iv_len: usize) -> i32;
@ -110,7 +117,14 @@ mod fruit_flavored {
assert_eq!(ciphertext.len(), 16);
unsafe {
let mut data_out_written = 0;
CCCryptorUpdate(self.0, plaintext.as_ptr().cast(), 16, ciphertext.as_mut_ptr().cast(), 16, &mut data_out_written);
CCCryptorUpdate(
self.0,
plaintext.as_ptr().cast(),
16,
ciphertext.as_mut_ptr().cast(),
16,
&mut data_out_written,
);
}
}
@ -119,7 +133,14 @@ mod fruit_flavored {
assert_eq!(data.len(), 16);
unsafe {
let mut data_out_written = 0;
CCCryptorUpdate(self.0, data.as_ptr().cast(), 16, data.as_mut_ptr().cast(), 16, &mut data_out_written);
CCCryptorUpdate(
self.0,
data.as_ptr().cast(),
16,
data.as_mut_ptr().cast(),
16,
&mut data_out_written,
);
}
}
@ -129,7 +150,14 @@ mod fruit_flavored {
assert_eq!(ciphertext.len(), 16);
unsafe {
let mut data_out_written = 0;
CCCryptorUpdate(self.1, ciphertext.as_ptr().cast(), 16, plaintext.as_mut_ptr().cast(), 16, &mut data_out_written);
CCCryptorUpdate(
self.1,
ciphertext.as_ptr().cast(),
16,
plaintext.as_mut_ptr().cast(),
16,
&mut data_out_written,
);
}
}
@ -138,7 +166,14 @@ mod fruit_flavored {
assert_eq!(data.len(), 16);
unsafe {
let mut data_out_written = 0;
CCCryptorUpdate(self.1, data.as_ptr().cast(), 16, data.as_mut_ptr().cast(), 16, &mut data_out_written);
CCCryptorUpdate(
self.1,
data.as_ptr().cast(),
16,
data.as_mut_ptr().cast(),
16,
&mut data_out_written,
);
}
}
}
@ -215,9 +250,15 @@ mod fruit_flavored {
unsafe {
assert_eq!(input.len(), output.len());
if self.1 {
assert_eq!(CCCryptorGCMEncrypt(self.0, input.as_ptr().cast(), input.len(), output.as_mut_ptr().cast()), 0);
assert_eq!(
CCCryptorGCMEncrypt(self.0, input.as_ptr().cast(), input.len(), output.as_mut_ptr().cast()),
0
);
} else {
assert_eq!(CCCryptorGCMDecrypt(self.0, input.as_ptr().cast(), input.len(), output.as_mut_ptr().cast()), 0);
assert_eq!(
CCCryptorGCMDecrypt(self.0, input.as_ptr().cast(), input.len(), output.as_mut_ptr().cast()),
0
);
}
}
}
@ -226,9 +267,15 @@ mod fruit_flavored {
pub fn crypt_in_place(&mut self, data: &mut [u8]) {
unsafe {
if self.1 {
assert_eq!(CCCryptorGCMEncrypt(self.0, data.as_ptr().cast(), data.len(), data.as_mut_ptr().cast()), 0);
assert_eq!(
CCCryptorGCMEncrypt(self.0, data.as_ptr().cast(), data.len(), data.as_mut_ptr().cast()),
0
);
} else {
assert_eq!(CCCryptorGCMDecrypt(self.0, data.as_ptr().cast(), data.len(), data.as_mut_ptr().cast()), 0);
assert_eq!(
CCCryptorGCMDecrypt(self.0, data.as_ptr().cast(), data.len(), data.as_mut_ptr().cast()),
0
);
}
}
}
@ -392,7 +439,12 @@ mod openssl_aes {
/// Encrypt or decrypt in place (same operation with CTR mode)
#[inline(always)]
pub fn crypt_in_place(&mut self, data: &mut [u8]) {
assert!(self.2.as_mut().unwrap().update(unsafe { &*std::slice::from_raw_parts(data.as_ptr(), data.len()) }, data).is_ok());
assert!(self
.2
.as_mut()
.unwrap()
.update(unsafe { &*std::slice::from_raw_parts(data.as_ptr(), data.len()) }, data)
.is_ok());
}
#[inline(always)]

110
crypto/src/aes_gmac_siv/impl_macos.rs
View file

@ -35,7 +35,14 @@ extern "C" {
options: i32,
cryyptor_ref: *mut *mut c_void,
) -> i32;
fn CCCryptorUpdate(cryptor_ref: *mut c_void, data_in: *const c_void, data_in_len: usize, data_out: *mut c_void, data_out_len: usize, data_out_written: *mut usize) -> i32;
fn CCCryptorUpdate(
cryptor_ref: *mut c_void,
data_in: *const c_void,
data_in_len: usize,
data_out: *mut c_void,
data_out_len: usize,
data_out_written: *mut usize,
) -> i32;
fn CCCryptorReset(cryptor_ref: *mut c_void, iv: *const c_void) -> i32;
fn CCCryptorRelease(cryptor_ref: *mut c_void) -> i32;
fn CCCryptorGCMSetIV(cryptor_ref: *mut c_void, iv: *const c_void, iv_len: usize) -> i32;
@ -65,7 +72,20 @@ impl AesCtr {
}
unsafe {
let mut ptr: *mut c_void = null_mut();
let result = CCCryptorCreateWithMode(kCCEncrypt, kCCModeCTR, kCCAlgorithmAES, 0, crate::ZEROES.as_ptr().cast(), k.as_ptr().cast(), k.len(), null(), 0, 0, 0, &mut ptr);
let result = CCCryptorCreateWithMode(
kCCEncrypt,
kCCModeCTR,
kCCAlgorithmAES,
0,
crate::ZEROES.as_ptr().cast(),
k.as_ptr().cast(),
k.len(),
null(),
0,
0,
0,
&mut ptr,
);
if result != 0 {
panic!("CCCryptorCreateWithMode for CTR mode returned {}", result);
}
@ -99,7 +119,14 @@ impl AesCtr {
unsafe {
assert!(output.len() >= input.len());
let mut data_out_written: usize = 0;
CCCryptorUpdate(self.0, input.as_ptr().cast(), input.len(), output.as_mut_ptr().cast(), output.len(), &mut data_out_written);
CCCryptorUpdate(
self.0,
input.as_ptr().cast(),
input.len(),
output.as_mut_ptr().cast(),
output.len(),
&mut data_out_written,
);
}
}
@ -108,7 +135,14 @@ impl AesCtr {
pub fn crypt_in_place(&mut self, data: &mut [u8]) {
unsafe {
let mut data_out_written: usize = 0;
CCCryptorUpdate(self.0, data.as_ptr().cast(), data.len(), data.as_mut_ptr().cast(), data.len(), &mut data_out_written);
CCCryptorUpdate(
self.0,
data.as_ptr().cast(),
data.len(),
data.as_mut_ptr().cast(),
data.len(),
&mut data_out_written,
);
}
}
}
@ -164,8 +198,20 @@ impl AesGmacSiv {
gmac: null_mut(),
};
unsafe {
let result =
CCCryptorCreateWithMode(kCCEncrypt, kCCModeCTR, kCCAlgorithmAES, 0, crate::ZEROES.as_ptr().cast(), k1.as_ptr().cast(), k1.len(), null(), 0, 0, 0, &mut c.ctr);
let result = CCCryptorCreateWithMode(
kCCEncrypt,
kCCModeCTR,
kCCAlgorithmAES,
0,
crate::ZEROES.as_ptr().cast(),
k1.as_ptr().cast(),
k1.len(),
null(),
0,
0,
0,
&mut c.ctr,
);
if result != 0 {
panic!("CCCryptorCreateWithMode for CTR mode returned {}", result);
}
@ -203,8 +249,20 @@ impl AesGmacSiv {
if result != 0 {
panic!("CCCryptorCreateWithMode for ECB decrypt mode returned {}", result);
}
let result =
CCCryptorCreateWithMode(kCCEncrypt, kCCModeGCM, kCCAlgorithmAES, 0, crate::ZEROES.as_ptr().cast(), k0.as_ptr().cast(), k0.len(), null(), 0, 0, 0, &mut c.gmac);
let result = CCCryptorCreateWithMode(
kCCEncrypt,
kCCModeGCM,
kCCAlgorithmAES,
0,
crate::ZEROES.as_ptr().cast(),
k0.as_ptr().cast(),
k0.len(),
null(),
0,
0,
0,
&mut c.gmac,
);
if result != 0 {
panic!("CCCryptorCreateWithMode for GCM (GMAC) mode returned {}", result);
}
@ -262,7 +320,14 @@ impl AesGmacSiv {
let tmp = self.tmp.as_mut_ptr().cast::<u64>();
*self.tag.as_mut_ptr().cast::<u64>().offset(1) = *tmp ^ *tmp.offset(1);
let mut data_out_written: usize = 0;
CCCryptorUpdate(self.ecb_enc, self.tag.as_ptr().cast(), 16, self.tag.as_mut_ptr().cast(), 16, &mut data_out_written);
CCCryptorUpdate(
self.ecb_enc,
self.tag.as_ptr().cast(),
16,
self.tag.as_mut_ptr().cast(),
16,
&mut data_out_written,
);
}
self.tmp.copy_from_slice(&self.tag);
self.tmp[12] &= 0x7f;
@ -280,7 +345,14 @@ impl AesGmacSiv {
unsafe {
assert!(ciphertext.len() >= plaintext.len());
let mut data_out_written: usize = 0;
CCCryptorUpdate(self.ctr, plaintext.as_ptr().cast(), plaintext.len(), ciphertext.as_mut_ptr().cast(), ciphertext.len(), &mut data_out_written);
CCCryptorUpdate(
self.ctr,
plaintext.as_ptr().cast(),
plaintext.len(),
ciphertext.as_mut_ptr().cast(),
ciphertext.len(),
&mut data_out_written,
);
}
}
@ -316,7 +388,14 @@ impl AesGmacSiv {
panic!("CCCryptorReset for CTR mode failed (old MacOS bug)");
}
let mut data_out_written = 0;
CCCryptorUpdate(self.ecb_dec, self.tag.as_ptr().cast(), 16, self.tag.as_mut_ptr().cast(), 16, &mut data_out_written);
CCCryptorUpdate(
self.ecb_dec,
self.tag.as_ptr().cast(),
16,
self.tag.as_mut_ptr().cast(),
16,
&mut data_out_written,
);
let tmp = self.tmp.as_mut_ptr().cast::<u64>();
*tmp = *self.tag.as_mut_ptr().cast::<u64>();
*tmp.add(1) = 0;
@ -345,7 +424,14 @@ impl AesGmacSiv {
pub fn decrypt(&mut self, ciphertext: &[u8], plaintext: &mut [u8]) {
unsafe {
let mut data_out_written = 0;
CCCryptorUpdate(self.ctr, ciphertext.as_ptr().cast(), ciphertext.len(), plaintext.as_mut_ptr().cast(), plaintext.len(), &mut data_out_written);
CCCryptorUpdate(
self.ctr,
ciphertext.as_ptr().cast(),
ciphertext.len(),
plaintext.as_mut_ptr().cast(),
plaintext.len(),
&mut data_out_written,
);
CCCryptorGCMAddAAD(self.gmac, plaintext.as_ptr().cast(), plaintext.len());
}
}

60
crypto/src/aes_gmac_siv/impl_openssl.rs
View file

@ -56,7 +56,9 @@ impl AesCtr {
/// If it's already been used, this also resets the cipher. There is no separate reset.
#[inline(always)]
pub fn init(&mut self, iv: &[u8]) {
let _ = self.1.replace(Crypter::new(aes_ctr_by_key_size(self.0.len()), Mode::Encrypt, self.0.as_slice(), Some(iv)).unwrap());
let _ = self
.1
.replace(Crypter::new(aes_ctr_by_key_size(self.0.len()), Mode::Encrypt, self.0.as_slice(), Some(iv)).unwrap());
}
/// Encrypt or decrypt (same operation with CTR mode)
@ -68,7 +70,11 @@ impl AesCtr {
/// Encrypt or decrypt in place (same operation with CTR mode)
#[inline(always)]
pub fn crypt_in_place(&mut self, data: &mut [u8]) {
let _ = self.1.as_mut().unwrap().update(unsafe { &*std::slice::from_raw_parts(data.as_ptr(), data.len()) }, data);
let _ = self
.1
.as_mut()
.unwrap()
.update(unsafe { &*std::slice::from_raw_parts(data.as_ptr(), data.len()) }, data);
}
}
@ -116,7 +122,15 @@ impl AesGmacSiv {
pub fn encrypt_init(&mut self, iv: &[u8]) {
self.tag[0..8].copy_from_slice(iv);
self.tag[8..12].fill(0);
let _ = self.gmac.replace(Crypter::new(aes_gcm_by_key_size(self.k0.len()), Mode::Encrypt, self.k0.as_slice(), Some(&self.tag[0..12])).unwrap());
let _ = self.gmac.replace(
Crypter::new(
aes_gcm_by_key_size(self.k0.len()),
Mode::Encrypt,
self.k0.as_slice(),
Some(&self.tag[0..12]),
)
.unwrap(),
);
}
/// Set additional authenticated data (data to be authenticated but not encrypted).
@ -164,7 +178,15 @@ impl AesGmacSiv {
self.tmp.copy_from_slice(&tag_tmp[0..16]);
self.tmp[12] &= 0x7f;
let _ = self.ctr.replace(Crypter::new(aes_ctr_by_key_size(self.k1.len()), Mode::Encrypt, self.k1.as_slice(), Some(&self.tmp)).unwrap());
let _ = self.ctr.replace(
Crypter::new(
aes_ctr_by_key_size(self.k1.len()),
Mode::Encrypt,
self.k1.as_slice(),
Some(&self.tmp),
)
.unwrap(),
);
}
/// Feed plaintext for second pass and write ciphertext to supplied buffer.
@ -178,7 +200,10 @@ impl AesGmacSiv {
/// This may be called more than once.
#[inline(always)]
pub fn encrypt_second_pass_in_place(&mut self, plaintext_to_ciphertext: &mut [u8]) {
let _ = self.ctr.as_mut().unwrap().update(unsafe { std::slice::from_raw_parts(plaintext_to_ciphertext.as_ptr(), plaintext_to_ciphertext.len()) }, plaintext_to_ciphertext);
let _ = self.ctr.as_mut().unwrap().update(
unsafe { std::slice::from_raw_parts(plaintext_to_ciphertext.as_ptr(), plaintext_to_ciphertext.len()) },
plaintext_to_ciphertext,
);
}
/// Finish second pass and return a reference to the tag for this message.
@ -194,7 +219,15 @@ impl AesGmacSiv {
pub fn decrypt_init(&mut self, tag: &[u8]) {
self.tmp.copy_from_slice(tag);
self.tmp[12] &= 0x7f;
let _ = self.ctr.replace(Crypter::new(aes_ctr_by_key_size(self.k1.len()), Mode::Decrypt, self.k1.as_slice(), Some(&self.tmp)).unwrap());
let _ = self.ctr.replace(
Crypter::new(
aes_ctr_by_key_size(self.k1.len()),
Mode::Decrypt,
self.k1.as_slice(),
Some(&self.tmp),
)
.unwrap(),
);
let mut tag_tmp = [0_u8; 32];
let mut ecb = Crypter::new(aes_ecb_by_key_size(self.k1.len()), Mode::Decrypt, self.k1.as_slice(), None).unwrap();
@ -204,7 +237,15 @@ impl AesGmacSiv {
}
self.tag.copy_from_slice(&tag_tmp[0..16]);
tag_tmp[8..12].fill(0);
let _ = self.gmac.replace(Crypter::new(aes_gcm_by_key_size(self.k0.len()), Mode::Encrypt, self.k0.as_slice(), Some(&tag_tmp[0..12])).unwrap());
let _ = self.gmac.replace(
Crypter::new(
aes_gcm_by_key_size(self.k0.len()),
Mode::Encrypt,
self.k0.as_slice(),
Some(&tag_tmp[0..12]),
)
.unwrap(),
);
}
/// Set additional authenticated data to be checked.
@ -225,7 +266,10 @@ impl AesGmacSiv {
/// This may be called more than once.
#[inline(always)]
pub fn decrypt_in_place(&mut self, ciphertext_to_plaintext: &mut [u8]) {
self.decrypt(unsafe { std::slice::from_raw_parts(ciphertext_to_plaintext.as_ptr(), ciphertext_to_plaintext.len()) }, ciphertext_to_plaintext);
self.decrypt(
unsafe { std::slice::from_raw_parts(ciphertext_to_plaintext.as_ptr(), ciphertext_to_plaintext.len()) },
ciphertext_to_plaintext,
);
}
/// Finish decryption and return true if authentication appears valid.

533
crypto/src/aes_gmac_siv/mod.rs
View file

@ -20,98 +20,441 @@ mod tests {
use sha2::Digest;
use std::time::SystemTime;
const TV0_KEYS: [&'static [u8]; 2] = ["00000000000000000000000000000000".as_bytes(), "11111111111111111111111111111111".as_bytes()];
const TV0_KEYS: [&'static [u8]; 2] = [
"00000000000000000000000000000000".as_bytes(),
"11111111111111111111111111111111".as_bytes(),
];
/// Test vectors consist of a series of input sizes, a SHA384 hash of a resulting ciphertext, and an expected tag.
/// Input is a standard byte array consisting of bytes 0, 1, 2, 3, ..., 255 and then cycling back to 0 over and over
/// and is provided both as ciphertext and associated data (AAD).
#[allow(unused)]
const TEST_VECTORS: [(usize, &'static str, &'static str); 85] = [
(0, "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b", "43847e644239134deccf5538162c861e"),
(777, "aabf892f18a620b9c3bae91bb03a74c84193e4a7b64916c6bc88b885b9ebed4134495e5f22f12e3046fbb3f26fa111a7", "b8c318b5dcc1d672114a6f7be54ef289"),
(1554, "648f551df29217f0e634b72ba6973c0eb95c7d4be8b135e550d8bcdf65b75980881bc0e03cf22589e04bedc7da1804cd", "535b8ddd51ec82a1e850906fe321b21a"),
(2331, "bfbfdffea40062e23bbdf0835e1d38d1623bebca7407908bbc6d5b3f2bfd062a2d237f091affda7348094fafda0bd1a7", "4f521876fbb2c563051196b33c20c822"),
(3108, "cc6035cab70f3a3298a5c4956ff07f179acf3771bb915c590a8a19fe5133d6d8a81c118148394dfb364af5c2fbdaadeb", "d3adfa578c8bcd738c55ffc527358cef"),
(3885, "15ec2760a21c25f9870a84ee757f3da2c261a950c2f692d75ff9e99b2d50c826c21e27e49c4cd3450fedc7e60371589f", "a4c22d6c3d773634c2dc057e1f7c6738"),
(4662, "c2afad6f034704300c34f143dcdcb86c9b954cec1ebf22e7071f288c58a2ae430d3e3748d214d1021472793d3f337dc6", "c0601cb6cd4883102f70570c2cdc0ab6"),
(5439, "8fee067f5a7a475a630f9db8b2eb80c1edc40eb4246a0f1c078e535df7d06451c6a9bde1a23ba70285690dd7100a8626", "7352239f2302b08844309d28b13fa867"),
(6216, "60095b4172438aee61e65f5379f4ef276c3632d4ac74eea7723a2201823432614aba7b4670d9bf7a5b9126ca38f3b88a", "c0f0b0aa651965f8514b473c5406285e"),
(6993, "10e754dd08b4d2a6c109fb01fce2b57d54743947e14a7e67d7efd0608baf91f7fc42a53328fe8c18d234abad8ebcdff0", "58444988a62a99060728a7637c8499eb"),
(7770, "1abc4a5dcd2696336bd0e8af20fe7fc261aa424b52cfb5ad80ee7c7c793ac44f11db3506cdbbbaed0f80000925d08d52", "e8065c563bc6018cdcbf9aaafef767e6"),
(8547, "26aaf74ae8bfc6aaf45ceee0476ea0a484304f5c36050d3e2265cb194a2f7c308213314232270608b6d3f1c11b834e33", "ec50e4b3f6e4b3de24b3476623d08157"),
(9324, "863206305d466aa9c0d0ec674572069f61fe5009767f99ec8832912725c28c49d6a106ad3f55372c922e4e169fc382ce", "0cfac64f49e0f128d0a18d293878f222"),
(10101, "bd0c0950b947a6c34f1fa6e877433b42c039a8ea7b37634c40fb47efae4958ba74ef0991cfedf3c82a0b87ef59635071", "e0220a02b74259eeebbebede847d50f9"),
(10878, "d7b9901af1dacf6a8c369b993ba1c607f9b7f073d02311c72d8449d3494d477ffc8344a1d8b488020ccfc7c80fbd27e1", "ebe3933146734a6ade2b434f2bcd78ae"),
(11655, "0ba265e3ef0bebf01a4f3490da462c7730aad6aa6c70bb9ce64a36d26d24fe213660e60e4d3301329170471f11ff8ca2", "ec3dd4bf4cb7d527a86dd559c773a87b"),
(12432, "c3b6755a1be922ec71c1e187ead36c4e6fc307c72969c64ca1e9b7339d61e1a93a74a315fd73bed8fa5797b78b19dbe5", "5b58dcf392749bcef91056ba9475d0ef"),
(13209, "2fb1a67151183daa2f0d7f0064534497357f173161349dd008499a8c1a123cc942662ecc426e2ad7743fe0ab9f5d7be1", "c011260d328d310e2ab606aa1ef8afd4"),
(13986, "6afae2a07ce9bfe30fbbfb7dcf32d755bcf357334dc5c309e58cab38ebe559f25b313a0b3ca32ff1dc41f7b99718f653", "011bf43cfbbb7ae5986f8e0fc87771a9"),
(14763, "cc6215c115eb6411f4712c2289f5bf0ccb5151635f9f9ceac7c1b62d8d2f4d26498079d0289f83aeb26e97b5b924ffc4", "a015034a8d5bc83cc76c6983a5ba19ab"),
(15540, "3cebce794e947341c4ceec444ca43c6ac57c6f58de462bfec7566cbd59a1b6f2eae774120e29521e76120a604d1a12d9", "d373cd2bd9000655141ac632880eca40"),
(16317, "899147b98d78bb5d137dc7c4f03be7eca82bcca19cc3a701261332923707aed2e6719d35d2f2bf067cd1d193a53529cf", "ed223b64529299c787f49d631ce181c1"),
(17094, "aecd1830958b994b2c331b90e7d8ff79f27c83a71f5797a65ade3a30b4fa5928e79140bcd03f375591d53df96fea1a4d", "948a7c253d54bb6b65d78530c0eb7aab"),
(17871, "e677ffd4ecaba5899659fefe5fe8e643004392be3be6dc5a801409870ac1e3398f47cc1d83f7a4c41925b6337e01f7fd", "156a600c336f3ac034ca90034aa22635"),
(18648, "4ee50f4a98d0bbd160add6acf76765ccdac0c1cd0bb2adbbcb22dd012a1121620b739a120df7dc4091e684ddf28eb726", "75873467b416a7b025f9f1b015bf653a"),
(19425, "aa025f32c0575af7209828fc7fc4591b41fa7cfb485e26c5401e63ca1fa05776f8b8af1769a15e81f2c663bca9b02ab3", "5679efa7a4404e1e5c9b372782a41bf2"),
(20202, "6e77ab62d2affeb27f4ef326191b3df3863c338a629f64a785505f4a5968ff59bc011c7a27951cb00e2e7d9b9bd32fec", "36a9c4515d34f9bb962d8876ab3b5c86"),
(20979, "1625b4f0e65fc66f11ba3ee6b3e20c732535654c447df6b517ced113107a1057a64477faa2af4a5ede4034bf3cff98ea", "9058044e0f71c28d4f8d3281a3aec024"),
(21756, "94efe6aa55bd77bfa58c185dec313a41003f9bef02568e72c337be4de1b46c6e5bb9a9329b4f108686489b8bc9d5f4f0", "8d6d2c90590268a26f5e7d76351f48c1"),
(22533, "7327a05fdb0ac92433dfc2c85c5e96e6ddcbdb01e079f8dafbee79c14cb4d5fd46047acd6bb0e09a98f6dd03dced2a0a", "4e0f0a394f85bca35c68ef667aa9c244"),
(23310, "93da9e356efbc8b5ae366256f4c6fc11c11fc347aaa879d591b7c1262d90adf98925f571914696054f1d09c74783561e", "8c83c157be439280afc790ee3fd667eb"),
(24087, "99b91be5ffca51b1cbc7410798b1540b5b1a3356f801ed4dc54812919c08ca5a9adc218bc51e594d97b46445a1515506", "9436ff05729a77f673e815e464aeaa75"),
(24864, "074253ad5d5a5d2b072e7aeaffa04a06119ec812a88ca43481fe5e2dce02cf6736952095cd342ec70b833c12fc1777f4", "69d8951b96866a08efbb65f2bc31cfbc"),
(25641, "c0a301f90597c05cf19e60c35378676764086b7156e455f4800347f8a6e733d644e4cc709fb9d95a9211f3e1e10c762a", "3561c9802143c306ecc5e07e3b976d9e"),
(26418, "3c839e59d945b841acb604e1b9ae3df36a291444ce0bcae336ee875beaf208bf10af7342b375429ecb92ec54d11a5907", "3032ffdb8daee11b2e739132c6175615"),
(27195, "3dc59b16603950dfc26a90bf036712eb088412e8de4d1b27c3fa6be6502ac12d89d194764fb53c3dc7d90fa696ba5a16", "49436717edff7cd67c9a1be16d524f07"),
(27972, "4fbc0d40ff13376b8ed5382890cdea337b4a0c9c31b477c4008d2ef8299bd5ab771ba70b1b4b743f8f7caa1f0164d1a1", "64a9856a3bb81dc81ff1bc1025192dc9"),
(28749, "6ab191aa6327f229cc94e8c7b1b7ee30bc723e6aeaf3050eb7d14cb491c3513254e9b19894c2b4f071d298401fd31945", "101f2ffea60f246a3b57c4a530d67cf1"),
(29526, "d06dece58e6c7345986aae4b7f15b3317653f5387d6262f389b5cbbe804568124a876eabb89204e96b3c0f7b552df3c4", "5c0e873adba65a9f4cb24cce4f194b18"),
(30303, "7a33c1268eafdc1f89ad460fa4ded8d3df9a3cabe4339706877878c64a2c8080cf3fa5ea7f2f24744e3341476b1eb5a5", "b7dc708fc46ce5cde24a31ad549fec83"),
(31080, "37bf1f9fca6d705b989b2d63259ca924dc860fc6027e07d9aad79b94841227739774f5d324590df45d8f41249ef742ea", "8ead50308c281e699b79b69dad7ecb91"),
(31857, "91b120c73be86f9d53326fa707cfa1411e5ac76ab998a2d7ebd73a75e3b1a04c9f0855d102184b8a3fd5d99818b0b134", "6056d09595bd16bfa317c6f87ce64bb7"),
(32634, "42cc255c06184ead57b27efd0cefb0f2c788c8962a6fd15db3f25533a7f49700bca85af916f9e985f1941a6e66943b38", "3b15e332d2f53bb97e1a9d03e6113b97"),
(33411, "737f8bb8f3fd03a9d13e50abba3a42f4491c36eda3eb215085abda733227ec490cb863ffbd68f915c8fb2926a899fbc3", "b2c647d25c46aab4d4a5ede4a3b4576d"),
(34188, "e9caa36505e19628175d1ce8b933267380099753a41e503fa2f894cea17b7692f0b27079ed33cdd1293db9a35722d561", "a2882adfd00f22823250215b12b3a1fd"),
(34965, "81ddc348ebbdfb963daa5d0c1b51bbb73cacd883d4fc4316db6bd3388779beff7be0655bbac73951f89dc53832199c11", "f33106eb8104f3780350c6d4f82333ad"),
(35742, "308ce31daf40dab707e2cb4c4a5307bc403e24c971ae1e30e998449f804a167fe5f2cf617d585851b6fe9f2b4209f09c", "44070ac90cbf350ab92289cc063e978c"),
(36519, "71f51b4bddbe8a52f18be75f9bdb3fca0773901b794de845450fb308c34775ede1a6da9a82b61e9682a29a3ef71274e2", "0e387704298c444bf3afba0edc0c1c1c"),
(37296, "478ac94eee8c5f96210003fcb478392b91f2ef6fc3a729774e5fe82a2d8d0abc54ae1d25b3eaefb061e2bd43b70ca4ea", "fb65ebeda52cd5848d303c0677cecb7f"),
(38073, "bc3a9390618da7d644be932627353e2c92024df939d2d8497fba61fae3dd822cdd3e130c1707f4a9d5d4a0cbb4b3e0b3", "d790d529a837ec79f7cc3f66ed9a399f"),
(38850, "ef0e63a53a10e56477c47e13320b8a7d330aee3a4363c850edc56c0707a2686478e5a5193f54ceb33467ab7e8a22aa21", "6f2c18742f106f16fc290767342fb62b"),
(39627, "c16f63533c099d872d9a01c326db7756e7eb488c756b9a6ebf575993d8ea2eb45c572b2e162f061e145710e0e21e8e18", "a57afde7938b223ae5e109a03db4ee4c"),
(40404, "ade484ae8c13465a73589ef14789bb6891c933453e198df84edd34b4ac5c83aa90f2cf61fa072fa4d8f5b5c4cd68fa9e", "a01d13009db86ac442f7afd39d83309f"),
(41181, "6c5c7eed0e043a0bd60bcac9b5b546e150028d70c1efefc9ff69037ef4dc1a36878b171b9f2a639df822d11054a0e405", "6321c8622ca5866c875d340206d06a28"),
(41958, "dd311c54222fb0d92858719cf5b1c51bb5e3ca2539ffd68f1dd6c7e38969495be935804855ccdcc4b4cf221fcdbda886", "cf401eb819b5dc5cd8c909aae9b3b34b"),
(42735, "31cda9d663199b32eff042dd16c0b909ba999641e77ba751c91752bfc4d595e17ec6467119e74a600b72da72ba287d0a", "12fd6298ab5d744eb6ade3106565afad"),
(43512, "11b014057d51a8384d549d5d083c4406b575df6a9295853dd8f2f84f078cc241bb90495a119126b10b9510efcb68c0d3", "a48a49eea5dc90359ef21f32132f8604"),
(44289, "b44f5dbeecd76ee7efe3fb4dfe10ba8135d7a5e4d104149f4a91c5c6ee9446d9be19fb4c9ba668b074466d3892e22228", "07e1cbb7a19174d9b1e4d5a2c741cc14"),
(45066, "d87bbba3a3c739cab622386c89aeb685a70009fab1a606bd34622adfa3a75a05b58d56ee6b9874d414db38a6a32927b3", "a27cd252712cd2a1a2d95dea39f888d4"),
(45843, "abb90e60ea13c6cb3b401b8e271637416b87fbede165dde7be1d34abe4427dae4b39b499352cacac909bc43fb94028c8", "df3ae762b9257936feda435a61a9c3a1"),
(46620, "56d1132ee6e0f85543950d2d9667244b66b0ce6414eacd1859b128ed0b9026b31a25bfdcce3d1a0ce7c39d99f609c89c", "cfe7c3c3f1cb615e2d210cc8136443e6"),
(47397, "ecb023ec4c23cf95d1848a38b359f1f590f172dee9d8fb1be6bc9c4fb2ce96f612d60d7b111de539ab8313a87b821176", "501d24752bf55cb12239863981898a07"),
(48174, "34236ab60f05bb510aa0880fec358fb2002903efa14c912cab8a399e09418f97223ca2f7b8d6798c11d39e79032eaaa8", "4ecaba4eae886aa429927188abab9623"),
(48951, "55e8b40fad90a3d8c85a0f4d5bcf5975b8a6e2fb78377109f5b607a5e367187fbbc9a1e978aab3228fbf43ad23d0ad13", "84c43bc30eb4a67230b6c634fe3c7782"),
(49728, "14b1f896d0d01ecff4e456c3c392b1ca2bad9f1ef07713f84cdd89e663aa27ca77d80213ed57a89431eb992b11d98749", "7f58c2f9a249f70fe1c6f9b4f65e5a1d"),
(50505, "1335b1fb56196e0b371fa53ab7445845fdefcea3eb2833478deb3526e2ec888945e95ee8239b52caae5b9920ba4f43bb", "5fd729126b236ce3e0686fc706dce20f"),
(51282, "0d1983a6cab870c5e78f89a11dd30e7d2c71a3882f8bba3e71dc1b96a2d9fc6cc6d91d683b74456b886de34df792cfda", "7731ae6e6c54dfde12f6116357e812ea"),
(52059, "9d619fb4aa8441baaefed7b778693c291f2c1441b206ec135930fac3529d26587ac36f4472949e0b198b51c0c5a9d0f8", "39db2c996aea28996e03d576c118630f"),
(52836, "31dca4fa285878ba3efc3b66a248a078b69a11c3c73f81077377c4ffcb7002627aad5faa955e3141c1d8508aad68c8f6", "32ac1e5a09e7e629ff95f30aa9b69c00"),
(53613, "931a9969cf2bb02302c32b1eecd4933805e2da403d85aaf98c82c68129fb95f089eb85c65a6fcbc7d81bedb39de0cabb", "1a6f54b87c12868da530eac94d99eb31"),
(54390, "2f0742565801a37810ecb3f50a6f782e73a369a790d1a6a85135e7ffa12fc063db8909ab9eca7cf7308832887a6149d1", "1b18ed6a8f901b7947626216839f0643"),
(55167, "901defbd308b54deef89acd0d94e4387b370f9d2e6f870d72da2e447ed3ebe69c5f9f144488bd6207a732102160bff47", "1e0e6a05fcc0794121f617e28cfac1a0"),
(55944, "df984a5f7475250155dc4733a746e98446dc93a56a3f3bff691ddfef7deefb32b1da1b0e7e15cce443831ebfb3e30ada", "876121af882d0ebeae38f111f3d4b6e8"),
(56721, "acb693ed837b33561408cb1eed636e0082ac404f3fd72d277fa146ae5cd81a1fde3645f4cdc7babd8ba044b78075cb67", "5b90ed6c7943fc6da623c536e2ff1352"),
(57498, "dffb54bf5938e812076cfbf15cd524d72a189566c7980363a49dd89fb49e230d9742ef0b0e1ac543dca14366d735d152", "22aee072457306e32747fbbbc3ae127c"),
(58275, "92dbc245a980fc78974f7a27e62c22b12a00be9d3ef8d3718ff85f6d5fbcbf1d9d1e0f0a3daeb8c2628d090550a0ff6b", "5fa348117faba4ac8c9d9317ff44cd2d"),
(59052, "57721475cb719691850696d9a8ad4c28ca8ef9a7d45874ca21df4df250cb87ea60c464f4e3252e2d6161ed36c4b56d75", "24d92ae7cac56d9c0276b06f7428d5df"),
(59829, "d0936026440b5276747cb9fb7dc96de5d4e7846c233ca5f6f9354b2b39f760333483cbe99ffa905facb347242f58a7ef", "05c57068e183f9d835e7f461202f923c"),
(60606, "7b3bb3527b73a8692f076f6a503b2e09b427119543c7812db73c7c7fb2d43af9ecbd2a8a1452ac8ada96ad0bad7bb185", "f958635a193fec0bfb958e97961381df"),
(61383, "ff0d00255a36747eced86acfccd0cf9ef09faa9f44c8cf382efec462e7ead66e562a971060c3f32798ba142d9e1640a2", "838159b222e56aadde8229ed56a14095"),
(62160, "15806e088ed1428cd73ede3fecf5b60e2a616f1925004dadd2cab8e847059f795659659e82a4554f270baf88bf60af63", "fed2aa0c9c0a73d499cc970aef21c52f"),
(62937, "cfad71b23b6da51256bd1ddbd1ac77977fe10b2ad0a830a23a794cef914bf71a9519d78a5f83fc411e8d8db996a45d4e", "e1ea412fd3e1bd91c24b6b6445e8ff43"),
(63714, "7d03a3698a79b1af1663e3e485c2efdc306ecd87b2644f2e01d83a35999d6cdf12241b6114d60d107c10c0d0c9cc0d23", "e6a3c3f3fd2d9cfcdc06cca2f59e9a83"),
(64491, "e12b168cce0e82ed1db88df549f39b3ff40b5884a09fceae69c4c3db13c1c37ea79531c47b2700d1c27774a1ab7e8b35", "4cbb14d789f5cd8eca49ce9e1d442ea1"),
(65268, "056c9d1172cfa76ce7f19c605e5969c284b82dca155dc9c1ed58062ab4d5a7704e27fe69f3aa745b73f45f1cd0ee57df", "8195187f092d52c2a8695b680568b934"),
(
0,
"38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b",
"43847e644239134deccf5538162c861e",
),
(
777,
"aabf892f18a620b9c3bae91bb03a74c84193e4a7b64916c6bc88b885b9ebed4134495e5f22f12e3046fbb3f26fa111a7",
"b8c318b5dcc1d672114a6f7be54ef289",
),
(
1554,
"648f551df29217f0e634b72ba6973c0eb95c7d4be8b135e550d8bcdf65b75980881bc0e03cf22589e04bedc7da1804cd",
"535b8ddd51ec82a1e850906fe321b21a",
),
(
2331,
"bfbfdffea40062e23bbdf0835e1d38d1623bebca7407908bbc6d5b3f2bfd062a2d237f091affda7348094fafda0bd1a7",
"4f521876fbb2c563051196b33c20c822",
),
(
3108,
"cc6035cab70f3a3298a5c4956ff07f179acf3771bb915c590a8a19fe5133d6d8a81c118148394dfb364af5c2fbdaadeb",
"d3adfa578c8bcd738c55ffc527358cef",
),
(
3885,
"15ec2760a21c25f9870a84ee757f3da2c261a950c2f692d75ff9e99b2d50c826c21e27e49c4cd3450fedc7e60371589f",
"a4c22d6c3d773634c2dc057e1f7c6738",
),
(
4662,
"c2afad6f034704300c34f143dcdcb86c9b954cec1ebf22e7071f288c58a2ae430d3e3748d214d1021472793d3f337dc6",
"c0601cb6cd4883102f70570c2cdc0ab6",
),
(
5439,
"8fee067f5a7a475a630f9db8b2eb80c1edc40eb4246a0f1c078e535df7d06451c6a9bde1a23ba70285690dd7100a8626",
"7352239f2302b08844309d28b13fa867",
),
(
6216,
"60095b4172438aee61e65f5379f4ef276c3632d4ac74eea7723a2201823432614aba7b4670d9bf7a5b9126ca38f3b88a",
"c0f0b0aa651965f8514b473c5406285e",
),
(
6993,
"10e754dd08b4d2a6c109fb01fce2b57d54743947e14a7e67d7efd0608baf91f7fc42a53328fe8c18d234abad8ebcdff0",
"58444988a62a99060728a7637c8499eb",
),
(
7770,
"1abc4a5dcd2696336bd0e8af20fe7fc261aa424b52cfb5ad80ee7c7c793ac44f11db3506cdbbbaed0f80000925d08d52",
"e8065c563bc6018cdcbf9aaafef767e6",
),
(
8547,
"26aaf74ae8bfc6aaf45ceee0476ea0a484304f5c36050d3e2265cb194a2f7c308213314232270608b6d3f1c11b834e33",
"ec50e4b3f6e4b3de24b3476623d08157",
),
(
9324,
"863206305d466aa9c0d0ec674572069f61fe5009767f99ec8832912725c28c49d6a106ad3f55372c922e4e169fc382ce",
"0cfac64f49e0f128d0a18d293878f222",
),
(
10101,
"bd0c0950b947a6c34f1fa6e877433b42c039a8ea7b37634c40fb47efae4958ba74ef0991cfedf3c82a0b87ef59635071",
"e0220a02b74259eeebbebede847d50f9",
),
(
10878,
"d7b9901af1dacf6a8c369b993ba1c607f9b7f073d02311c72d8449d3494d477ffc8344a1d8b488020ccfc7c80fbd27e1",
"ebe3933146734a6ade2b434f2bcd78ae",
),
(
11655,
"0ba265e3ef0bebf01a4f3490da462c7730aad6aa6c70bb9ce64a36d26d24fe213660e60e4d3301329170471f11ff8ca2",
"ec3dd4bf4cb7d527a86dd559c773a87b",
),
(
12432,
"c3b6755a1be922ec71c1e187ead36c4e6fc307c72969c64ca1e9b7339d61e1a93a74a315fd73bed8fa5797b78b19dbe5",
"5b58dcf392749bcef91056ba9475d0ef",
),
(
13209,
"2fb1a67151183daa2f0d7f0064534497357f173161349dd008499a8c1a123cc942662ecc426e2ad7743fe0ab9f5d7be1",
"c011260d328d310e2ab606aa1ef8afd4",
),
(
13986,
"6afae2a07ce9bfe30fbbfb7dcf32d755bcf357334dc5c309e58cab38ebe559f25b313a0b3ca32ff1dc41f7b99718f653",
"011bf43cfbbb7ae5986f8e0fc87771a9",
),
(
14763,
"cc6215c115eb6411f4712c2289f5bf0ccb5151635f9f9ceac7c1b62d8d2f4d26498079d0289f83aeb26e97b5b924ffc4",
"a015034a8d5bc83cc76c6983a5ba19ab",
),
(
15540,
"3cebce794e947341c4ceec444ca43c6ac57c6f58de462bfec7566cbd59a1b6f2eae774120e29521e76120a604d1a12d9",
"d373cd2bd9000655141ac632880eca40",
),
(
16317,
"899147b98d78bb5d137dc7c4f03be7eca82bcca19cc3a701261332923707aed2e6719d35d2f2bf067cd1d193a53529cf",
"ed223b64529299c787f49d631ce181c1",
),
(
17094,
"aecd1830958b994b2c331b90e7d8ff79f27c83a71f5797a65ade3a30b4fa5928e79140bcd03f375591d53df96fea1a4d",
"948a7c253d54bb6b65d78530c0eb7aab",
),
(
17871,
"e677ffd4ecaba5899659fefe5fe8e643004392be3be6dc5a801409870ac1e3398f47cc1d83f7a4c41925b6337e01f7fd",
"156a600c336f3ac034ca90034aa22635",
),
(
18648,
"4ee50f4a98d0bbd160add6acf76765ccdac0c1cd0bb2adbbcb22dd012a1121620b739a120df7dc4091e684ddf28eb726",
"75873467b416a7b025f9f1b015bf653a",
),
(
19425,
"aa025f32c0575af7209828fc7fc4591b41fa7cfb485e26c5401e63ca1fa05776f8b8af1769a15e81f2c663bca9b02ab3",
"5679efa7a4404e1e5c9b372782a41bf2",
),
(
20202,
"6e77ab62d2affeb27f4ef326191b3df3863c338a629f64a785505f4a5968ff59bc011c7a27951cb00e2e7d9b9bd32fec",
"36a9c4515d34f9bb962d8876ab3b5c86",
),
(
20979,
"1625b4f0e65fc66f11ba3ee6b3e20c732535654c447df6b517ced113107a1057a64477faa2af4a5ede4034bf3cff98ea",
"9058044e0f71c28d4f8d3281a3aec024",
),
(
21756,
"94efe6aa55bd77bfa58c185dec313a41003f9bef02568e72c337be4de1b46c6e5bb9a9329b4f108686489b8bc9d5f4f0",
"8d6d2c90590268a26f5e7d76351f48c1",
),
(
22533,
"7327a05fdb0ac92433dfc2c85c5e96e6ddcbdb01e079f8dafbee79c14cb4d5fd46047acd6bb0e09a98f6dd03dced2a0a",
"4e0f0a394f85bca35c68ef667aa9c244",
),
(
23310,
"93da9e356efbc8b5ae366256f4c6fc11c11fc347aaa879d591b7c1262d90adf98925f571914696054f1d09c74783561e",
"8c83c157be439280afc790ee3fd667eb",
),
(
24087,
"99b91be5ffca51b1cbc7410798b1540b5b1a3356f801ed4dc54812919c08ca5a9adc218bc51e594d97b46445a1515506",
"9436ff05729a77f673e815e464aeaa75",
),
(
24864,
"074253ad5d5a5d2b072e7aeaffa04a06119ec812a88ca43481fe5e2dce02cf6736952095cd342ec70b833c12fc1777f4",
"69d8951b96866a08efbb65f2bc31cfbc",
),
(
25641,
"c0a301f90597c05cf19e60c35378676764086b7156e455f4800347f8a6e733d644e4cc709fb9d95a9211f3e1e10c762a",
"3561c9802143c306ecc5e07e3b976d9e",
),
(
26418,
"3c839e59d945b841acb604e1b9ae3df36a291444ce0bcae336ee875beaf208bf10af7342b375429ecb92ec54d11a5907",
"3032ffdb8daee11b2e739132c6175615",
),
(
27195,
"3dc59b16603950dfc26a90bf036712eb088412e8de4d1b27c3fa6be6502ac12d89d194764fb53c3dc7d90fa696ba5a16",
"49436717edff7cd67c9a1be16d524f07",
),
(
27972,
"4fbc0d40ff13376b8ed5382890cdea337b4a0c9c31b477c4008d2ef8299bd5ab771ba70b1b4b743f8f7caa1f0164d1a1",
"64a9856a3bb81dc81ff1bc1025192dc9",
),
(
28749,
"6ab191aa6327f229cc94e8c7b1b7ee30bc723e6aeaf3050eb7d14cb491c3513254e9b19894c2b4f071d298401fd31945",
"101f2ffea60f246a3b57c4a530d67cf1",
),
(
29526,
"d06dece58e6c7345986aae4b7f15b3317653f5387d6262f389b5cbbe804568124a876eabb89204e96b3c0f7b552df3c4",
"5c0e873adba65a9f4cb24cce4f194b18",
),
(
30303,
"7a33c1268eafdc1f89ad460fa4ded8d3df9a3cabe4339706877878c64a2c8080cf3fa5ea7f2f24744e3341476b1eb5a5",
"b7dc708fc46ce5cde24a31ad549fec83",
),
(
31080,
"37bf1f9fca6d705b989b2d63259ca924dc860fc6027e07d9aad79b94841227739774f5d324590df45d8f41249ef742ea",
"8ead50308c281e699b79b69dad7ecb91",
),
(
31857,
"91b120c73be86f9d53326fa707cfa1411e5ac76ab998a2d7ebd73a75e3b1a04c9f0855d102184b8a3fd5d99818b0b134",
"6056d09595bd16bfa317c6f87ce64bb7",
),
(
32634,
"42cc255c06184ead57b27efd0cefb0f2c788c8962a6fd15db3f25533a7f49700bca85af916f9e985f1941a6e66943b38",
"3b15e332d2f53bb97e1a9d03e6113b97",
),
(
33411,
"737f8bb8f3fd03a9d13e50abba3a42f4491c36eda3eb215085abda733227ec490cb863ffbd68f915c8fb2926a899fbc3",
"b2c647d25c46aab4d4a5ede4a3b4576d",
),
(
34188,
"e9caa36505e19628175d1ce8b933267380099753a41e503fa2f894cea17b7692f0b27079ed33cdd1293db9a35722d561",
"a2882adfd00f22823250215b12b3a1fd",
),
(
34965,
"81ddc348ebbdfb963daa5d0c1b51bbb73cacd883d4fc4316db6bd3388779beff7be0655bbac73951f89dc53832199c11",
"f33106eb8104f3780350c6d4f82333ad",
),
(
35742,
"308ce31daf40dab707e2cb4c4a5307bc403e24c971ae1e30e998449f804a167fe5f2cf617d585851b6fe9f2b4209f09c",
"44070ac90cbf350ab92289cc063e978c",
),
(
36519,
"71f51b4bddbe8a52f18be75f9bdb3fca0773901b794de845450fb308c34775ede1a6da9a82b61e9682a29a3ef71274e2",
"0e387704298c444bf3afba0edc0c1c1c",
),
(
37296,
"478ac94eee8c5f96210003fcb478392b91f2ef6fc3a729774e5fe82a2d8d0abc54ae1d25b3eaefb061e2bd43b70ca4ea",
"fb65ebeda52cd5848d303c0677cecb7f",
),
(
38073,
"bc3a9390618da7d644be932627353e2c92024df939d2d8497fba61fae3dd822cdd3e130c1707f4a9d5d4a0cbb4b3e0b3",
"d790d529a837ec79f7cc3f66ed9a399f",
),
(
38850,
"ef0e63a53a10e56477c47e13320b8a7d330aee3a4363c850edc56c0707a2686478e5a5193f54ceb33467ab7e8a22aa21",
"6f2c18742f106f16fc290767342fb62b",
),
(
39627,
"c16f63533c099d872d9a01c326db7756e7eb488c756b9a6ebf575993d8ea2eb45c572b2e162f061e145710e0e21e8e18",
"a57afde7938b223ae5e109a03db4ee4c",
),
(
40404,
"ade484ae8c13465a73589ef14789bb6891c933453e198df84edd34b4ac5c83aa90f2cf61fa072fa4d8f5b5c4cd68fa9e",
"a01d13009db86ac442f7afd39d83309f",
),
(
41181,
"6c5c7eed0e043a0bd60bcac9b5b546e150028d70c1efefc9ff69037ef4dc1a36878b171b9f2a639df822d11054a0e405",
"6321c8622ca5866c875d340206d06a28",
),
(
41958,
"dd311c54222fb0d92858719cf5b1c51bb5e3ca2539ffd68f1dd6c7e38969495be935804855ccdcc4b4cf221fcdbda886",
"cf401eb819b5dc5cd8c909aae9b3b34b",
),
(
42735,
"31cda9d663199b32eff042dd16c0b909ba999641e77ba751c91752bfc4d595e17ec6467119e74a600b72da72ba287d0a",
"12fd6298ab5d744eb6ade3106565afad",
),
(
43512,
"11b014057d51a8384d549d5d083c4406b575df6a9295853dd8f2f84f078cc241bb90495a119126b10b9510efcb68c0d3",
"a48a49eea5dc90359ef21f32132f8604",
),
(
44289,
"b44f5dbeecd76ee7efe3fb4dfe10ba8135d7a5e4d104149f4a91c5c6ee9446d9be19fb4c9ba668b074466d3892e22228",
"07e1cbb7a19174d9b1e4d5a2c741cc14",
),
(
45066,
"d87bbba3a3c739cab622386c89aeb685a70009fab1a606bd34622adfa3a75a05b58d56ee6b9874d414db38a6a32927b3",
"a27cd252712cd2a1a2d95dea39f888d4",
),
(
45843,
"abb90e60ea13c6cb3b401b8e271637416b87fbede165dde7be1d34abe4427dae4b39b499352cacac909bc43fb94028c8",
"df3ae762b9257936feda435a61a9c3a1",
),
(
46620,
"56d1132ee6e0f85543950d2d9667244b66b0ce6414eacd1859b128ed0b9026b31a25bfdcce3d1a0ce7c39d99f609c89c",
"cfe7c3c3f1cb615e2d210cc8136443e6",
),
(
47397,
"ecb023ec4c23cf95d1848a38b359f1f590f172dee9d8fb1be6bc9c4fb2ce96f612d60d7b111de539ab8313a87b821176",
"501d24752bf55cb12239863981898a07",
),
(
48174,
"34236ab60f05bb510aa0880fec358fb2002903efa14c912cab8a399e09418f97223ca2f7b8d6798c11d39e79032eaaa8",
"4ecaba4eae886aa429927188abab9623",
),
(
48951,
"55e8b40fad90a3d8c85a0f4d5bcf5975b8a6e2fb78377109f5b607a5e367187fbbc9a1e978aab3228fbf43ad23d0ad13",
"84c43bc30eb4a67230b6c634fe3c7782",
),
(
49728,
"14b1f896d0d01ecff4e456c3c392b1ca2bad9f1ef07713f84cdd89e663aa27ca77d80213ed57a89431eb992b11d98749",
"7f58c2f9a249f70fe1c6f9b4f65e5a1d",
),
(
50505,
"1335b1fb56196e0b371fa53ab7445845fdefcea3eb2833478deb3526e2ec888945e95ee8239b52caae5b9920ba4f43bb",
"5fd729126b236ce3e0686fc706dce20f",
),
(
51282,
"0d1983a6cab870c5e78f89a11dd30e7d2c71a3882f8bba3e71dc1b96a2d9fc6cc6d91d683b74456b886de34df792cfda",
"7731ae6e6c54dfde12f6116357e812ea",
),
(
52059,
"9d619fb4aa8441baaefed7b778693c291f2c1441b206ec135930fac3529d26587ac36f4472949e0b198b51c0c5a9d0f8",
"39db2c996aea28996e03d576c118630f",
),
(
52836,
"31dca4fa285878ba3efc3b66a248a078b69a11c3c73f81077377c4ffcb7002627aad5faa955e3141c1d8508aad68c8f6",
"32ac1e5a09e7e629ff95f30aa9b69c00",
),
(
53613,
"931a9969cf2bb02302c32b1eecd4933805e2da403d85aaf98c82c68129fb95f089eb85c65a6fcbc7d81bedb39de0cabb",
"1a6f54b87c12868da530eac94d99eb31",
),
(
54390,
"2f0742565801a37810ecb3f50a6f782e73a369a790d1a6a85135e7ffa12fc063db8909ab9eca7cf7308832887a6149d1",
"1b18ed6a8f901b7947626216839f0643",
),
(
55167,
"901defbd308b54deef89acd0d94e4387b370f9d2e6f870d72da2e447ed3ebe69c5f9f144488bd6207a732102160bff47",
"1e0e6a05fcc0794121f617e28cfac1a0",
),
(
55944,
"df984a5f7475250155dc4733a746e98446dc93a56a3f3bff691ddfef7deefb32b1da1b0e7e15cce443831ebfb3e30ada",
"876121af882d0ebeae38f111f3d4b6e8",
),
(
56721,
"acb693ed837b33561408cb1eed636e0082ac404f3fd72d277fa146ae5cd81a1fde3645f4cdc7babd8ba044b78075cb67",
"5b90ed6c7943fc6da623c536e2ff1352",
),
(
57498,
"dffb54bf5938e812076cfbf15cd524d72a189566c7980363a49dd89fb49e230d9742ef0b0e1ac543dca14366d735d152",
"22aee072457306e32747fbbbc3ae127c",
),
(
58275,
"92dbc245a980fc78974f7a27e62c22b12a00be9d3ef8d3718ff85f6d5fbcbf1d9d1e0f0a3daeb8c2628d090550a0ff6b",
"5fa348117faba4ac8c9d9317ff44cd2d",
),
(
59052,
"57721475cb719691850696d9a8ad4c28ca8ef9a7d45874ca21df4df250cb87ea60c464f4e3252e2d6161ed36c4b56d75",
"24d92ae7cac56d9c0276b06f7428d5df",
),
(
59829,
"d0936026440b5276747cb9fb7dc96de5d4e7846c233ca5f6f9354b2b39f760333483cbe99ffa905facb347242f58a7ef",
"05c57068e183f9d835e7f461202f923c",
),
(
60606,
"7b3bb3527b73a8692f076f6a503b2e09b427119543c7812db73c7c7fb2d43af9ecbd2a8a1452ac8ada96ad0bad7bb185",
"f958635a193fec0bfb958e97961381df",
),
(
61383,
"ff0d00255a36747eced86acfccd0cf9ef09faa9f44c8cf382efec462e7ead66e562a971060c3f32798ba142d9e1640a2",
"838159b222e56aadde8229ed56a14095",
),
(
62160,
"15806e088ed1428cd73ede3fecf5b60e2a616f1925004dadd2cab8e847059f795659659e82a4554f270baf88bf60af63",
"fed2aa0c9c0a73d499cc970aef21c52f",
),
(
62937,
"cfad71b23b6da51256bd1ddbd1ac77977fe10b2ad0a830a23a794cef914bf71a9519d78a5f83fc411e8d8db996a45d4e",
"e1ea412fd3e1bd91c24b6b6445e8ff43",
),
(
63714,
"7d03a3698a79b1af1663e3e485c2efdc306ecd87b2644f2e01d83a35999d6cdf12241b6114d60d107c10c0d0c9cc0d23",
"e6a3c3f3fd2d9cfcdc06cca2f59e9a83",
),
(
64491,
"e12b168cce0e82ed1db88df549f39b3ff40b5884a09fceae69c4c3db13c1c37ea79531c47b2700d1c27774a1ab7e8b35",
"4cbb14d789f5cd8eca49ce9e1d442ea1",
),
(
65268,
"056c9d1172cfa76ce7f19c605e5969c284b82dca155dc9c1ed58062ab4d5a7704e27fe69f3aa745b73f45f1cd0ee57df",
"8195187f092d52c2a8695b680568b934",
),
];
fn to_hex(b: &[u8]) -> String {
@ -156,8 +499,12 @@ mod tests {
/// Test repeated encrypt/decrypt and run a benchmark. Run with --nocapture to see it.
#[test]
fn encrypt_decrypt() {
let aes_key_0: [u8; 32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32];
let aes_key_1: [u8; 32] = [2, 3, 4, 5, 6, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32];
let aes_key_0: [u8; 32] = [
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
];
let aes_key_1: [u8; 32] = [
2, 3, 4, 5, 6, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
];
let iv: [u8; 8] = [0, 1, 2, 3, 4, 5, 6, 7];
let mut buf = [0_u8; 12345];
@ -205,7 +552,10 @@ mod tests {
let _ = c.encrypt_second_pass_finish();
}
let duration = SystemTime::now().duration_since(start).unwrap();
println!("AES-GMAC-SIV (legacy) encrypt benchmark: {} MiB/sec", (((benchmark_iterations * buf.len()) as f64) / 1048576.0) / duration.as_secs_f64());
println!(
"AES-GMAC-SIV (legacy) encrypt benchmark: {} MiB/sec",
(((benchmark_iterations * buf.len()) as f64) / 1048576.0) / duration.as_secs_f64()
);
let start = SystemTime::now();
for _ in 0..benchmark_iterations {
c.reset();
@ -214,6 +564,9 @@ mod tests {
c.decrypt_finish();
}
let duration = SystemTime::now().duration_since(start).unwrap();
println!("AES-GMAC-SIV (legacy) decrypt benchmark: {} MiB/sec", (((benchmark_iterations * buf.len()) as f64) / 1048576.0) / duration.as_secs_f64());
println!(
"AES-GMAC-SIV (legacy) decrypt benchmark: {} MiB/sec",
(((benchmark_iterations * buf.len()) as f64) / 1048576.0) / duration.as_secs_f64()
);
}
}

196
crypto/src/p384.rs
View file

@ -1,6 +1,14 @@
// (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md.
#![allow(dead_code, mutable_transmutes, non_camel_case_types, non_snake_case, non_upper_case_globals, unused_assignments, unused_mut)]
#![allow(
dead_code,
mutable_transmutes,
non_camel_case_types,
non_snake_case,
non_upper_case_globals,
unused_assignments,
unused_mut
)]
pub const P384_PUBLIC_KEY_SIZE: usize = 49;
pub const P384_SECRET_KEY_SIZE: usize = 48;
@ -116,7 +124,8 @@ mod builtin {
/* Returns nonzero if bit p_bit of p_vli is set. */
unsafe fn vli_testBit(mut p_vli: *mut uint64_t, mut p_bit: uint) -> uint64_t {
return *p_vli.offset(p_bit.wrapping_div(64 as libc::c_int as libc::c_uint) as isize) & (1 as libc::c_int as uint64_t) << p_bit.wrapping_rem(64 as libc::c_int as libc::c_uint);
return *p_vli.offset(p_bit.wrapping_div(64 as libc::c_int as libc::c_uint) as isize)
& (1 as libc::c_int as uint64_t) << p_bit.wrapping_rem(64 as libc::c_int as libc::c_uint);
}
/* Counts the number of 64-bit "digits" in p_vli. */
@ -215,7 +224,9 @@ mod builtin {
let mut i: uint = 0;
i = 0 as libc::c_int as uint;
while i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint {
let mut l_sum: uint64_t = (*p_left.offset(i as isize)).wrapping_add(*p_right.offset(i as isize)).wrapping_add(l_carry);
let mut l_sum: uint64_t = (*p_left.offset(i as isize))
.wrapping_add(*p_right.offset(i as isize))
.wrapping_add(l_carry);
if l_sum != *p_left.offset(i as isize) {
l_carry = (l_sum < *p_left.offset(i as isize)) as libc::c_int as uint64_t
}
@ -231,7 +242,9 @@ mod builtin {
let mut i: uint = 0;
i = 0 as libc::c_int as uint;
while i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint {
let mut l_diff: uint64_t = (*p_left.offset(i as isize)).wrapping_sub(*p_right.offset(i as isize)).wrapping_sub(l_borrow);
let mut l_diff: uint64_t = (*p_left.offset(i as isize))
.wrapping_sub(*p_right.offset(i as isize))
.wrapping_sub(l_borrow);
if l_diff != *p_left.offset(i as isize) {
l_borrow = (l_diff > *p_left.offset(i as isize)) as libc::c_int as uint64_t
}
@ -253,11 +266,13 @@ mod builtin {
let mut l_min: uint = if k < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint {
0 as libc::c_int as libc::c_uint
} else {
k.wrapping_add(1 as libc::c_int as libc::c_uint).wrapping_sub((48 as libc::c_int / 8 as libc::c_int) as libc::c_uint)
k.wrapping_add(1 as libc::c_int as libc::c_uint)
.wrapping_sub((48 as libc::c_int / 8 as libc::c_int) as libc::c_uint)
};
i = l_min;
while i <= k && i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint {
let mut l_product: uint128_t = (*p_left.offset(i as isize) as uint128_t).wrapping_mul(*p_right.offset(k.wrapping_sub(i) as isize) as u128);
let mut l_product: uint128_t =
(*p_left.offset(i as isize) as uint128_t).wrapping_mul(*p_right.offset(k.wrapping_sub(i) as isize) as u128);
r01 = (r01 as u128).wrapping_add(l_product) as uint128_t as uint128_t;
r2 = (r2 as libc::c_ulong).wrapping_add((r01 < l_product) as libc::c_int as libc::c_ulong) as uint64_t as uint64_t;
i = i.wrapping_add(1)
@ -281,11 +296,13 @@ mod builtin {
let mut l_min: uint = if k < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint {
0 as libc::c_int as libc::c_uint
} else {
k.wrapping_add(1 as libc::c_int as libc::c_uint).wrapping_sub((48 as libc::c_int / 8 as libc::c_int) as libc::c_uint)
k.wrapping_add(1 as libc::c_int as libc::c_uint)
.wrapping_sub((48 as libc::c_int / 8 as libc::c_int) as libc::c_uint)
};
i = l_min;
while i <= k && i <= k.wrapping_sub(i) {
let mut l_product: uint128_t = (*p_left.offset(i as isize) as uint128_t).wrapping_mul(*p_left.offset(k.wrapping_sub(i) as isize) as u128);
let mut l_product: uint128_t =
(*p_left.offset(i as isize) as uint128_t).wrapping_mul(*p_left.offset(k.wrapping_sub(i) as isize) as u128);
if i < k.wrapping_sub(i) {
r2 = (r2 as u128).wrapping_add(l_product >> 127 as libc::c_int) as uint64_t as uint64_t;
l_product = (l_product as u128).wrapping_mul(2 as libc::c_int as u128) as uint128_t as uint128_t
@ -333,9 +350,16 @@ mod builtin {
/* Multiply by (2^128 + 2^96 - 2^32 + 1). */
vli_set(p_result, p_right); /* 1 */
l_carry = vli_lshift(l_tmp.as_mut_ptr(), p_right, 32 as libc::c_int as uint); /* 2^96 + 1 */
*p_result.offset((1 as libc::c_int + 48 as libc::c_int / 8 as libc::c_int) as isize) =
l_carry.wrapping_add(vli_add(p_result.offset(1 as libc::c_int as isize), p_result.offset(1 as libc::c_int as isize), l_tmp.as_mut_ptr())); /* 2^128 + 2^96 + 1 */
*p_result.offset((2 as libc::c_int + 48 as libc::c_int / 8 as libc::c_int) as isize) = vli_add(p_result.offset(2 as libc::c_int as isize), p_result.offset(2 as libc::c_int as isize), p_right); /* 2^128 + 2^96 - 2^32 + 1 */
*p_result.offset((1 as libc::c_int + 48 as libc::c_int / 8 as libc::c_int) as isize) = l_carry.wrapping_add(vli_add(
p_result.offset(1 as libc::c_int as isize),
p_result.offset(1 as libc::c_int as isize),
l_tmp.as_mut_ptr(),
)); /* 2^128 + 2^96 + 1 */
*p_result.offset((2 as libc::c_int + 48 as libc::c_int / 8 as libc::c_int) as isize) = vli_add(
p_result.offset(2 as libc::c_int as isize),
p_result.offset(2 as libc::c_int as isize),
p_right,
); /* 2^128 + 2^96 - 2^32 + 1 */
l_carry = (l_carry as libc::c_ulong).wrapping_add(vli_sub(p_result, p_result, l_tmp.as_mut_ptr())) as uint64_t as uint64_t;
l_diff = (*p_result.offset((48 as libc::c_int / 8 as libc::c_int) as isize)).wrapping_sub(l_carry);
if l_diff > *p_result.offset((48 as libc::c_int / 8 as libc::c_int) as isize) {
@ -365,12 +389,17 @@ mod builtin {
let mut i: uint = 0; /* p = c0 */
vli_clear(l_tmp.as_mut_ptr());
vli_clear(l_tmp.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize));
omega_mult(l_tmp.as_mut_ptr(), p_product.offset((48 as libc::c_int / 8 as libc::c_int) as isize));
omega_mult(
l_tmp.as_mut_ptr(),
p_product.offset((48 as libc::c_int / 8 as libc::c_int) as isize),
);
vli_clear(p_product.offset((48 as libc::c_int / 8 as libc::c_int) as isize));
/* (c1, c0) = c0 + w * c1 */
i = 0 as libc::c_int as uint;
while i < (48 as libc::c_int / 8 as libc::c_int + 3 as libc::c_int) as libc::c_uint {
let mut l_sum: uint64_t = (*p_product.offset(i as isize)).wrapping_add(l_tmp[i as usize]).wrapping_add(l_carry);
let mut l_sum: uint64_t = (*p_product.offset(i as isize))
.wrapping_add(l_tmp[i as usize])
.wrapping_add(l_carry);
if l_sum != *p_product.offset(i as isize) {
l_carry = (l_sum < *p_product.offset(i as isize)) as libc::c_int as uint64_t
}
@ -432,7 +461,8 @@ mod builtin {
vli_rshift1(u.as_mut_ptr());
if l_carry != 0 {
u[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] =
(u[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong | 0x8000000000000000 as libc::c_ulonglong) as uint64_t
(u[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong
| 0x8000000000000000 as libc::c_ulonglong) as uint64_t
}
} else if b[0 as libc::c_int as usize] & 1 as libc::c_int as libc::c_ulong == 0 {
vli_rshift1(b.as_mut_ptr());
@ -442,7 +472,8 @@ mod builtin {
vli_rshift1(v.as_mut_ptr());
if l_carry != 0 {
v[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] =
(v[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong | 0x8000000000000000 as libc::c_ulonglong) as uint64_t
(v[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong
| 0x8000000000000000 as libc::c_ulonglong) as uint64_t
}
} else if l_cmpResult > 0 as libc::c_int {
vli_sub(a.as_mut_ptr(), a.as_mut_ptr(), b.as_mut_ptr());
@ -457,7 +488,8 @@ mod builtin {
vli_rshift1(u.as_mut_ptr());
if l_carry != 0 {
u[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] =
(u[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong | 0x8000000000000000 as libc::c_ulonglong) as uint64_t
(u[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong
| 0x8000000000000000 as libc::c_ulonglong) as uint64_t
}
} else {
vli_sub(b.as_mut_ptr(), b.as_mut_ptr(), a.as_mut_ptr());
@ -472,7 +504,8 @@ mod builtin {
vli_rshift1(v.as_mut_ptr());
if l_carry != 0 {
v[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] =
(v[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong | 0x8000000000000000 as libc::c_ulonglong) as uint64_t
(v[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] as libc::c_ulonglong
| 0x8000000000000000 as libc::c_ulonglong) as uint64_t
}
}
}
@ -538,7 +571,13 @@ mod builtin {
}
/* P = (x1, y1) => 2P, (x2, y2) => P' */
unsafe fn XYcZ_initial_double(mut X1: *mut uint64_t, mut Y1: *mut uint64_t, mut X2: *mut uint64_t, mut Y2: *mut uint64_t, mut p_initialZ: *mut uint64_t) {
unsafe fn XYcZ_initial_double(
mut X1: *mut uint64_t,
mut Y1: *mut uint64_t,
mut X2: *mut uint64_t,
mut Y2: *mut uint64_t,
mut p_initialZ: *mut uint64_t,
) {
let mut z: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init();
vli_set(X2, X1);
vli_set(Y2, Y1);
@ -606,7 +645,12 @@ mod builtin {
vli_set(X1, t7.as_mut_ptr());
}
unsafe fn EccPoint_mult(mut p_result: *mut EccPoint, mut p_point: *mut EccPoint, mut p_scalar: *mut uint64_t, mut p_initialZ: *mut uint64_t) {
unsafe fn EccPoint_mult(
mut p_result: *mut EccPoint,
mut p_point: *mut EccPoint,
mut p_scalar: *mut uint64_t,
mut p_initialZ: *mut uint64_t,
) {
/* R0 and R1 */
let mut Rx: [[uint64_t; 6]; 2] = std::mem::MaybeUninit::uninit().assume_init();
let mut Ry: [[uint64_t; 6]; 2] = std::mem::MaybeUninit::uninit().assume_init();
@ -665,7 +709,11 @@ mod builtin {
Rx[(1 as libc::c_int - nb) as usize].as_mut_ptr(),
Ry[(1 as libc::c_int - nb) as usize].as_mut_ptr(),
);
apply_z(Rx[0 as libc::c_int as usize].as_mut_ptr(), Ry[0 as libc::c_int as usize].as_mut_ptr(), z.as_mut_ptr());
apply_z(
Rx[0 as libc::c_int as usize].as_mut_ptr(),
Ry[0 as libc::c_int as usize].as_mut_ptr(),
z.as_mut_ptr(),
);
vli_set((*p_result).x.as_mut_ptr(), Rx[0 as libc::c_int as usize].as_mut_ptr());
vli_set((*p_result).y.as_mut_ptr(), Ry[0 as libc::c_int as usize].as_mut_ptr());
}
@ -674,8 +722,11 @@ mod builtin {
let mut i: libc::c_uint = 0;
i = 0 as libc::c_int as libc::c_uint;
while i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint {
let mut p_digit: *const uint8_t =
p_bytes.offset((8 as libc::c_int as libc::c_uint).wrapping_mul(((48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as libc::c_uint).wrapping_sub(i)) as isize);
let mut p_digit: *const uint8_t = p_bytes.offset(
(8 as libc::c_int as libc::c_uint)
.wrapping_mul(((48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as libc::c_uint).wrapping_sub(i))
as isize,
);
*p_native.offset(i as isize) = (*p_digit.offset(0 as libc::c_int as isize) as uint64_t) << 56 as libc::c_int
| (*p_digit.offset(1 as libc::c_int as isize) as uint64_t) << 48 as libc::c_int
| (*p_digit.offset(2 as libc::c_int as isize) as uint64_t) << 40 as libc::c_int
@ -692,8 +743,11 @@ mod builtin {
let mut i: libc::c_uint = 0;
i = 0 as libc::c_int as libc::c_uint;
while i < (48 as libc::c_int / 8 as libc::c_int) as libc::c_uint {
let mut p_digit: *mut uint8_t =
p_bytes.offset((8 as libc::c_int as libc::c_uint).wrapping_mul(((48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as libc::c_uint).wrapping_sub(i)) as isize);
let mut p_digit: *mut uint8_t = p_bytes.offset(
(8 as libc::c_int as libc::c_uint)
.wrapping_mul(((48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as libc::c_uint).wrapping_sub(i))
as isize,
);
*p_digit.offset(0 as libc::c_int as isize) = (*p_native.offset(i as isize) >> 56 as libc::c_int) as uint8_t;
*p_digit.offset(1 as libc::c_int as isize) = (*p_native.offset(i as isize) >> 48 as libc::c_int) as uint8_t;
*p_digit.offset(2 as libc::c_int as isize) = (*p_native.offset(i as isize) >> 40 as libc::c_int) as uint8_t;
@ -730,11 +784,23 @@ mod builtin {
let mut _3: [uint64_t; 6] = [3 as libc::c_int as uint64_t, 0, 0, 0, 0, 0];
ecc_bytes2native((*p_point).x.as_mut_ptr(), p_compressed.offset(1 as libc::c_int as isize));
vli_modSquare_fast((*p_point).y.as_mut_ptr(), (*p_point).x.as_mut_ptr());
vli_modSub((*p_point).y.as_mut_ptr(), (*p_point).y.as_mut_ptr(), _3.as_mut_ptr(), curve_p.as_mut_ptr());
vli_modSub(
(*p_point).y.as_mut_ptr(),
(*p_point).y.as_mut_ptr(),
_3.as_mut_ptr(),
curve_p.as_mut_ptr(),
);
vli_modMult_fast((*p_point).y.as_mut_ptr(), (*p_point).y.as_mut_ptr(), (*p_point).x.as_mut_ptr());
vli_modAdd((*p_point).y.as_mut_ptr(), (*p_point).y.as_mut_ptr(), curve_b.as_mut_ptr(), curve_p.as_mut_ptr());
vli_modAdd(
(*p_point).y.as_mut_ptr(),
(*p_point).y.as_mut_ptr(),
curve_b.as_mut_ptr(),
curve_p.as_mut_ptr(),
);
mod_sqrt((*p_point).y.as_mut_ptr());
if (*p_point).y[0 as libc::c_int as usize] & 0x1 as libc::c_int as libc::c_ulong != (*p_compressed.offset(0 as libc::c_int as isize) as libc::c_int & 0x1 as libc::c_int) as libc::c_ulong {
if (*p_point).y[0 as libc::c_int as usize] & 0x1 as libc::c_int as libc::c_ulong
!= (*p_compressed.offset(0 as libc::c_int as isize) as libc::c_int & 0x1 as libc::c_int) as libc::c_ulong
{
vli_sub((*p_point).y.as_mut_ptr(), curve_p.as_mut_ptr(), (*p_point).y.as_mut_ptr());
};
}
@ -763,11 +829,20 @@ mod builtin {
}
}
ecc_native2bytes(p_privateKey, l_private.as_mut_ptr() as *const uint64_t);
ecc_native2bytes(p_publicKey.offset(1 as libc::c_int as isize), l_public.x.as_mut_ptr() as *const uint64_t);
*p_publicKey.offset(0 as libc::c_int as isize) = (2 as libc::c_int as libc::c_ulong).wrapping_add(l_public.y[0 as libc::c_int as usize] & 0x1 as libc::c_int as libc::c_ulong) as uint8_t;
ecc_native2bytes(
p_publicKey.offset(1 as libc::c_int as isize),
l_public.x.as_mut_ptr() as *const uint64_t,
);
*p_publicKey.offset(0 as libc::c_int as isize) = (2 as libc::c_int as libc::c_ulong)
.wrapping_add(l_public.y[0 as libc::c_int as usize] & 0x1 as libc::c_int as libc::c_ulong)
as uint8_t;
return 1 as libc::c_int;
}
pub unsafe fn ecdh_shared_secret(mut p_publicKey: *const uint8_t, mut p_privateKey: *const uint8_t, mut p_secret: *mut uint8_t) -> libc::c_int {
pub unsafe fn ecdh_shared_secret(
mut p_publicKey: *const uint8_t,
mut p_privateKey: *const uint8_t,
mut p_secret: *mut uint8_t,
) -> libc::c_int {
let mut l_public: EccPoint = std::mem::MaybeUninit::uninit().assume_init();
let mut l_private: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init();
let mut l_random: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init();
@ -794,7 +869,9 @@ mod builtin {
vli_mult(l_product.as_mut_ptr(), p_left, p_right);
l_productBits = vli_numBits(l_product.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize));
if l_productBits != 0 {
l_productBits = (l_productBits as libc::c_uint).wrapping_add((48 as libc::c_int / 8 as libc::c_int * 64 as libc::c_int) as libc::c_uint) as uint as uint
l_productBits = (l_productBits as libc::c_uint)
.wrapping_add((48 as libc::c_int / 8 as libc::c_int * 64 as libc::c_int) as libc::c_uint)
as uint as uint
} else {
l_productBits = vli_numBits(l_product.as_mut_ptr())
}
@ -807,8 +884,12 @@ mod builtin {
power of two possible while still resulting in a number less than p_left. */
vli_clear(l_modMultiple.as_mut_ptr());
vli_clear(l_modMultiple.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize));
l_digitShift = l_productBits.wrapping_sub(l_modBits).wrapping_div(64 as libc::c_int as libc::c_uint);
l_bitShift = l_productBits.wrapping_sub(l_modBits).wrapping_rem(64 as libc::c_int as libc::c_uint);
l_digitShift = l_productBits
.wrapping_sub(l_modBits)
.wrapping_div(64 as libc::c_int as libc::c_uint);
l_bitShift = l_productBits
.wrapping_sub(l_modBits)
.wrapping_rem(64 as libc::c_int as libc::c_uint);
if l_bitShift != 0 {
l_modMultiple[l_digitShift.wrapping_add((48 as libc::c_int / 8 as libc::c_int) as libc::c_uint) as usize] =
vli_lshift(l_modMultiple.as_mut_ptr().offset(l_digitShift as isize), p_mod, l_bitShift)
@ -818,12 +899,16 @@ mod builtin {
/* Subtract all multiples of p_mod to get the remainder. */
vli_clear(p_result); /* Use p_result as a temp var to store 1 (for subtraction) */
*p_result.offset(0 as libc::c_int as isize) = 1 as libc::c_int as uint64_t;
while l_productBits > (48 as libc::c_int / 8 as libc::c_int * 64 as libc::c_int) as libc::c_uint || vli_cmp(l_modMultiple.as_mut_ptr(), p_mod) >= 0 as libc::c_int {
while l_productBits > (48 as libc::c_int / 8 as libc::c_int * 64 as libc::c_int) as libc::c_uint
|| vli_cmp(l_modMultiple.as_mut_ptr(), p_mod) >= 0 as libc::c_int
{
let mut l_cmp: libc::c_int = vli_cmp(
l_modMultiple.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize),
l_product.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize),
);
if l_cmp < 0 as libc::c_int || l_cmp == 0 as libc::c_int && vli_cmp(l_modMultiple.as_mut_ptr(), l_product.as_mut_ptr()) <= 0 as libc::c_int {
if l_cmp < 0 as libc::c_int
|| l_cmp == 0 as libc::c_int && vli_cmp(l_modMultiple.as_mut_ptr(), l_product.as_mut_ptr()) <= 0 as libc::c_int
{
if vli_sub(l_product.as_mut_ptr(), l_product.as_mut_ptr(), l_modMultiple.as_mut_ptr()) != 0 {
/* borrow */
vli_sub(
@ -838,7 +923,8 @@ mod builtin {
l_modMultiple.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize),
);
}
let mut l_carry: uint64_t = (l_modMultiple[(48 as libc::c_int / 8 as libc::c_int) as usize] & 0x1 as libc::c_int as libc::c_ulong) << 63 as libc::c_int;
let mut l_carry: uint64_t =
(l_modMultiple[(48 as libc::c_int / 8 as libc::c_int) as usize] & 0x1 as libc::c_int as libc::c_ulong) << 63 as libc::c_int;
vli_rshift1(l_modMultiple.as_mut_ptr().offset((48 as libc::c_int / 8 as libc::c_int) as isize));
vli_rshift1(l_modMultiple.as_mut_ptr());
l_modMultiple[(48 as libc::c_int / 8 as libc::c_int - 1 as libc::c_int) as usize] |= l_carry;
@ -890,7 +976,11 @@ mod builtin {
ecc_native2bytes(p_signature.offset(48 as libc::c_int as isize), l_s.as_mut_ptr() as *const uint64_t);
return 1 as libc::c_int;
}
pub unsafe fn ecdsa_verify(mut p_publicKey: *const uint8_t, mut p_hash: *const uint8_t, mut p_signature: *const uint8_t) -> libc::c_int {
pub unsafe fn ecdsa_verify(
mut p_publicKey: *const uint8_t,
mut p_hash: *const uint8_t,
mut p_signature: *const uint8_t,
) -> libc::c_int {
let mut u1: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init();
let mut u2: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init();
let mut z: [uint64_t; 6] = std::mem::MaybeUninit::uninit().assume_init();
@ -910,7 +1000,9 @@ mod builtin {
/* r, s must not be 0. */
return 0 as libc::c_int;
}
if vli_cmp(curve_n.as_mut_ptr(), l_r.as_mut_ptr()) != 1 as libc::c_int || vli_cmp(curve_n.as_mut_ptr(), l_s.as_mut_ptr()) != 1 as libc::c_int {
if vli_cmp(curve_n.as_mut_ptr(), l_r.as_mut_ptr()) != 1 as libc::c_int
|| vli_cmp(curve_n.as_mut_ptr(), l_s.as_mut_ptr()) != 1 as libc::c_int
{
/* r, s must be < n. */
return 0 as libc::c_int;
}
@ -931,9 +1023,10 @@ mod builtin {
/* Use Shamir's trick to calculate u1*G + u2*Q */
let mut l_points: [*mut EccPoint; 4] = [0 as *mut EccPoint, &mut curve_G, &mut l_public, &mut l_sum]; /* Z = x2 - x1 */
let mut l_numBits: uint = umax(vli_numBits(u1.as_mut_ptr()), vli_numBits(u2.as_mut_ptr())); /* Z = 1/Z */
let mut l_point: *mut EccPoint = l_points[((vli_testBit(u1.as_mut_ptr(), l_numBits.wrapping_sub(1 as libc::c_int as libc::c_uint)) != 0) as libc::c_int
| ((vli_testBit(u2.as_mut_ptr(), l_numBits.wrapping_sub(1 as libc::c_int as libc::c_uint)) != 0) as libc::c_int) << 1 as libc::c_int)
as usize];
let mut l_point: *mut EccPoint = l_points[((vli_testBit(u1.as_mut_ptr(), l_numBits.wrapping_sub(1 as libc::c_int as libc::c_uint))
!= 0) as libc::c_int
| ((vli_testBit(u2.as_mut_ptr(), l_numBits.wrapping_sub(1 as libc::c_int as libc::c_uint)) != 0) as libc::c_int)
<< 1 as libc::c_int) as usize];
vli_set(rx.as_mut_ptr(), (*l_point).x.as_mut_ptr());
vli_set(ry.as_mut_ptr(), (*l_point).y.as_mut_ptr());
vli_clear(z.as_mut_ptr());
@ -942,7 +1035,8 @@ mod builtin {
i = l_numBits.wrapping_sub(2 as libc::c_int as libc::c_uint) as libc::c_int;
while i >= 0 as libc::c_int {
EccPoint_double_jacobian(rx.as_mut_ptr(), ry.as_mut_ptr(), z.as_mut_ptr());
let mut l_index: libc::c_int = (vli_testBit(u1.as_mut_ptr(), i as uint) != 0) as libc::c_int | ((vli_testBit(u2.as_mut_ptr(), i as uint) != 0) as libc::c_int) << 1 as libc::c_int;
let mut l_index: libc::c_int = (vli_testBit(u1.as_mut_ptr(), i as uint) != 0) as libc::c_int
| ((vli_testBit(u2.as_mut_ptr(), i as uint) != 0) as libc::c_int) << 1 as libc::c_int;
let mut l_point_0: *mut EccPoint = l_points[l_index as usize];
if !l_point_0.is_null() {
vli_set(tx.as_mut_ptr(), (*l_point_0).x.as_mut_ptr());
@ -1002,7 +1096,10 @@ mod builtin {
pub fn from_bytes(public_bytes: &[u8], secret_bytes: &[u8]) -> Option<P384KeyPair> {
if public_bytes.len() == 49 && secret_bytes.len() == 48 {
Some(Self(P384PublicKey(public_bytes.try_into().unwrap()), Secret(secret_bytes.try_into().unwrap())))
Some(Self(
P384PublicKey(public_bytes.try_into().unwrap()),
Secret(secret_bytes.try_into().unwrap()),
))
} else {
None
}
@ -1080,7 +1177,9 @@ mod openssl_based {
impl P384PublicKey {
fn new_from_point(key: &EcPointRef) -> Self {
let mut bnc = BigNumContext::new().unwrap();
let kb = key.to_bytes(GROUP_P384.as_ref(), PointConversionForm::COMPRESSED, &mut bnc).unwrap();
let kb = key
.to_bytes(GROUP_P384.as_ref(), PointConversionForm::COMPRESSED, &mut bnc)
.unwrap();
let mut bytes = [0_u8; 49];
bytes[(49 - kb.len())..].copy_from_slice(kb.as_slice());
Self {
@ -1219,7 +1318,14 @@ mod openssl_based {
pub fn agree(&self, other_public: &P384PublicKey) -> Option<Secret<P384_ECDH_SHARED_SECRET_SIZE>> {
unsafe {
let mut s: Secret<P384_ECDH_SHARED_SECRET_SIZE> = Secret::default();
if ECDH_compute_key(s.0.as_mut_ptr().cast(), 48, other_public.key.public_key().as_ptr().cast(), self.pair.as_ptr().cast(), null()) == 48 {
if ECDH_compute_key(
s.0.as_mut_ptr().cast(),
48,
other_public.key.public_key().as_ptr().cast(),
self.pair.as_ptr().cast(),
null(),
) == 48
{
Some(s)
} else {
None

22
crypto/src/poly1305.rs
View file

@ -10,7 +10,10 @@ pub const POLY1305_MAC_SIZE: usize = 16;
#[inline(always)]
pub fn compute(one_time_key: &[u8], message: &[u8]) -> [u8; POLY1305_MAC_SIZE] {
poly1305::Poly1305::new(poly1305::Key::from_slice(one_time_key)).compute_unpadded(message).into_bytes().into()
poly1305::Poly1305::new(poly1305::Key::from_slice(one_time_key))
.compute_unpadded(message)
.into_bytes()
.into()
}
#[cfg(test)]
@ -18,18 +21,25 @@ mod tests {
use crate::poly1305::*;
const TV0_INPUT: [u8; 32] = [
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
];
const TV0_KEY: [u8; 32] = [
0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35,
0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f,
0x72, 0x20, 0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35,
];
const TV0_TAG: [u8; 16] = [
0x49, 0xec, 0x78, 0x09, 0x0e, 0x48, 0x1e, 0xc6, 0xc2, 0x6b, 0x33, 0xb9, 0x1c, 0xcc, 0x03, 0x07,
];
const TV0_TAG: [u8; 16] = [0x49, 0xec, 0x78, 0x09, 0x0e, 0x48, 0x1e, 0xc6, 0xc2, 0x6b, 0x33, 0xb9, 0x1c, 0xcc, 0x03, 0x07];
const TV1_INPUT: [u8; 12] = [0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21];
const TV1_KEY: [u8; 32] = [
0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35,
0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20, 0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f,
0x72, 0x20, 0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35,
];
const TV1_TAG: [u8; 16] = [
0xa6, 0xf7, 0x45, 0x00, 0x8f, 0x81, 0xc9, 0x16, 0xa2, 0x0d, 0xcc, 0x74, 0xee, 0xf2, 0xb2, 0xf0,
];
const TV1_TAG: [u8; 16] = [0xa6, 0xf7, 0x45, 0x00, 0x8f, 0x81, 0xc9, 0x16, 0xa2, 0x0d, 0xcc, 0x74, 0xee, 0xf2, 0xb2, 0xf0];
#[test]
fn poly1305() {

42
crypto/src/salsa.rs
View file

@ -3,7 +3,12 @@
use std::convert::TryInto;
use std::ptr::{slice_from_raw_parts, slice_from_raw_parts_mut};
const CONSTANTS: [u32; 4] = [u32::from_le_bytes(*b"expa"), u32::from_le_bytes(*b"nd 3"), u32::from_le_bytes(*b"2-by"), u32::from_le_bytes(*b"te k")];
const CONSTANTS: [u32; 4] = [
u32::from_le_bytes(*b"expa"),
u32::from_le_bytes(*b"nd 3"),
u32::from_le_bytes(*b"2-by"),
u32::from_le_bytes(*b"te k"),
];
/// Salsa stream cipher implementation supporting 8, 12, or 20 rounds.
///
@ -65,8 +70,24 @@ impl<const ROUNDS: usize> Salsa<ROUNDS> {
);
while !plaintext.is_empty() {
let (mut x0, mut x1, mut x2, mut x3, mut x4, mut x5, mut x6, mut x7, mut x8, mut x9, mut x10, mut x11, mut x12, mut x13, mut x14, mut x15) =
(j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15);
let (
mut x0,
mut x1,
mut x2,
mut x3,
mut x4,
mut x5,
mut x6,
mut x7,
mut x8,
mut x9,
mut x10,
mut x11,
mut x12,
mut x13,
mut x14,
mut x15,
) = (j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15);
for _ in 0..(ROUNDS / 2) {
x4 ^= x0.wrapping_add(x12).rotate_left(7);
@ -206,7 +227,12 @@ impl<const ROUNDS: usize> Salsa<ROUNDS> {
#[inline(always)]
pub fn crypt_in_place(&mut self, data: &mut [u8]) {
unsafe { self.crypt(&*slice_from_raw_parts(data.as_ptr(), data.len()), &mut *slice_from_raw_parts_mut(data.as_mut_ptr(), data.len())) }
unsafe {
self.crypt(
&*slice_from_raw_parts(data.as_ptr(), data.len()),
&mut *slice_from_raw_parts_mut(data.as_mut_ptr(), data.len()),
)
}
}
}
@ -215,12 +241,14 @@ mod tests {
use crate::salsa::*;
const SALSA_20_TV0_KEY: [u8; 32] = [
0x0f, 0x62, 0xb5, 0x08, 0x5b, 0xae, 0x01, 0x54, 0xa7, 0xfa, 0x4d, 0xa0, 0xf3, 0x46, 0x99, 0xec, 0x3f, 0x92, 0xe5, 0x38, 0x8b, 0xde, 0x31, 0x84, 0xd7, 0x2a, 0x7d, 0xd0, 0x23, 0x76, 0xc9, 0x1c,
0x0f, 0x62, 0xb5, 0x08, 0x5b, 0xae, 0x01, 0x54, 0xa7, 0xfa, 0x4d, 0xa0, 0xf3, 0x46, 0x99, 0xec, 0x3f, 0x92, 0xe5, 0x38, 0x8b, 0xde,
0x31, 0x84, 0xd7, 0x2a, 0x7d, 0xd0, 0x23, 0x76, 0xc9, 0x1c,
];
const SALSA_20_TV0_IV: [u8; 8] = [0x28, 0x8f, 0xf6, 0x5d, 0xc4, 0x2b, 0x92, 0xf9];
const SALSA_20_TV0_KS: [u8; 64] = [
0x5e, 0x5e, 0x71, 0xf9, 0x01, 0x99, 0x34, 0x03, 0x04, 0xab, 0xb2, 0x2a, 0x37, 0xb6, 0x62, 0x5b, 0xf8, 0x83, 0xfb, 0x89, 0xce, 0x3b, 0x21, 0xf5, 0x4a, 0x10, 0xb8, 0x10, 0x66, 0xef, 0x87, 0xda,
0x30, 0xb7, 0x76, 0x99, 0xaa, 0x73, 0x79, 0xda, 0x59, 0x5c, 0x77, 0xdd, 0x59, 0x54, 0x2d, 0xa2, 0x08, 0xe5, 0x95, 0x4f, 0x89, 0xe4, 0x0e, 0xb7, 0xaa, 0x80, 0xa8, 0x4a, 0x61, 0x76, 0x66, 0x3f,
0x5e, 0x5e, 0x71, 0xf9, 0x01, 0x99, 0x34, 0x03, 0x04, 0xab, 0xb2, 0x2a, 0x37, 0xb6, 0x62, 0x5b, 0xf8, 0x83, 0xfb, 0x89, 0xce, 0x3b,
0x21, 0xf5, 0x4a, 0x10, 0xb8, 0x10, 0x66, 0xef, 0x87, 0xda, 0x30, 0xb7, 0x76, 0x99, 0xaa, 0x73, 0x79, 0xda, 0x59, 0x5c, 0x77, 0xdd,
0x59, 0x54, 0x2d, 0xa2, 0x08, 0xe5, 0x95, 0x4f, 0x89, 0xe4, 0x0e, 0xb7, 0xaa, 0x80, 0xa8, 0x4a, 0x61, 0x76, 0x66, 0x3f,
];
#[test]

6
crypto/src/x25519.rs
View file

@ -84,7 +84,11 @@ impl X25519KeyPair {
impl Clone for X25519KeyPair {
fn clone(&self) -> Self {
Self(x25519_dalek::StaticSecret::from(self.0.to_bytes()), self.1.clone(), x25519_dalek::PublicKey::from(self.1 .0.clone()))
Self(
x25519_dalek::StaticSecret::from(self.0.to_bytes()),
self.1.clone(),
x25519_dalek::PublicKey::from(self.1 .0.clone()),
)
}
}

265
crypto/src/zssp.rs
View file

@ -117,8 +117,9 @@ const KBKDF_KEY_USAGE_LABEL_RATCHETING: u8 = b'R';
const INITIAL_KEY: [u8; 64] = [
// macOS command line to generate:
// echo -n 'ZSSP_Noise_IKpsk2_NISTP384_?KYBER1024_AESGCM_SHA512' | shasum -a 512 | cut -d ' ' -f 1 | xxd -r -p | xxd -i
0x35, 0x6a, 0x75, 0xc0, 0xbf, 0xbe, 0xc3, 0x59, 0x70, 0x94, 0x50, 0x69, 0x4c, 0xa2, 0x08, 0x40, 0xc7, 0xdf, 0x67, 0xa8, 0x68, 0x52, 0x6e, 0xd5, 0xdd, 0x77, 0xec, 0x59, 0x6f, 0x8e, 0xa1, 0x99,
0xb4, 0x32, 0x85, 0xaf, 0x7f, 0x0d, 0xa9, 0x6c, 0x01, 0xfb, 0x72, 0x46, 0xc0, 0x09, 0x58, 0xb8, 0xe0, 0xa8, 0xcf, 0xb1, 0x58, 0x04, 0x6e, 0x32, 0xba, 0xa8, 0xb8, 0xf9, 0x0a, 0xa4, 0xbf, 0x36,
0x35, 0x6a, 0x75, 0xc0, 0xbf, 0xbe, 0xc3, 0x59, 0x70, 0x94, 0x50, 0x69, 0x4c, 0xa2, 0x08, 0x40, 0xc7, 0xdf, 0x67, 0xa8, 0x68, 0x52,
0x6e, 0xd5, 0xdd, 0x77, 0xec, 0x59, 0x6f, 0x8e, 0xa1, 0x99, 0xb4, 0x32, 0x85, 0xaf, 0x7f, 0x0d, 0xa9, 0x6c, 0x01, 0xfb, 0x72, 0x46,
0xc0, 0x09, 0x58, 0xb8, 0xe0, 0xa8, 0xcf, 0xb1, 0x58, 0x04, 0x6e, 0x32, 0xba, 0xa8, 0xb8, 0xf9, 0x0a, 0xa4, 0xbf, 0x36,
];
pub enum Error {
@ -232,7 +233,8 @@ impl SessionId {
pub fn new_from_reader<R: Read>(r: &mut R) -> std::io::Result<Option<SessionId>> {
let mut tmp = [0_u8; 8];
r.read_exact(&mut tmp[..SESSION_ID_SIZE]).map(|_| NonZeroU64::new(u64::from_le_bytes(tmp)).map(|i| Self(i)))
r.read_exact(&mut tmp[..SESSION_ID_SIZE])
.map(|_| NonZeroU64::new(u64::from_le_bytes(tmp)).map(|i| Self(i)))
}
}
@ -278,7 +280,12 @@ pub trait Host: Sized {
/// On success a tuple of local session ID, static secret, and associated object is returned. The
/// static secret is whatever results from agreement between the local and remote static public
/// keys.
fn accept_new_session(&self, receive_context: &ReceiveContext<Self>, remote_static_public: &[u8], remote_metadata: &[u8]) -> Option<(SessionId, Secret<64>, Self::AssociatedObject)>;
fn accept_new_session(
&self,
receive_context: &ReceiveContext<Self>,
remote_static_public: &[u8],
remote_metadata: &[u8],
) -> Option<(SessionId, Secret<64>, Self::AssociatedObject)>;
}
/// ZSSP bi-directional packet transport channel.
@ -338,7 +345,8 @@ impl<H: Host> Session<H> {
let send_counter = Counter::new();
let header_check_cipher = Aes::new(kbkdf512(ss.as_bytes(), KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>());
let remote_s_public_hash = SHA384::hash(remote_s_public);
let outgoing_init_header_check_cipher = Aes::new(kbkdf512(&remote_s_public_hash, KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>());
let outgoing_init_header_check_cipher =
Aes::new(kbkdf512(&remote_s_public_hash, KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>());
if let Ok(offer) = create_initial_offer(
&mut send,
send_counter.next(),
@ -381,7 +389,12 @@ impl<H: Host> Session<H> {
///
/// * `mtu_buffer` - A writable work buffer whose size must be equal to the wire MTU
/// * `data` - Data to send
pub fn send<SendFunction: FnMut(&mut [u8])>(&self, mut send: SendFunction, mtu_buffer: &mut [u8], mut data: &[u8]) -> Result<(), Error> {
pub fn send<SendFunction: FnMut(&mut [u8])>(
&self,
mut send: SendFunction,
mtu_buffer: &mut [u8],
mut data: &[u8],
) -> Result<(), Error> {
debug_assert!(mtu_buffer.len() >= MIN_MTU);
let state = self.state.read();
if let Some(remote_session_id) = state.remote_session_id {
@ -389,7 +402,14 @@ impl<H: Host> Session<H> {
let mut packet_len = data.len() + HEADER_SIZE + AES_GCM_TAG_SIZE;
let counter = self.send_counter.next();
create_packet_header(mtu_buffer, packet_len, mtu_buffer.len(), PACKET_TYPE_DATA, remote_session_id.into(), counter)?;
create_packet_header(
mtu_buffer,
packet_len,
mtu_buffer.len(),
PACKET_TYPE_DATA,
remote_session_id.into(),
counter,
)?;
let mut c = key.get_send_cipher(counter)?;
c.init(memory::as_byte_array::<Pseudoheader, 12>(&Pseudoheader::make(
@ -461,13 +481,24 @@ impl<H: Host> Session<H> {
/// * `mtu` - Physical MTU for sent packets
/// * `current_time` - Current monotonic time in milliseconds
/// * `force_rekey` - Re-key the session now regardless of key aging (still subject to rate limiting)
pub fn service<SendFunction: FnMut(&mut [u8])>(&self, host: &H, mut send: SendFunction, offer_metadata: &[u8], mtu: usize, current_time: i64, force_rekey: bool) {
pub fn service<SendFunction: FnMut(&mut [u8])>(
&self,
host: &H,
mut send: SendFunction,
offer_metadata: &[u8],
mtu: usize,
current_time: i64,
force_rekey: bool,
) {
let state = self.state.upgradable_read();
if (force_rekey
|| state.keys[state.key_ptr]
.as_ref()
.map_or(true, |key| key.lifetime.should_rekey(self.send_counter.current(), current_time)))
&& state.offer.as_ref().map_or(true, |o| (current_time - o.creation_time) > OFFER_RATE_LIMIT_MS)
&& state
.offer
.as_ref()
.map_or(true, |o| (current_time - o.creation_time) > OFFER_RATE_LIMIT_MS)
{
if let Some(remote_s_public_p384) = P384PublicKey::from_bytes(&self.remote_s_public_p384) {
let mut tmp_header_check_cipher = None;
@ -485,7 +516,9 @@ impl<H: Host> Session<H> {
if state.remote_session_id.is_some() {
&self.header_check_cipher
} else {
let _ = tmp_header_check_cipher.insert(Aes::new(kbkdf512(&self.remote_s_public_hash, KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>()));
let _ = tmp_header_check_cipher.insert(Aes::new(
kbkdf512(&self.remote_s_public_hash, KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>(),
));
tmp_header_check_cipher.as_ref().unwrap()
},
mtu,
@ -502,7 +535,9 @@ impl<H: Host> ReceiveContext<H> {
pub fn new(host: &H) -> Self {
Self {
initial_offer_defrag: Mutex::new(RingBufferMap::new(random::xorshift64_random() as u32)),
incoming_init_header_check_cipher: Aes::new(kbkdf512(host.get_local_s_public_hash(), KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>()),
incoming_init_header_check_cipher: Aes::new(
kbkdf512(host.get_local_s_public_hash(), KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>(),
),
}
}
@ -529,7 +564,9 @@ impl<H: Host> ReceiveContext<H> {
if let Some(local_session_id) = SessionId::new_from_u64(memory::u64_from_le_bytes(incoming_packet) & SessionId::MAX_BIT_MASK) {
if let Some(session) = host.session_lookup(local_session_id) {
if let Some((packet_type, fragment_count, fragment_no, counter)) = dearmor_header(incoming_packet, &session.header_check_cipher) {
if let Some((packet_type, fragment_count, fragment_no, counter)) =
dearmor_header(incoming_packet, &session.header_check_cipher)
{
let pseudoheader = Pseudoheader::make(u64::from(local_session_id), packet_type, counter);
if fragment_count > 1 {
if fragment_count <= (MAX_FRAGMENTS as u8) && fragment_no < fragment_count {
@ -576,7 +613,9 @@ impl<H: Host> ReceiveContext<H> {
}
} else {
unlikely_branch();
if let Some((packet_type, fragment_count, fragment_no, counter)) = dearmor_header(incoming_packet, &self.incoming_init_header_check_cipher) {
if let Some((packet_type, fragment_count, fragment_no, counter)) =
dearmor_header(incoming_packet, &self.incoming_init_header_check_cipher)
{
let pseudoheader = Pseudoheader::make(0, packet_type, counter);
if fragment_count > 1 {
let mut defrag = self.initial_offer_defrag.lock();
@ -670,13 +709,20 @@ impl<H: Host> ReceiveContext<H> {
key.return_receive_cipher(c);
return Err(Error::DataBufferTooSmall);
}
c.crypt(&tail[HEADER_SIZE..(tail.len() - AES_GCM_TAG_SIZE)], &mut data_buf[current_frag_data_start..data_len]);
c.crypt(
&tail[HEADER_SIZE..(tail.len() - AES_GCM_TAG_SIZE)],
&mut data_buf[current_frag_data_start..data_len],
);
let ok = c.finish_decrypt(&tail[(tail.len() - AES_GCM_TAG_SIZE)..]);
key.return_receive_cipher(c);
if ok {
// Select this key as the new default if it's newer than the current key.
if p > 0 && state.keys[state.key_ptr].as_ref().map_or(true, |old| old.establish_counter < key.establish_counter) {
if p > 0
&& state.keys[state.key_ptr]
.as_ref()
.map_or(true, |old| old.establish_counter < key.establish_counter)
{
drop(state);
let mut state = session.state.write();
state.key_ptr = key_ptr;
@ -741,7 +787,13 @@ impl<H: Host> ReceiveContext<H> {
let hmac1_end = incoming_packet_len - HMAC_SIZE;
// Check that the sender knows this host's identity before doing anything else.
if !hmac_sha384_2(host.get_local_s_public_hash(), pseudoheader, &incoming_packet[HEADER_SIZE..hmac1_end]).eq(&incoming_packet[hmac1_end..]) {
if !hmac_sha384_2(
host.get_local_s_public_hash(),
pseudoheader,
&incoming_packet[HEADER_SIZE..hmac1_end],
)
.eq(&incoming_packet[hmac1_end..])
{
return Err(Error::FailedAuthentication);
}
@ -755,15 +807,19 @@ impl<H: Host> ReceiveContext<H> {
}
// Key agreement: alice (remote) ephemeral NIST P-384 <> local static NIST P-384
let (alice_e0_public, e0s) = P384PublicKey::from_bytes(&incoming_packet[(HEADER_SIZE + 1)..(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)])
.and_then(|pk| host.get_local_s_keypair_p384().agree(&pk).map(move |s| (pk, s)))
.ok_or(Error::FailedAuthentication)?;
let (alice_e0_public, e0s) =
P384PublicKey::from_bytes(&incoming_packet[(HEADER_SIZE + 1)..(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)])
.and_then(|pk| host.get_local_s_keypair_p384().agree(&pk).map(move |s| (pk, s)))
.ok_or(Error::FailedAuthentication)?;
// Initial key derivation from starting point, mixing in alice's ephemeral public and the e0s.
let mut key = Secret(hmac_sha512(&hmac_sha512(&INITIAL_KEY, alice_e0_public.as_bytes()), e0s.as_bytes()));
// Decrypt the encrypted part of the packet payload and authenticate the above key exchange via AES-GCM auth.
let mut c = AesGcm::new(kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_ALICE_TO_BOB).first_n::<32>(), false);
let mut c = AesGcm::new(
kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_ALICE_TO_BOB).first_n::<32>(),
false,
);
c.init(pseudoheader);
c.crypt_in_place(&mut incoming_packet[(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)..payload_end]);
if !c.finish_decrypt(&incoming_packet[payload_end..aes_gcm_tag_end]) {
@ -812,7 +868,10 @@ impl<H: Host> ReceiveContext<H> {
let alice_s_public_p384 = H::extract_p384_static(alice_s_public).ok_or(Error::InvalidPacket)?;
// Key agreement: both sides' static P-384 keys.
let ss = host.get_local_s_keypair_p384().agree(&alice_s_public_p384).ok_or(Error::FailedAuthentication)?;
let ss = host
.get_local_s_keypair_p384()
.agree(&alice_s_public_p384)
.ok_or(Error::FailedAuthentication)?;
// Mix result of 'ss' agreement into master key.
key = Secret(hmac_sha512(key.as_bytes(), ss.as_bytes()));
@ -843,7 +902,9 @@ impl<H: Host> ReceiveContext<H> {
let new_session = if session.is_some() {
None
} else {
if let Some((new_session_id, psk, associated_object)) = host.accept_new_session(self, alice_s_public, alice_metadata) {
if let Some((new_session_id, psk, associated_object)) =
host.accept_new_session(self, alice_s_public, alice_metadata)
{
let header_check_cipher = Aes::new(kbkdf512(ss.as_bytes(), KBKDF_KEY_USAGE_LABEL_HEADER_CHECK).first_n::<16>());
Some(Session::<H> {
id: new_session_id,
@ -877,7 +938,10 @@ impl<H: Host> ReceiveContext<H> {
// fixed size so this shouldn't matter cryptographically.
key = Secret(hmac_sha512(
session.psk.as_bytes(),
&hmac_sha512(&hmac_sha512(&hmac_sha512(key.as_bytes(), bob_e0_keypair.public_key_bytes()), e0e0.as_bytes()), se0.as_bytes()),
&hmac_sha512(
&hmac_sha512(&hmac_sha512(key.as_bytes(), bob_e0_keypair.public_key_bytes()), e0e0.as_bytes()),
se0.as_bytes(),
),
));
// At this point we've completed Noise_IK key derivation with NIST P-384 ECDH, but now for hybrid and ratcheting...
@ -922,12 +986,23 @@ impl<H: Host> ReceiveContext<H> {
REPLY_BUF_LEN - rp.len()
};
create_packet_header(&mut reply_buf, reply_len, mtu, PACKET_TYPE_KEY_COUNTER_OFFER, alice_session_id.into(), reply_counter)?;
let reply_pseudoheader = Pseudoheader::make(alice_session_id.into(), PACKET_TYPE_KEY_COUNTER_OFFER, reply_counter.to_u32());
create_packet_header(
&mut reply_buf,
reply_len,
mtu,
PACKET_TYPE_KEY_COUNTER_OFFER,
alice_session_id.into(),
reply_counter,
)?;
let reply_pseudoheader =
Pseudoheader::make(alice_session_id.into(), PACKET_TYPE_KEY_COUNTER_OFFER, reply_counter.to_u32());
// Encrypt reply packet using final Noise_IK key BEFORE mixing hybrid or ratcheting, since the other side
// must decrypt before doing these things.
let mut c = AesGcm::new(kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_BOB_TO_ALICE).first_n::<32>(), true);
let mut c = AesGcm::new(
kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_BOB_TO_ALICE).first_n::<32>(),
true,
);
c.init(memory::as_byte_array::<Pseudoheader, 12>(&reply_pseudoheader));
c.crypt_in_place(&mut reply_buf[(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)..reply_len]);
let c = c.finish_encrypt();
@ -985,17 +1060,27 @@ impl<H: Host> ReceiveContext<H> {
if let Some(session) = session {
let state = session.state.upgradable_read();
if let Some(offer) = state.offer.as_ref() {
let (bob_e0_public, e0e0) = P384PublicKey::from_bytes(&incoming_packet[(HEADER_SIZE + 1)..(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)])
.and_then(|pk| offer.alice_e0_keypair.agree(&pk).map(move |s| (pk, s)))
let (bob_e0_public, e0e0) =
P384PublicKey::from_bytes(&incoming_packet[(HEADER_SIZE + 1)..(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)])
.and_then(|pk| offer.alice_e0_keypair.agree(&pk).map(move |s| (pk, s)))
.ok_or(Error::FailedAuthentication)?;
let se0 = host
.get_local_s_keypair_p384()
.agree(&bob_e0_public)
.ok_or(Error::FailedAuthentication)?;
let se0 = host.get_local_s_keypair_p384().agree(&bob_e0_public).ok_or(Error::FailedAuthentication)?;
let mut key = Secret(hmac_sha512(
session.psk.as_bytes(),
&hmac_sha512(&hmac_sha512(&hmac_sha512(offer.key.as_bytes(), bob_e0_public.as_bytes()), e0e0.as_bytes()), se0.as_bytes()),
&hmac_sha512(
&hmac_sha512(&hmac_sha512(offer.key.as_bytes(), bob_e0_public.as_bytes()), e0e0.as_bytes()),
se0.as_bytes(),
),
));
let mut c = AesGcm::new(kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_BOB_TO_ALICE).first_n::<32>(), false);
let mut c = AesGcm::new(
kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_BOB_TO_ALICE).first_n::<32>(),
false,
);
c.init(pseudoheader);
c.crypt_in_place(&mut incoming_packet[(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)..payload_end]);
if !c.finish_decrypt(&incoming_packet[payload_end..aes_gcm_tag_end]) {
@ -1046,10 +1131,21 @@ impl<H: Host> ReceiveContext<H> {
let key = SessionKey::new(key, Role::Alice, current_time, counter, ratchet_count + 1, e1e1.is_some());
let mut reply_buf = [0_u8; HEADER_SIZE + AES_GCM_TAG_SIZE];
create_packet_header(&mut reply_buf, HEADER_SIZE + AES_GCM_TAG_SIZE, mtu, PACKET_TYPE_NOP, bob_session_id.into(), counter)?;
create_packet_header(
&mut reply_buf,
HEADER_SIZE + AES_GCM_TAG_SIZE,
mtu,
PACKET_TYPE_NOP,
bob_session_id.into(),
counter,
)?;
let mut c = key.get_send_cipher(counter)?;
c.init(memory::as_byte_array::<Pseudoheader, 12>(&Pseudoheader::make(bob_session_id.into(), PACKET_TYPE_NOP, counter.to_u32())));
c.init(memory::as_byte_array::<Pseudoheader, 12>(&Pseudoheader::make(
bob_session_id.into(),
PACKET_TYPE_NOP,
counter.to_u32(),
)));
reply_buf[HEADER_SIZE..].copy_from_slice(&c.finish_encrypt());
key.return_send_cipher(c);
@ -1204,10 +1300,16 @@ fn create_initial_offer<SendFunction: FnMut(&mut [u8])>(
create_packet_header(&mut packet_buf, packet_len, mtu, PACKET_TYPE_KEY_OFFER, bob_session_id, counter)?;
let pseudoheader = Pseudoheader::make(bob_session_id, PACKET_TYPE_KEY_OFFER, counter.to_u32());
let key = Secret(hmac_sha512(&hmac_sha512(&INITIAL_KEY, alice_e0_keypair.public_key_bytes()), e0s.unwrap().as_bytes()));
let key = Secret(hmac_sha512(
&hmac_sha512(&INITIAL_KEY, alice_e0_keypair.public_key_bytes()),
e0s.unwrap().as_bytes(),
));
let gcm_tag = {
let mut c = AesGcm::new(kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_ALICE_TO_BOB).first_n::<32>(), true);
let mut c = AesGcm::new(
kbkdf512(key.as_bytes(), KBKDF_KEY_USAGE_LABEL_AES_GCM_ALICE_TO_BOB).first_n::<32>(),
true,
);
c.init(memory::as_byte_array::<Pseudoheader, 12>(&pseudoheader));
c.crypt_in_place(&mut packet_buf[(HEADER_SIZE + 1 + P384_PUBLIC_KEY_SIZE)..packet_len]);
c.finish_encrypt()
@ -1225,7 +1327,11 @@ fn create_initial_offer<SendFunction: FnMut(&mut [u8])>(
packet_buf[packet_len..(packet_len + HMAC_SIZE)].copy_from_slice(&hmac);
packet_len += HMAC_SIZE;
let hmac = hmac_sha384_2(bob_s_public_hash, memory::as_byte_array::<Pseudoheader, 12>(&pseudoheader), &packet_buf[HEADER_SIZE..packet_len]);
let hmac = hmac_sha384_2(
bob_s_public_hash,
memory::as_byte_array::<Pseudoheader, 12>(&pseudoheader),
&packet_buf[HEADER_SIZE..packet_len],
);
packet_buf[packet_len..(packet_len + HMAC_SIZE)].copy_from_slice(&hmac);
packet_len += HMAC_SIZE;
@ -1243,7 +1349,14 @@ fn create_initial_offer<SendFunction: FnMut(&mut [u8])>(
}
#[inline(always)]
fn create_packet_header(header: &mut [u8], packet_len: usize, mtu: usize, packet_type: u8, recipient_session_id: u64, counter: CounterValue) -> Result<(), Error> {
fn create_packet_header(
header: &mut [u8],
packet_len: usize,
mtu: usize,
packet_type: u8,
recipient_session_id: u64,
counter: CounterValue,
) -> Result<(), Error> {
let fragment_count = ((packet_len as f32) / (mtu - HEADER_SIZE) as f32).ceil() as usize;
debug_assert!(header.len() >= HEADER_SIZE);
@ -1255,7 +1368,9 @@ fn create_packet_header(header: &mut [u8], packet_len: usize, mtu: usize, packet
debug_assert!(recipient_session_id <= 0xffffffffffff); // session ID is 48 bits
if fragment_count <= MAX_FRAGMENTS {
header[0..8].copy_from_slice(&(recipient_session_id | (packet_type as u64).wrapping_shl(48) | ((fragment_count - 1) as u64).wrapping_shl(52)).to_le_bytes());
header[0..8].copy_from_slice(
&(recipient_session_id | (packet_type as u64).wrapping_shl(48) | ((fragment_count - 1) as u64).wrapping_shl(52)).to_le_bytes(),
);
header[8..12].copy_from_slice(&counter.to_u32().to_le_bytes());
header[12..16].fill(0);
Ok(())
@ -1265,7 +1380,12 @@ fn create_packet_header(header: &mut [u8], packet_len: usize, mtu: usize, packet
}
}
fn send_with_fragmentation<SendFunction: FnMut(&mut [u8])>(send: &mut SendFunction, packet: &mut [u8], mtu: usize, header_check_cipher: &Aes) {
fn send_with_fragmentation<SendFunction: FnMut(&mut [u8])>(
send: &mut SendFunction,
packet: &mut [u8],
mtu: usize,
header_check_cipher: &Aes,
) {
let packet_len = packet.len();
let mut fragment_start = 0;
let mut fragment_end = packet_len.min(mtu);
@ -1294,7 +1414,10 @@ fn armor_header(packet: &mut [u8], header_check_cipher: &Aes) {
debug_assert!(packet.len() >= MIN_PACKET_SIZE);
let mut header_pad = 0u128.to_ne_bytes();
header_check_cipher.encrypt_block(&packet[16..32], &mut header_pad);
packet[SESSION_ID_SIZE..HEADER_SIZE].iter_mut().zip(header_pad.iter()).for_each(|(x, y)| *x ^= *y);
packet[SESSION_ID_SIZE..HEADER_SIZE]
.iter_mut()
.zip(header_pad.iter())
.for_each(|(x, y)| *x ^= *y);
}
/// Dearmor the armored part of the header and return it if the 32-bit MAC matches.
@ -1326,7 +1449,10 @@ fn dearmor_header(packet: &[u8], header_check_cipher: &Aes) -> Option<(u8, u8, u
}
/// Parse KEY_OFFER and KEY_COUNTER_OFFER starting after the unencrypted public key part.
fn parse_key_offer_after_header(incoming_packet: &[u8], packet_type: u8) -> Result<([u8; 16], SessionId, &[u8], &[u8], &[u8], Option<[u8; 16]>), Error> {
fn parse_key_offer_after_header(
incoming_packet: &[u8],
packet_type: u8,
) -> Result<([u8; 16], SessionId, &[u8], &[u8], &[u8], Option<[u8; 16]>), Error> {
let mut p = &incoming_packet[..];
let mut offer_id = [0_u8; 16];
p.read_exact(&mut offer_id)?;
@ -1381,7 +1507,14 @@ fn parse_key_offer_after_header(incoming_packet: &[u8], packet_type: u8) -> Resu
} else {
None
};
Ok((offer_id, alice_session_id, alice_s_public, alice_metadata, alice_e1_public, alice_ratchet_key_fingerprint))
Ok((
offer_id,
alice_session_id,
alice_s_public,
alice_metadata,
alice_e1_public,
alice_ratchet_key_fingerprint,
))
}
enum Role {
@ -1399,9 +1532,13 @@ struct KeyLifetime {
impl KeyLifetime {
fn new(current_counter: CounterValue, current_time: i64) -> Self {
Self {
rekey_at_or_after_counter: current_counter.0 + REKEY_AFTER_USES + (random::next_u32_secure() % REKEY_AFTER_USES_MAX_JITTER) as u64,
rekey_at_or_after_counter: current_counter.0
+ REKEY_AFTER_USES
+ (random::next_u32_secure() % REKEY_AFTER_USES_MAX_JITTER) as u64,
hard_expire_at_counter: current_counter.0 + EXPIRE_AFTER_USES,
rekey_at_or_after_timestamp: current_time + REKEY_AFTER_TIME_MS + (random::next_u32_secure() % REKEY_AFTER_TIME_MS_MAX_JITTER) as i64,
rekey_at_or_after_timestamp: current_time
+ REKEY_AFTER_TIME_MS
+ (random::next_u32_secure() % REKEY_AFTER_TIME_MS_MAX_JITTER) as i64,
}
}
@ -1458,7 +1595,11 @@ impl SessionKey {
#[inline(always)]
fn get_send_cipher(&self, counter: CounterValue) -> Result<Box<AesGcm>, Error> {
if !self.lifetime.expired(counter) {
Ok(self.send_cipher_pool.lock().pop().unwrap_or_else(|| Box::new(AesGcm::new(self.send_key.as_bytes(), true))))
Ok(self
.send_cipher_pool
.lock()
.pop()
.unwrap_or_else(|| Box::new(AesGcm::new(self.send_key.as_bytes(), true))))
} else {
Err(Error::MaxKeyLifetimeExceeded)
}
@ -1471,7 +1612,10 @@ impl SessionKey {
#[inline(always)]
fn get_receive_cipher(&self) -> Box<AesGcm> {
self.receive_cipher_pool.lock().pop().unwrap_or_else(|| Box::new(AesGcm::new(self.receive_key.as_bytes(), false)))
self.receive_cipher_pool
.lock()
.pop()
.unwrap_or_else(|| Box::new(AesGcm::new(self.receive_key.as_bytes(), false)))
}
#[inline(always)]
@ -1574,7 +1718,12 @@ mod tests {
})
}
fn accept_new_session(&self, _: &ReceiveContext<Self>, _: &[u8], _: &[u8]) -> Option<(SessionId, Secret<64>, Self::AssociatedObject)> {
fn accept_new_session(
&self,
_: &ReceiveContext<Self>,
_: &[u8],
_: &[u8],
) -> Option<(SessionId, Secret<64>, Self::AssociatedObject)> {
loop {
let mut new_id = self.session_id_counter.lock();
*new_id += 1;
@ -1646,7 +1795,13 @@ mod tests {
assert!(!data.iter().any(|x| *x != 0x12));
}
ReceiveResult::OkNewSession(new_session) => {
println!("zssp: {} => {} ({}): OkNewSession ({})", host.other_name, host.this_name, qi_len, u64::from(new_session.id));
println!(
"zssp: {} => {} ({}): OkNewSession ({})",
host.other_name,
host.this_name,
qi_len,
u64::from(new_session.id)
);
let mut hs = host.session.lock();
assert!(hs.is_none());
let _ = hs.insert(Arc::new(new_session));
@ -1656,7 +1811,13 @@ mod tests {
}
}
} else {
println!("zssp: {} => {} ({}): error: {}", host.other_name, host.this_name, qi_len, r.err().unwrap().to_string());
println!(
"zssp: {} => {} ({}): error: {}",
host.other_name,
host.this_name,
qi_len,
r.err().unwrap().to_string()
);
panic!();
}
} else {
@ -1683,7 +1844,11 @@ mod tests {
}
for _ in 0..4 {
assert!(session
.send(send_to_other, &mut mtu_buffer, &data_buf[..((random::xorshift64_random() as usize) % data_buf.len())])
.send(
send_to_other,
&mut mtu_buffer,
&data_buf[..((random::xorshift64_random() as usize) % data_buf.len())]
)
.is_ok());
}
if (test_loop % 8) == 0 && test_loop >= 8 && host.this_name.eq("alice") {

13
network-hypervisor/src/networkhypervisor.rs
View file

@ -54,8 +54,17 @@ impl<I: Interface> NetworkHypervisor<I> {
/// Process a physical packet received over a network interface.
#[inline(always)]
pub async fn handle_incoming_physical_packet(&self, ii: &I, source_endpoint: &Endpoint, source_local_socket: &I::LocalSocket, source_local_interface: &I::LocalInterface, data: PooledPacketBuffer) {
self.vl1.handle_incoming_physical_packet(ii, &self.vl2, source_endpoint, source_local_socket, source_local_interface, data).await
pub async fn handle_incoming_physical_packet(
&self,
ii: &I,
source_endpoint: &Endpoint,
source_local_socket: &I::LocalSocket,
source_local_interface: &I::LocalInterface,
data: PooledPacketBuffer,
) {
self.vl1
.handle_incoming_physical_packet(ii, &self.vl2, source_endpoint, source_local_socket, source_local_interface, data)
.await
}
/// Add or update a root set.

13
network-hypervisor/src/util/testutil.rs
View file

@ -2,5 +2,16 @@
// from zeronsd
pub fn randstring(len: u8) -> String {
(0..len).map(|_| (rand::random::<u8>() % 26) + 'a' as u8).map(|c| if rand::random::<bool>() { (c as char).to_ascii_uppercase() } else { c as char }).map(|c| c.to_string()).collect::<Vec<String>>().join("")
(0..len)
.map(|_| (rand::random::<u8>() % 26) + 'a' as u8)
.map(|c| {
if rand::random::<bool>() {
(c as char).to_ascii_uppercase()
} else {
c as char
}
})
.map(|c| c.to_string())
.collect::<Vec<String>>()
.join("")
}

15
network-hypervisor/src/vl1/address.rs
View file

@ -78,7 +78,10 @@ impl Marshalable for Address {
#[inline(always)]
fn unmarshal<const BL: usize>(buf: &Buffer<BL>, cursor: &mut usize) -> std::io::Result<Self> {
Self::from_bytes_fixed(buf.read_bytes_fixed(cursor)?).map_or_else(|| Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "cannot be zero")), |a| Ok(a))
Self::from_bytes_fixed(buf.read_bytes_fixed(cursor)?).map_or_else(
|| Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "cannot be zero")),
|a| Ok(a),
)
}
}
@ -275,8 +278,14 @@ mod tests {
let addr = safe_address();
for _ in 0..1000 {
assert_eq!(serde_json::from_str::<super::Address>(&serde_json::to_string(&addr).unwrap()).unwrap(), addr);
assert_eq!(serde_cbor::from_slice::<super::Address>(&serde_cbor::to_vec(&addr).unwrap()).unwrap(), addr);
assert_eq!(
serde_json::from_str::<super::Address>(&serde_json::to_string(&addr).unwrap()).unwrap(),
addr
);
assert_eq!(
serde_cbor::from_slice::<super::Address>(&serde_cbor::to_vec(&addr).unwrap()).unwrap(),
addr
);
}
}
}

22
network-hypervisor/src/vl1/dictionary.rs
View file

@ -81,7 +81,9 @@ impl Dictionary {
}
pub fn get_str(&self, k: &str) -> Option<&str> {
self.0.get(k).map_or(None, |v| std::str::from_utf8(v.as_slice()).map_or(None, |s| Some(s)))
self.0
.get(k)
.map_or(None, |v| std::str::from_utf8(v.as_slice()).map_or(None, |s| Some(s)))
}
pub fn get_bytes(&self, k: &str) -> Option<&[u8]> {
@ -89,15 +91,19 @@ impl Dictionary {
}
pub fn get_u64(&self, k: &str) -> Option<u64> {
self.get_str(k).map_or(None, |s| u64::from_str_radix(s, 16).map_or(None, |i| Some(i)))
self.get_str(k)
.map_or(None, |s| u64::from_str_radix(s, 16).map_or(None, |i| Some(i)))
}
pub fn get_i64(&self, k: &str) -> Option<i64> {
self.get_str(k).map_or(None, |s| i64::from_str_radix(s, 16).map_or(None, |i| Some(i)))
self.get_str(k)
.map_or(None, |s| i64::from_str_radix(s, 16).map_or(None, |i| Some(i)))
}
pub fn get_bool(&self, k: &str) -> Option<bool> {
self.0.get(k).map_or(None, |v| v.first().map_or(Some(false), |c| Some(BOOL_TRUTH.contains(*c as char))))
self.0
.get(k)
.map_or(None, |v| v.first().map_or(Some(false), |c| Some(BOOL_TRUTH.contains(*c as char))))
}
pub fn set_str(&mut self, k: &str, v: &str) {
@ -238,7 +244,13 @@ mod tests {
match selection {
0 => d.set_str(&key, &randstring(10)),
1 => d.set_u64(&key, rand::random()),
2 => d.set_bytes(&key, (0..((rand::random::<usize>() % 10) + 1)).into_iter().map(|_| rand::random()).collect::<Vec<u8>>()),
2 => d.set_bytes(
&key,
(0..((rand::random::<usize>() % 10) + 1))
.into_iter()
.map(|_| rand::random())
.collect::<Vec<u8>>(),
),
3 => d.set_bool(&key, rand::random::<bool>()),
_ => unreachable!(),
}

50
network-hypervisor/src/vl1/endpoint.rs
View file

@ -190,19 +190,31 @@ impl Marshalable for Endpoint {
if type_byte < 16 {
if type_byte == 4 {
let b: &[u8; 6] = buf.read_bytes_fixed(cursor)?;
Ok(Endpoint::IpUdp(InetAddress::from_ip_port(&b[0..4], u16::from_be_bytes(b[4..6].try_into().unwrap()))))
Ok(Endpoint::IpUdp(InetAddress::from_ip_port(
&b[0..4],
u16::from_be_bytes(b[4..6].try_into().unwrap()),
)))
} else if type_byte == 6 {
let b: &[u8; 18] = buf.read_bytes_fixed(cursor)?;
Ok(Endpoint::IpUdp(InetAddress::from_ip_port(&b[0..16], u16::from_be_bytes(b[16..18].try_into().unwrap()))))
Ok(Endpoint::IpUdp(InetAddress::from_ip_port(
&b[0..16],
u16::from_be_bytes(b[16..18].try_into().unwrap()),
)))
} else {
Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "unrecognized endpoint type in stream"))
Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"unrecognized endpoint type in stream",
))
}
} else {
match type_byte - 16 {
TYPE_NIL => Ok(Endpoint::Nil),
TYPE_ZEROTIER => {
let zt = Address::unmarshal(buf, cursor)?;
Ok(Endpoint::ZeroTier(zt, buf.read_bytes_fixed::<IDENTITY_FINGERPRINT_SIZE>(cursor)?.clone()))
Ok(Endpoint::ZeroTier(
zt,
buf.read_bytes_fixed::<IDENTITY_FINGERPRINT_SIZE>(cursor)?.clone(),
))
}
TYPE_ETHERNET => Ok(Endpoint::Ethernet(MAC::unmarshal(buf, cursor)?)),
TYPE_WIFIDIRECT => Ok(Endpoint::WifiDirect(MAC::unmarshal(buf, cursor)?)),
@ -210,13 +222,20 @@ impl Marshalable for Endpoint {
TYPE_ICMP => Ok(Endpoint::Icmp(InetAddress::unmarshal(buf, cursor)?)),
TYPE_IPUDP => Ok(Endpoint::IpUdp(InetAddress::unmarshal(buf, cursor)?)),
TYPE_IPTCP => Ok(Endpoint::IpTcp(InetAddress::unmarshal(buf, cursor)?)),
TYPE_HTTP => Ok(Endpoint::Http(String::from_utf8_lossy(buf.read_bytes(buf.read_varint(cursor)? as usize, cursor)?).to_string())),
TYPE_WEBRTC => Ok(Endpoint::WebRTC(buf.read_bytes(buf.read_varint(cursor)? as usize, cursor)?.to_vec())),
TYPE_HTTP => Ok(Endpoint::Http(
String::from_utf8_lossy(buf.read_bytes(buf.read_varint(cursor)? as usize, cursor)?).to_string(),
)),
TYPE_WEBRTC => Ok(Endpoint::WebRTC(
buf.read_bytes(buf.read_varint(cursor)? as usize, cursor)?.to_vec(),
)),
TYPE_ZEROTIER_ENCAP => {
let zt = Address::unmarshal(buf, cursor)?;
Ok(Endpoint::ZeroTierEncap(zt, buf.read_bytes_fixed(cursor)?.clone()))
}
_ => Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "unrecognized endpoint type in stream")),
_ => Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"unrecognized endpoint type in stream",
)),
}
}
}
@ -342,7 +361,10 @@ impl FromStr for Endpoint {
if endpoint_type == "zt" {
return Ok(Endpoint::ZeroTier(Address::from_str(address)?, hash.as_slice().try_into().unwrap()));
} else {
return Ok(Endpoint::ZeroTierEncap(Address::from_str(address)?, hash.as_slice().try_into().unwrap()));
return Ok(Endpoint::ZeroTierEncap(
Address::from_str(address)?,
hash.as_slice().try_into().unwrap(),
));
}
}
}
@ -538,7 +560,11 @@ mod tests {
for _ in 0..1000 {
let mac = crate::vl1::MAC::from_u64(rand::random()).unwrap();
for e in [Endpoint::Ethernet(mac.clone()), Endpoint::WifiDirect(mac.clone()), Endpoint::Bluetooth(mac.clone())] {
for e in [
Endpoint::Ethernet(mac.clone()),
Endpoint::WifiDirect(mac.clone()),
Endpoint::Bluetooth(mac.clone()),
] {
let mut buf = Buffer::<7>::new();
let res = e.marshal(&mut buf);
@ -563,7 +589,11 @@ mod tests {
let inet = crate::vl1::InetAddress::from_ip_port(&v, 1234);
for e in [Endpoint::Icmp(inet.clone()), Endpoint::IpTcp(inet.clone()), Endpoint::IpUdp(inet.clone())] {
for e in [
Endpoint::Icmp(inet.clone()),
Endpoint::IpTcp(inet.clone()),
Endpoint::IpUdp(inet.clone()),
] {
let mut buf = Buffer::<20>::new();
let res = e.marshal(&mut buf);

106
network-hypervisor/src/vl1/identity.rs
View file

@ -75,7 +75,12 @@ fn concat_arrays_2<const A: usize, const B: usize, const S: usize>(a: &[u8; A],
}
#[inline(always)]
fn concat_arrays_4<const A: usize, const B: usize, const C: usize, const D: usize, const S: usize>(a: &[u8; A], b: &[u8; B], c: &[u8; C], d: &[u8; D]) -> [u8; S] {
fn concat_arrays_4<const A: usize, const B: usize, const C: usize, const D: usize, const S: usize>(
a: &[u8; A],
b: &[u8; B],
c: &[u8; C],
d: &[u8; D],
) -> [u8; S] {
assert_eq!(A + B + C + D, S);
let mut tmp = [0_u8; S];
tmp[..A].copy_from_slice(a);
@ -137,10 +142,19 @@ impl Identity {
pub const BYTE_LENGTH_X25519_SECRET: usize = Self::BYTE_LENGTH_X25519_PUBLIC + C25519_SECRET_KEY_SIZE + ED25519_SECRET_KEY_SIZE;
/// Length of a new dual-key public identity in byte array form.
pub const BYTE_LENGTH_X25519P384_PUBLIC: usize = Self::BYTE_LENGTH_X25519_PUBLIC + 1 + P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_ECDSA_SIGNATURE_SIZE + ED25519_SIGNATURE_SIZE;
pub const BYTE_LENGTH_X25519P384_PUBLIC: usize = Self::BYTE_LENGTH_X25519_PUBLIC
+ 1
+ P384_PUBLIC_KEY_SIZE
+ P384_PUBLIC_KEY_SIZE
+ P384_ECDSA_SIGNATURE_SIZE
+ ED25519_SIGNATURE_SIZE;
/// Length of a new dual-key secret identity in byte array form.
pub const BYTE_LENGTH_X25519P384_SECRET: usize = Self::BYTE_LENGTH_X25519P384_PUBLIC + C25519_SECRET_KEY_SIZE + ED25519_SECRET_KEY_SIZE + P384_SECRET_KEY_SIZE + P384_SECRET_KEY_SIZE;
pub const BYTE_LENGTH_X25519P384_SECRET: usize = Self::BYTE_LENGTH_X25519P384_PUBLIC
+ C25519_SECRET_KEY_SIZE
+ ED25519_SECRET_KEY_SIZE
+ P384_SECRET_KEY_SIZE
+ P384_SECRET_KEY_SIZE;
const ALGORITHM_X25519: u8 = 0x01;
const ALGORITHM_EC_NIST_P384: u8 = 0x02;
@ -203,14 +217,23 @@ impl Identity {
/// It would be possible to change this in the future, with care.
pub fn upgrade(&mut self) -> Result<bool, InvalidParameterError> {
if self.secret.is_none() {
return Err(InvalidParameterError("an identity can only be upgraded if it includes its private key"));
return Err(InvalidParameterError(
"an identity can only be upgraded if it includes its private key",
));
}
if self.p384.is_none() {
let p384_ecdh = P384KeyPair::generate();
let p384_ecdsa = P384KeyPair::generate();
let mut self_sign_buf: Vec<u8> =
Vec::with_capacity(ADDRESS_SIZE + C25519_PUBLIC_KEY_SIZE + ED25519_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_ECDSA_SIGNATURE_SIZE + 4);
let mut self_sign_buf: Vec<u8> = Vec::with_capacity(
ADDRESS_SIZE
+ C25519_PUBLIC_KEY_SIZE
+ ED25519_PUBLIC_KEY_SIZE
+ P384_PUBLIC_KEY_SIZE
+ P384_PUBLIC_KEY_SIZE
+ P384_ECDSA_SIGNATURE_SIZE
+ 4,
);
let _ = self_sign_buf.write_all(&self.address.to_bytes());
let _ = self_sign_buf.write_all(&self.x25519);
let _ = self_sign_buf.write_all(&self.ed25519);
@ -233,7 +256,12 @@ impl Identity {
ecdsa_self_signature,
ed25519_self_signature,
});
let _ = self.secret.as_mut().unwrap().p384.insert(IdentityP384Secret { ecdh: p384_ecdh, ecdsa: p384_ecdsa });
let _ = self
.secret
.as_mut()
.unwrap()
.p384
.insert(IdentityP384Secret { ecdh: p384_ecdh, ecdsa: p384_ecdsa });
self.fingerprint = SHA384::hash(self.to_public_bytes().as_bytes());
@ -259,7 +287,9 @@ impl Identity {
/// This is somewhat time consuming due to the memory-intensive work algorithm.
pub fn validate_identity(&self) -> bool {
if let Some(p384) = self.p384.as_ref() {
let mut self_sign_buf: Vec<u8> = Vec::with_capacity(ADDRESS_SIZE + 4 + C25519_PUBLIC_KEY_SIZE + ED25519_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE);
let mut self_sign_buf: Vec<u8> = Vec::with_capacity(
ADDRESS_SIZE + 4 + C25519_PUBLIC_KEY_SIZE + ED25519_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE,
);
let _ = self_sign_buf.write_all(&self.address.to_bytes());
let _ = self_sign_buf.write_all(&self.x25519);
let _ = self_sign_buf.write_all(&self.ed25519);
@ -469,7 +499,11 @@ impl Identity {
}
IdentityBytes::X25519Secret(b) => {
let b: &packed::V0S = bytes_as_flat_object(b);
if b.key_type == 0 && b.secret_length == (C25519_SECRET_KEY_SIZE + ED25519_SECRET_KEY_SIZE) as u8 && b.reserved == 0x03 && u16::from_be_bytes(b.ext_len) == 0 {
if b.key_type == 0
&& b.secret_length == (C25519_SECRET_KEY_SIZE + ED25519_SECRET_KEY_SIZE) as u8
&& b.reserved == 0x03
&& u16::from_be_bytes(b.ext_len) == 0
{
Some(Self {
address: Address::from_bytes_fixed(&b.address)?,
x25519: b.x25519,
@ -570,7 +604,12 @@ impl Identity {
};
IdentityBytes::try_from(&buf[..obj_len]).map_or_else(
|_| Err(std::io::Error::new(std::io::ErrorKind::Other, "invalid identity")),
|b| Identity::from_bytes(&b).map_or_else(|| Err(std::io::Error::new(std::io::ErrorKind::Other, "invalid identity")), |id| Ok(id)),
|b| {
Identity::from_bytes(&b).map_or_else(
|| Err(std::io::Error::new(std::io::ErrorKind::Other, "invalid identity")),
|id| Ok(id),
)
},
)
}
@ -594,14 +633,21 @@ impl Identity {
}
s.push_str(":2:"); // 2 == IDENTITY_ALGORITHM_EC_NIST_P384
let p384_joined: [u8; P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_ECDSA_SIGNATURE_SIZE + ED25519_SIGNATURE_SIZE] =
concat_arrays_4(p384.ecdh.as_bytes(), p384.ecdsa.as_bytes(), &p384.ecdsa_self_signature, &p384.ed25519_self_signature);
concat_arrays_4(
p384.ecdh.as_bytes(),
p384.ecdsa.as_bytes(),
&p384.ecdsa_self_signature,
&p384.ed25519_self_signature,
);
s.push_str(base64::encode_config(p384_joined, base64::URL_SAFE_NO_PAD).as_str());
if self.secret.is_some() && include_private {
let secret = self.secret.as_ref().unwrap();
if secret.p384.is_some() {
let p384_secret = secret.p384.as_ref().unwrap();
let p384_secret_joined: [u8; P384_SECRET_KEY_SIZE + P384_SECRET_KEY_SIZE] =
concat_arrays_2(p384_secret.ecdh.secret_key_bytes().as_bytes(), p384_secret.ecdsa.secret_key_bytes().as_bytes());
let p384_secret_joined: [u8; P384_SECRET_KEY_SIZE + P384_SECRET_KEY_SIZE] = concat_arrays_2(
p384_secret.ecdh.secret_key_bytes().as_bytes(),
p384_secret.ecdsa.secret_key_bytes().as_bytes(),
);
s.push(':');
s.push_str(base64::encode_config(p384_secret_joined, base64::URL_SAFE_NO_PAD).as_str());
}
@ -680,7 +726,9 @@ impl FromStr for Identity {
if keys[0].len() != C25519_PUBLIC_KEY_SIZE + ED25519_PUBLIC_KEY_SIZE {
return Err(InvalidFormatError);
}
if !keys[2].is_empty() && keys[2].len() != P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_ECDSA_SIGNATURE_SIZE + ED25519_SIGNATURE_SIZE {
if !keys[2].is_empty()
&& keys[2].len() != P384_PUBLIC_KEY_SIZE + P384_PUBLIC_KEY_SIZE + P384_ECDSA_SIGNATURE_SIZE + ED25519_SIGNATURE_SIZE
{
return Err(InvalidFormatError);
}
if !keys[3].is_empty() && keys[3].len() != P384_SECRET_KEY_SIZE + P384_SECRET_KEY_SIZE {
@ -710,10 +758,13 @@ impl FromStr for Identity {
Some(IdentityP384Public {
ecdh: ecdh.unwrap(),
ecdsa: ecdsa.unwrap(),
ecdsa_self_signature: keys[2].as_slice()[(P384_PUBLIC_KEY_SIZE * 2)..((P384_PUBLIC_KEY_SIZE * 2) + P384_ECDSA_SIGNATURE_SIZE)]
ecdsa_self_signature: keys[2].as_slice()
[(P384_PUBLIC_KEY_SIZE * 2)..((P384_PUBLIC_KEY_SIZE * 2) + P384_ECDSA_SIGNATURE_SIZE)]
.try_into()
.unwrap(),
ed25519_self_signature: keys[2].as_slice()[((P384_PUBLIC_KEY_SIZE * 2) + P384_ECDSA_SIGNATURE_SIZE)..]
.try_into()
.unwrap(),
ed25519_self_signature: keys[2].as_slice()[((P384_PUBLIC_KEY_SIZE * 2) + P384_ECDSA_SIGNATURE_SIZE)..].try_into().unwrap(),
})
},
secret: if keys[1].is_empty() {
@ -742,14 +793,20 @@ impl FromStr for Identity {
} else {
Some(IdentityP384Secret {
ecdh: {
let tmp = P384KeyPair::from_bytes(&keys[2].as_slice()[..P384_PUBLIC_KEY_SIZE], &keys[3].as_slice()[..P384_SECRET_KEY_SIZE]);
let tmp = P384KeyPair::from_bytes(
&keys[2].as_slice()[..P384_PUBLIC_KEY_SIZE],
&keys[3].as_slice()[..P384_SECRET_KEY_SIZE],
);
if tmp.is_none() {
return Err(InvalidFormatError);
}
tmp.unwrap()
},
ecdsa: {
let tmp = P384KeyPair::from_bytes(&keys[2].as_slice()[P384_PUBLIC_KEY_SIZE..(P384_PUBLIC_KEY_SIZE * 2)], &keys[3].as_slice()[P384_SECRET_KEY_SIZE..]);
let tmp = P384KeyPair::from_bytes(
&keys[2].as_slice()[P384_PUBLIC_KEY_SIZE..(P384_PUBLIC_KEY_SIZE * 2)],
&keys[3].as_slice()[P384_SECRET_KEY_SIZE..],
);
if tmp.is_none() {
return Err(InvalidFormatError);
}
@ -781,7 +838,9 @@ impl Eq for Identity {}
impl Ord for Identity {
#[inline(always)]
fn cmp(&self, other: &Self) -> Ordering {
self.address.cmp(&other.address).then_with(|| self.fingerprint.cmp(&other.fingerprint))
self.address
.cmp(&other.address)
.then_with(|| self.fingerprint.cmp(&other.fingerprint))
}
}
@ -973,7 +1032,8 @@ mod tests {
#[test]
fn v0_identity() {
let self_agree_expected = hex::from_string("de904fc90ff3a2b96b739b926e623113f5334c80841b654509b77916c4c4a6eb0ca69ec6ed01a7f04aee17c546b30ba4");
let self_agree_expected =
hex::from_string("de904fc90ff3a2b96b739b926e623113f5334c80841b654509b77916c4c4a6eb0ca69ec6ed01a7f04aee17c546b30ba4");
// Test self-agree with a known good x25519-only (v0) identity.
let id = Identity::from_str(
@ -1020,7 +1080,11 @@ mod tests {
let gen_unmarshaled = Identity::from_bytes(&bytes).unwrap();
assert!(gen_unmarshaled.secret.is_some());
if !gen_unmarshaled.eq(&gen) {
println!("{} != {}", hex::to_string(&gen_unmarshaled.fingerprint), hex::to_string(&gen.fingerprint));
println!(
"{} != {}",
hex::to_string(&gen_unmarshaled.fingerprint),
hex::to_string(&gen.fingerprint)
);
}
assert!(Identity::from_str(string.as_str()).unwrap().secret.is_some());

42
network-hypervisor/src/vl1/inetaddress.rs
View file

@ -82,8 +82,10 @@ impl ToSocketAddrs for InetAddress {
#[inline(always)]
fn to_socket_addrs(&self) -> std::io::Result<Self::Iter> {
self.try_into()
.map_or_else(|_| Err(std::io::Error::new(std::io::ErrorKind::Other, "not an IP address")), |sa| Ok(std::iter::once(sa)))
self.try_into().map_or_else(
|_| Err(std::io::Error::new(std::io::ErrorKind::Other, "not an IP address")),
|sa| Ok(std::iter::once(sa)),
)
}
}
@ -199,7 +201,10 @@ impl TryInto<SocketAddrV4> for &InetAddress {
fn try_into(self) -> Result<SocketAddrV4, Self::Error> {
unsafe {
match self.sa.sa_family {
AF_INET => Ok(SocketAddrV4::new(Ipv4Addr::from(self.sin.sin_addr.s_addr.to_ne_bytes()), u16::from_be(self.sin.sin_port as u16))),
AF_INET => Ok(SocketAddrV4::new(
Ipv4Addr::from(self.sin.sin_addr.s_addr.to_ne_bytes()),
u16::from_be(self.sin.sin_port as u16),
)),
_ => Err(crate::error::InvalidParameterError("not an IPv4 address")),
}
}
@ -222,7 +227,12 @@ impl TryInto<SocketAddrV6> for &InetAddress {
fn try_into(self) -> Result<SocketAddrV6, Self::Error> {
unsafe {
match self.sa.sa_family {
AF_INET6 => Ok(SocketAddrV6::new(Ipv6Addr::from(self.sin6.sin6_addr.s6_addr), u16::from_be(self.sin6.sin6_port as u16), 0, 0)),
AF_INET6 => Ok(SocketAddrV6::new(
Ipv6Addr::from(self.sin6.sin6_addr.s6_addr),
u16::from_be(self.sin6.sin6_port as u16),
0,
0,
)),
_ => Err(crate::error::InvalidParameterError("not an IPv6 address")),
}
}
@ -597,7 +607,9 @@ impl InetAddress {
AF_INET => {
if cidr_bits <= 32 {
let discard_bits = 32 - cidr_bits;
if u32::from_be(self.sin.sin_addr.s_addr as u32).wrapping_shr(discard_bits) == u32::from_be(cidr.sin.sin_addr.s_addr as u32).wrapping_shr(discard_bits) {
if u32::from_be(self.sin.sin_addr.s_addr as u32).wrapping_shr(discard_bits)
== u32::from_be(cidr.sin.sin_addr.s_addr as u32).wrapping_shr(discard_bits)
{
return true;
}
}
@ -784,7 +796,11 @@ impl Marshalable for InetAddress {
AF_INET6 => {
let b = buf.append_bytes_fixed_get_mut::<19>()?;
b[0] = 6;
copy_nonoverlapping((&(self.sin6.sin6_addr) as *const in6_addr).cast::<u8>(), b.as_mut_ptr().offset(1), 16);
copy_nonoverlapping(
(&(self.sin6.sin6_addr) as *const in6_addr).cast::<u8>(),
b.as_mut_ptr().offset(1),
16,
);
b[17] = *(&self.sin6.sin6_port as *const u16).cast::<u8>();
b[18] = *(&self.sin6.sin6_port as *const u16).cast::<u8>().offset(1);
Ok(())
@ -801,7 +817,10 @@ impl Marshalable for InetAddress {
Ok(InetAddress::from_ip_port(&b[0..4], u16::from_be_bytes(b[4..6].try_into().unwrap())))
} else if t == 6 {
let b: &[u8; 18] = buf.read_bytes_fixed(cursor)?;
Ok(InetAddress::from_ip_port(&b[0..16], u16::from_be_bytes(b[16..18].try_into().unwrap())))
Ok(InetAddress::from_ip_port(
&b[0..16],
u16::from_be_bytes(b[16..18].try_into().unwrap()),
))
} else {
Ok(InetAddress::new())
}
@ -880,7 +899,8 @@ impl PartialEq for InetAddress {
AF_INET => self.sin.sin_port == other.sin.sin_port && self.sin.sin_addr.s_addr == other.sin.sin_addr.s_addr,
AF_INET6 => {
if self.sin6.sin6_port == other.sin6.sin6_port {
(*(&(self.sin6.sin6_addr) as *const in6_addr).cast::<[u8; 16]>()).eq(&*(&(other.sin6.sin6_addr) as *const in6_addr).cast::<[u8; 16]>())
(*(&(self.sin6.sin6_addr) as *const in6_addr).cast::<[u8; 16]>())
.eq(&*(&(other.sin6.sin6_addr) as *const in6_addr).cast::<[u8; 16]>())
} else {
false
}
@ -912,7 +932,8 @@ impl Ord for InetAddress {
match self.sa.sa_family as AddressFamilyType {
0 => Ordering::Equal,
AF_INET => {
let ip_ordering = u32::from_be(self.sin.sin_addr.s_addr as u32).cmp(&u32::from_be(other.sin.sin_addr.s_addr as u32));
let ip_ordering =
u32::from_be(self.sin.sin_addr.s_addr as u32).cmp(&u32::from_be(other.sin.sin_addr.s_addr as u32));
if ip_ordering == Ordering::Equal {
u16::from_be(self.sin.sin_port as u16).cmp(&u16::from_be(other.sin.sin_port as u16))
} else {
@ -931,7 +952,8 @@ impl Ord for InetAddress {
}
_ => {
// This shouldn't be possible, but handle it for correctness.
(*slice_from_raw_parts((self as *const Self).cast::<u8>(), size_of::<Self>())).cmp(&*slice_from_raw_parts((other as *const Self).cast::<u8>(), size_of::<Self>()))
(*slice_from_raw_parts((self as *const Self).cast::<u8>(), size_of::<Self>()))
.cmp(&*slice_from_raw_parts((other as *const Self).cast::<u8>(), size_of::<Self>()))
}
}
} else {

34
network-hypervisor/src/vl1/mac.rs
View file

@ -33,7 +33,15 @@ impl MAC {
#[inline(always)]
pub fn from_bytes(b: &[u8]) -> Option<MAC> {
if b.len() >= 6 {
NonZeroU64::new((b[0] as u64) << 40 | (b[1] as u64) << 32 | (b[2] as u64) << 24 | (b[3] as u64) << 16 as u64 | (b[4] as u64) << 8 | b[5] as u64).map(|i| MAC(i))
NonZeroU64::new(
(b[0] as u64) << 40
| (b[1] as u64) << 32
| (b[2] as u64) << 24
| (b[3] as u64) << 16 as u64
| (b[4] as u64) << 8
| b[5] as u64,
)
.map(|i| MAC(i))
} else {
None
}
@ -41,13 +49,23 @@ impl MAC {
#[inline(always)]
pub fn from_bytes_fixed(b: &[u8; 6]) -> Option<MAC> {
NonZeroU64::new((b[0] as u64) << 40 | (b[1] as u64) << 32 | (b[2] as u64) << 24 | (b[3] as u64) << 16 as u64 | (b[4] as u64) << 8 | b[5] as u64).map(|i| MAC(i))
NonZeroU64::new(
(b[0] as u64) << 40 | (b[1] as u64) << 32 | (b[2] as u64) << 24 | (b[3] as u64) << 16 as u64 | (b[4] as u64) << 8 | b[5] as u64,
)
.map(|i| MAC(i))
}
#[inline(always)]
pub fn to_bytes(&self) -> [u8; 6] {
let i = self.0.get();
[(i >> 40) as u8, (i >> 32) as u8, (i >> 24) as u8, (i >> 16) as u8, (i >> 8) as u8, i as u8]
[
(i >> 40) as u8,
(i >> 32) as u8,
(i >> 24) as u8,
(i >> 16) as u8,
(i >> 8) as u8,
i as u8,
]
}
}
@ -75,14 +93,20 @@ impl Marshalable for MAC {
#[inline(always)]
fn unmarshal<const BL: usize>(buf: &Buffer<BL>, cursor: &mut usize) -> std::io::Result<Self> {
Self::from_bytes_fixed(buf.read_bytes_fixed(cursor)?).map_or_else(|| Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "cannot be zero")), |a| Ok(a))
Self::from_bytes_fixed(buf.read_bytes_fixed(cursor)?).map_or_else(
|| Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "cannot be zero")),
|a| Ok(a),
)
}
}
impl ToString for MAC {
fn to_string(&self) -> String {
let b: [u8; 6] = self.to_bytes();
format!("{:0>2x}:{:0>2x}:{:0>2x}:{:0>2x}:{:0>2x}:{:0>2x}", b[0], b[1], b[2], b[3], b[4], b[5])
format!(
"{:0>2x}:{:0>2x}:{:0>2x}:{:0>2x}:{:0>2x}:{:0>2x}",
b[0], b[1], b[2], b[3], b[4], b[5]
)
}
}

108
network-hypervisor/src/vl1/node.rs
View file

@ -70,10 +70,23 @@ pub trait SystemInterface: Sync + Send + 'static {
/// For endpoint types that support a packet TTL, the implementation may set the TTL
/// if the 'ttl' parameter is not zero. If the parameter is zero or TTL setting is not
/// supported, the default TTL should be used.
async fn wire_send(&self, endpoint: &Endpoint, local_socket: Option<&Self::LocalSocket>, local_interface: Option<&Self::LocalInterface>, data: &[&[u8]], packet_ttl: u8) -> bool;
async fn wire_send(
&self,
endpoint: &Endpoint,
local_socket: Option<&Self::LocalSocket>,
local_interface: Option<&Self::LocalInterface>,
data: &[&[u8]],
packet_ttl: u8,
) -> bool;
/// Called to check and see if a physical address should be used for ZeroTier traffic to a node.
async fn check_path(&self, id: &Identity, endpoint: &Endpoint, local_socket: Option<&Self::LocalSocket>, local_interface: Option<&Self::LocalInterface>) -> bool;
async fn check_path(
&self,
id: &Identity,
endpoint: &Endpoint,
local_socket: Option<&Self::LocalSocket>,
local_interface: Option<&Self::LocalInterface>,
) -> bool;
/// Called to look up any statically defined or memorized paths to known nodes.
async fn get_path_hints(&self, id: &Identity) -> Option<Vec<(Endpoint, Option<Self::LocalSocket>, Option<Self::LocalInterface>)>>;
@ -96,7 +109,8 @@ pub trait InnerProtocolInterface: Sync + Send + 'static {
/// Handle a packet, returning true if it was handled by the next layer.
///
/// Do not attempt to handle OK or ERROR. Instead implement handle_ok() and handle_error().
async fn handle_packet<SI: SystemInterface>(&self, source: &Peer<SI>, source_path: &Path<SI>, verb: u8, payload: &PacketBuffer) -> bool;
async fn handle_packet<SI: SystemInterface>(&self, source: &Peer<SI>, source_path: &Path<SI>, verb: u8, payload: &PacketBuffer)
-> bool;
/// Handle errors, returning true if the error was recognized.
async fn handle_error<SI: SystemInterface>(
@ -111,7 +125,15 @@ pub trait InnerProtocolInterface: Sync + Send + 'static {
) -> bool;
/// Handle an OK, returing true if the OK was recognized.
async fn handle_ok<SI: SystemInterface>(&self, source: &Peer<SI>, source_path: &Path<SI>, in_re_verb: u8, in_re_message_id: u64, payload: &PacketBuffer, cursor: &mut usize) -> bool;
async fn handle_ok<SI: SystemInterface>(
&self,
source: &Peer<SI>,
source_path: &Path<SI>,
in_re_verb: u8,
in_re_message_id: u64,
payload: &PacketBuffer,
cursor: &mut usize,
) -> bool;
/// Check if this peer should communicate with another at all.
fn should_communicate_with(&self, id: &Identity) -> bool;
@ -298,7 +320,12 @@ impl<SI: SystemInterface> Node<SI> {
let mut best_root = self.best_root.write();
if let Some(best_root) = best_root.as_mut() {
if !Arc::ptr_eq(best_root, best) {
debug_event!(si, "[vl1] new best root: {} (replaced {})", best.identity.address.to_string(), best_root.identity.address.to_string());
debug_event!(
si,
"[vl1] new best root: {} (replaced {})",
best.identity.address.to_string(),
best_root.identity.address.to_string()
);
*best_root = best.clone();
}
} else {
@ -409,8 +436,14 @@ impl<SI: SystemInterface> Node<SI> {
for m in rs.members.iter() {
if m.identity.eq(&self.identity) {
let _ = my_root_sets.get_or_insert_with(|| Vec::new()).write_all(rs.to_bytes().as_slice());
} else if self.peers.read().get(&m.identity.address).map_or(false, |p| !p.identity.eq(&m.identity))
|| address_collision_check.insert(m.identity.address, &m.identity).map_or(false, |old_id| !old_id.eq(&m.identity))
} else if self
.peers
.read()
.get(&m.identity.address)
.map_or(false, |p| !p.identity.eq(&m.identity))
|| address_collision_check
.insert(m.identity.address, &m.identity)
.map_or(false, |old_id| !old_id.eq(&m.identity))
{
address_collisions.push(m.identity.address);
}
@ -420,7 +453,8 @@ impl<SI: SystemInterface> Node<SI> {
for (_, rs) in roots.sets.iter() {
for m in rs.members.iter() {
if m.endpoints.is_some() && !address_collisions.contains(&m.identity.address) && !m.identity.eq(&self.identity) {
if m.endpoints.is_some() && !address_collisions.contains(&m.identity.address) && !m.identity.eq(&self.identity)
{
debug_event!(
si,
"[vl1] examining root {} with {} endpoints",
@ -493,7 +527,12 @@ impl<SI: SystemInterface> Node<SI> {
};
for (root, endpoints) in roots.iter() {
for ep in endpoints.iter() {
debug_event!(si, "sending HELLO to root {} (root interval: {})", root.identity.address.to_string(), ROOT_HELLO_INTERVAL);
debug_event!(
si,
"sending HELLO to root {} (root interval: {})",
root.identity.address.to_string(),
ROOT_HELLO_INTERVAL
);
root.send_hello(si, self, Some(ep)).await;
}
}
@ -538,7 +577,8 @@ impl<SI: SystemInterface> Node<SI> {
let ka = [tt as u8]; // send different bytes every time for keepalive in case some things filter zero packets
let ka2 = [&ka[..1]];
for ka in need_keepalive.iter() {
si.wire_send(&ka.endpoint, Some(&ka.local_socket), Some(&ka.local_interface), &ka2, 0).await;
si.wire_send(&ka.endpoint, Some(&ka.local_socket), Some(&ka.local_interface), &ka2, 0)
.await;
}
}
@ -563,7 +603,8 @@ impl<SI: SystemInterface> Node<SI> {
si,
"[vl1] {} -> #{} {}->{} length {} (on socket {}@{})",
source_endpoint.to_string(),
data.bytes_fixed_at::<8>(0).map_or("????????????????".into(), |pid| hex::to_string(pid)),
data.bytes_fixed_at::<8>(0)
.map_or("????????????????".into(), |pid| hex::to_string(pid)),
data.bytes_fixed_at::<5>(13).map_or("??????????".into(), |src| hex::to_string(src)),
data.bytes_fixed_at::<5>(8).map_or("??????????".into(), |dest| hex::to_string(dest)),
data.len(),
@ -589,7 +630,13 @@ impl<SI: SystemInterface> Node<SI> {
fragment_header.total_fragments()
);
if let Some(assembled_packet) = path.receive_fragment(fragment_header.packet_id(), fragment_header.fragment_no(), fragment_header.total_fragments(), data, time_ticks) {
if let Some(assembled_packet) = path.receive_fragment(
fragment_header.packet_id(),
fragment_header.fragment_no(),
fragment_header.total_fragments(),
data,
time_ticks,
) {
if let Some(frag0) = assembled_packet.frags[0].as_ref() {
#[cfg(debug_assertions)]
debug_event!(si, "[vl1] #{:0>16x} packet fully assembled!", fragment_header_id);
@ -597,8 +644,17 @@ impl<SI: SystemInterface> Node<SI> {
if let Ok(packet_header) = frag0.struct_at::<v1::PacketHeader>(0) {
if let Some(source) = Address::from_bytes(&packet_header.src) {
if let Some(peer) = self.peer(source) {
peer.receive(self, si, ph, time_ticks, &path, packet_header, frag0, &assembled_packet.frags[1..(assembled_packet.have as usize)])
.await;
peer.receive(
self,
si,
ph,
time_ticks,
&path,
packet_header,
frag0,
&assembled_packet.frags[1..(assembled_packet.have as usize)],
)
.await;
} else {
self.whois.query(self, si, source, Some(QueuedPacket::Fragmented(assembled_packet)));
}
@ -613,7 +669,8 @@ impl<SI: SystemInterface> Node<SI> {
if let Some(source) = Address::from_bytes(&packet_header.src) {
if let Some(peer) = self.peer(source) {
peer.receive(self, si, ph, time_ticks, &path, packet_header, data.as_ref(), &[]).await;
peer.receive(self, si, ph, time_ticks, &path, packet_header, data.as_ref(), &[])
.await;
} else {
self.whois.query(self, si, source, Some(QueuedPacket::Unfragmented(data)));
}
@ -628,7 +685,12 @@ impl<SI: SystemInterface> Node<SI> {
#[cfg(debug_assertions)]
{
debug_packet_id = u64::from_be_bytes(fragment_header.id);
debug_event!(si, "[vl1] #{:0>16x} forwarding packet fragment to {}", debug_packet_id, dest.to_string());
debug_event!(
si,
"[vl1] #{:0>16x} forwarding packet fragment to {}",
debug_packet_id,
dest.to_string()
);
}
if fragment_header.increment_hops() > v1::FORWARD_MAX_HOPS {
#[cfg(debug_assertions)]
@ -644,7 +706,11 @@ impl<SI: SystemInterface> Node<SI> {
}
if packet_header.increment_hops() > v1::FORWARD_MAX_HOPS {
#[cfg(debug_assertions)]
debug_event!(si, "[vl1] #{:0>16x} discarded: max hops exceeded!", u64::from_be_bytes(packet_header.id));
debug_event!(
si,
"[vl1] #{:0>16x} discarded: max hops exceeded!",
u64::from_be_bytes(packet_header.id)
);
return;
}
} else {
@ -714,7 +780,13 @@ impl<SI: SystemInterface> Node<SI> {
self.roots.read().my_root_sets.is_some()
}
pub(crate) fn canonical_path(&self, ep: &Endpoint, local_socket: &SI::LocalSocket, local_interface: &SI::LocalInterface, time_ticks: i64) -> Arc<Path<SI>> {
pub(crate) fn canonical_path(
&self,
ep: &Endpoint,
local_socket: &SI::LocalSocket,
local_interface: &SI::LocalInterface,
time_ticks: i64,
) -> Arc<Path<SI>> {
if let Some(path) = self.paths.read().get(&PathKey::Ref(ep, local_socket)) {
return path.clone();
}

13
network-hypervisor/src/vl1/path.rs
View file

@ -50,7 +50,14 @@ impl<SI: SystemInterface> Path<SI> {
/// Receive a fragment and return a FragmentedPacket if the entire packet was assembled.
/// This returns None if more fragments are needed to assemble the packet.
pub(crate) fn receive_fragment(&self, packet_id: PacketId, fragment_no: u8, fragment_expecting_count: u8, packet: PooledPacketBuffer, time_ticks: i64) -> Option<FragmentedPacket> {
pub(crate) fn receive_fragment(
&self,
packet_id: PacketId,
fragment_no: u8,
fragment_expecting_count: u8,
packet: PooledPacketBuffer,
time_ticks: i64,
) -> Option<FragmentedPacket> {
let mut fp = self.fragmented_packets.lock();
// Discard some old waiting packets if the total incoming fragments for a path exceeds a
@ -90,7 +97,9 @@ impl<SI: SystemInterface> Path<SI> {
}
pub(crate) fn service(&self, time_ticks: i64) -> PathServiceResult {
self.fragmented_packets.lock().retain(|_, frag| (time_ticks - frag.ts_ticks) < v1::FRAGMENT_EXPIRATION);
self.fragmented_packets
.lock()
.retain(|_, frag| (time_ticks - frag.ts_ticks) < v1::FRAGMENT_EXPIRATION);
if (time_ticks - self.last_receive_time_ticks.load(Ordering::Relaxed)) < PATH_EXPIRATION_TIME {
if (time_ticks - self.last_send_time_ticks.load(Ordering::Relaxed)) >= PATH_KEEPALIVE_INTERVAL {
self.last_send_time_ticks.store(time_ticks, Ordering::Relaxed);

231
network-hypervisor/src/vl1/peer.rs
View file

@ -210,7 +210,12 @@ impl<SI: SystemInterface> Peer<SI> {
pub fn version(&self) -> Option<[u16; 4]> {
let rv = self.remote_node_info.read().remote_version;
if rv != 0 {
Some([rv.wrapping_shr(48) as u16, rv.wrapping_shr(32) as u16, rv.wrapping_shr(16) as u16, rv as u16])
Some([
rv.wrapping_shr(48) as u16,
rv.wrapping_shr(32) as u16,
rv.wrapping_shr(16) as u16,
rv as u16,
])
} else {
None
}
@ -299,7 +304,12 @@ impl<SI: SystemInterface> Peer<SI> {
}
// Learn new path if it's not a duplicate or should not replace an existing path.
debug_event!(si, "[vl1] {} learned new path: {}", self.identity.address.to_string(), new_path.endpoint.to_string());
debug_event!(
si,
"[vl1] {} learned new path: {}",
self.identity.address.to_string(),
new_path.endpoint.to_string()
);
paths.push(PeerPath::<SI> {
path: Arc::downgrade(new_path),
last_receive_time_ticks: time_ticks,
@ -314,7 +324,10 @@ impl<SI: SystemInterface> Peer<SI> {
paths.retain(|p| ((time_ticks - p.last_receive_time_ticks) < PEER_EXPIRATION_TIME) && (p.path.strong_count() > 0));
prioritize_paths(&mut paths);
}
self.remote_node_info.write().reported_local_endpoints.retain(|_, ts| (time_ticks - *ts) < PEER_EXPIRATION_TIME);
self.remote_node_info
.write()
.reported_local_endpoints
.retain(|_, ts| (time_ticks - *ts) < PEER_EXPIRATION_TIME);
(time_ticks - self.last_receive_time_ticks.load(Ordering::Relaxed).max(self.create_time_ticks)) < PEER_EXPIRATION_TIME
}
@ -334,13 +347,17 @@ impl<SI: SystemInterface> Peer<SI> {
let packet_size = packet.len();
if packet_size > max_fragment_size {
let bytes = packet.as_bytes();
if !si.wire_send(endpoint, local_socket, local_interface, &[&bytes[0..UDP_DEFAULT_MTU]], 0).await {
if !si
.wire_send(endpoint, local_socket, local_interface, &[&bytes[0..UDP_DEFAULT_MTU]], 0)
.await
{
return false;
}
let mut pos = UDP_DEFAULT_MTU;
let overrun_size = (packet_size - UDP_DEFAULT_MTU) as u32;
let fragment_count = (overrun_size / (UDP_DEFAULT_MTU - v1::FRAGMENT_HEADER_SIZE) as u32) + (((overrun_size % (UDP_DEFAULT_MTU - v1::FRAGMENT_HEADER_SIZE) as u32) != 0) as u32);
let fragment_count = (overrun_size / (UDP_DEFAULT_MTU - v1::FRAGMENT_HEADER_SIZE) as u32)
+ (((overrun_size % (UDP_DEFAULT_MTU - v1::FRAGMENT_HEADER_SIZE) as u32) != 0) as u32);
debug_assert!(fragment_count <= v1::FRAGMENT_COUNT_MAX as u32);
let mut header = v1::FragmentHeader {
@ -355,7 +372,16 @@ impl<SI: SystemInterface> Peer<SI> {
loop {
header.total_and_fragment_no += 1;
let next_pos = pos + chunk_size;
if !si.wire_send(endpoint, local_socket, local_interface, &[header.as_bytes(), &bytes[pos..next_pos]], 0).await {
if !si
.wire_send(
endpoint,
local_socket,
local_interface,
&[header.as_bytes(), &bytes[pos..next_pos]],
0,
)
.await
{
return false;
}
pos = next_pos;
@ -376,7 +402,14 @@ impl<SI: SystemInterface> Peer<SI> {
/// via a root or some other route.
///
/// It encrypts and sets the MAC and cipher fields and packet ID and other things.
pub(crate) async fn send(&self, si: &SI, path: Option<&Arc<Path<SI>>>, node: &Node<SI>, time_ticks: i64, packet: &mut PacketBuffer) -> bool {
pub(crate) async fn send(
&self,
si: &SI,
path: Option<&Arc<Path<SI>>>,
node: &Node<SI>,
time_ticks: i64,
packet: &mut PacketBuffer,
) -> bool {
let mut _path_arc = None;
let path = if let Some(path) = path {
path
@ -402,7 +435,11 @@ impl<SI: SystemInterface> Peer<SI> {
let mut aes_gmac_siv = self.identity_symmetric_key.aes_gmac_siv.get();
aes_gmac_siv.encrypt_init(&self.next_message_id().to_ne_bytes());
aes_gmac_siv.encrypt_set_aad(&v1::get_packet_aad_bytes(self.identity.address, node.identity.address, flags_cipher_hops));
aes_gmac_siv.encrypt_set_aad(&v1::get_packet_aad_bytes(
self.identity.address,
node.identity.address,
flags_cipher_hops,
));
if let Ok(payload) = packet.as_bytes_starting_at_mut(v1::HEADER_SIZE) {
aes_gmac_siv.encrypt_first_pass(payload);
aes_gmac_siv.encrypt_first_pass_finish();
@ -419,7 +456,14 @@ impl<SI: SystemInterface> Peer<SI> {
}
if self
.internal_send(si, &path.endpoint, Some(&path.local_socket), Some(&path.local_interface), max_fragment_size, packet)
.internal_send(
si,
&path.endpoint,
Some(&path.local_socket),
Some(&path.local_interface),
max_fragment_size,
packet,
)
.await
{
self.last_send_time_ticks.store(time_ticks, Ordering::Relaxed);
@ -438,7 +482,16 @@ impl<SI: SystemInterface> Peer<SI> {
/// Intermediates don't need to adjust fragmentation.
pub(crate) async fn forward(&self, si: &SI, time_ticks: i64, packet: &PacketBuffer) -> bool {
if let Some(path) = self.direct_path() {
if si.wire_send(&path.endpoint, Some(&path.local_socket), Some(&path.local_interface), &[packet.as_bytes()], 0).await {
if si
.wire_send(
&path.endpoint,
Some(&path.local_socket),
Some(&path.local_interface),
&[packet.as_bytes()],
0,
)
.await
{
self.last_forward_time_ticks.store(time_ticks, Ordering::Relaxed);
return true;
}
@ -479,7 +532,8 @@ impl<SI: SystemInterface> Peer<SI> {
let message_id = self.next_message_id();
{
let f: &mut (v1::PacketHeader, v1::message_component_structs::HelloFixedHeaderFields) = packet.append_struct_get_mut().unwrap();
let f: &mut (v1::PacketHeader, v1::message_component_structs::HelloFixedHeaderFields) =
packet.append_struct_get_mut().unwrap();
f.0.id = message_id.to_ne_bytes();
f.0.dest = self.identity.address.to_bytes();
f.0.src = node.identity.address.to_bytes();
@ -495,17 +549,37 @@ impl<SI: SystemInterface> Peer<SI> {
debug_assert_eq!(packet.len(), 41);
assert!(packet.append_bytes((&node.identity.to_public_bytes()).into()).is_ok());
let (_, poly1305_key) = salsa_poly_create(&self.identity_symmetric_key, packet.struct_at::<v1::PacketHeader>(0).unwrap(), packet.len());
let (_, poly1305_key) = salsa_poly_create(
&self.identity_symmetric_key,
packet.struct_at::<v1::PacketHeader>(0).unwrap(),
packet.len(),
);
let mac = poly1305::compute(&poly1305_key, packet.as_bytes_starting_at(v1::HEADER_SIZE).unwrap());
packet.as_mut()[v1::MAC_FIELD_INDEX..v1::MAC_FIELD_INDEX + 8].copy_from_slice(&mac[0..8]);
self.last_send_time_ticks.store(time_ticks, Ordering::Relaxed);
debug_event!(si, "HELLO -> {} @ {} ({} bytes)", self.identity.address.to_string(), destination.to_string(), packet.len());
debug_event!(
si,
"HELLO -> {} @ {} ({} bytes)",
self.identity.address.to_string(),
destination.to_string(),
packet.len()
);
}
if let Some(p) = path.as_ref() {
if self.internal_send(si, destination, Some(&p.local_socket), Some(&p.local_interface), max_fragment_size, &packet).await {
if self
.internal_send(
si,
destination,
Some(&p.local_socket),
Some(&p.local_interface),
max_fragment_size,
&packet,
)
.await
{
p.log_send_anything(time_ticks);
true
} else {
@ -536,7 +610,13 @@ impl<SI: SystemInterface> Peer<SI> {
if let Ok(packet_frag0_payload_bytes) = frag0.as_bytes_starting_at(v1::VERB_INDEX) {
let mut payload = PacketBuffer::new();
let message_id = if let Some(message_id2) = try_aead_decrypt(&self.identity_symmetric_key, packet_frag0_payload_bytes, packet_header, fragments, &mut payload) {
let message_id = if let Some(message_id2) = try_aead_decrypt(
&self.identity_symmetric_key,
packet_frag0_payload_bytes,
packet_header,
fragments,
&mut payload,
) {
// Decryption successful with static secret.
message_id2
} else {
@ -582,13 +662,25 @@ impl<SI: SystemInterface> Peer<SI> {
if match verb {
verbs::VL1_NOP => true,
verbs::VL1_HELLO => self.handle_incoming_hello(si, ph, node, time_ticks, message_id, source_path, packet_header.hops(), &payload).await,
verbs::VL1_HELLO => {
self.handle_incoming_hello(si, ph, node, time_ticks, message_id, source_path, packet_header.hops(), &payload)
.await
}
verbs::VL1_ERROR => self.handle_incoming_error(si, ph, node, time_ticks, source_path, &payload).await,
verbs::VL1_OK => self.handle_incoming_ok(si, ph, node, time_ticks, source_path, packet_header.hops(), path_is_known, &payload).await,
verbs::VL1_OK => {
self.handle_incoming_ok(si, ph, node, time_ticks, source_path, packet_header.hops(), path_is_known, &payload)
.await
}
verbs::VL1_WHOIS => self.handle_incoming_whois(si, ph, node, time_ticks, message_id, &payload).await,
verbs::VL1_RENDEZVOUS => self.handle_incoming_rendezvous(si, node, time_ticks, message_id, source_path, &payload).await,
verbs::VL1_RENDEZVOUS => {
self.handle_incoming_rendezvous(si, node, time_ticks, message_id, source_path, &payload)
.await
}
verbs::VL1_ECHO => self.handle_incoming_echo(si, ph, node, time_ticks, message_id, &payload).await,
verbs::VL1_PUSH_DIRECT_PATHS => self.handle_incoming_push_direct_paths(si, node, time_ticks, source_path, &payload).await,
verbs::VL1_PUSH_DIRECT_PATHS => {
self.handle_incoming_push_direct_paths(si, node, time_ticks, source_path, &payload)
.await
}
verbs::VL1_USER_MESSAGE => self.handle_incoming_user_message(si, node, time_ticks, source_path, &payload).await,
_ => ph.handle_packet(self, &source_path, verb, &payload).await,
} {
@ -614,7 +706,11 @@ impl<SI: SystemInterface> Peer<SI> {
payload: &PacketBuffer,
) -> bool {
if !(ph.should_communicate_with(&self.identity) || node.this_node_is_root() || node.is_peer_root(self)) {
debug_event!(si, "[vl1] dropping HELLO from {} due to lack of trust relationship", self.identity.address.to_string());
debug_event!(
si,
"[vl1] dropping HELLO from {} due to lack of trust relationship",
self.identity.address.to_string()
);
return true; // packet wasn't invalid, just ignored
}
@ -633,7 +729,10 @@ impl<SI: SystemInterface> Peer<SI> {
let mut packet = PacketBuffer::new();
packet.set_size(v1::HEADER_SIZE);
{
let f: &mut (v1::message_component_structs::OkHeader, v1::message_component_structs::OkHelloFixedHeaderFields) = packet.append_struct_get_mut().unwrap();
let f: &mut (
v1::message_component_structs::OkHeader,
v1::message_component_structs::OkHelloFixedHeaderFields,
) = packet.append_struct_get_mut().unwrap();
f.0.verb = verbs::VL1_OK;
f.0.in_re_verb = verbs::VL1_HELLO;
f.0.in_re_message_id = message_id.to_ne_bytes();
@ -651,7 +750,15 @@ impl<SI: SystemInterface> Peer<SI> {
return false;
}
async fn handle_incoming_error<PH: InnerProtocolInterface>(&self, _si: &SI, ph: &PH, _node: &Node<SI>, _time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) -> bool {
async fn handle_incoming_error<PH: InnerProtocolInterface>(
&self,
_si: &SI,
ph: &PH,
_node: &Node<SI>,
_time_ticks: i64,
source_path: &Arc<Path<SI>>,
payload: &PacketBuffer,
) -> bool {
let mut cursor = 0;
if let Ok(error_header) = payload.read_struct::<v1::message_component_structs::ErrorHeader>(&mut cursor) {
let in_re_message_id: MessageId = u64::from_ne_bytes(error_header.in_re_message_id);
@ -659,7 +766,15 @@ impl<SI: SystemInterface> Peer<SI> {
match error_header.in_re_verb {
_ => {
return ph
.handle_error(self, &source_path, error_header.in_re_verb, in_re_message_id, error_header.error_code, payload, &mut cursor)
.handle_error(
self,
&source_path,
error_header.in_re_verb,
in_re_message_id,
error_header.error_code,
payload,
&mut cursor,
)
.await;
}
}
@ -685,12 +800,20 @@ impl<SI: SystemInterface> Peer<SI> {
if self.message_id_counter.load(Ordering::Relaxed).wrapping_sub(in_re_message_id) <= PACKET_RESPONSE_COUNTER_DELTA_MAX {
match ok_header.in_re_verb {
verbs::VL1_HELLO => {
if let Ok(ok_hello_fixed_header_fields) = payload.read_struct::<v1::message_component_structs::OkHelloFixedHeaderFields>(&mut cursor) {
if let Ok(ok_hello_fixed_header_fields) =
payload.read_struct::<v1::message_component_structs::OkHelloFixedHeaderFields>(&mut cursor)
{
if hops == 0 {
if let Ok(reported_endpoint) = Endpoint::unmarshal(&payload, &mut cursor) {
#[cfg(debug_assertions)]
let reported_endpoint2 = reported_endpoint.clone();
if self.remote_node_info.write().reported_local_endpoints.insert(reported_endpoint, time_ticks).is_none() {
if self
.remote_node_info
.write()
.reported_local_endpoints
.insert(reported_endpoint, time_ticks)
.is_none()
{
#[cfg(debug_assertions)]
debug_event!(
si,
@ -725,7 +848,9 @@ impl<SI: SystemInterface> Peer<SI> {
}
_ => {
return ph.handle_ok(self, &source_path, ok_header.in_re_verb, in_re_message_id, payload, &mut cursor).await;
return ph
.handle_ok(self, &source_path, ok_header.in_re_verb, in_re_message_id, payload, &mut cursor)
.await;
}
}
}
@ -733,7 +858,15 @@ impl<SI: SystemInterface> Peer<SI> {
return false;
}
async fn handle_incoming_whois<PH: InnerProtocolInterface>(&self, si: &SI, ph: &PH, node: &Node<SI>, time_ticks: i64, message_id: MessageId, payload: &PacketBuffer) -> bool {
async fn handle_incoming_whois<PH: InnerProtocolInterface>(
&self,
si: &SI,
ph: &PH,
node: &Node<SI>,
time_ticks: i64,
message_id: MessageId,
payload: &PacketBuffer,
) -> bool {
if node.this_node_is_root() || ph.should_communicate_with(&self.identity) {
let mut packet = PacketBuffer::new();
packet.set_size(v1::HEADER_SIZE);
@ -763,12 +896,28 @@ impl<SI: SystemInterface> Peer<SI> {
}
#[allow(unused)]
async fn handle_incoming_rendezvous(&self, si: &SI, node: &Node<SI>, time_ticks: i64, message_id: MessageId, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) -> bool {
async fn handle_incoming_rendezvous(
&self,
si: &SI,
node: &Node<SI>,
time_ticks: i64,
message_id: MessageId,
source_path: &Arc<Path<SI>>,
payload: &PacketBuffer,
) -> bool {
if node.is_peer_root(self) {}
return true;
}
async fn handle_incoming_echo<PH: InnerProtocolInterface>(&self, si: &SI, ph: &PH, node: &Node<SI>, time_ticks: i64, message_id: MessageId, payload: &PacketBuffer) -> bool {
async fn handle_incoming_echo<PH: InnerProtocolInterface>(
&self,
si: &SI,
ph: &PH,
node: &Node<SI>,
time_ticks: i64,
message_id: MessageId,
payload: &PacketBuffer,
) -> bool {
if ph.should_communicate_with(&self.identity) || node.is_peer_root(self) {
let mut packet = PacketBuffer::new();
packet.set_size(v1::HEADER_SIZE);
@ -784,18 +933,36 @@ impl<SI: SystemInterface> Peer<SI> {
false
}
} else {
debug_event!(si, "[vl1] dropping ECHO from {} due to lack of trust relationship", self.identity.address.to_string());
debug_event!(
si,
"[vl1] dropping ECHO from {} due to lack of trust relationship",
self.identity.address.to_string()
);
true // packet wasn't invalid, just ignored
}
}
#[allow(unused)]
async fn handle_incoming_push_direct_paths(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) -> bool {
async fn handle_incoming_push_direct_paths(
&self,
si: &SI,
node: &Node<SI>,
time_ticks: i64,
source_path: &Arc<Path<SI>>,
payload: &PacketBuffer,
) -> bool {
false
}
#[allow(unused)]
async fn handle_incoming_user_message(&self, si: &SI, node: &Node<SI>, time_ticks: i64, source_path: &Arc<Path<SI>>, payload: &PacketBuffer) -> bool {
async fn handle_incoming_user_message(
&self,
si: &SI,
node: &Node<SI>,
time_ticks: i64,
source_path: &Arc<Path<SI>>,
payload: &PacketBuffer,
) -> bool {
false
}
}

22
network-hypervisor/src/vl1/rootset.rs
View file

@ -90,7 +90,11 @@ impl RootSet {
/// Get the ZeroTier default root set, which contains roots run by ZeroTier Inc.
pub fn zerotier_default() -> Self {
let mut cursor = 0;
let rs = Self::unmarshal(&Buffer::from(include_bytes!("../../default-rootset/root.zerotier.com.bin")), &mut cursor).unwrap();
let rs = Self::unmarshal(
&Buffer::from(include_bytes!("../../default-rootset/root.zerotier.com.bin")),
&mut cursor,
)
.unwrap();
assert!(rs.verify());
rs
}
@ -156,7 +160,13 @@ impl RootSet {
}
/// Add a member to this definition, replacing any current entry with this address.
pub fn add<'a, I: Iterator<Item = &'a Endpoint>>(&mut self, member_identity: &Identity, endpoints: Option<I>, priority: u8, protocol_version: u8) {
pub fn add<'a, I: Iterator<Item = &'a Endpoint>>(
&mut self,
member_identity: &Identity,
endpoints: Option<I>,
priority: u8,
protocol_version: u8,
) {
self.members.retain(|m| m.identity.address != member_identity.address);
let _ = self.members.push(Root {
identity: member_identity.clone_without_secret(),
@ -263,11 +273,15 @@ impl Marshalable for RootSet {
}
let name_len = buf.read_varint(cursor)?;
rc.name = String::from_utf8(buf.read_bytes(name_len as usize, cursor)?.to_vec()).map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid UTF8"))?;
rc.name = String::from_utf8(buf.read_bytes(name_len as usize, cursor)?.to_vec())
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid UTF8"))?;
let url_len = buf.read_varint(cursor)?;
if url_len > 0 {
rc.url = Some(String::from_utf8(buf.read_bytes(url_len as usize, cursor)?.to_vec()).map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid UTF8"))?);
rc.url = Some(
String::from_utf8(buf.read_bytes(url_len as usize, cursor)?.to_vec())
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid UTF8"))?,
);
}
rc.revision = buf.read_varint(cursor)?;

5
network-hypervisor/src/vl2/networkid.rs
View file

@ -67,7 +67,10 @@ impl Marshalable for NetworkId {
#[inline(always)]
fn unmarshal<const BL: usize>(buf: &Buffer<BL>, cursor: &mut usize) -> std::io::Result<Self> {
Self::from_u64(buf.read_u64(cursor)?).map_or_else(|| Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "cannot be zero")), |a| Ok(a))
Self::from_u64(buf.read_u64(cursor)?).map_or_else(
|| Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "cannot be zero")),
|a| Ok(a),
)
}
}

21
network-hypervisor/src/vl2/switch.rs
View file

@ -18,12 +18,29 @@ impl InnerProtocolInterface for Switch {
}
#[allow(unused)]
async fn handle_error<SI: SystemInterface>(&self, peer: &Peer<SI>, source_path: &Path<SI>, in_re_verb: u8, in_re_message_id: u64, error_code: u8, payload: &PacketBuffer, cursor: &mut usize) -> bool {
async fn handle_error<SI: SystemInterface>(
&self,
peer: &Peer<SI>,
source_path: &Path<SI>,
in_re_verb: u8,
in_re_message_id: u64,
error_code: u8,
payload: &PacketBuffer,
cursor: &mut usize,
) -> bool {
false
}
#[allow(unused)]
async fn handle_ok<SI: SystemInterface>(&self, peer: &Peer<SI>, source_path: &Path<SI>, in_re_verb: u8, in_re_message_id: u64, payload: &PacketBuffer, cursor: &mut usize) -> bool {
async fn handle_ok<SI: SystemInterface>(
&self,
peer: &Peer<SI>,
source_path: &Path<SI>,
in_re_verb: u8,
in_re_message_id: u64,
payload: &PacketBuffer,
cursor: &mut usize,
) -> bool {
false
}

4
rustfmt.toml
View file

@ -1,8 +1,8 @@
#unstable_features = true
max_width = 200
max_width = 140
#use_small_heuristics = "Max"
edition = "2021"
empty_item_single_line = true
#empty_item_single_line = true
newline_style = "Unix"
struct_lit_width = 60
tab_spaces = 4

9
system-service/src/datadir.rs
View file

@ -32,7 +32,10 @@ impl DataDir {
if !base_path.is_dir() {
let _ = std::fs::create_dir_all(&base_path);
if !base_path.is_dir() {
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "base path not found and cannot be created"));
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"base path not found and cannot be created",
));
}
}
@ -57,7 +60,9 @@ impl DataDir {
/// Load identity.secret from data directory.
pub async fn load_identity(&self) -> std::io::Result<Identity> {
let id_data = Identity::from_str(String::from_utf8_lossy(read_limit(self.base_path.join(IDENTITY_SECRET_FILENAME), 4096).await?.as_slice()).as_ref());
let id_data = Identity::from_str(
String::from_utf8_lossy(read_limit(self.base_path.join(IDENTITY_SECRET_FILENAME), 4096).await?.as_slice()).as_ref(),
);
if id_data.is_err() {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, id_data.err().unwrap()));
}

16
system-service/src/getifaddrs.rs
View file

@ -27,9 +27,17 @@ pub fn for_each_address<F: FnMut(&InetAddress, &LocalInterface)>(mut f: F) {
let sa_family = (*(*i).ifa_addr).sa_family as u8;
if sa_family == libc::AF_INET as u8 {
copy_nonoverlapping((*i).ifa_addr.cast::<u8>(), (&mut a as *mut InetAddress).cast::<u8>(), size_of::<libc::sockaddr_in>());
copy_nonoverlapping(
(*i).ifa_addr.cast::<u8>(),
(&mut a as *mut InetAddress).cast::<u8>(),
size_of::<libc::sockaddr_in>(),
);
} else if sa_family == libc::AF_INET6 as u8 {
copy_nonoverlapping((*i).ifa_addr.cast::<u8>(), (&mut a as *mut InetAddress).cast::<u8>(), size_of::<libc::sockaddr_in6>());
copy_nonoverlapping(
(*i).ifa_addr.cast::<u8>(),
(&mut a as *mut InetAddress).cast::<u8>(),
size_of::<libc::sockaddr_in6>(),
);
} else {
i = (*i).ifa_next;
continue;
@ -87,7 +95,9 @@ mod tests {
#[test]
fn test_getifaddrs() {
println!("starting getifaddrs...");
crate::getifaddrs::for_each_address(|a: &InetAddress, interface: &LocalInterface| println!(" {} {}", interface.to_string(), a.to_string()));
crate::getifaddrs::for_each_address(|a: &InetAddress, interface: &LocalInterface| {
println!(" {} {}", interface.to_string(), a.to_string())
});
println!("done.")
}
}

6
system-service/src/ipv6.rs
View file

@ -56,7 +56,11 @@ mod freebsd_like {
let device_name_bytes = device_name.as_bytes();
assert!(device_name_bytes.len() <= 15);
ifr6.ifr_name[..device_name_bytes.len()].copy_from_slice(device_name_bytes);
std::ptr::copy_nonoverlapping((address as *const InetAddress).cast(), &mut ifr6.ifr_ifru.ifru_addr, size_of::<libc::sockaddr_in6>());
std::ptr::copy_nonoverlapping(
(address as *const InetAddress).cast(),
&mut ifr6.ifr_ifru.ifru_addr,
size_of::<libc::sockaddr_in6>(),
);
if libc::ioctl((*info_socket).as_(), SIOCGIFAFLAG_IN6, &mut ifr6 as *mut in6_ifreq) != -1 {
if (ifr6.ifr_ifru.ifru_flags6 & IN6_IFF_TEMPORARY) != 0 {
return true;

17
system-service/src/localconfig.rs
View file

@ -9,12 +9,17 @@ use zerotier_network_hypervisor::vl2::NetworkId;
/// A list of unassigned or obsolete ports under 1024 that could possibly be squatted.
pub const UNASSIGNED_PRIVILEGED_PORTS: [u16; 299] = [
4, 6, 8, 10, 12, 14, 15, 16, 26, 28, 30, 32, 34, 36, 40, 60, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 285, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 323, 324, 325, 326, 327, 328,
329, 330, 331, 332, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 703, 708, 713, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728, 732, 733, 734, 735, 736, 737, 738, 739, 740, 743, 745, 746, 755, 756, 766, 768, 778, 779,
781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, 802, 803, 804, 805, 806, 807, 808, 809, 811, 812, 813, 814, 815, 816, 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 834, 835, 836, 837, 838, 839,
840, 841, 842, 843, 844, 845, 846, 849, 850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, 884, 885, 889, 890, 891, 892, 893, 894, 895, 896, 897,
898, 899, 904, 905, 906, 907, 908, 909, 910, 911, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 952, 953,
954, 955, 956, 957, 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1023,
4, 6, 8, 10, 12, 14, 15, 16, 26, 28, 30, 32, 34, 36, 40, 60, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 285, 288, 289, 290,
291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332,
334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 703, 708, 713, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727,
728, 732, 733, 734, 735, 736, 737, 738, 739, 740, 743, 745, 746, 755, 756, 766, 768, 778, 779, 781, 782, 783, 784, 785, 786, 787, 788,
789, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, 802, 803, 804, 805, 806, 807, 808, 809, 811, 812, 813, 814, 815, 816, 817, 818,
819, 820, 821, 822, 823, 824, 825, 826, 827, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 849, 850, 851, 852, 853,
854, 855, 856, 857, 858, 859, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883,
884, 885, 889, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 904, 905, 906, 907, 908, 909, 910, 911, 914, 915, 916, 917, 918, 919,
920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, 946,
947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973,
974, 975, 976, 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1023,
];
/// Default primary ZeroTier port.

63
system-service/src/main.rs
View file

@ -99,17 +99,36 @@ fn main() {
.subcommand(Command::new("secondaryport").arg(Arg::new("port#").index(1).validator(utils::is_valid_port)))
.subcommand(
Command::new("blacklist")
.subcommand(Command::new("cidr").arg(Arg::new("ip_bits").index(1)).arg(Arg::new("boolean").index(2).validator(utils::is_valid_bool)))
.subcommand(Command::new("if").arg(Arg::new("prefix").index(1)).arg(Arg::new("boolean").index(2).validator(utils::is_valid_bool))),
.subcommand(
Command::new("cidr")
.arg(Arg::new("ip_bits").index(1))
.arg(Arg::new("boolean").index(2).validator(utils::is_valid_bool)),
)
.subcommand(
Command::new("if")
.arg(Arg::new("prefix").index(1))
.arg(Arg::new("boolean").index(2).validator(utils::is_valid_bool)),
),
)
.subcommand(Command::new("portmap").arg(Arg::new("boolean").index(1).validator(utils::is_valid_bool))),
)
.subcommand(Command::new("peer").subcommand(Command::new("show").arg(Arg::new("address").index(1).required(true))).subcommand(Command::new("list")).subcommand(Command::new("listroots")).subcommand(Command::new("try")))
.subcommand(
Command::new("peer")
.subcommand(Command::new("show").arg(Arg::new("address").index(1).required(true)))
.subcommand(Command::new("list"))
.subcommand(Command::new("listroots"))
.subcommand(Command::new("try")),
)
.subcommand(
Command::new("network")
.subcommand(Command::new("show").arg(Arg::new("nwid").index(1).required(true)))
.subcommand(Command::new("list"))
.subcommand(Command::new("set").arg(Arg::new("nwid").index(1).required(true)).arg(Arg::new("setting").index(2).required(false)).arg(Arg::new("value").index(3).required(false))),
.subcommand(
Command::new("set")
.arg(Arg::new("nwid").index(1).required(true))
.arg(Arg::new("setting").index(2).required(false))
.arg(Arg::new("value").index(3).required(false)),
),
)
.subcommand(Command::new("join").arg(Arg::new("nwid").index(1).required(true)))
.subcommand(Command::new("leave").arg(Arg::new("nwid").index(1).required(true)))
@ -120,15 +139,28 @@ fn main() {
.subcommand(Command::new("getpublic").arg(Arg::new("identity").index(1).required(true)))
.subcommand(Command::new("fingerprint").arg(Arg::new("identity").index(1).required(true)))
.subcommand(Command::new("validate").arg(Arg::new("identity").index(1).required(true)))
.subcommand(Command::new("sign").arg(Arg::new("identity").index(1).required(true)).arg(Arg::new("path").index(2).required(true)))
.subcommand(Command::new("verify").arg(Arg::new("identity").index(1).required(true)).arg(Arg::new("path").index(2).required(true)).arg(Arg::new("signature").index(3).required(true))),
.subcommand(
Command::new("sign")
.arg(Arg::new("identity").index(1).required(true))
.arg(Arg::new("path").index(2).required(true)),
)
.subcommand(
Command::new("verify")
.arg(Arg::new("identity").index(1).required(true))
.arg(Arg::new("path").index(2).required(true))
.arg(Arg::new("signature").index(3).required(true)),
),
)
.subcommand(
Command::new("rootset")
.subcommand(Command::new("add").arg(Arg::new("path").index(1).required(true)))
.subcommand(Command::new("remove").arg(Arg::new("name").index(1).required(true)))
.subcommand(Command::new("list"))
.subcommand(Command::new("sign").arg(Arg::new("path").index(1).required(true)).arg(Arg::new("secret").index(2).required(true)))
.subcommand(
Command::new("sign")
.arg(Arg::new("path").index(1).required(true))
.arg(Arg::new("secret").index(2).required(true)),
)
.subcommand(Command::new("verify").arg(Arg::new("path").index(1).required(true)))
.subcommand(Command::new("marshal").arg(Arg::new("path").index(1).required(true)))
.subcommand(Command::new("restoredefault")),
@ -163,7 +195,10 @@ fn main() {
if suggested.is_empty() {
eprintln!("Unrecognized option '{}'. Use 'help' for help.", invalid);
} else {
eprintln!("Unrecognized option '{}', did you mean {}? Use 'help' for help.", invalid, suggested);
eprintln!(
"Unrecognized option '{}', did you mean {}? Use 'help' for help.",
invalid, suggested
);
}
}
std::process::exit(exitcode::ERR_USAGE);
@ -173,10 +208,18 @@ fn main() {
let flags = Flags {
json_output: global_args.is_present("json"),
base_path: global_args.value_of("path").map_or_else(platform_default_home_path, |p| p.to_string()),
base_path: global_args
.value_of("path")
.map_or_else(platform_default_home_path, |p| p.to_string()),
auth_token_path_override: global_args.value_of("token_path").map(|p| p.to_string()),
auth_token_override: global_args.value_of("token").map(|t| t.to_string()),
};
std::process::exit(tokio::runtime::Builder::new_multi_thread().enable_all().build().unwrap().block_on(async_main(flags, global_args)));
std::process::exit(
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap()
.block_on(async_main(flags, global_args)),
);
}

39
system-service/src/service.rs
View file

@ -63,7 +63,11 @@ impl Service {
///
/// This launches a number of background tasks in the async runtime that will run as long as this object exists.
/// When this is dropped these tasks are killed.
pub async fn new<P: AsRef<Path>>(rt: tokio::runtime::Handle, base_path: P, auto_upgrade_identity: bool) -> Result<Self, Box<dyn Error>> {
pub async fn new<P: AsRef<Path>>(
rt: tokio::runtime::Handle,
base_path: P,
auto_upgrade_identity: bool,
) -> Result<Self, Box<dyn Error>> {
let mut si = ServiceImpl {
rt,
data: DataDir::open(base_path).await.map_err(|e| Box::new(e))?,
@ -126,8 +130,14 @@ impl ServiceImpl {
let mut buf = core.get_packet_buffer();
if let Ok((bytes, source)) = socket.recv_from(unsafe { buf.entire_buffer_mut() }).await {
unsafe { buf.set_size_unchecked(bytes) };
core.handle_incoming_physical_packet(&self2, &Endpoint::IpUdp(InetAddress::from(source)), &local_socket, &interface, buf)
.await;
core.handle_incoming_physical_packet(
&self2,
&Endpoint::IpUdp(InetAddress::from(source)),
&local_socket,
&interface,
buf,
)
.await;
} else {
break;
}
@ -144,7 +154,11 @@ impl ServiceImpl {
let config = self.data.config().await;
if let Some(errors) = self
.update_udp_bindings_for_port(config.settings.primary_port, &config.settings.interface_prefix_blacklist, &config.settings.cidr_blacklist)
.update_udp_bindings_for_port(
config.settings.primary_port,
&config.settings.interface_prefix_blacklist,
&config.settings.cidr_blacklist,
)
.await
{
for e in errors.iter() {
@ -193,7 +207,14 @@ impl SystemInterface for ServiceImpl {
assert!(self.data.save_identity(id).await.is_ok())
}
async fn wire_send(&self, endpoint: &Endpoint, local_socket: Option<&Self::LocalSocket>, local_interface: Option<&Self::LocalInterface>, data: &[&[u8]], packet_ttl: u8) -> bool {
async fn wire_send(
&self,
endpoint: &Endpoint,
local_socket: Option<&Self::LocalSocket>,
local_interface: Option<&Self::LocalInterface>,
data: &[&[u8]],
packet_ttl: u8,
) -> bool {
match endpoint {
Endpoint::IpUdp(address) => {
// This is the fast path -- the socket is known to the core so just send it.
@ -251,7 +272,13 @@ impl SystemInterface for ServiceImpl {
return false;
}
async fn check_path(&self, _id: &Identity, endpoint: &Endpoint, _local_socket: Option<&Self::LocalSocket>, _local_interface: Option<&Self::LocalInterface>) -> bool {
async fn check_path(
&self,
_id: &Identity,
endpoint: &Endpoint,
_local_socket: Option<&Self::LocalSocket>,
_local_interface: Option<&Self::LocalInterface>,
) -> bool {
let config = self.data.config().await;
if let Some(pps) = config.physical.get(endpoint) {
!pps.blacklist

115
system-service/src/udp.rs
View file

@ -52,7 +52,15 @@ impl BoundUdpSocket {
#[inline(always)]
fn set_ttl(&self, packet_ttl: u8) {
let ttl = packet_ttl as libc::c_int;
unsafe { libc::setsockopt(self.fd.as_(), libc::IPPROTO_IP.as_(), libc::IP_TOS.as_(), (&ttl as *const libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_()) };
unsafe {
libc::setsockopt(
self.fd.as_(),
libc::IPPROTO_IP.as_(),
libc::IP_TOS.as_(),
(&ttl as *const libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
)
};
}
#[cfg(any(target_os = "macos", target_os = "freebsd"))]
@ -65,7 +73,14 @@ impl BoundUdpSocket {
unsafe {
if b.len() == 1 {
let bb = *b.get_unchecked(0);
ok = libc::sendto(self.fd.as_(), bb.as_ptr().cast(), bb.len().as_(), 0, (dest as *const InetAddress).cast(), size_of::<InetAddress>().as_()) > 0;
ok = libc::sendto(
self.fd.as_(),
bb.as_ptr().cast(),
bb.len().as_(),
0,
(dest as *const InetAddress).cast(),
size_of::<InetAddress>().as_(),
) > 0;
} else {
let mut iov: [libc::iovec; 16] = MaybeUninit::uninit().assume_init();
assert!(b.len() <= iov.len());
@ -143,10 +158,17 @@ impl BoundUdpPort {
/// The caller can check the 'sockets' member variable after calling to determine which if any bindings were
/// successful. Any errors that occurred are returned as tuples of (interface, address, error). The second vector
/// returned contains newly bound sockets.
pub fn update_bindings(&mut self, interface_prefix_blacklist: &Vec<String>, cidr_blacklist: &Vec<InetAddress>) -> (Vec<(LocalInterface, InetAddress, std::io::Error)>, Vec<Arc<BoundUdpSocket>>) {
pub fn update_bindings(
&mut self,
interface_prefix_blacklist: &Vec<String>,
cidr_blacklist: &Vec<InetAddress>,
) -> (Vec<(LocalInterface, InetAddress, std::io::Error)>, Vec<Arc<BoundUdpSocket>>) {
let mut existing_bindings: HashMap<LocalInterface, HashMap<InetAddress, Arc<BoundUdpSocket>>> = HashMap::with_capacity(4);
for s in self.sockets.drain(..) {
existing_bindings.entry(s.interface).or_insert_with(|| HashMap::with_capacity(4)).insert(s.address.clone(), s);
existing_bindings
.entry(s.interface)
.or_insert_with(|| HashMap::with_capacity(4))
.insert(s.address.clone(), s);
}
let mut errors = Vec::new();
@ -156,7 +178,10 @@ impl BoundUdpPort {
let mut addr_with_port = address.clone();
addr_with_port.set_port(self.port);
if address.is_ip()
&& matches!(address.scope(), IpScope::Global | IpScope::PseudoPrivate | IpScope::Private | IpScope::Shared)
&& matches!(
address.scope(),
IpScope::Global | IpScope::PseudoPrivate | IpScope::Private | IpScope::Shared
)
&& !interface_prefix_blacklist.iter().any(|pfx| interface_str.starts_with(pfx.as_str()))
&& !cidr_blacklist.iter().any(|r| address.is_within(r))
&& !ipv6::is_ipv6_temporary(interface_str.as_str(), address)
@ -187,7 +212,11 @@ impl BoundUdpPort {
errors.push((interface.clone(), addr_with_port, s.err().unwrap()));
}
} else {
errors.push((interface.clone(), addr_with_port, std::io::Error::new(std::io::ErrorKind::Other, s.err().unwrap())));
errors.push((
interface.clone(),
addr_with_port,
std::io::Error::new(std::io::ErrorKind::Other, s.err().unwrap()),
));
}
}
}
@ -220,15 +249,33 @@ unsafe fn bind_udp_to_device(device_name: &str, address: &InetAddress) -> Result
let mut fl: libc::c_int;
fl = 1;
setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_REUSEPORT.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_());
setsockopt_results |= libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_REUSEPORT.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
);
debug_assert!(setsockopt_results == 0);
fl = 1;
setsockopt_results |= libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_BROADCAST.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_());
setsockopt_results |= libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_BROADCAST.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
);
debug_assert!(setsockopt_results == 0);
if af == AF_INET6 {
fl = 1;
setsockopt_results |= libc::setsockopt(s, libc::IPPROTO_IPV6.as_(), libc::IPV6_V6ONLY.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_());
setsockopt_results |= libc::setsockopt(
s,
libc::IPPROTO_IPV6.as_(),
libc::IPV6_V6ONLY.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
);
debug_assert!(setsockopt_results == 0);
}
@ -237,7 +284,13 @@ unsafe fn bind_udp_to_device(device_name: &str, address: &InetAddress) -> Result
if !device_name.is_empty() {
let _ = std::ffi::CString::new(device_name).map(|dn| {
let dnb = dn.as_bytes_with_nul();
let _ = libc::setsockopt(s.as_(), libc::SOL_SOCKET.as_(), libc::SO_BINDTODEVICE.as_(), dnb.as_ptr().cast(), (dnb.len() - 1).as_());
let _ = libc::setsockopt(
s.as_(),
libc::SOL_SOCKET.as_(),
libc::SO_BINDTODEVICE.as_(),
dnb.as_ptr().cast(),
(dnb.len() - 1).as_(),
);
});
}
}
@ -251,30 +304,62 @@ unsafe fn bind_udp_to_device(device_name: &str, address: &InetAddress) -> Result
#[cfg(not(target_os = "linux"))]
{
fl = 0;
libc::setsockopt(s, libc::IPPROTO_IP.as_(), libc::IP_DONTFRAG.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_());
libc::setsockopt(
s,
libc::IPPROTO_IP.as_(),
libc::IP_DONTFRAG.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
);
}
#[cfg(target_os = "linux")]
{
fl = libc::IP_PMTUDISC_DONT as libc::c_int;
libc::setsockopt(s, libc::IPPROTO_IP.as_(), libc::IP_MTU_DISCOVER.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_());
libc::setsockopt(
s,
libc::IPPROTO_IP.as_(),
libc::IP_MTU_DISCOVER.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
);
}
}
if af == AF_INET6 {
fl = 0;
libc::setsockopt(s, libc::IPPROTO_IPV6.as_(), libc::IPV6_DONTFRAG.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_());
libc::setsockopt(
s,
libc::IPPROTO_IPV6.as_(),
libc::IPV6_DONTFRAG.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
);
}
fl = 1048576;
while fl >= 65536 {
if libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_RCVBUF.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_()) == 0 {
if libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_RCVBUF.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
) == 0
{
break;
}
fl -= 65536;
}
fl = 1048576;
while fl >= 65536 {
if libc::setsockopt(s, libc::SOL_SOCKET.as_(), libc::SO_SNDBUF.as_(), (&mut fl as *mut libc::c_int).cast(), std::mem::size_of::<libc::c_int>().as_()) == 0 {
if libc::setsockopt(
s,
libc::SOL_SOCKET.as_(),
libc::SO_SNDBUF.as_(),
(&mut fl as *mut libc::c_int).cast(),
std::mem::size_of::<libc::c_int>().as_(),
) == 0
{
break;
}
fl -= 65536;

20
system-service/src/utils.rs
View file

@ -99,7 +99,11 @@ pub fn json_patch(target: &mut serde_json::value::Value, source: &serde_json::va
///
/// If there are no changes, None is returned. The depth limit is passed through to json_patch and
/// should be set to a sanity check value to prevent overflows.
pub fn json_patch_object<O: Serialize + DeserializeOwned + Eq>(obj: O, patch: &str, depth_limit: usize) -> Result<Option<O>, serde_json::Error> {
pub fn json_patch_object<O: Serialize + DeserializeOwned + Eq>(
obj: O,
patch: &str,
depth_limit: usize,
) -> Result<Option<O>, serde_json::Error> {
serde_json::from_str::<serde_json::value::Value>(patch).map_or_else(
|e| Err(e),
|patch| {
@ -156,7 +160,14 @@ pub async fn read_limit<P: AsRef<Path>>(path: P, limit: usize) -> std::io::Resul
pub fn fs_restrict_permissions<P: AsRef<Path>>(path: P) -> bool {
unsafe {
let c_path = std::ffi::CString::new(path.as_ref().to_str().unwrap()).unwrap();
libc::chmod(c_path.as_ptr(), if path.as_ref().is_dir() { 0o700 } else { 0o600 }) == 0
libc::chmod(
c_path.as_ptr(),
if path.as_ref().is_dir() {
0o700
} else {
0o600
},
) == 0
}
}
@ -177,7 +188,10 @@ pub async fn parse_cli_identity(input: &str, validate: bool) -> Result<Identity,
let input_p = Path::new(input);
if input_p.is_file() {
read_limit(input_p, 16384).await.map_or_else(|e| Err(e.to_string()), |v| String::from_utf8(v).map_or_else(|e| Err(e.to_string()), |s| parse_func(s.as_str())))
read_limit(input_p, 16384).await.map_or_else(
|e| Err(e.to_string()),
|v| String::from_utf8(v).map_or_else(|e| Err(e.to_string()), |s| parse_func(s.as_str())),
)
} else {
parse_func(input)
}

46
system-service/src/vnic/common.rs
View file

@ -9,7 +9,15 @@ use num_traits::AsPrimitive;
#[allow(unused_imports)]
use zerotier_network_hypervisor::vl1::MAC;
#[cfg(any(target_os = "macos", target_os = "ios", target_os = "netbsd", target_os = "openbsd", target_os = "dragonfly", target_os = "freebsd", target_os = "darwin"))]
#[cfg(any(
target_os = "macos",
target_os = "ios",
target_os = "netbsd",
target_os = "openbsd",
target_os = "dragonfly",
target_os = "freebsd",
target_os = "darwin"
))]
#[allow(non_camel_case_types)]
#[repr(C)]
struct ifmaddrs {
@ -19,13 +27,29 @@ struct ifmaddrs {
ifma_lladdr: *mut libc::sockaddr,
}
#[cfg(any(target_os = "macos", target_os = "ios", target_os = "netbsd", target_os = "openbsd", target_os = "dragonfly", target_os = "freebsd", target_os = "darwin"))]
#[cfg(any(
target_os = "macos",
target_os = "ios",
target_os = "netbsd",
target_os = "openbsd",
target_os = "dragonfly",
target_os = "freebsd",
target_os = "darwin"
))]
extern "C" {
fn getifmaddrs(ifmap: *mut *mut ifmaddrs) -> c_int;
fn freeifmaddrs(ifmp: *mut ifmaddrs);
}
#[cfg(any(target_os = "macos", target_os = "ios", target_os = "netbsd", target_os = "openbsd", target_os = "dragonfly", target_os = "freebsd", target_os = "darwin"))]
#[cfg(any(
target_os = "macos",
target_os = "ios",
target_os = "netbsd",
target_os = "openbsd",
target_os = "dragonfly",
target_os = "freebsd",
target_os = "darwin"
))]
pub fn get_l2_multicast_subscriptions(dev: &str) -> HashSet<MAC> {
let mut groups: HashSet<MAC> = HashSet::new();
let dev = dev.as_bytes();
@ -37,10 +61,20 @@ pub fn get_l2_multicast_subscriptions(dev: &str) -> HashSet<MAC> {
if !(*i).ifma_name.is_null() && !(*i).ifma_addr.is_null() && (*(*i).ifma_addr).sa_family as i32 == libc::AF_LINK as i32 {
let in_: &libc::sockaddr_dl = &*((*i).ifma_name.cast());
let la: &libc::sockaddr_dl = &*((*i).ifma_addr.cast());
if la.sdl_alen == 6 && in_.sdl_nlen <= dev.len().as_() && libc::memcmp(dev.as_ptr().cast(), in_.sdl_data.as_ptr().cast(), in_.sdl_nlen.as_()) == 0 {
if la.sdl_alen == 6
&& in_.sdl_nlen <= dev.len().as_()
&& libc::memcmp(dev.as_ptr().cast(), in_.sdl_data.as_ptr().cast(), in_.sdl_nlen.as_()) == 0
{
let mi = la.sdl_nlen as usize;
MAC::from_u64((la.sdl_data[mi] as u64) << 40 | (la.sdl_data[mi + 1] as u64) << 32 | (la.sdl_data[mi + 2] as u64) << 24 | (la.sdl_data[mi + 3] as u64) << 16 | (la.sdl_data[mi + 4] as u64) << 8 | la.sdl_data[mi + 5] as u64)
.map(|mac| groups.insert(mac));
MAC::from_u64(
(la.sdl_data[mi] as u64) << 40
| (la.sdl_data[mi + 1] as u64) << 32
| (la.sdl_data[mi + 2] as u64) << 24
| (la.sdl_data[mi + 3] as u64) << 16
| (la.sdl_data[mi + 4] as u64) << 8
| la.sdl_data[mi + 5] as u64,
)
.map(|mac| groups.insert(mac));
}
}
i = (*i).ifma_next;

212
system-service/src/vnic/mac_feth_tap.rs
View file

@ -225,7 +225,15 @@ fn device_ipv6_set_params(device: &String, perform_nud: bool, accept_ra: bool) -
}
let mut nd: in6_ndireq = zeroed();
copy_nonoverlapping(dev.as_ptr(), nd.ifname.as_mut_ptr().cast::<u8>(), if dev.len() > (nd.ifname.len() - 1) { nd.ifname.len() - 1 } else { dev.len() });
copy_nonoverlapping(
dev.as_ptr(),
nd.ifname.as_mut_ptr().cast::<u8>(),
if dev.len() > (nd.ifname.len() - 1) {
nd.ifname.len() - 1
} else {
dev.len()
},
);
if libc::ioctl(s, 76 /* SIOCGIFINFO_IN6 */, (&mut nd as *mut in6_ndireq).cast::<c_void>()) == 0 {
let oldflags = nd.ndi.flags;
if perform_nud {
@ -243,7 +251,15 @@ fn device_ipv6_set_params(device: &String, perform_nud: bool, accept_ra: bool) -
}
let mut ifr: in6_ifreq = zeroed();
copy_nonoverlapping(dev.as_ptr(), ifr.ifr_name.as_mut_ptr().cast::<u8>(), if dev.len() > (ifr.ifr_name.len() - 1) { ifr.ifr_name.len() - 1 } else { dev.len() });
copy_nonoverlapping(
dev.as_ptr(),
ifr.ifr_name.as_mut_ptr().cast::<u8>(),
if dev.len() > (ifr.ifr_name.len() - 1) {
ifr.ifr_name.len() - 1
} else {
dev.len()
},
);
if libc::ioctl(
s,
if accept_ra {
@ -323,7 +339,13 @@ impl MacFethTap {
/// given will not remain valid after it returns. Also note that F will be called
/// from another thread that is spawned here, so all its bound references must
/// be "Send" and "Sync" e.g. Arc<>.
pub fn new<F: Fn(&[u8]) + Send + Sync + 'static>(nwid: u64, mac: &MAC, mtu: i32, metric: i32, eth_frame_func: F) -> Result<MacFethTap, String> {
pub fn new<F: Fn(&[u8]) + Send + Sync + 'static>(
nwid: u64,
mac: &MAC,
mtu: i32,
metric: i32,
eth_frame_func: F,
) -> Result<MacFethTap, String> {
// This tracks BPF devices we are using so we don't try to reopen them, and also
// doubles as a global lock to ensure that only one feth tap is created at once per
// ZeroTier process per system.
@ -369,12 +391,20 @@ impl MacFethTap {
// Create pair of feth interfaces and create MacFethDevice struct.
let cmd = Command::new(IFCONFIG).arg(&device_name).arg("create").spawn();
if cmd.is_err() {
return Err(format!("unable to create device '{}': {}", device_name.as_str(), cmd.err().unwrap().to_string()));
return Err(format!(
"unable to create device '{}': {}",
device_name.as_str(),
cmd.err().unwrap().to_string()
));
}
let _ = cmd.unwrap().wait();
let cmd = Command::new(IFCONFIG).arg(&peer_device_name).arg("create").spawn();
if cmd.is_err() {
return Err(format!("unable to create device '{}': {}", peer_device_name.as_str(), cmd.err().unwrap().to_string()));
return Err(format!(
"unable to create device '{}': {}",
peer_device_name.as_str(),
cmd.err().unwrap().to_string()
));
}
let _ = cmd.unwrap().wait();
let device = MacFethDevice { name: device_name, peer_name: peer_device_name };
@ -382,28 +412,60 @@ impl MacFethTap {
// Set link-layer (MAC) address of primary interface.
let cmd = Command::new(IFCONFIG).arg(&device.name).arg("lladdr").arg(mac.to_string()).spawn();
if cmd.is_err() {
return Err(format!("unable to configure device '{}': {}", &device.name, cmd.err().unwrap().to_string()));
return Err(format!(
"unable to configure device '{}': {}",
&device.name,
cmd.err().unwrap().to_string()
));
}
let _ = cmd.unwrap().wait();
// Bind peer interfaces together.
let cmd = Command::new(IFCONFIG).arg(&device.peer_name).arg("peer").arg(device.name.as_str()).spawn();
let cmd = Command::new(IFCONFIG)
.arg(&device.peer_name)
.arg("peer")
.arg(device.name.as_str())
.spawn();
if cmd.is_err() {
return Err(format!("unable to configure device '{}': {}", &device.peer_name, cmd.err().unwrap().to_string()));
return Err(format!(
"unable to configure device '{}': {}",
&device.peer_name,
cmd.err().unwrap().to_string()
));
}
let _ = cmd.unwrap().wait();
// Set MTU of secondary peer interface, bring up.
let cmd = Command::new(IFCONFIG).arg(&device.peer_name).arg("mtu").arg(mtu.to_string()).arg("up").spawn();
let cmd = Command::new(IFCONFIG)
.arg(&device.peer_name)
.arg("mtu")
.arg(mtu.to_string())
.arg("up")
.spawn();
if cmd.is_err() {
return Err(format!("unable to configure device '{}': {}", &device.peer_name, cmd.err().unwrap().to_string()));
return Err(format!(
"unable to configure device '{}': {}",
&device.peer_name,
cmd.err().unwrap().to_string()
));
}
let _ = cmd.unwrap().wait();
// Set MTU and metric of primary interface, bring up.
let cmd = Command::new(IFCONFIG).arg(&device.name).arg("mtu").arg(mtu.to_string()).arg("metric").arg(metric.to_string()).arg("up").spawn();
let cmd = Command::new(IFCONFIG)
.arg(&device.name)
.arg("mtu")
.arg(mtu.to_string())
.arg("metric")
.arg(metric.to_string())
.arg("up")
.spawn();
if cmd.is_err() {
return Err(format!("unable to configure device '{}': {}", &device.name.as_str(), cmd.err().unwrap().to_string()));
return Err(format!(
"unable to configure device '{}': {}",
&device.name.as_str(),
cmd.err().unwrap().to_string()
));
}
let _ = cmd.unwrap().wait();
@ -449,7 +511,11 @@ impl MacFethTap {
// Set immediate mode for "live" capture.
fl = 1;
if unsafe {
libc::ioctl(bpf_fd as c_int, 112 /* BIOCIMMEDIATE */, (&mut fl as *mut c_int).cast::<c_void>())
libc::ioctl(
bpf_fd as c_int,
112, /* BIOCIMMEDIATE */
(&mut fl as *mut c_int).cast::<c_void>(),
)
} != 0
{
unsafe {
@ -461,7 +527,11 @@ impl MacFethTap {
// Do not send us back packets we inject or send.
fl = 0;
if unsafe {
libc::ioctl(bpf_fd as c_int, 119 /* BIOCSSEESENT */, (&mut fl as *mut c_int).cast::<c_void>())
libc::ioctl(
bpf_fd as c_int,
119, /* BIOCSSEESENT */
(&mut fl as *mut c_int).cast::<c_void>(),
)
} != 0
{
unsafe {
@ -474,10 +544,22 @@ impl MacFethTap {
let mut bpf_ifr: ifreq = unsafe { std::mem::zeroed() };
let peer_dev_name_bytes = device.peer_name.as_bytes();
unsafe {
copy_nonoverlapping(peer_dev_name_bytes.as_ptr(), bpf_ifr.ifr_name.as_mut_ptr().cast::<u8>(), if peer_dev_name_bytes.len() > (bpf_ifr.ifr_name.len() - 1) { bpf_ifr.ifr_name.len() - 1 } else { peer_dev_name_bytes.len() });
copy_nonoverlapping(
peer_dev_name_bytes.as_ptr(),
bpf_ifr.ifr_name.as_mut_ptr().cast::<u8>(),
if peer_dev_name_bytes.len() > (bpf_ifr.ifr_name.len() - 1) {
bpf_ifr.ifr_name.len() - 1
} else {
peer_dev_name_bytes.len()
},
);
}
if unsafe {
libc::ioctl(bpf_fd as c_int, 108 /* BIOCSETIF */, (&mut bpf_ifr as *mut ifreq).cast::<c_void>())
libc::ioctl(
bpf_fd as c_int,
108, /* BIOCSETIF */
(&mut bpf_ifr as *mut ifreq).cast::<c_void>(),
)
} != 0
{
unsafe {
@ -489,7 +571,11 @@ impl MacFethTap {
// Include Ethernet header in BPF captures.
fl = 1;
if unsafe {
libc::ioctl(bpf_fd as c_int, 117 /* BIOCSHDRCMPLT */, (&mut fl as *mut c_int).cast::<c_void>())
libc::ioctl(
bpf_fd as c_int,
117, /* BIOCSHDRCMPLT */
(&mut fl as *mut c_int).cast::<c_void>(),
)
} != 0
{
unsafe {
@ -501,7 +587,11 @@ impl MacFethTap {
// Set promiscuous mode so bridging can work.
fl = 1;
if unsafe {
libc::ioctl(bpf_fd as c_int, 105 /* BIOCPROMISC */, (&mut fl as *mut c_int).cast::<c_void>())
libc::ioctl(
bpf_fd as c_int,
105, /* BIOCPROMISC */
(&mut fl as *mut c_int).cast::<c_void>(),
)
} != 0
{
unsafe {
@ -558,9 +648,24 @@ impl MacFethTap {
ndrv_sa.snd_len = std::mem::size_of::<sockaddr_ndrv>() as c_uchar;
ndrv_sa.snd_family = 27 /* AF_NDRV */;
unsafe {
copy_nonoverlapping(peer_dev_name_bytes.as_ptr(), ndrv_sa.snd_name.as_mut_ptr().cast::<u8>(), if peer_dev_name_bytes.len() > (bpf_ifr.ifr_name.len() - 1) { bpf_ifr.ifr_name.len() - 1 } else { peer_dev_name_bytes.len() });
copy_nonoverlapping(
peer_dev_name_bytes.as_ptr(),
ndrv_sa.snd_name.as_mut_ptr().cast::<u8>(),
if peer_dev_name_bytes.len() > (bpf_ifr.ifr_name.len() - 1) {
bpf_ifr.ifr_name.len() - 1
} else {
peer_dev_name_bytes.len()
},
);
}
if unsafe { libc::bind(ndrv_fd, (&ndrv_sa as *const sockaddr_ndrv).cast(), std::mem::size_of::<sockaddr_ndrv>() as libc::socklen_t) } != 0 {
if unsafe {
libc::bind(
ndrv_fd,
(&ndrv_sa as *const sockaddr_ndrv).cast(),
std::mem::size_of::<sockaddr_ndrv>() as libc::socklen_t,
)
} != 0
{
unsafe {
libc::close(bpf_fd);
}
@ -569,7 +674,14 @@ impl MacFethTap {
}
return Err(String::from("unable to bind AF_NDRV socket"));
}
if unsafe { libc::connect(ndrv_fd, (&ndrv_sa as *const sockaddr_ndrv).cast(), std::mem::size_of::<sockaddr_ndrv>() as libc::socklen_t) } != 0 {
if unsafe {
libc::connect(
ndrv_fd,
(&ndrv_sa as *const sockaddr_ndrv).cast(),
std::mem::size_of::<sockaddr_ndrv>() as libc::socklen_t,
)
} != 0
{
unsafe {
libc::close(bpf_fd);
}
@ -605,7 +717,16 @@ impl MacFethTap {
impl VNIC for MacFethTap {
fn add_ip(&self, ip: &InetAddress) -> bool {
if !self.have_ip(ip) {
let cmd = Command::new(IFCONFIG).arg(&self.device.name).arg(if ip.is_v6() { "inet6" } else { "inet" }).arg(ip.to_string()).arg("alias").spawn();
let cmd = Command::new(IFCONFIG)
.arg(&self.device.name)
.arg(if ip.is_v6() {
"inet6"
} else {
"inet"
})
.arg(ip.to_string())
.arg("alias")
.spawn();
if cmd.is_ok() {
let _ = cmd.unwrap().wait();
}
@ -616,7 +737,16 @@ impl VNIC for MacFethTap {
fn remove_ip(&self, ip: &InetAddress) -> bool {
if self.have_ip(ip) {
let cmd = Command::new(IFCONFIG).arg(&self.device.name).arg(if ip.is_v6() { "inet6" } else { "inet" }).arg(ip.to_string()).arg("-alias").spawn();
let cmd = Command::new(IFCONFIG)
.arg(&self.device.name)
.arg(if ip.is_v6() {
"inet6"
} else {
"inet"
})
.arg(ip.to_string())
.arg("-alias")
.spawn();
if cmd.is_ok() {
let _ = cmd.unwrap().wait();
}
@ -644,18 +774,42 @@ impl VNIC for MacFethTap {
fn get_multicast_groups(&self) -> HashSet<MulticastGroup> {
let mut all_groups: HashSet<MulticastGroup> = HashSet::new();
crate::vnic::common::get_l2_multicast_subscriptions(self.device.name.as_str()).into_iter().for_each(|mac| {
all_groups.insert(MulticastGroup::from(&mac));
});
crate::vnic::common::get_l2_multicast_subscriptions(self.device.name.as_str())
.into_iter()
.for_each(|mac| {
all_groups.insert(MulticastGroup::from(&mac));
});
all_groups
}
#[inline(always)]
fn put(&self, source_mac: &zerotier_network_hypervisor::vl1::MAC, dest_mac: &zerotier_network_hypervisor::vl1::MAC, ethertype: u16, _vlan_id: u16, data: *const u8, len: usize) -> bool {
fn put(
&self,
source_mac: &zerotier_network_hypervisor::vl1::MAC,
dest_mac: &zerotier_network_hypervisor::vl1::MAC,
ethertype: u16,
_vlan_id: u16,
data: *const u8,
len: usize,
) -> bool {
let dm = dest_mac.0;
let sm = source_mac.0;
let mut hdr: [u8; 14] =
[(dm >> 40) as u8, (dm >> 32) as u8, (dm >> 24) as u8, (dm >> 16) as u8, (dm >> 8) as u8, dm as u8, (sm >> 40) as u8, (sm >> 32) as u8, (sm >> 24) as u8, (sm >> 16) as u8, (sm >> 8) as u8, sm as u8, (ethertype >> 8) as u8, ethertype as u8];
let mut hdr: [u8; 14] = [
(dm >> 40) as u8,
(dm >> 32) as u8,
(dm >> 24) as u8,
(dm >> 16) as u8,
(dm >> 8) as u8,
dm as u8,
(sm >> 40) as u8,
(sm >> 32) as u8,
(sm >> 24) as u8,
(sm >> 16) as u8,
(sm >> 8) as u8,
sm as u8,
(ethertype >> 8) as u8,
ethertype as u8,
];
unsafe {
let iov: [libc::iovec; 2] = [
libc::iovec { iov_base: hdr.as_mut_ptr().cast(), iov_len: 14 },

76
third_party/kyber/benches/api.rs vendored
View file

@ -9,57 +9,45 @@ const CT_HEX: &str = "EADD5ADA14DA57F0AEF3505F1CAA6485D4238D999A3EF4B0A59A1CDBE0
// Benchmarking key generation
fn keypair_bench(c: &mut Criterion) {
let mut rng = rand::thread_rng();
c.bench_function(
"Keypair Generation",
|b| b.iter(
|| {
let _keys = keypair(&mut rng);
}
)
);
let mut rng = rand::thread_rng();
c.bench_function("Keypair Generation", |b| {
b.iter(|| {
let _keys = keypair(&mut rng);
})
});
}
// Encapsulating a single public key
fn encap_bench(c: &mut Criterion) {
let pk = crate::decode_hex(PK_HEX);
let mut rng = rand::thread_rng();
c.bench_function(
"Encapsulate",
|b| b.iter(
|| {
let _enc = encapsulate(&pk, &mut rng);
}
)
);
let pk = crate::decode_hex(PK_HEX);
let mut rng = rand::thread_rng();
c.bench_function("Encapsulate", |b| {
b.iter(|| {
let _enc = encapsulate(&pk, &mut rng);
})
});
}
// Decapsulating a single correct ciphertext
fn decap_bench(c: &mut Criterion) {
let sk = decode_hex(SK_HEX);
let ct = decode_hex(CT_HEX);
c.bench_function(
"Decapsulate",
|b| b.iter(
|| {
let _dec = decapsulate(&ct, &sk);
}
)
);
let sk = decode_hex(SK_HEX);
let ct = decode_hex(CT_HEX);
c.bench_function("Decapsulate", |b| {
b.iter(|| {
let _dec = decapsulate(&ct, &sk);
})
});
}
// Decapsulating a single incorrect ciphertext
fn decap_fail_bench(c: &mut Criterion) {
let sk = decode_hex(BAD_SK);
let ct = decode_hex(CT_HEX);
c.bench_function(
"Decapsulate Failure",
|b| b.iter(
|| {
let _dec = decapsulate(&ct, &sk);
}
)
);
let sk = decode_hex(BAD_SK);
let ct = decode_hex(CT_HEX);
c.bench_function("Decapsulate Failure", |b| {
b.iter(|| {
let _dec = decapsulate(&ct, &sk);
})
});
}
criterion_group!(benches, keypair_bench, encap_bench, decap_bench, decap_fail_bench);
@ -67,8 +55,8 @@ criterion_main!(benches);
// Decodes a hex string into a vector of bytes
pub fn decode_hex(s: &str) -> Vec<u8> {
(0..s.len())
.step_by(2)
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("Hex string decoding"))
.collect::<Vec<u8>>()
}
(0..s.len())
.step_by(2)
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("Hex string decoding"))
.collect::<Vec<u8>>()
}

36
third_party/kyber/build.rs vendored
View file

@ -1,19 +1,19 @@
fn main() {
#[cfg(all(target_arch = "x86_64", not(feature = "reference")))]
cc::Build::new()
.include("src/avx2/")
.file("src/avx2/basemul.S")
.file("src/avx2/fq.S")
.file("src/avx2/invntt.S")
.file("src/avx2/ntt.S")
.file("src/avx2/shuffle.S")
.compile("pqc_kyber");
// #[cfg(
// all(
// any(target_arch = "arm", target_arch="aarch64"),
// not(feature = "reference")
// )
// )]
// cc::Build::new()
// .include("src/neon/");
}
#[cfg(all(target_arch = "x86_64", not(feature = "reference")))]
cc::Build::new()
.include("src/avx2/")
.file("src/avx2/basemul.S")
.file("src/avx2/fq.S")
.file("src/avx2/invntt.S")
.file("src/avx2/ntt.S")
.file("src/avx2/shuffle.S")
.compile("pqc_kyber");
// #[cfg(
// all(
// any(target_arch = "arm", target_arch="aarch64"),
// not(feature = "reference")
// )
// )]
// cc::Build::new()
// .include("src/neon/");
}

36
third_party/kyber/examples/ake.rs vendored
View file

@ -1,27 +1,25 @@
use pqc_kyber::*;
fn main() -> Result<(), KyberError> {
let mut rng = rand::thread_rng();
let mut rng = rand::thread_rng();
let mut alice = Ake::new();
let mut bob = Ake::new();
let alice_keys = keypair(&mut rng);
let bob_keys = keypair(&mut rng);
let mut alice = Ake::new();
let mut bob = Ake::new();
let alice_keys = keypair(&mut rng);
let bob_keys = keypair(&mut rng);
// Alice initiates key exchange with bob
let client_send = alice.client_init(&bob_keys.public, &mut rng);
// Bob receives the request and authenticates Alice, sends
// encapsulated shared secret back
let server_send = bob.server_receive(
client_send, &alice_keys.public, &bob_keys.secret, &mut rng
)?;
// Alice initiates key exchange with bob
let client_send = alice.client_init(&bob_keys.public, &mut rng);
// Alice autheticates and decapsulates
alice.client_confirm(server_send, &alice_keys.secret)?;
// Bob receives the request and authenticates Alice, sends
// encapsulated shared secret back
let server_send = bob.server_receive(client_send, &alice_keys.public, &bob_keys.secret, &mut rng)?;
// Both structs now have the shared secret
assert_eq!(alice.shared_secret, bob.shared_secret);
// Alice autheticates and decapsulates
alice.client_confirm(server_send, &alice_keys.secret)?;
Ok(())
}
// Both structs now have the shared secret
assert_eq!(alice.shared_secret, bob.shared_secret);
Ok(())
}

28
third_party/kyber/examples/kem.rs vendored
View file

@ -1,18 +1,18 @@
use pqc_kyber::*;
fn main () -> Result<(), KyberError> {
let mut rng = rand::thread_rng();
fn main() -> Result<(), KyberError> {
let mut rng = rand::thread_rng();
// Alice generates a keypair
let alice_keys = keypair(&mut rng);
// Alice generates a keypair
let alice_keys = keypair(&mut rng);
// Bob encapsulates a shared secret
let (ciphertext, shared_secret_bob) = encapsulate(&alice_keys.public, &mut rng)?;
// Alice decapsulates the shared secret
let shared_secret_alice = decapsulate(&ciphertext, &alice_keys.secret)?;
// Both can now communicate symetrically
assert_eq!(shared_secret_alice, shared_secret_bob);
Ok(())
}
// Bob encapsulates a shared secret
let (ciphertext, shared_secret_bob) = encapsulate(&alice_keys.public, &mut rng)?;
// Alice decapsulates the shared secret
let shared_secret_alice = decapsulate(&ciphertext, &alice_keys.secret)?;
// Both can now communicate symetrically
assert_eq!(shared_secret_alice, shared_secret_bob);
Ok(())
}

34
third_party/kyber/examples/uake.rs vendored
View file

@ -1,27 +1,25 @@
use pqc_kyber::*;
fn main() -> Result<(), KyberError> {
let mut rng = rand::thread_rng();
let mut rng = rand::thread_rng();
let mut alice = Uake::new();
let mut bob = Uake::new();
let mut alice = Uake::new();
let mut bob = Uake::new();
let bob_keys = keypair(&mut rng);
let bob_keys = keypair(&mut rng);
// Alice initiates key exchange with bob
let client_send = alice.client_init(&bob_keys.public, &mut rng);
// Bob receives the request and authenticates Alice, sends
// encapsulated shared secret back
let server_send = bob.server_receive(
client_send, &bob_keys.secret, &mut rng
)?;
// Alice initiates key exchange with bob
let client_send = alice.client_init(&bob_keys.public, &mut rng);
// Alice autheticates and decapsulates
alice.client_confirm(server_send)?;
// Bob receives the request and authenticates Alice, sends
// encapsulated shared secret back
let server_send = bob.server_receive(client_send, &bob_keys.secret, &mut rng)?;
// Both structs now have the shared secret
assert_eq!(alice.shared_secret, bob.shared_secret);
// Alice autheticates and decapsulates
alice.client_confirm(server_send)?;
Ok(())
}
// Both structs now have the shared secret
assert_eq!(alice.shared_secret, bob.shared_secret);
Ok(())
}

28
third_party/kyber/fuzz/src/bin/decap.rs vendored
View file

@ -2,16 +2,18 @@ use honggfuzz::fuzz;
use pqc_kyber::*;
fn main() -> Result<(), KyberError> {
let mut ss = [0u8; KYBER_SSBYTES];
const CTBYTES: usize = KYBER_CIPHERTEXTBYTES;
const SKBYTES: usize = KYBER_SECRETKEYBYTES;
loop {
fuzz!(|data: &[u8] | {
if data.len() != CTBYTES + SKBYTES {return};
match crypto_kem_dec(&mut ss, &data[..CTBYTES], &data[CTBYTES..SKBYTES]) {
Ok(_) => (),
Err(_) => ()
}
});
};
}
let mut ss = [0u8; KYBER_SSBYTES];
const CTBYTES: usize = KYBER_CIPHERTEXTBYTES;
const SKBYTES: usize = KYBER_SECRETKEYBYTES;
loop {
fuzz!(|data: &[u8]| {
if data.len() != CTBYTES + SKBYTES {
return;
};
match crypto_kem_dec(&mut ss, &data[..CTBYTES], &data[CTBYTES..SKBYTES]) {
Ok(_) => (),
Err(_) => (),
}
});
}
}

30
third_party/kyber/fuzz/src/bin/encap.rs vendored
View file

@ -1,19 +1,21 @@
use honggfuzz::fuzz;
use pqc_kyber::*;
use rand_xoshiro::rand_core::{SeedableRng, RngCore};
use rand_xoshiro::rand_core::{RngCore, SeedableRng};
use rand_xoshiro::Xoshiro256Plus;
fn main() -> Result<(), KyberError> {
let mut _rng = rand::thread_rng(); //placeholder
let mut rng = Xoshiro256Plus::seed_from_u64(0);
let mut ct = [0u8; KYBER_CIPHERTEXTBYTES];
let mut ss = [0u8; KYBER_SSBYTES];
let mut s1 = [0u8; 32];
loop {
rng.fill_bytes(&mut s1);
fuzz!(|data: &[u8] | {
if data.len() != KYBER_PUBLICKEYBYTES {return};
crypto_kem_enc(&mut ct, &mut ss, data, &mut _rng, Some(&s1));
});
};
}
let mut _rng = rand::thread_rng(); //placeholder
let mut rng = Xoshiro256Plus::seed_from_u64(0);
let mut ct = [0u8; KYBER_CIPHERTEXTBYTES];
let mut ss = [0u8; KYBER_SSBYTES];
let mut s1 = [0u8; 32];
loop {
rng.fill_bytes(&mut s1);
fuzz!(|data: &[u8]| {
if data.len() != KYBER_PUBLICKEYBYTES {
return;
};
crypto_kem_enc(&mut ct, &mut ss, data, &mut _rng, Some(&s1));
});
}
}

32
third_party/kyber/fuzz/src/bin/keypair.rs vendored
View file

@ -1,20 +1,20 @@
use honggfuzz::fuzz;
use pqc_kyber::*;
use rand_xoshiro::rand_core::{SeedableRng, RngCore};
use rand_xoshiro::rand_core::{RngCore, SeedableRng};
use rand_xoshiro::Xoshiro256Plus;
fn main() -> Result<(), KyberError> {
let mut _rng = rand::thread_rng(); //placeholder
let mut rng = Xoshiro256Plus::seed_from_u64(0);
let mut public = [0u8; KYBER_PUBLICKEYBYTES];
let mut secret = [0u8; KYBER_SECRETKEYBYTES];
let mut s1 = [0u8; 32];
let mut s2 = [0u8; 32];
loop {
rng.fill_bytes(&mut s1);
rng.fill_bytes(&mut s2);
fuzz!(|data: ()| {
crypto_kem_keypair(&mut public, &mut secret, &mut _rng, Some((&s1, &s2)));
});
};
}
let mut _rng = rand::thread_rng(); //placeholder
let mut rng = Xoshiro256Plus::seed_from_u64(0);
let mut public = [0u8; KYBER_PUBLICKEYBYTES];
let mut secret = [0u8; KYBER_SECRETKEYBYTES];
let mut s1 = [0u8; 32];
let mut s2 = [0u8; 32];
loop {
rng.fill_bytes(&mut s1);
rng.fill_bytes(&mut s2);
fuzz!(|data: ()| {
crypto_kem_keypair(&mut public, &mut secret, &mut _rng, Some((&s1, &s2)));
});
}
}

103
third_party/kyber/src/api.rs vendored
View file

@ -1,13 +1,13 @@
use crate::{
params::*,
error::KyberError,
RngCore, CryptoRng,
kem::*,
kex::{PublicKey, SecretKey, Encapsulated, Decapsulated}
error::KyberError,
kem::*,
kex::{Decapsulated, Encapsulated, PublicKey, SecretKey},
params::*,
CryptoRng, RngCore,
};
/// Keypair generation with a provided RNG.
///
///
/// ### Example
/// ```
/// # use pqc_kyber::*;
@ -16,13 +16,14 @@ use crate::{
/// let keys = keypair(&mut rng);
/// # Ok(())}
/// ```
pub fn keypair<R>(rng: &mut R) -> Keypair
where R: RngCore + CryptoRng
pub fn keypair<R>(rng: &mut R) -> Keypair
where
R: RngCore + CryptoRng,
{
let mut public = [0u8; KYBER_PUBLICKEYBYTES];
let mut secret = [0u8; KYBER_SECRETKEYBYTES];
crypto_kem_keypair(&mut public, &mut secret, rng, None);
Keypair { public, secret }
let mut public = [0u8; KYBER_PUBLICKEYBYTES];
let mut secret = [0u8; KYBER_SECRETKEYBYTES];
crypto_kem_keypair(&mut public, &mut secret, rng, None);
Keypair { public, secret }
}
/// Encapsulates a public key returning the ciphertext to send
@ -30,23 +31,24 @@ pub fn keypair<R>(rng: &mut R) -> Keypair
///
/// ### Example
/// ```
/// # use pqc_kyber::*;
/// # use pqc_kyber::*;
/// # fn main() -> Result<(), KyberError> {
/// let mut rng = rand::thread_rng();
/// let keys = keypair(&mut rng);
/// let (ciphertext, shared_secret) = encapsulate(&keys.public, &mut rng)?;
/// # Ok(())}
/// ```
pub fn encapsulate<R>(pk: &[u8], rng: &mut R) -> Encapsulated
where R: CryptoRng + RngCore
pub fn encapsulate<R>(pk: &[u8], rng: &mut R) -> Encapsulated
where
R: CryptoRng + RngCore,
{
if pk.len() != KYBER_PUBLICKEYBYTES {
return Err(KyberError::InvalidInput)
}
let mut ct = [0u8; KYBER_CIPHERTEXTBYTES];
let mut ss = [0u8; KYBER_SSBYTES];
crypto_kem_enc(&mut ct, &mut ss, pk, rng, None);
Ok((ct, ss))
if pk.len() != KYBER_PUBLICKEYBYTES {
return Err(KyberError::InvalidInput);
}
let mut ct = [0u8; KYBER_CIPHERTEXTBYTES];
let mut ss = [0u8; KYBER_SSBYTES];
crypto_kem_enc(&mut ct, &mut ss, pk, rng, None);
Ok((ct, ss))
}
/// Decapsulates ciphertext with a secret key, the result will contain
@ -63,41 +65,40 @@ pub fn encapsulate<R>(pk: &[u8], rng: &mut R) -> Encapsulated
/// assert_eq!(ss1, ss2);
/// # Ok(())}
/// ```
pub fn decapsulate(ct: &[u8], sk: &[u8]) -> Decapsulated
{
if ct.len() != KYBER_CIPHERTEXTBYTES || sk.len() != KYBER_SECRETKEYBYTES {
return Err(KyberError::InvalidInput)
}
let mut ss = [0u8; KYBER_SSBYTES];
match crypto_kem_dec(&mut ss, ct, sk) {
Ok(_) => Ok(ss),
Err(e) => Err(e)
}
pub fn decapsulate(ct: &[u8], sk: &[u8]) -> Decapsulated {
if ct.len() != KYBER_CIPHERTEXTBYTES || sk.len() != KYBER_SECRETKEYBYTES {
return Err(KyberError::InvalidInput);
}
let mut ss = [0u8; KYBER_SSBYTES];
match crypto_kem_dec(&mut ss, ct, sk) {
Ok(_) => Ok(ss),
Err(e) => Err(e),
}
}
/// A public/secret keypair for use with Kyber.
///
/// A public/secret keypair for use with Kyber.
///
/// Byte lengths of the keys are determined by the security level chosen.
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub struct Keypair {
pub public: PublicKey,
pub secret: SecretKey
pub secret: SecretKey,
}
impl Keypair {
/// Securely generates a new keypair`
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(), KyberError> {
/// let mut rng = rand::thread_rng();
/// let keys = Keypair::generate(&mut rng);
/// # let empty_keys = Keypair{
/// public: [0u8; KYBER_PUBLICKEYBYTES], secret: [0u8; KYBER_SECRETKEYBYTES]
/// };
/// # assert!(empty_keys != keys);
/// # Ok(()) }
/// ```
pub fn generate<R: CryptoRng + RngCore>(rng: &mut R) -> Keypair {
keypair(rng)
}
}
/// Securely generates a new keypair`
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(), KyberError> {
/// let mut rng = rand::thread_rng();
/// let keys = Keypair::generate(&mut rng);
/// # let empty_keys = Keypair{
/// public: [0u8; KYBER_PUBLICKEYBYTES], secret: [0u8; KYBER_SECRETKEYBYTES]
/// };
/// # assert!(empty_keys != keys);
/// # Ok(()) }
/// ```
pub fn generate<R: CryptoRng + RngCore>(rng: &mut R) -> Keypair {
keypair(rng)
}
}

279
third_party/kyber/src/avx2/aes256ctr.rs vendored
View file

@ -1,177 +1,170 @@
// Based heavily on public-domain code by Romain Dolbeau
// Different handling of nonce+counter than original version using
// separated 64-bit nonce and internal 64-bit counter, starting from zero
// Public Domain
#![cfg(feature="90s")]
// Public Domain
#![cfg(feature = "90s")]
use core::arch::x86_64::*;
#[derive(Clone, Copy)]
#[repr(C)]
pub(crate) struct Aes256CtrCtx {
pub rkeys: [__m128i; 16],
pub n: __m128i
pub rkeys: [__m128i; 16],
pub n: __m128i,
}
impl Aes256CtrCtx {
pub fn new() -> Self {
unsafe {
Self {
rkeys: [_mm_setzero_si128(); 16],
n: _mm_setzero_si128()
}
pub fn new() -> Self {
unsafe { Self { rkeys: [_mm_setzero_si128(); 16], n: _mm_setzero_si128() } }
}
}
}
unsafe fn aesni_encrypt4(out: &mut[u8], n :&mut __m128i, rkeys: &[__m128i; 16])
{
let idx: __m128i = _mm_set_epi8(8,9,10,11,12,13,14,15,7,6,5,4,3,2,1,0);
unsafe fn aesni_encrypt4(out: &mut [u8], n: &mut __m128i, rkeys: &[__m128i; 16]) {
let idx: __m128i = _mm_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 7, 6, 5, 4, 3, 2, 1, 0);
// Load current counter value
let mut f = _mm_load_si128(n);
// Load current counter value
let mut f = _mm_load_si128(n);
// Increase counter in 4 consecutive blocks
let mut f0 = _mm_shuffle_epi8(_mm_add_epi64(f,_mm_set_epi64x(0,0)),idx);
let mut f1 = _mm_shuffle_epi8(_mm_add_epi64(f,_mm_set_epi64x(1,0)),idx);
let mut f2 = _mm_shuffle_epi8(_mm_add_epi64(f,_mm_set_epi64x(2,0)),idx);
let mut f3 = _mm_shuffle_epi8(_mm_add_epi64(f,_mm_set_epi64x(3,0)),idx);
// Increase counter in 4 consecutive blocks
let mut f0 = _mm_shuffle_epi8(_mm_add_epi64(f, _mm_set_epi64x(0, 0)), idx);
let mut f1 = _mm_shuffle_epi8(_mm_add_epi64(f, _mm_set_epi64x(1, 0)), idx);
let mut f2 = _mm_shuffle_epi8(_mm_add_epi64(f, _mm_set_epi64x(2, 0)), idx);
let mut f3 = _mm_shuffle_epi8(_mm_add_epi64(f, _mm_set_epi64x(3, 0)), idx);
// Write counter for next iteration, increased by 4
_mm_store_si128(n as *mut __m128i,_mm_add_epi64(f,_mm_set_epi64x(4,0)));
// Write counter for next iteration, increased by 4
_mm_store_si128(n as *mut __m128i, _mm_add_epi64(f, _mm_set_epi64x(4, 0)));
// Actual AES encryption, 4x interleaved4
f = _mm_load_si128(&rkeys[0]);
f0 = _mm_xor_si128(f0,f);
f1 = _mm_xor_si128(f1,f);
f2 = _mm_xor_si128(f2,f);
f3 = _mm_xor_si128(f3,f);
// Actual AES encryption, 4x interleaved4
f = _mm_load_si128(&rkeys[0]);
f0 = _mm_xor_si128(f0, f);
f1 = _mm_xor_si128(f1, f);
f2 = _mm_xor_si128(f2, f);
f3 = _mm_xor_si128(f3, f);
for i in 1..14 {
f = _mm_load_si128(&rkeys[i]);
f0 = _mm_aesenc_si128(f0,f);
f1 = _mm_aesenc_si128(f1,f);
f2 = _mm_aesenc_si128(f2,f);
f3 = _mm_aesenc_si128(f3,f);
}
for i in 1..14 {
f = _mm_load_si128(&rkeys[i]);
f0 = _mm_aesenc_si128(f0, f);
f1 = _mm_aesenc_si128(f1, f);
f2 = _mm_aesenc_si128(f2, f);
f3 = _mm_aesenc_si128(f3, f);
}
f = _mm_load_si128(&rkeys[14]);
f0 = _mm_aesenclast_si128(f0,f);
f1 = _mm_aesenclast_si128(f1,f);
f2 = _mm_aesenclast_si128(f2,f);
f3 = _mm_aesenclast_si128(f3,f);
f = _mm_load_si128(&rkeys[14]);
f0 = _mm_aesenclast_si128(f0, f);
f1 = _mm_aesenclast_si128(f1, f);
f2 = _mm_aesenclast_si128(f2, f);
f3 = _mm_aesenclast_si128(f3, f);
// Write results
_mm_storeu_si128(out[..].as_mut_ptr() as *mut __m128i, f0);
_mm_storeu_si128(out[16..].as_mut_ptr() as *mut __m128i, f1);
_mm_storeu_si128(out[32..].as_mut_ptr() as *mut __m128i, f2);
_mm_storeu_si128(out[48..].as_mut_ptr() as *mut __m128i, f3);
// Write results
_mm_storeu_si128(out[..].as_mut_ptr() as *mut __m128i, f0);
_mm_storeu_si128(out[16..].as_mut_ptr() as *mut __m128i, f1);
_mm_storeu_si128(out[32..].as_mut_ptr() as *mut __m128i, f2);
_mm_storeu_si128(out[48..].as_mut_ptr() as *mut __m128i, f3);
}
// Casting aliases
unsafe fn cast_128i(x: __m128) -> __m128i
{
_mm_castps_si128(x)
unsafe fn cast_128i(x: __m128) -> __m128i {
_mm_castps_si128(x)
}
unsafe fn cast_128(x: __m128i) -> __m128
{
_mm_castsi128_ps(x)
unsafe fn cast_128(x: __m128i) -> __m128 {
_mm_castsi128_ps(x)
}
pub(crate) fn aes256ctr_init(state: &mut Aes256CtrCtx, key: &[u8], nonce: [u8; 12])
{
unsafe {
let mut idx = 0;
let key0 = _mm_loadu_si128(key.as_ptr() as *const __m128i);
let key1 = _mm_loadu_si128(key[16..].as_ptr() as *const __m128i);
pub(crate) fn aes256ctr_init(state: &mut Aes256CtrCtx, key: &[u8], nonce: [u8; 12]) {
unsafe {
let mut idx = 0;
let key0 = _mm_loadu_si128(key.as_ptr() as *const __m128i);
let key1 = _mm_loadu_si128(key[16..].as_ptr() as *const __m128i);
state.n = _mm_loadl_epi64(nonce[..].as_ptr() as *const __m128i);
state.rkeys[idx] = key0;
idx += 1;
let mut temp0 = key0;
let mut temp1;
let mut temp2 = key1;
let mut temp4 = _mm_setzero_si128();
macro_rules! block1 {
($imm:expr) => {
temp1 = _mm_aeskeygenassist_si128(temp2, $imm);
state.rkeys[idx] = temp2;
state.n = _mm_loadl_epi64(nonce[..].as_ptr() as *const __m128i);
state.rkeys[idx] = key0;
idx += 1;
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp0), 0x10));
temp0 = _mm_xor_si128(temp0, temp4);
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp0), 0x8c));
temp0 = _mm_xor_si128(temp0, temp4);
temp1 = cast_128i(_mm_shuffle_ps(cast_128(temp1), cast_128(temp1), 0xff));
temp0 = _mm_xor_si128(temp0, temp1)
};
let mut temp0 = key0;
let mut temp1;
let mut temp2 = key1;
let mut temp4 = _mm_setzero_si128();
macro_rules! block1 {
($imm:expr) => {
temp1 = _mm_aeskeygenassist_si128(temp2, $imm);
state.rkeys[idx] = temp2;
idx += 1;
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp0), 0x10));
temp0 = _mm_xor_si128(temp0, temp4);
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp0), 0x8c));
temp0 = _mm_xor_si128(temp0, temp4);
temp1 = cast_128i(_mm_shuffle_ps(cast_128(temp1), cast_128(temp1), 0xff));
temp0 = _mm_xor_si128(temp0, temp1)
};
}
macro_rules! block2 {
($imm:expr) => {
temp1 = _mm_aeskeygenassist_si128(temp0, $imm);
state.rkeys[idx] = temp0;
idx += 1;
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp2), 0x10));
temp2 = _mm_xor_si128(temp2, temp4);
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp2), 0x8c));
temp2 = _mm_xor_si128(temp2, temp4);
temp1 = cast_128i(_mm_shuffle_ps(cast_128(temp1), cast_128(temp1), 0xaa));
temp2 = _mm_xor_si128(temp2, temp1)
};
}
block1!(0x01);
block2!(0x01);
block1!(0x02);
block2!(0x02);
block1!(0x04);
block2!(0x04);
block1!(0x08);
block2!(0x08);
block1!(0x10);
block2!(0x10);
block1!(0x20);
block2!(0x20);
block1!(0x40);
state.rkeys[idx] = temp0;
}
}
pub(crate) fn aes256ctr_squeezeblocks(out: &mut [u8], nblocks: usize, state: &mut Aes256CtrCtx) {
let mut idx = 0;
for _ in 0..nblocks {
unsafe {
aesni_encrypt4(&mut out[idx..], &mut state.n, &state.rkeys);
}
idx += 64
}
}
#[cfg(feature = "90s")]
pub(crate) fn aes256ctr_prf(out: &mut [u8], mut outlen: usize, seed: &[u8], nonce: u8) {
let mut buf = [0u8; 64];
let mut idx = 0;
let mut pad_nonce = [0u8; 12];
let mut state = unsafe { Aes256CtrCtx { rkeys: [_mm_setzero_si128(); 16], n: _mm_setzero_si128() } };
pad_nonce[0] = nonce;
aes256ctr_init(&mut state, seed, pad_nonce);
while outlen >= 64 {
unsafe {
aesni_encrypt4(&mut out[idx..], &mut state.n, &state.rkeys);
}
outlen -= 64;
idx += 64;
}
macro_rules! block2 {
($imm:expr) => {
temp1 = _mm_aeskeygenassist_si128(temp0, $imm);
state.rkeys[idx] = temp0;
idx += 1;
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp2), 0x10));
temp2 = _mm_xor_si128(temp2, temp4);
temp4 = cast_128i(_mm_shuffle_ps(cast_128(temp4), cast_128(temp2), 0x8c));
temp2 = _mm_xor_si128(temp2, temp4);
temp1 = cast_128i(_mm_shuffle_ps(cast_128(temp1), cast_128(temp1), 0xaa));
temp2 = _mm_xor_si128(temp2, temp1)
};
}
block1!(0x01);
block2!(0x01);
block1!(0x02);
block2!(0x02);
block1!(0x04);
block2!(0x04);
block1!(0x08);
block2!(0x08);
block1!(0x10);
block2!(0x10);
block1!(0x20);
block2!(0x20);
block1!(0x40);
state.rkeys[idx] = temp0;
}
if outlen != 0 {
unsafe {
aesni_encrypt4(&mut buf, &mut state.n, &state.rkeys);
}
out[idx..][..outlen].copy_from_slice(&buf[..outlen]);
}
}
pub(crate) fn aes256ctr_squeezeblocks(out: &mut[u8], nblocks: usize, state: &mut Aes256CtrCtx)
{
let mut idx = 0;
for _ in 0..nblocks {
unsafe { aesni_encrypt4(&mut out[idx..], &mut state.n, &state.rkeys); }
idx += 64
}
}
#[cfg(feature="90s")]
pub(crate) fn aes256ctr_prf(out: &mut[u8], mut outlen: usize, seed: &[u8], nonce: u8)
{
let mut buf = [0u8; 64];
let mut idx = 0;
let mut pad_nonce = [0u8; 12];
let mut state = unsafe{
Aes256CtrCtx{rkeys: [ _mm_setzero_si128(); 16], n: _mm_setzero_si128()}
};
pad_nonce[0] = nonce;
aes256ctr_init(&mut state, seed, pad_nonce);
while outlen >= 64 {
unsafe { aesni_encrypt4(&mut out[idx..], &mut state.n, &state.rkeys); }
outlen -= 64;
idx += 64;
}
if outlen != 0 {
unsafe { aesni_encrypt4(&mut buf, &mut state.n, &state.rkeys); }
out[idx..][..outlen].copy_from_slice(&buf[..outlen]);
}
}

87
third_party/kyber/src/avx2/align.rs vendored
View file

@ -1,11 +1,11 @@
#![allow(dead_code)]
use core::arch::x86_64::*;
use crate::avx2::rejsample::REJ_UNIFORM_AVX_NBLOCKS;
use crate::fips202::{SHAKE128_RATE, SHAKE256_RATE};
use crate::params::*;
use crate::poly::NOISE_NBLOCKS;
use crate::fips202::{SHAKE128_RATE, SHAKE256_RATE};
use crate::symmetric::*;
use crate::avx2::rejsample::REJ_UNIFORM_AVX_NBLOCKS;
use core::arch::x86_64::*;
// Buffer unions
// #[derive(Copy, Clone)]
@ -26,85 +26,76 @@ use crate::avx2::rejsample::REJ_UNIFORM_AVX_NBLOCKS;
#[derive(Copy, Clone)]
#[repr(C, align(32))]
pub union GenMatrixBuf {
pub coeffs: [u8; REJ_UNIFORM_AVX_NBLOCKS*SHAKE128_RATE],
pub vec: [__m256i; (REJ_UNIFORM_AVX_NBLOCKS*SHAKE128_RATE+31)/32]
pub coeffs: [u8; REJ_UNIFORM_AVX_NBLOCKS * SHAKE128_RATE],
pub vec: [__m256i; (REJ_UNIFORM_AVX_NBLOCKS * SHAKE128_RATE + 31) / 32],
}
impl GenMatrixBuf {
pub fn new() -> Self {
Self { coeffs: [0u8; REJ_UNIFORM_AVX_NBLOCKS*SHAKE128_RATE]}
}
pub fn new() -> Self {
Self { coeffs: [0u8; REJ_UNIFORM_AVX_NBLOCKS * SHAKE128_RATE] }
}
}
#[cfg(feature="90s")]
#[cfg(feature = "90s")]
#[repr(C)]
pub union GenMatrixBuf90s {
pub coeffs:
[u8; REJ_UNIFORM_AVX_NBLOCKS*XOF_BLOCKBYTES],
pub vec:
[__m256i; (REJ_UNIFORM_AVX_NBLOCKS*XOF_BLOCKBYTES+31)/32]
pub coeffs: [u8; REJ_UNIFORM_AVX_NBLOCKS * XOF_BLOCKBYTES],
pub vec: [__m256i; (REJ_UNIFORM_AVX_NBLOCKS * XOF_BLOCKBYTES + 31) / 32],
}
#[cfg(feature="90s")]
#[cfg(feature = "90s")]
impl GenMatrixBuf90s {
pub fn new() -> Self {
Self {
coeffs: [0u8; REJ_UNIFORM_AVX_NBLOCKS*XOF_BLOCKBYTES]
pub fn new() -> Self {
Self { coeffs: [0u8; REJ_UNIFORM_AVX_NBLOCKS * XOF_BLOCKBYTES] }
}
}
#[cfg(debug_assertions)]
pub fn checksum(&self) -> i16 {
let mut out = 0;
for i in 0..REJ_UNIFORM_AVX_NBLOCKS*XOF_BLOCKBYTES {
unsafe { out ^= self.coeffs[i] as i16; }
#[cfg(debug_assertions)]
pub fn checksum(&self) -> i16 {
let mut out = 0;
for i in 0..REJ_UNIFORM_AVX_NBLOCKS * XOF_BLOCKBYTES {
unsafe {
out ^= self.coeffs[i] as i16;
}
}
out
}
out
}
}
#[repr(C)]
pub union IndcpaBuf {
pub coeffs: [u8;
(KYBER_ETA1*KYBER_N/4)
/XOF_BLOCKBYTES*XOF_BLOCKBYTES+32],
pub vec: [__m256i;
((KYBER_ETA1*KYBER_N/4)
/XOF_BLOCKBYTES*XOF_BLOCKBYTES+32+31)/32]
pub coeffs: [u8; (KYBER_ETA1 * KYBER_N / 4) / XOF_BLOCKBYTES * XOF_BLOCKBYTES + 32],
pub vec: [__m256i; ((KYBER_ETA1 * KYBER_N / 4) / XOF_BLOCKBYTES * XOF_BLOCKBYTES + 32 + 31) / 32],
}
impl IndcpaBuf {
pub fn new() -> Self {
Self {
coeffs: [0u8;
(KYBER_ETA1*KYBER_N/4)
/XOF_BLOCKBYTES*XOF_BLOCKBYTES+32]
pub fn new() -> Self {
Self {
coeffs: [0u8; (KYBER_ETA1 * KYBER_N / 4) / XOF_BLOCKBYTES * XOF_BLOCKBYTES + 32],
}
}
}
}
#[repr(C, align(8))]
pub union Eta2Buf {
pub coeffs: [u8; KYBER_ETA2*KYBER_N/4],
pub vec: [__m256i; (KYBER_ETA2*KYBER_N/4+31)/32]
pub coeffs: [u8; KYBER_ETA2 * KYBER_N / 4],
pub vec: [__m256i; (KYBER_ETA2 * KYBER_N / 4 + 31) / 32],
}
impl Eta2Buf {
pub fn new() -> Self {
Self { coeffs: [0u8; KYBER_ETA2*KYBER_N/4] }
}
pub fn new() -> Self {
Self { coeffs: [0u8; KYBER_ETA2 * KYBER_N / 4] }
}
}
#[derive(Copy, Clone)]
#[repr(C, align(8))]
pub union Eta4xBuf {
pub coeffs: [u8; NOISE_NBLOCKS*SHAKE256_RATE],
pub vec: [__m256i; (NOISE_NBLOCKS*SHAKE256_RATE+31)/32]
pub coeffs: [u8; NOISE_NBLOCKS * SHAKE256_RATE],
pub vec: [__m256i; (NOISE_NBLOCKS * SHAKE256_RATE + 31) / 32],
}
impl Eta4xBuf {
pub fn new() -> Self {
Self { coeffs: [0u8; NOISE_NBLOCKS*SHAKE256_RATE] }
}
pub fn new() -> Self {
Self { coeffs: [0u8; NOISE_NBLOCKS * SHAKE256_RATE] }
}
}

202
third_party/kyber/src/avx2/cbd.rs vendored
View file

@ -1,136 +1,128 @@
#![allow(non_snake_case, dead_code)]
use core::arch::x86_64::*;
use crate::align::Eta4xBuf;
#[cfg(feature = "90s")]
use crate::align::IndcpaBuf;
use crate::params::KYBER_N;
use crate::poly::*;
use crate::align::Eta4xBuf;
#[cfg(feature="90s")]
use crate::align::IndcpaBuf;
use core::arch::x86_64::*;
fn cbd2(r: &mut Poly, buf: &[__m256i]) {
unsafe {
let mask55: __m256i = _mm256_set1_epi32(0x55555555);
let mask33: __m256i = _mm256_set1_epi32(0x33333333);
let mask03: __m256i = _mm256_set1_epi32(0x03030303);
let mask0F: __m256i = _mm256_set1_epi32(0x0F0F0F0F);
let (mut f0, mut f1, mut f2, mut f3);
for i in 0..(KYBER_N/64) {
f0 = _mm256_load_si256(&buf[i]);
unsafe {
let mask55: __m256i = _mm256_set1_epi32(0x55555555);
let mask33: __m256i = _mm256_set1_epi32(0x33333333);
let mask03: __m256i = _mm256_set1_epi32(0x03030303);
let mask0F: __m256i = _mm256_set1_epi32(0x0F0F0F0F);
let (mut f0, mut f1, mut f2, mut f3);
for i in 0..(KYBER_N / 64) {
f0 = _mm256_load_si256(&buf[i]);
f1 = _mm256_srli_epi16(f0, 1);
f0 = _mm256_and_si256(mask55, f0);
f1 = _mm256_and_si256(mask55, f1);
f0 = _mm256_add_epi8(f0, f1);
f1 = _mm256_srli_epi16(f0, 1);
f0 = _mm256_and_si256(mask55, f0);
f1 = _mm256_and_si256(mask55, f1);
f0 = _mm256_add_epi8(f0, f1);
f1 = _mm256_srli_epi16(f0, 2);
f0 = _mm256_and_si256(mask33, f0);
f1 = _mm256_and_si256(mask33, f1);
f0 = _mm256_add_epi8(f0, mask33);
f0 = _mm256_sub_epi8(f0, f1);
f1 = _mm256_srli_epi16(f0, 2);
f0 = _mm256_and_si256(mask33, f0);
f1 = _mm256_and_si256(mask33, f1);
f0 = _mm256_add_epi8(f0, mask33);
f0 = _mm256_sub_epi8(f0, f1);
f1 = _mm256_srli_epi16(f0, 4);
f0 = _mm256_and_si256(mask0F, f0);
f1 = _mm256_and_si256(mask0F, f1);
f0 = _mm256_sub_epi8(f0, mask03);
f1 = _mm256_sub_epi8(f1, mask03);
f1 = _mm256_srli_epi16(f0, 4);
f0 = _mm256_and_si256(mask0F, f0);
f1 = _mm256_and_si256(mask0F, f1);
f0 = _mm256_sub_epi8(f0, mask03);
f1 = _mm256_sub_epi8(f1, mask03);
f2 = _mm256_unpacklo_epi8(f0, f1);
f3 = _mm256_unpackhi_epi8(f0, f1);
f2 = _mm256_unpacklo_epi8(f0, f1);
f3 = _mm256_unpackhi_epi8(f0, f1);
f0 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(f2));
f1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(f2,1));
f2 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(f3));
f3 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(f3,1));
f0 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(f2));
f1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(f2, 1));
f2 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(f3));
f3 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(f3, 1));
_mm256_store_si256(&mut r.vec[4*i+0], f0);
_mm256_store_si256(&mut r.vec[4*i+1], f2);
_mm256_store_si256(&mut r.vec[4*i+2], f1);
_mm256_store_si256(&mut r.vec[4*i+3], f3);
_mm256_store_si256(&mut r.vec[4 * i + 0], f0);
_mm256_store_si256(&mut r.vec[4 * i + 1], f2);
_mm256_store_si256(&mut r.vec[4 * i + 2], f1);
_mm256_store_si256(&mut r.vec[4 * i + 3], f3);
}
}
}
}
fn cbd3(r: &mut Poly, buf: &[u8]) {
unsafe {
let (mut f0, mut f1, mut f2, mut f3);
let mask249: __m256i = _mm256_set1_epi32(0x249249);
let mask6DB: __m256i = _mm256_set1_epi32(0x6DB6DB);
let mask07: __m256i = _mm256_set1_epi32(7);
let mask70: __m256i = _mm256_set1_epi32(7 << 16);
let mask: __m256i = _mm256_set1_epi16(3);
let shufbidx: __m256i = _mm256_set_epi8(
-1,15,14,13,-1,12,11,10,-1, 9, 8, 7,-1, 6, 5, 4,
-1,11,10, 9,-1, 8, 7, 6,-1, 5, 4, 3,-1, 2, 1, 0
);
unsafe {
let (mut f0, mut f1, mut f2, mut f3);
let mask249: __m256i = _mm256_set1_epi32(0x249249);
let mask6DB: __m256i = _mm256_set1_epi32(0x6DB6DB);
let mask07: __m256i = _mm256_set1_epi32(7);
let mask70: __m256i = _mm256_set1_epi32(7 << 16);
let mask: __m256i = _mm256_set1_epi16(3);
let shufbidx: __m256i = _mm256_set_epi8(
-1, 15, 14, 13, -1, 12, 11, 10, -1, 9, 8, 7, -1, 6, 5, 4, -1, 11, 10, 9, -1, 8, 7, 6, -1, 5, 4, 3, -1, 2, 1, 0,
);
for i in 0..(KYBER_N/32) {
f0 = _mm256_loadu_si256(buf[24*i..].as_ptr() as *const __m256i);
f0 = _mm256_permute4x64_epi64(f0,0x94);
f0 = _mm256_shuffle_epi8(f0,shufbidx);
for i in 0..(KYBER_N / 32) {
f0 = _mm256_loadu_si256(buf[24 * i..].as_ptr() as *const __m256i);
f0 = _mm256_permute4x64_epi64(f0, 0x94);
f0 = _mm256_shuffle_epi8(f0, shufbidx);
f1 = _mm256_srli_epi32(f0,1);
f2 = _mm256_srli_epi32(f0,2);
f0 = _mm256_and_si256(mask249,f0);
f1 = _mm256_and_si256(mask249,f1);
f2 = _mm256_and_si256(mask249,f2);
f0 = _mm256_add_epi32(f0,f1);
f0 = _mm256_add_epi32(f0,f2);
f1 = _mm256_srli_epi32(f0, 1);
f2 = _mm256_srli_epi32(f0, 2);
f0 = _mm256_and_si256(mask249, f0);
f1 = _mm256_and_si256(mask249, f1);
f2 = _mm256_and_si256(mask249, f2);
f0 = _mm256_add_epi32(f0, f1);
f0 = _mm256_add_epi32(f0, f2);
f1 = _mm256_srli_epi32(f0,3);
f0 = _mm256_add_epi32(f0,mask6DB);
f0 = _mm256_sub_epi32(f0,f1);
f1 = _mm256_srli_epi32(f0, 3);
f0 = _mm256_add_epi32(f0, mask6DB);
f0 = _mm256_sub_epi32(f0, f1);
f1 = _mm256_slli_epi32(f0,10);
f2 = _mm256_srli_epi32(f0,12);
f3 = _mm256_srli_epi32(f0, 2);
f0 = _mm256_and_si256(f0,mask07);
f1 = _mm256_and_si256(f1,mask70);
f2 = _mm256_and_si256(f2,mask07);
f3 = _mm256_and_si256(f3,mask70);
f0 = _mm256_add_epi16(f0,f1);
f1 = _mm256_add_epi16(f2,f3);
f0 = _mm256_sub_epi16(f0,mask);
f1 = _mm256_sub_epi16(f1,mask);
f1 = _mm256_slli_epi32(f0, 10);
f2 = _mm256_srli_epi32(f0, 12);
f3 = _mm256_srli_epi32(f0, 2);
f0 = _mm256_and_si256(f0, mask07);
f1 = _mm256_and_si256(f1, mask70);
f2 = _mm256_and_si256(f2, mask07);
f3 = _mm256_and_si256(f3, mask70);
f0 = _mm256_add_epi16(f0, f1);
f1 = _mm256_add_epi16(f2, f3);
f0 = _mm256_sub_epi16(f0, mask);
f1 = _mm256_sub_epi16(f1, mask);
f2 = _mm256_unpacklo_epi32(f0,f1);
f3 = _mm256_unpackhi_epi32(f0,f1);
f2 = _mm256_unpacklo_epi32(f0, f1);
f3 = _mm256_unpackhi_epi32(f0, f1);
f0 = _mm256_permute2x128_si256(f2,f3,0x20);
f1 = _mm256_permute2x128_si256(f2,f3,0x31);
f0 = _mm256_permute2x128_si256(f2, f3, 0x20);
f1 = _mm256_permute2x128_si256(f2, f3, 0x31);
_mm256_store_si256(&mut r.vec[2*i+0], f0);
_mm256_store_si256(&mut r.vec[2*i+1], f1);
_mm256_store_si256(&mut r.vec[2 * i + 0], f0);
_mm256_store_si256(&mut r.vec[2 * i + 1], f1);
}
}
}
}
pub fn poly_cbd_eta1(r: &mut Poly, buf: &Eta4xBuf)
{
unsafe {
if cfg!(feature="kyber512") {
cbd3(r, &buf.coeffs)
}
else {
cbd2(r, &buf.vec)
pub fn poly_cbd_eta1(r: &mut Poly, buf: &Eta4xBuf) {
unsafe {
if cfg!(feature = "kyber512") {
cbd3(r, &buf.coeffs)
} else {
cbd2(r, &buf.vec)
}
}
}
}
#[cfg(feature="90s")]
pub fn poly_cbd_eta1_90s(r: &mut Poly, buf: &IndcpaBuf)
{
unsafe {
if cfg!(feature="kyber512") {
cbd3(r, &buf.coeffs)
}
else {
cbd2(r, &buf.vec)
#[cfg(feature = "90s")]
pub fn poly_cbd_eta1_90s(r: &mut Poly, buf: &IndcpaBuf) {
unsafe {
if cfg!(feature = "kyber512") {
cbd3(r, &buf.coeffs)
} else {
cbd2(r, &buf.vec)
}
}
}
}
pub fn poly_cbd_eta2(r: &mut Poly, buf: &[__m256i])
{
cbd2(r, &buf)
}
pub fn poly_cbd_eta2(r: &mut Poly, buf: &[__m256i]) {
cbd2(r, &buf)
}

159
third_party/kyber/src/avx2/consts.rs vendored
View file

@ -1,116 +1,69 @@
use core::arch::x86_64::*;
use crate::params::KYBER_Q;
use core::arch::x86_64::*;
pub(crate) const Q: i16 = KYBER_Q as i16;
// pub(crate) const MONT: i16 = -1044; // 2^16 mod q
pub(crate) const QINV: i16 = -3327; // q^-1 mod 2^16
pub(crate) const V: i16 = 20159; // floor(2^26/q + 0.5)
pub(crate) const FHI: i16 = 1441; // mont^2/128
pub(crate) const FLO: i16 = -10079; // qinv*FHI
pub(crate) const MONTSQHI: i16 = 1353; // mont^2
pub(crate) const MONTSQLO: i16 = 20553; // qinv*MONTSQHI
pub(crate) const QINV: i16 = -3327; // q^-1 mod 2^16
pub(crate) const V: i16 = 20159; // floor(2^26/q + 0.5)
pub(crate) const FHI: i16 = 1441; // mont^2/128
pub(crate) const FLO: i16 = -10079; // qinv*FHI
pub(crate) const MONTSQHI: i16 = 1353; // mont^2
pub(crate) const MONTSQLO: i16 = 20553; // qinv*MONTSQHI
pub(crate) const MASK: i16 = 4095;
pub(crate) const SHIFT: i16 = 32;
pub(crate) const _16XQ: usize = 0;
pub(crate) const _16XQINV: usize = 16;
pub(crate) const _16XV: usize = 32;
pub(crate) const _16XFLO: usize = 48;
pub(crate) const _16XFHI: usize = 64;
pub(crate) const _16XMONTSQLO: usize = 80;
pub(crate) const _16XMONTSQHI: usize = 96;
pub(crate) const _16XMASK: usize = 112;
pub(crate) const _REVIDXB: usize = 128;
pub(crate) const _REVIDXD: usize = 144;
pub(crate) const _ZETAS_EXP: usize = 160;
pub(crate) const _16XSHIFT: usize = 624;
pub(crate) const _16XQ: usize = 0;
pub(crate) const _16XQINV: usize = 16;
pub(crate) const _16XV: usize = 32;
pub(crate) const _16XFLO: usize = 48;
pub(crate) const _16XFHI: usize = 64;
pub(crate) const _16XMONTSQLO: usize = 80;
pub(crate) const _16XMONTSQHI: usize = 96;
pub(crate) const _16XMASK: usize = 112;
pub(crate) const _REVIDXB: usize = 128;
pub(crate) const _REVIDXD: usize = 144;
pub(crate) const _ZETAS_EXP: usize = 160;
pub(crate) const _16XSHIFT: usize = 624;
#[repr(C, align(32))]
pub union Qdata {
pub coeffs: [i16; 640],
pub vec: [__m256i; 40]
pub coeffs: [i16; 640],
pub vec: [__m256i; 40],
}
pub const QDATA: Qdata = Qdata { coeffs:
[ Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q,
QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV,
QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV,
V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V,
FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO,
FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO,
FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI,
FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI,
MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO,
MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO,
MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO,
MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO,
MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI,
MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI,
MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI,
MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI,
MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK,
MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK,
3854, 3340, 2826, 2312, 1798, 1284, 770, 256,
3854, 3340, 2826, 2312, 1798, 1284, 770, 256,
7, 0, 6, 0, 5, 0, 4, 0, 3, 0, 2, 0, 1, 0, 0, 0,
31498, 31498, 31498, 31498, -758, -758, -758, -758,
5237, 5237, 5237, 5237, 1397, 1397, 1397, 1397,
14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745,
14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745,
-359, -359, -359, -359, -359, -359, -359, -359,
-359, -359, -359, -359, -359, -359, -359, -359,
13525, 13525, 13525, 13525, 13525, 13525, 13525, 13525,
-12402, -12402, -12402, -12402, -12402, -12402, -12402, -12402,
1493, 1493, 1493, 1493, 1493, 1493, 1493, 1493,
1422, 1422, 1422, 1422, 1422, 1422, 1422, 1422,
-20907, -20907, -20907, -20907, 27758, 27758, 27758, 27758,
-3799, -3799, -3799, -3799, -15690, -15690, -15690, -15690,
-171, -171, -171, -171, 622, 622, 622, 622,
1577, 1577, 1577, 1577, 182, 182, 182, 182,
-5827, -5827, 17363, 17363, -26360, -26360, -29057, -29057,
5571, 5571, -1102, -1102, 21438, 21438, -26242, -26242,
573, 573, -1325, -1325, 264, 264, 383, 383,
-829, -829, 1458, 1458, -1602, -1602, -130, -130,
-5689, -6516, 1496, 30967, -23565, 20179, 20710, 25080,
-12796, 26616, 16064, -12442, 9134, -650, -25986, 27837,
1223, 652, -552, 1015, -1293, 1491, -282, -1544,
516, -8, -320, -666, -1618, -1162, 126, 1469,
-335, -11477, -32227, 20494, -27738, 945, -14883, 6182,
32010, 10631, 29175, -28762, -18486, 17560, -14430, -5276,
-1103, 555, -1251, 1550, 422, 177, -291, 1574,
-246, 1159, -777, -602, -1590, -872, 418, -156,
11182, 13387, -14233, -21655, 13131, -4587, 23092, 5493,
-32502, 30317, -18741, 12639, 20100, 18525, 19529, -12619,
430, 843, 871, 105, 587, -235, -460, 1653,
778, -147, 1483, 1119, 644, 349, 329, -75,
787, 787, 787, 787, 787, 787, 787, 787,
787, 787, 787, 787, 787, 787, 787, 787,
-1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517,
-1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517,
28191, 28191, 28191, 28191, 28191, 28191, 28191, 28191,
-16694, -16694, -16694, -16694, -16694, -16694, -16694, -16694,
287, 287, 287, 287, 287, 287, 287, 287,
202, 202, 202, 202, 202, 202, 202, 202,
10690, 10690, 10690, 10690, 1358, 1358, 1358, 1358,
-11202, -11202, -11202, -11202, 31164, 31164, 31164, 31164,
962, 962, 962, 962, -1202, -1202, -1202, -1202,
-1474, -1474, -1474, -1474, 1468, 1468, 1468, 1468,
-28073, -28073, 24313, 24313, -10532, -10532, 8800, 8800,
18426, 18426, 8859, 8859, 26675, 26675, -16163, -16163,
-681, -681, 1017, 1017, 732, 732, 608, 608,
-1542, -1542, 411, 411, -205, -205, -1571, -1571,
19883, -28250, -15887, -8898, -28309, 9075, -30199, 18249,
13426, 14017, -29156, -12757, 16832, 4311, -24155, -17915,
-853, -90, -271, 830, 107, -1421, -247, -951,
-398, 961, -1508, -725, 448, -1065, 677, -1275,
-31183, 25435, -7382, 24391, -20927, 10946, 24214, 16989,
10335, -7934, -22502, 10906, 31636, 28644, 23998, -17422,
817, 603, 1322, -1465, -1215, 1218, -874, -1187,
-1185, -1278, -1510, -870, -108, 996, 958, 1522,
20297, 2146, 15355, -32384, -6280, -14903, -11044, 14469,
-21498, -20198, 23210, -17442, -23860, -20257, 7756, 23132,
1097, 610, -1285, 384, -136, -1335, 220, -1659,
-1530, 794, -854, 478, -308, 991, -1460, 1628,
SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT,
SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT ]
pub const QDATA: Qdata = Qdata {
coeffs: [
Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV, QINV,
QINV, QINV, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, V, FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO, FLO,
FLO, FLO, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, FHI, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO,
MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQLO, MONTSQHI,
MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI, MONTSQHI,
MONTSQHI, MONTSQHI, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, MASK, 3854, 3340,
2826, 2312, 1798, 1284, 770, 256, 3854, 3340, 2826, 2312, 1798, 1284, 770, 256, 7, 0, 6, 0, 5, 0, 4, 0, 3, 0, 2, 0, 1, 0, 0, 0,
31498, 31498, 31498, 31498, -758, -758, -758, -758, 5237, 5237, 5237, 5237, 1397, 1397, 1397, 1397, 14745, 14745, 14745, 14745,
14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, 14745, -359, -359, -359, -359, -359, -359, -359, -359,
-359, -359, -359, -359, -359, -359, -359, -359, 13525, 13525, 13525, 13525, 13525, 13525, 13525, 13525, -12402, -12402, -12402,
-12402, -12402, -12402, -12402, -12402, 1493, 1493, 1493, 1493, 1493, 1493, 1493, 1493, 1422, 1422, 1422, 1422, 1422, 1422, 1422,
1422, -20907, -20907, -20907, -20907, 27758, 27758, 27758, 27758, -3799, -3799, -3799, -3799, -15690, -15690, -15690, -15690, -171,
-171, -171, -171, 622, 622, 622, 622, 1577, 1577, 1577, 1577, 182, 182, 182, 182, -5827, -5827, 17363, 17363, -26360, -26360,
-29057, -29057, 5571, 5571, -1102, -1102, 21438, 21438, -26242, -26242, 573, 573, -1325, -1325, 264, 264, 383, 383, -829, -829,
1458, 1458, -1602, -1602, -130, -130, -5689, -6516, 1496, 30967, -23565, 20179, 20710, 25080, -12796, 26616, 16064, -12442, 9134,
-650, -25986, 27837, 1223, 652, -552, 1015, -1293, 1491, -282, -1544, 516, -8, -320, -666, -1618, -1162, 126, 1469, -335, -11477,
-32227, 20494, -27738, 945, -14883, 6182, 32010, 10631, 29175, -28762, -18486, 17560, -14430, -5276, -1103, 555, -1251, 1550, 422,
177, -291, 1574, -246, 1159, -777, -602, -1590, -872, 418, -156, 11182, 13387, -14233, -21655, 13131, -4587, 23092, 5493, -32502,
30317, -18741, 12639, 20100, 18525, 19529, -12619, 430, 843, 871, 105, 587, -235, -460, 1653, 778, -147, 1483, 1119, 644, 349, 329,
-75, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, -1517, -1517, -1517, -1517, -1517, -1517,
-1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, 28191, 28191, 28191, 28191, 28191, 28191, 28191, 28191,
-16694, -16694, -16694, -16694, -16694, -16694, -16694, -16694, 287, 287, 287, 287, 287, 287, 287, 287, 202, 202, 202, 202, 202,
202, 202, 202, 10690, 10690, 10690, 10690, 1358, 1358, 1358, 1358, -11202, -11202, -11202, -11202, 31164, 31164, 31164, 31164, 962,
962, 962, 962, -1202, -1202, -1202, -1202, -1474, -1474, -1474, -1474, 1468, 1468, 1468, 1468, -28073, -28073, 24313, 24313,
-10532, -10532, 8800, 8800, 18426, 18426, 8859, 8859, 26675, 26675, -16163, -16163, -681, -681, 1017, 1017, 732, 732, 608, 608,
-1542, -1542, 411, 411, -205, -205, -1571, -1571, 19883, -28250, -15887, -8898, -28309, 9075, -30199, 18249, 13426, 14017, -29156,
-12757, 16832, 4311, -24155, -17915, -853, -90, -271, 830, 107, -1421, -247, -951, -398, 961, -1508, -725, 448, -1065, 677, -1275,
-31183, 25435, -7382, 24391, -20927, 10946, 24214, 16989, 10335, -7934, -22502, 10906, 31636, 28644, 23998, -17422, 817, 603, 1322,
-1465, -1215, 1218, -874, -1187, -1185, -1278, -1510, -870, -108, 996, 958, 1522, 20297, 2146, 15355, -32384, -6280, -14903,
-11044, 14469, -21498, -20198, 23210, -17442, -23860, -20257, 7756, 23132, 1097, 610, -1285, 384, -136, -1335, 220, -1659, -1530,
794, -854, 478, -308, 991, -1460, 1628, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT, SHIFT,
SHIFT, SHIFT, SHIFT,
],
};

826
third_party/kyber/src/avx2/fips202.rs vendored
View file

@ -4,12 +4,11 @@ use crate::symmetric::KeccakState;
pub(crate) const SHAKE128_RATE: usize = 168;
pub(crate) const SHAKE256_RATE: usize = 136;
const SHA3_256_RATE: usize = 136;
const SHA3_512_RATE: usize = 72;
const SHA3_512_RATE: usize = 72;
const NROUNDS: usize = 24;
fn rol(a: u64, offset: u64) -> u64
{
(a << offset) ^ (a >> (64-offset))
fn rol(a: u64, offset: u64) -> u64 {
(a << offset) ^ (a >> (64 - offset))
}
// Name: load64
@ -19,13 +18,12 @@ fn rol(a: u64, offset: u64) -> u64
// Arguments: - const [u8] x: input byte array
//
// Returns the loaded 64-bit unsigned integer
pub fn load64(x: &[u8]) -> u64
{
let mut r = 0u64;
for i in 0..8 {
r |= (x[i] as u64) << (8 * i);
}
r
pub fn load64(x: &[u8]) -> u64 {
let mut r = 0u64;
for i in 0..8 {
r |= (x[i] as u64) << (8 * i);
}
r
}
// Name: store64
@ -34,40 +32,39 @@ pub fn load64(x: &[u8]) -> u64
//
// Arguments: - [u8] x: the output byte array
// - u64 u: input 64-bit unsigned integer
pub fn store64(x: &mut[u8], mut u: u64)
{
for i in x.iter_mut().take(8) {
*i = u as u8;
u >>= 8;
}
pub fn store64(x: &mut [u8], mut u: u64) {
for i in x.iter_mut().take(8) {
*i = u as u8;
u >>= 8;
}
}
// Keccak round constants
const KECCAKF_ROUNDCONSTANTS: [u64; NROUNDS] = [
0x0000000000000001,
0x0000000000008082,
0x800000000000808a,
0x8000000080008000,
0x000000000000808b,
0x0000000080000001,
0x8000000080008081,
0x8000000000008009,
0x000000000000008a,
0x0000000000000088,
0x0000000080008009,
0x000000008000000a,
0x000000008000808b,
0x800000000000008b,
0x8000000000008089,
0x8000000000008003,
0x8000000000008002,
0x8000000000000080,
0x000000000000800a,
0x800000008000000a,
0x8000000080008081,
0x8000000000008080,
0x0000000080000001,
0x8000000080008008
0x0000000000000001,
0x0000000000008082,
0x800000000000808a,
0x8000000080008000,
0x000000000000808b,
0x0000000080000001,
0x8000000080008081,
0x8000000000008009,
0x000000000000008a,
0x0000000000000088,
0x0000000080008009,
0x000000008000000a,
0x000000008000808b,
0x800000000000008b,
0x8000000000008089,
0x8000000000008003,
0x8000000000008002,
0x8000000000000080,
0x000000000000800a,
0x800000008000000a,
0x8000000080008081,
0x8000000000008080,
0x0000000080000001,
0x8000000080008008,
];
// Name: KeccakF1600_StatePermute
@ -75,252 +72,251 @@ const KECCAKF_ROUNDCONSTANTS: [u64; NROUNDS] = [
// Description: The Keccak F1600 Permutation
//
// Arguments: - u64 * state: in/output Keccak state
pub(crate) fn keccakf1600_statepermute(state: &mut[u64])
{
//copyFromState(A, state)
let mut aba = state[ 0];
let mut abe = state[ 1];
let mut abi = state[ 2];
let mut abo = state[ 3];
let mut abu = state[ 4];
let mut aga = state[ 5];
let mut age = state[ 6];
let mut agi = state[ 7];
let mut ago = state[ 8];
let mut agu = state[ 9];
let mut aka = state[10];
let mut ake = state[11];
let mut aki = state[12];
let mut ako = state[13];
let mut aku = state[14];
let mut ama = state[15];
let mut ame = state[16];
let mut ami = state[17];
let mut amo = state[18];
let mut amu = state[19];
let mut asa = state[20];
let mut ase = state[21];
let mut asi = state[22];
let mut aso = state[23];
let mut asu = state[24];
pub(crate) fn keccakf1600_statepermute(state: &mut [u64]) {
//copyFromState(A, state)
let mut aba = state[0];
let mut abe = state[1];
let mut abi = state[2];
let mut abo = state[3];
let mut abu = state[4];
let mut aga = state[5];
let mut age = state[6];
let mut agi = state[7];
let mut ago = state[8];
let mut agu = state[9];
let mut aka = state[10];
let mut ake = state[11];
let mut aki = state[12];
let mut ako = state[13];
let mut aku = state[14];
let mut ama = state[15];
let mut ame = state[16];
let mut ami = state[17];
let mut amo = state[18];
let mut amu = state[19];
let mut asa = state[20];
let mut ase = state[21];
let mut asi = state[22];
let mut aso = state[23];
let mut asu = state[24];
for round in (0..NROUNDS).step_by(2) {
// prepareTheta
let mut bca = aba^aga^aka^ama^asa;
let mut bce = abe^age^ake^ame^ase;
let mut bci = abi^agi^aki^ami^asi;
let mut bco = abo^ago^ako^amo^aso;
let mut bcu = abu^agu^aku^amu^asu;
for round in (0..NROUNDS).step_by(2) {
// prepareTheta
let mut bca = aba ^ aga ^ aka ^ ama ^ asa;
let mut bce = abe ^ age ^ ake ^ ame ^ ase;
let mut bci = abi ^ agi ^ aki ^ ami ^ asi;
let mut bco = abo ^ ago ^ ako ^ amo ^ aso;
let mut bcu = abu ^ agu ^ aku ^ amu ^ asu;
//thetaRhoPiChiIotaPrepareTheta(round , A, E)
let mut da = bcu^rol(bce, 1);
let mut de = bca^rol(bci, 1);
let mut di = bce^rol(bco, 1);
let mut d_o = bci^rol(bcu, 1);
let mut du = bco^rol(bca, 1);
//thetaRhoPiChiIotaPrepareTheta(round , A, E)
let mut da = bcu ^ rol(bce, 1);
let mut de = bca ^ rol(bci, 1);
let mut di = bce ^ rol(bco, 1);
let mut d_o = bci ^ rol(bcu, 1);
let mut du = bco ^ rol(bca, 1);
aba ^= da;
bca = aba;
age ^= de;
bce = rol(age, 44);
aki ^= di;
bci = rol(aki, 43);
amo ^= d_o;
bco = rol(amo, 21);
asu ^= du;
bcu = rol(asu, 14);
let mut eba = bca ^((!bce)& bci );
eba ^= KECCAKF_ROUNDCONSTANTS[round];
let mut ebe = bce ^((!bci)& bco );
let mut ebi = bci ^((!bco)& bcu );
let mut ebo = bco ^((!bcu)& bca );
let mut ebu = bcu ^((!bca)& bce );
aba ^= da;
bca = aba;
age ^= de;
bce = rol(age, 44);
aki ^= di;
bci = rol(aki, 43);
amo ^= d_o;
bco = rol(amo, 21);
asu ^= du;
bcu = rol(asu, 14);
let mut eba = bca ^ ((!bce) & bci);
eba ^= KECCAKF_ROUNDCONSTANTS[round];
let mut ebe = bce ^ ((!bci) & bco);
let mut ebi = bci ^ ((!bco) & bcu);
let mut ebo = bco ^ ((!bcu) & bca);
let mut ebu = bcu ^ ((!bca) & bce);
abo ^= d_o;
bca = rol(abo, 28);
agu ^= du;
bce = rol(agu, 20);
aka ^= da;
bci = rol(aka, 3);
ame ^= de;
bco = rol(ame, 45);
asi ^= di;
bcu = rol(asi, 61);
let mut ega = bca ^((!bce)& bci );
let mut ege = bce ^((!bci)& bco );
let mut egi = bci ^((!bco)& bcu );
let mut ego = bco ^((!bcu)& bca );
let mut egu = bcu ^((!bca)& bce );
abo ^= d_o;
bca = rol(abo, 28);
agu ^= du;
bce = rol(agu, 20);
aka ^= da;
bci = rol(aka, 3);
ame ^= de;
bco = rol(ame, 45);
asi ^= di;
bcu = rol(asi, 61);
let mut ega = bca ^ ((!bce) & bci);
let mut ege = bce ^ ((!bci) & bco);
let mut egi = bci ^ ((!bco) & bcu);
let mut ego = bco ^ ((!bcu) & bca);
let mut egu = bcu ^ ((!bca) & bce);
abe ^= de;
bca = rol(abe, 1);
agi ^= di;
bce = rol(agi, 6);
ako ^= d_o;
bci = rol(ako, 25);
amu ^= du;
bco = rol(amu, 8);
asa ^= da;
bcu = rol(asa, 18);
let mut eka = bca ^((!bce)& bci );
let mut eke = bce ^((!bci)& bco );
let mut eki = bci ^((!bco)& bcu );
let mut eko = bco ^((!bcu)& bca );
let mut eku = bcu ^((!bca)& bce );
abe ^= de;
bca = rol(abe, 1);
agi ^= di;
bce = rol(agi, 6);
ako ^= d_o;
bci = rol(ako, 25);
amu ^= du;
bco = rol(amu, 8);
asa ^= da;
bcu = rol(asa, 18);
let mut eka = bca ^ ((!bce) & bci);
let mut eke = bce ^ ((!bci) & bco);
let mut eki = bci ^ ((!bco) & bcu);
let mut eko = bco ^ ((!bcu) & bca);
let mut eku = bcu ^ ((!bca) & bce);
abu ^= du;
bca = rol(abu, 27);
aga ^= da;
bce = rol(aga, 36);
ake ^= de;
bci = rol(ake, 10);
ami ^= di;
bco = rol(ami, 15);
aso ^= d_o;
bcu = rol(aso, 56);
let mut ema = bca ^((!bce)& bci );
let mut eme = bce ^((!bci)& bco );
let mut emi = bci ^((!bco)& bcu );
let mut emo = bco ^((!bcu)& bca );
let mut emu = bcu ^((!bca)& bce );
abu ^= du;
bca = rol(abu, 27);
aga ^= da;
bce = rol(aga, 36);
ake ^= de;
bci = rol(ake, 10);
ami ^= di;
bco = rol(ami, 15);
aso ^= d_o;
bcu = rol(aso, 56);
let mut ema = bca ^ ((!bce) & bci);
let mut eme = bce ^ ((!bci) & bco);
let mut emi = bci ^ ((!bco) & bcu);
let mut emo = bco ^ ((!bcu) & bca);
let mut emu = bcu ^ ((!bca) & bce);
abi ^= di;
bca = rol(abi, 62);
ago ^= d_o;
bce = rol(ago, 55);
aku ^= du;
bci = rol(aku, 39);
ama ^= da;
bco = rol(ama, 41);
ase ^= de;
bcu = rol(ase, 2);
let mut esa = bca ^((!bce)& bci );
let mut ese = bce ^((!bci)& bco );
let mut esi = bci ^((!bco)& bcu );
let mut eso = bco ^((!bcu)& bca );
let mut esu = bcu ^((!bca)& bce );
abi ^= di;
bca = rol(abi, 62);
ago ^= d_o;
bce = rol(ago, 55);
aku ^= du;
bci = rol(aku, 39);
ama ^= da;
bco = rol(ama, 41);
ase ^= de;
bcu = rol(ase, 2);
let mut esa = bca ^ ((!bce) & bci);
let mut ese = bce ^ ((!bci) & bco);
let mut esi = bci ^ ((!bco) & bcu);
let mut eso = bco ^ ((!bcu) & bca);
let mut esu = bcu ^ ((!bca) & bce);
// prepareTheta
bca = eba^ega^eka^ema^esa;
bce = ebe^ege^eke^eme^ese;
bci = ebi^egi^eki^emi^esi;
bco = ebo^ego^eko^emo^eso;
bcu = ebu^egu^eku^emu^esu;
// prepareTheta
bca = eba ^ ega ^ eka ^ ema ^ esa;
bce = ebe ^ ege ^ eke ^ eme ^ ese;
bci = ebi ^ egi ^ eki ^ emi ^ esi;
bco = ebo ^ ego ^ eko ^ emo ^ eso;
bcu = ebu ^ egu ^ eku ^ emu ^ esu;
//thetaRhoPiChiIotaPrepareTheta(round+1, E, A)
da = bcu^rol(bce, 1);
de = bca^rol(bci, 1);
di = bce^rol(bco, 1);
d_o = bci^rol(bcu, 1);
du = bco^rol(bca, 1);
//thetaRhoPiChiIotaPrepareTheta(round+1, E, A)
da = bcu ^ rol(bce, 1);
de = bca ^ rol(bci, 1);
di = bce ^ rol(bco, 1);
d_o = bci ^ rol(bcu, 1);
du = bco ^ rol(bca, 1);
eba ^= da;
bca = eba;
ege ^= de;
bce = rol(ege, 44);
eki ^= di;
bci = rol(eki, 43);
emo ^= d_o;
bco = rol(emo, 21);
esu ^= du;
bcu = rol(esu, 14);
aba = bca ^((!bce)& bci );
aba ^= KECCAKF_ROUNDCONSTANTS[round+1];
abe = bce ^((!bci)& bco );
abi = bci ^((!bco)& bcu );
abo = bco ^((!bcu)& bca );
abu = bcu ^((!bca)& bce );
eba ^= da;
bca = eba;
ege ^= de;
bce = rol(ege, 44);
eki ^= di;
bci = rol(eki, 43);
emo ^= d_o;
bco = rol(emo, 21);
esu ^= du;
bcu = rol(esu, 14);
aba = bca ^ ((!bce) & bci);
aba ^= KECCAKF_ROUNDCONSTANTS[round + 1];
abe = bce ^ ((!bci) & bco);
abi = bci ^ ((!bco) & bcu);
abo = bco ^ ((!bcu) & bca);
abu = bcu ^ ((!bca) & bce);
ebo ^= d_o;
bca = rol(ebo, 28);
egu ^= du;
bce = rol(egu, 20);
eka ^= da;
bci = rol(eka, 3);
eme ^= de;
bco = rol(eme, 45);
esi ^= di;
bcu = rol(esi, 61);
aga = bca ^((!bce)& bci );
age = bce ^((!bci)& bco );
agi = bci ^((!bco)& bcu );
ago = bco ^((!bcu)& bca );
agu = bcu ^((!bca)& bce );
ebo ^= d_o;
bca = rol(ebo, 28);
egu ^= du;
bce = rol(egu, 20);
eka ^= da;
bci = rol(eka, 3);
eme ^= de;
bco = rol(eme, 45);
esi ^= di;
bcu = rol(esi, 61);
aga = bca ^ ((!bce) & bci);
age = bce ^ ((!bci) & bco);
agi = bci ^ ((!bco) & bcu);
ago = bco ^ ((!bcu) & bca);
agu = bcu ^ ((!bca) & bce);
ebe ^= de;
bca = rol(ebe, 1);
egi ^= di;
bce = rol(egi, 6);
eko ^= d_o;
bci = rol(eko, 25);
emu ^= du;
bco = rol(emu, 8);
esa ^= da;
bcu = rol(esa, 18);
aka = bca ^((!bce)& bci );
ake = bce ^((!bci)& bco );
aki = bci ^((!bco)& bcu );
ako = bco ^((!bcu)& bca );
aku = bcu ^((!bca)& bce );
ebe ^= de;
bca = rol(ebe, 1);
egi ^= di;
bce = rol(egi, 6);
eko ^= d_o;
bci = rol(eko, 25);
emu ^= du;
bco = rol(emu, 8);
esa ^= da;
bcu = rol(esa, 18);
aka = bca ^ ((!bce) & bci);
ake = bce ^ ((!bci) & bco);
aki = bci ^ ((!bco) & bcu);
ako = bco ^ ((!bcu) & bca);
aku = bcu ^ ((!bca) & bce);
ebu ^= du;
bca = rol(ebu, 27);
ega ^= da;
bce = rol(ega, 36);
eke ^= de;
bci = rol(eke, 10);
emi ^= di;
bco = rol(emi, 15);
eso ^= d_o;
bcu = rol(eso, 56);
ama = bca ^((!bce)& bci );
ame = bce ^((!bci)& bco );
ami = bci ^((!bco)& bcu );
amo = bco ^((!bcu)& bca );
amu = bcu ^((!bca)& bce );
ebu ^= du;
bca = rol(ebu, 27);
ega ^= da;
bce = rol(ega, 36);
eke ^= de;
bci = rol(eke, 10);
emi ^= di;
bco = rol(emi, 15);
eso ^= d_o;
bcu = rol(eso, 56);
ama = bca ^ ((!bce) & bci);
ame = bce ^ ((!bci) & bco);
ami = bci ^ ((!bco) & bcu);
amo = bco ^ ((!bcu) & bca);
amu = bcu ^ ((!bca) & bce);
ebi ^= di;
bca = rol(ebi, 62);
ego ^= d_o;
bce = rol(ego, 55);
eku ^= du;
bci = rol(eku, 39);
ema ^= da;
bco = rol(ema, 41);
ese ^= de;
bcu = rol(ese, 2);
asa = bca ^((!bce)& bci );
ase = bce ^((!bci)& bco );
asi = bci ^((!bco)& bcu );
aso = bco ^((!bcu)& bca );
asu = bcu ^((!bca)& bce );
}
ebi ^= di;
bca = rol(ebi, 62);
ego ^= d_o;
bce = rol(ego, 55);
eku ^= du;
bci = rol(eku, 39);
ema ^= da;
bco = rol(ema, 41);
ese ^= de;
bcu = rol(ese, 2);
asa = bca ^ ((!bce) & bci);
ase = bce ^ ((!bci) & bco);
asi = bci ^ ((!bco) & bcu);
aso = bco ^ ((!bcu) & bca);
asu = bcu ^ ((!bca) & bce);
}
state[ 0] = aba;
state[ 1] = abe;
state[ 2] = abi;
state[ 3] = abo;
state[ 4] = abu;
state[ 5] = aga;
state[ 6] = age;
state[ 7] = agi;
state[ 8] = ago;
state[ 9] = agu;
state[10] = aka;
state[11] = ake;
state[12] = aki;
state[13] = ako;
state[14] = aku;
state[15] = ama;
state[16] = ame;
state[17] = ami;
state[18] = amo;
state[19] = amu;
state[20] = asa;
state[21] = ase;
state[22] = asi;
state[23] = aso;
state[24] = asu;
state[0] = aba;
state[1] = abe;
state[2] = abi;
state[3] = abo;
state[4] = abu;
state[5] = aga;
state[6] = age;
state[7] = agi;
state[8] = ago;
state[9] = agu;
state[10] = aka;
state[11] = ake;
state[12] = aki;
state[13] = ako;
state[14] = aku;
state[15] = ama;
state[16] = ame;
state[17] = ami;
state[18] = amo;
state[19] = amu;
state[20] = asa;
state[21] = ase;
state[22] = asi;
state[23] = aso;
state[24] = asu;
}
// Name: keccak_absorb
@ -333,29 +329,22 @@ pub(crate) fn keccakf1600_statepermute(state: &mut[u64])
// - usize r: rate in bytes (e.g., 168 for SHAKE128)
// - const [u8] input: pointer to input to be absorbed into s
// - u64 inlen: length of input in bytes
pub(crate) fn keccak_absorb(
s: &mut[u64],
mut pos: usize,
r: usize,
input: &[u8],
mut inlen: usize
) -> usize
{
let mut idx = 0usize;
while pos+inlen >= r {
for i in pos..r {
s[i/8] ^= (input[idx] as u64) << 8 * (i%8);
idx += 1;
pub(crate) fn keccak_absorb(s: &mut [u64], mut pos: usize, r: usize, input: &[u8], mut inlen: usize) -> usize {
let mut idx = 0usize;
while pos + inlen >= r {
for i in pos..r {
s[i / 8] ^= (input[idx] as u64) << 8 * (i % 8);
idx += 1;
}
inlen -= r - pos;
keccakf1600_statepermute(s);
pos = 0;
}
inlen -= r-pos;
keccakf1600_statepermute(s);
pos = 0;
}
let new_pos = pos+inlen;
for i in pos..new_pos {
s[i/8] ^= (input[idx] as u64) << 8 * (i%8);
}
new_pos
let new_pos = pos + inlen;
for i in pos..new_pos {
s[i / 8] ^= (input[idx] as u64) << 8 * (i % 8);
}
new_pos
}
// Name: keccak_squeezeblocks
@ -368,17 +357,16 @@ pub(crate) fn keccak_absorb(
// - u64 nblocks: number of blocks to be squeezed (written to h)
// - u64 *s: in/output Keccak state
// - usize r: rate in bytes (e.g., 168 for SHAKE128)
pub(crate) fn keccak_squeezeblocks(h: &mut[u8], mut nblocks: usize, s: &mut [u64], r: usize)
{
let mut idx = 0usize;
while nblocks > 0 {
keccakf1600_statepermute(s);
for i in 0..r/8 {
store64(&mut h[idx+8*i..], s[i])
pub(crate) fn keccak_squeezeblocks(h: &mut [u8], mut nblocks: usize, s: &mut [u64], r: usize) {
let mut idx = 0usize;
while nblocks > 0 {
keccakf1600_statepermute(s);
for i in 0..r / 8 {
store64(&mut h[idx + 8 * i..], s[i])
}
idx += r;
nblocks -= 1;
}
idx += r;
nblocks -= 1;
}
}
// Name: shake128_absorb
@ -389,10 +377,9 @@ pub(crate) fn keccak_squeezeblocks(h: &mut[u8], mut nblocks: usize, s: &mut [u64
// Arguments: - u64 *s: (uninitialized) output Keccak state
// - const [u8] input: input to be absorbed into s
// - u64 inputByteLen: length of input in bytes
pub(crate) fn shake128_absorb(state: &mut KeccakState, input: &[u8], inlen: usize)
{
let pos = state.pos;
state.pos =keccak_absorb(&mut state.s, pos, SHAKE128_RATE, input, inlen);
pub(crate) fn shake128_absorb(state: &mut KeccakState, input: &[u8], inlen: usize) {
let pos = state.pos;
state.pos = keccak_absorb(&mut state.s, pos, SHAKE128_RATE, input, inlen);
}
// Name: shake128_squeezeblocks
@ -405,9 +392,8 @@ pub(crate) fn shake128_absorb(state: &mut KeccakState, input: &[u8], inlen: usiz
// Arguments: - [u8] out: pointer to output blocks
// - u64 nblocks: number of blocks to be squeezed (written to output)
// - KeccakState state: pointer to input/output Keccak state
pub(crate) fn shake128_squeezeblocks(out: &mut[u8], nblocks: usize, state: &mut KeccakState)
{
keccak_squeezeblocks(out, nblocks, &mut state.s, SHAKE128_RATE);
pub(crate) fn shake128_squeezeblocks(out: &mut [u8], nblocks: usize, state: &mut KeccakState) {
keccak_squeezeblocks(out, nblocks, &mut state.s, SHAKE128_RATE);
}
// Name: shake256
@ -418,16 +404,15 @@ pub(crate) fn shake128_squeezeblocks(out: &mut[u8], nblocks: usize, state: &mut
// - usize outlen: requested output length in bytes
// - [u8] input: input
// - usize inlen: length of input in bytes
pub(crate) fn shake256(out: &mut[u8], mut outlen: usize, input: &[u8], inlen: usize)
{
let mut state = KeccakState::new();
let mut idx = 0;
shake256_absorb_once(&mut state, input, inlen);
let nblocks = outlen/SHAKE256_RATE;
shake256_squeezeblocks(&mut out[idx..], nblocks, &mut state);
outlen -= nblocks*SHAKE256_RATE;
idx += nblocks*SHAKE256_RATE;
shake256_squeeze(&mut out[idx..], outlen, &mut state);
pub(crate) fn shake256(out: &mut [u8], mut outlen: usize, input: &[u8], inlen: usize) {
let mut state = KeccakState::new();
let mut idx = 0;
shake256_absorb_once(&mut state, input, inlen);
let nblocks = outlen / SHAKE256_RATE;
shake256_squeezeblocks(&mut out[idx..], nblocks, &mut state);
outlen -= nblocks * SHAKE256_RATE;
idx += nblocks * SHAKE256_RATE;
shake256_squeeze(&mut out[idx..], outlen, &mut state);
}
// Name: sha3_256
@ -437,14 +422,13 @@ pub(crate) fn shake256(out: &mut[u8], mut outlen: usize, input: &[u8], inlen: us
// Arguments: - [u8] h: output (32 bytes)
// - const [u8] input: input
// - usize inlen: length of input in bytes
pub(crate) fn sha3_256(h: &mut[u8], input: &[u8], inlen: usize)
{
let mut s = [0u64; 25];
keccak_absorb_once(&mut s, SHA3_256_RATE, input, inlen, 0x06);
keccakf1600_statepermute(&mut s);
for i in 0..4 {
store64(&mut h[8*i..], s[i]);
}
pub(crate) fn sha3_256(h: &mut [u8], input: &[u8], inlen: usize) {
let mut s = [0u64; 25];
keccak_absorb_once(&mut s, SHA3_256_RATE, input, inlen, 0x06);
keccakf1600_statepermute(&mut s);
for i in 0..4 {
store64(&mut h[8 * i..], s[i]);
}
}
// Name: sha3_512
@ -454,18 +438,15 @@ pub(crate) fn sha3_256(h: &mut[u8], input: &[u8], inlen: usize)
// Arguments: - [u8] h: output (64 bytes)
// - const [u8] input: input
// - usize inlen: length of input in bytes
pub(crate) fn sha3_512(h: &mut[u8], input: &[u8], inlen: usize)
{
let mut s = [0u64; 25];
keccak_absorb_once(&mut s, SHA3_512_RATE, input, inlen, 0x06);
keccakf1600_statepermute(&mut s);
for i in 0..8 {
store64(&mut h[8*i..], s[i]);
}
pub(crate) fn sha3_512(h: &mut [u8], input: &[u8], inlen: usize) {
let mut s = [0u64; 25];
keccak_absorb_once(&mut s, SHA3_512_RATE, input, inlen, 0x06);
keccakf1600_statepermute(&mut s);
for i in 0..8 {
store64(&mut h[8 * i..], s[i]);
}
}
// Name: keccak_finalize
//
// Description: Finalize absorb step.
@ -474,10 +455,9 @@ pub(crate) fn sha3_512(h: &mut[u8], input: &[u8], inlen: usize)
// - usize pos: position in current block to be absorbed
// - usize r: rate in bytes (e.g., 168 for SHAKE128)
// - u8 p: domain separation byte
fn keccak_finalize(s: &mut[u64], pos: usize, r: usize, p: u8)
{
s[pos/8] ^= (p as u64) << 8*(pos%8);
s[r/8-1] ^= 1u64 << 63;
fn keccak_finalize(s: &mut [u64], pos: usize, r: usize, p: u8) {
s[pos / 8] ^= (p as u64) << 8 * (pos % 8);
s[r / 8 - 1] ^= 1u64 << 63;
}
// Name: keccak_absorb_once
@ -490,34 +470,27 @@ fn keccak_finalize(s: &mut[u64], pos: usize, r: usize, p: u8)
// - const [u8] input: input to be absorbed into s
// - u64 mlen: length of input in bytes
// - [u8] p: domain-separation byte for different Keccak-derived functions
pub(crate) fn keccak_absorb_once(
s: &mut[u64],
r: usize,
input: &[u8],
mut inlen:
usize,
p: u8)
{
// Zero State
for i in s.iter_mut() {
*i = 0;
}
let mut idx = 0usize;
while inlen >= r {
for i in 0..(r/8) {
s[i] ^= load64(&input[idx+8*i..]);
pub(crate) fn keccak_absorb_once(s: &mut [u64], r: usize, input: &[u8], mut inlen: usize, p: u8) {
// Zero State
for i in s.iter_mut() {
*i = 0;
}
idx += r;
inlen -= r;
keccakf1600_statepermute(s);
}
for i in 0..inlen {
s[i/8] ^= (input[idx+i] as u64) << 8*(i%8);
}
s[inlen/8] ^= (p as u64) << 8*(inlen%8);
s[(r-1)/8] ^= 1u64 << 63;
let mut idx = 0usize;
while inlen >= r {
for i in 0..(r / 8) {
s[i] ^= load64(&input[idx + 8 * i..]);
}
idx += r;
inlen -= r;
keccakf1600_statepermute(s);
}
for i in 0..inlen {
s[i / 8] ^= (input[idx + i] as u64) << 8 * (i % 8);
}
s[inlen / 8] ^= (p as u64) << 8 * (inlen % 8);
s[(r - 1) / 8] ^= 1u64 << 63;
}
// Name: keccak_squeeze
@ -532,30 +505,23 @@ pub(crate) fn keccak_absorb_once(
// usize pos: number of bytes in current block already squeezed
// - usize r: rate in bytes (e.g., 168 for SHAKE128)
// Returns new position pos in current block
pub(crate) fn keccak_squeeze(
out: &mut[u8],
mut outlen: usize,
s: &mut [u64],
mut pos: usize,
r: usize
) -> usize
{
let mut idx = 0;
while outlen > 0 {
if pos == r {
keccakf1600_statepermute(s);
pos = 0
pub(crate) fn keccak_squeeze(out: &mut [u8], mut outlen: usize, s: &mut [u64], mut pos: usize, r: usize) -> usize {
let mut idx = 0;
while outlen > 0 {
if pos == r {
keccakf1600_statepermute(s);
pos = 0
}
let mut i = pos;
while i < r && i < pos + outlen {
out[idx] = (s[i / 8] >> 8 * (i % 8)) as u8;
i += 1;
idx += 1;
}
outlen -= i - pos;
pos = i;
}
let mut i = pos;
while i < r && i < pos+outlen {
out[idx] = (s[i/8] >> 8*(i%8)) as u8;
i += 1;
idx += 1;
}
outlen -= i-pos;
pos = i;
}
pos
pos
}
// Name: shake128_init
@ -563,21 +529,18 @@ pub(crate) fn keccak_squeeze(
// Description: Initializes Keccak state for use as SHAKE128 XOF
//
// Arguments: - keccak_state state: (uninitialized) Keccak state
fn shake128_init(state: &mut KeccakState)
{
state.reset()
fn shake128_init(state: &mut KeccakState) {
state.reset()
}
// Name: shake128_finalize
//
// Description: Finalize absorb step of the SHAKE128 XOF.
//
// Arguments: - keccak_state state: pointer to Keccak state
fn shake128_finalize(state: &mut KeccakState)
{
keccak_finalize(&mut state.s, state.pos, SHAKE128_RATE, 0x1F);
state.pos = SHAKE128_RATE;
fn shake128_finalize(state: &mut KeccakState) {
keccak_finalize(&mut state.s, state.pos, SHAKE128_RATE, 0x1F);
state.pos = SHAKE128_RATE;
}
// Name: shake128_squeeze
@ -588,9 +551,8 @@ fn shake128_finalize(state: &mut KeccakState)
// Arguments: - [u8] out: pointer to output blocks
// - usize outlen : number of bytes to be squeezed (written to output)
// - keccak_state s: pointer to input/output Keccak state
fn shake128_squeeze(out: &mut[u8], outlen: usize, state: &mut KeccakState)
{
state.pos = keccak_squeeze(out, outlen, &mut state.s, state.pos, SHAKE128_RATE);
fn shake128_squeeze(out: &mut [u8], outlen: usize, state: &mut KeccakState) {
state.pos = keccak_squeeze(out, outlen, &mut state.s, state.pos, SHAKE128_RATE);
}
// Name: shake128_absorb_once
@ -600,52 +562,44 @@ fn shake128_squeeze(out: &mut[u8], outlen: usize, state: &mut KeccakState)
// Arguments: - keccak_state state: pointer to (uninitialized) output Keccak state
// - const [u8] in: input to be absorbed into s
// - usize inlen: length of input in bytes
pub(crate) fn shake128_absorb_once(state: &mut KeccakState, input: &[u8], inlen: usize)
{
keccak_absorb_once(&mut state.s, SHAKE128_RATE, input, inlen, 0x1F);
state.pos = SHAKE128_RATE;
pub(crate) fn shake128_absorb_once(state: &mut KeccakState, input: &[u8], inlen: usize) {
keccak_absorb_once(&mut state.s, SHAKE128_RATE, input, inlen, 0x1F);
state.pos = SHAKE128_RATE;
}
fn shake256_init(state: &mut KeccakState) {
state.reset();
state.reset();
}
fn shake256_absorb(state: &mut KeccakState, input: &[u8], inlen: usize)
{
state.pos = keccak_absorb(&mut state.s, state.pos, SHAKE256_RATE, input, inlen);
fn shake256_absorb(state: &mut KeccakState, input: &[u8], inlen: usize) {
state.pos = keccak_absorb(&mut state.s, state.pos, SHAKE256_RATE, input, inlen);
}
fn shake256_finalize(state: &mut KeccakState)
{
keccak_finalize(&mut state.s, state.pos, SHAKE256_RATE, 0x1F);
state.pos = SHAKE256_RATE;
fn shake256_finalize(state: &mut KeccakState) {
keccak_finalize(&mut state.s, state.pos, SHAKE256_RATE, 0x1F);
state.pos = SHAKE256_RATE;
}
fn shake256_squeeze(out: &mut[u8], outlen: usize, state: &mut KeccakState)
{
state.pos = keccak_squeeze(out, outlen, &mut state.s, state.pos, SHAKE256_RATE);
fn shake256_squeeze(out: &mut [u8], outlen: usize, state: &mut KeccakState) {
state.pos = keccak_squeeze(out, outlen, &mut state.s, state.pos, SHAKE256_RATE);
}
pub(crate) fn shake256_absorb_once(state: &mut KeccakState, input: &[u8], inlen: usize)
{
keccak_absorb_once(&mut state.s, SHAKE256_RATE, input, inlen, 0x1F);
state.pos = SHAKE256_RATE;
pub(crate) fn shake256_absorb_once(state: &mut KeccakState, input: &[u8], inlen: usize) {
keccak_absorb_once(&mut state.s, SHAKE256_RATE, input, inlen, 0x1F);
state.pos = SHAKE256_RATE;
}
fn shake256_squeezeblocks(out: &mut[u8], nblocks: usize, state: &mut KeccakState)
{
keccak_squeezeblocks(out, nblocks, &mut state.s, SHAKE256_RATE);
fn shake256_squeezeblocks(out: &mut [u8], nblocks: usize, state: &mut KeccakState) {
keccak_squeezeblocks(out, nblocks, &mut state.s, SHAKE256_RATE);
}
fn shake128(out: &mut[u8], mut outlen: usize, input: &[u8], inlen: usize)
{
let mut state = KeccakState::new();
let mut idx = 0;
shake128_absorb_once(&mut state, input, inlen);
let nblocks = outlen/SHAKE128_RATE;
shake128_squeezeblocks(&mut out[idx..], nblocks, &mut state);
outlen -= nblocks*SHAKE128_RATE;
idx += nblocks*SHAKE128_RATE;
shake128_squeeze(&mut out[idx..], outlen, &mut state);
fn shake128(out: &mut [u8], mut outlen: usize, input: &[u8], inlen: usize) {
let mut state = KeccakState::new();
let mut idx = 0;
shake128_absorb_once(&mut state, input, inlen);
let nblocks = outlen / SHAKE128_RATE;
shake128_squeezeblocks(&mut out[idx..], nblocks, &mut state);
outlen -= nblocks * SHAKE128_RATE;
idx += nblocks * SHAKE128_RATE;
shake128_squeeze(&mut out[idx..], outlen, &mut state);
}

246
third_party/kyber/src/avx2/fips202x4.rs vendored
View file

@ -1,124 +1,106 @@
#![allow(dead_code)]
use core::arch::x86_64::*;
use crate::align::{Eta4xBuf, GenMatrixBuf};
use crate::fips202::*;
use crate::keccak4x::f1600_x4;
use crate::align::{GenMatrixBuf, Eta4xBuf};
use core::arch::x86_64::*;
#[repr(C)]
pub struct Keccakx4State {
s: [__m256i; 25]
s: [__m256i; 25],
}
impl Keccakx4State {
pub fn new() -> Self {
unsafe {Keccakx4State { s: [_mm256_setzero_si256(); 25]}}
}
pub fn new() -> Self {
unsafe { Keccakx4State { s: [_mm256_setzero_si256(); 25] } }
}
}
pub unsafe fn keccakx4_absorb_once(
s: &mut[__m256i; 25],
r: usize,
in0: &[u8],
in1: &[u8],
in2: &[u8],
in3: &[u8],
mut inlen: usize,
p: u8
)
{
let mut pos = 0i64;
let mut t;
for i in 0..25 {
s[i] = _mm256_setzero_si256();
}
let mut idx = _mm256_set_epi64x(
in3.as_ptr() as i64,
in2.as_ptr() as i64,
in1.as_ptr() as i64,
in0.as_ptr() as i64,
);
while inlen >= r {
for i in 0..(r/8) {
t = _mm256_i64gather_epi64(pos as *const i64, idx, 1);
s[i] = _mm256_xor_si256(s[i], t);
pos += 8;
s: &mut [__m256i; 25],
r: usize,
in0: &[u8],
in1: &[u8],
in2: &[u8],
in3: &[u8],
mut inlen: usize,
p: u8,
) {
let mut pos = 0i64;
let mut t;
for i in 0..25 {
s[i] = _mm256_setzero_si256();
}
inlen -= r;
f1600_x4(s);
}
let end = inlen/8;
for i in 0..end {
t = _mm256_i64gather_epi64(pos as *const i64, idx, 1);
s[i] = _mm256_xor_si256(s[i], t);
pos += 8;
}
inlen -= 8*end;
let mut idx = _mm256_set_epi64x(in3.as_ptr() as i64, in2.as_ptr() as i64, in1.as_ptr() as i64, in0.as_ptr() as i64);
while inlen >= r {
for i in 0..(r / 8) {
t = _mm256_i64gather_epi64(pos as *const i64, idx, 1);
s[i] = _mm256_xor_si256(s[i], t);
pos += 8;
}
inlen -= r;
f1600_x4(s);
}
let end = inlen / 8;
for i in 0..end {
t = _mm256_i64gather_epi64(pos as *const i64, idx, 1);
s[i] = _mm256_xor_si256(s[i], t);
pos += 8;
}
inlen -= 8 * end;
if inlen > 0 {
t = _mm256_i64gather_epi64(pos as *const i64, idx, 1);
idx = _mm256_set1_epi64x(((1u64 << (8*inlen)) - 1) as i64);
t = _mm256_and_si256(t, idx);
if inlen > 0 {
t = _mm256_i64gather_epi64(pos as *const i64, idx, 1);
idx = _mm256_set1_epi64x(((1u64 << (8 * inlen)) - 1) as i64);
t = _mm256_and_si256(t, idx);
s[end] = _mm256_xor_si256(s[end], t);
}
t = _mm256_set1_epi64x(((p as u64) << 8 * inlen) as i64);
s[end] = _mm256_xor_si256(s[end], t);
}
t = _mm256_set1_epi64x(((p as u64) << 8*inlen) as i64);
s[end] = _mm256_xor_si256(s[end], t);
t = _mm256_set1_epi64x((1u64 << 63) as i64);
s[r/8 - 1] = _mm256_xor_si256(s[r/8 - 1], t);
t = _mm256_set1_epi64x((1u64 << 63) as i64);
s[r / 8 - 1] = _mm256_xor_si256(s[r / 8 - 1], t);
}
pub unsafe fn keccakx4_squeezeblocks128(
out: &mut [GenMatrixBuf; 4],
mut nblocks: usize,
r: usize,
s: &mut [__m256i; 25]
)
{
let mut t;
let mut idx = 0usize;
while nblocks > 0 {
f1600_x4(s);
for i in 0..(r/8) {
t = _mm_castsi128_pd(_mm256_castsi256_si128(s[i]));
let out0_ptr = out[0].coeffs[idx+8*i..].as_mut_ptr();
let out1_ptr = out[1].coeffs[idx+8*i..].as_mut_ptr();
_mm_storel_pd(out0_ptr as *mut f64, t);
_mm_storeh_pd(out1_ptr as *mut f64, t);
t = _mm_castsi128_pd(_mm256_extracti128_si256(s[i],1));
let out2_ptr = out[2].coeffs[idx+8*i..].as_mut_ptr();
let out3_ptr = out[3].coeffs[idx+8*i..].as_mut_ptr();
_mm_storel_pd(out2_ptr as *mut f64, t);
_mm_storeh_pd(out3_ptr as *mut f64, t);
pub unsafe fn keccakx4_squeezeblocks128(out: &mut [GenMatrixBuf; 4], mut nblocks: usize, r: usize, s: &mut [__m256i; 25]) {
let mut t;
let mut idx = 0usize;
while nblocks > 0 {
f1600_x4(s);
for i in 0..(r / 8) {
t = _mm_castsi128_pd(_mm256_castsi256_si128(s[i]));
let out0_ptr = out[0].coeffs[idx + 8 * i..].as_mut_ptr();
let out1_ptr = out[1].coeffs[idx + 8 * i..].as_mut_ptr();
_mm_storel_pd(out0_ptr as *mut f64, t);
_mm_storeh_pd(out1_ptr as *mut f64, t);
t = _mm_castsi128_pd(_mm256_extracti128_si256(s[i], 1));
let out2_ptr = out[2].coeffs[idx + 8 * i..].as_mut_ptr();
let out3_ptr = out[3].coeffs[idx + 8 * i..].as_mut_ptr();
_mm_storel_pd(out2_ptr as *mut f64, t);
_mm_storeh_pd(out3_ptr as *mut f64, t);
}
idx += r;
nblocks -= 1;
}
idx += r;
nblocks -= 1;
}
}
pub unsafe fn keccakx4_squeezeblocks256(
out: &mut [Eta4xBuf; 4],
mut nblocks: usize,
r: usize,
s: &mut [__m256i; 25]
)
{
let mut t;
let mut idx = 0usize;
while nblocks > 0 {
f1600_x4(s);
for i in 0..(r/8) {
t = _mm_castsi128_pd(_mm256_castsi256_si128(s[i]));
_mm_storel_pd(out[0].coeffs[idx+8*i..].as_mut_ptr() as *mut f64, t);
_mm_storeh_pd(out[1].coeffs[idx+8*i..].as_mut_ptr() as *mut f64, t);
t = _mm_castsi128_pd(_mm256_extracti128_si256(s[i],1));
_mm_storel_pd(out[2].coeffs[idx+8*i..].as_mut_ptr() as *mut f64, t);
_mm_storeh_pd(out[3].coeffs[idx+8*i..].as_mut_ptr() as *mut f64, t);
pub unsafe fn keccakx4_squeezeblocks256(out: &mut [Eta4xBuf; 4], mut nblocks: usize, r: usize, s: &mut [__m256i; 25]) {
let mut t;
let mut idx = 0usize;
while nblocks > 0 {
f1600_x4(s);
for i in 0..(r / 8) {
t = _mm_castsi128_pd(_mm256_castsi256_si128(s[i]));
_mm_storel_pd(out[0].coeffs[idx + 8 * i..].as_mut_ptr() as *mut f64, t);
_mm_storeh_pd(out[1].coeffs[idx + 8 * i..].as_mut_ptr() as *mut f64, t);
t = _mm_castsi128_pd(_mm256_extracti128_si256(s[i], 1));
_mm_storel_pd(out[2].coeffs[idx + 8 * i..].as_mut_ptr() as *mut f64, t);
_mm_storeh_pd(out[3].coeffs[idx + 8 * i..].as_mut_ptr() as *mut f64, t);
}
idx += r;
nblocks -= 1;
}
idx += r;
nblocks -= 1;
}
}
// pub unsafe fn keccakx4_squeezeonce128(
@ -156,66 +138,20 @@ pub unsafe fn keccakx4_squeezeblocks256(
// }
// }
pub unsafe fn shake128x4_absorb_once(
state: &mut Keccakx4State,
in0: &[u8],
in1: &[u8],
in2: &[u8],
in3: &[u8],
inlen: usize,
)
{
keccakx4_absorb_once(
&mut state.s,
SHAKE128_RATE,
in0, in1, in2, in3, inlen,
0x1F
)
pub unsafe fn shake128x4_absorb_once(state: &mut Keccakx4State, in0: &[u8], in1: &[u8], in2: &[u8], in3: &[u8], inlen: usize) {
keccakx4_absorb_once(&mut state.s, SHAKE128_RATE, in0, in1, in2, in3, inlen, 0x1F)
}
pub unsafe fn shake128x4_squeezeblocks(
out: &mut[GenMatrixBuf; 4],
nblocks: usize,
state: &mut Keccakx4State
)
{
keccakx4_squeezeblocks128(
out,
nblocks,
SHAKE128_RATE,
&mut state.s
);
pub unsafe fn shake128x4_squeezeblocks(out: &mut [GenMatrixBuf; 4], nblocks: usize, state: &mut Keccakx4State) {
keccakx4_squeezeblocks128(out, nblocks, SHAKE128_RATE, &mut state.s);
}
pub unsafe fn shake256x4_absorb_once(
state: &mut Keccakx4State,
in0: &[u8],
in1: &[u8],
in2: &[u8],
in3: &[u8],
inlen: usize,
)
{
keccakx4_absorb_once(
&mut state.s,
SHAKE256_RATE,
in0, in1, in2, in3, inlen,
0x1F
)
pub unsafe fn shake256x4_absorb_once(state: &mut Keccakx4State, in0: &[u8], in1: &[u8], in2: &[u8], in3: &[u8], inlen: usize) {
keccakx4_absorb_once(&mut state.s, SHAKE256_RATE, in0, in1, in2, in3, inlen, 0x1F)
}
pub unsafe fn shake256x4_squeezeblocks(
out: &mut[Eta4xBuf; 4],
nblocks: usize,
state: &mut Keccakx4State
)
{
keccakx4_squeezeblocks256(
out,
nblocks,
SHAKE256_RATE,
&mut state.s
);
pub unsafe fn shake256x4_squeezeblocks(out: &mut [Eta4xBuf; 4], nblocks: usize, state: &mut Keccakx4State) {
keccakx4_squeezeblocks256(out, nblocks, SHAKE256_RATE, &mut state.s);
}
// pub unsafe fn shake128x4(
@ -234,7 +170,7 @@ pub unsafe fn shake256x4_squeezeblocks(
// shake128x4_absorb_once(&mut state, in0, in1, in2, in3, inlen);
// shake128x4_squeezeblocks(out, nblocks, &mut state);
// let idx = nblocks*SHAKE128_RATE;
// let idx = nblocks*SHAKE128_RATE;
// outlen -= idx;
// if outlen > 0 {
@ -277,4 +213,4 @@ pub unsafe fn shake256x4_squeezeblocks(
// out[3].coeffs[idx+i] = t[3][i];
// }
// }
// }
// }

1031
third_party/kyber/src/avx2/indcpa.rs vendored
View file

File diff suppressed because it is too large Load diff

523
third_party/kyber/src/avx2/keccak4x.rs vendored
View file

@ -1,4 +1,4 @@
// Macro and function code structure is the work of Marek Kotewicz
// Macro and function code structure is the work of Marek Kotewicz
// plus contributors to the tiny-keccak crate licensed under
// Creative Commons CC0 1.0 Universal. Thankyou.
// https://github.com/debris/tiny-keccak
@ -16,205 +16,392 @@ use core::arch::x86_64::*;
#[repr(C, align(32))]
union RC_Data {
vecs: [__m256i; 24],
u: [u64; 96]
vecs: [__m256i; 24],
u: [u64; 96],
}
#[repr(C, align(32))]
union Temp {
pub vec: __m256i,
pub u: [u64; 4]
pub vec: __m256i,
pub u: [u64; 4],
}
const RHO: [u32; 24] = [
1, 3, 6, 10, 15, 21, 28, 36,
45, 55, 2, 14, 27, 41, 56, 8,
25, 43, 62, 18, 39, 61, 20, 44,
1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44,
];
const PI: [usize; 24] = [
10, 7, 11, 17, 18, 3, 5, 16,
8, 21, 24, 4, 15, 23, 19, 13,
12, 2, 20, 14, 22, 9, 6, 1,
10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1,
];
// Set __mm256i constants with a union
const RC_X4: RC_Data = RC_Data { u: [
0x0000000000000001, 0x0000000000000001, 0x0000000000000001, 0x0000000000000001,
0x0000000000008082, 0x0000000000008082, 0x0000000000008082, 0x0000000000008082,
0x800000000000808a, 0x800000000000808a, 0x800000000000808a, 0x800000000000808a,
0x8000000080008000, 0x8000000080008000, 0x8000000080008000, 0x8000000080008000,
0x000000000000808b, 0x000000000000808b, 0x000000000000808b, 0x000000000000808b,
0x0000000080000001, 0x0000000080000001, 0x0000000080000001, 0x0000000080000001,
0x8000000080008081, 0x8000000080008081, 0x8000000080008081, 0x8000000080008081,
0x8000000000008009, 0x8000000000008009, 0x8000000000008009, 0x8000000000008009,
0x000000000000008a, 0x000000000000008a, 0x000000000000008a, 0x000000000000008a,
0x0000000000000088, 0x0000000000000088, 0x0000000000000088, 0x0000000000000088,
0x0000000080008009, 0x0000000080008009, 0x0000000080008009, 0x0000000080008009,
0x000000008000000a, 0x000000008000000a, 0x000000008000000a, 0x000000008000000a,
0x000000008000808b, 0x000000008000808b, 0x000000008000808b, 0x000000008000808b,
0x800000000000008b, 0x800000000000008b, 0x800000000000008b, 0x800000000000008b,
0x8000000000008089, 0x8000000000008089, 0x8000000000008089, 0x8000000000008089,
0x8000000000008003, 0x8000000000008003, 0x8000000000008003, 0x8000000000008003,
0x8000000000008002, 0x8000000000008002, 0x8000000000008002, 0x8000000000008002,
0x8000000000000080, 0x8000000000000080, 0x8000000000000080, 0x8000000000000080,
0x000000000000800a, 0x000000000000800a, 0x000000000000800a, 0x000000000000800a,
0x800000008000000a, 0x800000008000000a, 0x800000008000000a, 0x800000008000000a,
0x8000000080008081, 0x8000000080008081, 0x8000000080008081, 0x8000000080008081,
0x8000000000008080, 0x8000000000008080, 0x8000000000008080, 0x8000000000008080,
0x0000000080000001, 0x0000000080000001, 0x0000000080000001, 0x0000000080000001,
0x8000000080008008, 0x8000000080008008, 0x8000000080008008, 0x8000000080008008
]};
const RC_X4: RC_Data = RC_Data {
u: [
0x0000000000000001,
0x0000000000000001,
0x0000000000000001,
0x0000000000000001,
0x0000000000008082,
0x0000000000008082,
0x0000000000008082,
0x0000000000008082,
0x800000000000808a,
0x800000000000808a,
0x800000000000808a,
0x800000000000808a,
0x8000000080008000,
0x8000000080008000,
0x8000000080008000,
0x8000000080008000,
0x000000000000808b,
0x000000000000808b,
0x000000000000808b,
0x000000000000808b,
0x0000000080000001,
0x0000000080000001,
0x0000000080000001,
0x0000000080000001,
0x8000000080008081,
0x8000000080008081,
0x8000000080008081,
0x8000000080008081,
0x8000000000008009,
0x8000000000008009,
0x8000000000008009,
0x8000000000008009,
0x000000000000008a,
0x000000000000008a,
0x000000000000008a,
0x000000000000008a,
0x0000000000000088,
0x0000000000000088,
0x0000000000000088,
0x0000000000000088,
0x0000000080008009,
0x0000000080008009,
0x0000000080008009,
0x0000000080008009,
0x000000008000000a,
0x000000008000000a,
0x000000008000000a,
0x000000008000000a,
0x000000008000808b,
0x000000008000808b,
0x000000008000808b,
0x000000008000808b,
0x800000000000008b,
0x800000000000008b,
0x800000000000008b,
0x800000000000008b,
0x8000000000008089,
0x8000000000008089,
0x8000000000008089,
0x8000000000008089,
0x8000000000008003,
0x8000000000008003,
0x8000000000008003,
0x8000000000008003,
0x8000000000008002,
0x8000000000008002,
0x8000000000008002,
0x8000000000008002,
0x8000000000000080,
0x8000000000000080,
0x8000000000000080,
0x8000000000000080,
0x000000000000800a,
0x000000000000800a,
0x000000000000800a,
0x000000000000800a,
0x800000008000000a,
0x800000008000000a,
0x800000008000000a,
0x800000008000000a,
0x8000000080008081,
0x8000000080008081,
0x8000000080008081,
0x8000000080008081,
0x8000000000008080,
0x8000000000008080,
0x8000000000008080,
0x8000000000008080,
0x0000000080000001,
0x0000000080000001,
0x0000000080000001,
0x0000000080000001,
0x8000000080008008,
0x8000000080008008,
0x8000000080008008,
0x8000000080008008,
],
};
macro_rules! unroll5 {
($var:ident, $body:block) => {
{ const $var: usize = 0; $body; }
{ const $var: usize = 1; $body; }
{ const $var: usize = 2; $body; }
{ const $var: usize = 3; $body; }
{ const $var: usize = 4; $body; }
};
($var:ident, $body:block) => {{
const $var: usize = 0;
$body;
}
{
const $var: usize = 1;
$body;
}
{
const $var: usize = 2;
$body;
}
{
const $var: usize = 3;
$body;
}
{
const $var: usize = 4;
$body;
}};
}
macro_rules! unroll24 {
($var: ident, $body: block) => {
{ const $var: usize = 0; $body; }
{ const $var: usize = 1; $body; }
{ const $var: usize = 2; $body; }
{ const $var: usize = 3; $body; }
{ const $var: usize = 4; $body; }
{ const $var: usize = 5; $body; }
{ const $var: usize = 6; $body; }
{ const $var: usize = 7; $body; }
{ const $var: usize = 8; $body; }
{ const $var: usize = 9; $body; }
{ const $var: usize = 10; $body; }
{ const $var: usize = 11; $body; }
{ const $var: usize = 12; $body; }
{ const $var: usize = 13; $body; }
{ const $var: usize = 14; $body; }
{ const $var: usize = 15; $body; }
{ const $var: usize = 16; $body; }
{ const $var: usize = 17; $body; }
{ const $var: usize = 18; $body; }
{ const $var: usize = 19; $body; }
{ const $var: usize = 20; $body; }
{ const $var: usize = 21; $body; }
{ const $var: usize = 22; $body; }
{ const $var: usize = 23; $body; }
};
($var: ident, $body: block) => {{
const $var: usize = 0;
$body;
}
{
const $var: usize = 1;
$body;
}
{
const $var: usize = 2;
$body;
}
{
const $var: usize = 3;
$body;
}
{
const $var: usize = 4;
$body;
}
{
const $var: usize = 5;
$body;
}
{
const $var: usize = 6;
$body;
}
{
const $var: usize = 7;
$body;
}
{
const $var: usize = 8;
$body;
}
{
const $var: usize = 9;
$body;
}
{
const $var: usize = 10;
$body;
}
{
const $var: usize = 11;
$body;
}
{
const $var: usize = 12;
$body;
}
{
const $var: usize = 13;
$body;
}
{
const $var: usize = 14;
$body;
}
{
const $var: usize = 15;
$body;
}
{
const $var: usize = 16;
$body;
}
{
const $var: usize = 17;
$body;
}
{
const $var: usize = 18;
$body;
}
{
const $var: usize = 19;
$body;
}
{
const $var: usize = 20;
$body;
}
{
const $var: usize = 21;
$body;
}
{
const $var: usize = 22;
$body;
}
{
const $var: usize = 23;
$body;
}};
}
#[allow(unused_assignments, non_upper_case_globals)]
pub fn f1600_x4(a: &mut [__m256i]) {
unsafe {
for i in 0..24 {
let mut array = [_mm256_setzero_si256(); 5];
// Theta
unroll5!(x, {
unroll5!(y, {
array[x] = _mm256_xor_si256(array[x], a[5 * y + x]);
});
});
unroll5!(x, {
unroll5!(y, {
let t1 = array[(x + 4) % 5];
let mut t2 = Temp {vec: array[(x + 1) % 5]};
for i in 0..4 {
t2.u[i] = t2.u[i].rotate_left(1);
}
a[5 * y + x] = _mm256_xor_si256(a[5 * y + x], _mm256_xor_si256(t1, t2.vec));
});
});
unsafe {
for i in 0..24 {
let mut array = [_mm256_setzero_si256(); 5];
// Rho and pi
let mut last = a[1];
unroll24!(x, {
array[0] = a[PI[x]];
let mut temp_last = Temp {vec: last};
for i in 0..4 {
temp_last.u[i] = temp_last.u[i].rotate_left(RHO[x]);
// Theta
unroll5!(x, {
unroll5!(y, {
array[x] = _mm256_xor_si256(array[x], a[5 * y + x]);
});
});
unroll5!(x, {
unroll5!(y, {
let t1 = array[(x + 4) % 5];
let mut t2 = Temp { vec: array[(x + 1) % 5] };
for i in 0..4 {
t2.u[i] = t2.u[i].rotate_left(1);
}
a[5 * y + x] = _mm256_xor_si256(a[5 * y + x], _mm256_xor_si256(t1, t2.vec));
});
});
// Rho and pi
let mut last = a[1];
unroll24!(x, {
array[0] = a[PI[x]];
let mut temp_last = Temp { vec: last };
for i in 0..4 {
temp_last.u[i] = temp_last.u[i].rotate_left(RHO[x]);
}
a[PI[x]] = temp_last.vec;
last = array[0];
});
// Chi
unroll5!(y_step, {
let y = 5 * y_step;
unroll5!(x, {
array[x] = a[y + x];
});
unroll5!(x, {
let t1 = array[(x + 1) % 5];
let t2 = array[(x + 2) % 5];
let tmp = _mm256_xor_si256(array[x], _mm256_andnot_si256(t1, t2));
a[y + x] = tmp;
});
});
a[0] = _mm256_xor_si256(a[0], RC_X4.vecs[i]);
}
a[PI[x]] = temp_last.vec;
last = array[0];
});
// Chi
unroll5!(y_step, {
let y = 5 * y_step;
unroll5!(x, {
array[x] = a[y + x];
});
unroll5!(x, {
let t1 = array[(x + 1) % 5];
let t2 = array[(x + 2) % 5];
let tmp = _mm256_xor_si256(array[x], _mm256_andnot_si256(t1, t2));
a[y+x] = tmp;
});
});
a[0] = _mm256_xor_si256(a[0], RC_X4.vecs[i]);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
const PLEN: usize = 25;
// Test vectors from XKCP
// https://github.com/XKCP/XKCP/blob/master/tests/TestVectors/KeccakF-1600-IntermediateValues.txt
use super::*;
const PLEN: usize = 25;
// Test vectors from XKCP
// https://github.com/XKCP/XKCP/blob/master/tests/TestVectors/KeccakF-1600-IntermediateValues.txt
#[test]
fn known_vectors() {
let vec1: [u64; 25] = [
0xF1258F7940E1DDE7, 0x84D5CCF933C0478A, 0xD598261EA65AA9EE, 0xBD1547306F80494D,
0x8B284E056253D057, 0xFF97A42D7F8E6FD4, 0x90FEE5A0A44647C4, 0x8C5BDA0CD6192E76,
0xAD30A6F71B19059C, 0x30935AB7D08FFC64, 0xEB5AA93F2317D635, 0xA9A6E6260D712103,
0x81A57C16DBCF555F, 0x43B831CD0347C826, 0x01F22F1A11A5569F, 0x05E5635A21D9AE61,
0x64BEFEF28CC970F2, 0x613670957BC46611, 0xB87C5A554FD00ECB, 0x8C3EE88A1CCF32C8,
0x940C7922AE3A2614, 0x1841F924A2C509E4, 0x16F53526E70465C2, 0x75F644E97F30A13B,
0xEAF1FF7B5CECA249
];
let vec2: [u64; 25] = [
0x2D5C954DF96ECB3C, 0x6A332CD07057B56D, 0x093D8D1270D76B6C, 0x8A20D9B25569D094,
0x4F9C4F99E5E7F156, 0xF957B9A2DA65FB38, 0x85773DAE1275AF0D, 0xFAF4F247C3D810F7,
0x1F1B9EE6F79A8759, 0xE4FECC0FEE98B425, 0x68CE61B6B9CE68A1, 0xDEEA66C4BA8F974F,
0x33C43D836EAFB1F5, 0xE00654042719DBD9, 0x7CF8A9F009831265, 0xFD5449A6BF174743,
0x97DDAD33D8994B40, 0x48EAD5FC5D0BE774, 0xE3B8C8EE55B7B03C, 0x91A0226E649E42E9,
0x900E3129E7BADD7B, 0x202A9EC5FAA3CCE8, 0x5B3402464E1C3DB6, 0x609F4E62A44C1059,
0x20D06CD26A8FBF5C
];
fn known_vectors() {
let vec1: [u64; 25] = [
0xF1258F7940E1DDE7,
0x84D5CCF933C0478A,
0xD598261EA65AA9EE,
0xBD1547306F80494D,
0x8B284E056253D057,
0xFF97A42D7F8E6FD4,
0x90FEE5A0A44647C4,
0x8C5BDA0CD6192E76,
0xAD30A6F71B19059C,
0x30935AB7D08FFC64,
0xEB5AA93F2317D635,
0xA9A6E6260D712103,
0x81A57C16DBCF555F,
0x43B831CD0347C826,
0x01F22F1A11A5569F,
0x05E5635A21D9AE61,
0x64BEFEF28CC970F2,
0x613670957BC46611,
0xB87C5A554FD00ECB,
0x8C3EE88A1CCF32C8,
0x940C7922AE3A2614,
0x1841F924A2C509E4,
0x16F53526E70465C2,
0x75F644E97F30A13B,
0xEAF1FF7B5CECA249,
];
// repeat values to check all lanes
let tvec1 = expand(vec1);
let tvec2 = expand(vec2);
let vec2: [u64; 25] = [
0x2D5C954DF96ECB3C,
0x6A332CD07057B56D,
0x093D8D1270D76B6C,
0x8A20D9B25569D094,
0x4F9C4F99E5E7F156,
0xF957B9A2DA65FB38,
0x85773DAE1275AF0D,
0xFAF4F247C3D810F7,
0x1F1B9EE6F79A8759,
0xE4FECC0FEE98B425,
0x68CE61B6B9CE68A1,
0xDEEA66C4BA8F974F,
0x33C43D836EAFB1F5,
0xE00654042719DBD9,
0x7CF8A9F009831265,
0xFD5449A6BF174743,
0x97DDAD33D8994B40,
0x48EAD5FC5D0BE774,
0xE3B8C8EE55B7B03C,
0x91A0226E649E42E9,
0x900E3129E7BADD7B,
0x202A9EC5FAA3CCE8,
0x5B3402464E1C3DB6,
0x609F4E62A44C1059,
0x20D06CD26A8FBF5C,
];
unsafe {
let mut data = Data { u: [0u64;100] };
f1600_x4(&mut data.lanes);
assert_eq!(&data.u , &tvec1);
f1600_x4(&mut data.lanes);
assert_eq!(data.u, tvec2);
// repeat values to check all lanes
let tvec1 = expand(vec1);
let tvec2 = expand(vec2);
unsafe {
let mut data = Data { u: [0u64; 100] };
f1600_x4(&mut data.lanes);
assert_eq!(&data.u, &tvec1);
f1600_x4(&mut data.lanes);
assert_eq!(data.u, tvec2);
}
}
}
#[repr(C)]
pub union Data {
pub lanes: [__m256i; PLEN],
pub u: [u64; PLEN * 4]
}
// [0,1...] expands to [0,0,0,0,1,1,1,1...]
fn expand(vec: [u64; PLEN]) -> [u64; 100] {
let mut out = [0u64; 100];
for (i,u) in vec.iter().enumerate() {
out[i*4..][..4].copy_from_slice(&[*u; 4]);
#[repr(C)]
pub union Data {
pub lanes: [__m256i; PLEN],
pub u: [u64; PLEN * 4],
}
// [0,1...] expands to [0,0,0,0,1,1,1,1...]
fn expand(vec: [u64; PLEN]) -> [u64; 100] {
let mut out = [0u64; 100];
for (i, u) in vec.iter().enumerate() {
out[i * 4..][..4].copy_from_slice(&[*u; 4]);
}
out
}
out
}
}

3
third_party/kyber/src/avx2/mod.rs vendored
View file

@ -1,4 +1,3 @@
pub mod aes256ctr;
pub mod align;
pub mod cbd;
@ -10,4 +9,4 @@ pub mod keccak4x;
pub mod poly;
pub mod polyvec;
pub mod rejsample;
pub mod verify;
pub mod verify;

733
third_party/kyber/src/avx2/poly.rs vendored
View file

@ -1,417 +1,388 @@
#![allow(unused_imports)]
use crate::{align::*, cbd::*, consts::*, fips202::*, fips202x4::*, params::*, symmetric::*};
use core::arch::x86_64::*;
use crate::{
align::*,
cbd::*,
consts::*,
fips202::*,
fips202x4::*,
params::*,
symmetric::*,
};
pub(crate) const NOISE_NBLOCKS: usize =
(KYBER_ETA1*KYBER_N/4+SHAKE256_RATE-1)/SHAKE256_RATE;
pub(crate) const NOISE_NBLOCKS: usize = (KYBER_ETA1 * KYBER_N / 4 + SHAKE256_RATE - 1) / SHAKE256_RATE;
#[derive(Clone)]
#[repr(C)]
pub union Poly {
pub coeffs: [i16; KYBER_N],
pub vec: [__m256i; (KYBER_N+15)/16]
pub vec: [__m256i; (KYBER_N + 15) / 16],
}
impl Copy for Poly {}
impl Poly {
pub fn new() -> Self {
Poly {
coeffs: [0i16; KYBER_N]
pub fn new() -> Self {
Poly { coeffs: [0i16; KYBER_N] }
}
}
// Basic polynomial value checking for development
// #[cfg(debug_assertions)]
// fn checksum(&self) -> i16 {
// unsafe{
// let mut out = 0;
// for x in &self.coeffs {
// out ^= x;
// }
// out
// }
// }
// Basic polynomial value checking for development
// #[cfg(debug_assertions)]
// fn checksum(&self) -> i16 {
// unsafe{
// let mut out = 0;
// for x in &self.coeffs {
// out ^= x;
// }
// out
// }
// }
}
extern {
fn ntt_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn invntt_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn nttunpack_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn basemul_avx(
r: &mut[i16; KYBER_N],
a: &[i16; KYBER_N],
b: &[i16; KYBER_N],
q_data: &[i16; 640]
);
fn tomont_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn reduce_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn ntttobytes_avx(r: *mut u8 , a: &[i16; KYBER_N] , q_data: &[i16; 640]);
fn nttfrombytes_avx(r: *mut i16, a: *const u8, q_data: &[i16; 640]);
extern "C" {
fn ntt_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn invntt_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn nttunpack_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn basemul_avx(r: &mut [i16; KYBER_N], a: &[i16; KYBER_N], b: &[i16; KYBER_N], q_data: &[i16; 640]);
fn tomont_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn reduce_avx(r: &mut [i16; KYBER_N], q_data: &[i16; 640]);
fn ntttobytes_avx(r: *mut u8, a: &[i16; KYBER_N], q_data: &[i16; 640]);
fn nttfrombytes_avx(r: *mut i16, a: *const u8, q_data: &[i16; 640]);
}
// #[target_feature(enable = "avx2")]
#[cfg(any(feature="kyber512", not(feature="kyber1024")))]
pub unsafe fn poly_compress(r: &mut[u8], a: Poly)
{
let (mut f0, mut f1, mut f2, mut f3);
let v: __m256i = _mm256_load_si256(QDATA.vec[_16XV/16..].as_ptr());
let shift1: __m256i = _mm256_set1_epi16(1 << 9);
let mask: __m256i = _mm256_set1_epi16(15);
let shift2: __m256i = _mm256_set1_epi16((16 << 8) + 1);
let permdidx: __m256i = _mm256_set_epi32(7,3,6,2,5,1,4,0);
#[cfg(any(feature = "kyber512", not(feature = "kyber1024")))]
pub unsafe fn poly_compress(r: &mut [u8], a: Poly) {
let (mut f0, mut f1, mut f2, mut f3);
let v: __m256i = _mm256_load_si256(QDATA.vec[_16XV / 16..].as_ptr());
let shift1: __m256i = _mm256_set1_epi16(1 << 9);
let mask: __m256i = _mm256_set1_epi16(15);
let shift2: __m256i = _mm256_set1_epi16((16 << 8) + 1);
let permdidx: __m256i = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
for i in 0..KYBER_N/64 {
f0 = _mm256_load_si256(&a.vec[4*i+0]);
f1 = _mm256_load_si256(&a.vec[4*i+1]);
f2 = _mm256_load_si256(&a.vec[4*i+2]);
f3 = _mm256_load_si256(&a.vec[4*i+3]);
f0 = _mm256_mulhi_epi16(f0,v);
f1 = _mm256_mulhi_epi16(f1,v);
f2 = _mm256_mulhi_epi16(f2,v);
f3 = _mm256_mulhi_epi16(f3,v);
f0 = _mm256_mulhrs_epi16(f0,shift1);
f1 = _mm256_mulhrs_epi16(f1,shift1);
f2 = _mm256_mulhrs_epi16(f2,shift1);
f3 = _mm256_mulhrs_epi16(f3,shift1);
f0 = _mm256_and_si256(f0,mask);
f1 = _mm256_and_si256(f1,mask);
f2 = _mm256_and_si256(f2,mask);
f3 = _mm256_and_si256(f3,mask);
f0 = _mm256_packus_epi16(f0,f1);
f2 = _mm256_packus_epi16(f2,f3);
f0 = _mm256_maddubs_epi16(f0,shift2);
f2 = _mm256_maddubs_epi16(f2,shift2);
f0 = _mm256_packus_epi16(f0,f2);
f0 = _mm256_permutevar8x32_epi32(f0,permdidx);
_mm256_storeu_si256(r[32*i..].as_mut_ptr() as *mut __m256i,f0);
}
}
// #[target_feature(enable = "avx2")]
#[cfg(any(feature="kyber512", not(feature="kyber1024")))]
pub unsafe fn poly_decompress(r: &mut Poly, a: &[u8])
{
let (mut t, mut f);
let q: __m256i = _mm256_load_si256(QDATA.vec[_16XQ/16..].as_ptr());
let shufbidx: __m256i = _mm256_set_epi8(
7,7,7,7,6,6,6,6,5,5,5,5,4,4,4,4,
3,3,3,3,2,2,2,2,1,1,1,1,0,0,0,0
);
let mask: __m256i = _mm256_set1_epi32(0x00F0000F);
let shift: __m256i = _mm256_set1_epi32((128 << 16) + 2048);
for i in 0..KYBER_N/16 {
t = _mm_loadl_epi64(a[8*i..].as_ptr() as *const __m128i);
f = _mm256_broadcastsi128_si256(t);
f = _mm256_shuffle_epi8(f,shufbidx);
f = _mm256_and_si256(f,mask);
f = _mm256_mullo_epi16(f,shift);
f = _mm256_mulhrs_epi16(f,q);
_mm256_store_si256(&mut r.vec[i],f);
}
}
// #[target_feature(enable = "avx2")]
#[cfg(feature="kyber1024")]
pub unsafe fn poly_compress(r: &mut[u8], a: Poly)
{
let (mut f0, mut f1);
let (mut t0, mut t1);
let mut tmp;
let v: __m256i = _mm256_load_si256(&QDATA.vec[_16XV/16]);
let shift1: __m256i = _mm256_set1_epi16(1 << 10);
let mask: __m256i = _mm256_set1_epi16(31);
let shift2: __m256i = _mm256_set1_epi16((32 << 8) + 1);
let shift3: __m256i = _mm256_set1_epi32((1024 << 16) + 1);
let sllvdidx: __m256i = _mm256_set1_epi64x(12);
let shufbidx: __m256i = _mm256_set_epi8(
8,-1,-1,-1,-1,-1, 4, 3, 2, 1, 0,-1,12,11,10, 9,
-1,12,11,10, 9, 8,-1,-1,-1,-1,-1 ,4, 3, 2, 1, 0
);
for i in 0..(KYBER_N/32) {
f0 = _mm256_load_si256(&a.vec[2*i+0]);
f1 = _mm256_load_si256(&a.vec[2*i+1]);
f0 = _mm256_mulhi_epi16(f0,v);
f1 = _mm256_mulhi_epi16(f1,v);
f0 = _mm256_mulhrs_epi16(f0,shift1);
f1 = _mm256_mulhrs_epi16(f1,shift1);
f0 = _mm256_and_si256(f0,mask);
f1 = _mm256_and_si256(f1,mask);
f0 = _mm256_packus_epi16(f0,f1);
f0 = _mm256_maddubs_epi16(f0,shift2);
f0 = _mm256_madd_epi16(f0,shift3);
f0 = _mm256_sllv_epi32(f0,sllvdidx);
f0 = _mm256_srlv_epi64(f0,sllvdidx);
f0 = _mm256_shuffle_epi8(f0,shufbidx);
t0 = _mm256_castsi256_si128(f0);
t1 = _mm256_extracti128_si256(f0,1);
t0 = _mm_blendv_epi8(t0,t1,_mm256_castsi256_si128(shufbidx));
_mm_storeu_si128(r[20*i+ 0..].as_mut_ptr() as *mut __m128i,t0);
tmp = _mm_cvtsi128_si32(t1);
r[20*i+16..20*i+20].copy_from_slice(&tmp.to_le_bytes());
}
}
// #[target_feature(enable = "avx2")]
#[cfg(feature="kyber1024")]
pub unsafe fn poly_decompress(r: &mut Poly, a: &[u8])
{
let (mut t, mut f, mut ti);
let q = _mm256_load_si256(&QDATA.vec[_16XQ/16]);
let shufbidx = _mm256_set_epi8(
9,9,9,8,8,8,8,7,7,6,6,6,6,5,5,5,
4,4,4,3,3,3,3,2,2,1,1,1,1,0,0,0
);
let mask = _mm256_set_epi16(
248,1984,62,496,3968,124,992,31,
248,1984,62,496,3968,124,992,31
);
let shift = _mm256_set_epi16(
128,16,512,64,8,256,32,1024,
128,16,512,64,8,256,32,1024
);
for i in 0..KYBER_N/16 {
t = _mm_loadl_epi64(a[10*i+0..].as_ptr() as *const __m128i);
ti = i32::from_le_bytes([a[10*i+8], a[10*i+9], 0, 0]);
t = _mm_insert_epi16(t, ti, 4);
f = _mm256_broadcastsi128_si256(t);
f = _mm256_shuffle_epi8(f,shufbidx);
f = _mm256_and_si256(f,mask);
f = _mm256_mullo_epi16(f,shift);
f = _mm256_mulhrs_epi16(f,q);
_mm256_store_si256(r.vec[i..].as_mut_ptr() as *mut __m256i,f);
}
}
pub fn poly_frombytes(r: &mut Poly, a: &[u8])
{
unsafe {
nttfrombytes_avx(r.coeffs.as_mut_ptr(), a.as_ptr(), &QDATA.coeffs);
}
}
pub fn poly_tobytes(r: &mut[u8], a: Poly)
{
let mut buf = [0u8; KYBER_POLYBYTES];
unsafe { ntttobytes_avx(buf.as_mut_ptr(), &a.coeffs, &QDATA.coeffs); }
r[..KYBER_POLYBYTES].copy_from_slice(&buf[..]);
}
// #[target_feature(enable = "avx2")]
pub unsafe fn poly_frommsg(r: &mut Poly, msg: &[u8])
{
let shift = _mm256_broadcastsi128_si256(_mm_set_epi32(0,1,2,3));
let idx = _mm256_broadcastsi128_si256(
_mm_set_epi8(15,14,11,10,7,6,3,2,13,12,9,8,5,4,1,0)
);
let hqs: __m256i = _mm256_set1_epi16((KYBER_Q+1) as i16/2);
let f = _mm256_loadu_si256(msg.as_ptr() as *const __m256i);
let mut frommsg64 = |i: usize, mut g3: __m256i| {
g3 = _mm256_sllv_epi32(g3,shift);
g3 = _mm256_shuffle_epi8(g3,idx);
let mut g0 = _mm256_slli_epi16(g3,12);
let mut g1 = _mm256_slli_epi16(g3,8);
let mut g2 = _mm256_slli_epi16(g3,4);
g0 = _mm256_srai_epi16(g0,15);
g1 = _mm256_srai_epi16(g1,15);
g2 = _mm256_srai_epi16(g2,15);
g3 = _mm256_srai_epi16(g3,15);
g0 = _mm256_and_si256(g0,hqs); // 19 18 17 16 3 2 1 0
g1 = _mm256_and_si256(g1,hqs); // 23 22 21 20 7 6 5 4
g2 = _mm256_and_si256(g2,hqs); // 27 26 25 24 11 10 9 8
g3 = _mm256_and_si256(g3,hqs); // 31 30 29 28 15 14 13 12
let h0 = _mm256_unpacklo_epi64(g0,g1);
let h2 = _mm256_unpackhi_epi64(g0,g1);
let h1 = _mm256_unpacklo_epi64(g2,g3);
let h3 = _mm256_unpackhi_epi64(g2,g3);
g0 = _mm256_permute2x128_si256(h0,h1,0x20);
g2 = _mm256_permute2x128_si256(h0,h1,0x31);
g1 = _mm256_permute2x128_si256(h2,h3,0x20);
g3 = _mm256_permute2x128_si256(h2,h3,0x31);
_mm256_store_si256(&mut r.vec[0+2*i+0],g0);
_mm256_store_si256(&mut r.vec[0+2*i+1],g1);
_mm256_store_si256(&mut r.vec[8+2*i+0],g2);
_mm256_store_si256(&mut r.vec[8+2*i+1],g3);
};
frommsg64(0, _mm256_shuffle_epi32(f, 0));
frommsg64(1, _mm256_shuffle_epi32(f, 85));
frommsg64(2, _mm256_shuffle_epi32(f, 170));
frommsg64(3, _mm256_shuffle_epi32(f, 255));
}
// #[target_feature(enable = "avx2")]
pub fn poly_tomsg(msg: &mut[u8], a: Poly)
{
unsafe {
let (mut f0, mut f1, mut g0, mut g1);
let hq: __m256i = _mm256_set1_epi16((KYBER_Q - 1) as i16/2);
let hhq: __m256i = _mm256_set1_epi16((KYBER_Q - 1) as i16/4);
for i in 0..KYBER_N/32 {
f0 = _mm256_load_si256(&a.vec[2*i+0]);
f1 = _mm256_load_si256(&a.vec[2*i+1]);
f0 = _mm256_sub_epi16(hq, f0);
f1 = _mm256_sub_epi16(hq, f1);
g0 = _mm256_srai_epi16(f0, 15);
g1 = _mm256_srai_epi16(f1, 15);
f0 = _mm256_xor_si256(f0, g0);
f1 = _mm256_xor_si256(f1, g1);
f0 = _mm256_sub_epi16(f0, hhq);
f1 = _mm256_sub_epi16(f1, hhq);
f0 = _mm256_packs_epi16(f0, f1);
f0 = _mm256_permute4x64_epi64(f0, 0xD8);
let small = _mm256_movemask_epi8(f0);
msg[4*i..][..4].copy_from_slice(&small.to_ne_bytes());
for i in 0..KYBER_N / 64 {
f0 = _mm256_load_si256(&a.vec[4 * i + 0]);
f1 = _mm256_load_si256(&a.vec[4 * i + 1]);
f2 = _mm256_load_si256(&a.vec[4 * i + 2]);
f3 = _mm256_load_si256(&a.vec[4 * i + 3]);
f0 = _mm256_mulhi_epi16(f0, v);
f1 = _mm256_mulhi_epi16(f1, v);
f2 = _mm256_mulhi_epi16(f2, v);
f3 = _mm256_mulhi_epi16(f3, v);
f0 = _mm256_mulhrs_epi16(f0, shift1);
f1 = _mm256_mulhrs_epi16(f1, shift1);
f2 = _mm256_mulhrs_epi16(f2, shift1);
f3 = _mm256_mulhrs_epi16(f3, shift1);
f0 = _mm256_and_si256(f0, mask);
f1 = _mm256_and_si256(f1, mask);
f2 = _mm256_and_si256(f2, mask);
f3 = _mm256_and_si256(f3, mask);
f0 = _mm256_packus_epi16(f0, f1);
f2 = _mm256_packus_epi16(f2, f3);
f0 = _mm256_maddubs_epi16(f0, shift2);
f2 = _mm256_maddubs_epi16(f2, shift2);
f0 = _mm256_packus_epi16(f0, f2);
f0 = _mm256_permutevar8x32_epi32(f0, permdidx);
_mm256_storeu_si256(r[32 * i..].as_mut_ptr() as *mut __m256i, f0);
}
}
// #[target_feature(enable = "avx2")]
#[cfg(any(feature = "kyber512", not(feature = "kyber1024")))]
pub unsafe fn poly_decompress(r: &mut Poly, a: &[u8]) {
let (mut t, mut f);
let q: __m256i = _mm256_load_si256(QDATA.vec[_16XQ / 16..].as_ptr());
let shufbidx: __m256i = _mm256_set_epi8(
7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0,
);
let mask: __m256i = _mm256_set1_epi32(0x00F0000F);
let shift: __m256i = _mm256_set1_epi32((128 << 16) + 2048);
for i in 0..KYBER_N / 16 {
t = _mm_loadl_epi64(a[8 * i..].as_ptr() as *const __m128i);
f = _mm256_broadcastsi128_si256(t);
f = _mm256_shuffle_epi8(f, shufbidx);
f = _mm256_and_si256(f, mask);
f = _mm256_mullo_epi16(f, shift);
f = _mm256_mulhrs_epi16(f, q);
_mm256_store_si256(&mut r.vec[i], f);
}
}
}
#[cfg(all(any(feature="kyber1024", feature="kyber512"), not(feature="90s")))]
pub fn poly_getnoise_eta2(r: &mut Poly, seed: &[u8], nonce: u8)
{
let mut buf = Eta2Buf::new();
unsafe {
prf(&mut buf.coeffs, KYBER_ETA2*KYBER_N/4, seed, nonce);
poly_cbd_eta2(r, &buf.vec);
}
// #[target_feature(enable = "avx2")]
#[cfg(feature = "kyber1024")]
pub unsafe fn poly_compress(r: &mut [u8], a: Poly) {
let (mut f0, mut f1);
let (mut t0, mut t1);
let mut tmp;
let v: __m256i = _mm256_load_si256(&QDATA.vec[_16XV / 16]);
let shift1: __m256i = _mm256_set1_epi16(1 << 10);
let mask: __m256i = _mm256_set1_epi16(31);
let shift2: __m256i = _mm256_set1_epi16((32 << 8) + 1);
let shift3: __m256i = _mm256_set1_epi32((1024 << 16) + 1);
let sllvdidx: __m256i = _mm256_set1_epi64x(12);
let shufbidx: __m256i = _mm256_set_epi8(
8, -1, -1, -1, -1, -1, 4, 3, 2, 1, 0, -1, 12, 11, 10, 9, -1, 12, 11, 10, 9, 8, -1, -1, -1, -1, -1, 4, 3, 2, 1, 0,
);
for i in 0..(KYBER_N / 32) {
f0 = _mm256_load_si256(&a.vec[2 * i + 0]);
f1 = _mm256_load_si256(&a.vec[2 * i + 1]);
f0 = _mm256_mulhi_epi16(f0, v);
f1 = _mm256_mulhi_epi16(f1, v);
f0 = _mm256_mulhrs_epi16(f0, shift1);
f1 = _mm256_mulhrs_epi16(f1, shift1);
f0 = _mm256_and_si256(f0, mask);
f1 = _mm256_and_si256(f1, mask);
f0 = _mm256_packus_epi16(f0, f1);
f0 = _mm256_maddubs_epi16(f0, shift2);
f0 = _mm256_madd_epi16(f0, shift3);
f0 = _mm256_sllv_epi32(f0, sllvdidx);
f0 = _mm256_srlv_epi64(f0, sllvdidx);
f0 = _mm256_shuffle_epi8(f0, shufbidx);
t0 = _mm256_castsi256_si128(f0);
t1 = _mm256_extracti128_si256(f0, 1);
t0 = _mm_blendv_epi8(t0, t1, _mm256_castsi256_si128(shufbidx));
_mm_storeu_si128(r[20 * i + 0..].as_mut_ptr() as *mut __m128i, t0);
tmp = _mm_cvtsi128_si32(t1);
r[20 * i + 16..20 * i + 20].copy_from_slice(&tmp.to_le_bytes());
}
}
#[cfg(not(feature="90s"))]
// #[target_feature(enable = "avx2")]
#[cfg(feature = "kyber1024")]
pub unsafe fn poly_decompress(r: &mut Poly, a: &[u8]) {
let (mut t, mut f, mut ti);
let q = _mm256_load_si256(&QDATA.vec[_16XQ / 16]);
let shufbidx = _mm256_set_epi8(
9, 9, 9, 8, 8, 8, 8, 7, 7, 6, 6, 6, 6, 5, 5, 5, 4, 4, 4, 3, 3, 3, 3, 2, 2, 1, 1, 1, 1, 0, 0, 0,
);
let mask = _mm256_set_epi16(248, 1984, 62, 496, 3968, 124, 992, 31, 248, 1984, 62, 496, 3968, 124, 992, 31);
let shift = _mm256_set_epi16(128, 16, 512, 64, 8, 256, 32, 1024, 128, 16, 512, 64, 8, 256, 32, 1024);
for i in 0..KYBER_N / 16 {
t = _mm_loadl_epi64(a[10 * i + 0..].as_ptr() as *const __m128i);
ti = i32::from_le_bytes([a[10 * i + 8], a[10 * i + 9], 0, 0]);
t = _mm_insert_epi16(t, ti, 4);
f = _mm256_broadcastsi128_si256(t);
f = _mm256_shuffle_epi8(f, shufbidx);
f = _mm256_and_si256(f, mask);
f = _mm256_mullo_epi16(f, shift);
f = _mm256_mulhrs_epi16(f, q);
_mm256_store_si256(r.vec[i..].as_mut_ptr() as *mut __m256i, f);
}
}
pub fn poly_frombytes(r: &mut Poly, a: &[u8]) {
unsafe {
nttfrombytes_avx(r.coeffs.as_mut_ptr(), a.as_ptr(), &QDATA.coeffs);
}
}
pub fn poly_tobytes(r: &mut [u8], a: Poly) {
let mut buf = [0u8; KYBER_POLYBYTES];
unsafe {
ntttobytes_avx(buf.as_mut_ptr(), &a.coeffs, &QDATA.coeffs);
}
r[..KYBER_POLYBYTES].copy_from_slice(&buf[..]);
}
// #[target_feature(enable = "avx2")]
pub unsafe fn poly_frommsg(r: &mut Poly, msg: &[u8]) {
let shift = _mm256_broadcastsi128_si256(_mm_set_epi32(0, 1, 2, 3));
let idx = _mm256_broadcastsi128_si256(_mm_set_epi8(15, 14, 11, 10, 7, 6, 3, 2, 13, 12, 9, 8, 5, 4, 1, 0));
let hqs: __m256i = _mm256_set1_epi16((KYBER_Q + 1) as i16 / 2);
let f = _mm256_loadu_si256(msg.as_ptr() as *const __m256i);
let mut frommsg64 = |i: usize, mut g3: __m256i| {
g3 = _mm256_sllv_epi32(g3, shift);
g3 = _mm256_shuffle_epi8(g3, idx);
let mut g0 = _mm256_slli_epi16(g3, 12);
let mut g1 = _mm256_slli_epi16(g3, 8);
let mut g2 = _mm256_slli_epi16(g3, 4);
g0 = _mm256_srai_epi16(g0, 15);
g1 = _mm256_srai_epi16(g1, 15);
g2 = _mm256_srai_epi16(g2, 15);
g3 = _mm256_srai_epi16(g3, 15);
g0 = _mm256_and_si256(g0, hqs); // 19 18 17 16 3 2 1 0
g1 = _mm256_and_si256(g1, hqs); // 23 22 21 20 7 6 5 4
g2 = _mm256_and_si256(g2, hqs); // 27 26 25 24 11 10 9 8
g3 = _mm256_and_si256(g3, hqs); // 31 30 29 28 15 14 13 12
let h0 = _mm256_unpacklo_epi64(g0, g1);
let h2 = _mm256_unpackhi_epi64(g0, g1);
let h1 = _mm256_unpacklo_epi64(g2, g3);
let h3 = _mm256_unpackhi_epi64(g2, g3);
g0 = _mm256_permute2x128_si256(h0, h1, 0x20);
g2 = _mm256_permute2x128_si256(h0, h1, 0x31);
g1 = _mm256_permute2x128_si256(h2, h3, 0x20);
g3 = _mm256_permute2x128_si256(h2, h3, 0x31);
_mm256_store_si256(&mut r.vec[0 + 2 * i + 0], g0);
_mm256_store_si256(&mut r.vec[0 + 2 * i + 1], g1);
_mm256_store_si256(&mut r.vec[8 + 2 * i + 0], g2);
_mm256_store_si256(&mut r.vec[8 + 2 * i + 1], g3);
};
frommsg64(0, _mm256_shuffle_epi32(f, 0));
frommsg64(1, _mm256_shuffle_epi32(f, 85));
frommsg64(2, _mm256_shuffle_epi32(f, 170));
frommsg64(3, _mm256_shuffle_epi32(f, 255));
}
// #[target_feature(enable = "avx2")]
pub fn poly_tomsg(msg: &mut [u8], a: Poly) {
unsafe {
let (mut f0, mut f1, mut g0, mut g1);
let hq: __m256i = _mm256_set1_epi16((KYBER_Q - 1) as i16 / 2);
let hhq: __m256i = _mm256_set1_epi16((KYBER_Q - 1) as i16 / 4);
for i in 0..KYBER_N / 32 {
f0 = _mm256_load_si256(&a.vec[2 * i + 0]);
f1 = _mm256_load_si256(&a.vec[2 * i + 1]);
f0 = _mm256_sub_epi16(hq, f0);
f1 = _mm256_sub_epi16(hq, f1);
g0 = _mm256_srai_epi16(f0, 15);
g1 = _mm256_srai_epi16(f1, 15);
f0 = _mm256_xor_si256(f0, g0);
f1 = _mm256_xor_si256(f1, g1);
f0 = _mm256_sub_epi16(f0, hhq);
f1 = _mm256_sub_epi16(f1, hhq);
f0 = _mm256_packs_epi16(f0, f1);
f0 = _mm256_permute4x64_epi64(f0, 0xD8);
let small = _mm256_movemask_epi8(f0);
msg[4 * i..][..4].copy_from_slice(&small.to_ne_bytes());
}
}
}
#[cfg(all(any(feature = "kyber1024", feature = "kyber512"), not(feature = "90s")))]
pub fn poly_getnoise_eta2(r: &mut Poly, seed: &[u8], nonce: u8) {
let mut buf = Eta2Buf::new();
unsafe {
prf(&mut buf.coeffs, KYBER_ETA2 * KYBER_N / 4, seed, nonce);
poly_cbd_eta2(r, &buf.vec);
}
}
#[cfg(not(feature = "90s"))]
pub fn poly_getnoise_eta1_4x(
r0: &mut Poly, r1: &mut Poly, r2: &mut Poly, r3: &mut Poly, seed: &[u8],
nonce0: u8, nonce1: u8, nonce2: u8, nonce3: u8
)
{
unsafe {
r0: &mut Poly,
r1: &mut Poly,
r2: &mut Poly,
r3: &mut Poly,
seed: &[u8],
nonce0: u8,
nonce1: u8,
nonce2: u8,
nonce3: u8,
) {
unsafe {
let mut buf = [Eta4xBuf::new(); 4];
let mut state = Keccakx4State::new();
let f = _mm256_loadu_si256(seed.as_ptr() as *const __m256i);
_mm256_store_si256(buf[0].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[1].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[2].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[3].vec.as_mut_ptr(), f);
buf[0].coeffs[32] = nonce0;
buf[1].coeffs[32] = nonce1;
buf[2].coeffs[32] = nonce2;
buf[3].coeffs[32] = nonce3;
shake256x4_absorb_once(&mut state, &buf[0].coeffs, &buf[1].coeffs, &buf[2].coeffs, &buf[3].coeffs, 33);
shake256x4_squeezeblocks(&mut buf, NOISE_NBLOCKS, &mut state);
poly_cbd_eta1(r0, &buf[0]);
poly_cbd_eta1(r1, &buf[1]);
poly_cbd_eta1(r2, &buf[2]);
poly_cbd_eta1(r3, &buf[3]);
}
}
#[cfg(all(feature = "kyber512", not(feature = "90s")))]
pub fn poly_getnoise_eta1122_4x(
r0: &mut Poly,
r1: &mut Poly,
r2: &mut Poly,
r3: &mut Poly,
seed: &[u8],
nonce0: u8,
nonce1: u8,
nonce2: u8,
nonce3: u8,
) {
let mut buf = [Eta4xBuf::new(); 4];
let mut state = Keccakx4State::new();
let f = _mm256_loadu_si256(seed.as_ptr() as *const __m256i);
_mm256_store_si256(buf[0].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[1].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[2].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[3].vec.as_mut_ptr(), f);
buf[0].coeffs[32] = nonce0;
buf[1].coeffs[32] = nonce1;
buf[2].coeffs[32] = nonce2;
buf[3].coeffs[32] = nonce3;
shake256x4_absorb_once(
&mut state,
&buf[0].coeffs, &buf[1].coeffs,
&buf[2].coeffs, &buf[3].coeffs,
33
);
shake256x4_squeezeblocks(&mut buf, NOISE_NBLOCKS, &mut state);
poly_cbd_eta1(r0, &buf[0]);
poly_cbd_eta1(r1, &buf[1]);
poly_cbd_eta1(r2, &buf[2]);
poly_cbd_eta1(r3, &buf[3]);
}
}
#[cfg(all(feature="kyber512", not(feature="90s")))]
pub fn poly_getnoise_eta1122_4x(
r0: &mut Poly, r1: &mut Poly, r2: &mut Poly, r3: &mut Poly, seed: &[u8],
nonce0: u8, nonce1: u8, nonce2: u8, nonce3: u8,
)
{
let mut buf = [Eta4xBuf::new(); 4];
let mut state = Keccakx4State::new();
unsafe {
let f = _mm256_loadu_si256(seed.as_ptr() as *const __m256i);
_mm256_store_si256(buf[0].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[1].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[2].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[3].vec.as_mut_ptr(), f);
buf[0].coeffs[32] = nonce0;
buf[1].coeffs[32] = nonce1;
buf[2].coeffs[32] = nonce2;
buf[3].coeffs[32] = nonce3;
shake256x4_absorb_once(
&mut state,
&buf[0].coeffs, &buf[1].coeffs,
&buf[2].coeffs, &buf[3].coeffs,
33
);
shake256x4_squeezeblocks(&mut buf, NOISE_NBLOCKS, &mut state);
poly_cbd_eta1(r0, &buf[0]);
poly_cbd_eta1(r1, &buf[1]);
poly_cbd_eta2(r2, &buf[2].vec);
poly_cbd_eta2(r3, &buf[3].vec);
}
}
pub fn poly_ntt(r: &mut Poly)
{
unsafe { ntt_avx(&mut r.coeffs, &QDATA.coeffs); }
}
pub fn poly_invntt_tomont(r: &mut Poly)
{
unsafe { invntt_avx(&mut r.coeffs, &QDATA.coeffs); }
}
pub fn poly_nttunpack(r: &mut Poly)
{
unsafe { nttunpack_avx(&mut r.coeffs, &QDATA.coeffs); }
}
pub fn poly_basemul(r: &mut Poly, a: &Poly, b: &Poly)
{
unsafe { basemul_avx(&mut r.coeffs, &a.coeffs, &b.coeffs, &QDATA.coeffs); }
}
pub fn poly_tomont(r: &mut Poly)
{
unsafe { tomont_avx(&mut r.coeffs, &QDATA.coeffs); }
}
pub fn poly_reduce(r: &mut Poly)
{
unsafe { reduce_avx(&mut r.coeffs, &QDATA.coeffs); }
}
pub fn poly_add(r: &mut Poly, b: &Poly)
{
let (mut f0, mut f1);
for i in 0..(KYBER_N/16) {
unsafe {
f0 = _mm256_load_si256(&r.vec[i]);
f1 = _mm256_load_si256(&b.vec[i]);
f0 = _mm256_add_epi16(f0, f1);
_mm256_store_si256(&mut r.vec[i] , f0);
let f = _mm256_loadu_si256(seed.as_ptr() as *const __m256i);
_mm256_store_si256(buf[0].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[1].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[2].vec.as_mut_ptr(), f);
_mm256_store_si256(buf[3].vec.as_mut_ptr(), f);
buf[0].coeffs[32] = nonce0;
buf[1].coeffs[32] = nonce1;
buf[2].coeffs[32] = nonce2;
buf[3].coeffs[32] = nonce3;
shake256x4_absorb_once(&mut state, &buf[0].coeffs, &buf[1].coeffs, &buf[2].coeffs, &buf[3].coeffs, 33);
shake256x4_squeezeblocks(&mut buf, NOISE_NBLOCKS, &mut state);
poly_cbd_eta1(r0, &buf[0]);
poly_cbd_eta1(r1, &buf[1]);
poly_cbd_eta2(r2, &buf[2].vec);
poly_cbd_eta2(r3, &buf[3].vec);
}
}
}
pub fn poly_sub(r: &mut Poly, a: &Poly)
{
let (mut f0, mut f1);
for i in 0..(KYBER_N/16) {
pub fn poly_ntt(r: &mut Poly) {
unsafe {
f0 = _mm256_load_si256(&a.vec[i]);
f1 = _mm256_load_si256(&r.vec[i]);
f0 = _mm256_sub_epi16(f0, f1);
_mm256_store_si256(&mut r.vec[i], f0);
ntt_avx(&mut r.coeffs, &QDATA.coeffs);
}
}
}
pub fn poly_invntt_tomont(r: &mut Poly) {
unsafe {
invntt_avx(&mut r.coeffs, &QDATA.coeffs);
}
}
pub fn poly_nttunpack(r: &mut Poly) {
unsafe {
nttunpack_avx(&mut r.coeffs, &QDATA.coeffs);
}
}
pub fn poly_basemul(r: &mut Poly, a: &Poly, b: &Poly) {
unsafe {
basemul_avx(&mut r.coeffs, &a.coeffs, &b.coeffs, &QDATA.coeffs);
}
}
pub fn poly_tomont(r: &mut Poly) {
unsafe {
tomont_avx(&mut r.coeffs, &QDATA.coeffs);
}
}
pub fn poly_reduce(r: &mut Poly) {
unsafe {
reduce_avx(&mut r.coeffs, &QDATA.coeffs);
}
}
pub fn poly_add(r: &mut Poly, b: &Poly) {
let (mut f0, mut f1);
for i in 0..(KYBER_N / 16) {
unsafe {
f0 = _mm256_load_si256(&r.vec[i]);
f1 = _mm256_load_si256(&b.vec[i]);
f0 = _mm256_add_epi16(f0, f1);
_mm256_store_si256(&mut r.vec[i], f0);
}
}
}
pub fn poly_sub(r: &mut Poly, a: &Poly) {
let (mut f0, mut f1);
for i in 0..(KYBER_N / 16) {
unsafe {
f0 = _mm256_load_si256(&a.vec[i]);
f1 = _mm256_load_si256(&r.vec[i]);
f0 = _mm256_sub_epi16(f0, f1);
_mm256_store_si256(&mut r.vec[i], f0);
}
}
}

421
third_party/kyber/src/avx2/polyvec.rs vendored
View file

@ -1,222 +1,197 @@
use crate::{consts::*, params::*, poly::*};
use core::arch::x86_64::*;
use crate::{
poly::*,
params::*,
consts::*
};
#[derive(Clone)]
pub struct Polyvec {
pub vec: [Poly; KYBER_K]
pub vec: [Poly; KYBER_K],
}
impl Copy for Polyvec {}
impl Polyvec {
pub fn new() -> Self {
Polyvec {
vec: [Poly::new(); KYBER_K]
pub fn new() -> Self {
Polyvec { vec: [Poly::new(); KYBER_K] }
}
}
// Basic polynomial value check for development
// #[cfg(debug_assertions)]
// pub fn checksum(&self) -> i16 {
// unsafe {
// let mut out = 0i16;
// for i in 0..KYBER_K {
// for j in 0..KYBER_N {
// out ^= &self.vec[i].coeffs[j];
// }
// }
// out
// }
// }
// Basic polynomial value check for development
// #[cfg(debug_assertions)]
// pub fn checksum(&self) -> i16 {
// unsafe {
// let mut out = 0i16;
// for i in 0..KYBER_K {
// for j in 0..KYBER_N {
// out ^= &self.vec[i].coeffs[j];
// }
// }
// out
// }
// }
}
// #[target_feature(enable = "avx")]
pub unsafe fn poly_compress10(r: &mut[u8], a: &Poly)
{
let (mut f0, mut f1, mut f2);
let (mut t0, mut t1);
pub unsafe fn poly_compress10(r: &mut [u8], a: &Poly) {
let (mut f0, mut f1, mut f2);
let (mut t0, mut t1);
let v = _mm256_load_si256(QDATA.vec[_16XV/16..].as_ptr());
let v8 = _mm256_slli_epi16(v,3);
let off = _mm256_set1_epi16(15);
let shift1 = _mm256_set1_epi16(1 << 12);
let mask = _mm256_set1_epi16(1023);
let shift2 = _mm256_set1_epi64x(
((1024u64 << 48) + (1u64 << 32) + (1024 << 16) + 1) as i64
);
let sllvdidx = _mm256_set1_epi64x(12);
let shufbidx = _mm256_set_epi8(
8, 4, 3, 2, 1, 0,-1,-1,-1,-1,-1,-1,12,11,10, 9,
-1,-1,-1,-1,-1,-1,12,11,10, 9, 8, 4, 3, 2, 1, 0
);
let v = _mm256_load_si256(QDATA.vec[_16XV / 16..].as_ptr());
let v8 = _mm256_slli_epi16(v, 3);
let off = _mm256_set1_epi16(15);
let shift1 = _mm256_set1_epi16(1 << 12);
let mask = _mm256_set1_epi16(1023);
let shift2 = _mm256_set1_epi64x(((1024u64 << 48) + (1u64 << 32) + (1024 << 16) + 1) as i64);
let sllvdidx = _mm256_set1_epi64x(12);
let shufbidx = _mm256_set_epi8(
8, 4, 3, 2, 1, 0, -1, -1, -1, -1, -1, -1, 12, 11, 10, 9, -1, -1, -1, -1, -1, -1, 12, 11, 10, 9, 8, 4, 3, 2, 1, 0,
);
for i in 0..(KYBER_N/16) {
f0 = _mm256_load_si256(&a.vec[i]);
f1 = _mm256_mullo_epi16(f0,v8);
f2 = _mm256_add_epi16(f0,off);
f0 = _mm256_slli_epi16(f0,3);
f0 = _mm256_mulhi_epi16(f0,v);
f2 = _mm256_sub_epi16(f1,f2);
f1 = _mm256_andnot_si256(f1,f2);
f1 = _mm256_srli_epi16(f1,15);
f0 = _mm256_sub_epi16(f0,f1);
f0 = _mm256_mulhrs_epi16(f0,shift1);
f0 = _mm256_and_si256(f0,mask);
f0 = _mm256_madd_epi16(f0,shift2);
f0 = _mm256_sllv_epi32(f0,sllvdidx);
f0 = _mm256_srli_epi64(f0,12);
f0 = _mm256_shuffle_epi8(f0,shufbidx);
t0 = _mm256_castsi256_si128(f0);
t1 = _mm256_extracti128_si256(f0,1);
t0 = _mm_blend_epi16(t0,t1,0xE0);
_mm_storeu_si128(r[20*i..].as_mut_ptr() as *mut __m128i,t0);
_mm_storeu_si128(r[20*i+16..].as_mut_ptr() as *mut __m128i, t1);
}
}
// #[target_feature(enable = "avx")]
pub unsafe fn poly_decompress10(r: &mut Poly, a: &[u8])
{
let mut f;
let q = _mm256_set1_epi32(((KYBER_Q as i32) << 16) + 4*KYBER_Q as i32);
let shufbidx = _mm256_set_epi8(11,10,10, 9, 9, 8, 8, 7,
6, 5, 5, 4, 4, 3, 3, 2,
9, 8, 8, 7, 7, 6, 6, 5,
4, 3, 3, 2, 2, 1, 1, 0);
let sllvdidx = _mm256_set1_epi64x(4);
let mask = _mm256_set1_epi32((32736 << 16) + 8184);
for i in 0..KYBER_N/16 {
f = _mm256_loadu_si256(a[20*i..].as_ptr() as *const __m256i);
f = _mm256_permute4x64_epi64(f,0x94);
f = _mm256_shuffle_epi8(f,shufbidx);
f = _mm256_sllv_epi32(f,sllvdidx);
f = _mm256_srli_epi16(f,1);
f = _mm256_and_si256(f,mask);
f = _mm256_mulhrs_epi16(f,q);
_mm256_store_si256(&mut r.vec[i],f);
}
}
// #[target_feature(enable = "avx")]
pub unsafe fn poly_compress11(r: &mut[u8], a: &Poly)
{
let (mut f0, mut f1, mut f2);
let (mut t0, mut t1);
let v = _mm256_load_si256(QDATA.vec[_16XV/16..].as_ptr());
let v8 = _mm256_slli_epi16(v,3);
let off = _mm256_set1_epi16(36);
let shift1 = _mm256_set1_epi16(1 << 13);
let mask = _mm256_set1_epi16(2047);
let shift2 = _mm256_set1_epi64x(
((2048u64 << 48) + (1u64 << 32) + (2048 << 16) + 1) as i64
);
let sllvdidx = _mm256_set1_epi64x(10);
let srlvqidx = _mm256_set_epi64x(30,10,30,10);
let shufbidx = _mm256_set_epi8(
4, 3, 2, 1, 0, 0,-1,-1,-1,-1,10, 9, 8, 7, 6, 5,
-1,-1,-1,-1,-1,10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
);
for i in 0..KYBER_N/16 {
f0 = _mm256_load_si256(&a.vec[i]);
f1 = _mm256_mullo_epi16(f0,v8);
f2 = _mm256_add_epi16(f0,off);
f0 = _mm256_slli_epi16(f0,3);
f0 = _mm256_mulhi_epi16(f0,v);
f2 = _mm256_sub_epi16(f1,f2);
f1 = _mm256_andnot_si256(f1,f2);
f1 = _mm256_srli_epi16(f1,15);
f0 = _mm256_sub_epi16(f0,f1);
f0 = _mm256_mulhrs_epi16(f0,shift1);
f0 = _mm256_and_si256(f0,mask);
f0 = _mm256_madd_epi16(f0,shift2);
f0 = _mm256_sllv_epi32(f0,sllvdidx);
f1 = _mm256_bsrli_epi128(f0,8);
f0 = _mm256_srlv_epi64(f0,srlvqidx);
f1 = _mm256_slli_epi64(f1,34);
f0 = _mm256_add_epi64(f0,f1);
f0 = _mm256_shuffle_epi8(f0,shufbidx);
t0 = _mm256_castsi256_si128(f0);
t1 = _mm256_extracti128_si256(f0,1);
t0 = _mm_blendv_epi8(t0,t1,_mm256_castsi256_si128(shufbidx));
_mm_storeu_si128(r[22*i+ 0..].as_mut_ptr() as *mut __m128i,t0);
_mm_storel_epi64(r[22*i+16..].as_mut_ptr() as *mut __m128i,t1);
}
}
// #[target_feature(enable = "avx")]
pub unsafe fn poly_decompress11(r: &mut Poly, a: &[u8])
{
let mut f;
let q = _mm256_load_si256(QDATA.vec[_16XQ/16..].as_ptr());
let shufbidx = _mm256_set_epi8(
13,12,12,11,10, 9, 9, 8,
8, 7, 6, 5, 5, 4, 4, 3,
10, 9, 9, 8, 7, 6, 6, 5,
5, 4, 3, 2, 2, 1, 1, 0
);
let srlvdidx = _mm256_set_epi32(0,0,1,0,0,0,1,0);
let srlvqidx = _mm256_set_epi64x(2,0,2,0);
let shift = _mm256_set_epi16(4,32,1,8,32,1,4,32,4,32,1,8,32,1,4,32);
let mask = _mm256_set1_epi16(32752);
for i in 0..(KYBER_N/16) {
f = _mm256_loadu_si256(a[22*i..].as_ptr() as *const __m256i);
f = _mm256_permute4x64_epi64(f,0x94);
f = _mm256_shuffle_epi8(f,shufbidx);
f = _mm256_srlv_epi32(f,srlvdidx);
f = _mm256_srlv_epi64(f,srlvqidx);
f = _mm256_mullo_epi16(f,shift);
f = _mm256_srli_epi16(f,1);
f = _mm256_and_si256(f,mask);
f = _mm256_mulhrs_epi16(f,q);
_mm256_store_si256(&mut r.vec[i],f);
}
}
pub unsafe fn polyvec_compress(r: &mut[u8], a: &Polyvec)
{
if cfg!(feature="kyber1024") {
for i in 0..KYBER_K {
poly_compress11(&mut r[352*i..], &a.vec[i]);
}
} else {
for i in 0..KYBER_K {
poly_compress10(&mut r[320*i..], &a.vec[i]);
for i in 0..(KYBER_N / 16) {
f0 = _mm256_load_si256(&a.vec[i]);
f1 = _mm256_mullo_epi16(f0, v8);
f2 = _mm256_add_epi16(f0, off);
f0 = _mm256_slli_epi16(f0, 3);
f0 = _mm256_mulhi_epi16(f0, v);
f2 = _mm256_sub_epi16(f1, f2);
f1 = _mm256_andnot_si256(f1, f2);
f1 = _mm256_srli_epi16(f1, 15);
f0 = _mm256_sub_epi16(f0, f1);
f0 = _mm256_mulhrs_epi16(f0, shift1);
f0 = _mm256_and_si256(f0, mask);
f0 = _mm256_madd_epi16(f0, shift2);
f0 = _mm256_sllv_epi32(f0, sllvdidx);
f0 = _mm256_srli_epi64(f0, 12);
f0 = _mm256_shuffle_epi8(f0, shufbidx);
t0 = _mm256_castsi256_si128(f0);
t1 = _mm256_extracti128_si256(f0, 1);
t0 = _mm_blend_epi16(t0, t1, 0xE0);
_mm_storeu_si128(r[20 * i..].as_mut_ptr() as *mut __m128i, t0);
_mm_storeu_si128(r[20 * i + 16..].as_mut_ptr() as *mut __m128i, t1);
}
}
}
pub unsafe fn polyvec_decompress(r: &mut Polyvec, a: &[u8])
{
if cfg!(feature="kyber1024") {
for i in 0..KYBER_K {
poly_decompress11(&mut r.vec[i], &a[352*i..]);
}
} else {
for i in 0..KYBER_K {
poly_decompress10(&mut r.vec[i], &a[320*i..]);
// #[target_feature(enable = "avx")]
pub unsafe fn poly_decompress10(r: &mut Poly, a: &[u8]) {
let mut f;
let q = _mm256_set1_epi32(((KYBER_Q as i32) << 16) + 4 * KYBER_Q as i32);
let shufbidx = _mm256_set_epi8(
11, 10, 10, 9, 9, 8, 8, 7, 6, 5, 5, 4, 4, 3, 3, 2, 9, 8, 8, 7, 7, 6, 6, 5, 4, 3, 3, 2, 2, 1, 1, 0,
);
let sllvdidx = _mm256_set1_epi64x(4);
let mask = _mm256_set1_epi32((32736 << 16) + 8184);
for i in 0..KYBER_N / 16 {
f = _mm256_loadu_si256(a[20 * i..].as_ptr() as *const __m256i);
f = _mm256_permute4x64_epi64(f, 0x94);
f = _mm256_shuffle_epi8(f, shufbidx);
f = _mm256_sllv_epi32(f, sllvdidx);
f = _mm256_srli_epi16(f, 1);
f = _mm256_and_si256(f, mask);
f = _mm256_mulhrs_epi16(f, q);
_mm256_store_si256(&mut r.vec[i], f);
}
}
}
pub fn polyvec_tobytes(r: &mut[u8], a: &Polyvec)
{
for i in 0..KYBER_K {
poly_tobytes(&mut r[i*KYBER_POLYBYTES..], a.vec[i]);
}
// #[target_feature(enable = "avx")]
pub unsafe fn poly_compress11(r: &mut [u8], a: &Poly) {
let (mut f0, mut f1, mut f2);
let (mut t0, mut t1);
let v = _mm256_load_si256(QDATA.vec[_16XV / 16..].as_ptr());
let v8 = _mm256_slli_epi16(v, 3);
let off = _mm256_set1_epi16(36);
let shift1 = _mm256_set1_epi16(1 << 13);
let mask = _mm256_set1_epi16(2047);
let shift2 = _mm256_set1_epi64x(((2048u64 << 48) + (1u64 << 32) + (2048 << 16) + 1) as i64);
let sllvdidx = _mm256_set1_epi64x(10);
let srlvqidx = _mm256_set_epi64x(30, 10, 30, 10);
let shufbidx = _mm256_set_epi8(
4, 3, 2, 1, 0, 0, -1, -1, -1, -1, 10, 9, 8, 7, 6, 5, -1, -1, -1, -1, -1, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
);
for i in 0..KYBER_N / 16 {
f0 = _mm256_load_si256(&a.vec[i]);
f1 = _mm256_mullo_epi16(f0, v8);
f2 = _mm256_add_epi16(f0, off);
f0 = _mm256_slli_epi16(f0, 3);
f0 = _mm256_mulhi_epi16(f0, v);
f2 = _mm256_sub_epi16(f1, f2);
f1 = _mm256_andnot_si256(f1, f2);
f1 = _mm256_srli_epi16(f1, 15);
f0 = _mm256_sub_epi16(f0, f1);
f0 = _mm256_mulhrs_epi16(f0, shift1);
f0 = _mm256_and_si256(f0, mask);
f0 = _mm256_madd_epi16(f0, shift2);
f0 = _mm256_sllv_epi32(f0, sllvdidx);
f1 = _mm256_bsrli_epi128(f0, 8);
f0 = _mm256_srlv_epi64(f0, srlvqidx);
f1 = _mm256_slli_epi64(f1, 34);
f0 = _mm256_add_epi64(f0, f1);
f0 = _mm256_shuffle_epi8(f0, shufbidx);
t0 = _mm256_castsi256_si128(f0);
t1 = _mm256_extracti128_si256(f0, 1);
t0 = _mm_blendv_epi8(t0, t1, _mm256_castsi256_si128(shufbidx));
_mm_storeu_si128(r[22 * i + 0..].as_mut_ptr() as *mut __m128i, t0);
_mm_storel_epi64(r[22 * i + 16..].as_mut_ptr() as *mut __m128i, t1);
}
}
pub unsafe fn polyvec_frombytes(r: &mut Polyvec, a: &[u8])
{
for i in 0..KYBER_K {
poly_frombytes(&mut r.vec[i], &a[i*KYBER_POLYBYTES..]);
}
// #[target_feature(enable = "avx")]
pub unsafe fn poly_decompress11(r: &mut Poly, a: &[u8]) {
let mut f;
let q = _mm256_load_si256(QDATA.vec[_16XQ / 16..].as_ptr());
let shufbidx = _mm256_set_epi8(
13, 12, 12, 11, 10, 9, 9, 8, 8, 7, 6, 5, 5, 4, 4, 3, 10, 9, 9, 8, 7, 6, 6, 5, 5, 4, 3, 2, 2, 1, 1, 0,
);
let srlvdidx = _mm256_set_epi32(0, 0, 1, 0, 0, 0, 1, 0);
let srlvqidx = _mm256_set_epi64x(2, 0, 2, 0);
let shift = _mm256_set_epi16(4, 32, 1, 8, 32, 1, 4, 32, 4, 32, 1, 8, 32, 1, 4, 32);
let mask = _mm256_set1_epi16(32752);
for i in 0..(KYBER_N / 16) {
f = _mm256_loadu_si256(a[22 * i..].as_ptr() as *const __m256i);
f = _mm256_permute4x64_epi64(f, 0x94);
f = _mm256_shuffle_epi8(f, shufbidx);
f = _mm256_srlv_epi32(f, srlvdidx);
f = _mm256_srlv_epi64(f, srlvqidx);
f = _mm256_mullo_epi16(f, shift);
f = _mm256_srli_epi16(f, 1);
f = _mm256_and_si256(f, mask);
f = _mm256_mulhrs_epi16(f, q);
_mm256_store_si256(&mut r.vec[i], f);
}
}
pub unsafe fn polyvec_compress(r: &mut [u8], a: &Polyvec) {
if cfg!(feature = "kyber1024") {
for i in 0..KYBER_K {
poly_compress11(&mut r[352 * i..], &a.vec[i]);
}
} else {
for i in 0..KYBER_K {
poly_compress10(&mut r[320 * i..], &a.vec[i]);
}
}
}
pub unsafe fn polyvec_decompress(r: &mut Polyvec, a: &[u8]) {
if cfg!(feature = "kyber1024") {
for i in 0..KYBER_K {
poly_decompress11(&mut r.vec[i], &a[352 * i..]);
}
} else {
for i in 0..KYBER_K {
poly_decompress10(&mut r.vec[i], &a[320 * i..]);
}
}
}
pub fn polyvec_tobytes(r: &mut [u8], a: &Polyvec) {
for i in 0..KYBER_K {
poly_tobytes(&mut r[i * KYBER_POLYBYTES..], a.vec[i]);
}
}
pub unsafe fn polyvec_frombytes(r: &mut Polyvec, a: &[u8]) {
for i in 0..KYBER_K {
poly_frombytes(&mut r.vec[i], &a[i * KYBER_POLYBYTES..]);
}
}
// Name: polyvec_ntt
@ -224,11 +199,10 @@ pub unsafe fn polyvec_frombytes(r: &mut Polyvec, a: &[u8])
// Description: Apply forward NTT to all elements of a vector of polynomials
//
// Arguments: - Polyvec r: in/output vector of polynomials
pub fn polyvec_ntt(r: &mut Polyvec)
{
for i in 0..KYBER_K {
poly_ntt(&mut r.vec[i]);
}
pub fn polyvec_ntt(r: &mut Polyvec) {
for i in 0..KYBER_K {
poly_ntt(&mut r.vec[i]);
}
}
// Name: polyvec_invntt
@ -236,11 +210,10 @@ pub fn polyvec_ntt(r: &mut Polyvec)
// Description: Apply inverse NTT to all elements of a vector of polynomials
//
// Arguments: - Polyvec r: in/output vector of polynomials
pub fn polyvec_invntt_tomont(r: &mut Polyvec)
{
for i in 0..KYBER_K {
poly_invntt_tomont(&mut r.vec[i]);
}
pub fn polyvec_invntt_tomont(r: &mut Polyvec) {
for i in 0..KYBER_K {
poly_invntt_tomont(&mut r.vec[i]);
}
}
// Name: polyvec_basemul_acc_montgomery
@ -250,31 +223,28 @@ pub fn polyvec_invntt_tomont(r: &mut Polyvec)
// Arguments: - poly *r: output polynomial
// - const Polyvec a: first input vector of polynomials
// - const Polyvec b: second input vector of polynomials
pub fn polyvec_basemul_acc_montgomery(r: &mut Poly, a: &Polyvec, b: &Polyvec)
{
let mut t = Poly::new();
poly_basemul(r, &a.vec[0], &b.vec[0]);
for i in 1..KYBER_K {
poly_basemul(&mut t, &a.vec[i], &b.vec[i]);
poly_add(r, &t);
}
pub fn polyvec_basemul_acc_montgomery(r: &mut Poly, a: &Polyvec, b: &Polyvec) {
let mut t = Poly::new();
poly_basemul(r, &a.vec[0], &b.vec[0]);
for i in 1..KYBER_K {
poly_basemul(&mut t, &a.vec[i], &b.vec[i]);
poly_add(r, &t);
}
}
// Name: polyvec_reduce
//
// Description: Applies Barrett reduction to each coefficient
// Description: Applies Barrett reduction to each coefficient
// of each element of a vector of polynomials
// for details of the Barrett reduction see comments in reduce.c
//
// Arguments: - poly *r: input/output polynomial
pub fn polyvec_reduce(r: &mut Polyvec)
{
for i in 0..KYBER_K {
poly_reduce(&mut r.vec[i]);
}
pub fn polyvec_reduce(r: &mut Polyvec) {
for i in 0..KYBER_K {
poly_reduce(&mut r.vec[i]);
}
}
// Name: polyvec_add
//
// Description: Add vectors of polynomials
@ -282,9 +252,8 @@ pub fn polyvec_reduce(r: &mut Polyvec)
// Arguments: - Polyvec r: output vector of polynomials
// - const Polyvec a: first input vector of polynomials
// - const Polyvec b: second input vector of polynomials
pub fn polyvec_add(r: &mut Polyvec, b: &Polyvec)
{
for i in 0..KYBER_K {
poly_add(&mut r.vec[i], &b.vec[i]);
}
pub fn polyvec_add(r: &mut Polyvec, b: &Polyvec) {
for i in 0..KYBER_K {
poly_add(&mut r.vec[i], &b.vec[i]);
}
}

710
third_party/kyber/src/avx2/rejsample.rs vendored
View file

@ -1,380 +1,372 @@
use crate::{consts::*, params::*, symmetric::*};
use core::arch::x86_64::*;
use crate::{
params::*,
consts::*,
symmetric::*
};
pub(crate) const REJ_UNIFORM_AVX_NBLOCKS: usize =
(12*KYBER_N/8*(1 << 12)/KYBER_Q + XOF_BLOCKBYTES)/XOF_BLOCKBYTES;
const REJ_UNIFORM_AVX_BUFLEN: usize = REJ_UNIFORM_AVX_NBLOCKS*XOF_BLOCKBYTES;
pub(crate) const REJ_UNIFORM_AVX_NBLOCKS: usize = (12 * KYBER_N / 8 * (1 << 12) / KYBER_Q + XOF_BLOCKBYTES) / XOF_BLOCKBYTES;
const REJ_UNIFORM_AVX_BUFLEN: usize = REJ_UNIFORM_AVX_NBLOCKS * XOF_BLOCKBYTES;
pub unsafe fn _mm256_cmpge_epu16(a: __m256i, b: __m256i) -> __m256i {
_mm256_cmpeq_epi16(_mm256_max_epu16(a, b), a)
_mm256_cmpeq_epi16(_mm256_max_epu16(a, b), a)
}
pub unsafe fn _mm_cmpge_epu16(a: __m128i, b: __m128i) -> __m128i {
_mm_cmpeq_epi16(_mm_max_epu16(a, b), a)
_mm_cmpeq_epi16(_mm_max_epu16(a, b), a)
}
pub unsafe fn rej_uniform_avx(r: &mut[i16], buf: &[u8]) -> usize {
let mut ctr = 0;
let mut pos = 0;
let mut good: usize;
let (mut val0, mut val1);
let (mut f0, mut f1, mut g0, mut g1, mut g2, mut g3);
let (mut f, mut t, mut pilo, mut pihi);
let qdata_ptr = QDATA.coeffs[_16XQ..].as_ptr();
let bound = _mm256_load_si256(qdata_ptr as *const __m256i);
let ones = _mm256_set1_epi8(1);
let mask = _mm256_set1_epi16(0xFFF);
let idx8 = _mm256_set_epi8(
15,14,14,13,12,11,11,10,
9, 8, 8, 7, 6, 5, 5, 4,
11,10,10, 9, 8, 7, 7, 6,
5, 4, 4, 3, 2, 1, 1, 0
);
while ctr <= KYBER_N - 32 && pos <= REJ_UNIFORM_AVX_BUFLEN - 48 {
f0 = _mm256_loadu_si256(buf[pos..].as_ptr() as *const __m256i);
f1 = _mm256_loadu_si256(buf[pos+24..].as_ptr() as *const __m256i);
f0 = _mm256_permute4x64_epi64(f0, 0x94);
f1 = _mm256_permute4x64_epi64(f1, 0x94);
f0 = _mm256_shuffle_epi8(f0, idx8);
f1 = _mm256_shuffle_epi8(f1, idx8);
g0 = _mm256_srli_epi16(f0, 4);
g1 = _mm256_srli_epi16(f1, 4);
f0 = _mm256_blend_epi16(f0, g0, 0xAA);
f1 = _mm256_blend_epi16(f1, g1, 0xAA);
f0 = _mm256_and_si256(f0, mask);
f1 = _mm256_and_si256(f1, mask);
pos += 48;
pub unsafe fn rej_uniform_avx(r: &mut [i16], buf: &[u8]) -> usize {
let mut ctr = 0;
let mut pos = 0;
let mut good: usize;
let (mut val0, mut val1);
let (mut f0, mut f1, mut g0, mut g1, mut g2, mut g3);
let (mut f, mut t, mut pilo, mut pihi);
let qdata_ptr = QDATA.coeffs[_16XQ..].as_ptr();
let bound = _mm256_load_si256(qdata_ptr as *const __m256i);
let ones = _mm256_set1_epi8(1);
let mask = _mm256_set1_epi16(0xFFF);
let idx8 = _mm256_set_epi8(
15, 14, 14, 13, 12, 11, 11, 10, 9, 8, 8, 7, 6, 5, 5, 4, 11, 10, 10, 9, 8, 7, 7, 6, 5, 4, 4, 3, 2, 1, 1, 0,
);
while ctr <= KYBER_N - 32 && pos <= REJ_UNIFORM_AVX_BUFLEN - 48 {
f0 = _mm256_loadu_si256(buf[pos..].as_ptr() as *const __m256i);
f1 = _mm256_loadu_si256(buf[pos + 24..].as_ptr() as *const __m256i);
f0 = _mm256_permute4x64_epi64(f0, 0x94);
f1 = _mm256_permute4x64_epi64(f1, 0x94);
f0 = _mm256_shuffle_epi8(f0, idx8);
f1 = _mm256_shuffle_epi8(f1, idx8);
g0 = _mm256_srli_epi16(f0, 4);
g1 = _mm256_srli_epi16(f1, 4);
f0 = _mm256_blend_epi16(f0, g0, 0xAA);
f1 = _mm256_blend_epi16(f1, g1, 0xAA);
f0 = _mm256_and_si256(f0, mask);
f1 = _mm256_and_si256(f1, mask);
pos += 48;
g0 = _mm256_cmpgt_epi16(bound, f0);
g1 = _mm256_cmpgt_epi16(bound, f1);
g0 = _mm256_cmpgt_epi16(bound, f0);
g1 = _mm256_cmpgt_epi16(bound, f1);
g0 = _mm256_packs_epi16(g0, g1);
good = _mm256_movemask_epi8(g0) as usize;
g0 = _mm256_packs_epi16(g0, g1);
good = _mm256_movemask_epi8(g0) as usize;
let mut l0 = _mm_loadl_epi64(IDX[(good >> 0) & 0xFF].as_ptr() as * const __m128i);
g0 = _mm256_castsi128_si256(l0);
let mut l1 = _mm_loadl_epi64(IDX[(good >> 8) & 0xFF].as_ptr() as *const __m128i);
g1 = _mm256_castsi128_si256(l1);
let mut l0 = _mm_loadl_epi64(IDX[(good >> 0) & 0xFF].as_ptr() as *const __m128i);
g0 = _mm256_castsi128_si256(l0);
let mut l1 = _mm_loadl_epi64(IDX[(good >> 8) & 0xFF].as_ptr() as *const __m128i);
g1 = _mm256_castsi128_si256(l1);
l0 = _mm_loadl_epi64(IDX[(good >> 16) & 0xFF].as_ptr() as *const __m128i);
g0 = _mm256_inserti128_si256(g0, l0, 1);
l1 = _mm_loadl_epi64(IDX[(good >> 24) & 0xFF].as_ptr() as *const __m128i);
g1 = _mm256_inserti128_si256(g1, l1, 1);
g2 = _mm256_add_epi8(g0, ones);
g3 = _mm256_add_epi8(g1, ones);
g0 = _mm256_unpacklo_epi8(g0, g2);
g1 = _mm256_unpacklo_epi8(g1, g3);
l0 = _mm_loadl_epi64(IDX[(good >> 16) & 0xFF].as_ptr() as *const __m128i);
g0 = _mm256_inserti128_si256(g0, l0, 1);
l1 = _mm_loadl_epi64(IDX[(good >> 24) & 0xFF].as_ptr() as *const __m128i);
g1 = _mm256_inserti128_si256(g1, l1, 1);
f0 = _mm256_shuffle_epi8(f0, g0);
f1 = _mm256_shuffle_epi8(f1, g1);
g2 = _mm256_add_epi8(g0, ones);
g3 = _mm256_add_epi8(g1, ones);
g0 = _mm256_unpacklo_epi8(g0, g2);
g1 = _mm256_unpacklo_epi8(g1, g3);
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(f0));
ctr += _popcnt32(((good >> 0) & 0xFF) as i32) as usize;
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_extracti128_si256(f0, 1));
ctr += _popcnt32(((good >> 16) & 0xFF) as i32) as usize;
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(f1));
ctr += _popcnt32(((good >> 8) & 0xFF) as i32) as usize;
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_extracti128_si256(f1, 1));
ctr += _popcnt32(((good >> 24) & 0xFF) as i32) as usize;
}
f0 = _mm256_shuffle_epi8(f0, g0);
f1 = _mm256_shuffle_epi8(f1, g1);
while ctr <= KYBER_N - 8 && pos <= REJ_UNIFORM_AVX_BUFLEN - 12 {
f = _mm_loadu_si128(buf[pos..].as_ptr() as *const __m128i);
f = _mm_shuffle_epi8(f, _mm256_castsi256_si128(idx8));
t = _mm_srli_epi16(f, 4);
f = _mm_blend_epi16(f, t, 0xAA);
f = _mm_and_si128(f, _mm256_castsi256_si128(mask));
pos += 12;
t = _mm_cmpgt_epi16(_mm256_castsi256_si128(bound), f);
good = _mm_movemask_epi8(t) as usize;
let good = _pext_u32(good as u32, 0x5555) as usize;
pilo = _mm_loadl_epi64(IDX[good][..].as_ptr() as *const __m128i);
pihi = _mm_add_epi8(pilo, _mm256_castsi256_si128(ones));
pilo = _mm_unpacklo_epi8(pilo, pihi);
f = _mm_shuffle_epi8(f, pilo);
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, f);
ctr += _popcnt32(good as i32) as usize;
}
while ctr < KYBER_N && pos <= REJ_UNIFORM_AVX_BUFLEN - 3 {
val0 = (buf[pos+0] >> 0) as u16 | ((buf[pos+1] as u16) << 8) & 0xFFF;
val1 = (buf[pos+1] >> 4) as u16 | ((buf[pos+2] as u16) << 4);
pos += 3;
if (val0 as usize) < KYBER_Q {
r[ctr] = val0 as i16;
ctr += 1;
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(f0));
ctr += _popcnt32(((good >> 0) & 0xFF) as i32) as usize;
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_extracti128_si256(f0, 1));
ctr += _popcnt32(((good >> 16) & 0xFF) as i32) as usize;
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(f1));
ctr += _popcnt32(((good >> 8) & 0xFF) as i32) as usize;
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, _mm256_extracti128_si256(f1, 1));
ctr += _popcnt32(((good >> 24) & 0xFF) as i32) as usize;
}
if (val1 as usize) < KYBER_Q && ctr < KYBER_N {
r[ctr] = val1 as i16;
ctr += 1;
while ctr <= KYBER_N - 8 && pos <= REJ_UNIFORM_AVX_BUFLEN - 12 {
f = _mm_loadu_si128(buf[pos..].as_ptr() as *const __m128i);
f = _mm_shuffle_epi8(f, _mm256_castsi256_si128(idx8));
t = _mm_srli_epi16(f, 4);
f = _mm_blend_epi16(f, t, 0xAA);
f = _mm_and_si128(f, _mm256_castsi256_si128(mask));
pos += 12;
t = _mm_cmpgt_epi16(_mm256_castsi256_si128(bound), f);
good = _mm_movemask_epi8(t) as usize;
let good = _pext_u32(good as u32, 0x5555) as usize;
pilo = _mm_loadl_epi64(IDX[good][..].as_ptr() as *const __m128i);
pihi = _mm_add_epi8(pilo, _mm256_castsi256_si128(ones));
pilo = _mm_unpacklo_epi8(pilo, pihi);
f = _mm_shuffle_epi8(f, pilo);
_mm_storeu_si128(r[ctr..].as_mut_ptr() as *mut __m128i, f);
ctr += _popcnt32(good as i32) as usize;
}
}
ctr
while ctr < KYBER_N && pos <= REJ_UNIFORM_AVX_BUFLEN - 3 {
val0 = (buf[pos + 0] >> 0) as u16 | ((buf[pos + 1] as u16) << 8) & 0xFFF;
val1 = (buf[pos + 1] >> 4) as u16 | ((buf[pos + 2] as u16) << 4);
pos += 3;
if (val0 as usize) < KYBER_Q {
r[ctr] = val0 as i16;
ctr += 1;
}
if (val1 as usize) < KYBER_Q && ctr < KYBER_N {
r[ctr] = val1 as i16;
ctr += 1;
}
}
ctr
}
const IDX: [[i8; 8]; 256] = [
[-1, -1, -1, -1, -1, -1, -1, -1],
[ 0, -1, -1, -1, -1, -1, -1, -1],
[ 2, -1, -1, -1, -1, -1, -1, -1],
[ 0, 2, -1, -1, -1, -1, -1, -1],
[ 4, -1, -1, -1, -1, -1, -1, -1],
[ 0, 4, -1, -1, -1, -1, -1, -1],
[ 2, 4, -1, -1, -1, -1, -1, -1],
[ 0, 2, 4, -1, -1, -1, -1, -1],
[ 6, -1, -1, -1, -1, -1, -1, -1],
[ 0, 6, -1, -1, -1, -1, -1, -1],
[ 2, 6, -1, -1, -1, -1, -1, -1],
[ 0, 2, 6, -1, -1, -1, -1, -1],
[ 4, 6, -1, -1, -1, -1, -1, -1],
[ 0, 4, 6, -1, -1, -1, -1, -1],
[ 2, 4, 6, -1, -1, -1, -1, -1],
[ 0, 2, 4, 6, -1, -1, -1, -1],
[ 8, -1, -1, -1, -1, -1, -1, -1],
[ 0, 8, -1, -1, -1, -1, -1, -1],
[ 2, 8, -1, -1, -1, -1, -1, -1],
[ 0, 2, 8, -1, -1, -1, -1, -1],
[ 4, 8, -1, -1, -1, -1, -1, -1],
[ 0, 4, 8, -1, -1, -1, -1, -1],
[ 2, 4, 8, -1, -1, -1, -1, -1],
[ 0, 2, 4, 8, -1, -1, -1, -1],
[ 6, 8, -1, -1, -1, -1, -1, -1],
[ 0, 6, 8, -1, -1, -1, -1, -1],
[ 2, 6, 8, -1, -1, -1, -1, -1],
[ 0, 2, 6, 8, -1, -1, -1, -1],
[ 4, 6, 8, -1, -1, -1, -1, -1],
[ 0, 4, 6, 8, -1, -1, -1, -1],
[ 2, 4, 6, 8, -1, -1, -1, -1],
[ 0, 2, 4, 6, 8, -1, -1, -1],
[10, -1, -1, -1, -1, -1, -1, -1],
[ 0, 10, -1, -1, -1, -1, -1, -1],
[ 2, 10, -1, -1, -1, -1, -1, -1],
[ 0, 2, 10, -1, -1, -1, -1, -1],
[ 4, 10, -1, -1, -1, -1, -1, -1],
[ 0, 4, 10, -1, -1, -1, -1, -1],
[ 2, 4, 10, -1, -1, -1, -1, -1],
[ 0, 2, 4, 10, -1, -1, -1, -1],
[ 6, 10, -1, -1, -1, -1, -1, -1],
[ 0, 6, 10, -1, -1, -1, -1, -1],
[ 2, 6, 10, -1, -1, -1, -1, -1],
[ 0, 2, 6, 10, -1, -1, -1, -1],
[ 4, 6, 10, -1, -1, -1, -1, -1],
[ 0, 4, 6, 10, -1, -1, -1, -1],
[ 2, 4, 6, 10, -1, -1, -1, -1],
[ 0, 2, 4, 6, 10, -1, -1, -1],
[ 8, 10, -1, -1, -1, -1, -1, -1],
[ 0, 8, 10, -1, -1, -1, -1, -1],
[ 2, 8, 10, -1, -1, -1, -1, -1],
[ 0, 2, 8, 10, -1, -1, -1, -1],
[ 4, 8, 10, -1, -1, -1, -1, -1],
[ 0, 4, 8, 10, -1, -1, -1, -1],
[ 2, 4, 8, 10, -1, -1, -1, -1],
[ 0, 2, 4, 8, 10, -1, -1, -1],
[ 6, 8, 10, -1, -1, -1, -1, -1],
[ 0, 6, 8, 10, -1, -1, -1, -1],
[ 2, 6, 8, 10, -1, -1, -1, -1],
[ 0, 2, 6, 8, 10, -1, -1, -1],
[ 4, 6, 8, 10, -1, -1, -1, -1],
[ 0, 4, 6, 8, 10, -1, -1, -1],
[ 2, 4, 6, 8, 10, -1, -1, -1],
[ 0, 2, 4, 6, 8, 10, -1, -1],
[12, -1, -1, -1, -1, -1, -1, -1],
[ 0, 12, -1, -1, -1, -1, -1, -1],
[ 2, 12, -1, -1, -1, -1, -1, -1],
[ 0, 2, 12, -1, -1, -1, -1, -1],
[ 4, 12, -1, -1, -1, -1, -1, -1],
[ 0, 4, 12, -1, -1, -1, -1, -1],
[ 2, 4, 12, -1, -1, -1, -1, -1],
[ 0, 2, 4, 12, -1, -1, -1, -1],
[ 6, 12, -1, -1, -1, -1, -1, -1],
[ 0, 6, 12, -1, -1, -1, -1, -1],
[ 2, 6, 12, -1, -1, -1, -1, -1],
[ 0, 2, 6, 12, -1, -1, -1, -1],
[ 4, 6, 12, -1, -1, -1, -1, -1],
[ 0, 4, 6, 12, -1, -1, -1, -1],
[ 2, 4, 6, 12, -1, -1, -1, -1],
[ 0, 2, 4, 6, 12, -1, -1, -1],
[ 8, 12, -1, -1, -1, -1, -1, -1],
[ 0, 8, 12, -1, -1, -1, -1, -1],
[ 2, 8, 12, -1, -1, -1, -1, -1],
[ 0, 2, 8, 12, -1, -1, -1, -1],
[ 4, 8, 12, -1, -1, -1, -1, -1],
[ 0, 4, 8, 12, -1, -1, -1, -1],
[ 2, 4, 8, 12, -1, -1, -1, -1],
[ 0, 2, 4, 8, 12, -1, -1, -1],
[ 6, 8, 12, -1, -1, -1, -1, -1],
[ 0, 6, 8, 12, -1, -1, -1, -1],
[ 2, 6, 8, 12, -1, -1, -1, -1],
[ 0, 2, 6, 8, 12, -1, -1, -1],
[ 4, 6, 8, 12, -1, -1, -1, -1],
[ 0, 4, 6, 8, 12, -1, -1, -1],
[ 2, 4, 6, 8, 12, -1, -1, -1],
[ 0, 2, 4, 6, 8, 12, -1, -1],
[10, 12, -1, -1, -1, -1, -1, -1],
[ 0, 10, 12, -1, -1, -1, -1, -1],
[ 2, 10, 12, -1, -1, -1, -1, -1],
[ 0, 2, 10, 12, -1, -1, -1, -1],
[ 4, 10, 12, -1, -1, -1, -1, -1],
[ 0, 4, 10, 12, -1, -1, -1, -1],
[ 2, 4, 10, 12, -1, -1, -1, -1],
[ 0, 2, 4, 10, 12, -1, -1, -1],
[ 6, 10, 12, -1, -1, -1, -1, -1],
[ 0, 6, 10, 12, -1, -1, -1, -1],
[ 2, 6, 10, 12, -1, -1, -1, -1],
[ 0, 2, 6, 10, 12, -1, -1, -1],
[ 4, 6, 10, 12, -1, -1, -1, -1],
[ 0, 4, 6, 10, 12, -1, -1, -1],
[ 2, 4, 6, 10, 12, -1, -1, -1],
[ 0, 2, 4, 6, 10, 12, -1, -1],
[ 8, 10, 12, -1, -1, -1, -1, -1],
[ 0, 8, 10, 12, -1, -1, -1, -1],
[ 2, 8, 10, 12, -1, -1, -1, -1],
[ 0, 2, 8, 10, 12, -1, -1, -1],
[ 4, 8, 10, 12, -1, -1, -1, -1],
[ 0, 4, 8, 10, 12, -1, -1, -1],
[ 2, 4, 8, 10, 12, -1, -1, -1],
[ 0, 2, 4, 8, 10, 12, -1, -1],
[ 6, 8, 10, 12, -1, -1, -1, -1],
[ 0, 6, 8, 10, 12, -1, -1, -1],
[ 2, 6, 8, 10, 12, -1, -1, -1],
[ 0, 2, 6, 8, 10, 12, -1, -1],
[ 4, 6, 8, 10, 12, -1, -1, -1],
[ 0, 4, 6, 8, 10, 12, -1, -1],
[ 2, 4, 6, 8, 10, 12, -1, -1],
[ 0, 2, 4, 6, 8, 10, 12, -1],
[14, -1, -1, -1, -1, -1, -1, -1],
[ 0, 14, -1, -1, -1, -1, -1, -1],
[ 2, 14, -1, -1, -1, -1, -1, -1],
[ 0, 2, 14, -1, -1, -1, -1, -1],
[ 4, 14, -1, -1, -1, -1, -1, -1],
[ 0, 4, 14, -1, -1, -1, -1, -1],
[ 2, 4, 14, -1, -1, -1, -1, -1],
[ 0, 2, 4, 14, -1, -1, -1, -1],
[ 6, 14, -1, -1, -1, -1, -1, -1],
[ 0, 6, 14, -1, -1, -1, -1, -1],
[ 2, 6, 14, -1, -1, -1, -1, -1],
[ 0, 2, 6, 14, -1, -1, -1, -1],
[ 4, 6, 14, -1, -1, -1, -1, -1],
[ 0, 4, 6, 14, -1, -1, -1, -1],
[ 2, 4, 6, 14, -1, -1, -1, -1],
[ 0, 2, 4, 6, 14, -1, -1, -1],
[ 8, 14, -1, -1, -1, -1, -1, -1],
[ 0, 8, 14, -1, -1, -1, -1, -1],
[ 2, 8, 14, -1, -1, -1, -1, -1],
[ 0, 2, 8, 14, -1, -1, -1, -1],
[ 4, 8, 14, -1, -1, -1, -1, -1],
[ 0, 4, 8, 14, -1, -1, -1, -1],
[ 2, 4, 8, 14, -1, -1, -1, -1],
[ 0, 2, 4, 8, 14, -1, -1, -1],
[ 6, 8, 14, -1, -1, -1, -1, -1],
[ 0, 6, 8, 14, -1, -1, -1, -1],
[ 2, 6, 8, 14, -1, -1, -1, -1],
[ 0, 2, 6, 8, 14, -1, -1, -1],
[ 4, 6, 8, 14, -1, -1, -1, -1],
[ 0, 4, 6, 8, 14, -1, -1, -1],
[ 2, 4, 6, 8, 14, -1, -1, -1],
[ 0, 2, 4, 6, 8, 14, -1, -1],
[10, 14, -1, -1, -1, -1, -1, -1],
[ 0, 10, 14, -1, -1, -1, -1, -1],
[ 2, 10, 14, -1, -1, -1, -1, -1],
[ 0, 2, 10, 14, -1, -1, -1, -1],
[ 4, 10, 14, -1, -1, -1, -1, -1],
[ 0, 4, 10, 14, -1, -1, -1, -1],
[ 2, 4, 10, 14, -1, -1, -1, -1],
[ 0, 2, 4, 10, 14, -1, -1, -1],
[ 6, 10, 14, -1, -1, -1, -1, -1],
[ 0, 6, 10, 14, -1, -1, -1, -1],
[ 2, 6, 10, 14, -1, -1, -1, -1],
[ 0, 2, 6, 10, 14, -1, -1, -1],
[ 4, 6, 10, 14, -1, -1, -1, -1],
[ 0, 4, 6, 10, 14, -1, -1, -1],
[ 2, 4, 6, 10, 14, -1, -1, -1],
[ 0, 2, 4, 6, 10, 14, -1, -1],
[ 8, 10, 14, -1, -1, -1, -1, -1],
[ 0, 8, 10, 14, -1, -1, -1, -1],
[ 2, 8, 10, 14, -1, -1, -1, -1],
[ 0, 2, 8, 10, 14, -1, -1, -1],
[ 4, 8, 10, 14, -1, -1, -1, -1],
[ 0, 4, 8, 10, 14, -1, -1, -1],
[ 2, 4, 8, 10, 14, -1, -1, -1],
[ 0, 2, 4, 8, 10, 14, -1, -1],
[ 6, 8, 10, 14, -1, -1, -1, -1],
[ 0, 6, 8, 10, 14, -1, -1, -1],
[ 2, 6, 8, 10, 14, -1, -1, -1],
[ 0, 2, 6, 8, 10, 14, -1, -1],
[ 4, 6, 8, 10, 14, -1, -1, -1],
[ 0, 4, 6, 8, 10, 14, -1, -1],
[ 2, 4, 6, 8, 10, 14, -1, -1],
[ 0, 2, 4, 6, 8, 10, 14, -1],
[12, 14, -1, -1, -1, -1, -1, -1],
[ 0, 12, 14, -1, -1, -1, -1, -1],
[ 2, 12, 14, -1, -1, -1, -1, -1],
[ 0, 2, 12, 14, -1, -1, -1, -1],
[ 4, 12, 14, -1, -1, -1, -1, -1],
[ 0, 4, 12, 14, -1, -1, -1, -1],
[ 2, 4, 12, 14, -1, -1, -1, -1],
[ 0, 2, 4, 12, 14, -1, -1, -1],
[ 6, 12, 14, -1, -1, -1, -1, -1],
[ 0, 6, 12, 14, -1, -1, -1, -1],
[ 2, 6, 12, 14, -1, -1, -1, -1],
[ 0, 2, 6, 12, 14, -1, -1, -1],
[ 4, 6, 12, 14, -1, -1, -1, -1],
[ 0, 4, 6, 12, 14, -1, -1, -1],
[ 2, 4, 6, 12, 14, -1, -1, -1],
[ 0, 2, 4, 6, 12, 14, -1, -1],
[ 8, 12, 14, -1, -1, -1, -1, -1],
[ 0, 8, 12, 14, -1, -1, -1, -1],
[ 2, 8, 12, 14, -1, -1, -1, -1],
[ 0, 2, 8, 12, 14, -1, -1, -1],
[ 4, 8, 12, 14, -1, -1, -1, -1],
[ 0, 4, 8, 12, 14, -1, -1, -1],
[ 2, 4, 8, 12, 14, -1, -1, -1],
[ 0, 2, 4, 8, 12, 14, -1, -1],
[ 6, 8, 12, 14, -1, -1, -1, -1],
[ 0, 6, 8, 12, 14, -1, -1, -1],
[ 2, 6, 8, 12, 14, -1, -1, -1],
[ 0, 2, 6, 8, 12, 14, -1, -1],
[ 4, 6, 8, 12, 14, -1, -1, -1],
[ 0, 4, 6, 8, 12, 14, -1, -1],
[ 2, 4, 6, 8, 12, 14, -1, -1],
[ 0, 2, 4, 6, 8, 12, 14, -1],
[10, 12, 14, -1, -1, -1, -1, -1],
[ 0, 10, 12, 14, -1, -1, -1, -1],
[ 2, 10, 12, 14, -1, -1, -1, -1],
[ 0, 2, 10, 12, 14, -1, -1, -1],
[ 4, 10, 12, 14, -1, -1, -1, -1],
[ 0, 4, 10, 12, 14, -1, -1, -1],
[ 2, 4, 10, 12, 14, -1, -1, -1],
[ 0, 2, 4, 10, 12, 14, -1, -1],
[ 6, 10, 12, 14, -1, -1, -1, -1],
[ 0, 6, 10, 12, 14, -1, -1, -1],
[ 2, 6, 10, 12, 14, -1, -1, -1],
[ 0, 2, 6, 10, 12, 14, -1, -1],
[ 4, 6, 10, 12, 14, -1, -1, -1],
[ 0, 4, 6, 10, 12, 14, -1, -1],
[ 2, 4, 6, 10, 12, 14, -1, -1],
[ 0, 2, 4, 6, 10, 12, 14, -1],
[ 8, 10, 12, 14, -1, -1, -1, -1],
[ 0, 8, 10, 12, 14, -1, -1, -1],
[ 2, 8, 10, 12, 14, -1, -1, -1],
[ 0, 2, 8, 10, 12, 14, -1, -1],
[ 4, 8, 10, 12, 14, -1, -1, -1],
[ 0, 4, 8, 10, 12, 14, -1, -1],
[ 2, 4, 8, 10, 12, 14, -1, -1],
[ 0, 2, 4, 8, 10, 12, 14, -1],
[ 6, 8, 10, 12, 14, -1, -1, -1],
[ 0, 6, 8, 10, 12, 14, -1, -1],
[ 2, 6, 8, 10, 12, 14, -1, -1],
[ 0, 2, 6, 8, 10, 12, 14, -1],
[ 4, 6, 8, 10, 12, 14, -1, -1],
[ 0, 4, 6, 8, 10, 12, 14, -1],
[ 2, 4, 6, 8, 10, 12, 14, -1],
[ 0, 2, 4, 6, 8, 10, 12, 14]
];
[-1, -1, -1, -1, -1, -1, -1, -1],
[0, -1, -1, -1, -1, -1, -1, -1],
[2, -1, -1, -1, -1, -1, -1, -1],
[0, 2, -1, -1, -1, -1, -1, -1],
[4, -1, -1, -1, -1, -1, -1, -1],
[0, 4, -1, -1, -1, -1, -1, -1],
[2, 4, -1, -1, -1, -1, -1, -1],
[0, 2, 4, -1, -1, -1, -1, -1],
[6, -1, -1, -1, -1, -1, -1, -1],
[0, 6, -1, -1, -1, -1, -1, -1],
[2, 6, -1, -1, -1, -1, -1, -1],
[0, 2, 6, -1, -1, -1, -1, -1],
[4, 6, -1, -1, -1, -1, -1, -1],
[0, 4, 6, -1, -1, -1, -1, -1],
[2, 4, 6, -1, -1, -1, -1, -1],
[0, 2, 4, 6, -1, -1, -1, -1],
[8, -1, -1, -1, -1, -1, -1, -1],
[0, 8, -1, -1, -1, -1, -1, -1],
[2, 8, -1, -1, -1, -1, -1, -1],
[0, 2, 8, -1, -1, -1, -1, -1],
[4, 8, -1, -1, -1, -1, -1, -1],
[0, 4, 8, -1, -1, -1, -1, -1],
[2, 4, 8, -1, -1, -1, -1, -1],
[0, 2, 4, 8, -1, -1, -1, -1],
[6, 8, -1, -1, -1, -1, -1, -1],
[0, 6, 8, -1, -1, -1, -1, -1],
[2, 6, 8, -1, -1, -1, -1, -1],
[0, 2, 6, 8, -1, -1, -1, -1],
[4, 6, 8, -1, -1, -1, -1, -1],
[0, 4, 6, 8, -1, -1, -1, -1],
[2, 4, 6, 8, -1, -1, -1, -1],
[0, 2, 4, 6, 8, -1, -1, -1],
[10, -1, -1, -1, -1, -1, -1, -1],
[0, 10, -1, -1, -1, -1, -1, -1],
[2, 10, -1, -1, -1, -1, -1, -1],
[0, 2, 10, -1, -1, -1, -1, -1],
[4, 10, -1, -1, -1, -1, -1, -1],
[0, 4, 10, -1, -1, -1, -1, -1],
[2, 4, 10, -1, -1, -1, -1, -1],
[0, 2, 4, 10, -1, -1, -1, -1],
[6, 10, -1, -1, -1, -1, -1, -1],
[0, 6, 10, -1, -1, -1, -1, -1],
[2, 6, 10, -1, -1, -1, -1, -1],
[0, 2, 6, 10, -1, -1, -1, -1],
[4, 6, 10, -1, -1, -1, -1, -1],
[0, 4, 6, 10, -1, -1, -1, -1],
[2, 4, 6, 10, -1, -1, -1, -1],
[0, 2, 4, 6, 10, -1, -1, -1],
[8, 10, -1, -1, -1, -1, -1, -1],
[0, 8, 10, -1, -1, -1, -1, -1],
[2, 8, 10, -1, -1, -1, -1, -1],
[0, 2, 8, 10, -1, -1, -1, -1],
[4, 8, 10, -1, -1, -1, -1, -1],
[0, 4, 8, 10, -1, -1, -1, -1],
[2, 4, 8, 10, -1, -1, -1, -1],
[0, 2, 4, 8, 10, -1, -1, -1],
[6, 8, 10, -1, -1, -1, -1, -1],
[0, 6, 8, 10, -1, -1, -1, -1],
[2, 6, 8, 10, -1, -1, -1, -1],
[0, 2, 6, 8, 10, -1, -1, -1],
[4, 6, 8, 10, -1, -1, -1, -1],
[0, 4, 6, 8, 10, -1, -1, -1],
[2, 4, 6, 8, 10, -1, -1, -1],
[0, 2, 4, 6, 8, 10, -1, -1],
[12, -1, -1, -1, -1, -1, -1, -1],
[0, 12, -1, -1, -1, -1, -1, -1],
[2, 12, -1, -1, -1, -1, -1, -1],
[0, 2, 12, -1, -1, -1, -1, -1],
[4, 12, -1, -1, -1, -1, -1, -1],
[0, 4, 12, -1, -1, -1, -1, -1],
[2, 4, 12, -1, -1, -1, -1, -1],
[0, 2, 4, 12, -1, -1, -1, -1],
[6, 12, -1, -1, -1, -1, -1, -1],
[0, 6, 12, -1, -1, -1, -1, -1],
[2, 6, 12, -1, -1, -1, -1, -1],
[0, 2, 6, 12, -1, -1, -1, -1],
[4, 6, 12, -1, -1, -1, -1, -1],
[0, 4, 6, 12, -1, -1, -1, -1],
[2, 4, 6, 12, -1, -1, -1, -1],
[0, 2, 4, 6, 12, -1, -1, -1],
[8, 12, -1, -1, -1, -1, -1, -1],
[0, 8, 12, -1, -1, -1, -1, -1],
[2, 8, 12, -1, -1, -1, -1, -1],
[0, 2, 8, 12, -1, -1, -1, -1],
[4, 8, 12, -1, -1, -1, -1, -1],
[0, 4, 8, 12, -1, -1, -1, -1],
[2, 4, 8, 12, -1, -1, -1, -1],
[0, 2, 4, 8, 12, -1, -1, -1],
[6, 8, 12, -1, -1, -1, -1, -1],
[0, 6, 8, 12, -1, -1, -1, -1],
[2, 6, 8, 12, -1, -1, -1, -1],
[0, 2, 6, 8, 12, -1, -1, -1],
[4, 6, 8, 12, -1, -1, -1, -1],
[0, 4, 6, 8, 12, -1, -1, -1],
[2, 4, 6, 8, 12, -1, -1, -1],
[0, 2, 4, 6, 8, 12, -1, -1],
[10, 12, -1, -1, -1, -1, -1, -1],
[0, 10, 12, -1, -1, -1, -1, -1],
[2, 10, 12, -1, -1, -1, -1, -1],
[0, 2, 10, 12, -1, -1, -1, -1],
[4, 10, 12, -1, -1, -1, -1, -1],
[0, 4, 10, 12, -1, -1, -1, -1],
[2, 4, 10, 12, -1, -1, -1, -1],
[0, 2, 4, 10, 12, -1, -1, -1],
[6, 10, 12, -1, -1, -1, -1, -1],
[0, 6, 10, 12, -1, -1, -1, -1],
[2, 6, 10, 12, -1, -1, -1, -1],
[0, 2, 6, 10, 12, -1, -1, -1],
[4, 6, 10, 12, -1, -1, -1, -1],
[0, 4, 6, 10, 12, -1, -1, -1],
[2, 4, 6, 10, 12, -1, -1, -1],
[0, 2, 4, 6, 10, 12, -1, -1],
[8, 10, 12, -1, -1, -1, -1, -1],
[0, 8, 10, 12, -1, -1, -1, -1],
[2, 8, 10, 12, -1, -1, -1, -1],
[0, 2, 8, 10, 12, -1, -1, -1],
[4, 8, 10, 12, -1, -1, -1, -1],
[0, 4, 8, 10, 12, -1, -1, -1],
[2, 4, 8, 10, 12, -1, -1, -1],
[0, 2, 4, 8, 10, 12, -1, -1],
[6, 8, 10, 12, -1, -1, -1, -1],
[0, 6, 8, 10, 12, -1, -1, -1],
[2, 6, 8, 10, 12, -1, -1, -1],
[0, 2, 6, 8, 10, 12, -1, -1],
[4, 6, 8, 10, 12, -1, -1, -1],
[0, 4, 6, 8, 10, 12, -1, -1],
[2, 4, 6, 8, 10, 12, -1, -1],
[0, 2, 4, 6, 8, 10, 12, -1],
[14, -1, -1, -1, -1, -1, -1, -1],
[0, 14, -1, -1, -1, -1, -1, -1],
[2, 14, -1, -1, -1, -1, -1, -1],
[0, 2, 14, -1, -1, -1, -1, -1],
[4, 14, -1, -1, -1, -1, -1, -1],
[0, 4, 14, -1, -1, -1, -1, -1],
[2, 4, 14, -1, -1, -1, -1, -1],
[0, 2, 4, 14, -1, -1, -1, -1],
[6, 14, -1, -1, -1, -1, -1, -1],
[0, 6, 14, -1, -1, -1, -1, -1],
[2, 6, 14, -1, -1, -1, -1, -1],
[0, 2, 6, 14, -1, -1, -1, -1],
[4, 6, 14, -1, -1, -1, -1, -1],
[0, 4, 6, 14, -1, -1, -1, -1],
[2, 4, 6, 14, -1, -1, -1, -1],
[0, 2, 4, 6, 14, -1, -1, -1],
[8, 14, -1, -1, -1, -1, -1, -1],
[0, 8, 14, -1, -1, -1, -1, -1],
[2, 8, 14, -1, -1, -1, -1, -1],
[0, 2, 8, 14, -1, -1, -1, -1],
[4, 8, 14, -1, -1, -1, -1, -1],
[0, 4, 8, 14, -1, -1, -1, -1],
[2, 4, 8, 14, -1, -1, -1, -1],
[0, 2, 4, 8, 14, -1, -1, -1],
[6, 8, 14, -1, -1, -1, -1, -1],
[0, 6, 8, 14, -1, -1, -1, -1],
[2, 6, 8, 14, -1, -1, -1, -1],
[0, 2, 6, 8, 14, -1, -1, -1],
[4, 6, 8, 14, -1, -1, -1, -1],
[0, 4, 6, 8, 14, -1, -1, -1],
[2, 4, 6, 8, 14, -1, -1, -1],
[0, 2, 4, 6, 8, 14, -1, -1],
[10, 14, -1, -1, -1, -1, -1, -1],
[0, 10, 14, -1, -1, -1, -1, -1],
[2, 10, 14, -1, -1, -1, -1, -1],
[0, 2, 10, 14, -1, -1, -1, -1],
[4, 10, 14, -1, -1, -1, -1, -1],
[0, 4, 10, 14, -1, -1, -1, -1],
[2, 4, 10, 14, -1, -1, -1, -1],
[0, 2, 4, 10, 14, -1, -1, -1],
[6, 10, 14, -1, -1, -1, -1, -1],
[0, 6, 10, 14, -1, -1, -1, -1],
[2, 6, 10, 14, -1, -1, -1, -1],
[0, 2, 6, 10, 14, -1, -1, -1],
[4, 6, 10, 14, -1, -1, -1, -1],
[0, 4, 6, 10, 14, -1, -1, -1],
[2, 4, 6, 10, 14, -1, -1, -1],
[0, 2, 4, 6, 10, 14, -1, -1],
[8, 10, 14, -1, -1, -1, -1, -1],
[0, 8, 10, 14, -1, -1, -1, -1],
[2, 8, 10, 14, -1, -1, -1, -1],
[0, 2, 8, 10, 14, -1, -1, -1],
[4, 8, 10, 14, -1, -1, -1, -1],
[0, 4, 8, 10, 14, -1, -1, -1],
[2, 4, 8, 10, 14, -1, -1, -1],
[0, 2, 4, 8, 10, 14, -1, -1],
[6, 8, 10, 14, -1, -1, -1, -1],
[0, 6, 8, 10, 14, -1, -1, -1],
[2, 6, 8, 10, 14, -1, -1, -1],
[0, 2, 6, 8, 10, 14, -1, -1],
[4, 6, 8, 10, 14, -1, -1, -1],
[0, 4, 6, 8, 10, 14, -1, -1],
[2, 4, 6, 8, 10, 14, -1, -1],
[0, 2, 4, 6, 8, 10, 14, -1],
[12, 14, -1, -1, -1, -1, -1, -1],
[0, 12, 14, -1, -1, -1, -1, -1],
[2, 12, 14, -1, -1, -1, -1, -1],
[0, 2, 12, 14, -1, -1, -1, -1],
[4, 12, 14, -1, -1, -1, -1, -1],
[0, 4, 12, 14, -1, -1, -1, -1],
[2, 4, 12, 14, -1, -1, -1, -1],
[0, 2, 4, 12, 14, -1, -1, -1],
[6, 12, 14, -1, -1, -1, -1, -1],
[0, 6, 12, 14, -1, -1, -1, -1],
[2, 6, 12, 14, -1, -1, -1, -1],
[0, 2, 6, 12, 14, -1, -1, -1],
[4, 6, 12, 14, -1, -1, -1, -1],
[0, 4, 6, 12, 14, -1, -1, -1],
[2, 4, 6, 12, 14, -1, -1, -1],
[0, 2, 4, 6, 12, 14, -1, -1],
[8, 12, 14, -1, -1, -1, -1, -1],
[0, 8, 12, 14, -1, -1, -1, -1],
[2, 8, 12, 14, -1, -1, -1, -1],
[0, 2, 8, 12, 14, -1, -1, -1],
[4, 8, 12, 14, -1, -1, -1, -1],
[0, 4, 8, 12, 14, -1, -1, -1],
[2, 4, 8, 12, 14, -1, -1, -1],
[0, 2, 4, 8, 12, 14, -1, -1],
[6, 8, 12, 14, -1, -1, -1, -1],
[0, 6, 8, 12, 14, -1, -1, -1],
[2, 6, 8, 12, 14, -1, -1, -1],
[0, 2, 6, 8, 12, 14, -1, -1],
[4, 6, 8, 12, 14, -1, -1, -1],
[0, 4, 6, 8, 12, 14, -1, -1],
[2, 4, 6, 8, 12, 14, -1, -1],
[0, 2, 4, 6, 8, 12, 14, -1],
[10, 12, 14, -1, -1, -1, -1, -1],
[0, 10, 12, 14, -1, -1, -1, -1],
[2, 10, 12, 14, -1, -1, -1, -1],
[0, 2, 10, 12, 14, -1, -1, -1],
[4, 10, 12, 14, -1, -1, -1, -1],
[0, 4, 10, 12, 14, -1, -1, -1],
[2, 4, 10, 12, 14, -1, -1, -1],
[0, 2, 4, 10, 12, 14, -1, -1],
[6, 10, 12, 14, -1, -1, -1, -1],
[0, 6, 10, 12, 14, -1, -1, -1],
[2, 6, 10, 12, 14, -1, -1, -1],
[0, 2, 6, 10, 12, 14, -1, -1],
[4, 6, 10, 12, 14, -1, -1, -1],
[0, 4, 6, 10, 12, 14, -1, -1],
[2, 4, 6, 10, 12, 14, -1, -1],
[0, 2, 4, 6, 10, 12, 14, -1],
[8, 10, 12, 14, -1, -1, -1, -1],
[0, 8, 10, 12, 14, -1, -1, -1],
[2, 8, 10, 12, 14, -1, -1, -1],
[0, 2, 8, 10, 12, 14, -1, -1],
[4, 8, 10, 12, 14, -1, -1, -1],
[0, 4, 8, 10, 12, 14, -1, -1],
[2, 4, 8, 10, 12, 14, -1, -1],
[0, 2, 4, 8, 10, 12, 14, -1],
[6, 8, 10, 12, 14, -1, -1, -1],
[0, 6, 8, 10, 12, 14, -1, -1],
[2, 6, 8, 10, 12, 14, -1, -1],
[0, 2, 6, 8, 10, 12, 14, -1],
[4, 6, 8, 10, 12, 14, -1, -1],
[0, 4, 6, 8, 10, 12, 14, -1],
[2, 4, 6, 8, 10, 12, 14, -1],
[0, 2, 4, 6, 8, 10, 12, 14],
];

72
third_party/kyber/src/avx2/verify.rs vendored
View file

@ -1,43 +1,41 @@
use core::arch::x86_64::*;
pub fn verify(a: &[u8], b: &[u8], mut len: usize) -> u8
{
let (mut f, mut g);
let mut r: u64;
unsafe {
let mut h = _mm256_setzero_si256();
for i in 0..(len/32) {
f = _mm256_loadu_si256(a[32*i..].as_ptr() as *const __m256i);
g = _mm256_loadu_si256(b[32*i..].as_ptr() as *const __m256i);
f = _mm256_xor_si256(f,g);
h = _mm256_or_si256(h,f);
pub fn verify(a: &[u8], b: &[u8], mut len: usize) -> u8 {
let (mut f, mut g);
let mut r: u64;
unsafe {
let mut h = _mm256_setzero_si256();
for i in 0..(len / 32) {
f = _mm256_loadu_si256(a[32 * i..].as_ptr() as *const __m256i);
g = _mm256_loadu_si256(b[32 * i..].as_ptr() as *const __m256i);
f = _mm256_xor_si256(f, g);
h = _mm256_or_si256(h, f);
}
r = 1 - _mm256_testz_si256(h, h) as u64;
}
r = 1 - _mm256_testz_si256(h,h) as u64;
}
let idx = 32*(len/32);
len -= idx;
for i in 0..len {
r |= (a[idx+i] ^ b[idx+i]) as u64;
}
(r.wrapping_neg() >> 63) as u8
let idx = 32 * (len / 32);
len -= idx;
for i in 0..len {
r |= (a[idx + i] ^ b[idx + i]) as u64;
}
(r.wrapping_neg() >> 63) as u8
}
pub fn cmov(r: &mut[u8], x: &[u8], mut len: usize, mut b: u8)
{
let (mut xvec, mut rvec);
unsafe {
let bvec = _mm256_set1_epi64x(b as i64);
for i in 0..(len/32) {
rvec = _mm256_loadu_si256(r[32*i..].as_ptr() as *const __m256i);
xvec = _mm256_loadu_si256(x[32*i..].as_ptr() as *const __m256i);
rvec = _mm256_blendv_epi8(rvec,xvec,bvec);
_mm256_storeu_si256(r[32*i..].as_mut_ptr() as *mut __m256i,rvec);
pub fn cmov(r: &mut [u8], x: &[u8], mut len: usize, mut b: u8) {
let (mut xvec, mut rvec);
unsafe {
let bvec = _mm256_set1_epi64x(b as i64);
for i in 0..(len / 32) {
rvec = _mm256_loadu_si256(r[32 * i..].as_ptr() as *const __m256i);
xvec = _mm256_loadu_si256(x[32 * i..].as_ptr() as *const __m256i);
rvec = _mm256_blendv_epi8(rvec, xvec, bvec);
_mm256_storeu_si256(r[32 * i..].as_mut_ptr() as *mut __m256i, rvec);
}
}
}
let idx = 32*(len/32);
len -= idx;
b = b.wrapping_neg();
for i in 0..len {
r[idx+i] ^= b & (x[idx+i] ^ r[idx+i]);
}
}
let idx = 32 * (len / 32);
len -= idx;
b = b.wrapping_neg();
for i in 0..len {
r[idx + i] ^= b & (x[idx + i] ^ r[idx + i]);
}
}

22
third_party/kyber/src/error.rs vendored
View file

@ -1,19 +1,19 @@
#[derive(Debug, PartialEq)]
/// Error types for the failure modes
pub enum KyberError {
/// One or more inputs to a function are incorrectly sized. A likely cause of this is two parties using different security
/// levels while trying to negotiate a key exchange.
InvalidInput,
/// The ciphertext was unable to be authenticated.
/// The shared secret was not decapsulated.
Decapsulation,
/// One or more inputs to a function are incorrectly sized. A likely cause of this is two parties using different security
/// levels while trying to negotiate a key exchange.
InvalidInput,
/// The ciphertext was unable to be authenticated.
/// The shared secret was not decapsulated.
Decapsulation,
}
impl core::fmt::Display for KyberError {
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
match *self {
KyberError::InvalidInput => write!(f, "Function input is of incorrect length"),
KyberError::Decapsulation => write!(f, "Decapsulation Failure, unable to obtain shared secret from ciphertext"),
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
match *self {
KyberError::InvalidInput => write!(f, "Function input is of incorrect length"),
KyberError::Decapsulation => write!(f, "Decapsulation Failure, unable to obtain shared secret from ciphertext"),
}
}
}
}

241
third_party/kyber/src/kem.rs vendored
View file

@ -1,128 +1,113 @@
#[cfg(not(feature="KATs"))]
use crate::rng::randombytes;
use rand_core::{RngCore, CryptoRng};
use crate::{
params::*,
indcpa::*,
symmetric::*,
error::KyberError,
verify::*
};
// Name: crypto_kem_keypair
//
// Description: Generates public and private key
// for CCA-secure Kyber key encapsulation mechanism
//
// Arguments: - [u8] pk: output public key (an already allocated array of CRYPTO_PUBLICKEYBYTES bytes)
// - [u8] sk: output private key (an already allocated array of CRYPTO_SECRETKEYBYTES bytes)
pub fn crypto_kem_keypair<R>(
pk: &mut[u8], sk: &mut[u8], _rng: &mut R, _seed: Option<(&[u8], &[u8])>
)
where R: RngCore + CryptoRng
{
const PK_START: usize = KYBER_SECRETKEYBYTES - (2 * KYBER_SYMBYTES);
const SK_START: usize = KYBER_SECRETKEYBYTES-KYBER_SYMBYTES;
const END: usize = KYBER_INDCPA_PUBLICKEYBYTES + KYBER_INDCPA_SECRETKEYBYTES;
indcpa_keypair(pk, sk, _seed, _rng);
sk[KYBER_INDCPA_SECRETKEYBYTES..END]
.copy_from_slice(&pk[..KYBER_INDCPA_PUBLICKEYBYTES]);
hash_h(&mut sk[PK_START..], pk, KYBER_PUBLICKEYBYTES);
#[cfg(feature="KATs")]
sk[SK_START..].copy_from_slice(&_seed.expect("KATs feature only for testing").1);
#[cfg(not(feature="KATs"))]
randombytes(&mut sk[SK_START..],KYBER_SYMBYTES, _rng);
}
// Name: crypto_kem_enc
//
// Description: Generates cipher text and shared
// secret for given public key
//
// Arguments: - [u8] ct: output cipher text (an already allocated array of CRYPTO_CIPHERTEXTBYTES bytes)
// - [u8] ss: output shared secret (an already allocated array of CRYPTO_BYTES bytes)
// - const [u8] pk: input public key (an already allocated array of CRYPTO_PUBLICKEYBYTES bytes)
pub fn crypto_kem_enc<R>(
ct: &mut[u8], ss: &mut[u8], pk: &[u8], _rng: &mut R,_seed: Option<&[u8]>
)
where R: RngCore + CryptoRng
{
let mut kr = [0u8; 2*KYBER_SYMBYTES];
let mut buf = [0u8; 2*KYBER_SYMBYTES];
let mut randbuf = [0u8; 2*KYBER_SYMBYTES];
#[cfg(not(feature="KATs"))]
randombytes(&mut randbuf, KYBER_SYMBYTES, _rng);
// Deterministic randbuf for KAT's
#[cfg(feature="KATs")]
randbuf[..KYBER_SYMBYTES]
.copy_from_slice(&_seed.expect("KATs feature only works with `cargo test`"));
// Don't release system RNG output
hash_h(&mut buf, &randbuf, KYBER_SYMBYTES);
// Multitarget countermeasure for coins + contributory KEM
hash_h(&mut buf[KYBER_SYMBYTES..], pk, KYBER_PUBLICKEYBYTES);
hash_g(&mut kr, &buf, 2*KYBER_SYMBYTES);
// coins are in kr[KYBER_SYMBYTES..]
indcpa_enc(ct, &buf, pk, &kr[KYBER_SYMBYTES..]);
// overwrite coins in kr with H(c)
hash_h(&mut kr[KYBER_SYMBYTES..], ct, KYBER_CIPHERTEXTBYTES);
// hash concatenation of pre-k and H(c) to k
kdf(ss, &kr, 2*KYBER_SYMBYTES);
}
// Name: crypto_kem_dec
//
// Description: Generates shared secret for given
// cipher text and private key
//
// Arguments: - [u8] ss: output shared secret (an already allocated array of CRYPTO_BYTES bytes)
// - const [u8] ct: input cipher text (an already allocated array of CRYPTO_CIPHERTEXTBYTES bytes)
// - const [u8] sk: input private key (an already allocated array of CRYPTO_SECRETKEYBYTES bytes)
//
// On failure, ss will contain a pseudo-random value.
pub fn crypto_kem_dec(
ss: &mut[u8], ct: &[u8], sk: &[u8]
)
-> Result<(), KyberError>
{
let mut buf = [0u8; 2*KYBER_SYMBYTES];
let mut kr = [0u8; 2*KYBER_SYMBYTES];
let mut cmp = [0u8; KYBER_CIPHERTEXTBYTES];
let mut pk = [0u8; KYBER_INDCPA_PUBLICKEYBYTES + 2*KYBER_SYMBYTES];
pk.copy_from_slice(&sk[KYBER_INDCPA_SECRETKEYBYTES..]);
indcpa_dec(&mut buf, ct, sk);
// Multitarget countermeasure for coins + contributory KEM
const START: usize = KYBER_SECRETKEYBYTES-2*KYBER_SYMBYTES;
const END: usize = KYBER_SECRETKEYBYTES-KYBER_SYMBYTES;
buf[KYBER_SYMBYTES..].copy_from_slice(&sk[START..END]);
hash_g(&mut kr, &buf, 2*KYBER_SYMBYTES);
// coins are in kr[KYBER_SYMBYTES..]
indcpa_enc(&mut cmp, &buf, &pk, &kr[KYBER_SYMBYTES..]);
let fail = verify(ct, &cmp, KYBER_CIPHERTEXTBYTES);
// overwrite coins in kr with H(c)
hash_h(&mut kr[KYBER_SYMBYTES..], ct, KYBER_CIPHERTEXTBYTES);
// Overwrite pre-k with z on re-encryption failure
cmov(&mut kr, &sk[END..], KYBER_SYMBYTES, fail);
// hash concatenation of pre-k and H(c) to k
kdf(ss, &kr, 2*KYBER_SYMBYTES);
match fail {
0 => Ok(()),
_ => Err(KyberError::Decapsulation)
}
}
#[cfg(not(feature = "KATs"))]
use crate::rng::randombytes;
use crate::{error::KyberError, indcpa::*, params::*, symmetric::*, verify::*};
use rand_core::{CryptoRng, RngCore};
// Name: crypto_kem_keypair
//
// Description: Generates public and private key
// for CCA-secure Kyber key encapsulation mechanism
//
// Arguments: - [u8] pk: output public key (an already allocated array of CRYPTO_PUBLICKEYBYTES bytes)
// - [u8] sk: output private key (an already allocated array of CRYPTO_SECRETKEYBYTES bytes)
pub fn crypto_kem_keypair<R>(pk: &mut [u8], sk: &mut [u8], _rng: &mut R, _seed: Option<(&[u8], &[u8])>)
where
R: RngCore + CryptoRng,
{
const PK_START: usize = KYBER_SECRETKEYBYTES - (2 * KYBER_SYMBYTES);
const SK_START: usize = KYBER_SECRETKEYBYTES - KYBER_SYMBYTES;
const END: usize = KYBER_INDCPA_PUBLICKEYBYTES + KYBER_INDCPA_SECRETKEYBYTES;
indcpa_keypair(pk, sk, _seed, _rng);
sk[KYBER_INDCPA_SECRETKEYBYTES..END].copy_from_slice(&pk[..KYBER_INDCPA_PUBLICKEYBYTES]);
hash_h(&mut sk[PK_START..], pk, KYBER_PUBLICKEYBYTES);
#[cfg(feature = "KATs")]
sk[SK_START..].copy_from_slice(&_seed.expect("KATs feature only for testing").1);
#[cfg(not(feature = "KATs"))]
randombytes(&mut sk[SK_START..], KYBER_SYMBYTES, _rng);
}
// Name: crypto_kem_enc
//
// Description: Generates cipher text and shared
// secret for given public key
//
// Arguments: - [u8] ct: output cipher text (an already allocated array of CRYPTO_CIPHERTEXTBYTES bytes)
// - [u8] ss: output shared secret (an already allocated array of CRYPTO_BYTES bytes)
// - const [u8] pk: input public key (an already allocated array of CRYPTO_PUBLICKEYBYTES bytes)
pub fn crypto_kem_enc<R>(ct: &mut [u8], ss: &mut [u8], pk: &[u8], _rng: &mut R, _seed: Option<&[u8]>)
where
R: RngCore + CryptoRng,
{
let mut kr = [0u8; 2 * KYBER_SYMBYTES];
let mut buf = [0u8; 2 * KYBER_SYMBYTES];
let mut randbuf = [0u8; 2 * KYBER_SYMBYTES];
#[cfg(not(feature = "KATs"))]
randombytes(&mut randbuf, KYBER_SYMBYTES, _rng);
// Deterministic randbuf for KAT's
#[cfg(feature = "KATs")]
randbuf[..KYBER_SYMBYTES].copy_from_slice(&_seed.expect("KATs feature only works with `cargo test`"));
// Don't release system RNG output
hash_h(&mut buf, &randbuf, KYBER_SYMBYTES);
// Multitarget countermeasure for coins + contributory KEM
hash_h(&mut buf[KYBER_SYMBYTES..], pk, KYBER_PUBLICKEYBYTES);
hash_g(&mut kr, &buf, 2 * KYBER_SYMBYTES);
// coins are in kr[KYBER_SYMBYTES..]
indcpa_enc(ct, &buf, pk, &kr[KYBER_SYMBYTES..]);
// overwrite coins in kr with H(c)
hash_h(&mut kr[KYBER_SYMBYTES..], ct, KYBER_CIPHERTEXTBYTES);
// hash concatenation of pre-k and H(c) to k
kdf(ss, &kr, 2 * KYBER_SYMBYTES);
}
// Name: crypto_kem_dec
//
// Description: Generates shared secret for given
// cipher text and private key
//
// Arguments: - [u8] ss: output shared secret (an already allocated array of CRYPTO_BYTES bytes)
// - const [u8] ct: input cipher text (an already allocated array of CRYPTO_CIPHERTEXTBYTES bytes)
// - const [u8] sk: input private key (an already allocated array of CRYPTO_SECRETKEYBYTES bytes)
//
// On failure, ss will contain a pseudo-random value.
pub fn crypto_kem_dec(ss: &mut [u8], ct: &[u8], sk: &[u8]) -> Result<(), KyberError> {
let mut buf = [0u8; 2 * KYBER_SYMBYTES];
let mut kr = [0u8; 2 * KYBER_SYMBYTES];
let mut cmp = [0u8; KYBER_CIPHERTEXTBYTES];
let mut pk = [0u8; KYBER_INDCPA_PUBLICKEYBYTES + 2 * KYBER_SYMBYTES];
pk.copy_from_slice(&sk[KYBER_INDCPA_SECRETKEYBYTES..]);
indcpa_dec(&mut buf, ct, sk);
// Multitarget countermeasure for coins + contributory KEM
const START: usize = KYBER_SECRETKEYBYTES - 2 * KYBER_SYMBYTES;
const END: usize = KYBER_SECRETKEYBYTES - KYBER_SYMBYTES;
buf[KYBER_SYMBYTES..].copy_from_slice(&sk[START..END]);
hash_g(&mut kr, &buf, 2 * KYBER_SYMBYTES);
// coins are in kr[KYBER_SYMBYTES..]
indcpa_enc(&mut cmp, &buf, &pk, &kr[KYBER_SYMBYTES..]);
let fail = verify(ct, &cmp, KYBER_CIPHERTEXTBYTES);
// overwrite coins in kr with H(c)
hash_h(&mut kr[KYBER_SYMBYTES..], ct, KYBER_CIPHERTEXTBYTES);
// Overwrite pre-k with z on re-encryption failure
cmov(&mut kr, &sk[END..], KYBER_SYMBYTES, fail);
// hash concatenation of pre-k and H(c) to k
kdf(ss, &kr, 2 * KYBER_SYMBYTES);
match fail {
0 => Ok(()),
_ => Err(KyberError::Decapsulation),
}
}

513
third_party/kyber/src/kex.rs vendored
View file

@ -1,22 +1,17 @@
use rand_core::{RngCore, CryptoRng};
use crate::{
kem::*,
symmetric::kdf,
params::*,
KyberError
};
use crate::{kem::*, params::*, symmetric::kdf, KyberError};
use rand_core::{CryptoRng, RngCore};
/// Unilateral Key Exchange Initiation Byte Length
/// Unilateral Key Exchange Initiation Byte Length
pub const UAKE_INIT_BYTES: usize = KYBER_PUBLICKEYBYTES + KYBER_CIPHERTEXTBYTES;
/// Unilateral Key Exchange Response Byte Length
pub const UAKE_RESPONSE_BYTES: usize = KYBER_CIPHERTEXTBYTES;
/// Mutual Key Exchange Initiation Byte Length
/// Mutual Key Exchange Initiation Byte Length
pub const AKE_INIT_BYTES: usize = KYBER_PUBLICKEYBYTES + KYBER_CIPHERTEXTBYTES;
/// Mutual Key Exchange Response Byte Length
pub const AKE_RESPONSE_BYTES: usize = 2 * KYBER_CIPHERTEXTBYTES;
/// Result of encapsulating a public key which includes the ciphertext and shared secret
pub type Encapsulated = Result<([u8; KYBER_CIPHERTEXTBYTES], [u8; KYBER_SSBYTES]), KyberError>;
pub type Encapsulated = Result<([u8; KYBER_CIPHERTEXTBYTES], [u8; KYBER_SSBYTES]), KyberError>;
/// The result of decapsulating a ciphertext which produces a shared secret when confirmed
pub type Decapsulated = Result<[u8; KYBER_SSBYTES], KyberError>;
/// Kyber public key
@ -24,15 +19,15 @@ pub type PublicKey = [u8; KYBER_PUBLICKEYBYTES];
/// Kyber secret key
pub type SecretKey = [u8; KYBER_SECRETKEYBYTES];
/// Kyber Shared Secret
pub type SharedSecret = [u8; KYBER_SSBYTES];
pub type SharedSecret = [u8; KYBER_SSBYTES];
/// Bytes to send when initiating a unilateral key exchange
pub type UakeSendInit = [u8; UAKE_INIT_BYTES];
pub type UakeSendInit = [u8; UAKE_INIT_BYTES];
/// Bytes to send when responding to a unilateral key exchange
pub type UakeSendResponse = [u8; UAKE_RESPONSE_BYTES];
pub type UakeSendResponse = [u8; UAKE_RESPONSE_BYTES];
/// Bytes to send when initiating a mutual key exchange
pub type AkeSendInit = [u8; AKE_INIT_BYTES];
pub type AkeSendInit = [u8; AKE_INIT_BYTES];
/// Bytes to send when responding to a mutual key exchange
pub type AkeSendResponse = [u8; AKE_RESPONSE_BYTES];
pub type AkeSendResponse = [u8; AKE_RESPONSE_BYTES];
// Ephemeral keys
type TempKey = [u8; KYBER_SSBYTES];
@ -40,349 +35,293 @@ type Eska = [u8; KYBER_SECRETKEYBYTES];
// TODO: implement zeroise feature
/// Used for unilaterally authenticated key exchange between two parties.
///
///
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> {
/// let mut rng = rand::thread_rng();
///
///
/// let mut alice = Uake::new();
/// let mut bob = Uake::new();
/// let bob_keys = keypair(&mut rng);
///
///
/// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng)?;
/// let client_confirm = alice.client_confirm(server_send);
///
///
/// assert_eq!(alice.shared_secret, bob.shared_secret);
/// # Ok(()) }
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub struct Uake {
/// The resulting shared secret from a key exchange
pub shared_secret: SharedSecret,
/// Sent when initiating a key exchange
send_a: UakeSendInit,
/// Response to a key exchange initiation
send_b: UakeSendResponse,
// Epheremal keys
temp_key: TempKey,
eska: Eska
/// The resulting shared secret from a key exchange
pub shared_secret: SharedSecret,
/// Sent when initiating a key exchange
send_a: UakeSendInit,
/// Response to a key exchange initiation
send_b: UakeSendResponse,
// Epheremal keys
temp_key: TempKey,
eska: Eska,
}
impl Default for Uake {
fn default() -> Self {
Uake {
shared_secret: [0u8; KYBER_SSBYTES],
send_a: [0u8; UAKE_INIT_BYTES],
send_b: [0u8; UAKE_RESPONSE_BYTES],
temp_key: [0u8; KYBER_SSBYTES],
eska: [0u8; KYBER_SECRETKEYBYTES],
fn default() -> Self {
Uake {
shared_secret: [0u8; KYBER_SSBYTES],
send_a: [0u8; UAKE_INIT_BYTES],
send_b: [0u8; UAKE_RESPONSE_BYTES],
temp_key: [0u8; KYBER_SSBYTES],
eska: [0u8; KYBER_SECRETKEYBYTES],
}
}
}
}
impl Uake {
/// Builds new UAKE struct
/// ```
/// # use pqc_kyber::Uake;
/// let mut kex = Uake::new();
/// ```
pub fn new() -> Self {
Self::default()
}
/// Builds new UAKE struct
/// ```
/// # use pqc_kyber::Uake;
/// let mut kex = Uake::new();
/// ```
pub fn new() -> Self {
Self::default()
}
/// Initiates a Unilaterally Authenticated Key Exchange.
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> {
/// let mut rng = rand::thread_rng();
/// let mut alice = Uake::new();
/// let bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// # Ok(()) }
/// ```
pub fn client_init<R>(&mut self, pubkey: &PublicKey, rng: &mut R)
-> UakeSendInit
where R: CryptoRng + RngCore
{
uake_init_a(
&mut self.send_a, &mut self.temp_key,
&mut self.eska, pubkey, rng
);
self.send_a
}
/// Initiates a Unilaterally Authenticated Key Exchange.
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> {
/// let mut rng = rand::thread_rng();
/// let mut alice = Uake::new();
/// let bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// # Ok(()) }
/// ```
pub fn client_init<R>(&mut self, pubkey: &PublicKey, rng: &mut R) -> UakeSendInit
where
R: CryptoRng + RngCore,
{
uake_init_a(&mut self.send_a, &mut self.temp_key, &mut self.eska, pubkey, rng);
self.send_a
}
/// Handles the output of a `client_init()` request
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> {
/// # let mut rng = rand::thread_rng();
/// let mut alice = Uake::new();
/// let mut bob = Uake::new();
/// let mut bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng)?;
/// # Ok(()) }
pub fn server_receive<R>(
&mut self, send_a: UakeSendInit, secretkey: &SecretKey, rng: &mut R
)
-> Result<UakeSendResponse, KyberError>
where R: CryptoRng + RngCore
{
uake_shared_b(
&mut self.send_b, &mut self.shared_secret,
&send_a, secretkey, rng
)?;
Ok(self.send_b)
}
/// Handles the output of a `client_init()` request
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> {
/// # let mut rng = rand::thread_rng();
/// let mut alice = Uake::new();
/// let mut bob = Uake::new();
/// let mut bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng)?;
/// # Ok(()) }
pub fn server_receive<R>(&mut self, send_a: UakeSendInit, secretkey: &SecretKey, rng: &mut R) -> Result<UakeSendResponse, KyberError>
where
R: CryptoRng + RngCore,
{
uake_shared_b(&mut self.send_b, &mut self.shared_secret, &send_a, secretkey, rng)?;
Ok(self.send_b)
}
/// Decapsulates and authenticates the shared secret from the output of
/// `server_receive()`
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> {
/// # let mut rng = rand::thread_rng();
/// # let mut alice = Uake::new();
/// # let mut bob = Uake::new();
/// # let bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng)?;
/// let client_confirm = alice.client_confirm(server_send);
/// assert_eq!(alice.shared_secret, bob.shared_secret);
/// # Ok(()) }
pub fn client_confirm(&mut self, send_b: UakeSendResponse)
-> Result<(), KyberError>
{
uake_shared_a(
&mut self.shared_secret, &send_b,
&self.temp_key, &self.eska
)?;
Ok(())
}
/// Decapsulates and authenticates the shared secret from the output of
/// `server_receive()`
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> {
/// # let mut rng = rand::thread_rng();
/// # let mut alice = Uake::new();
/// # let mut bob = Uake::new();
/// # let bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng)?;
/// let client_confirm = alice.client_confirm(server_send);
/// assert_eq!(alice.shared_secret, bob.shared_secret);
/// # Ok(()) }
pub fn client_confirm(&mut self, send_b: UakeSendResponse) -> Result<(), KyberError> {
uake_shared_a(&mut self.shared_secret, &send_b, &self.temp_key, &self.eska)?;
Ok(())
}
}
/// Used for mutually authenticated key exchange between two parties.
///
///
/// # Example:
/// ```
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> {
/// let mut rng = rand::thread_rng();
///
///
/// let mut alice = Ake::new();
/// let mut bob = Ake::new();
///
///
/// let alice_keys = keypair(&mut rng);
/// let bob_keys = keypair(&mut rng);
///
///
/// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng)?;
/// let client_confirm = alice.client_confirm(server_send, &alice_keys.secret);
///
///
/// assert_eq!(alice.shared_secret, bob.shared_secret);
/// # Ok(()) }
/// ```
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub struct Ake {
/// The resulting shared secret from a key exchange
pub shared_secret: SharedSecret,
/// Sent when initiating a key exchange
send_a: AkeSendInit,
/// Sent back responding to a key exchange initiation
send_b: AkeSendResponse,
// Epheremal keys
temp_key: TempKey,
eska: Eska
/// The resulting shared secret from a key exchange
pub shared_secret: SharedSecret,
/// Sent when initiating a key exchange
send_a: AkeSendInit,
/// Sent back responding to a key exchange initiation
send_b: AkeSendResponse,
// Epheremal keys
temp_key: TempKey,
eska: Eska,
}
impl Default for Ake {
fn default() -> Self {
Ake {
shared_secret: [0u8; KYBER_SSBYTES],
send_a: [0u8; AKE_INIT_BYTES],
send_b: [0u8; AKE_RESPONSE_BYTES],
temp_key: [0u8; KYBER_SSBYTES],
eska: [0u8; KYBER_SECRETKEYBYTES],
fn default() -> Self {
Ake {
shared_secret: [0u8; KYBER_SSBYTES],
send_a: [0u8; AKE_INIT_BYTES],
send_b: [0u8; AKE_RESPONSE_BYTES],
temp_key: [0u8; KYBER_SSBYTES],
eska: [0u8; KYBER_SECRETKEYBYTES],
}
}
}
}
impl Ake {
/// Builds a new AKE struct
/// ```
/// # use pqc_kyber::Ake;
/// let mut kex = Ake::new();
/// ```
pub fn new() -> Self {
Self::default()
}
/// Builds a new AKE struct
/// ```
/// # use pqc_kyber::Ake;
/// let mut kex = Ake::new();
/// ```
pub fn new() -> Self {
Self::default()
}
/// Initiates a Mutually Authenticated Key Exchange.
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> {
/// let mut rng = rand::thread_rng();
/// let mut alice = Ake::new();
/// let bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// # Ok(()) }
/// ```
pub fn client_init<R>(&mut self, pubkey: &PublicKey, rng: &mut R)
-> AkeSendInit
where R: CryptoRng + RngCore
{
ake_init_a(
&mut self.send_a, &mut self.temp_key,
&mut self.eska, pubkey, rng
);
self.send_a
}
/// Initiates a Mutually Authenticated Key Exchange.
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> {
/// let mut rng = rand::thread_rng();
/// let mut alice = Ake::new();
/// let bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// # Ok(()) }
/// ```
pub fn client_init<R>(&mut self, pubkey: &PublicKey, rng: &mut R) -> AkeSendInit
where
R: CryptoRng + RngCore,
{
ake_init_a(&mut self.send_a, &mut self.temp_key, &mut self.eska, pubkey, rng);
self.send_a
}
/// Handles and authenticates the output of a `client_init()` request
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> {
/// # let mut rng = rand::thread_rng();
/// let mut alice = Ake::new();
/// let mut bob = Ake::new();
/// let alice_keys = keypair(&mut rng);
/// let bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng)?;
/// # Ok(()) }
pub fn server_receive<R>(
&mut self, ake_send_a: AkeSendInit, pubkey: &PublicKey,
secretkey: &SecretKey, rng: &mut R
)
-> Result<AkeSendResponse, KyberError>
where R: CryptoRng + RngCore
{
ake_shared_b(
&mut self.send_b, &mut self.shared_secret,
&ake_send_a, secretkey, pubkey, rng
)?;
Ok(self.send_b)
}
/// Handles and authenticates the output of a `client_init()` request
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> {
/// # let mut rng = rand::thread_rng();
/// let mut alice = Ake::new();
/// let mut bob = Ake::new();
/// let alice_keys = keypair(&mut rng);
/// let bob_keys = keypair(&mut rng);
/// let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng)?;
/// # Ok(()) }
pub fn server_receive<R>(
&mut self,
ake_send_a: AkeSendInit,
pubkey: &PublicKey,
secretkey: &SecretKey,
rng: &mut R,
) -> Result<AkeSendResponse, KyberError>
where
R: CryptoRng + RngCore,
{
ake_shared_b(&mut self.send_b, &mut self.shared_secret, &ake_send_a, secretkey, pubkey, rng)?;
Ok(self.send_b)
}
/// Decapsulates and authenticates the shared secret from the output of
/// `server_receive()`
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> {
/// # let mut rng = rand::thread_rng();
/// # let mut alice = Ake::new();
/// # let mut bob = Ake::new();
/// # let alice_keys = keypair(&mut rng);
/// # let bob_keys = keypair(&mut rng);
/// # let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng)?;
/// let client_confirm = alice.client_confirm(server_send, &alice_keys.secret);
/// assert_eq!(alice.shared_secret, bob.shared_secret);
/// # Ok(()) }
pub fn client_confirm(&mut self, send_b: AkeSendResponse, secretkey: &SecretKey)
-> Result<(), KyberError>
{
ake_shared_a(
&mut self.shared_secret, &send_b,
&self.temp_key, &self.eska, secretkey
)?;
Ok(())
}
/// Decapsulates and authenticates the shared secret from the output of
/// `server_receive()`
/// ```
/// # use pqc_kyber::*;
/// # fn main() -> Result<(),KyberError> {
/// # let mut rng = rand::thread_rng();
/// # let mut alice = Ake::new();
/// # let mut bob = Ake::new();
/// # let alice_keys = keypair(&mut rng);
/// # let bob_keys = keypair(&mut rng);
/// # let client_init = alice.client_init(&bob_keys.public, &mut rng);
/// let server_send = bob.server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng)?;
/// let client_confirm = alice.client_confirm(server_send, &alice_keys.secret);
/// assert_eq!(alice.shared_secret, bob.shared_secret);
/// # Ok(()) }
pub fn client_confirm(&mut self, send_b: AkeSendResponse, secretkey: &SecretKey) -> Result<(), KyberError> {
ake_shared_a(&mut self.shared_secret, &send_b, &self.temp_key, &self.eska, secretkey)?;
Ok(())
}
}
// Unilaterally Authenticated Key Exchange initiation
fn uake_init_a<R>(
send: &mut[u8],
tk: &mut[u8],
sk: &mut[u8],
pkb: &[u8],
rng: &mut R
)
where R: CryptoRng + RngCore
fn uake_init_a<R>(send: &mut [u8], tk: &mut [u8], sk: &mut [u8], pkb: &[u8], rng: &mut R)
where
R: CryptoRng + RngCore,
{
crypto_kem_keypair(send, sk, rng, None);
crypto_kem_enc(&mut send[KYBER_PUBLICKEYBYTES..], tk, pkb, rng, None);
crypto_kem_keypair(send, sk, rng, None);
crypto_kem_enc(&mut send[KYBER_PUBLICKEYBYTES..], tk, pkb, rng, None);
}
// Unilaterally authenticated key exchange computation by Bob
fn uake_shared_b<R>(
send: &mut[u8],
k: &mut[u8],
recv: &[u8],
skb: &[u8],
rng: &mut R
) -> Result<(), KyberError>
where R: CryptoRng + RngCore
// Unilaterally authenticated key exchange computation by Bob
fn uake_shared_b<R>(send: &mut [u8], k: &mut [u8], recv: &[u8], skb: &[u8], rng: &mut R) -> Result<(), KyberError>
where
R: CryptoRng + RngCore,
{
let mut buf = [0u8; 2*KYBER_SYMBYTES];
crypto_kem_enc(send, &mut buf, recv, rng, None);
crypto_kem_dec(&mut buf[KYBER_SYMBYTES..], &recv[KYBER_PUBLICKEYBYTES..], skb)?;
kdf(k, &buf, 2*KYBER_SYMBYTES);
Ok(())
let mut buf = [0u8; 2 * KYBER_SYMBYTES];
crypto_kem_enc(send, &mut buf, recv, rng, None);
crypto_kem_dec(&mut buf[KYBER_SYMBYTES..], &recv[KYBER_PUBLICKEYBYTES..], skb)?;
kdf(k, &buf, 2 * KYBER_SYMBYTES);
Ok(())
}
// Unilaterally authenticated key exchange computation by Alice
fn uake_shared_a(
k: &mut[u8],
recv: &[u8],
tk: &[u8],
sk: &[u8]
) -> Result<(), KyberError>
{
let mut buf = [0u8; 2*KYBER_SYMBYTES];
crypto_kem_dec(&mut buf, recv, sk)?;
buf[KYBER_SYMBYTES..].copy_from_slice(&tk[..]);
kdf(k, &buf, 2*KYBER_SYMBYTES);
Ok(())
fn uake_shared_a(k: &mut [u8], recv: &[u8], tk: &[u8], sk: &[u8]) -> Result<(), KyberError> {
let mut buf = [0u8; 2 * KYBER_SYMBYTES];
crypto_kem_dec(&mut buf, recv, sk)?;
buf[KYBER_SYMBYTES..].copy_from_slice(&tk[..]);
kdf(k, &buf, 2 * KYBER_SYMBYTES);
Ok(())
}
// Authenticated key exchange initiation by Alice
fn ake_init_a<R>(
send: &mut[u8],
tk: &mut[u8],
sk: &mut[u8],
pkb: &[u8],
rng: &mut R
)
where R: CryptoRng + RngCore
fn ake_init_a<R>(send: &mut [u8], tk: &mut [u8], sk: &mut [u8], pkb: &[u8], rng: &mut R)
where
R: CryptoRng + RngCore,
{
crypto_kem_keypair(send, sk, rng, None);
crypto_kem_enc(&mut send[KYBER_PUBLICKEYBYTES..], tk, pkb, rng, None);
crypto_kem_keypair(send, sk, rng, None);
crypto_kem_enc(&mut send[KYBER_PUBLICKEYBYTES..], tk, pkb, rng, None);
}
// Mutually authenticated key exchange computation by Bob
fn ake_shared_b<R>(
send: &mut[u8],
k: &mut[u8],
recv: &[u8],
skb: &[u8],
pka: &[u8],
rng: &mut R
) -> Result<(), KyberError>
where R: CryptoRng + RngCore
fn ake_shared_b<R>(send: &mut [u8], k: &mut [u8], recv: &[u8], skb: &[u8], pka: &[u8], rng: &mut R) -> Result<(), KyberError>
where
R: CryptoRng + RngCore,
{
let mut buf = [0u8; 3*KYBER_SYMBYTES];
crypto_kem_enc(send, &mut buf, recv, rng, None);
crypto_kem_enc(&mut send[KYBER_CIPHERTEXTBYTES..], &mut buf[KYBER_SYMBYTES..], pka, rng, None);
crypto_kem_dec(&mut buf[2*KYBER_SYMBYTES..], &recv[KYBER_PUBLICKEYBYTES..], skb)?;
kdf(k, &buf, 3*KYBER_SYMBYTES);
Ok(())
let mut buf = [0u8; 3 * KYBER_SYMBYTES];
crypto_kem_enc(send, &mut buf, recv, rng, None);
crypto_kem_enc(&mut send[KYBER_CIPHERTEXTBYTES..], &mut buf[KYBER_SYMBYTES..], pka, rng, None);
crypto_kem_dec(&mut buf[2 * KYBER_SYMBYTES..], &recv[KYBER_PUBLICKEYBYTES..], skb)?;
kdf(k, &buf, 3 * KYBER_SYMBYTES);
Ok(())
}
// Mutually authenticated key exchange computation by Alice
fn ake_shared_a(
k: &mut[u8],
recv: &[u8],
tk: &[u8],
sk: &[u8],
ska: &[u8]
) -> Result<(), KyberError>
{
let mut buf = [0u8; 3*KYBER_SYMBYTES];
crypto_kem_dec(&mut buf, recv, sk)?;
crypto_kem_dec(&mut buf[KYBER_SYMBYTES..], &recv[KYBER_CIPHERTEXTBYTES..], ska)?;
buf[2*KYBER_SYMBYTES..].copy_from_slice(&tk[..]);
kdf(k, &buf, 3*KYBER_SYMBYTES);
Ok(())
}
fn ake_shared_a(k: &mut [u8], recv: &[u8], tk: &[u8], sk: &[u8], ska: &[u8]) -> Result<(), KyberError> {
let mut buf = [0u8; 3 * KYBER_SYMBYTES];
crypto_kem_dec(&mut buf, recv, sk)?;
crypto_kem_dec(&mut buf[KYBER_SYMBYTES..], &recv[KYBER_CIPHERTEXTBYTES..], ska)?;
buf[2 * KYBER_SYMBYTES..].copy_from_slice(&tk[..]);
kdf(k, &buf, 3 * KYBER_SYMBYTES);
Ok(())
}

134
third_party/kyber/src/params.rs vendored
View file

@ -1,67 +1,67 @@
/// The security level of Kyber
///
/// Defaults to 3 (kyber768), will be 2 or 4 repsectively when
/// kyber512 or kyber1024 are selected with feature flags.
///
/// * Kyber-512 aims at security roughly equivalent to AES-128
/// * Kyber-768 aims at security roughly equivalent to AES-192
/// * Kyber-1024 aims at security roughly equivalent to AES-256
#[cfg(not(any(feature = "kyber512", feature = "kyber1024")))]
pub const KYBER_K: usize = 3;
#[cfg(feature = "kyber512")]
pub const KYBER_K: usize = 2;
#[cfg(feature = "kyber1024")]
pub const KYBER_K: usize = 4;
/// A boolean flag for whether 90's mode is activated.
///
/// If true AES-CTR and SHA2 will be used as cryptographic primitives instead,
/// which may have hardware speed-ups on certain platforms.
///
/// Defaults to false, set`features = ["90s"]` in Cargo.toml to enable.
#[cfg(not(feature = "90s"))]
pub const KYBER_90S: bool = false;
#[cfg(feature = "90s")]
pub const KYBER_90S: bool = true;
pub(crate) const KYBER_N: usize = 256;
pub(crate) const KYBER_Q: usize = 3329;
#[cfg(feature = "kyber512")]
pub(crate) const KYBER_ETA1: usize = 3;
#[cfg(not(feature = "kyber512"))]
pub(crate) const KYBER_ETA1: usize = 2;
pub(crate) const KYBER_ETA2: usize = 2;
// Size of the hashes and seeds
pub(crate) const KYBER_SYMBYTES: usize = 32;
/// Size of the shared key
pub const KYBER_SSBYTES: usize = 32;
pub(crate) const KYBER_POLYBYTES: usize = 384;
pub(crate) const KYBER_POLYVECBYTES: usize = KYBER_K * KYBER_POLYBYTES;
#[cfg(not(feature = "kyber1024"))]
pub(crate) const KYBER_POLYCOMPRESSEDBYTES: usize = 128;
#[cfg(not(feature = "kyber1024"))]
pub(crate) const KYBER_POLYVECCOMPRESSEDBYTES: usize = KYBER_K * 320;
#[cfg(feature = "kyber1024")]
pub(crate) const KYBER_POLYCOMPRESSEDBYTES: usize = 160;
#[cfg(feature = "kyber1024")]
pub(crate) const KYBER_POLYVECCOMPRESSEDBYTES: usize = KYBER_K * 352;
pub(crate) const KYBER_INDCPA_PUBLICKEYBYTES: usize = KYBER_POLYVECBYTES + KYBER_SYMBYTES;
pub(crate) const KYBER_INDCPA_SECRETKEYBYTES: usize = KYBER_POLYVECBYTES;
pub(crate) const KYBER_INDCPA_BYTES: usize = KYBER_POLYVECCOMPRESSEDBYTES + KYBER_POLYCOMPRESSEDBYTES;
/// Size in bytes of the Kyber public key
pub const KYBER_PUBLICKEYBYTES: usize = KYBER_INDCPA_PUBLICKEYBYTES;
/// Size in bytes of the Kyber secret key
pub const KYBER_SECRETKEYBYTES: usize = KYBER_INDCPA_SECRETKEYBYTES + KYBER_INDCPA_PUBLICKEYBYTES + 2*KYBER_SYMBYTES;
/// Size in bytes of the Kyber ciphertext
pub const KYBER_CIPHERTEXTBYTES: usize = KYBER_INDCPA_BYTES;
/// The security level of Kyber
///
/// Defaults to 3 (kyber768), will be 2 or 4 repsectively when
/// kyber512 or kyber1024 are selected with feature flags.
///
/// * Kyber-512 aims at security roughly equivalent to AES-128
/// * Kyber-768 aims at security roughly equivalent to AES-192
/// * Kyber-1024 aims at security roughly equivalent to AES-256
#[cfg(not(any(feature = "kyber512", feature = "kyber1024")))]
pub const KYBER_K: usize = 3;
#[cfg(feature = "kyber512")]
pub const KYBER_K: usize = 2;
#[cfg(feature = "kyber1024")]
pub const KYBER_K: usize = 4;
/// A boolean flag for whether 90's mode is activated.
///
/// If true AES-CTR and SHA2 will be used as cryptographic primitives instead,
/// which may have hardware speed-ups on certain platforms.
///
/// Defaults to false, set`features = ["90s"]` in Cargo.toml to enable.
#[cfg(not(feature = "90s"))]
pub const KYBER_90S: bool = false;
#[cfg(feature = "90s")]
pub const KYBER_90S: bool = true;
pub(crate) const KYBER_N: usize = 256;
pub(crate) const KYBER_Q: usize = 3329;
#[cfg(feature = "kyber512")]
pub(crate) const KYBER_ETA1: usize = 3;
#[cfg(not(feature = "kyber512"))]
pub(crate) const KYBER_ETA1: usize = 2;
pub(crate) const KYBER_ETA2: usize = 2;
// Size of the hashes and seeds
pub(crate) const KYBER_SYMBYTES: usize = 32;
/// Size of the shared key
pub const KYBER_SSBYTES: usize = 32;
pub(crate) const KYBER_POLYBYTES: usize = 384;
pub(crate) const KYBER_POLYVECBYTES: usize = KYBER_K * KYBER_POLYBYTES;
#[cfg(not(feature = "kyber1024"))]
pub(crate) const KYBER_POLYCOMPRESSEDBYTES: usize = 128;
#[cfg(not(feature = "kyber1024"))]
pub(crate) const KYBER_POLYVECCOMPRESSEDBYTES: usize = KYBER_K * 320;
#[cfg(feature = "kyber1024")]
pub(crate) const KYBER_POLYCOMPRESSEDBYTES: usize = 160;
#[cfg(feature = "kyber1024")]
pub(crate) const KYBER_POLYVECCOMPRESSEDBYTES: usize = KYBER_K * 352;
pub(crate) const KYBER_INDCPA_PUBLICKEYBYTES: usize = KYBER_POLYVECBYTES + KYBER_SYMBYTES;
pub(crate) const KYBER_INDCPA_SECRETKEYBYTES: usize = KYBER_POLYVECBYTES;
pub(crate) const KYBER_INDCPA_BYTES: usize = KYBER_POLYVECCOMPRESSEDBYTES + KYBER_POLYCOMPRESSEDBYTES;
/// Size in bytes of the Kyber public key
pub const KYBER_PUBLICKEYBYTES: usize = KYBER_INDCPA_PUBLICKEYBYTES;
/// Size in bytes of the Kyber secret key
pub const KYBER_SECRETKEYBYTES: usize = KYBER_INDCPA_SECRETKEYBYTES + KYBER_INDCPA_PUBLICKEYBYTES + 2 * KYBER_SYMBYTES;
/// Size in bytes of the Kyber ciphertext
pub const KYBER_CIPHERTEXTBYTES: usize = KYBER_INDCPA_BYTES;

1225
third_party/kyber/src/reference/aes256ctr.rs vendored
View file

File diff suppressed because it is too large Load diff

189
third_party/kyber/src/reference/cbd.rs vendored
View file

@ -1,98 +1,91 @@
use crate::poly::Poly;
use crate::params::KYBER_N;
// Name: load32_littleendian
//
// Description: load 4 bytes into a 32-bit integer
// in little-endian order
//
// Arguments: - const [u8] x: input byte array
//
// Returns 32-bit unsigned integer loaded from x
fn load32_littleendian(x: &[u8]) -> u32
{
let mut r = x[0] as u32;
r |= (x[1] as u32) << 8;
r |= (x[2] as u32) << 16;
r |= (x[3] as u32) << 24;
r
}
// Name: load32_littleendian
//
// Description: load 3 bytes into a 32-bit integer
// in little-endian order
// This function is only needed for Kyber-512
//
// Arguments: - const [u8] x: input byte array
//
// Returns 32-bit unsigned integer loaded from x
fn load24_littleendian(x: &[u8]) -> u32
{
let mut r = x[0] as u32;
r |= (x[1] as u32) << 8;
r |= (x[2] as u32) << 16;
r
}
// Name: cbd2
//
// Description: Given an array of uniformly random bytes, compute
// polynomial with coefficients distributed according to
// a centered binomial distribution with parameter eta=2
//
// Arguments: - poly *r: output polynomial
// - const [u8] buf: input byte array
pub fn cbd2(r: &mut Poly, buf: &[u8])
{
let (mut d, mut t, mut a, mut b);
for i in 0..(KYBER_N/8) {
t = load32_littleendian(&buf[4*i..]);
d = t & 0x55555555;
d += (t>>1) & 0x55555555;
for j in 0..8 {
a = ((d >> (4*j)) & 0x3) as i16;
b = ((d >> (4*j+2)) & 0x3) as i16;
r.coeffs[8*i+j] = a - b;
}
}
}
// Name: cbd3
//
// Description: Given an array of uniformly random bytes, compute
// polynomial with coefficients distributed according to
// a centered binomial distribution with parameter eta=3
// This function is only needed for Kyber-512
// Arguments: - poly *r: output polynomial
// - const [u8] buf: input byte array
pub fn cbd3(r: &mut Poly, buf: &[u8])
{
let (mut d, mut t, mut a, mut b);
for i in 0..(KYBER_N/4) {
t = load24_littleendian(&buf[3*i..]);
d = t & 0x00249249;
d += (t>>1) & 0x00249249;
d += (t>>2) & 0x00249249;
for j in 0..4 {
a = ((d >> (6*j)) & 0x7) as i16;
b = ((d >> (6*j+3)) & 0x7) as i16;
r.coeffs[4*i+j] = a - b;
}
}
}
pub fn poly_cbd_eta1(r: &mut Poly, buf: &[u8])
{
if cfg!(feature="kyber512") {
cbd3(r, buf)
}
else {
cbd2(r, buf)
}
}
pub fn poly_cbd_eta2(r: &mut Poly, buf: &[u8])
{
cbd2(r, buf)
}
use crate::params::KYBER_N;
use crate::poly::Poly;
// Name: load32_littleendian
//
// Description: load 4 bytes into a 32-bit integer
// in little-endian order
//
// Arguments: - const [u8] x: input byte array
//
// Returns 32-bit unsigned integer loaded from x
fn load32_littleendian(x: &[u8]) -> u32 {
let mut r = x[0] as u32;
r |= (x[1] as u32) << 8;
r |= (x[2] as u32) << 16;
r |= (x[3] as u32) << 24;
r
}
// Name: load32_littleendian
//
// Description: load 3 bytes into a 32-bit integer
// in little-endian order
// This function is only needed for Kyber-512
//
// Arguments: - const [u8] x: input byte array
//
// Returns 32-bit unsigned integer loaded from x
fn load24_littleendian(x: &[u8]) -> u32 {
let mut r = x[0] as u32;
r |= (x[1] as u32) << 8;
r |= (x[2] as u32) << 16;
r
}
// Name: cbd2
//
// Description: Given an array of uniformly random bytes, compute
// polynomial with coefficients distributed according to
// a centered binomial distribution with parameter eta=2
//
// Arguments: - poly *r: output polynomial
// - const [u8] buf: input byte array
pub fn cbd2(r: &mut Poly, buf: &[u8]) {
let (mut d, mut t, mut a, mut b);
for i in 0..(KYBER_N / 8) {
t = load32_littleendian(&buf[4 * i..]);
d = t & 0x55555555;
d += (t >> 1) & 0x55555555;
for j in 0..8 {
a = ((d >> (4 * j)) & 0x3) as i16;
b = ((d >> (4 * j + 2)) & 0x3) as i16;
r.coeffs[8 * i + j] = a - b;
}
}
}
// Name: cbd3
//
// Description: Given an array of uniformly random bytes, compute
// polynomial with coefficients distributed according to
// a centered binomial distribution with parameter eta=3
// This function is only needed for Kyber-512
// Arguments: - poly *r: output polynomial
// - const [u8] buf: input byte array
pub fn cbd3(r: &mut Poly, buf: &[u8]) {
let (mut d, mut t, mut a, mut b);
for i in 0..(KYBER_N / 4) {
t = load24_littleendian(&buf[3 * i..]);
d = t & 0x00249249;
d += (t >> 1) & 0x00249249;
d += (t >> 2) & 0x00249249;
for j in 0..4 {
a = ((d >> (6 * j)) & 0x7) as i16;
b = ((d >> (6 * j + 3)) & 0x7) as i16;
r.coeffs[4 * i + j] = a - b;
}
}
}
pub fn poly_cbd_eta1(r: &mut Poly, buf: &[u8]) {
if cfg!(feature = "kyber512") {
cbd3(r, buf)
} else {
cbd2(r, buf)
}
}
pub fn poly_cbd_eta2(r: &mut Poly, buf: &[u8]) {
cbd2(r, buf)
}

1258
third_party/kyber/src/reference/fips202.rs vendored
View file

File diff suppressed because it is too large Load diff

605
third_party/kyber/src/reference/indcpa.rs vendored
View file

@ -1,316 +1,289 @@
#[cfg(not(feature="KATs"))]
use crate::rng::randombytes;
use crate::{
poly::*,
polyvec::*,
symmetric::*,
params::*,
RngCore,
CryptoRng,
};
// Name: pack_pk
//
// Description: Serialize the public key as concatenation of the
// serialized vector of polynomials pk
// and the public seed used to generate the matrix A.
//
// Arguments: [u8] r: the output serialized public key
// const poly *pk: the input public-key polynomial
// const [u8] seed: the input public seed
fn pack_pk(r: &mut[u8], pk: &mut Polyvec, seed: &[u8])
{
const END: usize = KYBER_SYMBYTES + KYBER_POLYVECBYTES;
polyvec_tobytes(r, pk);
r[KYBER_POLYVECBYTES..END].copy_from_slice(&seed[..KYBER_SYMBYTES]);
}
// Name: unpack_pk
//
// Description: De-serialize public key from a byte array;
// approximate inverse of pack_pk
//
// Arguments: - Polyvec pk: output public-key vector of polynomials
// - [u8] seed: output seed to generate matrix A
// - const [u8] packedpk: input serialized public key
fn unpack_pk(pk: &mut Polyvec, seed: &mut[u8], packedpk: &[u8])
{
const END: usize = KYBER_SYMBYTES + KYBER_POLYVECBYTES;
polyvec_frombytes(pk, packedpk);
seed[..KYBER_SYMBYTES].copy_from_slice(&packedpk[KYBER_POLYVECBYTES..END]);
}
// Name: pack_sk
//
// Description: Serialize the secret key
//
// Arguments: - [u8] r: output serialized secret key
// - const Polyvec sk: input vector of polynomials (secret key)
fn pack_sk(r: &mut[u8], sk: &mut Polyvec)
{
polyvec_tobytes(r, sk);
}
// Name: unpack_sk
//
// Description: De-serialize the secret key, inverse of pack_sk
//
// Arguments: - Polyvec sk: output vector of polynomials (secret key)
// - const [u8] packedsk: input serialized secret key
fn unpack_sk(sk: &mut Polyvec, packedsk: &[u8])
{
polyvec_frombytes(sk, packedsk);
}
// Name: pack_ciphertext
//
// Description: Serialize the ciphertext as concatenation of the
// compressed and serialized vector of polynomials b
// and the compressed and serialized polynomial v
//
// Arguments: [u8] r: the output serialized ciphertext
// const poly *pk: the input vector of polynomials b
// const [u8] seed: the input polynomial v
fn pack_ciphertext(r: &mut[u8], b: &mut Polyvec, v: Poly)
{
polyvec_compress(r, *b);
poly_compress(&mut r[KYBER_POLYVECCOMPRESSEDBYTES..], v);
}
// Name: unpack_ciphertext
//
// Description: De-serialize and decompress ciphertext from a byte array;
// approximate inverse of pack_ciphertext
//
// Arguments: - Polyvec b: output vector of polynomials b
// - poly *v: output polynomial v
// - const [u8] c: input serialized ciphertext
fn unpack_ciphertext(b: &mut Polyvec, v: &mut Poly, c: &[u8])
{
polyvec_decompress(b, c);
poly_decompress(v, &c[KYBER_POLYVECCOMPRESSEDBYTES..]);
}
// Name: rej_uniform
//
// Description: Run rejection sampling on uniform random bytes to generate
// uniform random integers mod q
//
// Arguments: - i16 *r: output buffer
// - usize len: requested number of 16-bit integers (uniform mod q)
// - const [u8] buf: input buffer (assumed to be uniform random bytes)
// - usize buflen: length of input buffer in bytes
//
// Returns number of sampled 16-bit integers (at most len)
fn rej_uniform(r: &mut[i16], len: usize, buf: &[u8], buflen: usize) -> usize
{
let (mut ctr, mut pos) = (0usize, 0usize);
let (mut val0, mut val1);
while ctr < len && pos + 3 <= buflen {
val0 = ((buf[pos+0] >> 0) as u16 | (buf[pos+1] as u16) << 8) & 0xFFF;
val1 = ((buf[pos+1] >> 4) as u16 | (buf[pos+2] as u16) << 4) & 0xFFF;
pos += 3;
if val0 < KYBER_Q as u16 {
r[ctr] = val0 as i16;
ctr += 1;
}
if ctr < len && val1 < KYBER_Q as u16 {
r[ctr] = val1 as i16;
ctr += 1;
}
}
ctr
}
fn gen_a(a: &mut [Polyvec], b: &[u8])
{
gen_matrix(a, b, false);
}
fn gen_at(a: &mut [Polyvec], b: &[u8])
{
gen_matrix(a, b, true);
}
// Name: gen_matrix
//
// Description: Deterministically generate matrix A (or the transpose of A)
// from a seed. Entries of the matrix are polynomials that look
// uniformly random. Performs rejection sampling on output of
// a XOF
//
// Arguments: - Polyvec a: ouptput matrix A
// - const [u8] seed: input seed
// - bool transposed: boolean deciding whether A or A^T is generated
fn gen_matrix(a: &mut [Polyvec], seed: &[u8], transposed: bool)
{
let mut ctr;
// 530 is expected number of required bytes
const GEN_MATRIX_NBLOCKS: usize =
(12*KYBER_N/8*(1 << 12)/KYBER_Q + XOF_BLOCKBYTES)/XOF_BLOCKBYTES;
const BUFLEN: usize = GEN_MATRIX_NBLOCKS*XOF_BLOCKBYTES;
let mut buf = [0u8; BUFLEN+2];
let mut off: usize;
let mut state = XofState::new();
for i in 0..KYBER_K {
for j in 0..KYBER_K {
if transposed {
xof_absorb(&mut state, seed, i as u8, j as u8);
}
else {
xof_absorb(&mut state, seed, j as u8, i as u8);
}
xof_squeezeblocks(&mut buf, GEN_MATRIX_NBLOCKS, &mut state);
ctr = rej_uniform(&mut a[i].vec[j].coeffs, KYBER_N, &buf, BUFLEN);
while ctr < KYBER_N
{
off = BUFLEN % 3;
for k in 0..off {
buf[k] = buf[BUFLEN - off + k];
}
xof_squeezeblocks(&mut buf[off..], 1, &mut state);
ctr += rej_uniform(&mut a[i].vec[j].coeffs[ctr..], KYBER_N - ctr, &buf, BUFLEN);
}
}
}
}
// Name: indcpa_keypair
//
// Description: Generates public and private key for the CPA-secure
// public-key encryption scheme underlying Kyber
//
// Arguments: - [u8] pk: output public key (length KYBER_INDCPA_PUBLICKEYBYTES)
// - [u8] sk: output private key (length KYBER_INDCPA_SECRETKEYBYTES)
pub fn indcpa_keypair<R>(
pk : &mut[u8],
sk: &mut[u8],
_seed: Option<(&[u8], &[u8])>,
_rng: &mut R
)
where R: CryptoRng + RngCore
{
let mut a = [Polyvec::new(); KYBER_K];
let (mut e, mut pkpv, mut skpv) = (Polyvec::new(), Polyvec::new(), Polyvec::new());
let mut nonce = 0u8;
let mut buf = [0u8; 2*KYBER_SYMBYTES];
let mut randbuf = [0u8; 2*KYBER_SYMBYTES];
#[cfg(not(feature="KATs"))]
randombytes(&mut randbuf, KYBER_SYMBYTES, _rng);
// Use rng seed for test vectors
#[cfg(feature="KATs")]
randbuf[..KYBER_SYMBYTES].copy_from_slice(&_seed.expect("KAT feature only for testing").0);
hash_g(&mut buf, &randbuf, KYBER_SYMBYTES);
let (publicseed, noiseseed) = buf.split_at(KYBER_SYMBYTES);
gen_a(&mut a, publicseed);
for i in 0..KYBER_K {
poly_getnoise_eta1(&mut skpv.vec[i], noiseseed, nonce);
nonce += 1;
}
for i in 0..KYBER_K {
poly_getnoise_eta1(&mut e.vec[i], noiseseed, nonce);
nonce += 1;
}
polyvec_ntt(&mut skpv);
polyvec_ntt(&mut e);
// matrix-vector multiplication
for i in 0..KYBER_K {
polyvec_basemul_acc_montgomery(&mut pkpv.vec[i], &a[i], &skpv);
poly_frommont(&mut pkpv.vec[i]);
}
polyvec_add(&mut pkpv, &e);
polyvec_reduce(&mut pkpv);
pack_sk(sk, &mut skpv);
pack_pk(pk, &mut pkpv, publicseed);
}
// Name: indcpa_enc
//
// Description: Encryption function of the CPA-secure
// public-key encryption scheme underlying Kyber.
//
// Arguments: - [u8] c: output ciphertext (length KYBER_INDCPA_BYTES)
// - const [u8] m: input message (length KYBER_INDCPA_MSGBYTES)
// - const [u8] pk: input public key (length KYBER_INDCPA_PUBLICKEYBYTES)
// - const [u8] coin: input random coins used as seed (length KYBER_SYMBYTES)
// to deterministically generate all randomness
pub fn indcpa_enc(c: &mut[u8], m: &[u8], pk: &[u8], coins: &[u8])
{
let mut at = [Polyvec::new(); KYBER_K];
let (mut sp, mut pkpv, mut ep, mut b) =
(Polyvec::new(),Polyvec::new(), Polyvec::new(), Polyvec::new());
let (mut v, mut k, mut epp) = (Poly::new(), Poly::new(), Poly::new());
let mut seed = [0u8; KYBER_SYMBYTES];
let mut nonce = 0u8;
unpack_pk(&mut pkpv, &mut seed, pk);
poly_frommsg(&mut k, m);
gen_at(&mut at, &seed);
for i in 0..KYBER_K {
poly_getnoise_eta1(&mut sp.vec[i], coins, nonce);
nonce += 1;
}
for i in 0..KYBER_K {
poly_getnoise_eta2(&mut ep.vec[i], coins, nonce);
nonce += 1;
}
poly_getnoise_eta2(&mut epp, coins, nonce);
polyvec_ntt(&mut sp);
// matrix-vector multiplication
for i in 0..KYBER_K {
polyvec_basemul_acc_montgomery(&mut b.vec[i], &at[i], &sp);
}
polyvec_basemul_acc_montgomery(&mut v, &pkpv, &sp);
polyvec_invntt_tomont(&mut b);
poly_invntt_tomont(&mut v);
polyvec_add(&mut b, &ep);
poly_add(&mut v, &epp);
poly_add(&mut v, &k);
polyvec_reduce(&mut b);
poly_reduce(&mut v);
pack_ciphertext(c, &mut b, v);
}
// Name: indcpa_dec
//
// Description: Decryption function of the CPA-secure
// public-key encryption scheme underlying Kyber.
//
// Arguments: - [u8] m: output decrypted message (of length KYBER_INDCPA_MSGBYTES)
// - const [u8] c: input ciphertext (of length KYBER_INDCPA_BYTES)
// - const [u8] sk: input secret key (of length KYBER_INDCPA_SECRETKEYBYTES)
pub fn indcpa_dec(m: &mut[u8], c: &[u8], sk: &[u8])
{
let (mut b, mut skpv) = (Polyvec::new(),Polyvec::new());
let (mut v, mut mp) = (Poly::new(),Poly::new());
unpack_ciphertext(&mut b, &mut v, c);
unpack_sk(&mut skpv, sk);
polyvec_ntt(&mut b);
polyvec_basemul_acc_montgomery(&mut mp, &skpv, &b);
poly_invntt_tomont(&mut mp);
poly_sub(&mut mp, &v);
poly_reduce(&mut mp);
poly_tomsg(m, mp);
}
#[cfg(not(feature = "KATs"))]
use crate::rng::randombytes;
use crate::{params::*, poly::*, polyvec::*, symmetric::*, CryptoRng, RngCore};
// Name: pack_pk
//
// Description: Serialize the public key as concatenation of the
// serialized vector of polynomials pk
// and the public seed used to generate the matrix A.
//
// Arguments: [u8] r: the output serialized public key
// const poly *pk: the input public-key polynomial
// const [u8] seed: the input public seed
fn pack_pk(r: &mut [u8], pk: &mut Polyvec, seed: &[u8]) {
const END: usize = KYBER_SYMBYTES + KYBER_POLYVECBYTES;
polyvec_tobytes(r, pk);
r[KYBER_POLYVECBYTES..END].copy_from_slice(&seed[..KYBER_SYMBYTES]);
}
// Name: unpack_pk
//
// Description: De-serialize public key from a byte array;
// approximate inverse of pack_pk
//
// Arguments: - Polyvec pk: output public-key vector of polynomials
// - [u8] seed: output seed to generate matrix A
// - const [u8] packedpk: input serialized public key
fn unpack_pk(pk: &mut Polyvec, seed: &mut [u8], packedpk: &[u8]) {
const END: usize = KYBER_SYMBYTES + KYBER_POLYVECBYTES;
polyvec_frombytes(pk, packedpk);
seed[..KYBER_SYMBYTES].copy_from_slice(&packedpk[KYBER_POLYVECBYTES..END]);
}
// Name: pack_sk
//
// Description: Serialize the secret key
//
// Arguments: - [u8] r: output serialized secret key
// - const Polyvec sk: input vector of polynomials (secret key)
fn pack_sk(r: &mut [u8], sk: &mut Polyvec) {
polyvec_tobytes(r, sk);
}
// Name: unpack_sk
//
// Description: De-serialize the secret key, inverse of pack_sk
//
// Arguments: - Polyvec sk: output vector of polynomials (secret key)
// - const [u8] packedsk: input serialized secret key
fn unpack_sk(sk: &mut Polyvec, packedsk: &[u8]) {
polyvec_frombytes(sk, packedsk);
}
// Name: pack_ciphertext
//
// Description: Serialize the ciphertext as concatenation of the
// compressed and serialized vector of polynomials b
// and the compressed and serialized polynomial v
//
// Arguments: [u8] r: the output serialized ciphertext
// const poly *pk: the input vector of polynomials b
// const [u8] seed: the input polynomial v
fn pack_ciphertext(r: &mut [u8], b: &mut Polyvec, v: Poly) {
polyvec_compress(r, *b);
poly_compress(&mut r[KYBER_POLYVECCOMPRESSEDBYTES..], v);
}
// Name: unpack_ciphertext
//
// Description: De-serialize and decompress ciphertext from a byte array;
// approximate inverse of pack_ciphertext
//
// Arguments: - Polyvec b: output vector of polynomials b
// - poly *v: output polynomial v
// - const [u8] c: input serialized ciphertext
fn unpack_ciphertext(b: &mut Polyvec, v: &mut Poly, c: &[u8]) {
polyvec_decompress(b, c);
poly_decompress(v, &c[KYBER_POLYVECCOMPRESSEDBYTES..]);
}
// Name: rej_uniform
//
// Description: Run rejection sampling on uniform random bytes to generate
// uniform random integers mod q
//
// Arguments: - i16 *r: output buffer
// - usize len: requested number of 16-bit integers (uniform mod q)
// - const [u8] buf: input buffer (assumed to be uniform random bytes)
// - usize buflen: length of input buffer in bytes
//
// Returns number of sampled 16-bit integers (at most len)
fn rej_uniform(r: &mut [i16], len: usize, buf: &[u8], buflen: usize) -> usize {
let (mut ctr, mut pos) = (0usize, 0usize);
let (mut val0, mut val1);
while ctr < len && pos + 3 <= buflen {
val0 = ((buf[pos + 0] >> 0) as u16 | (buf[pos + 1] as u16) << 8) & 0xFFF;
val1 = ((buf[pos + 1] >> 4) as u16 | (buf[pos + 2] as u16) << 4) & 0xFFF;
pos += 3;
if val0 < KYBER_Q as u16 {
r[ctr] = val0 as i16;
ctr += 1;
}
if ctr < len && val1 < KYBER_Q as u16 {
r[ctr] = val1 as i16;
ctr += 1;
}
}
ctr
}
fn gen_a(a: &mut [Polyvec], b: &[u8]) {
gen_matrix(a, b, false);
}
fn gen_at(a: &mut [Polyvec], b: &[u8]) {
gen_matrix(a, b, true);
}
// Name: gen_matrix
//
// Description: Deterministically generate matrix A (or the transpose of A)
// from a seed. Entries of the matrix are polynomials that look
// uniformly random. Performs rejection sampling on output of
// a XOF
//
// Arguments: - Polyvec a: ouptput matrix A
// - const [u8] seed: input seed
// - bool transposed: boolean deciding whether A or A^T is generated
fn gen_matrix(a: &mut [Polyvec], seed: &[u8], transposed: bool) {
let mut ctr;
// 530 is expected number of required bytes
const GEN_MATRIX_NBLOCKS: usize = (12 * KYBER_N / 8 * (1 << 12) / KYBER_Q + XOF_BLOCKBYTES) / XOF_BLOCKBYTES;
const BUFLEN: usize = GEN_MATRIX_NBLOCKS * XOF_BLOCKBYTES;
let mut buf = [0u8; BUFLEN + 2];
let mut off: usize;
let mut state = XofState::new();
for i in 0..KYBER_K {
for j in 0..KYBER_K {
if transposed {
xof_absorb(&mut state, seed, i as u8, j as u8);
} else {
xof_absorb(&mut state, seed, j as u8, i as u8);
}
xof_squeezeblocks(&mut buf, GEN_MATRIX_NBLOCKS, &mut state);
ctr = rej_uniform(&mut a[i].vec[j].coeffs, KYBER_N, &buf, BUFLEN);
while ctr < KYBER_N {
off = BUFLEN % 3;
for k in 0..off {
buf[k] = buf[BUFLEN - off + k];
}
xof_squeezeblocks(&mut buf[off..], 1, &mut state);
ctr += rej_uniform(&mut a[i].vec[j].coeffs[ctr..], KYBER_N - ctr, &buf, BUFLEN);
}
}
}
}
// Name: indcpa_keypair
//
// Description: Generates public and private key for the CPA-secure
// public-key encryption scheme underlying Kyber
//
// Arguments: - [u8] pk: output public key (length KYBER_INDCPA_PUBLICKEYBYTES)
// - [u8] sk: output private key (length KYBER_INDCPA_SECRETKEYBYTES)
pub fn indcpa_keypair<R>(pk: &mut [u8], sk: &mut [u8], _seed: Option<(&[u8], &[u8])>, _rng: &mut R)
where
R: CryptoRng + RngCore,
{
let mut a = [Polyvec::new(); KYBER_K];
let (mut e, mut pkpv, mut skpv) = (Polyvec::new(), Polyvec::new(), Polyvec::new());
let mut nonce = 0u8;
let mut buf = [0u8; 2 * KYBER_SYMBYTES];
let mut randbuf = [0u8; 2 * KYBER_SYMBYTES];
#[cfg(not(feature = "KATs"))]
randombytes(&mut randbuf, KYBER_SYMBYTES, _rng);
// Use rng seed for test vectors
#[cfg(feature = "KATs")]
randbuf[..KYBER_SYMBYTES].copy_from_slice(&_seed.expect("KAT feature only for testing").0);
hash_g(&mut buf, &randbuf, KYBER_SYMBYTES);
let (publicseed, noiseseed) = buf.split_at(KYBER_SYMBYTES);
gen_a(&mut a, publicseed);
for i in 0..KYBER_K {
poly_getnoise_eta1(&mut skpv.vec[i], noiseseed, nonce);
nonce += 1;
}
for i in 0..KYBER_K {
poly_getnoise_eta1(&mut e.vec[i], noiseseed, nonce);
nonce += 1;
}
polyvec_ntt(&mut skpv);
polyvec_ntt(&mut e);
// matrix-vector multiplication
for i in 0..KYBER_K {
polyvec_basemul_acc_montgomery(&mut pkpv.vec[i], &a[i], &skpv);
poly_frommont(&mut pkpv.vec[i]);
}
polyvec_add(&mut pkpv, &e);
polyvec_reduce(&mut pkpv);
pack_sk(sk, &mut skpv);
pack_pk(pk, &mut pkpv, publicseed);
}
// Name: indcpa_enc
//
// Description: Encryption function of the CPA-secure
// public-key encryption scheme underlying Kyber.
//
// Arguments: - [u8] c: output ciphertext (length KYBER_INDCPA_BYTES)
// - const [u8] m: input message (length KYBER_INDCPA_MSGBYTES)
// - const [u8] pk: input public key (length KYBER_INDCPA_PUBLICKEYBYTES)
// - const [u8] coin: input random coins used as seed (length KYBER_SYMBYTES)
// to deterministically generate all randomness
pub fn indcpa_enc(c: &mut [u8], m: &[u8], pk: &[u8], coins: &[u8]) {
let mut at = [Polyvec::new(); KYBER_K];
let (mut sp, mut pkpv, mut ep, mut b) = (Polyvec::new(), Polyvec::new(), Polyvec::new(), Polyvec::new());
let (mut v, mut k, mut epp) = (Poly::new(), Poly::new(), Poly::new());
let mut seed = [0u8; KYBER_SYMBYTES];
let mut nonce = 0u8;
unpack_pk(&mut pkpv, &mut seed, pk);
poly_frommsg(&mut k, m);
gen_at(&mut at, &seed);
for i in 0..KYBER_K {
poly_getnoise_eta1(&mut sp.vec[i], coins, nonce);
nonce += 1;
}
for i in 0..KYBER_K {
poly_getnoise_eta2(&mut ep.vec[i], coins, nonce);
nonce += 1;
}
poly_getnoise_eta2(&mut epp, coins, nonce);
polyvec_ntt(&mut sp);
// matrix-vector multiplication
for i in 0..KYBER_K {
polyvec_basemul_acc_montgomery(&mut b.vec[i], &at[i], &sp);
}
polyvec_basemul_acc_montgomery(&mut v, &pkpv, &sp);
polyvec_invntt_tomont(&mut b);
poly_invntt_tomont(&mut v);
polyvec_add(&mut b, &ep);
poly_add(&mut v, &epp);
poly_add(&mut v, &k);
polyvec_reduce(&mut b);
poly_reduce(&mut v);
pack_ciphertext(c, &mut b, v);
}
// Name: indcpa_dec
//
// Description: Decryption function of the CPA-secure
// public-key encryption scheme underlying Kyber.
//
// Arguments: - [u8] m: output decrypted message (of length KYBER_INDCPA_MSGBYTES)
// - const [u8] c: input ciphertext (of length KYBER_INDCPA_BYTES)
// - const [u8] sk: input secret key (of length KYBER_INDCPA_SECRETKEYBYTES)
pub fn indcpa_dec(m: &mut [u8], c: &[u8], sk: &[u8]) {
let (mut b, mut skpv) = (Polyvec::new(), Polyvec::new());
let (mut v, mut mp) = (Poly::new(), Poly::new());
unpack_ciphertext(&mut b, &mut v, c);
unpack_sk(&mut skpv, sk);
polyvec_ntt(&mut b);
polyvec_basemul_acc_montgomery(&mut mp, &skpv, &b);
poly_invntt_tomont(&mut mp);
poly_sub(&mut mp, &v);
poly_reduce(&mut mp);
poly_tomsg(m, mp);
}

2
third_party/kyber/src/reference/mod.rs vendored
View file

@ -6,4 +6,4 @@ pub mod ntt;
pub mod poly;
pub mod polyvec;
pub mod reduce;
pub mod verify;
pub mod verify;

294
third_party/kyber/src/reference/ntt.rs vendored
View file

@ -1,154 +1,140 @@
use crate::reduce::*;
// Code to generate zetas used in the number-theoretic transform:
//
//#define KYBER_ROOT_OF_UNITY 17
//
//static const uint8_t tree[128] = {
// 0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120,
// 4, 68, 36, 100, 20, 84, 52, 116, 12, 76, 44, 108, 28, 92, 60, 124,
// 2, 66, 34, 98, 18, 82, 50, 114, 10, 74, 42, 106, 26, 90, 58, 122,
// 6, 70, 38, 102, 22, 86, 54, 118, 14, 78, 46, 110, 30, 94, 62, 126,
// 1, 65, 33, 97, 17, 81, 49, 113, 9, 73, 41, 105, 25, 89, 57, 121,
// 5, 69, 37, 101, 21, 85, 53, 117, 13, 77, 45, 109, 29, 93, 61, 125,
// 3, 67, 35, 99, 19, 83, 51, 115, 11, 75, 43, 107, 27, 91, 59, 123,
// 7, 71, 39, 103, 23, 87, 55, 119, 15, 79, 47, 111, 31, 95, 63, 127};
//
//
// static int16_t fqmul(int16_t a, int16_t b) {
// return montgomery_reduce((int32_t)a*b);
//}
//
// void init_ntt() {
// unsigned int i;
// int16_t tmp[128];
//
// tmp[0] = MONT;
// for(i = 1; i < 128; ++i)
// tmp[i] = fqmul(tmp[i-1], MONT*KYBER_ROOT_OF_UNITY % KYBER_Q);
//
// for(i = 0; i < 128; ++i)
// zetas[i] = tmp[tree[i]];
//
// if(zetas[i] > KYBER_Q/2)
// zetas[i] -= KYBER_Q;
// if(zetas[i] < -KYBER_Q/2)
// zetas[i] += KYBER_Q;
//}
pub const ZETAS: [i16; 128] = [
-1044, -758, -359, -1517, 1493, 1422, 287, 202,
-171, 622, 1577, 182, 962, -1202, -1474, 1468,
573, -1325, 264, 383, -829, 1458, -1602, -130,
-681, 1017, 732, 608, -1542, 411, -205, -1571,
1223, 652, -552, 1015, -1293, 1491, -282, -1544,
516, -8, -320, -666, -1618, -1162, 126, 1469,
-853, -90, -271, 830, 107, -1421, -247, -951,
-398, 961, -1508, -725, 448, -1065, 677, -1275,
-1103, 430, 555, 843, -1251, 871, 1550, 105,
422, 587, 177, -235, -291, -460, 1574, 1653,
-246, 778, 1159, -147, -777, 1483, -602, 1119,
-1590, 644, -872, 349, 418, 329, -156, -75,
817, 1097, 603, 610, 1322, -1285, -1465, 384,
-1215, -136, 1218, -1335, -874, 220, -1187, -1659,
-1185, -1530, -1278, 794, -1510, -854, -870, 478,
-108, -308, 996, 991, 958, -1460, 1522, 1628
];
// Name: fqmul
//
// Description: Multiplication followed by Montgomery reduction
//
// Arguments: - i16 a: first factor
// - i16 b: second factor
//
// Returns 16-bit integer congruent to a*b*R^{-1} mod q
pub fn fqmul(a: i16, b: i16) -> i16
{
montgomery_reduce(a as i32 * b as i32)
}
// Name: ntt
//
// Description: Inplace number-theoretic transform (NTT) in Rq
// input is in standard order, output is in bitreversed order
//
// Arguments: - i16 r[256]: input/output vector of elements of Zq
pub fn ntt(r: &mut[i16])
{
let mut j;
let mut k = 1usize;
let mut len = 128;
let (mut t, mut zeta);
while len >= 2 {
let mut start = 0;
while start < 256 {
zeta = ZETAS[k];
k += 1;
j = start;
while j < (start + len) {
t = fqmul(zeta, r[j + len]);
r[j + len] = r[j] - t;
r[j] += t;
j += 1;
}
start = j + len;
}
len >>= 1;
}
}
// Name: invntt
//
// Description: Inplace inverse number-theoretic transform in Rq
// input is in bitreversed order, output is in standard order
//
// Arguments: - i16 r[256]: input/output vector of elements of Zq
pub fn invntt(r: &mut[i16])
{
let mut j;
let mut k = 127usize;
let mut len = 2;
let (mut t, mut zeta);
const F: i16 = 1441; // mont^2/128
while len <= 128 {
let mut start = 0;
while start < 256 {
zeta = ZETAS[k];
k -= 1;
j = start;
while j < (start + len) {
t = r[j];
r[j] = barrett_reduce(t + r[j + len]);
r[j + len] = r[j + len] -t;
r[j + len] = fqmul(zeta, r[j + len]);
j += 1
}
start = j + len;
}
len <<= 1;
}
for j in 0..256 {
r[j] = fqmul(r[j], F);
}
}
// Name: basemul
//
// Description: Multiplication of polynomials in Zq[X]/((X^2-zeta))
// used for multiplication of elements in Rq in NTT domain
//
// Arguments: - i16 r[2]: the output polynomial
// - const i16 a[2]: the first factor
// - const i16 b[2]: the second factor
// - i16 zeta: integer defining the reduction polynomial
pub fn basemul(r: &mut[i16], a: &[i16], b: &[i16], zeta: i16)
{
r[0] = fqmul(a[1], b[1]);
r[0] = fqmul(r[0], zeta);
r[0] += fqmul(a[0], b[0]);
r[1] = fqmul(a[0], b[1]);
r[1] += fqmul(a[1], b[0]);
}
use crate::reduce::*;
// Code to generate zetas used in the number-theoretic transform:
//
//#define KYBER_ROOT_OF_UNITY 17
//
//static const uint8_t tree[128] = {
// 0, 64, 32, 96, 16, 80, 48, 112, 8, 72, 40, 104, 24, 88, 56, 120,
// 4, 68, 36, 100, 20, 84, 52, 116, 12, 76, 44, 108, 28, 92, 60, 124,
// 2, 66, 34, 98, 18, 82, 50, 114, 10, 74, 42, 106, 26, 90, 58, 122,
// 6, 70, 38, 102, 22, 86, 54, 118, 14, 78, 46, 110, 30, 94, 62, 126,
// 1, 65, 33, 97, 17, 81, 49, 113, 9, 73, 41, 105, 25, 89, 57, 121,
// 5, 69, 37, 101, 21, 85, 53, 117, 13, 77, 45, 109, 29, 93, 61, 125,
// 3, 67, 35, 99, 19, 83, 51, 115, 11, 75, 43, 107, 27, 91, 59, 123,
// 7, 71, 39, 103, 23, 87, 55, 119, 15, 79, 47, 111, 31, 95, 63, 127};
//
//
// static int16_t fqmul(int16_t a, int16_t b) {
// return montgomery_reduce((int32_t)a*b);
//}
//
// void init_ntt() {
// unsigned int i;
// int16_t tmp[128];
//
// tmp[0] = MONT;
// for(i = 1; i < 128; ++i)
// tmp[i] = fqmul(tmp[i-1], MONT*KYBER_ROOT_OF_UNITY % KYBER_Q);
//
// for(i = 0; i < 128; ++i)
// zetas[i] = tmp[tree[i]];
//
// if(zetas[i] > KYBER_Q/2)
// zetas[i] -= KYBER_Q;
// if(zetas[i] < -KYBER_Q/2)
// zetas[i] += KYBER_Q;
//}
pub const ZETAS: [i16; 128] = [
-1044, -758, -359, -1517, 1493, 1422, 287, 202, -171, 622, 1577, 182, 962, -1202, -1474, 1468, 573, -1325, 264, 383, -829, 1458, -1602,
-130, -681, 1017, 732, 608, -1542, 411, -205, -1571, 1223, 652, -552, 1015, -1293, 1491, -282, -1544, 516, -8, -320, -666, -1618,
-1162, 126, 1469, -853, -90, -271, 830, 107, -1421, -247, -951, -398, 961, -1508, -725, 448, -1065, 677, -1275, -1103, 430, 555, 843,
-1251, 871, 1550, 105, 422, 587, 177, -235, -291, -460, 1574, 1653, -246, 778, 1159, -147, -777, 1483, -602, 1119, -1590, 644, -872,
349, 418, 329, -156, -75, 817, 1097, 603, 610, 1322, -1285, -1465, 384, -1215, -136, 1218, -1335, -874, 220, -1187, -1659, -1185,
-1530, -1278, 794, -1510, -854, -870, 478, -108, -308, 996, 991, 958, -1460, 1522, 1628,
];
// Name: fqmul
//
// Description: Multiplication followed by Montgomery reduction
//
// Arguments: - i16 a: first factor
// - i16 b: second factor
//
// Returns 16-bit integer congruent to a*b*R^{-1} mod q
pub fn fqmul(a: i16, b: i16) -> i16 {
montgomery_reduce(a as i32 * b as i32)
}
// Name: ntt
//
// Description: Inplace number-theoretic transform (NTT) in Rq
// input is in standard order, output is in bitreversed order
//
// Arguments: - i16 r[256]: input/output vector of elements of Zq
pub fn ntt(r: &mut [i16]) {
let mut j;
let mut k = 1usize;
let mut len = 128;
let (mut t, mut zeta);
while len >= 2 {
let mut start = 0;
while start < 256 {
zeta = ZETAS[k];
k += 1;
j = start;
while j < (start + len) {
t = fqmul(zeta, r[j + len]);
r[j + len] = r[j] - t;
r[j] += t;
j += 1;
}
start = j + len;
}
len >>= 1;
}
}
// Name: invntt
//
// Description: Inplace inverse number-theoretic transform in Rq
// input is in bitreversed order, output is in standard order
//
// Arguments: - i16 r[256]: input/output vector of elements of Zq
pub fn invntt(r: &mut [i16]) {
let mut j;
let mut k = 127usize;
let mut len = 2;
let (mut t, mut zeta);
const F: i16 = 1441; // mont^2/128
while len <= 128 {
let mut start = 0;
while start < 256 {
zeta = ZETAS[k];
k -= 1;
j = start;
while j < (start + len) {
t = r[j];
r[j] = barrett_reduce(t + r[j + len]);
r[j + len] = r[j + len] - t;
r[j + len] = fqmul(zeta, r[j + len]);
j += 1
}
start = j + len;
}
len <<= 1;
}
for j in 0..256 {
r[j] = fqmul(r[j], F);
}
}
// Name: basemul
//
// Description: Multiplication of polynomials in Zq[X]/((X^2-zeta))
// used for multiplication of elements in Rq in NTT domain
//
// Arguments: - i16 r[2]: the output polynomial
// - const i16 a[2]: the first factor
// - const i16 b[2]: the second factor
// - i16 zeta: integer defining the reduction polynomial
pub fn basemul(r: &mut [i16], a: &[i16], b: &[i16], zeta: i16) {
r[0] = fqmul(a[1], b[1]);
r[0] = fqmul(r[0], zeta);
r[0] += fqmul(a[0], b[0]);
r[1] = fqmul(a[0], b[1]);
r[1] += fqmul(a[1], b[0]);
}

639
third_party/kyber/src/reference/poly.rs vendored
View file

@ -1,335 +1,304 @@
use crate::{
params::*,
ntt::*,
reduce::*,
cbd::*,
symmetric::*
};
#[derive(Clone)]
pub struct Poly {
pub coeffs: [i16; KYBER_N]
}
impl Copy for Poly {}
impl Default for Poly {
fn default() -> Self {
Poly {
coeffs: [0i16; KYBER_N]
}
}
}
// new() is nicer
impl Poly {
pub fn new() -> Self {
Self::default()
}
}
// Name: poly_compress
//
// Description: Compression and subsequent serialization of a polynomial
//
// Arguments: - [u8] r: output byte array (needs space for KYBER_POLYCOMPRESSEDBYTES bytes)
// - const poly *a: input polynomial
pub fn poly_compress(r: &mut[u8], a: Poly)
{
let mut t = [0u8; 8];
let mut k = 0usize;
let mut u: i16;
match KYBER_POLYCOMPRESSEDBYTES {
128 => {
for i in 0..KYBER_N/8 {
for j in 0..8 {
// map to positive standard representatives
u = a.coeffs[8*i+j];
u += (u >> 15) & KYBER_Q as i16;
t[j] = (((((u as u16) << 4) + KYBER_Q as u16 /2) / KYBER_Q as u16) & 15) as u8;
}
r[k] = t[0] | (t[1] << 4);
r[k+1] = t[2] | (t[3] << 4);
r[k+2] = t[4] | (t[5] << 4);
r[k+3] = t[6] | (t[7] << 4);
k += 4;
}
},
160 => {
for i in 0..(KYBER_N/8) {
for j in 0..8 {
// map to positive standard representatives
u = a.coeffs[8*i+j];
u += (u >> 15) & KYBER_Q as i16;
t[j] = (((((u as u32) << 5) + KYBER_Q as u32/2) / KYBER_Q as u32) & 31) as u8;
}
r[k] = t[0] | (t[1] << 5);
r[k+1] = (t[1] >> 3) | (t[2] << 2) | (t[3] << 7);
r[k+2] = (t[3] >> 1) | (t[4] << 4);
r[k+3] = (t[4] >> 4) | (t[5] << 1) | (t[6] << 6);
r[k+4] = (t[6] >> 2) | (t[7] << 3);
k += 5;
}
},
_ => panic!("KYBER_POLYCOMPRESSEDBYTES needs to be one of (128, 160)")
}
}
// Name: poly_decompress
//
// Description: De-serialization and subsequent decompression of a polynomial;
// approximate inverse of poly_compress
//
// Arguments: - poly *r: output polynomial
// - const [u8] a: input byte array (of length KYBER_POLYCOMPRESSEDBYTES bytes)
pub fn poly_decompress(r: &mut Poly, a: &[u8])
{
match KYBER_POLYCOMPRESSEDBYTES {
128 => {
let mut idx = 0usize;
for i in 0..KYBER_N/2 {
r.coeffs[2*i+0] = ((((a[idx] & 15) as usize * KYBER_Q) + 8) >> 4) as i16;
r.coeffs[2*i+1] = ((((a[idx] >> 4) as usize * KYBER_Q) + 8) >> 4) as i16;
idx += 1;
}
},
160 => {
let mut idx = 0usize;
let mut t = [0u8;8];
for i in 0..KYBER_N/8 {
t[0] = a[idx+0];
t[1] = (a[idx+0] >> 5) | (a[idx+1] << 3);
t[2] = a[idx+1] >> 2;
t[3] = (a[idx+1] >> 7) | (a[idx+2] << 1);
t[4] = (a[idx+2] >> 4) | (a[idx+3] << 4);
t[5] = a[idx+3] >> 1;
t[6] = (a[idx+3] >> 6) | (a[idx+4] << 2);
t[7] = a[idx+4] >> 3;
idx += 5;
for j in 0..8 {
r.coeffs[8*i+j] = ((((t[j] as u32) & 31)*KYBER_Q as u32 + 16) >> 5) as i16;
}
}
},
_ => panic!("KYBER_POLYCOMPRESSEDBYTES needs to be either (128, 160)")
}
}
// Name: poly_tobytes
//
// Description: Serialization of a polynomial
//
// Arguments: - [u8] r: output byte array (needs space for KYBER_POLYBYTES bytes)
// - const poly *a: input polynomial
pub fn poly_tobytes(r: &mut[u8], a: Poly)
{
let (mut t0, mut t1);
for i in 0..(KYBER_N/2) {
// map to positive standard representatives
t0 = a.coeffs[2*i];
t0 += (t0 >> 15) & KYBER_Q as i16;
t1 = a.coeffs[2*i+1];
t1 += (t1 >> 15) & KYBER_Q as i16;
r[3*i+0] = (t0 >> 0) as u8;
r[3*i+1] = ((t0 >> 8) | (t1 << 4)) as u8;
r[3*i+2] = (t1 >> 4) as u8;
}
}
// Name: poly_frombytes
//
// Description: De-serialization of a polynomial;
// inverse of poly_tobytes
//
// Arguments: - poly *r: output polynomial
// - const [u8] a: input byte array (of KYBER_POLYBYTES bytes)
pub fn poly_frombytes(r: &mut Poly, a: &[u8])
{
for i in 0..(KYBER_N/2) {
r.coeffs[2*i+0] = ((a[3*i+0] >> 0) as u16 | ((a[3*i+1] as u16) << 8) & 0xFFF) as i16;
r.coeffs[2*i+1] = ((a[3*i+1] >> 4) as u16 | ((a[3*i+2] as u16) << 4) & 0xFFF) as i16;
}
}
// Name: poly_getnoise_eta1
//
// Description: Sample a polynomial deterministically from a seed and a nonce,
// with output polynomial close to centered binomial distribution
// with parameter KYBER_ETA1
//
// Arguments: - poly *r: output polynomial
// - const [u8] seed: input seed (pointing to array of length KYBER_SYMBYTES bytes)
// - [u8] nonce: one-byte input nonce
pub fn poly_getnoise_eta1(r: &mut Poly, seed: &[u8], nonce: u8)
{
const LENGTH: usize = KYBER_ETA1*KYBER_N/4;
let mut buf = [0u8; LENGTH];
prf(&mut buf, LENGTH, seed, nonce);
poly_cbd_eta1(r, &buf);
}
// Name: poly_getnoise_eta2
//
// Description: Sample a polynomial deterministically from a seed and a nonce,
// with output polynomial close to centered binomial distribution
// with parameter KYBER_ETA2
//
// Arguments: - poly *r: output polynomial
// - const [u8] seed: input seed (pointing to array of length KYBER_SYMBYTES bytes)
// - [u8] nonce: one-byte input nonce
pub fn poly_getnoise_eta2(r: &mut Poly, seed: &[u8], nonce: u8)
{
const LENGTH: usize = KYBER_ETA2*KYBER_N/4;
let mut buf = [0u8; LENGTH];
prf(&mut buf, LENGTH, seed, nonce);
poly_cbd_eta2(r, &buf);
}
// Name: poly_ntt
//
// Description: Computes negacyclic number-theoretic transform (NTT) of
// a polynomial in place;
// inputs assumed to be in normal order, output in bitreversed order
//
// Arguments: - Poly r: in/output polynomial
pub fn poly_ntt(r: &mut Poly)
{
ntt(&mut r.coeffs);
poly_reduce(r);
}
// Name: poly_invntt
//
// Description: Computes inverse of negacyclic number-theoretic transform (NTT) of
// a polynomial in place;
// inputs assumed to be in bitreversed order, output in normal order
//
// Arguments: - Poly a: in/output polynomial
pub fn poly_invntt_tomont(r: &mut Poly)
{
invntt(&mut r.coeffs);
}
// Name: poly_basemul
//
// Description: Multiplication of two polynomials in NTT domain
//
// Arguments: - poly *r: output polynomial
// - const poly *a: first input polynomial
// - const poly *b: second input polynomial
pub fn poly_basemul(r: &mut Poly, a: &Poly, b: &Poly)
{
for i in 0..(KYBER_N/4) {
basemul(
&mut r.coeffs[4*i..],
&a.coeffs[4*i..],
&b.coeffs[4*i..],
ZETAS[64 + i]
);
basemul(
&mut r.coeffs[4*i+2..],
&a.coeffs[4*i+2..],
&b.coeffs[4*i+2..],
-(ZETAS[64 + i]));
}
}
// Name: poly_frommont
//
// Description: Inplace conversion of all coefficients of a polynomial
// from Montgomery domain to normal domain
//
// Arguments: - poly *r: input/output polynomial
pub fn poly_frommont(r: &mut Poly)
{
let f = ((1u64 << 32) % KYBER_Q as u64) as i16;
for i in 0..KYBER_N {
let a = r.coeffs[i] as i32 * f as i32;
r.coeffs[i] = montgomery_reduce(a);
}
}
// Name: poly_reduce
//
// Description: Applies Barrett reduction to all coefficients of a polynomial
// for details of the Barrett reduction see comments in reduce.c
//
// Arguments: - poly *r: input/output polynomial
pub fn poly_reduce(r: &mut Poly)
{
for i in 0..KYBER_N {
r.coeffs[i] = barrett_reduce(r.coeffs[i]);
}
}
// Name: poly_add
//
// Description: Add two polynomials; no modular reduction is performed
//
// Arguments: - poly *r: output polynomial
// - const poly *a: first input polynomial
// - const poly *b: second input polynomial
pub fn poly_add(r: &mut Poly, b: &Poly)
{
for i in 0..KYBER_N {
r.coeffs[i] += b.coeffs[i];
}
}
// Name: poly_sub
//
// Description: Subtract two polynomials; no modular reduction is performed
//
// Arguments: - poly *r: output polynomial
// - const poly *a: first input polynomial
// - const poly *b: second input polynomial
pub fn poly_sub(r: &mut Poly, a: &Poly)
{
for i in 0..KYBER_N {
r.coeffs[i] = a.coeffs[i] - r.coeffs[i];
}
}
// Name: poly_frommsg
//
// Description: Convert 32-byte message to polynomial
//
// Arguments: - poly *r: output polynomial
// - const [u8] msg: input message
pub fn poly_frommsg(r: &mut Poly, msg: &[u8])
{
let mut mask;
for i in 0..KYBER_SYMBYTES {
for j in 0..8 {
mask = ((msg[i] as u16 >> j) & 1 ).wrapping_neg();
r.coeffs[8*i+j] = (mask & ((KYBER_Q+1)/2) as u16) as i16;
}
}
}
// Name: poly_tomsg
//
// Description: Convert polynomial to 32-byte message
//
// Arguments: - [u8] msg: output message
// - const poly *a: input polynomial
pub fn poly_tomsg(msg: &mut[u8], a: Poly)
{
let mut t;
for i in 0..KYBER_SYMBYTES {
msg[i] = 0;
for j in 0..8 {
t = a.coeffs[8*i+j];
t += (t >> 15) & KYBER_Q as i16;
t = (((t << 1) + KYBER_Q as i16 /2) / KYBER_Q as i16) & 1;
msg[i] |= (t << j) as u8;
}
}
}
use crate::{cbd::*, ntt::*, params::*, reduce::*, symmetric::*};
#[derive(Clone)]
pub struct Poly {
pub coeffs: [i16; KYBER_N],
}
impl Copy for Poly {}
impl Default for Poly {
fn default() -> Self {
Poly { coeffs: [0i16; KYBER_N] }
}
}
// new() is nicer
impl Poly {
pub fn new() -> Self {
Self::default()
}
}
// Name: poly_compress
//
// Description: Compression and subsequent serialization of a polynomial
//
// Arguments: - [u8] r: output byte array (needs space for KYBER_POLYCOMPRESSEDBYTES bytes)
// - const poly *a: input polynomial
pub fn poly_compress(r: &mut [u8], a: Poly) {
let mut t = [0u8; 8];
let mut k = 0usize;
let mut u: i16;
match KYBER_POLYCOMPRESSEDBYTES {
128 => {
for i in 0..KYBER_N / 8 {
for j in 0..8 {
// map to positive standard representatives
u = a.coeffs[8 * i + j];
u += (u >> 15) & KYBER_Q as i16;
t[j] = (((((u as u16) << 4) + KYBER_Q as u16 / 2) / KYBER_Q as u16) & 15) as u8;
}
r[k] = t[0] | (t[1] << 4);
r[k + 1] = t[2] | (t[3] << 4);
r[k + 2] = t[4] | (t[5] << 4);
r[k + 3] = t[6] | (t[7] << 4);
k += 4;
}
}
160 => {
for i in 0..(KYBER_N / 8) {
for j in 0..8 {
// map to positive standard representatives
u = a.coeffs[8 * i + j];
u += (u >> 15) & KYBER_Q as i16;
t[j] = (((((u as u32) << 5) + KYBER_Q as u32 / 2) / KYBER_Q as u32) & 31) as u8;
}
r[k] = t[0] | (t[1] << 5);
r[k + 1] = (t[1] >> 3) | (t[2] << 2) | (t[3] << 7);
r[k + 2] = (t[3] >> 1) | (t[4] << 4);
r[k + 3] = (t[4] >> 4) | (t[5] << 1) | (t[6] << 6);
r[k + 4] = (t[6] >> 2) | (t[7] << 3);
k += 5;
}
}
_ => panic!("KYBER_POLYCOMPRESSEDBYTES needs to be one of (128, 160)"),
}
}
// Name: poly_decompress
//
// Description: De-serialization and subsequent decompression of a polynomial;
// approximate inverse of poly_compress
//
// Arguments: - poly *r: output polynomial
// - const [u8] a: input byte array (of length KYBER_POLYCOMPRESSEDBYTES bytes)
pub fn poly_decompress(r: &mut Poly, a: &[u8]) {
match KYBER_POLYCOMPRESSEDBYTES {
128 => {
let mut idx = 0usize;
for i in 0..KYBER_N / 2 {
r.coeffs[2 * i + 0] = ((((a[idx] & 15) as usize * KYBER_Q) + 8) >> 4) as i16;
r.coeffs[2 * i + 1] = ((((a[idx] >> 4) as usize * KYBER_Q) + 8) >> 4) as i16;
idx += 1;
}
}
160 => {
let mut idx = 0usize;
let mut t = [0u8; 8];
for i in 0..KYBER_N / 8 {
t[0] = a[idx + 0];
t[1] = (a[idx + 0] >> 5) | (a[idx + 1] << 3);
t[2] = a[idx + 1] >> 2;
t[3] = (a[idx + 1] >> 7) | (a[idx + 2] << 1);
t[4] = (a[idx + 2] >> 4) | (a[idx + 3] << 4);
t[5] = a[idx + 3] >> 1;
t[6] = (a[idx + 3] >> 6) | (a[idx + 4] << 2);
t[7] = a[idx + 4] >> 3;
idx += 5;
for j in 0..8 {
r.coeffs[8 * i + j] = ((((t[j] as u32) & 31) * KYBER_Q as u32 + 16) >> 5) as i16;
}
}
}
_ => panic!("KYBER_POLYCOMPRESSEDBYTES needs to be either (128, 160)"),
}
}
// Name: poly_tobytes
//
// Description: Serialization of a polynomial
//
// Arguments: - [u8] r: output byte array (needs space for KYBER_POLYBYTES bytes)
// - const poly *a: input polynomial
pub fn poly_tobytes(r: &mut [u8], a: Poly) {
let (mut t0, mut t1);
for i in 0..(KYBER_N / 2) {
// map to positive standard representatives
t0 = a.coeffs[2 * i];
t0 += (t0 >> 15) & KYBER_Q as i16;
t1 = a.coeffs[2 * i + 1];
t1 += (t1 >> 15) & KYBER_Q as i16;
r[3 * i + 0] = (t0 >> 0) as u8;
r[3 * i + 1] = ((t0 >> 8) | (t1 << 4)) as u8;
r[3 * i + 2] = (t1 >> 4) as u8;
}
}
// Name: poly_frombytes
//
// Description: De-serialization of a polynomial;
// inverse of poly_tobytes
//
// Arguments: - poly *r: output polynomial
// - const [u8] a: input byte array (of KYBER_POLYBYTES bytes)
pub fn poly_frombytes(r: &mut Poly, a: &[u8]) {
for i in 0..(KYBER_N / 2) {
r.coeffs[2 * i + 0] = ((a[3 * i + 0] >> 0) as u16 | ((a[3 * i + 1] as u16) << 8) & 0xFFF) as i16;
r.coeffs[2 * i + 1] = ((a[3 * i + 1] >> 4) as u16 | ((a[3 * i + 2] as u16) << 4) & 0xFFF) as i16;
}
}
// Name: poly_getnoise_eta1
//
// Description: Sample a polynomial deterministically from a seed and a nonce,
// with output polynomial close to centered binomial distribution
// with parameter KYBER_ETA1
//
// Arguments: - poly *r: output polynomial
// - const [u8] seed: input seed (pointing to array of length KYBER_SYMBYTES bytes)
// - [u8] nonce: one-byte input nonce
pub fn poly_getnoise_eta1(r: &mut Poly, seed: &[u8], nonce: u8) {
const LENGTH: usize = KYBER_ETA1 * KYBER_N / 4;
let mut buf = [0u8; LENGTH];
prf(&mut buf, LENGTH, seed, nonce);
poly_cbd_eta1(r, &buf);
}
// Name: poly_getnoise_eta2
//
// Description: Sample a polynomial deterministically from a seed and a nonce,
// with output polynomial close to centered binomial distribution
// with parameter KYBER_ETA2
//
// Arguments: - poly *r: output polynomial
// - const [u8] seed: input seed (pointing to array of length KYBER_SYMBYTES bytes)
// - [u8] nonce: one-byte input nonce
pub fn poly_getnoise_eta2(r: &mut Poly, seed: &[u8], nonce: u8) {
const LENGTH: usize = KYBER_ETA2 * KYBER_N / 4;
let mut buf = [0u8; LENGTH];
prf(&mut buf, LENGTH, seed, nonce);
poly_cbd_eta2(r, &buf);
}
// Name: poly_ntt
//
// Description: Computes negacyclic number-theoretic transform (NTT) of
// a polynomial in place;
// inputs assumed to be in normal order, output in bitreversed order
//
// Arguments: - Poly r: in/output polynomial
pub fn poly_ntt(r: &mut Poly) {
ntt(&mut r.coeffs);
poly_reduce(r);
}
// Name: poly_invntt
//
// Description: Computes inverse of negacyclic number-theoretic transform (NTT) of
// a polynomial in place;
// inputs assumed to be in bitreversed order, output in normal order
//
// Arguments: - Poly a: in/output polynomial
pub fn poly_invntt_tomont(r: &mut Poly) {
invntt(&mut r.coeffs);
}
// Name: poly_basemul
//
// Description: Multiplication of two polynomials in NTT domain
//
// Arguments: - poly *r: output polynomial
// - const poly *a: first input polynomial
// - const poly *b: second input polynomial
pub fn poly_basemul(r: &mut Poly, a: &Poly, b: &Poly) {
for i in 0..(KYBER_N / 4) {
basemul(&mut r.coeffs[4 * i..], &a.coeffs[4 * i..], &b.coeffs[4 * i..], ZETAS[64 + i]);
basemul(
&mut r.coeffs[4 * i + 2..],
&a.coeffs[4 * i + 2..],
&b.coeffs[4 * i + 2..],
-(ZETAS[64 + i]),
);
}
}
// Name: poly_frommont
//
// Description: Inplace conversion of all coefficients of a polynomial
// from Montgomery domain to normal domain
//
// Arguments: - poly *r: input/output polynomial
pub fn poly_frommont(r: &mut Poly) {
let f = ((1u64 << 32) % KYBER_Q as u64) as i16;
for i in 0..KYBER_N {
let a = r.coeffs[i] as i32 * f as i32;
r.coeffs[i] = montgomery_reduce(a);
}
}
// Name: poly_reduce
//
// Description: Applies Barrett reduction to all coefficients of a polynomial
// for details of the Barrett reduction see comments in reduce.c
//
// Arguments: - poly *r: input/output polynomial
pub fn poly_reduce(r: &mut Poly) {
for i in 0..KYBER_N {
r.coeffs[i] = barrett_reduce(r.coeffs[i]);
}
}
// Name: poly_add
//
// Description: Add two polynomials; no modular reduction is performed
//
// Arguments: - poly *r: output polynomial
// - const poly *a: first input polynomial
// - const poly *b: second input polynomial
pub fn poly_add(r: &mut Poly, b: &Poly) {
for i in 0..KYBER_N {
r.coeffs[i] += b.coeffs[i];
}
}
// Name: poly_sub
//
// Description: Subtract two polynomials; no modular reduction is performed
//
// Arguments: - poly *r: output polynomial
// - const poly *a: first input polynomial
// - const poly *b: second input polynomial
pub fn poly_sub(r: &mut Poly, a: &Poly) {
for i in 0..KYBER_N {
r.coeffs[i] = a.coeffs[i] - r.coeffs[i];
}
}
// Name: poly_frommsg
//
// Description: Convert 32-byte message to polynomial
//
// Arguments: - poly *r: output polynomial
// - const [u8] msg: input message
pub fn poly_frommsg(r: &mut Poly, msg: &[u8]) {
let mut mask;
for i in 0..KYBER_SYMBYTES {
for j in 0..8 {
mask = ((msg[i] as u16 >> j) & 1).wrapping_neg();
r.coeffs[8 * i + j] = (mask & ((KYBER_Q + 1) / 2) as u16) as i16;
}
}
}
// Name: poly_tomsg
//
// Description: Convert polynomial to 32-byte message
//
// Arguments: - [u8] msg: output message
// - const poly *a: input polynomial
pub fn poly_tomsg(msg: &mut [u8], a: Poly) {
let mut t;
for i in 0..KYBER_SYMBYTES {
msg[i] = 0;
for j in 0..8 {
t = a.coeffs[8 * i + j];
t += (t >> 15) & KYBER_Q as i16;
t = (((t << 1) + KYBER_Q as i16 / 2) / KYBER_Q as i16) & 1;
msg[i] |= (t << j) as u8;
}
}
}

463
third_party/kyber/src/reference/polyvec.rs vendored
View file

@ -1,241 +1,222 @@
#![allow(clippy::precedence)]
use crate::{
poly::*,
params::*
};
#[derive(Clone)]
pub struct Polyvec {
pub vec: [Poly; KYBER_K]
}
impl Copy for Polyvec {}
impl Polyvec {
pub fn new() -> Self {
Polyvec {
vec: [Poly::new(); KYBER_K]
}
}
// #[cfg(debug_assertions)]
// pub fn checksum(&self) -> i16 {
// let mut out = 0i16;
// for i in 0..KYBER_K {
// for j in 0..KYBER_N {
// out ^= &self.vec[i].coeffs[j]
// }
// }
// out
// }
}
// Name: polyvec_compress
//
// Description: Compress and serialize vector of polynomials
//
// Arguments: - [u8] r: output byte array (needs space for KYBER_POLYVECCOMPRESSEDBYTES)
// - const Polyvec a: input vector of polynomials
pub fn polyvec_compress(r: &mut[u8], a: Polyvec)
{
#[cfg(feature="kyber1024")]
{
let mut t = [0u16; 8];
let mut idx = 0usize;
for i in 0..KYBER_K {
for j in 0..KYBER_N/8 {
for k in 0..8 {
t[k] = a.vec[i].coeffs[8*j+k] as u16;
t[k] = t[k].wrapping_add((((t[k] as i16) >> 15) & KYBER_Q as i16) as u16);
t[k] = (((((t[k] as u32) << 11) + KYBER_Q as u32/2)/KYBER_Q as u32) & 0x7ff ) as u16;
}
r[idx+0] = (t[0] >> 0) as u8;
r[idx+1] = ((t[0] >> 8) | (t[1] << 3)) as u8;
r[idx+2] = ((t[1] >> 5) | (t[2] << 6)) as u8;
r[idx+3] = (t[2] >> 2) as u8;
r[idx+4] = ((t[2] >> 10) | (t[3] << 1)) as u8;
r[idx+5] = ((t[3] >> 7) | (t[4] << 4)) as u8;
r[idx+6] = ((t[4] >> 4) | (t[5] << 7)) as u8;
r[idx+7] = (t[5] >> 1) as u8;
r[idx+8] = ((t[5] >> 9) | (t[6] << 2)) as u8;
r[idx+9] = ((t[6] >> 6) | (t[7] << 5)) as u8;
r[idx+10] = (t[7] >> 3) as u8;
idx += 11
}
}
}
#[cfg(not(feature="kyber1024"))]
{
let mut t = [0u16; 4];
let mut idx = 0usize;
for i in 0..KYBER_K {
for j in 0..KYBER_N/4 {
for k in 0..4 {
t[k] = a.vec[i].coeffs[4*j+k] as u16;
t[k] = t[k].wrapping_add((((t[k] as i16) >> 15) & KYBER_Q as i16) as u16);
t[k] =
(((((t[k] as u32) << 10) + KYBER_Q as u32/2)/ KYBER_Q as u32) & 0x3ff) as u16;
}
r[idx+0] = (t[0] >> 0) as u8;
r[idx+1] = ((t[0] >> 8) | (t[1] << 2)) as u8;
r[idx+2] = ((t[1] >> 6) | (t[2] << 4)) as u8;
r[idx+3] = ((t[2] >> 4) | (t[3] << 6)) as u8;
r[idx+4] = (t[3] >> 2) as u8;
idx += 5;
}
}
}
}
// Name: polyvec_decompress
//
// Description: De-serialize and decompress vector of polynomials;
// approximate inverse of polyvec_compress
//
// Arguments: - Polyvec r: output vector of polynomials
// - [u8] a: input byte array (of length KYBER_POLYVECCOMPRESSEDBYTES)
pub fn polyvec_decompress(r: &mut Polyvec, a: &[u8])
{
#[cfg(feature="kyber1024")]
{
let mut t = [0u16; 8];
let mut idx = 0usize;
for i in 0..KYBER_K {
for j in 0..KYBER_N/8 {
t[0] = (a[idx+0] >> 0) as u16 | (a[idx+ 1] as u16) << 8;
t[1] = (a[idx+1] >> 3) as u16 | (a[idx+ 2] as u16) << 5;
t[2] = (a[idx+2] >> 6) as u16 | (a[idx+ 3] as u16) << 2 | (a[idx+4] as u16) << 10;
t[3] = (a[idx+4] >> 1) as u16 | (a[idx+ 5] as u16) << 7;
t[4] = (a[idx+5] >> 4) as u16 | (a[idx+ 6] as u16) << 4;
t[5] = (a[idx+6] >> 7) as u16 | (a[idx+ 7] as u16) << 1 | (a[idx+8] as u16) << 9;
t[6] = (a[idx+8] >> 2) as u16 | (a[idx+ 9] as u16) << 6;
t[7] = (a[idx+9] >> 5) as u16 | (a[idx+10] as u16) << 3;
idx += 11;
for k in 0..8 {
r.vec[i].coeffs[8*j+k] =
(((t[k] & 0x7FF)as u32 * KYBER_Q as u32 + 1024) >> 11) as i16;
}
}
}
}
#[cfg(not(feature="kyber1024"))]
{
let mut idx = 0usize;
let mut t = [0u16; 4];
for i in 0..KYBER_K {
for j in 0..KYBER_N/4 {
t[0] = (a[idx+0] >> 0) as u16 | (a[idx+1] as u16) << 8;
t[1] = (a[idx+1] >> 2) as u16 | (a[idx+2] as u16) << 6;
t[2] = (a[idx+2] >> 4) as u16 | (a[idx+3] as u16) << 4;
t[3] = (a[idx+3] >> 6) as u16 | (a[idx+4] as u16) << 2;
idx += 5;
for k in 0..4 {
r.vec[i].coeffs[4*j+k] =
((((t[k] as u32) & 0x3FF) * KYBER_Q as u32 + 512) >> 10) as i16;
}
}
}
}
}
// Name: polyvec_tobytes
//
// Description: Serialize vector of polynomials
//
// Arguments: - [u8] r: output byte array (needs space for KYBER_POLYVECBYTES)
// - const Polyvec a: input vector of polynomials
pub fn polyvec_tobytes(r: &mut[u8], a: &Polyvec)
{
for i in 0..KYBER_K {
poly_tobytes(&mut r[i*KYBER_POLYBYTES..], a.vec[i]);
}
}
// Name: polyvec_frombytes
//
// Description: De-serialize vector of polynomials;
// inverse of polyvec_tobytes
//
// Arguments: - [u8] r: output byte array
// - const Polyvec a: input vector of polynomials (of length KYBER_POLYVECBYTES)
pub fn polyvec_frombytes(r: &mut Polyvec, a: &[u8])
{
for i in 0..KYBER_K {
poly_frombytes(&mut r.vec[i], &a[i*KYBER_POLYBYTES..]);
}
}
// Name: polyvec_ntt
//
// Description: Apply forward NTT to all elements of a vector of polynomials
//
// Arguments: - Polyvec r: in/output vector of polynomials
pub fn polyvec_ntt(r: &mut Polyvec)
{
for i in 0..KYBER_K {
poly_ntt(&mut r.vec[i]);
}
}
// Name: polyvec_invntt
//
// Description: Apply inverse NTT to all elements of a vector of polynomials
//
// Arguments: - Polyvec r: in/output vector of polynomials
pub fn polyvec_invntt_tomont(r: &mut Polyvec)
{
for i in 0..KYBER_K {
poly_invntt_tomont(&mut r.vec[i]);
}
}
// Name: polyvec_basemul_acc_montgomery
//
// Description: Pointwise multiply elements of a and b and accumulate into r
//
// Arguments: - poly *r: output polynomial
// - const Polyvec a: first input vector of polynomials
// - const Polyvec b: second input vector of polynomials
pub fn polyvec_basemul_acc_montgomery(r: &mut Poly, a: &Polyvec, b: &Polyvec)
{
let mut t = Poly::new();
poly_basemul(r, &a.vec[0], &b.vec[0]);
for i in 1..KYBER_K {
poly_basemul(&mut t, &a.vec[i], &b.vec[i]);
poly_add(r, &t);
}
poly_reduce(r);
}
// Name: polyvec_reduce
//
// Description: Applies Barrett reduction to each coefficient
// of each element of a vector of polynomials
// for details of the Barrett reduction see comments in reduce.c
//
// Arguments: - poly *r: input/output polynomial
pub fn polyvec_reduce(r: &mut Polyvec)
{
for i in 0..KYBER_K {
poly_reduce(&mut r.vec[i]);
}
}
// Name: polyvec_add
//
// Description: Add vectors of polynomials
//
// Arguments: - Polyvec r: output vector of polynomials
// - const Polyvec b: second input vector of polynomials
pub fn polyvec_add(r: &mut Polyvec, b: &Polyvec)
{
for i in 0..KYBER_K {
poly_add(&mut r.vec[i], &b.vec[i]);
}
}
#![allow(clippy::precedence)]
use crate::{params::*, poly::*};
#[derive(Clone)]
pub struct Polyvec {
pub vec: [Poly; KYBER_K],
}
impl Copy for Polyvec {}
impl Polyvec {
pub fn new() -> Self {
Polyvec { vec: [Poly::new(); KYBER_K] }
}
// #[cfg(debug_assertions)]
// pub fn checksum(&self) -> i16 {
// let mut out = 0i16;
// for i in 0..KYBER_K {
// for j in 0..KYBER_N {
// out ^= &self.vec[i].coeffs[j]
// }
// }
// out
// }
}
// Name: polyvec_compress
//
// Description: Compress and serialize vector of polynomials
//
// Arguments: - [u8] r: output byte array (needs space for KYBER_POLYVECCOMPRESSEDBYTES)
// - const Polyvec a: input vector of polynomials
pub fn polyvec_compress(r: &mut [u8], a: Polyvec) {
#[cfg(feature = "kyber1024")]
{
let mut t = [0u16; 8];
let mut idx = 0usize;
for i in 0..KYBER_K {
for j in 0..KYBER_N / 8 {
for k in 0..8 {
t[k] = a.vec[i].coeffs[8 * j + k] as u16;
t[k] = t[k].wrapping_add((((t[k] as i16) >> 15) & KYBER_Q as i16) as u16);
t[k] = (((((t[k] as u32) << 11) + KYBER_Q as u32 / 2) / KYBER_Q as u32) & 0x7ff) as u16;
}
r[idx + 0] = (t[0] >> 0) as u8;
r[idx + 1] = ((t[0] >> 8) | (t[1] << 3)) as u8;
r[idx + 2] = ((t[1] >> 5) | (t[2] << 6)) as u8;
r[idx + 3] = (t[2] >> 2) as u8;
r[idx + 4] = ((t[2] >> 10) | (t[3] << 1)) as u8;
r[idx + 5] = ((t[3] >> 7) | (t[4] << 4)) as u8;
r[idx + 6] = ((t[4] >> 4) | (t[5] << 7)) as u8;
r[idx + 7] = (t[5] >> 1) as u8;
r[idx + 8] = ((t[5] >> 9) | (t[6] << 2)) as u8;
r[idx + 9] = ((t[6] >> 6) | (t[7] << 5)) as u8;
r[idx + 10] = (t[7] >> 3) as u8;
idx += 11
}
}
}
#[cfg(not(feature = "kyber1024"))]
{
let mut t = [0u16; 4];
let mut idx = 0usize;
for i in 0..KYBER_K {
for j in 0..KYBER_N / 4 {
for k in 0..4 {
t[k] = a.vec[i].coeffs[4 * j + k] as u16;
t[k] = t[k].wrapping_add((((t[k] as i16) >> 15) & KYBER_Q as i16) as u16);
t[k] = (((((t[k] as u32) << 10) + KYBER_Q as u32 / 2) / KYBER_Q as u32) & 0x3ff) as u16;
}
r[idx + 0] = (t[0] >> 0) as u8;
r[idx + 1] = ((t[0] >> 8) | (t[1] << 2)) as u8;
r[idx + 2] = ((t[1] >> 6) | (t[2] << 4)) as u8;
r[idx + 3] = ((t[2] >> 4) | (t[3] << 6)) as u8;
r[idx + 4] = (t[3] >> 2) as u8;
idx += 5;
}
}
}
}
// Name: polyvec_decompress
//
// Description: De-serialize and decompress vector of polynomials;
// approximate inverse of polyvec_compress
//
// Arguments: - Polyvec r: output vector of polynomials
// - [u8] a: input byte array (of length KYBER_POLYVECCOMPRESSEDBYTES)
pub fn polyvec_decompress(r: &mut Polyvec, a: &[u8]) {
#[cfg(feature = "kyber1024")]
{
let mut t = [0u16; 8];
let mut idx = 0usize;
for i in 0..KYBER_K {
for j in 0..KYBER_N / 8 {
t[0] = (a[idx + 0] >> 0) as u16 | (a[idx + 1] as u16) << 8;
t[1] = (a[idx + 1] >> 3) as u16 | (a[idx + 2] as u16) << 5;
t[2] = (a[idx + 2] >> 6) as u16 | (a[idx + 3] as u16) << 2 | (a[idx + 4] as u16) << 10;
t[3] = (a[idx + 4] >> 1) as u16 | (a[idx + 5] as u16) << 7;
t[4] = (a[idx + 5] >> 4) as u16 | (a[idx + 6] as u16) << 4;
t[5] = (a[idx + 6] >> 7) as u16 | (a[idx + 7] as u16) << 1 | (a[idx + 8] as u16) << 9;
t[6] = (a[idx + 8] >> 2) as u16 | (a[idx + 9] as u16) << 6;
t[7] = (a[idx + 9] >> 5) as u16 | (a[idx + 10] as u16) << 3;
idx += 11;
for k in 0..8 {
r.vec[i].coeffs[8 * j + k] = (((t[k] & 0x7FF) as u32 * KYBER_Q as u32 + 1024) >> 11) as i16;
}
}
}
}
#[cfg(not(feature = "kyber1024"))]
{
let mut idx = 0usize;
let mut t = [0u16; 4];
for i in 0..KYBER_K {
for j in 0..KYBER_N / 4 {
t[0] = (a[idx + 0] >> 0) as u16 | (a[idx + 1] as u16) << 8;
t[1] = (a[idx + 1] >> 2) as u16 | (a[idx + 2] as u16) << 6;
t[2] = (a[idx + 2] >> 4) as u16 | (a[idx + 3] as u16) << 4;
t[3] = (a[idx + 3] >> 6) as u16 | (a[idx + 4] as u16) << 2;
idx += 5;
for k in 0..4 {
r.vec[i].coeffs[4 * j + k] = ((((t[k] as u32) & 0x3FF) * KYBER_Q as u32 + 512) >> 10) as i16;
}
}
}
}
}
// Name: polyvec_tobytes
//
// Description: Serialize vector of polynomials
//
// Arguments: - [u8] r: output byte array (needs space for KYBER_POLYVECBYTES)
// - const Polyvec a: input vector of polynomials
pub fn polyvec_tobytes(r: &mut [u8], a: &Polyvec) {
for i in 0..KYBER_K {
poly_tobytes(&mut r[i * KYBER_POLYBYTES..], a.vec[i]);
}
}
// Name: polyvec_frombytes
//
// Description: De-serialize vector of polynomials;
// inverse of polyvec_tobytes
//
// Arguments: - [u8] r: output byte array
// - const Polyvec a: input vector of polynomials (of length KYBER_POLYVECBYTES)
pub fn polyvec_frombytes(r: &mut Polyvec, a: &[u8]) {
for i in 0..KYBER_K {
poly_frombytes(&mut r.vec[i], &a[i * KYBER_POLYBYTES..]);
}
}
// Name: polyvec_ntt
//
// Description: Apply forward NTT to all elements of a vector of polynomials
//
// Arguments: - Polyvec r: in/output vector of polynomials
pub fn polyvec_ntt(r: &mut Polyvec) {
for i in 0..KYBER_K {
poly_ntt(&mut r.vec[i]);
}
}
// Name: polyvec_invntt
//
// Description: Apply inverse NTT to all elements of a vector of polynomials
//
// Arguments: - Polyvec r: in/output vector of polynomials
pub fn polyvec_invntt_tomont(r: &mut Polyvec) {
for i in 0..KYBER_K {
poly_invntt_tomont(&mut r.vec[i]);
}
}
// Name: polyvec_basemul_acc_montgomery
//
// Description: Pointwise multiply elements of a and b and accumulate into r
//
// Arguments: - poly *r: output polynomial
// - const Polyvec a: first input vector of polynomials
// - const Polyvec b: second input vector of polynomials
pub fn polyvec_basemul_acc_montgomery(r: &mut Poly, a: &Polyvec, b: &Polyvec) {
let mut t = Poly::new();
poly_basemul(r, &a.vec[0], &b.vec[0]);
for i in 1..KYBER_K {
poly_basemul(&mut t, &a.vec[i], &b.vec[i]);
poly_add(r, &t);
}
poly_reduce(r);
}
// Name: polyvec_reduce
//
// Description: Applies Barrett reduction to each coefficient
// of each element of a vector of polynomials
// for details of the Barrett reduction see comments in reduce.c
//
// Arguments: - poly *r: input/output polynomial
pub fn polyvec_reduce(r: &mut Polyvec) {
for i in 0..KYBER_K {
poly_reduce(&mut r.vec[i]);
}
}
// Name: polyvec_add
//
// Description: Add vectors of polynomials
//
// Arguments: - Polyvec r: output vector of polynomials
// - const Polyvec b: second input vector of polynomials
pub fn polyvec_add(r: &mut Polyvec, b: &Polyvec) {
for i in 0..KYBER_K {
poly_add(&mut r.vec[i], &b.vec[i]);
}
}

78
third_party/kyber/src/reference/reduce.rs vendored
View file

@ -1,41 +1,37 @@
use crate::params::*;
const QINV: i32 = 62209; // q^(-1) mod 2^16
// Name: montgomery_reduce
//
// Description: Montgomery reduction; given a 32-bit integer a, computes
// 16-bit integer congruent to a * R^-1 mod q,
// where R=2^16
//
// Arguments: - i32 a: input integer to be reduced; has to be in {-q2^15,...,q2^15-1}
//
// Returns: integer in {-q+1,...,q-1} congruent to a * R^-1 modulo q.
pub fn montgomery_reduce(a: i32) -> i16
{
let ua = a.wrapping_mul(QINV) as i16;
let u = ua as i32;
let mut t = u * KYBER_Q as i32;
t = a - t;
t >>= 16;
t as i16
}
// Name: barrett_reduce
//
// Description: Barrett reduction; given a 16-bit integer a, computes
// centered representative congruent to a mod q in {-(q-1)/2,...,(q-1)/2}
//
// Arguments: - i16 a: input integer to be reduced
//
// Returns: i16 in {-(q-1)/2,...,(q-1)/2} congruent to a modulo q.
pub fn barrett_reduce(a: i16) -> i16
{
let v = ((1u32 << 26)/KYBER_Q as u32 + 1) as i32;
let mut t = v * a as i32 + (1 << 25);
t >>= 26;
t *= KYBER_Q as i32;
a - t as i16
}
use crate::params::*;
const QINV: i32 = 62209; // q^(-1) mod 2^16
// Name: montgomery_reduce
//
// Description: Montgomery reduction; given a 32-bit integer a, computes
// 16-bit integer congruent to a * R^-1 mod q,
// where R=2^16
//
// Arguments: - i32 a: input integer to be reduced; has to be in {-q2^15,...,q2^15-1}
//
// Returns: integer in {-q+1,...,q-1} congruent to a * R^-1 modulo q.
pub fn montgomery_reduce(a: i32) -> i16 {
let ua = a.wrapping_mul(QINV) as i16;
let u = ua as i32;
let mut t = u * KYBER_Q as i32;
t = a - t;
t >>= 16;
t as i16
}
// Name: barrett_reduce
//
// Description: Barrett reduction; given a 16-bit integer a, computes
// centered representative congruent to a mod q in {-(q-1)/2,...,(q-1)/2}
//
// Arguments: - i16 a: input integer to be reduced
//
// Returns: i16 in {-(q-1)/2,...,(q-1)/2} congruent to a modulo q.
pub fn barrett_reduce(a: i16) -> i16 {
let v = ((1u32 << 26) / KYBER_Q as u32 + 1) as i32;
let mut t = v * a as i32 + (1 << 25);
t >>= 26;
t *= KYBER_Q as i32;
a - t as i16
}

72
third_party/kyber/src/reference/verify.rs vendored
View file

@ -1,37 +1,35 @@
// Name: verify
//
// Description: Compare two arrays for equality in constant time.
//
// Arguments: const [u8] a: first byte array
// const [u8] b: second byte array
// size_t len: length of the byte arrays
//
// Returns 0 if the byte arrays are equal, 1 otherwise
pub fn verify(a: &[u8], b: &[u8], len: usize) -> u8
{
let mut r = 0u64;
for i in 0..len {
r |= (a[i] ^ b[i]) as u64;
}
r = r.wrapping_neg() >> 63;
r as u8
}
// Name: cmov
//
// Description: Copy len bytes from x to r if b is 1;
// don't modify x if b is 0. Requires b to be in {0,1};
// assumes two's complement representation of negative integers.
// Runs in constant time.
//
// Arguments: [u8] r: output byte array
// const [u8] x: input byte array
// size_t len: Amount of bytes to be copied
// [u8] b: Condition bit; has to be in {0,1}
pub fn cmov(r: &mut[u8], x: &[u8], len: usize, mut b: u8)
{
b = b.wrapping_neg();
for i in 0..len {
r[i] ^= b & (x[i] ^ r[i]);
}
}
// Name: verify
//
// Description: Compare two arrays for equality in constant time.
//
// Arguments: const [u8] a: first byte array
// const [u8] b: second byte array
// size_t len: length of the byte arrays
//
// Returns 0 if the byte arrays are equal, 1 otherwise
pub fn verify(a: &[u8], b: &[u8], len: usize) -> u8 {
let mut r = 0u64;
for i in 0..len {
r |= (a[i] ^ b[i]) as u64;
}
r = r.wrapping_neg() >> 63;
r as u8
}
// Name: cmov
//
// Description: Copy len bytes from x to r if b is 1;
// don't modify x if b is 0. Requires b to be in {0,1};
// assumes two's complement representation of negative integers.
// Runs in constant time.
//
// Arguments: [u8] r: output byte array
// const [u8] x: input byte array
// size_t len: Amount of bytes to be copied
// [u8] b: Condition bit; has to be in {0,1}
pub fn cmov(r: &mut [u8], x: &[u8], len: usize, mut b: u8) {
b = b.wrapping_neg();
for i in 0..len {
r[i] ^= b & (x[i] ^ r[i]);
}
}

21
third_party/kyber/src/rng.rs vendored
View file

@ -1,10 +1,11 @@
#![cfg(not(feature="KATs"))]
use rand_core::*;
// Fills buffer x with len bytes, RNG must satisfy the
// RngCore trait and CryptoRng marker trait requirements
pub fn randombytes<R>(x: &mut [u8], len: usize, rng: &mut R)
where R: RngCore + CryptoRng,
{
rng.fill_bytes(&mut x[..len]);
}
#![cfg(not(feature = "KATs"))]
use rand_core::*;
// Fills buffer x with len bytes, RNG must satisfy the
// RngCore trait and CryptoRng marker trait requirements
pub fn randombytes<R>(x: &mut [u8], len: usize, rng: &mut R)
where
R: RngCore + CryptoRng,
{
rng.fill_bytes(&mut x[..len]);
}

386
third_party/kyber/src/symmetric.rs vendored
View file

@ -1,205 +1,181 @@
#![allow(dead_code)]
#[cfg(not(feature = "90s"))] use crate::{fips202::*, params::*};
#[cfg(feature = "90s")] use crate::aes256ctr::*;
#[cfg(feature = "90s")] use sha2::{Sha256, Sha512, Digest};
// TODO: Rustrypto AES-CTR feature
// #[cfg(feature = "90s")] use aes_ctr::Aes256Ctr;
// #[cfg(feature = "90s")] use aes_ctr::cipher::{
// generic_array::GenericArray,
// stream::{NewStreamCipher, SyncStreamCipher}
// };
#[cfg(feature = "90s")]
pub(crate) const AES256CTR_BLOCKBYTES: usize = 64;
#[cfg(feature = "90s")]
pub(crate) const XOF_BLOCKBYTES: usize = AES256CTR_BLOCKBYTES;
#[cfg(not(feature = "90s"))]
pub(crate) const XOF_BLOCKBYTES: usize = SHAKE128_RATE;
#[cfg(not(feature = "90s"))]
pub(crate) type XofState = KeccakState;
#[cfg(feature = "90s")]
pub(crate) type XofState = Aes256CtrCtx;
#[derive(Copy, Clone)]
pub(crate) struct KeccakState {
pub s: [u64; 25],
pub pos: usize
}
impl KeccakState {
pub fn new() -> Self {
KeccakState {
s: [0u64; 25],
pos: 0usize
}
}
pub fn reset(&mut self) {
self.s = [0u64; 25];
self.pos = 0;
}
}
// SHA3-256
#[cfg(not(feature = "90s"))]
pub(crate) fn hash_h(out: &mut[u8], input: &[u8], inlen: usize)
{
sha3_256(out, input, inlen);
}
// 90s mode SHA2-256
#[cfg(feature = "90s")]
pub(crate) fn hash_h(out: &mut[u8], input: &[u8], inlen: usize)
{
let mut hasher = Sha256::new();
hasher.update(&input[..inlen]);
let digest = hasher.finalize();
out[..digest.len()].copy_from_slice(&digest);
}
#[cfg(not(feature = "90s"))]
pub(crate) fn hash_g(out: &mut[u8], input: &[u8], inlen: usize)
{
sha3_512(out, input, inlen);
}
#[cfg(feature = "90s")]
pub(crate) fn hash_g(out: &mut[u8], input: &[u8], inlen: usize)
{
let mut hasher = Sha512::new();
hasher.update(&input[..inlen]);
let digest = hasher.finalize();
out[..digest.len()].copy_from_slice(&digest);
}
#[cfg(not(feature = "90s"))]
pub(crate) fn xof_absorb(state: &mut XofState, input: &[u8], x: u8, y: u8)
{
kyber_shake128_absorb(state, &input, x, y);
}
#[cfg(feature = "90s")]
pub(crate) fn xof_absorb(state: &mut XofState, input: &[u8], x: u8, y: u8)
{
let mut nonce = [0u8; 12];
nonce[0] = x;
nonce[1] = y;
aes256ctr_init(state, &input, nonce);
}
#[cfg(not(feature = "90s"))]
pub(crate) fn xof_squeezeblocks(out: &mut[u8], outblocks: usize, state: &mut XofState)
{
kyber_shake128_squeezeblocks(out, outblocks, state);
}
#[cfg(feature = "90s")]
pub(crate) fn xof_squeezeblocks(out: &mut[u8], outblocks: usize, state: &mut XofState)
{
aes256ctr_squeezeblocks(out, outblocks, state);
}
#[cfg(not(feature = "90s"))]
pub(crate) fn prf(out: &mut[u8], outbytes: usize, key: &[u8], nonce: u8)
{
shake256_prf(out, outbytes, &key, nonce);
}
#[cfg(feature = "90s")]
pub(crate) fn prf(out: &mut[u8], outbytes: usize, key: &[u8], nonce: u8)
{
aes256ctr_prf(out, outbytes, &key, nonce);
// TODO: Add feature to use RustCrypto AES_CTR
// implementation with no lookup tables
// Perhaps add an option for ring also.
// Working RustCrypto code:
// if cfg!(feature = "rustcrypto-aes") {
// let mut expnonce = [0u8; 16];
// expnonce[0] = nonce;
// let key = GenericArray::from_slice(key);
// let iv = GenericArray::from_slice(&expnonce);
// let mut cipher = Aes256Ctr::new(&key, &iv);
// cipher.apply_keystream(out);
// }
}
#[cfg(not(feature = "90s"))]
pub(crate) fn kdf(out: &mut[u8], input: &[u8], inlen: usize)
{
shake256(out, KYBER_SSBYTES, input, inlen);
}
#[cfg(feature = "90s")]
pub(crate) fn kdf(out: &mut[u8], input: &[u8], inlen: usize)
{
let mut hasher = Sha256::new();
hasher.update(&input[..inlen]);
let digest = hasher.finalize();
out[..digest.len()].copy_from_slice(&digest);
}
// Name: kyber_shake128_absorb
//
// Description: Absorb step of the SHAKE128 specialized for the Kyber context.
//
// Arguments: - u64 *s: (uninitialized) output Keccak state
// - const [u8] input: KYBER_SYMBYTES input to be absorbed into s
// - u8 x additional byte of input
// - u8 y additional byte of input
#[cfg(not(feature = "90s"))]
fn kyber_shake128_absorb(
s: &mut KeccakState,
input: &[u8],
x: u8,
y: u8
)
{
let mut extseed = [0u8; KYBER_SYMBYTES + 2];
extseed[..KYBER_SYMBYTES].copy_from_slice(input);
extseed[KYBER_SYMBYTES] = x;
extseed[KYBER_SYMBYTES+1] = y;
shake128_absorb_once(s, &extseed, KYBER_SYMBYTES + 2);
}
// Name: kyber_shake128_squeezeblocks
//
// Description: Squeeze step of SHAKE128 XOF. Squeezes full blocks of SHAKE128_RATE bytes each.
// Modifies the state. Can be called multiple times to keep squeezing,
// i.e., is incremental.
//
// Arguments: - [u8] output: output blocks
// - u64 nblocks: number of blocks to be squeezed (written to output)
// - keccak_state *s: in/output Keccak state
#[cfg(not(feature = "90s"))]
fn kyber_shake128_squeezeblocks(
output: &mut[u8],
nblocks: usize,
s: &mut KeccakState
)
{
shake128_squeezeblocks(output, nblocks, s);
}
// Name: shake256_prf
//
// Description: Usage of SHAKE256 as a PRF, concatenates secret and public input
// and then generates outlen bytes of SHAKE256 output
//
// Arguments: - [u8] output: output
// - u64 outlen: number of requested output bytes
// - const [u8] key: the key (of length KYBER_SYMBYTES)
// - const [u8] nonce: single-byte nonce (public PRF input)
#[cfg(not(feature = "90s"))]
fn shake256_prf(output: &mut[u8], outlen: usize, key: &[u8], nonce: u8)
{
let mut extkey = [0u8; KYBER_SYMBYTES+1];
extkey[..KYBER_SYMBYTES].copy_from_slice(key);
extkey[KYBER_SYMBYTES] = nonce;
shake256(output, outlen, &extkey, KYBER_SYMBYTES + 1);
}
#![allow(dead_code)]
#[cfg(feature = "90s")]
use crate::aes256ctr::*;
#[cfg(not(feature = "90s"))]
use crate::{fips202::*, params::*};
#[cfg(feature = "90s")]
use sha2::{Digest, Sha256, Sha512};
// TODO: Rustrypto AES-CTR feature
// #[cfg(feature = "90s")] use aes_ctr::Aes256Ctr;
// #[cfg(feature = "90s")] use aes_ctr::cipher::{
// generic_array::GenericArray,
// stream::{NewStreamCipher, SyncStreamCipher}
// };
#[cfg(feature = "90s")]
pub(crate) const AES256CTR_BLOCKBYTES: usize = 64;
#[cfg(feature = "90s")]
pub(crate) const XOF_BLOCKBYTES: usize = AES256CTR_BLOCKBYTES;
#[cfg(not(feature = "90s"))]
pub(crate) const XOF_BLOCKBYTES: usize = SHAKE128_RATE;
#[cfg(not(feature = "90s"))]
pub(crate) type XofState = KeccakState;
#[cfg(feature = "90s")]
pub(crate) type XofState = Aes256CtrCtx;
#[derive(Copy, Clone)]
pub(crate) struct KeccakState {
pub s: [u64; 25],
pub pos: usize,
}
impl KeccakState {
pub fn new() -> Self {
KeccakState { s: [0u64; 25], pos: 0usize }
}
pub fn reset(&mut self) {
self.s = [0u64; 25];
self.pos = 0;
}
}
// SHA3-256
#[cfg(not(feature = "90s"))]
pub(crate) fn hash_h(out: &mut [u8], input: &[u8], inlen: usize) {
sha3_256(out, input, inlen);
}
// 90s mode SHA2-256
#[cfg(feature = "90s")]
pub(crate) fn hash_h(out: &mut [u8], input: &[u8], inlen: usize) {
let mut hasher = Sha256::new();
hasher.update(&input[..inlen]);
let digest = hasher.finalize();
out[..digest.len()].copy_from_slice(&digest);
}
#[cfg(not(feature = "90s"))]
pub(crate) fn hash_g(out: &mut [u8], input: &[u8], inlen: usize) {
sha3_512(out, input, inlen);
}
#[cfg(feature = "90s")]
pub(crate) fn hash_g(out: &mut [u8], input: &[u8], inlen: usize) {
let mut hasher = Sha512::new();
hasher.update(&input[..inlen]);
let digest = hasher.finalize();
out[..digest.len()].copy_from_slice(&digest);
}
#[cfg(not(feature = "90s"))]
pub(crate) fn xof_absorb(state: &mut XofState, input: &[u8], x: u8, y: u8) {
kyber_shake128_absorb(state, &input, x, y);
}
#[cfg(feature = "90s")]
pub(crate) fn xof_absorb(state: &mut XofState, input: &[u8], x: u8, y: u8) {
let mut nonce = [0u8; 12];
nonce[0] = x;
nonce[1] = y;
aes256ctr_init(state, &input, nonce);
}
#[cfg(not(feature = "90s"))]
pub(crate) fn xof_squeezeblocks(out: &mut [u8], outblocks: usize, state: &mut XofState) {
kyber_shake128_squeezeblocks(out, outblocks, state);
}
#[cfg(feature = "90s")]
pub(crate) fn xof_squeezeblocks(out: &mut [u8], outblocks: usize, state: &mut XofState) {
aes256ctr_squeezeblocks(out, outblocks, state);
}
#[cfg(not(feature = "90s"))]
pub(crate) fn prf(out: &mut [u8], outbytes: usize, key: &[u8], nonce: u8) {
shake256_prf(out, outbytes, &key, nonce);
}
#[cfg(feature = "90s")]
pub(crate) fn prf(out: &mut [u8], outbytes: usize, key: &[u8], nonce: u8) {
aes256ctr_prf(out, outbytes, &key, nonce);
// TODO: Add feature to use RustCrypto AES_CTR
// implementation with no lookup tables
// Perhaps add an option for ring also.
// Working RustCrypto code:
// if cfg!(feature = "rustcrypto-aes") {
// let mut expnonce = [0u8; 16];
// expnonce[0] = nonce;
// let key = GenericArray::from_slice(key);
// let iv = GenericArray::from_slice(&expnonce);
// let mut cipher = Aes256Ctr::new(&key, &iv);
// cipher.apply_keystream(out);
// }
}
#[cfg(not(feature = "90s"))]
pub(crate) fn kdf(out: &mut [u8], input: &[u8], inlen: usize) {
shake256(out, KYBER_SSBYTES, input, inlen);
}
#[cfg(feature = "90s")]
pub(crate) fn kdf(out: &mut [u8], input: &[u8], inlen: usize) {
let mut hasher = Sha256::new();
hasher.update(&input[..inlen]);
let digest = hasher.finalize();
out[..digest.len()].copy_from_slice(&digest);
}
// Name: kyber_shake128_absorb
//
// Description: Absorb step of the SHAKE128 specialized for the Kyber context.
//
// Arguments: - u64 *s: (uninitialized) output Keccak state
// - const [u8] input: KYBER_SYMBYTES input to be absorbed into s
// - u8 x additional byte of input
// - u8 y additional byte of input
#[cfg(not(feature = "90s"))]
fn kyber_shake128_absorb(s: &mut KeccakState, input: &[u8], x: u8, y: u8) {
let mut extseed = [0u8; KYBER_SYMBYTES + 2];
extseed[..KYBER_SYMBYTES].copy_from_slice(input);
extseed[KYBER_SYMBYTES] = x;
extseed[KYBER_SYMBYTES + 1] = y;
shake128_absorb_once(s, &extseed, KYBER_SYMBYTES + 2);
}
// Name: kyber_shake128_squeezeblocks
//
// Description: Squeeze step of SHAKE128 XOF. Squeezes full blocks of SHAKE128_RATE bytes each.
// Modifies the state. Can be called multiple times to keep squeezing,
// i.e., is incremental.
//
// Arguments: - [u8] output: output blocks
// - u64 nblocks: number of blocks to be squeezed (written to output)
// - keccak_state *s: in/output Keccak state
#[cfg(not(feature = "90s"))]
fn kyber_shake128_squeezeblocks(output: &mut [u8], nblocks: usize, s: &mut KeccakState) {
shake128_squeezeblocks(output, nblocks, s);
}
// Name: shake256_prf
//
// Description: Usage of SHAKE256 as a PRF, concatenates secret and public input
// and then generates outlen bytes of SHAKE256 output
//
// Arguments: - [u8] output: output
// - u64 outlen: number of requested output bytes
// - const [u8] key: the key (of length KYBER_SYMBYTES)
// - const [u8] nonce: single-byte nonce (public PRF input)
#[cfg(not(feature = "90s"))]
fn shake256_prf(output: &mut [u8], outlen: usize, key: &[u8], nonce: u8) {
let mut extkey = [0u8; KYBER_SYMBYTES + 1];
extkey[..KYBER_SYMBYTES].copy_from_slice(key);
extkey[KYBER_SYMBYTES] = nonce;
shake256(output, outlen, &extkey, KYBER_SYMBYTES + 1);
}

122
third_party/kyber/src/wasm.rs vendored
View file

@ -3,43 +3,43 @@ use wasm_bindgen::prelude::*;
#[wasm_bindgen]
pub fn keypair() -> Keys {
let mut pk = [0u8; KYBER_PUBLICKEYBYTES];
let mut sk = [0u8; KYBER_SECRETKEYBYTES];
kem::crypto_kem_keypair(&mut pk, &mut sk, None);
Keys{
pubkey: Box::new(pk),
secret: Box::new(sk),
..Default::default()
}
let mut pk = [0u8; KYBER_PUBLICKEYBYTES];
let mut sk = [0u8; KYBER_SECRETKEYBYTES];
kem::crypto_kem_keypair(&mut pk, &mut sk, None);
Keys {
pubkey: Box::new(pk),
secret: Box::new(sk),
..Default::default()
}
}
#[wasm_bindgen]
pub fn encapsulate(pk: Box<[u8]>) -> Result<Keys, JsValue> {
if pk.len() != KYBER_PUBLICKEYBYTES {
return Err(JsValue::null())
}
let mut ct = [0u8; KYBER_CIPHERTEXTBYTES];
let mut ss = [0u8; KYBER_SSBYTES];
kem::crypto_kem_enc(&mut ct, &mut ss, &pk, None);
Ok(Keys{
ciphertext: Box::new(ct),
shared_secret: Box::new(ss),
..Default::default()
})
if pk.len() != KYBER_PUBLICKEYBYTES {
return Err(JsValue::null());
}
let mut ct = [0u8; KYBER_CIPHERTEXTBYTES];
let mut ss = [0u8; KYBER_SSBYTES];
kem::crypto_kem_enc(&mut ct, &mut ss, &pk, None);
Ok(Keys {
ciphertext: Box::new(ct),
shared_secret: Box::new(ss),
..Default::default()
})
}
#[wasm_bindgen]
pub fn decapsulate(ct: Box<[u8]>, sk: Box<[u8]>) -> Result<Keys, JsValue> {
let mut ss = [0u8; KYBER_SSBYTES];
match kem::crypto_kem_dec(&mut ss, &ct, &sk) {
Ok(_) => Ok(Keys {shared_secret: Box::new(ss), ..Default::default()}),
Err(_) => Err(JsValue::null())
}
let mut ss = [0u8; KYBER_SSBYTES];
match kem::crypto_kem_dec(&mut ss, &ct, &sk) {
Ok(_) => Ok(Keys { shared_secret: Box::new(ss), ..Default::default() }),
Err(_) => Err(JsValue::null()),
}
}
#[wasm_bindgen]
#[derive(Default, Clone, Debug)]
pub struct Keys{
pub struct Keys {
pubkey: Box<[u8]>,
secret: Box<[u8]>,
ciphertext: Box<[u8]>,
@ -48,48 +48,48 @@ pub struct Keys{
#[wasm_bindgen]
impl Keys {
#[wasm_bindgen(constructor)]
pub fn new() -> Self {
Keys::default()
}
#[wasm_bindgen(constructor)]
pub fn new() -> Self {
Keys::default()
}
#[wasm_bindgen(getter)]
pub fn pubkey(&self) -> Box<[u8]> {
self.pubkey.clone()
}
#[wasm_bindgen(getter)]
pub fn pubkey(&self) -> Box<[u8]> {
self.pubkey.clone()
}
#[wasm_bindgen(getter)]
pub fn secret(&self) -> Box<[u8]> {
self.secret.clone()
}
#[wasm_bindgen(getter)]
pub fn secret(&self) -> Box<[u8]> {
self.secret.clone()
}
#[wasm_bindgen(getter)]
pub fn ciphertext(&self) -> Box<[u8]> {
self.ciphertext.clone()
}
#[wasm_bindgen(getter)]
pub fn ciphertext(&self) -> Box<[u8]> {
self.ciphertext.clone()
}
#[wasm_bindgen(getter)]
pub fn shared_secret(&self) -> Box<[u8]> {
self.shared_secret.clone()
}
#[wasm_bindgen(getter)]
pub fn shared_secret(&self) -> Box<[u8]> {
self.shared_secret.clone()
}
#[wasm_bindgen(setter)]
pub fn set_pubkey(&mut self, pubkey: Box<[u8]>) {
self.pubkey = pubkey;
}
#[wasm_bindgen(setter)]
pub fn set_pubkey(&mut self, pubkey: Box<[u8]>) {
self.pubkey = pubkey;
}
#[wasm_bindgen(setter)]
pub fn set_secret(&mut self, secret: Box<[u8]>) {
self.secret = secret;
}
#[wasm_bindgen(setter)]
pub fn set_secret(&mut self, secret: Box<[u8]>) {
self.secret = secret;
}
#[wasm_bindgen(setter)]
pub fn set_ciphertext(&mut self, ciphertext: Box<[u8]>) {
self.ciphertext = ciphertext;
}
#[wasm_bindgen(setter)]
pub fn set_ciphertext(&mut self, ciphertext: Box<[u8]>) {
self.ciphertext = ciphertext;
}
#[wasm_bindgen(setter)]
pub fn set_shared_secret(&mut self, shared_secret: Box<[u8]>) {
self.shared_secret = shared_secret;
}
#[wasm_bindgen(setter)]
pub fn set_shared_secret(&mut self, shared_secret: Box<[u8]>) {
self.shared_secret = shared_secret;
}
}

91
third_party/kyber/tests/kat.rs vendored
View file

@ -1,75 +1,74 @@
#![cfg(feature="KATs")]
#![cfg(feature = "KATs")]
mod load;
use pqc_kyber::*;
use load::*;
use pqc_kyber::*;
// Generate KAT keypairs from seeds.
#[test]
fn keypairs() {
let kats = build_kats();
let mut _rng = rand::thread_rng(); // placeholder
for kat in kats {
let known_pk = decode_hex(&kat.pk);
let known_sk = decode_hex(&kat.sk);
let buf1 = decode_hex(&kat.keygen_buffer1);
let buf2 = decode_hex(&kat.keygen_buffer2);
let bufs = Some((&buf1[..], &buf2[..]));
let mut pk = [0u8; KYBER_PUBLICKEYBYTES];
let mut sk = [0u8; KYBER_SECRETKEYBYTES];
crypto_kem_keypair(&mut pk, &mut sk, &mut _rng, bufs);
assert_eq!(&pk[..], &known_pk[..], "Public key KAT mismatch");
assert_eq!(&sk[..], &known_sk[..], "Secret key KAT mismatch");
}
let kats = build_kats();
let mut _rng = rand::thread_rng(); // placeholder
for kat in kats {
let known_pk = decode_hex(&kat.pk);
let known_sk = decode_hex(&kat.sk);
let buf1 = decode_hex(&kat.keygen_buffer1);
let buf2 = decode_hex(&kat.keygen_buffer2);
let bufs = Some((&buf1[..], &buf2[..]));
let mut pk = [0u8; KYBER_PUBLICKEYBYTES];
let mut sk = [0u8; KYBER_SECRETKEYBYTES];
crypto_kem_keypair(&mut pk, &mut sk, &mut _rng, bufs);
assert_eq!(&pk[..], &known_pk[..], "Public key KAT mismatch");
assert_eq!(&sk[..], &known_sk[..], "Secret key KAT mismatch");
}
}
// Encapsulating KAT's using deterministic rand buffers
#[test]
fn encaps() {
let kats = build_kats();
let mut _rng = rand::thread_rng(); // placeholder
for kat in kats {
let known_ss = decode_hex(&kat.ss);
let pk = decode_hex(&kat.pk);
let buf1 = decode_hex(&kat.encap_buffer);
let encap_buf = Some(&buf1[..]);
let mut ct = [0u8; KYBER_CIPHERTEXTBYTES];
let mut ss = [0u8; KYBER_SSBYTES];
crypto_kem_enc(&mut ct, &mut ss, &pk, &mut _rng, encap_buf);
assert_eq!(&ss[..], &known_ss[..], "Shared secret KAT mismatch");
}
let kats = build_kats();
let mut _rng = rand::thread_rng(); // placeholder
for kat in kats {
let known_ss = decode_hex(&kat.ss);
let pk = decode_hex(&kat.pk);
let buf1 = decode_hex(&kat.encap_buffer);
let encap_buf = Some(&buf1[..]);
let mut ct = [0u8; KYBER_CIPHERTEXTBYTES];
let mut ss = [0u8; KYBER_SSBYTES];
crypto_kem_enc(&mut ct, &mut ss, &pk, &mut _rng, encap_buf);
assert_eq!(&ss[..], &known_ss[..], "Shared secret KAT mismatch");
}
}
// Decapsulating KAT's
#[test]
fn decaps() {
let kats = build_kats();
for kat in kats {
let sk = decode_hex(&kat.sk);
let ct = decode_hex(&kat.ct);
let known_ss = decode_hex(&kat.ss);
let decap_result = decapsulate(&ct, &sk);
assert!(decap_result.is_ok(), "KEM decapsulation failure");
assert_eq!(&decap_result.unwrap()[..], &known_ss[..], "Shared secret KAT doesn't match")
}
let kats = build_kats();
for kat in kats {
let sk = decode_hex(&kat.sk);
let ct = decode_hex(&kat.ct);
let known_ss = decode_hex(&kat.ss);
let decap_result = decapsulate(&ct, &sk);
assert!(decap_result.is_ok(), "KEM decapsulation failure");
assert_eq!(&decap_result.unwrap()[..], &known_ss[..], "Shared secret KAT doesn't match")
}
}
// Helper functions
// Encodes byte slice into a hex string
pub fn encode_hex(bytes: &[u8]) -> String {
let mut output = String::new();
for b in bytes {
let mut output = String::new();
for b in bytes {
output.push_str(&format!("{:02X}", b));
}
output
output
}
// Decodes hex string into a vector of bytes
pub fn decode_hex(s: &str) -> Vec<u8> {
(0..s.len())
.step_by(2)
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("Hex string decoding"))
.collect::<Vec<u8>>()
}
(0..s.len())
.step_by(2)
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("Hex string decoding"))
.collect::<Vec<u8>>()
}

13
third_party/kyber/tests/kem.rs vendored
View file

@ -1,12 +1,11 @@
#![cfg(not(feature="KATs"))]
#![cfg(not(feature = "KATs"))]
use pqc_kyber::*;
#[test]
fn keypair_encap_decap_kem() {
let mut rng = rand::thread_rng();
let keys = keypair(&mut rng);
let (ct, ss1) = encapsulate(&keys.public, &mut rng).unwrap();
let ss2 = decapsulate(&ct, &keys.secret).unwrap();
assert_eq!(ss1, ss2);
let mut rng = rand::thread_rng();
let keys = keypair(&mut rng);
let (ct, ss1) = encapsulate(&keys.public, &mut rng).unwrap();
let ss2 = decapsulate(&ct, &keys.secret).unwrap();
assert_eq!(ss1, ss2);
}

38
third_party/kyber/tests/kex.rs vendored
View file

@ -1,28 +1,30 @@
#![cfg(not(feature="KATs"))]
#![cfg(not(feature = "KATs"))]
use pqc_kyber::*;
// Kyber struct uake and ake functions
#[test]
fn kyber_uake() {
let mut rng = rand::thread_rng();
let mut alice = Uake::new();
let mut bob = Uake::new();
let bob_keys = keypair(&mut rng);
let client_init = alice.client_init(&bob_keys.public, &mut rng);
let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng).unwrap();
let _client_confirm = alice.client_confirm(server_send).unwrap();
assert_eq!(alice.shared_secret, bob.shared_secret);
let mut rng = rand::thread_rng();
let mut alice = Uake::new();
let mut bob = Uake::new();
let bob_keys = keypair(&mut rng);
let client_init = alice.client_init(&bob_keys.public, &mut rng);
let server_send = bob.server_receive(client_init, &bob_keys.secret, &mut rng).unwrap();
let _client_confirm = alice.client_confirm(server_send).unwrap();
assert_eq!(alice.shared_secret, bob.shared_secret);
}
#[test]
fn kyber_ake() {
let mut rng = rand::thread_rng();
let mut alice = Ake::new();
let mut bob = Ake::new();
let alice_keys = keypair(&mut rng);
let bob_keys = keypair(&mut rng);
let client_init = alice.client_init(&bob_keys.public, &mut rng);
let server_send = bob.server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng).unwrap();
let _client_confirm = alice.client_confirm(server_send, &alice_keys.secret).unwrap();
assert_eq!(alice.shared_secret, bob.shared_secret);
let mut rng = rand::thread_rng();
let mut alice = Ake::new();
let mut bob = Ake::new();
let alice_keys = keypair(&mut rng);
let bob_keys = keypair(&mut rng);
let client_init = alice.client_init(&bob_keys.public, &mut rng);
let server_send = bob
.server_receive(client_init, &alice_keys.public, &bob_keys.secret, &mut rng)
.unwrap();
let _client_confirm = alice.client_confirm(server_send, &alice_keys.secret).unwrap();
assert_eq!(alice.shared_secret, bob.shared_secret);
}

128
third_party/kyber/tests/load/mod.rs vendored
View file

@ -1,100 +1,94 @@
use pqc_kyber::{KYBER_90S, KYBER_K};
use std::fs::File;
use std::path::PathBuf;
use std::io::{prelude::*, BufReader};
use pqc_kyber::{KYBER_K, KYBER_90S};
use std::path::PathBuf;
// Known Answer Tests
#[derive(Debug)]
pub struct Kat {
// Deterministic RNG buffers
pub keygen_buffer1: String,
pub keygen_buffer2: String,
pub encap_buffer: String,
// Keys, Ciphertext and Shared Secret
pub pk: String,
pub sk: String,
pub ct: String,
pub ss: String
// Deterministic RNG buffers
pub keygen_buffer1: String,
pub keygen_buffer2: String,
pub encap_buffer: String,
// Keys, Ciphertext and Shared Secret
pub pk: String,
pub sk: String,
pub ct: String,
pub ss: String,
}
// Converts string octuples from tvec files into Kat structs
impl From<&[String]> for Kat {
fn from(kat: &[String]) -> Self {
// Extract values from key:value lines
let values: Vec<String> = kat.iter()
.map(
|katline| {
let val: Vec<&str> = katline.split(": ").collect();
if val.len() > 1 {
val[1].into()
} else {
val[0].into()
}
fn from(kat: &[String]) -> Self {
// Extract values from key:value lines
let values: Vec<String> = kat
.iter()
.map(|katline| {
let val: Vec<&str> = katline.split(": ").collect();
if val.len() > 1 {
val[1].into()
} else {
val[0].into()
}
})
.collect();
// Build KAT from values
Kat {
keygen_buffer1: values[0].clone(),
keygen_buffer2: values[1].clone(),
pk: values[2].clone(),
sk: values[3].clone(),
encap_buffer: values[4].clone(),
ct: values[5].clone(),
ss: values[6].clone(),
}
)
.collect();
// Build KAT from values
Kat {
keygen_buffer1: values[0].clone(),
keygen_buffer2: values[1].clone(),
pk: values[2].clone(),
sk: values[3].clone(),
encap_buffer: values[4].clone(),
ct: values[5].clone(),
ss: values[6].clone(),
}
}
}
// Get KAT filename based on security level and if 90s mode
fn get_filename() -> String {
let mut filename = match KYBER_K {
2 => "tvecs512".to_string(),
3 => "tvecs768".to_string(),
4 => "tvecs1024".to_string(),
_ => panic!("No security level set")
};
if KYBER_90S {
filename.push_str("-90s");
}
println!("Using KAT file: {}", &filename);
filename
let mut filename = match KYBER_K {
2 => "tvecs512".to_string(),
3 => "tvecs768".to_string(),
4 => "tvecs1024".to_string(),
_ => panic!("No security level set"),
};
if KYBER_90S {
filename.push_str("-90s");
}
println!("Using KAT file: {}", &filename);
filename
}
// Base dir
fn get_test_dir() -> PathBuf {
let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
path.extend(&["tests"]);
path
let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
path.extend(&["tests"]);
path
}
// KATs path
fn get_kat_filepath() -> PathBuf {
let mut path = get_test_dir();
path.extend(&["KATs"]);
path.extend(&[get_filename()]);
path
let mut path = get_test_dir();
path.extend(&["KATs"]);
path.extend(&[get_filename()]);
path
}
fn load_file(filepath: PathBuf) -> File {
File::open(filepath).expect("Error loading KAT file")
File::open(filepath).expect("Error loading KAT file")
}
fn parse_kats() -> Vec<String> {
let file = load_file(get_kat_filepath());
let buf = BufReader::new(file);
buf.lines()
.map(|l| l.expect("Unable to parse line"))
.collect()
let file = load_file(get_kat_filepath());
let buf = BufReader::new(file);
buf.lines().map(|l| l.expect("Unable to parse line")).collect()
}
// Packs chunks of lines into Kat structs
// Packs chunks of lines into Kat structs
pub fn build_kats() -> Vec<Kat> {
let lines = parse_kats();
let kats = lines.chunks_exact(8);
// Map String slices into Vec<KAT>
kats.map(
|c| {c.into()}
)
.collect::<Vec<Kat>>()
}
let lines = parse_kats();
let kats = lines.chunks_exact(8);
// Map String slices into Vec<KAT>
kats.map(|c| c.into()).collect::<Vec<Kat>>()
}

6
utils/src/gatherarray.rs
View file

@ -47,7 +47,11 @@ impl<T, const C: usize> GatherArray<T, C> {
if (self.have_count as usize) == goal {
debug_assert_eq!(0xffffffffffffffffu64.wrapping_shr(64 - goal as u32), have);
let mut tmp = ArrayVec::new();
copy_nonoverlapping(self.a.as_ptr().cast::<u8>(), tmp.a.as_mut_ptr().cast::<u8>(), size_of::<MaybeUninit<T>>() * goal);
copy_nonoverlapping(
self.a.as_ptr().cast::<u8>(),
tmp.a.as_mut_ptr().cast::<u8>(),
size_of::<MaybeUninit<T>>() * goal,
);
tmp.s = goal;
self.goal = 0;
return Some(tmp);

4
utils/src/hex.rs
View file

@ -1,6 +1,8 @@
// (c) 2020-2022 ZeroTier, Inc. -- currently propritery pending actual release and licensing. See LICENSE.md.
pub const HEX_CHARS: [u8; 16] = [b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f'];
pub const HEX_CHARS: [u8; 16] = [
b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'a', b'b', b'c', b'd', b'e', b'f',
];
/// Encode a byte slice to a hexadecimal string.
pub fn to_string(b: &[u8]) -> String {

5
utils/src/pool.rs
View file

@ -43,7 +43,10 @@ impl<O: Send, F: PoolFactory<O>> Pooled<O, F> {
#[inline(always)]
pub unsafe fn into_raw(self) -> *mut O {
// Verify that the structure is not padded before 'obj'.
assert_eq!((&self.0.as_ref().obj as *const O).cast::<u8>(), (self.0.as_ref() as *const PoolEntry<O, F>).cast::<u8>());
assert_eq!(
(&self.0.as_ref().obj as *const O).cast::<u8>(),
(self.0.as_ref() as *const PoolEntry<O, F>).cast::<u8>()
);
let ptr = self.0.as_ptr().cast::<O>();
std::mem::forget(self);