diff --git a/Makefile b/Makefile index 7ae8565a9..41d97d583 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ all: clean: FORCE - rm -rf zerotier-core-crypto/target zerotier-network-hypervisor/target zerotier-system-service/target + rm -rf zerotier-core-crypto/target zerotier-network-hypervisor/target zerotier-system-service/target syncwhole/target aes-gmac-siv/target FORCE: diff --git a/egads/Cargo.lock b/egads/Cargo.lock deleted file mode 100644 index 4760312e4..000000000 --- a/egads/Cargo.lock +++ /dev/null @@ -1,702 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "aes-gmac-siv" -version = "0.1.0" -dependencies = [ - "openssl", -] - -[[package]] -name = "aho-corasick" -version = "0.7.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" -dependencies = [ - "memchr", -] - -[[package]] -name = "atomic-polyfill" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee6adc1648f03fbc1bc1b5cf0f2fdfb5edbc96215b711edcfe6ce2641ef9b347" -dependencies = [ - "critical-section", - "riscv-target", -] - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "bare-metal" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5deb64efa5bd81e31fcd1938615a6d98c82eafcbcd787162b6f63b91d6bac5b3" -dependencies = [ - "rustc_version", -] - -[[package]] -name = "bare-metal" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fe8f5a8a398345e52358e18ff07cc17a568fbca5c6f73873d3a62056309603" - -[[package]] -name = "bit_field" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4" - -[[package]] -name = "bitfield" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46afbd2983a5d5a7bd740ccb198caf5b82f45c40c09c0eed36052d91cb92e719" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - -[[package]] -name = "byteorder" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" - -[[package]] -name = "cc" -version = "1.0.72" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "cortex-m" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ff967e867ca14eba0c34ac25cd71ea98c678e741e3915d923999bb2fe7c826" -dependencies = [ - "bare-metal 0.2.5", - "bitfield", - "embedded-hal", - "volatile-register", -] - -[[package]] -name = "cpufeatures" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" -dependencies = [ - "libc", -] - -[[package]] -name = "critical-section" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01e191a5a6f6edad9b679777ef6b6c0f2bdd4a333f2ecb8f61c3e28109a03d70" -dependencies = [ - "bare-metal 1.0.0", - "cfg-if", - "cortex-m", - "riscv", -] - -[[package]] -name = "curve25519-dalek" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61" -dependencies = [ - "byteorder", - "digest", - "rand_core", - "subtle", - "zeroize", -] - -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - -[[package]] -name = "ed25519" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74e1069e39f1454367eb2de793ed062fac4c35c2934b76a81d90dd9abcd28816" -dependencies = [ - "signature", -] - -[[package]] -name = "ed25519-dalek" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" -dependencies = [ - "curve25519-dalek", - "ed25519", - "rand", - "serde", - "sha2", - "zeroize", -] - -[[package]] -name = "egads" -version = "0.1.0" -dependencies = [ - "rmp-serde", - "serde", - "zerotier-core-crypto", -] - -[[package]] -name = "embedded-hal" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35949884794ad573cf46071e41c9b60efb0cb311e3ca01f7af807af1debc66ff" -dependencies = [ - "nb 0.1.3", - "void", -] - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "generic-array" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "hash32" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" -dependencies = [ - "byteorder", -] - -[[package]] -name = "heapless" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d076121838e03f862871315477528debffdb7462fb229216ecef91b1a3eb31eb" -dependencies = [ - "atomic-polyfill", - "hash32", - "spin", - "stable_deref_trait", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "libc" -version = "0.2.118" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06e509672465a0504304aa87f9f176f2b2b716ed8fb105ebe5c02dc6dce96a94" - -[[package]] -name = "lock_api" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" -dependencies = [ - "scopeguard", -] - -[[package]] -name = "memchr" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" - -[[package]] -name = "nb" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "801d31da0513b6ec5214e9bf433a77966320625a37860f910be265be6e18d06f" -dependencies = [ - "nb 1.0.0", -] - -[[package]] -name = "nb" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "546c37ac5d9e56f55e73b677106873d9d9f5190605e41a856503623648488cae" - -[[package]] -name = "num-traits" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" -dependencies = [ - "autocfg", -] - -[[package]] -name = "once_cell" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" - -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - -[[package]] -name = "openssl" -version = "0.10.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-sys", -] - -[[package]] -name = "openssl-sys" -version = "0.9.72" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" -dependencies = [ - "autocfg", - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "pkg-config" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" - -[[package]] -name = "poly1305" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "048aeb476be11a4b6ca432ca569e375810de9294ae78f4774e78ea98a9246ede" -dependencies = [ - "cpufeatures", - "opaque-debug", - "universal-hash", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" - -[[package]] -name = "proc-macro2" -version = "1.0.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" -dependencies = [ - "unicode-xid", -] - -[[package]] -name = "quote" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom", - "libc", - "rand_chacha", - "rand_core", - "rand_hc", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core", -] - -[[package]] -name = "regex" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.6.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" - -[[package]] -name = "riscv" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6907ccdd7a31012b70faf2af85cd9e5ba97657cc3987c4f13f8e4d2c2a088aba" -dependencies = [ - "bare-metal 1.0.0", - "bit_field", - "riscv-target", -] - -[[package]] -name = "riscv-target" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88aa938cda42a0cf62a20cfe8d139ff1af20c2e681212b5b34adb5a58333f222" -dependencies = [ - "lazy_static", - "regex", -] - -[[package]] -name = "rmp" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f55e5fa1446c4d5dd1f5daeed2a4fe193071771a2636274d0d7a3b082aa7ad6" -dependencies = [ - "byteorder", - "num-traits", -] - -[[package]] -name = "rmp-serde" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723ecff9ad04f4ad92fe1c8ca6c20d2196d9286e9c60727c4cb5511629260e9d" -dependencies = [ - "byteorder", - "rmp", - "serde", -] - -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver", -] - -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - -[[package]] -name = "serde" -version = "1.0.136" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.136" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer", - "cfg-if", - "cpufeatures", - "digest", - "opaque-debug", -] - -[[package]] -name = "signature" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f054c6c1a6e95179d6f23ed974060dcefb2d9388bb7256900badad682c499de4" - -[[package]] -name = "spin" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5" -dependencies = [ - "lock_api", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "subtle" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" - -[[package]] -name = "syn" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - -[[package]] -name = "synstructure" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", -] - -[[package]] -name = "typenum" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" - -[[package]] -name = "unicode-xid" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" - -[[package]] -name = "universal-hash" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" -dependencies = [ - "generic-array", - "subtle", -] - -[[package]] -name = "vcell" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77439c1b53d2303b20d9459b1ade71a83c716e3f9c34f3228c00e6f185d6c002" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" - -[[package]] -name = "volatile-register" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ee8f19f9d74293faf70901bc20ad067dc1ad390d2cbf1e3f75f721ffee908b6" -dependencies = [ - "vcell", -] - -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - -[[package]] -name = "x25519-dalek" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2392b6b94a576b4e2bf3c5b2757d63f10ada8020a2e4d08ac849ebcf6ea8e077" -dependencies = [ - "curve25519-dalek", - "rand_core", - "zeroize", -] - -[[package]] -name = "zeroize" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81e8f13fef10b63c06356d65d416b070798ddabcadc10d3ece0c5be9b3c7eddb" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerotier-core-crypto" -version = "0.1.0" -dependencies = [ - "aes-gmac-siv", - "ed25519-dalek", - "foreign-types", - "heapless", - "lazy_static", - "openssl", - "poly1305", - "rand_core", - "subtle", - "x25519-dalek", -] diff --git a/egads/Cargo.toml b/egads/Cargo.toml deleted file mode 100644 index c5ae1707a..000000000 --- a/egads/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "egads" -version = "0.1.0" -edition = "2021" - -[dependencies] -zerotier-core-crypto = { path = "../zerotier-core-crypto" } -serde = { version = "^1", features = ["derive"], default-features = false } -rmp-serde = "^0" diff --git a/egads/README.md b/egads/README.md deleted file mode 100644 index d7ffb5195..000000000 --- a/egads/README.md +++ /dev/null @@ -1,3 +0,0 @@ -EGADS: Embeddable Globally Authenticated Data Store -===== - diff --git a/egads/src/block.rs b/egads/src/block.rs deleted file mode 100644 index 532ab4634..000000000 --- a/egads/src/block.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub struct Block<'a> { - pub validator: &'a [u8], - pub parent: &'a [u8], - pub timestamp: u64, - pub height: u64, - pub records: &'a [u8], - pub signature: &'a [u8], -} diff --git a/egads/src/lib.rs b/egads/src/lib.rs deleted file mode 100644 index fc9210db1..000000000 --- a/egads/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -mod block; diff --git a/syncwhole/Cargo.lock b/syncwhole/Cargo.lock new file mode 100644 index 000000000..721bb5dfc --- /dev/null +++ b/syncwhole/Cargo.lock @@ -0,0 +1,330 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "libc" +version = "0.2.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4" + +[[package]] +name = "lock_api" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "mio" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba272f85fa0b41fc91872be579b3bbe0f56b792aa361a380eb669469f68dafb2" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + +[[package]] +name = "ntapi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "parking_lot" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" + +[[package]] +name = "proc-macro2" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rmp" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f55e5fa1446c4d5dd1f5daeed2a4fe193071771a2636274d0d7a3b082aa7ad6" +dependencies = [ + "byteorder", + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3eedffbfcc6a428f230c04baf8f59bd73c1781361e4286111fe900849aaddaf" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "smallvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" + +[[package]] +name = "socket2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "syn" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "syncwhole" +version = "0.1.0" +dependencies = [ + "rmp", + "rmp-serde", + "serde", + "tokio", +] + +[[package]] +name = "tokio" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +dependencies = [ + "bytes", + "libc", + "memchr", + "mio", + "parking_lot", + "pin-project-lite", + "socket2", + "winapi", +] + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" + +[[package]] +name = "windows_i686_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" + +[[package]] +name = "windows_i686_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" diff --git a/syncwhole/Cargo.toml b/syncwhole/Cargo.toml new file mode 100644 index 000000000..e3873a8e8 --- /dev/null +++ b/syncwhole/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "syncwhole" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" +authors = ["Adam Ierymenko "] + +[dependencies] +tokio = { version = "^1", features = ["net", "rt", "parking_lot", "time", "io-std", "io-util", "sync"], default-features = false } +serde = { version = "^1", features = ["derive"], default-features = false } +rmp = "^0" +rmp-serde = "^1" diff --git a/syncwhole/src/database.rs b/syncwhole/src/database.rs new file mode 100644 index 000000000..bf8c4a1c2 --- /dev/null +++ b/syncwhole/src/database.rs @@ -0,0 +1,192 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * (c)2022 ZeroTier, Inc. + * https://www.zerotier.com/ + */ + +use std::ops::Bound::Included; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +/// Result returned by DB::load(). +pub enum LoadResult + Send> { + /// Object was found. + Ok(V), + + /// Object was not found, including the case of being excluded due to the value of reference_time. + NotFound, + + /// Supplied reference_time is outside what is available (usually too old). + TimeNotAvailable +} + +/// Result returned by DB::store(). +pub enum StoreResult { + /// Entry was accepted (whether or not an old value was replaced). + Ok, + + /// Entry was rejected as a duplicate but was otherwise valid. + Duplicate, + + /// Entry was rejected as invalid. + /// + /// An invalid entry is one that is malformed, fails a signature check, etc., and returning + /// this causes the synchronization service to drop the link to the node that sent it. + Rejected +} + +/// API to be implemented by the data store we want to replicate. +/// +/// The API specified here supports temporally subjective data sets. These are data sets +/// where the existence or non-existence of a record may depend on the (real world) time. +/// A parameter for reference time allows a remote querying node to send its own "this is +/// what time I think it is" value to be considered locally so that data can be replicated +/// as of any given time. +/// +/// The KEY_IS_COMPUTED constant must be set to indicate whether keys are a pure function of +/// values. If this is true, get_key() must be implemented. +/// +/// The implementation must be thread safe. +pub trait Database: Sync + Send { + /// Type to be enclosed in the Ok() enum value in LoadResult. + type LoadResultValueType: AsRef<[u8]> + Send; + + /// Size of keys, which must be fixed in length. These are typically hashes. + const KEY_SIZE: usize; + + /// Maximum size of a value in bytes. + const MAX_VALUE_SIZE: usize; + + /// This should be true if the key is computed, such as by hashing the value. + /// + /// If this is true only values are sent over the wire and get_key() is used to compute + /// keys from values. If this is false both keys and values are replicated. + const KEY_IS_COMPUTED: bool; + + /// Get the key corresponding to a value. + /// + /// If KEY_IS_COMPUTED is true this must be implemented. The default implementation + /// panics to indicate this. If KEY_IS_COMPUTED is false this is never called. + #[allow(unused_variables)] + fn get_key(value: &[u8], key: &mut [u8]) { + panic!("get_key() must be implemented if KEY_IS_COMPUTED is true"); + } + + /// Get an item if it exists as of a given reference time. + /// + /// The supplied key must be of length KEY_SIZE or this may panic. + fn load(&self, reference_time: i64, key: &[u8]) -> LoadResult; + + /// Store an item in the data store and return Ok, Duplicate, or Rejected. + /// + /// The supplied key must be of length KEY_SIZE or this may panic. + /// + /// Note that no time is supplied here. The data store must determine this in an implementation + /// dependent manner if this is a temporally subjective data store. It could be determined by + /// the wall clock, from the object itself, etc. + fn store(&self, key: &[u8], value: &[u8]) -> StoreResult; + + /// Get the number of items under a prefix as of a given reference time. + /// + /// The default implementation uses for_each(). This can be specialized if it can be done + /// more efficiently than that. + fn count(&self, reference_time: i64, key_prefix: &[u8]) -> u64 { + let mut cnt: u64 = 0; + self.for_each(reference_time, key_prefix, |_, _| { + cnt += 1; + true + }); + cnt + } + + /// Iterate through keys beneath a key prefix, stopping at the end or if the function returns false. + /// + /// The default implementation uses for_each(). + fn for_each_key bool>(&self, reference_time: i64, key_prefix: &[u8], mut f: F) { + self.for_each(reference_time, key_prefix, |k, _| f(k)); + } + + /// Iterate through keys and values beneath a key prefix, stopping at the end or if the function returns false. + fn for_each bool>(&self, reference_time: i64, key_prefix: &[u8], f: F); +} + +/// A simple in-memory data store backed by a BTreeMap. +pub struct MemoryDatabase { + max_age: i64, + db: Mutex)>> +} + +impl MemoryDatabase { + pub fn new(max_age: i64) -> Self { + Self { + max_age: if max_age > 0 { max_age } else { i64::MAX }, + db: Mutex::new(BTreeMap::new()) + } + } +} + +impl Database for MemoryDatabase { + type LoadResultValueType = Arc<[u8]>; + const KEY_SIZE: usize = KEY_SIZE; + const MAX_VALUE_SIZE: usize = 65536; + const KEY_IS_COMPUTED: bool = false; + + fn load(&self, reference_time: i64, key: &[u8]) -> LoadResult { + let db = self.db.lock().unwrap(); + let e = db.get(key); + if e.is_some() { + let e = e.unwrap(); + if (reference_time - e.0) <= self.max_age { + LoadResult::Ok(e.1.clone()) + } else { + LoadResult::NotFound + } + } else { + LoadResult::NotFound + } + } + + fn store(&self, key: &[u8], value: &[u8]) -> StoreResult { + let ts = crate::ms_since_epoch(); + let mut isdup = false; + self.db.lock().unwrap().entry(key.try_into().unwrap()).and_modify(|e| { + if e.1.as_ref().eq(value) { + isdup = true; + } else { + *e = (ts, Arc::from(value)); + } + }).or_insert_with(|| { + (ts, Arc::from(value)) + }); + if isdup { + StoreResult::Duplicate + } else { + StoreResult::Ok + } + } + + fn for_each bool>(&self, reference_time: i64, key_prefix: &[u8], mut f: F) { + let mut r_start = [0_u8; KEY_SIZE]; + let mut r_end = [0xff_u8; KEY_SIZE]; + (&mut r_start[0..key_prefix.len()]).copy_from_slice(key_prefix); + (&mut r_end[0..key_prefix.len()]).copy_from_slice(key_prefix); + for (k, v) in self.db.lock().unwrap().range((Included(r_start), Included(r_end))) { + if (reference_time - v.0) <= self.max_age { + if !f(k, &v.1) { + break; + } + } + } + } +} + +impl PartialEq for MemoryDatabase { + fn eq(&self, other: &Self) -> bool { + self.max_age == other.max_age && self.db.lock().unwrap().eq(&*other.db.lock().unwrap()) + } +} + +impl Eq for MemoryDatabase {} + diff --git a/syncwhole/src/host.rs b/syncwhole/src/host.rs new file mode 100644 index 000000000..b205fcafa --- /dev/null +++ b/syncwhole/src/host.rs @@ -0,0 +1,38 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * (c)2022 ZeroTier, Inc. + * https://www.zerotier.com/ + */ + +use std::collections::HashSet; +use std::error::Error; +use std::net::SocketAddr; + +/// A trait that users of syncwhole implement to provide configuration information and listen for events. +pub trait Host: Sync + Send { + /// Compute SHA512. + fn sha512(msg: &[u8]) -> [u8; 64]; + + /// Get a list of endpoints to which we always want to try to stay connected. + /// + /// The node will repeatedly try to connect to these until a link is established and + /// reconnect on link failure. They should be stable well known nodes for this domain. + fn get_static_endpoints(&self) -> &[SocketAddr]; + + /// Get additional endpoints to try. + /// + /// This should return any endpoints not in the supplied endpoint set if the size + /// of the set is less than the minimum active link count the host wishes to maintain. + fn get_more_endpoints(&self, current_endpoints: &HashSet) -> Vec; + + /// Called whenever we have successfully connected to a remote endpoint to (possibly) remember it. + fn on_connect_success(&self, endpoint: &SocketAddr); + + /// Called whenever an outgoing connection fails. + fn on_connect_failure(&self, endpoint: &SocketAddr, reason: Box); + + /// Fill a buffer with secure random bytes. + fn get_secure_random(&self, buf: &mut [u8]); +} diff --git a/syncwhole/src/iblt.rs b/syncwhole/src/iblt.rs new file mode 100644 index 000000000..f12cd7454 --- /dev/null +++ b/syncwhole/src/iblt.rs @@ -0,0 +1,261 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * (c)2022 ZeroTier, Inc. + * https://www.zerotier.com/ + */ + +use std::io::{Read, Write}; +use std::mem::{size_of, zeroed}; +use std::ptr::write_bytes; + +use crate::varint; + +// max value: 6, 5 was determined to be good via empirical testing +const KEY_MAPPING_ITERATIONS: usize = 5; + +#[inline(always)] +fn xorshift64(mut x: u64) -> u64 { + x ^= x.wrapping_shl(13); + x ^= x.wrapping_shr(7); + x ^= x.wrapping_shl(17); + x +} + +#[inline(always)] +fn splitmix64(mut x: u64) -> u64 { + x = u64::from_le(x); + x ^= x.wrapping_shr(30); + x = x.wrapping_mul(0xbf58476d1ce4e5b9); + x ^= x.wrapping_shr(27); + x = x.wrapping_mul(0x94d049bb133111eb); + x ^= x.wrapping_shr(31); + x.to_le() +} + +#[inline(always)] +fn splitmix64_inverse(mut x: u64) -> u64 { + x = u64::from_le(x); + x ^= x.wrapping_shr(31) ^ x.wrapping_shr(62); + x = x.wrapping_mul(0x319642b2d24d8ec3); + x ^= x.wrapping_shr(27) ^ x.wrapping_shr(54); + x = x.wrapping_mul(0x96de1b173f119089); + x ^= x.wrapping_shr(30) ^ x.wrapping_shr(60); + x.to_le() +} + +// https://nullprogram.com/blog/2018/07/31/ +#[inline(always)] +fn triple32(mut x: u32) -> u32 { + x ^= x.wrapping_shr(17); + x = x.wrapping_mul(0xed5ad4bb); + x ^= x.wrapping_shr(11); + x = x.wrapping_mul(0xac4c1b51); + x ^= x.wrapping_shr(15); + x = x.wrapping_mul(0x31848bab); + x ^= x.wrapping_shr(14); + x +} + +#[inline(always)] +fn next_iteration_index(prev_iteration_index: u64, salt: u64) -> u64 { + prev_iteration_index.wrapping_add(triple32(prev_iteration_index.wrapping_shr(32) as u32) as u64).wrapping_add(salt) +} + +#[derive(Clone, PartialEq, Eq)] +struct IBLTEntry { + key_sum: u64, + check_hash_sum: u64, + count: i64 +} + +impl IBLTEntry { + #[inline(always)] + fn is_singular(&self) -> bool { + if self.count == 1 || self.count == -1 { + xorshift64(self.key_sum) == self.check_hash_sum + } else { + false + } + } +} + +/// An Invertible Bloom Lookup Table for set reconciliation with 64-bit hashes. +#[derive(Clone, PartialEq, Eq)] +pub struct IBLT { + salt: u64, + map: [IBLTEntry; B] +} + +impl IBLT { + pub const BUCKETS: usize = B; + + pub fn new(salt: u64) -> Self { + Self { + salt, + map: unsafe { zeroed() } + } + } + + pub fn reset(&mut self, salt: u64) { + self.salt = salt; + unsafe { write_bytes((&mut self.map as *mut IBLTEntry).cast::(), 0, size_of::<[IBLTEntry; B]>()) }; + } + + pub fn read(&mut self, r: &mut R) -> std::io::Result<()> { + r.read_exact(unsafe { &mut *(&mut self.salt as *mut u64).cast::<[u8; 8]>() })?; + let mut prev_c = 0_i64; + for b in self.map.iter_mut() { + r.read_exact(unsafe { &mut *(&mut b.key_sum as *mut u64).cast::<[u8; 8]>() })?; + r.read_exact(unsafe { &mut *(&mut b.check_hash_sum as *mut u64).cast::<[u8; 8]>() })?; + let mut c = varint::read(r)? as i64; + if (c & 1) == 0 { + c = c.wrapping_shr(1); + } else { + c = -c.wrapping_shr(1); + } + b.count = c + prev_c; + prev_c = b.count; + } + Ok(()) + } + + pub fn write(&self, w: &mut W) -> std::io::Result<()> { + w.write_all(unsafe { &*(&self.salt as *const u64).cast::<[u8; 8]>() })?; + let mut prev_c = 0_i64; + for b in self.map.iter() { + w.write_all(unsafe { &*(&b.key_sum as *const u64).cast::<[u8; 8]>() })?; + w.write_all(unsafe { &*(&b.check_hash_sum as *const u64).cast::<[u8; 8]>() })?; + let mut c = (b.count - prev_c).wrapping_shl(1); + prev_c = b.count; + if c < 0 { + c = -c | 1; + } + varint::write(w, c as u64)?; + } + Ok(()) + } + + fn ins_rem(&mut self, mut key: u64, delta: i64) { + key = splitmix64(key ^ self.salt); + let check_hash = xorshift64(key); + let mut iteration_index = u64::from_le(key); + for _ in 0..KEY_MAPPING_ITERATIONS { + iteration_index = next_iteration_index(iteration_index, self.salt); + let b = unsafe { self.map.get_unchecked_mut((iteration_index as usize) % B) }; + b.key_sum ^= key; + b.check_hash_sum ^= check_hash; + b.count += delta; + } + } + + /// Insert a 64-bit key. + /// Panics if the key is shorter than 64 bits. If longer, bits beyond 64 are ignored. + #[inline(always)] + pub fn insert(&mut self, key: &[u8]) { + assert!(key.len() >= 8); + self.ins_rem(unsafe { u64::from_ne_bytes(*(key.as_ptr().cast::<[u8; 8]>())) }, 1); + } + + /// Remove a 64-bit key. + /// Panics if the key is shorter than 64 bits. If longer, bits beyond 64 are ignored. + #[inline(always)] + pub fn remove(&mut self, key: &[u8]) { + assert!(key.len() >= 8); + self.ins_rem(unsafe { u64::from_ne_bytes(*(key.as_ptr().cast::<[u8; 8]>())) }, -1); + } + + pub fn subtract(&mut self, other: &Self) { + for b in 0..B { + let s = &mut self.map[b]; + let o = &other.map[b]; + s.key_sum ^= o.key_sum; + s.check_hash_sum ^= o.check_hash_sum; + s.count += o.count; + } + } + + pub fn list(mut self, mut f: F) -> bool { + let mut singular_buckets = [0_usize; B]; + let mut singular_bucket_count = 0_usize; + + for b in 0..B { + if self.map[b].is_singular() { + singular_buckets[singular_bucket_count] = b; + singular_bucket_count += 1; + } + } + + while singular_bucket_count > 0 { + singular_bucket_count -= 1; + let b = &self.map[singular_buckets[singular_bucket_count]]; + + if b.is_singular() { + let key = b.key_sum; + let check_hash = xorshift64(key); + let mut iteration_index = u64::from_le(key); + + f(&(splitmix64_inverse(key) ^ self.salt).to_ne_bytes()); + + for _ in 0..KEY_MAPPING_ITERATIONS { + iteration_index = next_iteration_index(iteration_index, self.salt); + let b_idx = (iteration_index as usize) % B; + let b = unsafe { self.map.get_unchecked_mut(b_idx) }; + b.key_sum ^= key; + b.check_hash_sum ^= check_hash; + b.count -= 1; + + if b.is_singular() { + if singular_bucket_count >= B { + // This would indicate an invalid IBLT. + return false; + } + singular_buckets[singular_bucket_count] = b_idx; + singular_bucket_count += 1; + } + } + } + } + + return true; + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + use std::time::SystemTime; + use crate::iblt::*; + + #[test] + fn splitmix_is_invertiblex() { + for i in 1..1024_u64 { + assert_eq!(i, splitmix64_inverse(splitmix64(i))) + } + } + + #[test] + fn insert_and_list() { + let mut rn = xorshift64(SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap().as_nanos() as u64); + for _ in 0..256 { + let mut alice: IBLT<1024> = IBLT::new(rn); + rn = xorshift64(rn); + let mut expected: HashSet = HashSet::with_capacity(1024); + let count = 600; + for _ in 0..count { + let x = rn; + rn = xorshift64(rn); + expected.insert(x); + alice.insert(&x.to_ne_bytes()); + } + let mut cnt = 0; + alice.list(|x| { + let x = u64::from_ne_bytes(*x); + cnt += 1; + assert!(expected.contains(&x)); + }); + assert_eq!(cnt, count); + } + } +} diff --git a/syncwhole/src/lib.rs b/syncwhole/src/lib.rs new file mode 100644 index 000000000..38ffc9caa --- /dev/null +++ b/syncwhole/src/lib.rs @@ -0,0 +1,14 @@ +pub(crate) mod varint; +pub(crate) mod protocol; +pub(crate) mod iblt; +pub mod database; +pub mod node; +pub mod host; + +pub(crate) fn ms_since_epoch() -> i64 { + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis() as i64 +} + +pub(crate) fn ms_monotonic() -> i64 { + std::time::Instant::now().elapsed().as_millis() as i64 +} diff --git a/syncwhole/src/node.rs b/syncwhole/src/node.rs new file mode 100644 index 000000000..608b8961c --- /dev/null +++ b/syncwhole/src/node.rs @@ -0,0 +1,288 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * (c)2022 ZeroTier, Inc. + * https://www.zerotier.com/ + */ + +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::{Arc, Weak}; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::time::Duration; +use serde::{Deserialize, Serialize}; + +use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader}; +use tokio::net::{TcpListener, TcpSocket, TcpStream}; +use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; +use tokio::sync::Mutex; +use tokio::task::JoinHandle; + +use crate::database::Database; +use crate::host::Host; +use crate::ms_monotonic; +use crate::protocol::*; +use crate::varint; + +const CONNECTION_TIMEOUT: i64 = 60000; +const CONNECTION_KEEPALIVE_AFTER: i64 = 20000; + +struct Connection { + writer: Mutex, + last_send_time: AtomicI64, + last_receive_time: AtomicI64, + io_task: std::sync::Mutex>>>, + incoming: bool +} + +impl Connection { + async fn send(&self, data: &[u8], now: i64) -> std::io::Result<()> { + self.writer.lock().await.write_all(data).await.map(|_| { + self.last_send_time.store(now, Ordering::Relaxed); + }) + } + + async fn send_obj(&self, message_type: u8, obj: &O, now: i64) -> std::io::Result<()> { + let data = rmp_serde::encode::to_vec_named(&obj); + if data.is_ok() { + let data = data.unwrap(); + let mut tmp = [0_u8; 16]; + tmp[0] = message_type; + let len = 1 + varint::encode(&mut tmp[1..], data.len() as u64); + let mut writer = self.writer.lock().await; + writer.write_all(&tmp[0..len]).await?; + writer.write_all(data.as_slice()).await?; + self.last_send_time.store(now, Ordering::Relaxed); + Ok(()) + } else { + Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "serialize failure")) + } + } + + fn kill(&self) { + let _ = self.io_task.lock().unwrap().take().map(|h| h.abort()); + } + + async fn read_msg<'a>(&self, reader: &mut BufReader, buf: &'a mut Vec, message_size: usize, now: i64) -> std::io::Result<&'a [u8]> { + if message_size > buf.len() { + buf.resize(((message_size / 4096) + 1) * 4096, 0); + } + let b = &mut buf.as_mut_slice()[0..message_size]; + reader.read_exact(b).await?; + self.last_receive_time.store(now, Ordering::Relaxed); + Ok(b) + } + + async fn read_obj<'a, O: Deserialize<'a>>(&self, reader: &mut BufReader, buf: &'a mut Vec, message_size: usize, now: i64) -> std::io::Result { + rmp_serde::from_slice(self.read_msg(reader, buf, message_size, now).await?).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) + } +} + +pub struct NodeInternal { + anti_loopback_secret: [u8; 64], + db: Arc, + host: Arc, + bind_address: SocketAddr, + connections: Mutex>>, +} + +impl NodeInternal { + async fn housekeeping_task_main(self: Arc) { + loop { + tokio::time::sleep(Duration::from_millis(CONNECTION_KEEPALIVE_AFTER as u64)).await; + + let mut to_ping: Vec> = Vec::new(); + let mut connections = self.connections.lock().await; + let now = ms_monotonic(); + connections.retain(|_, c| { + c.upgrade().map_or(false, |c| { + if (now - c.last_receive_time.load(Ordering::Relaxed)) < CONNECTION_TIMEOUT { + if (now - c.last_send_time.load(Ordering::Relaxed)) > CONNECTION_KEEPALIVE_AFTER { + to_ping.push(c); + } + } else { + c.kill(); + return false; + } + return true; + }) + }); + drop(connections); // release lock + for c in to_ping.iter() { + if c.send(&[MESSAGE_TYPE_NOP, 0], now).await.is_err() { + c.kill(); + } + } + } + } + + async fn listener_task_main(self: Arc, listener: TcpListener) { + loop { + let socket = listener.accept().await; + if socket.is_ok() { + let (stream, endpoint) = socket.unwrap(); + Self::connection_start(&self, endpoint, stream, true).await; + } + } + } + + async fn connection_io_task_main(self: Arc, connection: Arc, reader: OwnedReadHalf) -> std::io::Result<()> { + let mut challenge = [0_u8; 16]; + self.host.get_secure_random(&mut challenge); + connection.send_obj(MESSAGE_TYPE_INIT, &msg::Init { + anti_loopback_challenge: &challenge, + domain: String::new(), // TODO + key_size: D::KEY_SIZE as u16, + max_value_size: D::MAX_VALUE_SIZE as u64, + node_name: None, + node_contact: None, + preferred_ipv4: None, + preferred_ipv6: None + }, ms_monotonic()).await?; + + let mut initialized = false; + let mut reader = BufReader::with_capacity(65536, reader); + let mut buf: Vec = Vec::new(); + buf.resize(4096, 0); + loop { + reader.read_exact(&mut buf.as_mut_slice()[0..1]).await?; + let message_type = unsafe { *buf.get_unchecked(0) }; + let message_size = varint::read_async(&mut reader).await?; + if message_size > (D::MAX_VALUE_SIZE + ((D::KEY_SIZE + 10) * 256) + 65536) as u64 { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "message too large")); + } + let now = ms_monotonic(); + + match message_type { + MESSAGE_TYPE_INIT => { + if initialized { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "duplicate init")); + } + let msg: msg::Init = connection.read_obj(&mut reader, &mut buf, message_size as usize, now).await?; + + let mut antiloop = msg.anti_loopback_challenge.to_vec(); + let _ = std::io::Write::write_all(&mut antiloop, &self.anti_loopback_secret); + let antiloop = H::sha512(antiloop.as_slice()); + + connection.send_obj(MESSAGE_TYPE_INIT_RESPONSE, &msg::InitResponse { + anti_loopback_response: &antiloop[0..16] + }, now).await?; + + initialized = true; + }, + _ => { + // Skip messages that aren't recognized or don't need to be parsed like NOP. + let mut remaining = message_size as usize; + while remaining > 0 { + let s = remaining.min(buf.len()); + reader.read_exact(&mut buf.as_mut_slice()[0..s]).await?; + remaining -= s; + } + connection.last_receive_time.store(ms_monotonic(), Ordering::Relaxed); + } + } + } + } + + async fn connection_start(self: &Arc, endpoint: SocketAddr, stream: TcpStream, incoming: bool) -> bool { + let (reader, writer) = stream.into_split(); + let mut ok = false; + let _ = self.connections.lock().await.entry(endpoint.clone()).or_insert_with(|| { + ok = true; + let now = ms_monotonic(); + let connection = Arc::new(Connection { + writer: Mutex::new(writer), + last_send_time: AtomicI64::new(now), + last_receive_time: AtomicI64::new(now), + io_task: std::sync::Mutex::new(None), + incoming + }); + let _ = connection.io_task.lock().unwrap().insert(tokio::spawn(Self::connection_io_task_main(self.clone(), connection.clone(), reader))); + Arc::downgrade(&connection) + }); + ok + } + + async fn connect(self: &Arc, endpoint: &SocketAddr) -> std::io::Result { + if !self.connections.lock().await.contains_key(endpoint) { + let stream = if endpoint.is_ipv4() { TcpSocket::new_v4() } else { TcpSocket::new_v6() }?; + if stream.set_reuseport(true).is_err() { + stream.set_reuseaddr(true)?; + } + stream.bind(self.bind_address.clone())?; + let stream = stream.connect(endpoint.clone()).await?; + Ok(self.connection_start(endpoint.clone(), stream, false).await) + } else { + Ok(false) + } + } +} + +impl Drop for NodeInternal { + fn drop(&mut self) { + for (_, c) in self.connections.blocking_lock().drain() { + let _ = c.upgrade().map(|c| c.kill()); + } + } +} + +pub struct Node { + internal: Arc>, + housekeeping_task: JoinHandle<()>, + listener_task: JoinHandle<()> +} + +impl Node { + pub async fn new(db: Arc, host: Arc, bind_address: SocketAddr) -> std::io::Result { + let listener = if bind_address.is_ipv4() { TcpSocket::new_v4() } else { TcpSocket::new_v6() }?; + if listener.set_reuseport(true).is_err() { + listener.set_reuseaddr(true)?; + } + listener.bind(bind_address.clone())?; + let listener = listener.listen(1024)?; + + let internal = Arc::new(NodeInternal:: { + anti_loopback_secret: { + let mut tmp = [0_u8; 64]; + host.get_secure_random(&mut tmp); + tmp + }, + db: db.clone(), + host: host.clone(), + bind_address, + connections: Mutex::new(HashMap::with_capacity(64)), + }); + Ok(Self { + internal: internal.clone(), + housekeeping_task: tokio::spawn(internal.clone().housekeeping_task_main()), + listener_task: tokio::spawn(internal.listener_task_main(listener)), + }) + } + + #[inline(always)] + pub async fn connect(&self, endpoint: &SocketAddr) -> std::io::Result { + self.internal.connect(endpoint).await + } + + pub fn list_connections(&self) -> Vec { + let mut connections = self.internal.connections.blocking_lock(); + let mut cl: Vec = Vec::with_capacity(connections.len()); + connections.retain(|sa, c| { + if c.strong_count() > 0 { + cl.push(sa.clone()); + true + } else { + false + } + }); + cl + } +} + +impl Drop for Node { + fn drop(&mut self) { + self.housekeeping_task.abort(); + self.listener_task.abort(); + } +} diff --git a/syncwhole/src/protocol.rs b/syncwhole/src/protocol.rs new file mode 100644 index 000000000..896925e48 --- /dev/null +++ b/syncwhole/src/protocol.rs @@ -0,0 +1,59 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * (c)2022 ZeroTier, Inc. + * https://www.zerotier.com/ + */ + +pub const MESSAGE_TYPE_NOP: u8 = 0; +pub const MESSAGE_TYPE_INIT: u8 = 1; +pub const MESSAGE_TYPE_INIT_RESPONSE: u8 = 2; +pub const MESSAGE_TYPE_HAVE_RECORDS: u8 = 3; +pub const MESSAGE_TYPE_GET_RECORDS: u8 = 4; + +pub mod msg { + use serde::{Serialize, Deserialize}; + + #[derive(Serialize, Deserialize)] + pub struct IPv4 { + #[serde(rename = "i")] + pub ip: [u8; 4], + #[serde(rename = "p")] + pub port: u16 + } + + #[derive(Serialize, Deserialize)] + pub struct IPv6 { + #[serde(rename = "i")] + pub ip: [u8; 16], + #[serde(rename = "p")] + pub port: u16 + } + + #[derive(Serialize, Deserialize)] + pub struct Init<'a> { + #[serde(rename = "alc")] + pub anti_loopback_challenge: &'a [u8], + #[serde(rename = "d")] + pub domain: String, + #[serde(rename = "ks")] + pub key_size: u16, + #[serde(rename = "mvs")] + pub max_value_size: u64, + #[serde(rename = "nn")] + pub node_name: Option, + #[serde(rename = "nc")] + pub node_contact: Option, + #[serde(rename = "pi4")] + pub preferred_ipv4: Option, + #[serde(rename = "pi6")] + pub preferred_ipv6: Option, + } + + #[derive(Serialize, Deserialize)] + pub struct InitResponse<'a> { + #[serde(rename = "alr")] + pub anti_loopback_response: &'a [u8], + } +} diff --git a/syncwhole/src/varint.rs b/syncwhole/src/varint.rs new file mode 100644 index 000000000..c656944bb --- /dev/null +++ b/syncwhole/src/varint.rs @@ -0,0 +1,76 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * (c)2022 ZeroTier, Inc. + * https://www.zerotier.com/ + */ + +use std::io::{Read, Write}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +const VARINT_MAX_SIZE_BYTES: usize = 10; + +pub fn encode(b: &mut [u8], mut v: u64) -> usize { + let mut i = 0; + loop { + if v > 0x7f { + b[i] = (v as u8) & 0x7f; + i += 1; + v = v.wrapping_shr(7); + } else { + b[i] = (v as u8) | 0x80; + i += 1; + break; + } + } + i +} + +#[inline(always)] +pub async fn write_async(w: &mut W, v: u64) -> std::io::Result<()> { + let mut b = [0_u8; VARINT_MAX_SIZE_BYTES]; + let i = encode(&mut b, v); + w.write_all(&b[0..i]).await +} + +pub async fn read_async(r: &mut R) -> std::io::Result { + let mut v = 0_u64; + let mut buf = [0_u8; 1]; + let mut pos = 0; + loop { + let _ = r.read_exact(&mut buf).await?; + let b = buf[0]; + if b <= 0x7f { + v |= (b as u64).wrapping_shl(pos); + pos += 7; + } else { + v |= ((b & 0x7f) as u64).wrapping_shl(pos); + return Ok(v); + } + } +} + +#[inline(always)] +pub fn write(w: &mut W, v: u64) -> std::io::Result<()> { + let mut b = [0_u8; VARINT_MAX_SIZE_BYTES]; + let i = encode(&mut b, v); + w.write_all(&b[0..i]) +} + +pub fn read(r: &mut R) -> std::io::Result { + let mut v = 0_u64; + let mut buf = [0_u8; 1]; + let mut pos = 0; + loop { + let _ = r.read_exact(&mut buf)?; + let b = buf[0]; + if b <= 0x7f { + v |= (b as u64).wrapping_shl(pos); + pos += 7; + } else { + v |= ((b & 0x7f) as u64).wrapping_shl(pos); + return Ok(v); + } + } +} diff --git a/zerotier-network-hypervisor/src/vl1/identity.rs b/zerotier-network-hypervisor/src/vl1/identity.rs index b0725bdbe..836c24061 100644 --- a/zerotier-network-hypervisor/src/vl1/identity.rs +++ b/zerotier-network-hypervisor/src/vl1/identity.rs @@ -137,14 +137,15 @@ impl Identity { fingerprint: [0_u8; 64] // replaced in upgrade() }; assert!(id.upgrade().is_ok()); + assert!(id.p384.is_some() && id.secret.as_ref().unwrap().p384.is_some()); id } /// Upgrade older x25519-only identities to hybrid identities with both x25519 and NIST P-384 curves. /// - /// The identity must contain its x25519 secret key or an error occurs. If the identity is already - /// a new form hybrid identity nothing happens and Ok is returned. - pub fn upgrade(&mut self) -> Result<(), InvalidParameterError> { + /// The boolean indicates whether or not an upgrade occurred. An error occurs if this identity is + /// invalid or missing its private key(s). This does nothing if no upgrades are possible. + pub fn upgrade(&mut self) -> Result { if self.secret.is_none() { return Err(InvalidParameterError("an identity can only be upgraded if it includes its private key")); } @@ -179,10 +180,11 @@ impl Identity { ecdh: p384_ecdh, ecdsa: p384_ecdsa, }); - self.fingerprint = SHA512::hash(self_sign_buf.as_slice()); + + return Ok(true); } - return Ok(()); + return Ok(false); } #[inline(always)] diff --git a/zerotier-network-hypervisor/src/vl1/node.rs b/zerotier-network-hypervisor/src/vl1/node.rs index a7a7aa5f2..244870c4e 100644 --- a/zerotier-network-hypervisor/src/vl1/node.rs +++ b/zerotier-network-hypervisor/src/vl1/node.rs @@ -35,10 +35,6 @@ pub trait SystemInterface: Sync + Send { /// Node is shutting down. fn event_node_is_down(&self); - /// A root signaled an identity collision. - /// This should cause the external code to shut down this node, delete its identity, and recreate. - fn event_identity_collision(&self); - /// Node has gone online or offline. fn event_online_status_change(&self, online: bool); @@ -49,8 +45,7 @@ pub trait SystemInterface: Sync + Send { fn load_node_identity(&self) -> Option>; /// Save this node's identity. - /// Note that this is only called on first startup (after up) and after identity_changed. - fn save_node_identity(&self, id: &Identity, public: &[u8], secret: &[u8]); + fn save_node_identity(&self, id: &Identity); /// Called to send a packet over the physical network (virtual -> physical). /// @@ -145,14 +140,14 @@ pub struct Node { impl Node { /// Create a new Node. pub fn new(si: &SI, auto_generate_identity: bool) -> Result { - let id = { + let mut id = { let id_str = si.load_node_identity(); if id_str.is_none() { if !auto_generate_identity { return Err(InvalidParameterError("no identity found and auto-generate not enabled")); } else { let id = Identity::generate(); - si.save_node_identity(&id, id.to_string().as_bytes(), id.to_secret_string().as_bytes()); + si.save_node_identity(&id); id } } else { @@ -166,6 +161,11 @@ impl Node { } }; + // Automatically upgrade old type identities to add P-384 keys. + if id.upgrade()? { + si.save_node_identity(&id); + } + Ok(Self { instance_id: zerotier_core_crypto::random::next_u64_secure(), identity: id, @@ -203,6 +203,14 @@ impl Node { let mut intervals = self.intervals.lock(); let tt = si.time_ticks(); + if intervals.peers.gate(tt) { + self.peers.retain(|_, peer| { + peer.call_every_interval(si, tt); + todo!(); + true + }); + } + if intervals.paths.gate(tt) { self.paths.retain(|_, path| { path.upgrade().map_or(false, |p| { @@ -212,14 +220,6 @@ impl Node { }); } - if intervals.peers.gate(tt) { - self.peers.retain(|_, peer| { - peer.call_every_interval(si, tt); - todo!(); - true - }); - } - if intervals.whois.gate(tt) { self.whois.call_every_interval(self, si, tt); }