From fbec58576fc469123a4c1499180ac0193e119c0e Mon Sep 17 00:00:00 2001 From: Zicklag Date: Wed, 15 Sep 2021 01:24:02 -0500 Subject: [PATCH 01/12] Start Work on Initial Server Structure --- .gitignore | 5 +- Cargo.lock | 744 +++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 31 +- playground/Cargo.toml | 16 + playground/src/main.rs | 54 +++ src/engine.rs | 101 ++++++ src/graph.rs | 233 +++++++++++++ src/lib.rs | 35 +- src/protocol.rs | 0 src/store.rs | 197 +++++++++++ 10 files changed, 1411 insertions(+), 5 deletions(-) create mode 100644 Cargo.lock create mode 100644 playground/Cargo.toml create mode 100644 playground/src/main.rs create mode 100644 src/engine.rs create mode 100644 src/graph.rs create mode 100644 src/protocol.rs create mode 100644 src/store.rs diff --git a/.gitignore b/.gitignore index 396db90..c77bb18 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,7 @@ Cargo.lock # MSVC Windows builds of rustc generate these, which store debugging information *.pdb -*.DS_store \ No newline at end of file +*.DS_store + +# Data generated by running the demo server +data/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..a1f2b24 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,744 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1" + +[[package]] +name = "async-channel" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-executor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "once_cell", + "slab", +] + +[[package]] +name = "async-task" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" + +[[package]] +name = "async-timer" +version = "1.0.0-beta.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d962799a5863fdf06fbf594e04102130582d010379137e9a98a7e2e693a5885" +dependencies = [ + "error-code", + "libc", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "async-trait" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "blocking" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9" +dependencies = [ + "async-channel", + "async-task", + "atomic-waker", + "fastrand", + "futures-lite", + "once_cell", +] + +[[package]] +name = "bumpalo" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" + +[[package]] +name = "cache-padded" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "winapi", +] + +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + +[[package]] +name = "easy-parallel" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd4afd79212583ff429b913ad6605242ed7eec277e950b1438f300748f948f4" + +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + +[[package]] +name = "error-code" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5115567ac25674e0043e472be13d14e537f37ea8aa4bdc4aef0c89add1db1ff" +dependencies = [ + "libc", + "str-buf", +] + +[[package]] +name = "event-listener" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" + +[[package]] +name = "fastrand" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b394ed3d285a429378d3b384b9eb1285267e7df4b166df24b7a6939a04dc392e" +dependencies = [ + "instant", +] + +[[package]] +name = "futures-core" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d" + +[[package]] +name = "futures-io" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" + +[[package]] +name = "futures-lite" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "instant" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21" + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "matchers" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" + +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + +[[package]] +name = "pin-project-lite" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" + +[[package]] +name = "playground" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-executor", + "async-timer", + "atty", + "futures-lite", + "rod", + "serde_json", + "tracing", + "tracing-error", + "tracing-subscriber", +] + +[[package]] +name = "proc-macro2" +version = "1.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", + "serde", +] + +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "rod" +version = "0.1.0" +dependencies = [ + "async-executor", + "async-timer", + "async-trait", + "base64", + "blocking", + "easy-parallel", + "futures-lite", + "num_cpus", + "once_cell", + "radix_trie", + "scc", + "serde", + "serde_json", + "thiserror", + "tracing", + "uuid", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "scc" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d602af27b0129ed0b3a9becec8c3eadca7c6e0d34bd7595c5fc012d015c0b6" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7f9e390c27c3c0ce8bc5d725f6e4d30a29d26659494aa4b17535f7522c5c950" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "740223c51853f3145fe7c90360d2d4232f2b62e3449489c207eccde818979982" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "slab" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "str-buf" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d44a3643b4ff9caf57abcee9c2c621d6c03d9135e0d8b589bd9afb5992cb176a" + +[[package]] +name = "syn" +version = "1.0.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f107db402c2c2055242dbf4d2af0e69197202e9faacbef9571bbe47f5a1b84" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "thiserror" +version = "1.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602eca064b2d83369e2b2f34b09c70b605402801927c65c11071ac911d299b88" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad553cc2c78e8de258400763a647e80e6d1b31ee237275d756f6836d204494c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tracing" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2ba9ab62b7d6497a8638dfda5e5c4fb3b2d5a7fca4118f2b96151c8ef1a437e" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98863d0dd09fa59a1b79c6750ad80dbda6b75f4e71c437a6a1a8cb91a8bcbd77" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46125608c26121c81b0c6d693eab5a420e416da7e43c426d2e8f7df8da8a3acf" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-error" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4d7c0b83d4a500748fa5879461652b361edf5c9d51ede2a2ac03875ca185e24" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-log" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62af966210b88ad5776ee3ba12d5f35b8d6a2b2a12168f3080cf02b814d7376b" +dependencies = [ + "ansi_term", + "chrono", + "lazy_static", + "matchers", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "wasm-bindgen" +version = "0.2.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e68338db6becec24d3c7977b5bf8a48be992c934b5d07177e3931f5dc9b076c" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f34c405b4f0658583dba0c1c7c9b694f3cac32655db463b56c254a1c75269523" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d5a6580be83b19dc570a8f9c324251687ab2184e57086f71625feb57ec77c8" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3775a030dc6f5a0afd8a84981a21cc92a781eb429acef9ecce476d0c9113e92" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c279e376c7a8e8752a8f1eaa35b7b0bee6bb9fb0cdacfa97cc3f1f289c87e2b4" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/Cargo.toml b/Cargo.toml index 5e0473c..37b6cda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,4 +3,33 @@ name = "rod" version = "0.1.0" edition = "2018" -[dependencies] \ No newline at end of file +[workspace] +members = [ + ".", + "playground", +] + +[dependencies] +# Data structures +uuid = { version = "0.8.2", features = ["v4", "serde"] } +radix_trie = { version = "0.2", features = ["serde"] } +once_cell = "1.8.0" +scc = "0.5.2" +thiserror = "1.0.29" + +# Logging +tracing = "0.1.27" + +# Serialization +serde = { version = "1.0.130", features = ["derive"] } +serde_json = "1.0.67" +base64 = "0.13.0" + +# Async +num_cpus = "1.13.0" +easy-parallel = "3.1.0" +async-executor = "1.4.1" +futures-lite = "1.12.0" +blocking = "1.0.2" +async-trait = "0.1.51" +async-timer = "1.0.0-beta.7" diff --git a/playground/Cargo.toml b/playground/Cargo.toml new file mode 100644 index 0000000..3ddc741 --- /dev/null +++ b/playground/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "playground" +version = "0.1.0" +edition = "2018" + +[dependencies] +rod = { path = "../"} +serde_json = "1.0.67" +async-executor = "1.4.1" +futures-lite = "1.12.0" +anyhow = "1.0.44" +async-timer = "1.0.0-beta.7" +tracing-subscriber = "0.2.22" +tracing-error = "0.1.2" +atty = "0.2.14" +tracing = "0.1.27" \ No newline at end of file diff --git a/playground/src/main.rs b/playground/src/main.rs new file mode 100644 index 0000000..dabe398 --- /dev/null +++ b/playground/src/main.rs @@ -0,0 +1,54 @@ +use std::{process, time::Duration}; +use tracing as trc; + +use rod::engine::Rod; + +use futures_lite::future; + +fn main() { + let ex = async_executor::Executor::new(); + if let Err(e) = future::block_on(ex.run(start())) { + eprintln!("Error: {:#?}", e); + process::exit(1); + } +} + +async fn start() -> anyhow::Result<()> { + install_tracing(); + + trc::info!("Staring server"); + + let engine = Rod::new().await?; + + // Just prevent the process from exiting + let mut interval = async_timer::interval(Duration::from_secs(1)); + loop { + interval.wait().await; + } +} + +fn install_tracing() { + use tracing_error::ErrorLayer; + use tracing_subscriber::prelude::*; + use tracing_subscriber::{fmt, fmt::format::FmtSpan, EnvFilter}; + + // Build the tracing layers + let fmt_layer = + fmt::layer() + .with_span_events(FmtSpan::FULL) + .with_ansi(if atty::is(atty::Stream::Stdout) { + true + } else { + false + }); + let filter_layer = EnvFilter::try_from_default_env() + .or_else(|_| EnvFilter::try_new("info")) + .unwrap(); + + // Add all of the layers to the subscriber and initialize it + tracing_subscriber::registry() + .with(filter_layer) + .with(fmt_layer) + .with(ErrorLayer::default()) + .init(); +} diff --git a/src/engine.rs b/src/engine.rs new file mode 100644 index 0000000..7bf4383 --- /dev/null +++ b/src/engine.rs @@ -0,0 +1,101 @@ +use std::{sync::Arc, time::Duration}; + +use tracing as trc; +use uuid::Uuid; + +use crate::{ + executor, + graph::Node, + store::{get_default_store, Store, StoreError}, +}; + +/// The Rod engine, responsible for managing connections and performing the various database +/// synchronization task +/// +/// [`Rod`] is the primary public API for accessing the database. +/// +/// The [`Rod`] instance is cheap to clone and can be sent and shared across threads to allow +/// accessing the database concurrently from different threads. +#[derive(Clone)] +pub struct Rod { + /// The inner data of the [`Rod`] instance + inner: Arc, +} + +/// The interval at which modifications to the database will be flushed to disk +/// +/// TODO: Make this configuable per [`Rod`] instance +const FLUSH_INTERVAL: Duration = Duration::from_secs(2); + +struct RodInner { + /// The list of nodes cached in memory + nodes: scc::HashMap, + /// Nodes that have been modified in memory and need to be flushed to disk + dirty_nodes: scc::HashMap, + /// The backing data store for this engine + store: Box, +} + +impl Rod { + /// Initialize a new [`Rod`] instance + /// + /// TODO: Use an `EngineBuilder` to construct an engine with customized store and peers list + pub async fn new() -> Result { + trc::trace!("Creating new instance"); + // Initialize data store + let store = Box::new(get_default_store().await?); + + // Create clonable inner data + let inner = Arc::new(RodInner { + nodes: Default::default(), + dirty_nodes: Default::default(), + store, + }); + + // Create Rod instance + let instance = Rod { inner }; + + // Spawn a job to flush the dirty nodes to disk + let instance_ = instance.clone(); + executor::spawn(async move { + trc::debug!("Staring periodic node flush"); + let db = instance_.inner; + + // Loop on and interval + let mut interval = async_timer::interval(FLUSH_INTERVAL); + loop { + trc::debug!("Flushing dirty nodes to disk"); + + // Flush all dirty nodes to disk + let count = db.dirty_nodes.len(); + let mut dirty_nodes = Vec::with_capacity(count); + db.dirty_nodes.retain(|uuid, _| { + dirty_nodes.push(uuid.clone()); + true + }); + for uuid in dirty_nodes { + if let Some(node) = db.nodes.read(&uuid, |_, node| node.clone()) { + if let Err(err) = db + .store + .put_string( + &uuid.to_string(), + &serde_json::to_string_pretty(&node).expect("Serialize node"), + ) + .await + { + trc::error!(%err, "Could not flush node data to the store. The store may be out-of-date!"); + } + } + } + + trc::debug!(%count, " Flushed nodes to disk"); + + interval.wait().await; + } + }).detach(); + + Ok(instance) + } + + // pub async fn get(key: &str) -> +} diff --git a/src/graph.rs b/src/graph.rs new file mode 100644 index 0000000..f2af343 --- /dev/null +++ b/src/graph.rs @@ -0,0 +1,233 @@ +//! Data structures used to build the rod data graph + +use std::{ + cmp, + collections::HashMap, + mem, + ops::Deref, + time::{Duration, SystemTime}, +}; + +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +pub trait LexicalCmp { + fn lexical_cmp(&self, other: &Self) -> cmp::Ordering; +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Node { + pub id: Uuid, + pub fields: HashMap, +} + +impl Default for Node { + fn default() -> Self { + Self { + id: Uuid::new_v4(), + fields: Default::default(), + } + } +} + +impl Node { + pub fn new() -> Self { + Self::default() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Field { + /// Time that this field value was updated as relative to the + /// [`UNIX_EPOCH`][std::time::SystemTime::UNIX_EPOCH] + updated_at: Duration, + value: Value, +} + +/// If an update comes in that is more than this amount of time in the future, we will assume that +/// the node that sent the update is lying and trying to make it's update take precedence over the +/// current value of the field. +const FUTURE_UPDATE_THREASHOLD: Duration = Duration::from_secs(60 * 10); + +impl Field { + /// Merge the new value into this field, using the [HAM] merge conflict resolution strategy + /// + /// [HAM]: https://github.com/amark/gun/wiki/Conflict-Resolution-with-Guns + pub fn merge_with(&mut self, field: &Field) { + let current_time = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("TODO: system time"); + + // If the new field has the same timestamp + if field.updated_at == self.updated_at { + match self.lexical_cmp(field) { + // Totally equal, do nothing + cmp::Ordering::Equal => return, + // Keep our value, do nothing + cmp::Ordering::Less => return, + // Keep the new value + cmp::Ordering::Greater => self.value = field.value.clone(), + } + + // If the other field is and older update than the one we have, just ignore it + } else if field.updated_at < self.updated_at { + return; + + // If the other field is in the future + } else if field.updated_at > current_time { + // If the field is too far in the future, ignore it + if field.updated_at - current_time > FUTURE_UPDATE_THREASHOLD { + return; + } + + // Wait to apply this update until later + unimplemented!( + "Use async logic to apply this update once our system clock \ + reaches the future time" + ); + } else { + unreachable!() + } + } +} + +impl Field { + pub fn new(value: Value) -> Self { + Self { + updated_at: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("Could not get system time"), + value, + } + } + + pub fn value(&self) -> &Value { + &self.value + } + + pub fn state(&self) -> &Duration { + &self.updated_at + } +} + +impl Deref for Field { + type Target = Value; + + fn deref(&self) -> &Self::Target { + &self.value + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Value { + None, + Bool(bool), + Int(i64), + Float(f64), + String(String), + Binary(Vec), + Node(Uuid), +} + +impl From for Value { + fn from(s: String) -> Self { + Self::String(s) + } +} + +impl From<&str> for Value { + fn from(s: &str) -> Self { + Self::String(s.to_string()) + } +} + +impl From for Value { + fn from(b: bool) -> Self { + Self::Bool(b) + } +} + +impl From for Value { + fn from(i: i64) -> Self { + Self::Int(i) + } +} + +impl From for Value { + fn from(i: i32) -> Self { + Self::Int(i as i64) + } +} + +impl From for Value { + fn from(f: f32) -> Self { + Self::Float(f as f64) + } +} + +impl From for Value { + fn from(f: f64) -> Self { + Self::Float(f) + } +} + +impl From> for Value { + fn from(b: Vec) -> Self { + Self::Binary(b) + } +} + +impl From for Value { + fn from(u: Uuid) -> Self { + Self::Node(u) + } +} + +impl LexicalCmp for Value { + fn lexical_cmp(&self, other: &Self) -> cmp::Ordering { + use Value::*; + if mem::discriminant(self) == mem::discriminant(other) { + match (self, other) { + (Bool(x), Bool(y)) => match (x, y) { + (true, true) => cmp::Ordering::Equal, + (true, false) => cmp::Ordering::Less, + (false, true) => cmp::Ordering::Greater, + (false, false) => cmp::Ordering::Equal, + }, + (Int(x), Int(y)) => x.cmp(y), + (Float(x), Float(y)) => x.partial_cmp(y).unwrap_or(cmp::Ordering::Less), + (String(x), String(y)) => x.cmp(y), + (Binary(x), Binary(y)) => x.cmp(y), + (Node(x), Node(y)) => x.cmp(y), + _ => unreachable!(), + } + } else { + let self_rank = match self { + None => 0, + Bool(_) => 1, + Int(_) => 2, + Float(_) => 3, + String(_) => 4, + Binary(_) => 5, + Node(_) => 6, + }; + let other_rank = match other { + None => 0, + Bool(_) => 1, + Int(_) => 2, + Float(_) => 3, + String(_) => 4, + Binary(_) => 5, + Node(_) => 6, + }; + + if self_rank < other_rank { + cmp::Ordering::Less + } else if self_rank > other_rank { + cmp::Ordering::Greater + } else { + unreachable!() + } + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 29b0977..04a9a77 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,32 @@ -mod object; -mod all; -mod rod; \ No newline at end of file +pub mod engine; +pub mod graph; +pub mod protocol; +pub mod store; + +pub(crate) mod executor { + use std::{future::Future, panic, thread}; + + use async_executor::{Executor, Task}; + use futures_lite::future; + use once_cell::sync::Lazy; + + pub fn spawn(future: impl Future + Send + 'static) -> Task { + static GLOBAL: Lazy> = Lazy::new(|| { + for n in 1..=num_cpus::get() { + thread::Builder::new() + .name(format!("rod-worker-{}", n)) + .spawn(|| loop { + panic::catch_unwind(|| { + future::block_on(GLOBAL.run(future::pending::<()>())) + }) + .ok(); + }) + .expect("cannot spawn executor thread"); + } + + Executor::new() + }); + + GLOBAL.spawn(future) + } +} diff --git a/src/protocol.rs b/src/protocol.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/store.rs b/src/store.rs new file mode 100644 index 0000000..2cfdb25 --- /dev/null +++ b/src/store.rs @@ -0,0 +1,197 @@ +use std::{ + fs::{self, OpenOptions}, + io::{Read, Write}, + path::{Path, PathBuf}, +}; + +use blocking::unblock; + +/// Get the default store implementation for the current platform +/// +/// Currently on non-WASM platforms this is the [`SimpleFsStore`], configured to use the `./data` +/// directory for storage. +/// +/// The default store implementation on WASM will use the browser's IndexedDB, but this is not yet +/// implemented. +pub async fn get_default_store() -> Result { + #[cfg(not(target_arch = "wasm32"))] + let store = SimpleFsStore::new(Path::new("./data")).await; + + #[cfg(target_arch = "wasm32")] + compile_error!("TODO: Implement WASM default store implementation"); + + store +} + +/// A simple, raw data storage interface +/// +/// [`Store`] is designed to be implemented over any persistant storage interface such as the +/// filesystem, S3, browser LocalStorage or IndexedDB, etc. +/// +/// The interface is a simple key-value store, where each key could either have no data asociated to +/// it, or it could have a value that may either be either string data or binary data. +/// +/// Because some data stores such as browser local storage can only store string data, separate +/// functions are used for getting and setting string and binary data. This allows the +/// implementation to choose to base64 encode/decode binary values when writing to a string-only +/// data store. +/// +/// Calling [`Store::get_binary`] on a key on a string value or calling [`Store::get_string`] on a +/// binary value may either fail with an error, or simply interpert the contents of the key as the +/// other value type depending on the implementation. In-general you should only call the "get" +/// function that corresponds with the value at the given key to avoid unexpected behavior. +#[async_trait::async_trait] +pub trait Store { + /// Get a string value from the store + /// + /// # Errors + /// + /// - This function **will** error if there is a problem accessing the store + /// - This funciton **may** error or return an unexpected value if you try to read a key that + /// has a binary value currently set. + async fn get_string(&self, key: &str) -> Result, StoreError>; + /// Get a binary value from the store + /// + /// # Errors + /// + /// - This function **will** error if there is a problem accessing the store + /// - This funciton **may** error or return an unexpected value if you try to read a key that + /// has a string value currently set. + async fn get_binary(&self, key: &str) -> Result>, StoreError>; + /// Store a string value in the store + async fn put_string(&self, key: &str, data: &str) -> Result<(), StoreError>; + /// Store a binary value in the store + async fn put_binary(&self, key: &str, data: &[u8]) -> Result<(), StoreError>; + /// Delete a key + async fn delete(&self, key: &str) -> Result<(), StoreError>; +} + +/// An error that can occur in a [`Store`] +#[derive(Debug, thiserror::Error)] +pub enum StoreError { + /// Attempted to read a binary value as a string + #[error("Attempted to read binary data as a string")] + ReadBinaryAsString, + /// Attempted to read a string value as binary + #[error("Attempted to read string data as binary")] + ReadStringAsBinary, + /// Other, implementation-specific error + #[error("Error: {0}")] + Other(#[from] Box), +} + +/// Ultra-simple filesystem [`Store`] implementation that uses a separate file for each string key +/// +/// The names of each file will be the base64-encoded key and the value will be the string or binary +/// data associated to the key. +pub struct SimpleFsStore { + /// The root directory for the filesystem store + root_dir: PathBuf, +} + +impl SimpleFsStore { + /// Create a new [`FsStore`] that puts files in the specified `root_dir` + pub async fn new(root_dir: &Path) -> Result { + let root_dir = root_dir.to_owned(); + + unblock(move || { + fs::create_dir_all(&root_dir).map_err(box_error)?; + + Ok(Self { + root_dir: root_dir.to_owned(), + }) + }) + .await + } +} + +#[async_trait::async_trait] +impl Store for SimpleFsStore { + async fn get_binary(&self, key: &str) -> Result>, StoreError> { + // Get the path to the file + let file_path = self.root_dir.join(key); + + // Perform blocking operations on a thread pool + unblock(move || { + // Check if the file exists + if !file_path.exists() { + return Ok(None); + } + + // Open the file + let mut file = OpenOptions::new() + .read(true) + .open(file_path) + .map_err(box_error)?; + + // Read the file into buffer + let mut buf = Vec::new(); + file.read_to_end(&mut buf).map_err(box_error)?; + + // And return the buffer + Ok(Some(buf)) + }) + .await + } + + async fn put_binary(&self, key: &str, data: &[u8]) -> Result<(), StoreError> { + // Get the path to the file + let file_path = self.root_dir.join(key); + // Clone the data + let data = data.to_vec(); + + // Perform blocking operations on a thread pool + unblock(move || { + // Open the file + let mut file = OpenOptions::new() + .write(true) + .truncate(true) + .create(true) + .open(file_path) + .map_err(box_error)?; + + // Write the data to the file + Ok(file.write_all(&data).map_err(box_error)?) + }) + .await + } + + async fn get_string(&self, key: &str) -> Result, StoreError> { + // Get the binary data if it exists + if let Some(binary) = self.get_binary(key).await? { + // Parse the binary data as a UTF-8 String + Ok(Some(String::from_utf8(binary).map_err(box_error)?)) + + // The key doesn't exist + } else { + Ok(None) + } + } + + async fn put_string(&self, key: &str, data: &str) -> Result<(), StoreError> { + // Store the string as UTF-8 bytes + self.put_binary(key, data.as_bytes()).await + } + + async fn delete(&self, key: &str) -> Result<(), StoreError> { + // Get the path to the file + let file_path = self.root_dir.join(key); + + // Perform blocking operation on a thread pool + unblock(move || { + // Delete the file + fs::remove_file(file_path).map_err(box_error)?; + + // Write the data to the file + Ok(()) + }) + .await + } +} + +/// Helper to box the an error +fn box_error( + e: impl std::error::Error + Sync + Send + 'static, +) -> Box { + Box::new(e) +} From a6cffaff3ebc2d2d6d87f1b1501367ee87e7bc60 Mon Sep 17 00:00:00 2001 From: Zicklag Date: Wed, 15 Sep 2021 02:26:54 -0500 Subject: [PATCH 02/12] Test Some Binary Size Optimization Stuff --- Cargo.lock | 57 +++++++++++++++++++++++++++++++++++++++---- Cargo.toml | 13 +++++++++- playground/Cargo.toml | 2 +- src/executor.rs | 25 +++++++++++++++++++ src/lib.rs | 32 ++++++------------------ src/store.rs | 29 +++++++++++++++++++++- wasmtest/.gitignore | 1 + wasmtest/Cargo.toml | 12 +++++++++ wasmtest/src/main.rs | 18 ++++++++++++++ 9 files changed, 156 insertions(+), 33 deletions(-) create mode 100644 src/executor.rs create mode 100644 wasmtest/.gitignore create mode 100644 wasmtest/Cargo.toml create mode 100644 wasmtest/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index a1f2b24..079f1a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -126,6 +126,12 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -223,9 +229,11 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -243,7 +251,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -252,6 +260,15 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "js-sys" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1866b355d9c878e5e607473cbe3f63282c0b7aad2db1dbebf55076c686918254" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -270,7 +287,7 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -288,6 +305,12 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memory_units" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" + [[package]] name = "nibble_vec" version = "0.1.0" @@ -424,6 +447,7 @@ dependencies = [ "blocking", "easy-parallel", "futures-lite", + "getrandom", "num_cpus", "once_cell", "radix_trie", @@ -433,6 +457,7 @@ dependencies = [ "thiserror", "tracing", "uuid", + "wee_alloc", ] [[package]] @@ -560,7 +585,7 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2ba9ab62b7d6497a8638dfda5e5c4fb3b2d5a7fca4118f2b96151c8ef1a437e" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -673,7 +698,7 @@ version = "0.2.77" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e68338db6becec24d3c7977b5bf8a48be992c934b5d07177e3931f5dc9b076c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen-macro", ] @@ -721,6 +746,28 @@ version = "0.2.77" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c279e376c7a8e8752a8f1eaa35b7b0bee6bb9fb0cdacfa97cc3f1f289c87e2b4" +[[package]] +name = "wasmtest" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-executor", + "futures-lite", + "rod", +] + +[[package]] +name = "wee_alloc" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb3b5a6b2bb17cb6ad44a2e68a43e8d2722c997da10e928665c72ec6c0a0b8e" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "memory_units", + "winapi", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 37b6cda..38a1b14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,11 +7,12 @@ edition = "2018" members = [ ".", "playground", + "wasmtest", ] [dependencies] # Data structures -uuid = { version = "0.8.2", features = ["v4", "serde"] } +uuid = { version = "0.8.2", features = ["v4", "serde", "wasm-bindgen"] } radix_trie = { version = "0.2", features = ["serde"] } once_cell = "1.8.0" scc = "0.5.2" @@ -33,3 +34,13 @@ futures-lite = "1.12.0" blocking = "1.0.2" async-trait = "0.1.51" async-timer = "1.0.0-beta.7" +getrandom = { version = "*", features = ["js"] } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +wee_alloc = "0.4.5" + +[profile.release] +lto = true +opt-level = "z" +panic = "abort" +debug = false \ No newline at end of file diff --git a/playground/Cargo.toml b/playground/Cargo.toml index 3ddc741..6c9b016 100644 --- a/playground/Cargo.toml +++ b/playground/Cargo.toml @@ -5,9 +5,9 @@ edition = "2018" [dependencies] rod = { path = "../"} -serde_json = "1.0.67" async-executor = "1.4.1" futures-lite = "1.12.0" +serde_json = "1.0.67" anyhow = "1.0.44" async-timer = "1.0.0-beta.7" tracing-subscriber = "0.2.22" diff --git a/src/executor.rs b/src/executor.rs new file mode 100644 index 0000000..24166dc --- /dev/null +++ b/src/executor.rs @@ -0,0 +1,25 @@ +use std::{future::Future, panic, thread}; + +use async_executor::{Executor, Task}; +use futures_lite::future; +use once_cell::sync::Lazy; + +pub fn spawn(future: impl Future + Send + 'static) -> Task { + static GLOBAL: Lazy> = Lazy::new(|| { + for n in 1..=num_cpus::get() { + thread::Builder::new() + .name(format!("rod-worker-{}", n)) + .spawn(|| loop { + panic::catch_unwind(|| { + future::block_on(GLOBAL.run(future::pending::<()>())) + }) + .ok(); + }) + .expect("cannot spawn executor thread"); + } + + Executor::new() + }); + + GLOBAL.spawn(future) +} diff --git a/src/lib.rs b/src/lib.rs index 04a9a77..d9d8acb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,30 +3,12 @@ pub mod graph; pub mod protocol; pub mod store; -pub(crate) mod executor { - use std::{future::Future, panic, thread}; +pub(crate) mod executor; - use async_executor::{Executor, Task}; - use futures_lite::future; - use once_cell::sync::Lazy; +#[cfg(target_arch = "wasm32")] +extern crate wee_alloc; - pub fn spawn(future: impl Future + Send + 'static) -> Task { - static GLOBAL: Lazy> = Lazy::new(|| { - for n in 1..=num_cpus::get() { - thread::Builder::new() - .name(format!("rod-worker-{}", n)) - .spawn(|| loop { - panic::catch_unwind(|| { - future::block_on(GLOBAL.run(future::pending::<()>())) - }) - .ok(); - }) - .expect("cannot spawn executor thread"); - } - - Executor::new() - }); - - GLOBAL.spawn(future) - } -} +// Use `wee_alloc` as the global allocator for WASM +#[cfg(target_arch = "wasm32")] +#[global_allocator] +static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; diff --git a/src/store.rs b/src/store.rs index 2cfdb25..64107e5 100644 --- a/src/store.rs +++ b/src/store.rs @@ -18,11 +18,38 @@ pub async fn get_default_store() -> Result { let store = SimpleFsStore::new(Path::new("./data")).await; #[cfg(target_arch = "wasm32")] - compile_error!("TODO: Implement WASM default store implementation"); + let store = Ok(IndexedDbStore); store } +#[cfg(target_arch = "wasm32")] +struct IndexedDbStore; + +#[cfg(target_arch = "wasm32")] +#[async_trait::async_trait] +impl Store for IndexedDbStore { + async fn get_string(&self, key: &str) -> Result, StoreError> { + todo!() + } + + async fn get_binary(&self, key: &str) -> Result>, StoreError> { + todo!() + } + + async fn put_string(&self, key: &str, data: &str) -> Result<(), StoreError> { + todo!() + } + + async fn put_binary(&self, key: &str, data: &[u8]) -> Result<(), StoreError> { + todo!() + } + + async fn delete(&self, key: &str) -> Result<(), StoreError> { + todo!() + } +} + /// A simple, raw data storage interface /// /// [`Store`] is designed to be implemented over any persistant storage interface such as the diff --git a/wasmtest/.gitignore b/wasmtest/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/wasmtest/.gitignore @@ -0,0 +1 @@ +/target diff --git a/wasmtest/Cargo.toml b/wasmtest/Cargo.toml new file mode 100644 index 0000000..bb316f9 --- /dev/null +++ b/wasmtest/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "wasmtest" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rod = { path = "../"} +async-executor = "1.4.1" +futures-lite = "1.12.0" +anyhow = "1.0.44" \ No newline at end of file diff --git a/wasmtest/src/main.rs b/wasmtest/src/main.rs new file mode 100644 index 0000000..37a0322 --- /dev/null +++ b/wasmtest/src/main.rs @@ -0,0 +1,18 @@ +use std::{time::Duration}; + +use rod::engine::Rod; + +use futures_lite::future; + +fn main() { + let ex = async_executor::Executor::new(); + future::block_on(ex.run(start())).expect("Error"); +} + +async fn start() -> anyhow::Result<()> { + let engine = Rod::new().await?; + + future::pending::<()>().await; + + Ok(()) +} From 649ae3387739c945b0ae472d6fcf2cf239ca8a99 Mon Sep 17 00:00:00 2001 From: Zicklag Date: Wed, 22 Sep 2021 19:24:46 -0500 Subject: [PATCH 03/12] Stores Should Store Nodes, Not Binary/Strings --- .gitignore | 2 + Cargo.lock | 286 +++++++++++++++++++++++++++++++++++------ Cargo.toml | 25 ++-- playground/src/main.rs | 24 +++- src/all.rs | 4 - src/engine.rs | 28 ++-- src/executor.rs | 53 +++++--- src/graph.rs | 65 +++++++--- src/lib.rs | 3 + src/main.rs | 16 --- src/object.rs | 12 -- src/rod.rs | 20 --- src/store.rs | 194 +++------------------------- src/store/native.rs | 138 ++++++++++++++++++++ src/store/wasm.rs | 14 ++ src/ulid.rs | 57 ++++++++ 16 files changed, 606 insertions(+), 335 deletions(-) delete mode 100644 src/all.rs delete mode 100644 src/main.rs delete mode 100644 src/object.rs delete mode 100644 src/rod.rs create mode 100644 src/store/native.rs create mode 100644 src/store/wasm.rs create mode 100644 src/ulid.rs diff --git a/.gitignore b/.gitignore index c77bb18..5d5e583 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ Cargo.lock # Data generated by running the demo server data/ + +.vscode/settings.json \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 079f1a0..9460327 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "ahash" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" + [[package]] name = "ansi_term" version = "0.12.1" @@ -88,6 +94,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + [[package]] name = "autocfg" version = "1.0.1" @@ -95,10 +107,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] -name = "base64" -version = "0.13.0" +name = "bitflags" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "blocking" @@ -114,6 +126,51 @@ dependencies = [ "once_cell", ] +[[package]] +name = "borsh" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18dda7dc709193c0d86a1a51050a926dc3df1cf262ec46a23a25dba421ea1924" +dependencies = [ + "borsh-derive", + "hashbrown", +] + +[[package]] +name = "borsh-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "684155372435f578c0fa1acd13ebbb182cc19d6b38b64ae7901da4393217d264" +dependencies = [ + "borsh-derive-internal", + "borsh-schema-derive-internal", + "proc-macro-crate", + "proc-macro2", + "syn", +] + +[[package]] +name = "borsh-derive-internal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2102f62f8b6d3edeab871830782285b64cc1830168094db05c8e458f209bc5c3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "borsh-schema-derive-internal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196c978c4c9b0b142d446ef3240690bf5a8a33497074a113ff9a337ccb750483" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "bumpalo" version = "3.7.0" @@ -144,12 +201,24 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ + "js-sys", "libc", "num-integer", "num-traits", + "time", + "wasm-bindgen", "winapi", ] +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + [[package]] name = "concurrent-queue" version = "1.2.2" @@ -165,12 +234,6 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dd4afd79212583ff429b913ad6605242ed7eec277e950b1438f300748f948f4" -[[package]] -name = "endian-type" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" - [[package]] name = "error-code" version = "2.3.0" @@ -196,6 +259,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "futures-core" version = "0.3.17" @@ -236,6 +305,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +dependencies = [ + "ahash", +] + [[package]] name = "hermit-abi" version = "0.1.19" @@ -311,22 +389,13 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" -[[package]] -name = "nibble_vec" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" -dependencies = [ - "smallvec", -] - [[package]] name = "num-integer" version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ - "autocfg", + "autocfg 1.0.1", "num-traits", ] @@ -336,7 +405,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ - "autocfg", + "autocfg 1.0.1", ] [[package]] @@ -383,6 +452,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "proc-macro-crate" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml", +] + [[package]] name = "proc-macro2" version = "1.0.29" @@ -402,14 +480,119 @@ dependencies = [ ] [[package]] -name = "radix_trie" -version = "0.2.1" +name = "rand" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" dependencies = [ - "endian-type", - "nibble_vec", - "serde", + "autocfg 0.1.7", + "libc", + "rand_chacha", + "rand_core 0.4.2", + "rand_hc", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", ] [[package]] @@ -443,20 +626,19 @@ dependencies = [ "async-executor", "async-timer", "async-trait", - "base64", "blocking", + "borsh", "easy-parallel", "futures-lite", "getrandom", "num_cpus", "once_cell", - "radix_trie", "scc", "serde", "serde_json", "thiserror", "tracing", - "uuid", + "ulid", "wee_alloc", ] @@ -579,6 +761,26 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi", + "winapi", +] + +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + [[package]] name = "tracing" version = "0.1.27" @@ -665,21 +867,23 @@ dependencies = [ ] [[package]] -name = "unicode-xid" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" - -[[package]] -name = "uuid" -version = "0.8.2" +name = "ulid" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +checksum = "b7e95a59b292ca0cf9b45be2e52294d1ca6cb24eb11b08ef4376f73f1a00c549" dependencies = [ - "getrandom", + "chrono", + "lazy_static", + "rand", "serde", ] +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + [[package]] name = "waker-fn" version = "1.1.0" @@ -688,9 +892,9 @@ checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" [[package]] name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" +version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" diff --git a/Cargo.toml b/Cargo.toml index 38a1b14..5d640ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,10 +10,14 @@ members = [ "wasmtest", ] +[features] +default = ["borsh"] +json = ["serde", "serde_json", "ulid/serde"] + [dependencies] # Data structures -uuid = { version = "0.8.2", features = ["v4", "serde", "wasm-bindgen"] } -radix_trie = { version = "0.2", features = ["serde"] } +ulid = { version = "0.4.1" } +# radix_trie = { version = "0.2", features = ["serde"] } once_cell = "1.8.0" scc = "0.5.2" thiserror = "1.0.29" @@ -22,20 +26,23 @@ thiserror = "1.0.29" tracing = "0.1.27" # Serialization -serde = { version = "1.0.130", features = ["derive"] } -serde_json = "1.0.67" -base64 = "0.13.0" +serde = { version = "1.0.130", features = ["derive"], optional = true } +serde_json = { version = "1.0.67", optional = true } +borsh = { version = "0.9.1", optional = true } # Async -num_cpus = "1.13.0" -easy-parallel = "3.1.0" -async-executor = "1.4.1" futures-lite = "1.12.0" -blocking = "1.0.2" async-trait = "0.1.51" async-timer = "1.0.0-beta.7" getrandom = { version = "*", features = ["js"] } +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +# Async +easy-parallel = "3.1.0" +num_cpus = "1.13.0" +blocking = "1.0.2" +async-executor = "1.4.1" + [target.'cfg(target_arch = "wasm32")'.dependencies] wee_alloc = "0.4.5" diff --git a/playground/src/main.rs b/playground/src/main.rs index dabe398..a6a072e 100644 --- a/playground/src/main.rs +++ b/playground/src/main.rs @@ -1,7 +1,11 @@ use std::{process, time::Duration}; use tracing as trc; -use rod::engine::Rod; +use rod::{ + engine::Rod, + graph::{Field, Node, Value}, + store::{get_default_store, Store}, +}; use futures_lite::future; @@ -18,7 +22,23 @@ async fn start() -> anyhow::Result<()> { trc::info!("Staring server"); - let engine = Rod::new().await?; + let rod = Rod::new().await?; + + let store = get_default_store().await?; + + store.put("key1", Node::new()).await?; + store + .put( + "key2", + Node::new_with_fields(vec![( + "hello".into(), + Field::new(Value::String("world".into())), + )]), + ) + .await?; + + dbg!(store.get("hello").await)?; + dbg!(store.get("hello2").await)?; // Just prevent the process from exiting let mut interval = async_timer::interval(Duration::from_secs(1)); diff --git a/src/all.rs b/src/all.rs deleted file mode 100644 index d0bbf2f..0000000 --- a/src/all.rs +++ /dev/null @@ -1,4 +0,0 @@ -use std::collections::HashMap; -use crate::object::Object; - -pub type All = HashMap; \ No newline at end of file diff --git a/src/engine.rs b/src/engine.rs index 7bf4383..cee236d 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -1,12 +1,12 @@ use std::{sync::Arc, time::Duration}; use tracing as trc; -use uuid::Uuid; use crate::{ executor, graph::Node, store::{get_default_store, Store, StoreError}, + Ulid, }; /// The Rod engine, responsible for managing connections and performing the various database @@ -29,9 +29,9 @@ const FLUSH_INTERVAL: Duration = Duration::from_secs(2); struct RodInner { /// The list of nodes cached in memory - nodes: scc::HashMap, + nodes: scc::HashMap, /// Nodes that have been modified in memory and need to be flushed to disk - dirty_nodes: scc::HashMap, + dirty_nodes: scc::HashMap, /// The backing data store for this engine store: Box, } @@ -39,9 +39,10 @@ struct RodInner { impl Rod { /// Initialize a new [`Rod`] instance /// - /// TODO: Use an `EngineBuilder` to construct an engine with customized store and peers list + /// TODO: Use an `RodBuilder` to construct an engine with customized store and peers list pub async fn new() -> Result { - trc::trace!("Creating new instance"); + trc::trace!("Creating new Rod instance"); + // Initialize data store let store = Box::new(get_default_store().await?); @@ -55,7 +56,7 @@ impl Rod { // Create Rod instance let instance = Rod { inner }; - // Spawn a job to flush the dirty nodes to disk + // Spawn a job to flush the dirty nodes to disk periodically let instance_ = instance.clone(); executor::spawn(async move { trc::debug!("Staring periodic node flush"); @@ -75,14 +76,7 @@ impl Rod { }); for uuid in dirty_nodes { if let Some(node) = db.nodes.read(&uuid, |_, node| node.clone()) { - if let Err(err) = db - .store - .put_string( - &uuid.to_string(), - &serde_json::to_string_pretty(&node).expect("Serialize node"), - ) - .await - { + if let Err(err) = db.store.put(&uuid.to_string(), node).await { trc::error!(%err, "Could not flush node data to the store. The store may be out-of-date!"); } } @@ -92,10 +86,12 @@ impl Rod { interval.wait().await; } - }).detach(); + }); Ok(instance) } - // pub async fn get(key: &str) -> + // pub async fn get(&self, key: &str) -> Value { + // self.inner.store. + // } } diff --git a/src/executor.rs b/src/executor.rs index 24166dc..3fc297d 100644 --- a/src/executor.rs +++ b/src/executor.rs @@ -1,25 +1,40 @@ -use std::{future::Future, panic, thread}; +pub use implementation::*; -use async_executor::{Executor, Task}; -use futures_lite::future; -use once_cell::sync::Lazy; +#[cfg(not(target_arch = "wasm32"))] +mod implementation { + use async_executor::Executor; + use futures_lite::future; + use once_cell::sync::Lazy; -pub fn spawn(future: impl Future + Send + 'static) -> Task { - static GLOBAL: Lazy> = Lazy::new(|| { - for n in 1..=num_cpus::get() { - thread::Builder::new() - .name(format!("rod-worker-{}", n)) - .spawn(|| loop { - panic::catch_unwind(|| { - future::block_on(GLOBAL.run(future::pending::<()>())) + use std::{future::Future, panic, thread}; + + #[cfg(not(target_arch = "wasm32"))] + pub fn spawn(future: impl Future + Send + 'static) { + static GLOBAL: Lazy> = Lazy::new(|| { + for n in 1..=num_cpus::get() { + thread::Builder::new() + .name(format!("rod-worker-{}", n)) + .spawn(|| loop { + panic::catch_unwind(|| { + future::block_on(GLOBAL.run(future::pending::<()>())) + }) + .ok(); }) - .ok(); - }) - .expect("cannot spawn executor thread"); - } + .expect("cannot spawn executor thread"); + } + + Executor::new() + }); + + GLOBAL.spawn(future).detach() + } +} - Executor::new() - }); +#[cfg(target_arch = "wasm32")] +mod implementation { + use std::future::Future; - GLOBAL.spawn(future) + pub fn spawn(_future: impl Future + Send + 'static) { + todo!(); + } } diff --git a/src/graph.rs b/src/graph.rs index f2af343..13f412a 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -1,30 +1,34 @@ //! Data structures used to build the rod data graph -use std::{ - cmp, - collections::HashMap, - mem, - ops::Deref, - time::{Duration, SystemTime}, -}; +use std::{cmp, collections::HashMap, mem, ops::Deref, time::SystemTime}; +use crate::Ulid; + +#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; -use uuid::Uuid; + +#[cfg(feature = "borsh")] +use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; pub trait LexicalCmp { fn lexical_cmp(&self, other: &Self) -> cmp::Ordering; } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr( + feature = "borsh", + derive(BorshSerialize, BorshDeserialize, BorshSchema) +)] pub struct Node { - pub id: Uuid, + pub id: Ulid, pub fields: HashMap, } impl Default for Node { fn default() -> Self { Self { - id: Uuid::new_v4(), + id: Ulid::new(), fields: Default::default(), } } @@ -34,20 +38,32 @@ impl Node { pub fn new() -> Self { Self::default() } + + pub fn new_with_fields(fields: Vec<(String, Field)>) -> Self { + Self { + id: Ulid::new(), + fields: fields.into_iter().collect(), + } + } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr( + feature = "borsh", + derive(BorshSerialize, BorshDeserialize, BorshSchema) +)] pub struct Field { /// Time that this field value was updated as relative to the /// [`UNIX_EPOCH`][std::time::SystemTime::UNIX_EPOCH] - updated_at: Duration, + updated_at: f64, value: Value, } /// If an update comes in that is more than this amount of time in the future, we will assume that /// the node that sent the update is lying and trying to make it's update take precedence over the /// current value of the field. -const FUTURE_UPDATE_THREASHOLD: Duration = Duration::from_secs(60 * 10); +const FUTURE_UPDATE_THREASHOLD: f64 = 600.0; impl Field { /// Merge the new value into this field, using the [HAM] merge conflict resolution strategy @@ -56,7 +72,8 @@ impl Field { pub fn merge_with(&mut self, field: &Field) { let current_time = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) - .expect("TODO: system time"); + .expect("TODO: system time") + .as_secs_f64(); // If the new field has the same timestamp if field.updated_at == self.updated_at { @@ -96,7 +113,8 @@ impl Field { Self { updated_at: SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) - .expect("Could not get system time"), + .expect("Could not get system time") + .as_secs_f64(), value, } } @@ -105,7 +123,7 @@ impl Field { &self.value } - pub fn state(&self) -> &Duration { + pub fn state(&self) -> &f64 { &self.updated_at } } @@ -118,7 +136,12 @@ impl Deref for Field { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr( + feature = "borsh", + derive(BorshSerialize, BorshDeserialize, BorshSchema) +)] pub enum Value { None, Bool(bool), @@ -126,7 +149,7 @@ pub enum Value { Float(f64), String(String), Binary(Vec), - Node(Uuid), + Node(Ulid), } impl From for Value { @@ -177,8 +200,8 @@ impl From> for Value { } } -impl From for Value { - fn from(u: Uuid) -> Self { +impl From for Value { + fn from(u: Ulid) -> Self { Self::Node(u) } } diff --git a/src/lib.rs b/src/lib.rs index d9d8acb..41483f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,9 @@ pub mod engine; pub mod graph; pub mod protocol; pub mod store; +mod ulid; + +pub use self::ulid::Ulid; pub(crate) mod executor; diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index 1e786ad..0000000 --- a/src/main.rs +++ /dev/null @@ -1,16 +0,0 @@ -pub mod object; -pub mod all; -pub mod rod; - -use crate::object::Value; -use crate::object::Object; -use crate::all::All; -use crate::rod::Rod; - -fn main() { - let mut all = All::new(); - Rod::put(&mut all, "todo", "milk", Value::Text("1gallon whole".to_string())); - for id in all.keys() { - println!("{}: {:?}", id, all.get(id)); - } -} \ No newline at end of file diff --git a/src/object.rs b/src/object.rs deleted file mode 100644 index 744ba2d..0000000 --- a/src/object.rs +++ /dev/null @@ -1,12 +0,0 @@ -use std::collections::HashMap; - -pub type Object = HashMap; - -#[derive(Debug)] -pub enum Value { - Null(Option), - Bit(bool), - Number(f32), - Text(String), - Link(Object) -} \ No newline at end of file diff --git a/src/rod.rs b/src/rod.rs deleted file mode 100644 index bb49146..0000000 --- a/src/rod.rs +++ /dev/null @@ -1,20 +0,0 @@ -use crate::object::Object; -use crate::object::Value; -use crate::all::All; - -pub struct Rod { - all: All -} -impl Rod { - pub fn new() -> Rod { - let mut all = All::new(); - Rod {all: all} - } - pub fn get(&mut self, id: &str){ - } - pub fn put(all: &mut All, id: &str, has: &str, value: Value){ - let mut obj = Object::new(); - obj.insert(id.to_string(), value); - all.insert(has.to_string(), obj); - } -} \ No newline at end of file diff --git a/src/store.rs b/src/store.rs index 64107e5..c6d1cbe 100644 --- a/src/store.rs +++ b/src/store.rs @@ -1,10 +1,10 @@ -use std::{ - fs::{self, OpenOptions}, - io::{Read, Write}, - path::{Path, PathBuf}, -}; +use crate::graph::Node; -use blocking::unblock; +#[cfg(not(target_arch = "wasm32"))] +mod native; + +#[cfg(target_arch = "wasm32")] +mod wasm; /// Get the default store implementation for the current platform /// @@ -15,41 +15,14 @@ use blocking::unblock; /// implemented. pub async fn get_default_store() -> Result { #[cfg(not(target_arch = "wasm32"))] - let store = SimpleFsStore::new(Path::new("./data")).await; + let store = native::get_default_store().await; #[cfg(target_arch = "wasm32")] - let store = Ok(IndexedDbStore); + let store = wasm::get_default_store().await; store } -#[cfg(target_arch = "wasm32")] -struct IndexedDbStore; - -#[cfg(target_arch = "wasm32")] -#[async_trait::async_trait] -impl Store for IndexedDbStore { - async fn get_string(&self, key: &str) -> Result, StoreError> { - todo!() - } - - async fn get_binary(&self, key: &str) -> Result>, StoreError> { - todo!() - } - - async fn put_string(&self, key: &str, data: &str) -> Result<(), StoreError> { - todo!() - } - - async fn put_binary(&self, key: &str, data: &[u8]) -> Result<(), StoreError> { - todo!() - } - - async fn delete(&self, key: &str) -> Result<(), StoreError> { - todo!() - } -} - /// A simple, raw data storage interface /// /// [`Store`] is designed to be implemented over any persistant storage interface such as the @@ -61,34 +34,18 @@ impl Store for IndexedDbStore { /// Because some data stores such as browser local storage can only store string data, separate /// functions are used for getting and setting string and binary data. This allows the /// implementation to choose to base64 encode/decode binary values when writing to a string-only -/// data store. +/// data store, or to just write the raw bytes directly if binary storage is supported. /// /// Calling [`Store::get_binary`] on a key on a string value or calling [`Store::get_string`] on a -/// binary value may either fail with an error, or simply interpert the contents of the key as the -/// other value type depending on the implementation. In-general you should only call the "get" -/// function that corresponds with the value at the given key to avoid unexpected behavior. +/// binary value should fail with an error. #[async_trait::async_trait] pub trait Store { - /// Get a string value from the store - /// - /// # Errors - /// - /// - This function **will** error if there is a problem accessing the store - /// - This funciton **may** error or return an unexpected value if you try to read a key that - /// has a binary value currently set. - async fn get_string(&self, key: &str) -> Result, StoreError>; - /// Get a binary value from the store - /// - /// # Errors - /// - /// - This function **will** error if there is a problem accessing the store - /// - This funciton **may** error or return an unexpected value if you try to read a key that - /// has a string value currently set. - async fn get_binary(&self, key: &str) -> Result>, StoreError>; - /// Store a string value in the store - async fn put_string(&self, key: &str, data: &str) -> Result<(), StoreError>; - /// Store a binary value in the store - async fn put_binary(&self, key: &str, data: &[u8]) -> Result<(), StoreError>; + /// Get a value from the store + async fn get(&self, key: &str) -> Result, StoreError>; + + // Put a value in the store + async fn put(&self, key: &str, value: Node) -> Result<(), StoreError>; + /// Delete a key async fn delete(&self, key: &str) -> Result<(), StoreError>; } @@ -102,123 +59,10 @@ pub enum StoreError { /// Attempted to read a string value as binary #[error("Attempted to read string data as binary")] ReadStringAsBinary, + /// Found unrecognized data in storage medium + #[error("Found unrecognized data in storage medium")] + UnrecognizedData, /// Other, implementation-specific error #[error("Error: {0}")] Other(#[from] Box), } - -/// Ultra-simple filesystem [`Store`] implementation that uses a separate file for each string key -/// -/// The names of each file will be the base64-encoded key and the value will be the string or binary -/// data associated to the key. -pub struct SimpleFsStore { - /// The root directory for the filesystem store - root_dir: PathBuf, -} - -impl SimpleFsStore { - /// Create a new [`FsStore`] that puts files in the specified `root_dir` - pub async fn new(root_dir: &Path) -> Result { - let root_dir = root_dir.to_owned(); - - unblock(move || { - fs::create_dir_all(&root_dir).map_err(box_error)?; - - Ok(Self { - root_dir: root_dir.to_owned(), - }) - }) - .await - } -} - -#[async_trait::async_trait] -impl Store for SimpleFsStore { - async fn get_binary(&self, key: &str) -> Result>, StoreError> { - // Get the path to the file - let file_path = self.root_dir.join(key); - - // Perform blocking operations on a thread pool - unblock(move || { - // Check if the file exists - if !file_path.exists() { - return Ok(None); - } - - // Open the file - let mut file = OpenOptions::new() - .read(true) - .open(file_path) - .map_err(box_error)?; - - // Read the file into buffer - let mut buf = Vec::new(); - file.read_to_end(&mut buf).map_err(box_error)?; - - // And return the buffer - Ok(Some(buf)) - }) - .await - } - - async fn put_binary(&self, key: &str, data: &[u8]) -> Result<(), StoreError> { - // Get the path to the file - let file_path = self.root_dir.join(key); - // Clone the data - let data = data.to_vec(); - - // Perform blocking operations on a thread pool - unblock(move || { - // Open the file - let mut file = OpenOptions::new() - .write(true) - .truncate(true) - .create(true) - .open(file_path) - .map_err(box_error)?; - - // Write the data to the file - Ok(file.write_all(&data).map_err(box_error)?) - }) - .await - } - - async fn get_string(&self, key: &str) -> Result, StoreError> { - // Get the binary data if it exists - if let Some(binary) = self.get_binary(key).await? { - // Parse the binary data as a UTF-8 String - Ok(Some(String::from_utf8(binary).map_err(box_error)?)) - - // The key doesn't exist - } else { - Ok(None) - } - } - - async fn put_string(&self, key: &str, data: &str) -> Result<(), StoreError> { - // Store the string as UTF-8 bytes - self.put_binary(key, data.as_bytes()).await - } - - async fn delete(&self, key: &str) -> Result<(), StoreError> { - // Get the path to the file - let file_path = self.root_dir.join(key); - - // Perform blocking operation on a thread pool - unblock(move || { - // Delete the file - fs::remove_file(file_path).map_err(box_error)?; - - // Write the data to the file - Ok(()) - }) - .await - } -} - -/// Helper to box the an error -fn box_error( - e: impl std::error::Error + Sync + Send + 'static, -) -> Box { - Box::new(e) -} diff --git a/src/store/native.rs b/src/store/native.rs new file mode 100644 index 0000000..8760749 --- /dev/null +++ b/src/store/native.rs @@ -0,0 +1,138 @@ +use std::{ + fs::{self, OpenOptions}, + io::{Read, Write}, + path::{Path, PathBuf}, +}; + +use blocking::unblock; + +use crate::graph::Node; + +use super::{Store, StoreError}; + +pub async fn get_default_store() -> Result { + SimpleFsStore::new(Path::new("./data")).await +} + +// TODO: SledDB filesystem store + +/// Ultra-simple filesystem [`Store`] implementation that uses a separate file for each string key +/// +/// The names of each file will be the base64-encoded key and the value will be the string or binary +/// data associated to the key. +pub struct SimpleFsStore { + /// The root directory for the filesystem store + root_dir: PathBuf, +} + +impl SimpleFsStore { + /// Create a new [`FsStore`] that puts files in the specified `root_dir` + pub async fn new(root_dir: &Path) -> Result { + #[cfg(not(feature = "borsh"))] + compile_error!("`borsh` feature required to use `SimpleFsStore`"); + + let root_dir = root_dir.to_owned(); + + unblock(move || { + fs::create_dir_all(&root_dir).map_err(box_error)?; + + Ok(Self { + root_dir: root_dir.to_owned(), + }) + }) + .await + } +} + +async fn load_file(file_path: PathBuf) -> Result>, StoreError> { + // Perform blocking operations on a thread pool + unblock(move || { + // Check if the file exists + if !file_path.exists() { + return Ok(None); + } + + // Open the file + let mut file = OpenOptions::new() + .read(true) + .open(file_path) + .map_err(box_error)?; + + // Read the file into buffer + let mut buf = Vec::new(); + file.read_to_end(&mut buf).map_err(box_error)?; + + // And return the buffer + Ok(Some(buf)) + }) + .await +} + +async fn write_file(file_path: PathBuf, data: Vec) -> Result<(), StoreError> { + // Perform blocking operations on a thread pool + unblock(move || { + // Open the file + let mut file = OpenOptions::new() + .write(true) + .truncate(true) + .create(true) + .open(file_path) + .map_err(box_error)?; + + // Write the data to the file + Ok(file.write_all(&data).map_err(box_error)?) + }) + .await +} + +#[async_trait::async_trait] +impl Store for SimpleFsStore { + async fn get(&self, key: &str) -> Result, StoreError> { + use borsh::BorshDeserialize; + + // Get the path to the file + let file_path = self.root_dir.join(key); + + if let Some(buf) = load_file(file_path).await? { + let node = Node::deserialize(&mut buf.as_slice()).map_err(box_error)?; + + Ok(Some(node)) + } else { + Ok(None) + } + } + + async fn put(&self, key: &str, data: Node) -> Result<(), StoreError> { + use borsh::BorshSerialize; + + // Get the path to the file + let file_path = self.root_dir.join(key); + // Clone the data + let data = data.try_to_vec().expect("Impossible IO error"); + + // Write the file + write_file(file_path, data).await + } + + async fn delete(&self, key: &str) -> Result<(), StoreError> { + // Get the path to the file + let file_path = self.root_dir.join(key); + + // Perform blocking operation on a thread pool + unblock(move || { + // Delete the file + fs::remove_file(file_path).map_err(box_error)?; + + // Write the data to the file + Ok(()) + }) + .await + } +} + +/// Helper to box the an error +fn box_error( + e: impl std::error::Error + Sync + Send + 'static, +) -> Box { + Box::new(e) +} diff --git a/src/store/wasm.rs b/src/store/wasm.rs new file mode 100644 index 0000000..64b170d --- /dev/null +++ b/src/store/wasm.rs @@ -0,0 +1,14 @@ +use super::Store; + +pub async fn get_default_store() -> Result { + let store = Ok(IndexedDbStore); +} + +pub struct IndexedDbStore; + +#[async_trait::async_trait] +impl Store for IndexedDbStore { + async fn get(&self, key: &str) -> Result, StoreError> {} + async fn put(&self, key: &str, value: Value) -> Result<(), StoreError> {} + async fn delete(&self, key: &str) -> Result<(), StoreError> {} +} diff --git a/src/ulid.rs b/src/ulid.rs new file mode 100644 index 0000000..9f7c25e --- /dev/null +++ b/src/ulid.rs @@ -0,0 +1,57 @@ +use ulid::Ulid as ActualUlid; + +/// Wrapper around UUID that implements the borsh traits +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Ulid(ActualUlid); + +impl Ulid { + pub fn new() -> Self { + Ulid(ActualUlid::new()) + } +} + +impl std::ops::Deref for Ulid { + type Target = ActualUlid; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::DerefMut for Ulid { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +#[cfg(feature = "borsh")] +impl borsh::BorshSerialize for Ulid { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + let id: u128 = self.0.into(); + borsh::BorshSerialize::serialize(&id, writer) + } +} + +#[cfg(feature = "borsh")] +impl borsh::BorshDeserialize for Ulid { + fn deserialize(buf: &mut &[u8]) -> std::io::Result { + let id: u128 = borsh::BorshDeserialize::deserialize(buf)?; + Ok(Ulid(ActualUlid::from(id))) + } +} + +#[cfg(feature = "borsh")] +impl borsh::BorshSchema for Ulid { + fn add_definitions_recursively( + definitions: &mut std::collections::HashMap< + borsh::schema::Declaration, + borsh::schema::Definition, + >, + ) { + ::add_definitions_recursively(definitions) + } + + fn declaration() -> borsh::schema::Declaration { + ::declaration() + } +} From 7e401aab016b61c7f195bec6c1ecabc6f0c4a937 Mon Sep 17 00:00:00 2001 From: Zicklag Date: Thu, 23 Sep 2021 20:12:56 -0500 Subject: [PATCH 04/12] Custom Serializer Matching GUN's JSON Structure --- Cargo.lock | 7 + Cargo.toml | 3 +- playground/Cargo.toml | 2 +- playground/src/main.rs | 27 ++- src/engine.rs | 42 +---- src/executor.rs | 2 + src/graph.rs | 265 +++++---------------------- src/graph/repr.rs | 400 +++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 4 +- src/merge.rs | 124 +++++++++++++ src/store.rs | 15 +- src/store/native.rs | 21 ++- src/ulid.rs | 57 ------ 13 files changed, 621 insertions(+), 348 deletions(-) create mode 100644 src/graph/repr.rs create mode 100644 src/merge.rs delete mode 100644 src/ulid.rs diff --git a/Cargo.lock b/Cargo.lock index 9460327..5708f86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -106,6 +106,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bitflags" version = "1.3.2" @@ -626,6 +632,7 @@ dependencies = [ "async-executor", "async-timer", "async-trait", + "base64", "blocking", "borsh", "easy-parallel", diff --git a/Cargo.toml b/Cargo.toml index 5d640ce..449a055 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ [features] default = ["borsh"] -json = ["serde", "serde_json", "ulid/serde"] +json = ["serde", "serde_json", "ulid/serde", "base64"] [dependencies] # Data structures @@ -29,6 +29,7 @@ tracing = "0.1.27" serde = { version = "1.0.130", features = ["derive"], optional = true } serde_json = { version = "1.0.67", optional = true } borsh = { version = "0.9.1", optional = true } +base64 = { version = "0.13.0", optional = true } # Async futures-lite = "1.12.0" diff --git a/playground/Cargo.toml b/playground/Cargo.toml index 6c9b016..18c92b5 100644 --- a/playground/Cargo.toml +++ b/playground/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2018" [dependencies] -rod = { path = "../"} +rod = { path = "../", features = ["json"] } async-executor = "1.4.1" futures-lite = "1.12.0" serde_json = "1.0.67" diff --git a/playground/src/main.rs b/playground/src/main.rs index a6a072e..4e02f39 100644 --- a/playground/src/main.rs +++ b/playground/src/main.rs @@ -3,8 +3,9 @@ use tracing as trc; use rod::{ engine::Rod, - graph::{Field, Node, Value}, + graph::{repr::repr_json::JsonNode, Field, Node, Value}, store::{get_default_store, Store}, + Ulid, }; use futures_lite::future; @@ -22,7 +23,7 @@ async fn start() -> anyhow::Result<()> { trc::info!("Staring server"); - let rod = Rod::new().await?; + let _rod = Rod::new().await?; let store = get_default_store().await?; @@ -30,15 +31,25 @@ async fn start() -> anyhow::Result<()> { store .put( "key2", - Node::new_with_fields(vec![( - "hello".into(), - Field::new(Value::String("world".into())), - )]), + Node::new_with_fields(vec![ + ("hello".into(), Field::new(Value::String("world".into()))), + ( + "someJunk".into(), + Field::new(Value::Binary(vec![1, 2, 3, 4])), + ), + ("age".into(), Field::new(Value::Float(30.0))), + ("nothing".into(), Field::new(Value::None)), + ("anotherNode".into(), Field::new(Value::Node(Ulid::new()))), + ]), ) .await?; - dbg!(store.get("hello").await)?; - dbg!(store.get("hello2").await)?; + let node2 = store.get("key2").await?.unwrap(); + let json = serde_json::to_string_pretty(&JsonNode::from(node2))?; + println!("{}", json); + + let parsed: JsonNode = serde_json::from_str(&json).unwrap(); + println!("{:#?}", Node::from(parsed)); // Just prevent the process from exiting let mut interval = async_timer::interval(Duration::from_secs(1)); diff --git a/src/engine.rs b/src/engine.rs index cee236d..6a0af81 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -1,12 +1,12 @@ use std::{sync::Arc, time::Duration}; use tracing as trc; +use ulid::Ulid; use crate::{ executor, graph::Node, store::{get_default_store, Store, StoreError}, - Ulid, }; /// The Rod engine, responsible for managing connections and performing the various database @@ -22,16 +22,9 @@ pub struct Rod { inner: Arc, } -/// The interval at which modifications to the database will be flushed to disk -/// -/// TODO: Make this configuable per [`Rod`] instance -const FLUSH_INTERVAL: Duration = Duration::from_secs(2); - struct RodInner { /// The list of nodes cached in memory nodes: scc::HashMap, - /// Nodes that have been modified in memory and need to be flushed to disk - dirty_nodes: scc::HashMap, /// The backing data store for this engine store: Box, } @@ -49,45 +42,12 @@ impl Rod { // Create clonable inner data let inner = Arc::new(RodInner { nodes: Default::default(), - dirty_nodes: Default::default(), store, }); // Create Rod instance let instance = Rod { inner }; - // Spawn a job to flush the dirty nodes to disk periodically - let instance_ = instance.clone(); - executor::spawn(async move { - trc::debug!("Staring periodic node flush"); - let db = instance_.inner; - - // Loop on and interval - let mut interval = async_timer::interval(FLUSH_INTERVAL); - loop { - trc::debug!("Flushing dirty nodes to disk"); - - // Flush all dirty nodes to disk - let count = db.dirty_nodes.len(); - let mut dirty_nodes = Vec::with_capacity(count); - db.dirty_nodes.retain(|uuid, _| { - dirty_nodes.push(uuid.clone()); - true - }); - for uuid in dirty_nodes { - if let Some(node) = db.nodes.read(&uuid, |_, node| node.clone()) { - if let Err(err) = db.store.put(&uuid.to_string(), node).await { - trc::error!(%err, "Could not flush node data to the store. The store may be out-of-date!"); - } - } - } - - trc::debug!(%count, " Flushed nodes to disk"); - - interval.wait().await; - } - }); - Ok(instance) } diff --git a/src/executor.rs b/src/executor.rs index 3fc297d..f9ef8e1 100644 --- a/src/executor.rs +++ b/src/executor.rs @@ -1,3 +1,5 @@ +//! Implementation of async executor functions such as [`spawn`] + pub use implementation::*; #[cfg(not(target_arch = "wasm32"))] diff --git a/src/graph.rs b/src/graph.rs index 13f412a..f5f9770 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -1,256 +1,83 @@ //! Data structures used to build the rod data graph -use std::{cmp, collections::HashMap, mem, ops::Deref, time::SystemTime}; +use std::{collections::HashMap, ops::Deref}; -use crate::Ulid; +use ulid::Ulid; -#[cfg(feature = "serde")] -use serde::{Deserialize, Serialize}; - -#[cfg(feature = "borsh")] -use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; - -pub trait LexicalCmp { - fn lexical_cmp(&self, other: &Self) -> cmp::Ordering; -} +pub mod repr; +/// [`Node`] is the core data structure in the data graph +/// +/// A grpah is made up of a collection of nodes #[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr( - feature = "borsh", - derive(BorshSerialize, BorshDeserialize, BorshSchema) -)] pub struct Node { + /// The node's universally unique identifier pub id: Ulid, + /// The fields in the node pub fields: HashMap, } -impl Default for Node { - fn default() -> Self { - Self { - id: Ulid::new(), - fields: Default::default(), - } - } -} - -impl Node { - pub fn new() -> Self { - Self::default() - } - - pub fn new_with_fields(fields: Vec<(String, Field)>) -> Self { - Self { - id: Ulid::new(), - fields: fields.into_iter().collect(), - } - } -} - +/// A [`Field`] is a named item in a node +/// +/// A field encompases the last time that the field was modified, and the value of the field #[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr( - feature = "borsh", - derive(BorshSerialize, BorshDeserialize, BorshSchema) -)] pub struct Field { - /// Time that this field value was updated as relative to the + /// The time in seconds that this field value was updated as relative to the /// [`UNIX_EPOCH`][std::time::SystemTime::UNIX_EPOCH] - updated_at: f64, - value: Value, -} - -/// If an update comes in that is more than this amount of time in the future, we will assume that -/// the node that sent the update is lying and trying to make it's update take precedence over the -/// current value of the field. -const FUTURE_UPDATE_THREASHOLD: f64 = 600.0; - -impl Field { - /// Merge the new value into this field, using the [HAM] merge conflict resolution strategy - /// - /// [HAM]: https://github.com/amark/gun/wiki/Conflict-Resolution-with-Guns - pub fn merge_with(&mut self, field: &Field) { - let current_time = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .expect("TODO: system time") - .as_secs_f64(); - - // If the new field has the same timestamp - if field.updated_at == self.updated_at { - match self.lexical_cmp(field) { - // Totally equal, do nothing - cmp::Ordering::Equal => return, - // Keep our value, do nothing - cmp::Ordering::Less => return, - // Keep the new value - cmp::Ordering::Greater => self.value = field.value.clone(), - } - - // If the other field is and older update than the one we have, just ignore it - } else if field.updated_at < self.updated_at { - return; - - // If the other field is in the future - } else if field.updated_at > current_time { - // If the field is too far in the future, ignore it - if field.updated_at - current_time > FUTURE_UPDATE_THREASHOLD { - return; - } - - // Wait to apply this update until later - unimplemented!( - "Use async logic to apply this update once our system clock \ - reaches the future time" - ); - } else { - unreachable!() - } - } -} - -impl Field { - pub fn new(value: Value) -> Self { - Self { - updated_at: SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .expect("Could not get system time") - .as_secs_f64(), - value, - } - } - - pub fn value(&self) -> &Value { - &self.value - } - - pub fn state(&self) -> &f64 { - &self.updated_at - } -} - -impl Deref for Field { - type Target = Value; - - fn deref(&self) -> &Self::Target { - &self.value - } + pub updated_at: f64, + /// The value of the field + pub value: Value, } +/// A value represents the different data types that a field value can take #[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr( - feature = "borsh", - derive(BorshSerialize, BorshDeserialize, BorshSchema) -)] pub enum Value { + /// An empty value None, + /// A boolean value Bool(bool), + /// A signed integer value Int(i64), + /// A floating point value Float(f64), + /// A string value String(String), + /// A binary data value Binary(Vec), + /// A reference to the unique ID of another node Node(Ulid), } -impl From for Value { - fn from(s: String) -> Self { - Self::String(s) - } -} - -impl From<&str> for Value { - fn from(s: &str) -> Self { - Self::String(s.to_string()) - } -} - -impl From for Value { - fn from(b: bool) -> Self { - Self::Bool(b) - } -} - -impl From for Value { - fn from(i: i64) -> Self { - Self::Int(i) - } -} - -impl From for Value { - fn from(i: i32) -> Self { - Self::Int(i as i64) - } -} - -impl From for Value { - fn from(f: f32) -> Self { - Self::Float(f as f64) - } -} +mod impls { + use super::*; -impl From for Value { - fn from(f: f64) -> Self { - Self::Float(f) + impl Default for Node { + fn default() -> Self { + Self { + id: Ulid::new(), + fields: Default::default(), + } + } } -} -impl From> for Value { - fn from(b: Vec) -> Self { - Self::Binary(b) - } -} + impl Node { + pub fn new() -> Self { + Self::default() + } -impl From for Value { - fn from(u: Ulid) -> Self { - Self::Node(u) + pub fn new_with_fields(fields: Vec<(String, Field)>) -> Self { + Self { + id: Ulid::new(), + fields: fields.into_iter().collect(), + } + } } -} -impl LexicalCmp for Value { - fn lexical_cmp(&self, other: &Self) -> cmp::Ordering { - use Value::*; - if mem::discriminant(self) == mem::discriminant(other) { - match (self, other) { - (Bool(x), Bool(y)) => match (x, y) { - (true, true) => cmp::Ordering::Equal, - (true, false) => cmp::Ordering::Less, - (false, true) => cmp::Ordering::Greater, - (false, false) => cmp::Ordering::Equal, - }, - (Int(x), Int(y)) => x.cmp(y), - (Float(x), Float(y)) => x.partial_cmp(y).unwrap_or(cmp::Ordering::Less), - (String(x), String(y)) => x.cmp(y), - (Binary(x), Binary(y)) => x.cmp(y), - (Node(x), Node(y)) => x.cmp(y), - _ => unreachable!(), - } - } else { - let self_rank = match self { - None => 0, - Bool(_) => 1, - Int(_) => 2, - Float(_) => 3, - String(_) => 4, - Binary(_) => 5, - Node(_) => 6, - }; - let other_rank = match other { - None => 0, - Bool(_) => 1, - Int(_) => 2, - Float(_) => 3, - String(_) => 4, - Binary(_) => 5, - Node(_) => 6, - }; + impl Deref for Field { + type Target = Value; - if self_rank < other_rank { - cmp::Ordering::Less - } else if self_rank > other_rank { - cmp::Ordering::Greater - } else { - unreachable!() - } + fn deref(&self) -> &Self::Target { + &self.value } } } diff --git a/src/graph/repr.rs b/src/graph/repr.rs new file mode 100644 index 0000000..983b85b --- /dev/null +++ b/src/graph/repr.rs @@ -0,0 +1,400 @@ +//! Representations of the graph data structures used for serialization/deserialization + +use super::*; + +/// [borsh] representation +/// +/// [borsh]: https://github.com/near/borsh +#[cfg(feature = "borsh")] +pub mod repr_borsh { + use borsh::{BorshDeserialize, BorshSerialize}; + + use std::collections::HashMap; + + use super::*; + + #[derive(BorshDeserialize, BorshSerialize)] + pub struct BorshNode { + pub id: u128, + pub fields: HashMap, + } + + #[derive(BorshDeserialize, BorshSerialize)] + pub struct BorshField { + pub updated_at: f64, + pub value: BorshValue, + } + + #[derive(BorshDeserialize, BorshSerialize)] + pub enum BorshValue { + None, + Bool(bool), + Int(i64), + Float(f64), + String(String), + Binary(Vec), + Node(u128), + } + + mod to_borsh { + use super::*; + + impl From for BorshNode { + fn from(node: Node) -> Self { + Self { + id: node.id.into(), + fields: node + .fields + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(), + } + } + } + impl From for BorshField { + fn from(field: Field) -> Self { + Self { + updated_at: field.updated_at, + value: field.value.into(), + } + } + } + impl From for BorshValue { + fn from(value: Value) -> Self { + match value { + Value::None => BorshValue::None, + Value::Bool(b) => BorshValue::Bool(b), + Value::Int(i) => BorshValue::Int(i), + Value::Float(f) => BorshValue::Float(f), + Value::String(s) => BorshValue::String(s), + Value::Binary(b) => BorshValue::Binary(b), + Value::Node(n) => BorshValue::Node(n.into()), + } + } + } + } + + mod from_borsh { + use super::*; + + impl From for Node { + fn from(node: BorshNode) -> Self { + Self { + id: node.id.into(), + fields: node + .fields + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(), + } + } + } + impl From for Field { + fn from(field: BorshField) -> Self { + Self { + updated_at: field.updated_at, + value: field.value.into(), + } + } + } + impl From for Value { + fn from(value: BorshValue) -> Self { + match value { + BorshValue::None => Value::None, + BorshValue::Bool(b) => Value::Bool(b), + BorshValue::Int(i) => Value::Int(i), + BorshValue::Float(f) => Value::Float(f), + BorshValue::String(s) => Value::String(s), + BorshValue::Binary(b) => Value::Binary(b), + BorshValue::Node(n) => Value::Node(n.into()), + } + } + } + } +} + +#[cfg(feature = "json")] +pub mod repr_json { + use serde::{de::Visitor, Deserialize, Deserializer, Serialize}; + + use std::{collections::HashMap, convert::TryFrom}; + + use super::*; + + #[derive(Deserialize, Serialize)] + pub struct JsonNode { + #[serde(rename = "_")] + pub meta: JsonNodeMeta, + #[serde(flatten)] + pub fields: HashMap, + } + + #[derive(Deserialize, Serialize)] + pub struct JsonNodeMeta { + #[serde(rename = "#")] + id: Ulid, + #[serde(rename = ">")] + field_states: HashMap, + } + + pub enum JsonValue { + None, + Bool(bool), + Int(i64), + Float(f64), + String(String), + Binary { data: BinaryData }, + Node { id: Ulid }, + } + + #[derive(Serialize, Deserialize, Clone)] + #[serde(try_from = "String", into = "String")] + pub struct BinaryData(Vec); + + impl Into for BinaryData { + fn into(self) -> String { + base64::encode(self.0) + } + } + + impl TryFrom for BinaryData { + type Error = &'static str; + + fn try_from(value: String) -> Result { + base64::decode(value) + .map(|x| BinaryData(x)) + .map_err(|_| "String is not valid base64") + } + } + + mod to_json { + use super::*; + + impl From for JsonNode { + fn from(node: Node) -> Self { + let mut field_states = HashMap::with_capacity(node.fields.len()); + let mut fields = HashMap::with_capacity(node.fields.len()); + for (k, v) in node.fields { + field_states.insert(k.clone(), v.updated_at); + fields.insert(k, v.value.into()); + } + Self { + meta: JsonNodeMeta { + id: node.id, + field_states, + }, + fields, + } + } + } + + impl From for JsonValue { + fn from(value: Value) -> Self { + match value { + Value::None => JsonValue::None, + Value::Bool(b) => JsonValue::Bool(b), + Value::Int(i) => JsonValue::Int(i), + Value::Float(f) => JsonValue::Float(f), + Value::String(s) => JsonValue::String(s), + Value::Binary(b) => JsonValue::Binary { + data: BinaryData(b), + }, + Value::Node(n) => JsonValue::Node { id: n.into() }, + } + } + } + } + + mod from_json { + use super::*; + + impl From for Node { + fn from(node: JsonNode) -> Self { + let JsonNode { mut meta, fields } = node; + Self { + id: meta.id.into(), + fields: fields + .into_iter() + .map(|(k, v)| { + let field = Field { + updated_at: meta.field_states.remove(&k).unwrap(), + value: v.into(), + }; + (k, field) + }) + .collect(), + } + } + } + impl From for Value { + fn from(value: JsonValue) -> Self { + match value { + JsonValue::None => Value::None, + JsonValue::Bool(b) => Value::Bool(b), + JsonValue::Int(i) => Value::Int(i), + JsonValue::Float(f) => Value::Float(f), + JsonValue::String(s) => Value::String(s), + JsonValue::Binary { data } => Value::Binary(data.0), + JsonValue::Node { id } => Value::Node(id.into()), + } + } + } + } + + mod serde_impls { + use serde::ser::SerializeMap; + + use super::*; + + impl Serialize for JsonValue { + fn serialize(&self, s: S) -> Result + where + S: serde::Serializer, + { + match self { + JsonValue::None => s.serialize_none(), + JsonValue::Bool(b) => s.serialize_bool(*b), + JsonValue::Int(i) => s.serialize_i64(*i), + JsonValue::Float(f) => s.serialize_f64(*f), + JsonValue::String(string) => s.serialize_str(string), + JsonValue::Binary { data } => { + s.serialize_str(&format!("$base64${}", base64::encode(data.0.clone()))) + } + JsonValue::Node { id } => { + let mut map = s.serialize_map(Some(1))?; + map.serialize_entry("#", &id.to_string())?; + map.end() + } + } + } + } + + struct JsonValueVisitor; + + macro_rules! visit_int { + ($fn:ident, $int:ident) => { + fn $fn(self, v: $int) -> Result + where + E: serde::de::Error, + { + Ok(JsonValue::Int(v as i64)) + } + }; + } + macro_rules! visit_float { + ($fn:ident, $float:ident) => { + fn $fn(self, v: $float) -> Result + where + E: serde::de::Error, + { + Ok(JsonValue::Float(v as f64)) + } + }; + } + + impl<'de> Visitor<'de> for JsonValueVisitor { + type Value = JsonValue; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str( + "null, a boolean, an integer, a float, a string, base64 encoded binary data as a \ + string starting with `$base64$`, or a map with a single field `#` set to a uuid", + ) + } + + fn visit_bool(self, v: bool) -> Result + where + E: serde::de::Error, + { + Ok(JsonValue::Bool(v)) + } + + visit_int!(visit_i64, i64); + visit_int!(visit_i32, i32); + visit_int!(visit_i16, i16); + visit_int!(visit_i8, i8); + visit_int!(visit_u64, u64); + visit_int!(visit_u32, u32); + visit_int!(visit_u16, u16); + visit_int!(visit_u8, u8); + visit_float!(visit_f32, f32); + visit_float!(visit_f64, f64); + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + let base64_prefix = "$base64$"; + if v.starts_with(base64_prefix) { + let data_base64 = v.strip_prefix(base64_prefix).unwrap_or(""); + + let data = base64::decode(data_base64).map_err(|_| { + serde::de::Error::invalid_value( + serde::de::Unexpected::Str("$base64$[invalid base64 data]"), + &"valid base64 encoded data", + ) + })?; + + Ok(JsonValue::Binary { + data: BinaryData(data), + }) + } else { + Ok(JsonValue::String(v.to_owned())) + } + } + + fn visit_string(self, v: String) -> Result + where + E: serde::de::Error, + { + self.visit_str(&v) + } + + fn visit_none(self) -> Result + where + E: serde::de::Error, + { + Ok(JsonValue::None) + } + + fn visit_unit(self) -> Result + where + E: serde::de::Error, + { + Ok(JsonValue::None) + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + if let Some(key) = map.next_key()? { + if key == String::from("#") { + let ulid_str: &str = map.next_value()?; + Ok(JsonValue::Node { + id: Ulid::from_string(ulid_str).map_err(|_| { + serde::de::Error::invalid_value( + serde::de::Unexpected::Str(ulid_str), + &"Valid ULID", + ) + })?, + }) + } else { + Err(serde::de::Error::unknown_field(key, &["#"])) + } + } else { + Err(serde::de::Error::missing_field("#")) + } + } + } + + impl<'de> Deserialize<'de> for JsonValue { + fn deserialize(d: D) -> Result + where + D: Deserializer<'de>, + { + d.deserialize_any(JsonValueVisitor) + } + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 41483f8..52cdcce 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,9 +2,9 @@ pub mod engine; pub mod graph; pub mod protocol; pub mod store; -mod ulid; +pub mod merge; -pub use self::ulid::Ulid; +pub use ulid::Ulid; pub(crate) mod executor; diff --git a/src/merge.rs b/src/merge.rs new file mode 100644 index 0000000..7872ef2 --- /dev/null +++ b/src/merge.rs @@ -0,0 +1,124 @@ +use std::{cmp, mem, time::SystemTime}; + +use crate::graph::{Field, Value}; + +pub trait LexicalCmp { + fn lexical_cmp(&self, other: &Self) -> cmp::Ordering; +} + +/// If an update comes in that is more than this amount of time in the future, we will assume that +/// the node that sent the update is lying and trying to make it's update take precedence over the +/// current value of the field. +const FUTURE_UPDATE_THREASHOLD: f64 = 600.0; + +impl Field { + /// Merge the new value into this field, using the [HAM] merge conflict resolution strategy + /// + /// [HAM]: https://github.com/amark/gun/wiki/Conflict-Resolution-with-Guns + pub fn merge_with(&mut self, field: &Field) { + let current_time = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("TODO: system time") + .as_secs_f64(); + + // If the new field has the same timestamp + if field.updated_at == self.updated_at { + match self.lexical_cmp(field) { + // Totally equal, do nothing + cmp::Ordering::Equal => return, + // Keep our value, do nothing + cmp::Ordering::Less => return, + // Keep the new value + cmp::Ordering::Greater => self.value = field.value.clone(), + } + + // If the other field is and older update than the one we have, just ignore it + } else if field.updated_at < self.updated_at { + return; + + // If the other field is in the future + } else if field.updated_at > current_time { + // If the field is too far in the future, ignore it + if field.updated_at - current_time > FUTURE_UPDATE_THREASHOLD { + return; + } + + // Wait to apply this update until later + unimplemented!( + "Use async logic to apply this update once our system clock \ + reaches the future time" + ); + } else { + unreachable!() + } + } +} + +impl Field { + pub fn new(value: Value) -> Self { + Self { + updated_at: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("Could not get system time") + .as_secs_f64(), + value, + } + } + + pub fn value(&self) -> &Value { + &self.value + } + + pub fn state(&self) -> &f64 { + &self.updated_at + } +} + +impl LexicalCmp for Value { + fn lexical_cmp(&self, other: &Self) -> cmp::Ordering { + use Value::*; + if mem::discriminant(self) == mem::discriminant(other) { + match (self, other) { + (Bool(x), Bool(y)) => match (x, y) { + (true, true) => cmp::Ordering::Equal, + (true, false) => cmp::Ordering::Less, + (false, true) => cmp::Ordering::Greater, + (false, false) => cmp::Ordering::Equal, + }, + (Int(x), Int(y)) => x.cmp(y), + (Float(x), Float(y)) => x.partial_cmp(y).unwrap_or(cmp::Ordering::Less), + (String(x), String(y)) => x.cmp(y), + (Binary(x), Binary(y)) => x.cmp(y), + (Node(x), Node(y)) => x.cmp(y), + _ => unreachable!(), + } + } else { + let self_rank = match self { + None => 0, + Bool(_) => 1, + Int(_) => 2, + Float(_) => 3, + String(_) => 4, + Binary(_) => 5, + Node(_) => 6, + }; + let other_rank = match other { + None => 0, + Bool(_) => 1, + Int(_) => 2, + Float(_) => 3, + String(_) => 4, + Binary(_) => 5, + Node(_) => 6, + }; + + if self_rank < other_rank { + cmp::Ordering::Less + } else if self_rank > other_rank { + cmp::Ordering::Greater + } else { + unreachable!() + } + } + } +} diff --git a/src/store.rs b/src/store.rs index c6d1cbe..340d412 100644 --- a/src/store.rs +++ b/src/store.rs @@ -8,7 +8,7 @@ mod wasm; /// Get the default store implementation for the current platform /// -/// Currently on non-WASM platforms this is the [`SimpleFsStore`], configured to use the `./data` +/// Currently on non-WASM platforms this is the `SimpleFsStore`, configured to use the `./data` /// directory for storage. /// /// The default store implementation on WASM will use the browser's IndexedDB, but this is not yet @@ -23,21 +23,10 @@ pub async fn get_default_store() -> Result { store } -/// A simple, raw data storage interface +/// A simple, [`Node`] storage interface /// /// [`Store`] is designed to be implemented over any persistant storage interface such as the /// filesystem, S3, browser LocalStorage or IndexedDB, etc. -/// -/// The interface is a simple key-value store, where each key could either have no data asociated to -/// it, or it could have a value that may either be either string data or binary data. -/// -/// Because some data stores such as browser local storage can only store string data, separate -/// functions are used for getting and setting string and binary data. This allows the -/// implementation to choose to base64 encode/decode binary values when writing to a string-only -/// data store, or to just write the raw bytes directly if binary storage is supported. -/// -/// Calling [`Store::get_binary`] on a key on a string value or calling [`Store::get_string`] on a -/// binary value should fail with an error. #[async_trait::async_trait] pub trait Store { /// Get a value from the store diff --git a/src/store/native.rs b/src/store/native.rs index 8760749..f7fb8e0 100644 --- a/src/store/native.rs +++ b/src/store/native.rs @@ -6,7 +6,7 @@ use std::{ use blocking::unblock; -use crate::graph::Node; +use crate::graph::{repr::repr_borsh::BorshNode, Node}; use super::{Store, StoreError}; @@ -42,6 +42,10 @@ impl SimpleFsStore { }) .await } + + fn file_path(&self, key: &str) -> PathBuf { + self.root_dir.join(key) + } } async fn load_file(file_path: PathBuf) -> Result>, StoreError> { @@ -91,10 +95,12 @@ impl Store for SimpleFsStore { use borsh::BorshDeserialize; // Get the path to the file - let file_path = self.root_dir.join(key); + let file_path = self.file_path(key); if let Some(buf) = load_file(file_path).await? { - let node = Node::deserialize(&mut buf.as_slice()).map_err(box_error)?; + let node = BorshNode::deserialize(&mut buf.as_slice()) + .map_err(box_error)? + .into(); Ok(Some(node)) } else { @@ -106,9 +112,12 @@ impl Store for SimpleFsStore { use borsh::BorshSerialize; // Get the path to the file - let file_path = self.root_dir.join(key); + let file_path = self.file_path(key); + // Clone the data - let data = data.try_to_vec().expect("Impossible IO error"); + let data = BorshNode::from(data) + .try_to_vec() + .expect("Unreachable: IO error"); // Write the file write_file(file_path, data).await @@ -116,7 +125,7 @@ impl Store for SimpleFsStore { async fn delete(&self, key: &str) -> Result<(), StoreError> { // Get the path to the file - let file_path = self.root_dir.join(key); + let file_path = self.file_path(key); // Perform blocking operation on a thread pool unblock(move || { diff --git a/src/ulid.rs b/src/ulid.rs deleted file mode 100644 index 9f7c25e..0000000 --- a/src/ulid.rs +++ /dev/null @@ -1,57 +0,0 @@ -use ulid::Ulid as ActualUlid; - -/// Wrapper around UUID that implements the borsh traits -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct Ulid(ActualUlid); - -impl Ulid { - pub fn new() -> Self { - Ulid(ActualUlid::new()) - } -} - -impl std::ops::Deref for Ulid { - type Target = ActualUlid; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl std::ops::DerefMut for Ulid { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -#[cfg(feature = "borsh")] -impl borsh::BorshSerialize for Ulid { - fn serialize(&self, writer: &mut W) -> std::io::Result<()> { - let id: u128 = self.0.into(); - borsh::BorshSerialize::serialize(&id, writer) - } -} - -#[cfg(feature = "borsh")] -impl borsh::BorshDeserialize for Ulid { - fn deserialize(buf: &mut &[u8]) -> std::io::Result { - let id: u128 = borsh::BorshDeserialize::deserialize(buf)?; - Ok(Ulid(ActualUlid::from(id))) - } -} - -#[cfg(feature = "borsh")] -impl borsh::BorshSchema for Ulid { - fn add_definitions_recursively( - definitions: &mut std::collections::HashMap< - borsh::schema::Declaration, - borsh::schema::Definition, - >, - ) { - ::add_definitions_recursively(definitions) - } - - fn declaration() -> borsh::schema::Declaration { - ::declaration() - } -} From 54f590a3cf8d8eff22a7bc52057b3fb1bde3f8b8 Mon Sep 17 00:00:00 2001 From: Zicklag Date: Thu, 23 Sep 2021 20:52:38 -0500 Subject: [PATCH 05/12] Move Repr Modules to Their Own Files --- src/graph/repr.rs | 396 +---------------------------------- src/graph/repr/repr_borsh.rs | 108 ++++++++++ src/graph/repr/repr_json.rs | 286 +++++++++++++++++++++++++ 3 files changed, 396 insertions(+), 394 deletions(-) create mode 100644 src/graph/repr/repr_borsh.rs create mode 100644 src/graph/repr/repr_json.rs diff --git a/src/graph/repr.rs b/src/graph/repr.rs index 983b85b..1dd71c0 100644 --- a/src/graph/repr.rs +++ b/src/graph/repr.rs @@ -1,400 +1,8 @@ //! Representations of the graph data structures used for serialization/deserialization use super::*; - -/// [borsh] representation -/// -/// [borsh]: https://github.com/near/borsh #[cfg(feature = "borsh")] -pub mod repr_borsh { - use borsh::{BorshDeserialize, BorshSerialize}; - - use std::collections::HashMap; - - use super::*; - - #[derive(BorshDeserialize, BorshSerialize)] - pub struct BorshNode { - pub id: u128, - pub fields: HashMap, - } - - #[derive(BorshDeserialize, BorshSerialize)] - pub struct BorshField { - pub updated_at: f64, - pub value: BorshValue, - } - - #[derive(BorshDeserialize, BorshSerialize)] - pub enum BorshValue { - None, - Bool(bool), - Int(i64), - Float(f64), - String(String), - Binary(Vec), - Node(u128), - } - - mod to_borsh { - use super::*; - - impl From for BorshNode { - fn from(node: Node) -> Self { - Self { - id: node.id.into(), - fields: node - .fields - .into_iter() - .map(|(k, v)| (k, v.into())) - .collect(), - } - } - } - impl From for BorshField { - fn from(field: Field) -> Self { - Self { - updated_at: field.updated_at, - value: field.value.into(), - } - } - } - impl From for BorshValue { - fn from(value: Value) -> Self { - match value { - Value::None => BorshValue::None, - Value::Bool(b) => BorshValue::Bool(b), - Value::Int(i) => BorshValue::Int(i), - Value::Float(f) => BorshValue::Float(f), - Value::String(s) => BorshValue::String(s), - Value::Binary(b) => BorshValue::Binary(b), - Value::Node(n) => BorshValue::Node(n.into()), - } - } - } - } - - mod from_borsh { - use super::*; - - impl From for Node { - fn from(node: BorshNode) -> Self { - Self { - id: node.id.into(), - fields: node - .fields - .into_iter() - .map(|(k, v)| (k, v.into())) - .collect(), - } - } - } - impl From for Field { - fn from(field: BorshField) -> Self { - Self { - updated_at: field.updated_at, - value: field.value.into(), - } - } - } - impl From for Value { - fn from(value: BorshValue) -> Self { - match value { - BorshValue::None => Value::None, - BorshValue::Bool(b) => Value::Bool(b), - BorshValue::Int(i) => Value::Int(i), - BorshValue::Float(f) => Value::Float(f), - BorshValue::String(s) => Value::String(s), - BorshValue::Binary(b) => Value::Binary(b), - BorshValue::Node(n) => Value::Node(n.into()), - } - } - } - } -} +pub mod repr_borsh; #[cfg(feature = "json")] -pub mod repr_json { - use serde::{de::Visitor, Deserialize, Deserializer, Serialize}; - - use std::{collections::HashMap, convert::TryFrom}; - - use super::*; - - #[derive(Deserialize, Serialize)] - pub struct JsonNode { - #[serde(rename = "_")] - pub meta: JsonNodeMeta, - #[serde(flatten)] - pub fields: HashMap, - } - - #[derive(Deserialize, Serialize)] - pub struct JsonNodeMeta { - #[serde(rename = "#")] - id: Ulid, - #[serde(rename = ">")] - field_states: HashMap, - } - - pub enum JsonValue { - None, - Bool(bool), - Int(i64), - Float(f64), - String(String), - Binary { data: BinaryData }, - Node { id: Ulid }, - } - - #[derive(Serialize, Deserialize, Clone)] - #[serde(try_from = "String", into = "String")] - pub struct BinaryData(Vec); - - impl Into for BinaryData { - fn into(self) -> String { - base64::encode(self.0) - } - } - - impl TryFrom for BinaryData { - type Error = &'static str; - - fn try_from(value: String) -> Result { - base64::decode(value) - .map(|x| BinaryData(x)) - .map_err(|_| "String is not valid base64") - } - } - - mod to_json { - use super::*; - - impl From for JsonNode { - fn from(node: Node) -> Self { - let mut field_states = HashMap::with_capacity(node.fields.len()); - let mut fields = HashMap::with_capacity(node.fields.len()); - for (k, v) in node.fields { - field_states.insert(k.clone(), v.updated_at); - fields.insert(k, v.value.into()); - } - Self { - meta: JsonNodeMeta { - id: node.id, - field_states, - }, - fields, - } - } - } - - impl From for JsonValue { - fn from(value: Value) -> Self { - match value { - Value::None => JsonValue::None, - Value::Bool(b) => JsonValue::Bool(b), - Value::Int(i) => JsonValue::Int(i), - Value::Float(f) => JsonValue::Float(f), - Value::String(s) => JsonValue::String(s), - Value::Binary(b) => JsonValue::Binary { - data: BinaryData(b), - }, - Value::Node(n) => JsonValue::Node { id: n.into() }, - } - } - } - } - - mod from_json { - use super::*; - - impl From for Node { - fn from(node: JsonNode) -> Self { - let JsonNode { mut meta, fields } = node; - Self { - id: meta.id.into(), - fields: fields - .into_iter() - .map(|(k, v)| { - let field = Field { - updated_at: meta.field_states.remove(&k).unwrap(), - value: v.into(), - }; - (k, field) - }) - .collect(), - } - } - } - impl From for Value { - fn from(value: JsonValue) -> Self { - match value { - JsonValue::None => Value::None, - JsonValue::Bool(b) => Value::Bool(b), - JsonValue::Int(i) => Value::Int(i), - JsonValue::Float(f) => Value::Float(f), - JsonValue::String(s) => Value::String(s), - JsonValue::Binary { data } => Value::Binary(data.0), - JsonValue::Node { id } => Value::Node(id.into()), - } - } - } - } - - mod serde_impls { - use serde::ser::SerializeMap; - - use super::*; - - impl Serialize for JsonValue { - fn serialize(&self, s: S) -> Result - where - S: serde::Serializer, - { - match self { - JsonValue::None => s.serialize_none(), - JsonValue::Bool(b) => s.serialize_bool(*b), - JsonValue::Int(i) => s.serialize_i64(*i), - JsonValue::Float(f) => s.serialize_f64(*f), - JsonValue::String(string) => s.serialize_str(string), - JsonValue::Binary { data } => { - s.serialize_str(&format!("$base64${}", base64::encode(data.0.clone()))) - } - JsonValue::Node { id } => { - let mut map = s.serialize_map(Some(1))?; - map.serialize_entry("#", &id.to_string())?; - map.end() - } - } - } - } - - struct JsonValueVisitor; - - macro_rules! visit_int { - ($fn:ident, $int:ident) => { - fn $fn(self, v: $int) -> Result - where - E: serde::de::Error, - { - Ok(JsonValue::Int(v as i64)) - } - }; - } - macro_rules! visit_float { - ($fn:ident, $float:ident) => { - fn $fn(self, v: $float) -> Result - where - E: serde::de::Error, - { - Ok(JsonValue::Float(v as f64)) - } - }; - } - - impl<'de> Visitor<'de> for JsonValueVisitor { - type Value = JsonValue; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str( - "null, a boolean, an integer, a float, a string, base64 encoded binary data as a \ - string starting with `$base64$`, or a map with a single field `#` set to a uuid", - ) - } - - fn visit_bool(self, v: bool) -> Result - where - E: serde::de::Error, - { - Ok(JsonValue::Bool(v)) - } - - visit_int!(visit_i64, i64); - visit_int!(visit_i32, i32); - visit_int!(visit_i16, i16); - visit_int!(visit_i8, i8); - visit_int!(visit_u64, u64); - visit_int!(visit_u32, u32); - visit_int!(visit_u16, u16); - visit_int!(visit_u8, u8); - visit_float!(visit_f32, f32); - visit_float!(visit_f64, f64); - - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - let base64_prefix = "$base64$"; - if v.starts_with(base64_prefix) { - let data_base64 = v.strip_prefix(base64_prefix).unwrap_or(""); - - let data = base64::decode(data_base64).map_err(|_| { - serde::de::Error::invalid_value( - serde::de::Unexpected::Str("$base64$[invalid base64 data]"), - &"valid base64 encoded data", - ) - })?; - - Ok(JsonValue::Binary { - data: BinaryData(data), - }) - } else { - Ok(JsonValue::String(v.to_owned())) - } - } - - fn visit_string(self, v: String) -> Result - where - E: serde::de::Error, - { - self.visit_str(&v) - } - - fn visit_none(self) -> Result - where - E: serde::de::Error, - { - Ok(JsonValue::None) - } - - fn visit_unit(self) -> Result - where - E: serde::de::Error, - { - Ok(JsonValue::None) - } - - fn visit_map(self, mut map: A) -> Result - where - A: serde::de::MapAccess<'de>, - { - if let Some(key) = map.next_key()? { - if key == String::from("#") { - let ulid_str: &str = map.next_value()?; - Ok(JsonValue::Node { - id: Ulid::from_string(ulid_str).map_err(|_| { - serde::de::Error::invalid_value( - serde::de::Unexpected::Str(ulid_str), - &"Valid ULID", - ) - })?, - }) - } else { - Err(serde::de::Error::unknown_field(key, &["#"])) - } - } else { - Err(serde::de::Error::missing_field("#")) - } - } - } - - impl<'de> Deserialize<'de> for JsonValue { - fn deserialize(d: D) -> Result - where - D: Deserializer<'de>, - { - d.deserialize_any(JsonValueVisitor) - } - } - } -} +pub mod repr_json; diff --git a/src/graph/repr/repr_borsh.rs b/src/graph/repr/repr_borsh.rs new file mode 100644 index 0000000..a72ce74 --- /dev/null +++ b/src/graph/repr/repr_borsh.rs @@ -0,0 +1,108 @@ +//! [borsh] representation +//! +//! [borsh]: https://github.com/near/borsh + +use borsh::{BorshDeserialize, BorshSerialize}; + +use std::collections::HashMap; + +use super::*; + +#[derive(BorshDeserialize, BorshSerialize)] +pub struct BorshNode { + pub id: u128, + pub fields: HashMap, +} + +#[derive(BorshDeserialize, BorshSerialize)] +pub struct BorshField { + pub updated_at: f64, + pub value: BorshValue, +} + +#[derive(BorshDeserialize, BorshSerialize)] +pub enum BorshValue { + None, + Bool(bool), + Int(i64), + Float(f64), + String(String), + Binary(Vec), + Node(u128), +} + +mod to_borsh { + use super::*; + + impl From for BorshNode { + fn from(node: Node) -> Self { + Self { + id: node.id.into(), + fields: node + .fields + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(), + } + } + } + impl From for BorshField { + fn from(field: Field) -> Self { + Self { + updated_at: field.updated_at, + value: field.value.into(), + } + } + } + impl From for BorshValue { + fn from(value: Value) -> Self { + match value { + Value::None => BorshValue::None, + Value::Bool(b) => BorshValue::Bool(b), + Value::Int(i) => BorshValue::Int(i), + Value::Float(f) => BorshValue::Float(f), + Value::String(s) => BorshValue::String(s), + Value::Binary(b) => BorshValue::Binary(b), + Value::Node(n) => BorshValue::Node(n.into()), + } + } + } +} + +mod from_borsh { + use super::*; + + impl From for Node { + fn from(node: BorshNode) -> Self { + Self { + id: node.id.into(), + fields: node + .fields + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(), + } + } + } + impl From for Field { + fn from(field: BorshField) -> Self { + Self { + updated_at: field.updated_at, + value: field.value.into(), + } + } + } + impl From for Value { + fn from(value: BorshValue) -> Self { + match value { + BorshValue::None => Value::None, + BorshValue::Bool(b) => Value::Bool(b), + BorshValue::Int(i) => Value::Int(i), + BorshValue::Float(f) => Value::Float(f), + BorshValue::String(s) => Value::String(s), + BorshValue::Binary(b) => Value::Binary(b), + BorshValue::Node(n) => Value::Node(n.into()), + } + } + } +} diff --git a/src/graph/repr/repr_json.rs b/src/graph/repr/repr_json.rs new file mode 100644 index 0000000..ee2413b --- /dev/null +++ b/src/graph/repr/repr_json.rs @@ -0,0 +1,286 @@ +//! JSON representation of graph data +//! +//! Used to match the official JavaScript GUN implementation's format + +use serde::{de::Visitor, Deserialize, Deserializer, Serialize}; + +use std::{collections::HashMap, convert::TryFrom}; + +use super::*; + +#[derive(Deserialize, Serialize)] +pub struct JsonNode { + #[serde(rename = "_")] + pub meta: JsonNodeMeta, + #[serde(flatten)] + pub fields: HashMap, +} + +#[derive(Deserialize, Serialize)] +pub struct JsonNodeMeta { + #[serde(rename = "#")] + id: Ulid, + #[serde(rename = ">")] + field_states: HashMap, +} + +pub enum JsonValue { + None, + Bool(bool), + Int(i64), + Float(f64), + String(String), + Binary { data: BinaryData }, + Node { id: Ulid }, +} + +#[derive(Serialize, Deserialize, Clone)] +#[serde(try_from = "String", into = "String")] +pub struct BinaryData(Vec); + +impl Into for BinaryData { + fn into(self) -> String { + base64::encode(self.0) + } +} + +impl TryFrom for BinaryData { + type Error = &'static str; + + fn try_from(value: String) -> Result { + base64::decode(value) + .map(|x| BinaryData(x)) + .map_err(|_| "String is not valid base64") + } +} + +mod to_json { + use super::*; + + impl From for JsonNode { + fn from(node: Node) -> Self { + let mut field_states = HashMap::with_capacity(node.fields.len()); + let mut fields = HashMap::with_capacity(node.fields.len()); + for (k, v) in node.fields { + field_states.insert(k.clone(), v.updated_at); + fields.insert(k, v.value.into()); + } + Self { + meta: JsonNodeMeta { + id: node.id, + field_states, + }, + fields, + } + } + } + + impl From for JsonValue { + fn from(value: Value) -> Self { + match value { + Value::None => JsonValue::None, + Value::Bool(b) => JsonValue::Bool(b), + Value::Int(i) => JsonValue::Int(i), + Value::Float(f) => JsonValue::Float(f), + Value::String(s) => JsonValue::String(s), + Value::Binary(b) => JsonValue::Binary { + data: BinaryData(b), + }, + Value::Node(n) => JsonValue::Node { id: n.into() }, + } + } + } +} + +mod from_json { + use super::*; + + impl From for Node { + fn from(node: JsonNode) -> Self { + let JsonNode { mut meta, fields } = node; + Self { + id: meta.id.into(), + fields: fields + .into_iter() + .map(|(k, v)| { + let field = Field { + updated_at: meta.field_states.remove(&k).unwrap(), + value: v.into(), + }; + (k, field) + }) + .collect(), + } + } + } + impl From for Value { + fn from(value: JsonValue) -> Self { + match value { + JsonValue::None => Value::None, + JsonValue::Bool(b) => Value::Bool(b), + JsonValue::Int(i) => Value::Int(i), + JsonValue::Float(f) => Value::Float(f), + JsonValue::String(s) => Value::String(s), + JsonValue::Binary { data } => Value::Binary(data.0), + JsonValue::Node { id } => Value::Node(id.into()), + } + } + } +} + +mod serde_impls { + use serde::ser::SerializeMap; + + use super::*; + + impl Serialize for JsonValue { + fn serialize(&self, s: S) -> Result + where + S: serde::Serializer, + { + match self { + JsonValue::None => s.serialize_none(), + JsonValue::Bool(b) => s.serialize_bool(*b), + JsonValue::Int(i) => s.serialize_i64(*i), + JsonValue::Float(f) => s.serialize_f64(*f), + JsonValue::String(string) => s.serialize_str(string), + JsonValue::Binary { data } => { + s.serialize_str(&format!("$base64${}", base64::encode(data.0.clone()))) + } + JsonValue::Node { id } => { + let mut map = s.serialize_map(Some(1))?; + map.serialize_entry("#", &id.to_string())?; + map.end() + } + } + } + } + + struct JsonValueVisitor; + + macro_rules! visit_int { + ($fn:ident, $int:ident) => { + fn $fn(self, v: $int) -> Result + where + E: serde::de::Error, + { + Ok(JsonValue::Int(v as i64)) + } + }; + } + macro_rules! visit_float { + ($fn:ident, $float:ident) => { + fn $fn(self, v: $float) -> Result + where + E: serde::de::Error, + { + Ok(JsonValue::Float(v as f64)) + } + }; + } + + impl<'de> Visitor<'de> for JsonValueVisitor { + type Value = JsonValue; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str( + "null, a boolean, an integer, a float, a string, base64 encoded binary data as a \ + string starting with `$base64$`, or a map with a single field `#` set to a uuid", + ) + } + + fn visit_bool(self, v: bool) -> Result + where + E: serde::de::Error, + { + Ok(JsonValue::Bool(v)) + } + + visit_int!(visit_i64, i64); + visit_int!(visit_i32, i32); + visit_int!(visit_i16, i16); + visit_int!(visit_i8, i8); + visit_int!(visit_u64, u64); + visit_int!(visit_u32, u32); + visit_int!(visit_u16, u16); + visit_int!(visit_u8, u8); + visit_float!(visit_f32, f32); + visit_float!(visit_f64, f64); + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + let base64_prefix = "$base64$"; + if v.starts_with(base64_prefix) { + let data_base64 = v.strip_prefix(base64_prefix).unwrap_or(""); + + let data = base64::decode(data_base64).map_err(|_| { + serde::de::Error::invalid_value( + serde::de::Unexpected::Str("$base64$[invalid base64 data]"), + &"valid base64 encoded data", + ) + })?; + + Ok(JsonValue::Binary { + data: BinaryData(data), + }) + } else { + Ok(JsonValue::String(v.to_owned())) + } + } + + fn visit_string(self, v: String) -> Result + where + E: serde::de::Error, + { + self.visit_str(&v) + } + + fn visit_none(self) -> Result + where + E: serde::de::Error, + { + Ok(JsonValue::None) + } + + fn visit_unit(self) -> Result + where + E: serde::de::Error, + { + Ok(JsonValue::None) + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + if let Some(key) = map.next_key()? { + if key == String::from("#") { + let ulid_str: &str = map.next_value()?; + Ok(JsonValue::Node { + id: Ulid::from_string(ulid_str).map_err(|_| { + serde::de::Error::invalid_value( + serde::de::Unexpected::Str(ulid_str), + &"Valid ULID", + ) + })?, + }) + } else { + Err(serde::de::Error::unknown_field(key, &["#"])) + } + } else { + Err(serde::de::Error::missing_field("#")) + } + } + } + + impl<'de> Deserialize<'de> for JsonValue { + fn deserialize(d: D) -> Result + where + D: Deserializer<'de>, + { + d.deserialize_any(JsonValueVisitor) + } + } +} From c2e4f572f4e113e8f6675823f103a7a25652be57 Mon Sep 17 00:00:00 2001 From: Zicklag Date: Thu, 23 Sep 2021 21:34:19 -0500 Subject: [PATCH 06/12] Write Som Docs and Reoganize --- src/{merge.rs => crdt.rs} | 5 +++++ src/engine.rs | 5 +++-- src/executor.rs | 14 +++++++++++++- src/lib.rs | 12 +++++++++--- src/protocol.rs | 1 + src/store.rs | 6 ++++-- src/store/native.rs | 5 ++++- src/store/wasm.rs | 3 +++ 8 files changed, 42 insertions(+), 9 deletions(-) rename src/{merge.rs => crdt.rs} (95%) diff --git a/src/merge.rs b/src/crdt.rs similarity index 95% rename from src/merge.rs rename to src/crdt.rs index 7872ef2..9033ec3 100644 --- a/src/merge.rs +++ b/src/crdt.rs @@ -1,8 +1,13 @@ +//! CRDTs ( Conflict-free Replicated Data Types ), including the implementation of GUN's HAM +//! algorithm + use std::{cmp, mem, time::SystemTime}; use crate::graph::{Field, Value}; +/// Trait implemented by structs that can be lexically sorted pub trait LexicalCmp { + /// Compare two object lexographically fn lexical_cmp(&self, other: &Self) -> cmp::Ordering; } diff --git a/src/engine.rs b/src/engine.rs index 6a0af81..db80a33 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -1,10 +1,11 @@ -use std::{sync::Arc, time::Duration}; +//! Contains the main [`Rod`] struct, used to access the replicated database + +use std::sync::Arc; use tracing as trc; use ulid::Ulid; use crate::{ - executor, graph::Node, store::{get_default_store, Store, StoreError}, }; diff --git a/src/executor.rs b/src/executor.rs index f9ef8e1..6b6b5b6 100644 --- a/src/executor.rs +++ b/src/executor.rs @@ -1,4 +1,14 @@ -//! Implementation of async executor functions such as [`spawn`] +//! Functions abstracted over the current async executor such as [`spawn`] +//! +//! Different async executors may be used on different platforms and eventually there will be Cargo +//! features for building for the desired executor. +//! +//! Currently [`smol`] will be used on native targets and [`wasm-bindgen-futures::spawn_local`] +//! +//! [`smol`]: https://docs.rs/smol +//! +//! [`wasm-bindgen-futures::spawn_local`]: +//! https://docs.rs/wasm-bindgen-futures/0.4.28/wasm_bindgen_futures/fn.spawn_local.html pub use implementation::*; @@ -10,6 +20,7 @@ mod implementation { use std::{future::Future, panic, thread}; + /// Spawn an async task to run in the background #[cfg(not(target_arch = "wasm32"))] pub fn spawn(future: impl Future + Send + 'static) { static GLOBAL: Lazy> = Lazy::new(|| { @@ -36,6 +47,7 @@ mod implementation { mod implementation { use std::future::Future; + /// Spawn an async task to run in the background pub fn spawn(_future: impl Future + Send + 'static) { todo!(); } diff --git a/src/lib.rs b/src/lib.rs index 52cdcce..288a00a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,13 +1,19 @@ +//! The Rust implementation of the [GUN] decentralized database syncronization protocol. +//! +//! Rod is attempting to be compatible with the official JavaScript implementation of GUN while also +//! supporting extra features such as a binary serialization +//! +//! [GUN]: https://github.com/amark/gun + +pub mod crdt; pub mod engine; +pub mod executor; pub mod graph; pub mod protocol; pub mod store; -pub mod merge; pub use ulid::Ulid; -pub(crate) mod executor; - #[cfg(target_arch = "wasm32")] extern crate wee_alloc; diff --git a/src/protocol.rs b/src/protocol.rs index e69de29..571c62e 100644 --- a/src/protocol.rs +++ b/src/protocol.rs @@ -0,0 +1 @@ +//! Wire protocol data types and representations \ No newline at end of file diff --git a/src/store.rs b/src/store.rs index 340d412..87e5b52 100644 --- a/src/store.rs +++ b/src/store.rs @@ -1,10 +1,12 @@ +//! Backing data stores used for persistant data + use crate::graph::Node; #[cfg(not(target_arch = "wasm32"))] -mod native; +pub mod native; #[cfg(target_arch = "wasm32")] -mod wasm; +pub mod wasm; /// Get the default store implementation for the current platform /// diff --git a/src/store/native.rs b/src/store/native.rs index f7fb8e0..30e9107 100644 --- a/src/store/native.rs +++ b/src/store/native.rs @@ -1,3 +1,5 @@ +//! Store implementatons for native targets + use std::{ fs::{self, OpenOptions}, io::{Read, Write}, @@ -10,6 +12,7 @@ use crate::graph::{repr::repr_borsh::BorshNode, Node}; use super::{Store, StoreError}; +/// Get the default native store pub async fn get_default_store() -> Result { SimpleFsStore::new(Path::new("./data")).await } @@ -26,7 +29,7 @@ pub struct SimpleFsStore { } impl SimpleFsStore { - /// Create a new [`FsStore`] that puts files in the specified `root_dir` + /// Create a new [`SimpleFsStore`] that puts files in the specified `root_dir` pub async fn new(root_dir: &Path) -> Result { #[cfg(not(feature = "borsh"))] compile_error!("`borsh` feature required to use `SimpleFsStore`"); diff --git a/src/store/wasm.rs b/src/store/wasm.rs index 64b170d..3d664a4 100644 --- a/src/store/wasm.rs +++ b/src/store/wasm.rs @@ -1,5 +1,8 @@ +//! Store implementatons for WASM targets + use super::Store; +/// Get the default WASM store pub async fn get_default_store() -> Result { let store = Ok(IndexedDbStore); } From a642de72fea9d80f16c82b7fd321f850fa81fd97 Mon Sep 17 00:00:00 2001 From: Zicklag Date: Fri, 24 Sep 2021 12:44:25 -0500 Subject: [PATCH 07/12] Make WASM Leaner! --- Cargo.lock | 62 ++++++++++---------------------------------- Cargo.toml | 9 ++----- src/lib.rs | 8 ------ src/store/wasm.rs | 20 ++++++++++---- wasmtest/Cargo.toml | 4 +-- wasmtest/src/main.rs | 15 +++-------- 6 files changed, 35 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5708f86..a4021b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -189,12 +189,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.0" @@ -298,19 +292,6 @@ dependencies = [ "waker-fn", ] -[[package]] -name = "getrandom" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" -dependencies = [ - "cfg-if 1.0.0", - "js-sys", - "libc", - "wasi", - "wasm-bindgen", -] - [[package]] name = "hashbrown" version = "0.9.1" @@ -335,7 +316,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -371,7 +352,7 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -389,12 +370,6 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" -[[package]] -name = "memory_units" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" - [[package]] name = "num-integer" version = "0.1.44" @@ -458,6 +433,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "pollster" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb20dcc30536a1508e75d47dd0e399bb2fe7354dcf35cda9127f2bf1ed92e30e" + [[package]] name = "proc-macro-crate" version = "0.1.5" @@ -630,14 +611,12 @@ name = "rod" version = "0.1.0" dependencies = [ "async-executor", - "async-timer", "async-trait", "base64", "blocking", "borsh", "easy-parallel", "futures-lite", - "getrandom", "num_cpus", "once_cell", "scc", @@ -646,7 +625,6 @@ dependencies = [ "thiserror", "tracing", "ulid", - "wee_alloc", ] [[package]] @@ -657,9 +635,9 @@ checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" [[package]] name = "scc" -version = "0.5.2" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d602af27b0129ed0b3a9becec8c3eadca7c6e0d34bd7595c5fc012d015c0b6" +checksum = "f78f4f64a22f8ca49d465148ed18b7ebc476273a1698852808a3714ef0bf4c35" dependencies = [ "scopeguard", ] @@ -794,7 +772,7 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2ba9ab62b7d6497a8638dfda5e5c4fb3b2d5a7fca4118f2b96151c8ef1a437e" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -909,7 +887,7 @@ version = "0.2.77" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e68338db6becec24d3c7977b5bf8a48be992c934b5d07177e3931f5dc9b076c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen-macro", ] @@ -961,24 +939,10 @@ checksum = "c279e376c7a8e8752a8f1eaa35b7b0bee6bb9fb0cdacfa97cc3f1f289c87e2b4" name = "wasmtest" version = "0.1.0" dependencies = [ - "anyhow", - "async-executor", - "futures-lite", + "pollster", "rod", ] -[[package]] -name = "wee_alloc" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb3b5a6b2bb17cb6ad44a2e68a43e8d2722c997da10e928665c72ec6c0a0b8e" -dependencies = [ - "cfg-if 0.1.10", - "libc", - "memory_units", - "winapi", -] - [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 449a055..3848016 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,23 +32,18 @@ borsh = { version = "0.9.1", optional = true } base64 = { version = "0.13.0", optional = true } # Async -futures-lite = "1.12.0" async-trait = "0.1.51" -async-timer = "1.0.0-beta.7" -getrandom = { version = "*", features = ["js"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Async +futures-lite = "1.12.0" easy-parallel = "3.1.0" num_cpus = "1.13.0" blocking = "1.0.2" async-executor = "1.4.1" -[target.'cfg(target_arch = "wasm32")'.dependencies] -wee_alloc = "0.4.5" - [profile.release] lto = true opt-level = "z" panic = "abort" -debug = false \ No newline at end of file +debug = false diff --git a/src/lib.rs b/src/lib.rs index 288a00a..a028aab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,11 +13,3 @@ pub mod protocol; pub mod store; pub use ulid::Ulid; - -#[cfg(target_arch = "wasm32")] -extern crate wee_alloc; - -// Use `wee_alloc` as the global allocator for WASM -#[cfg(target_arch = "wasm32")] -#[global_allocator] -static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; diff --git a/src/store/wasm.rs b/src/store/wasm.rs index 3d664a4..32f3ea9 100644 --- a/src/store/wasm.rs +++ b/src/store/wasm.rs @@ -1,17 +1,27 @@ //! Store implementatons for WASM targets -use super::Store; +use super::{Store, StoreError}; + +use crate::graph::Value; /// Get the default WASM store pub async fn get_default_store() -> Result { - let store = Ok(IndexedDbStore); + Ok(IndexedDbStore) } pub struct IndexedDbStore; #[async_trait::async_trait] impl Store for IndexedDbStore { - async fn get(&self, key: &str) -> Result, StoreError> {} - async fn put(&self, key: &str, value: Value) -> Result<(), StoreError> {} - async fn delete(&self, key: &str) -> Result<(), StoreError> {} + async fn get(&self, _key: &str) -> Result, StoreError> { + todo!() + } + + async fn put(&self, _key: &str, _value: crate::graph::Node) -> Result<(), StoreError> { + todo!() + } + + async fn delete(&self, _key: &str) -> Result<(), StoreError> { + todo!() + } } diff --git a/wasmtest/Cargo.toml b/wasmtest/Cargo.toml index bb316f9..31e285d 100644 --- a/wasmtest/Cargo.toml +++ b/wasmtest/Cargo.toml @@ -7,6 +7,4 @@ edition = "2018" [dependencies] rod = { path = "../"} -async-executor = "1.4.1" -futures-lite = "1.12.0" -anyhow = "1.0.44" \ No newline at end of file +pollster = "0.2.4" \ No newline at end of file diff --git a/wasmtest/src/main.rs b/wasmtest/src/main.rs index 37a0322..4254d34 100644 --- a/wasmtest/src/main.rs +++ b/wasmtest/src/main.rs @@ -1,18 +1,11 @@ -use std::{time::Duration}; - use rod::engine::Rod; -use futures_lite::future; - fn main() { - let ex = async_executor::Executor::new(); - future::block_on(ex.run(start())).expect("Error"); + pollster::block_on(start()); } -async fn start() -> anyhow::Result<()> { - let engine = Rod::new().await?; - - future::pending::<()>().await; +async fn start() { + let _engine = Rod::new().await.unwrap(); - Ok(()) + std::future::pending::<()>().await; } From fdd792870d248baa755723bc292d2dc0c13867b6 Mon Sep 17 00:00:00 2001 From: Zicklag Date: Fri, 24 Sep 2021 15:54:20 -0500 Subject: [PATCH 08/12] More Work on Storage Interface - Now we have get() and put() functions on the Rod struct --- Cargo.lock | 48 +++++++++++++++++------ Cargo.toml | 8 +++- playground/src/main.rs | 52 +++++++++++-------------- src/crdt.rs | 38 +++++++++--------- src/engine.rs | 37 +++++++++++++----- src/executor.rs | 6 ++- src/store.rs | 22 ++++++++--- src/store/native.rs | 87 ++++++++++++++++++++++++++++++------------ src/store/wasm.rs | 13 ++++++- 9 files changed, 203 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a4021b0..7324e7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -327,9 +327,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "js-sys" -version = "0.3.54" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1866b355d9c878e5e607473cbe3f63282c0b7aad2db1dbebf55076c686918254" +checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84" dependencies = [ "wasm-bindgen", ] @@ -625,6 +625,8 @@ dependencies = [ "thiserror", "tracing", "ulid", + "wasm-bindgen", + "wasm-bindgen-futures", ] [[package]] @@ -883,9 +885,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.77" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e68338db6becec24d3c7977b5bf8a48be992c934b5d07177e3931f5dc9b076c" +checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -893,9 +895,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.77" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34c405b4f0658583dba0c1c7c9b694f3cac32655db463b56c254a1c75269523" +checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" dependencies = [ "bumpalo", "lazy_static", @@ -906,11 +908,23 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" -version = "0.2.77" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d5a6580be83b19dc570a8f9c324251687ab2184e57086f71625feb57ec77c8" +checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -918,9 +932,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.77" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3775a030dc6f5a0afd8a84981a21cc92a781eb429acef9ecce476d0c9113e92" +checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" dependencies = [ "proc-macro2", "quote", @@ -931,9 +945,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.77" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c279e376c7a8e8752a8f1eaa35b7b0bee6bb9fb0cdacfa97cc3f1f289c87e2b4" +checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" [[package]] name = "wasmtest" @@ -943,6 +957,16 @@ dependencies = [ "rod", ] +[[package]] +name = "web-sys" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 3848016..5dba4cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ json = ["serde", "serde_json", "ulid/serde", "base64"] [dependencies] # Data structures ulid = { version = "0.4.1" } -# radix_trie = { version = "0.2", features = ["serde"] } + once_cell = "1.8.0" scc = "0.5.2" thiserror = "1.0.29" @@ -42,8 +42,12 @@ num_cpus = "1.13.0" blocking = "1.0.2" async-executor = "1.4.1" +[target.'cfg(target_arch = "wasm32")'.dependencies] +# Async +wasm-bindgen = "0.2.78" +wasm-bindgen-futures = "0.4.28" + [profile.release] lto = true opt-level = "z" -panic = "abort" debug = false diff --git a/playground/src/main.rs b/playground/src/main.rs index 4e02f39..af18f75 100644 --- a/playground/src/main.rs +++ b/playground/src/main.rs @@ -1,12 +1,7 @@ use std::{process, time::Duration}; use tracing as trc; -use rod::{ - engine::Rod, - graph::{repr::repr_json::JsonNode, Field, Node, Value}, - store::{get_default_store, Store}, - Ulid, -}; +use rod::engine::Rod; use futures_lite::future; @@ -23,33 +18,30 @@ async fn start() -> anyhow::Result<()> { trc::info!("Staring server"); - let _rod = Rod::new().await?; + let rod = Rod::new().await?; - let store = get_default_store().await?; + // { + // use rod::{ + // graph::{Field, Node, Value}, + // Ulid, + // }; + // let node1 = Node::new(); + // let node2 = Node::new_with_fields(vec![ + // ("hello".into(), Field::new(Value::String("world".into()))), + // ( + // "someJunk".into(), + // Field::new(Value::Binary(vec![1, 2, 3, 4])), + // ), + // ("age".into(), Field::new(Value::Float(30.0))), + // ("nothing".into(), Field::new(Value::None)), + // ("anotherNode".into(), Field::new(Value::Node(Ulid::new()))), + // ]); - store.put("key1", Node::new()).await?; - store - .put( - "key2", - Node::new_with_fields(vec![ - ("hello".into(), Field::new(Value::String("world".into()))), - ( - "someJunk".into(), - Field::new(Value::Binary(vec![1, 2, 3, 4])), - ), - ("age".into(), Field::new(Value::Float(30.0))), - ("nothing".into(), Field::new(Value::None)), - ("anotherNode".into(), Field::new(Value::Node(Ulid::new()))), - ]), - ) - .await?; + // rod.put("node1", node1).await?; + // rod.put("node2", node2).await?; + // } - let node2 = store.get("key2").await?.unwrap(); - let json = serde_json::to_string_pretty(&JsonNode::from(node2))?; - println!("{}", json); - - let parsed: JsonNode = serde_json::from_str(&json).unwrap(); - println!("{:#?}", Node::from(parsed)); + dbg!(rod.get("node2").await?); // Just prevent the process from exiting let mut interval = async_timer::interval(Duration::from_secs(1)); diff --git a/src/crdt.rs b/src/crdt.rs index 9033ec3..c33a358 100644 --- a/src/crdt.rs +++ b/src/crdt.rs @@ -17,6 +17,24 @@ pub trait LexicalCmp { const FUTURE_UPDATE_THREASHOLD: f64 = 600.0; impl Field { + pub fn new(value: Value) -> Self { + Self { + updated_at: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("Could not get system time") + .as_secs_f64(), + value, + } + } + + pub fn value(&self) -> &Value { + &self.value + } + + pub fn state(&self) -> &f64 { + &self.updated_at + } + /// Merge the new value into this field, using the [HAM] merge conflict resolution strategy /// /// [HAM]: https://github.com/amark/gun/wiki/Conflict-Resolution-with-Guns @@ -59,26 +77,6 @@ impl Field { } } -impl Field { - pub fn new(value: Value) -> Self { - Self { - updated_at: SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .expect("Could not get system time") - .as_secs_f64(), - value, - } - } - - pub fn value(&self) -> &Value { - &self.value - } - - pub fn state(&self) -> &f64 { - &self.updated_at - } -} - impl LexicalCmp for Value { fn lexical_cmp(&self, other: &Self) -> cmp::Ordering { use Value::*; diff --git a/src/engine.rs b/src/engine.rs index db80a33..0a4ebca 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use tracing as trc; -use ulid::Ulid; use crate::{ graph::Node, @@ -24,8 +23,6 @@ pub struct Rod { } struct RodInner { - /// The list of nodes cached in memory - nodes: scc::HashMap, /// The backing data store for this engine store: Box, } @@ -41,10 +38,7 @@ impl Rod { let store = Box::new(get_default_store().await?); // Create clonable inner data - let inner = Arc::new(RodInner { - nodes: Default::default(), - store, - }); + let inner = Arc::new(RodInner { store }); // Create Rod instance let instance = Rod { inner }; @@ -52,7 +46,30 @@ impl Rod { Ok(instance) } - // pub async fn get(&self, key: &str) -> Value { - // self.inner.store. - // } + /// Get a node from the database + pub async fn get(&self, key: &str) -> Result, StoreError> { + let this = &self.inner; + + let id = if let Some(id) = this.store.get_id(key).await?.flatten() { + id + } else { + return Ok(None); + }; + + if let Some(node) = this.store.get_node(&id).await? { + return Ok(Some(node)); + } else { + return Ok(None); + } + } + + pub async fn put(&self, key: &str, node: Node) -> Result<(), StoreError> { + let this = &self.inner; + + let id = node.id.clone(); + this.store.set_id(key, Some(id)).await?; + this.store.put_node(node).await?; + + Ok(()) + } } diff --git a/src/executor.rs b/src/executor.rs index 6b6b5b6..27442a0 100644 --- a/src/executor.rs +++ b/src/executor.rs @@ -48,7 +48,9 @@ mod implementation { use std::future::Future; /// Spawn an async task to run in the background - pub fn spawn(_future: impl Future + Send + 'static) { - todo!(); + pub fn spawn(future: impl Future + Send + 'static) { + wasm_bindgen_futures::spawn_local(async move { + future.await; + }); } } diff --git a/src/store.rs b/src/store.rs index 87e5b52..214c979 100644 --- a/src/store.rs +++ b/src/store.rs @@ -1,5 +1,7 @@ //! Backing data stores used for persistant data +use ulid::Ulid; + use crate::graph::Node; #[cfg(not(target_arch = "wasm32"))] @@ -31,14 +33,22 @@ pub async fn get_default_store() -> Result { /// filesystem, S3, browser LocalStorage or IndexedDB, etc. #[async_trait::async_trait] pub trait Store { - /// Get a value from the store - async fn get(&self, key: &str) -> Result, StoreError>; + /// Get a node from the store using it's ULID + async fn get_node(&self, id: &Ulid) -> Result, StoreError>; + + /// Put a node into the store + /// + /// The node can later be retrieved using it's ULID + async fn put_node(&self, node: Node) -> Result<(), StoreError>; + + /// Delete a node from the store using it's ULID + async fn delete_node(&self, id: &Ulid) -> Result<(), StoreError>; - // Put a value in the store - async fn put(&self, key: &str, value: Node) -> Result<(), StoreError>; + /// Point a string key in the database to a node's ULID + async fn set_id(&self, key: &str, id: Option) -> Result<(), StoreError>; - /// Delete a key - async fn delete(&self, key: &str) -> Result<(), StoreError>; + /// Get the ULID pointed at by the string key in the database + async fn get_id(&self, key: &str) -> Result>, StoreError>; } /// An error that can occur in a [`Store`] diff --git a/src/store/native.rs b/src/store/native.rs index 30e9107..bc970a4 100644 --- a/src/store/native.rs +++ b/src/store/native.rs @@ -7,6 +7,8 @@ use std::{ }; use blocking::unblock; +use borsh::BorshDeserialize; +use ulid::Ulid; use crate::graph::{repr::repr_borsh::BorshNode, Node}; @@ -24,8 +26,10 @@ pub async fn get_default_store() -> Result { /// The names of each file will be the base64-encoded key and the value will be the string or binary /// data associated to the key. pub struct SimpleFsStore { - /// The root directory for the filesystem store - root_dir: PathBuf, + /// The directory to store nodes in + node_dir: PathBuf, + /// The directory to store id mappings in + id_dir: PathBuf, } impl SimpleFsStore { @@ -37,17 +41,25 @@ impl SimpleFsStore { let root_dir = root_dir.to_owned(); unblock(move || { - fs::create_dir_all(&root_dir).map_err(box_error)?; + let store = Self { + node_dir: root_dir.join("nodes"), + id_dir: root_dir.join("ids"), + }; - Ok(Self { - root_dir: root_dir.to_owned(), - }) + fs::create_dir_all(&store.node_dir).boxed_err()?; + fs::create_dir_all(&store.id_dir).boxed_err()?; + + Ok(store) }) .await } - fn file_path(&self, key: &str) -> PathBuf { - self.root_dir.join(key) + fn node_path(&self, key: &Ulid) -> PathBuf { + self.node_dir.join(key.to_string()) + } + + fn id_path(&self, id: &str) -> PathBuf { + self.id_dir.join(id.to_string()) } } @@ -60,14 +72,11 @@ async fn load_file(file_path: PathBuf) -> Result>, StoreError> { } // Open the file - let mut file = OpenOptions::new() - .read(true) - .open(file_path) - .map_err(box_error)?; + let mut file = OpenOptions::new().read(true).open(file_path).boxed_err()?; // Read the file into buffer let mut buf = Vec::new(); - file.read_to_end(&mut buf).map_err(box_error)?; + file.read_to_end(&mut buf).boxed_err()?; // And return the buffer Ok(Some(buf)) @@ -84,25 +93,25 @@ async fn write_file(file_path: PathBuf, data: Vec) -> Result<(), StoreError> .truncate(true) .create(true) .open(file_path) - .map_err(box_error)?; + .boxed_err()?; // Write the data to the file - Ok(file.write_all(&data).map_err(box_error)?) + Ok(file.write_all(&data).boxed_err()?) }) .await } #[async_trait::async_trait] impl Store for SimpleFsStore { - async fn get(&self, key: &str) -> Result, StoreError> { + async fn get_node(&self, key: &Ulid) -> Result, StoreError> { use borsh::BorshDeserialize; // Get the path to the file - let file_path = self.file_path(key); + let file_path = self.node_path(key); if let Some(buf) = load_file(file_path).await? { let node = BorshNode::deserialize(&mut buf.as_slice()) - .map_err(box_error)? + .boxed_err()? .into(); Ok(Some(node)) @@ -111,14 +120,14 @@ impl Store for SimpleFsStore { } } - async fn put(&self, key: &str, data: Node) -> Result<(), StoreError> { + async fn put_node(&self, node: Node) -> Result<(), StoreError> { use borsh::BorshSerialize; // Get the path to the file - let file_path = self.file_path(key); + let file_path = self.node_path(&node.id); // Clone the data - let data = BorshNode::from(data) + let data = BorshNode::from(node) .try_to_vec() .expect("Unreachable: IO error"); @@ -126,20 +135,50 @@ impl Store for SimpleFsStore { write_file(file_path, data).await } - async fn delete(&self, key: &str) -> Result<(), StoreError> { + async fn delete_node(&self, key: &Ulid) -> Result<(), StoreError> { // Get the path to the file - let file_path = self.file_path(key); + let file_path = self.node_path(key); // Perform blocking operation on a thread pool unblock(move || { // Delete the file - fs::remove_file(file_path).map_err(box_error)?; + fs::remove_file(file_path).boxed_err()?; // Write the data to the file Ok(()) }) .await } + + async fn set_id(&self, key: &str, id: Option) -> Result<(), StoreError> { + let file_path = self.id_path(key); + let data = borsh::to_vec(&id.map(|x| u128::from(x))).boxed_err()?; + + write_file(file_path, data).await + } + + async fn get_id(&self, key: &str) -> Result>, StoreError> { + let file_path = self.id_path(key); + if let Some(buf) = load_file(file_path).await? { + let id = Option::::deserialize(&mut buf.as_slice()) + .boxed_err()? + .map(Ulid::from); + + Ok(Some(id)) + } else { + Ok(None) + } + } +} + +trait BoxedError { + fn boxed_err(self) -> Result>; +} + +impl BoxedError for Result { + fn boxed_err(self) -> Result> { + self.map_err(box_error) + } } /// Helper to box the an error diff --git a/src/store/wasm.rs b/src/store/wasm.rs index 32f3ea9..2ade017 100644 --- a/src/store/wasm.rs +++ b/src/store/wasm.rs @@ -2,8 +2,6 @@ use super::{Store, StoreError}; -use crate::graph::Value; - /// Get the default WASM store pub async fn get_default_store() -> Result { Ok(IndexedDbStore) @@ -24,4 +22,15 @@ impl Store for IndexedDbStore { async fn delete(&self, _key: &str) -> Result<(), StoreError> { todo!() } + + async fn put_radix( + &self, + _tree: radix_trie::Trie, + ) -> Result<(), StoreError> { + todo!() + } + + async fn get_radix(&self) -> Result, StoreError> { + todo!() + } } From c103d942b7115dfb7d4cce577982decff68af1f2 Mon Sep 17 00:00:00 2001 From: Zicklag Date: Fri, 24 Sep 2021 16:44:37 -0500 Subject: [PATCH 09/12] Improve API Ergonomics --- playground/src/main.rs | 53 +++++++++++++++-------------------- src/engine.rs | 28 +++++++++++++++++-- src/graph.rs | 63 +++++++++++++++++++++++++++++++++++++++++- src/store.rs | 8 ++++-- src/store/native.rs | 4 +-- 5 files changed, 119 insertions(+), 37 deletions(-) diff --git a/playground/src/main.rs b/playground/src/main.rs index af18f75..e043146 100644 --- a/playground/src/main.rs +++ b/playground/src/main.rs @@ -1,4 +1,4 @@ -use std::{process, time::Duration}; +use std::process; use tracing as trc; use rod::engine::Rod; @@ -18,36 +18,29 @@ async fn start() -> anyhow::Result<()> { trc::info!("Staring server"); - let rod = Rod::new().await?; - - // { - // use rod::{ - // graph::{Field, Node, Value}, - // Ulid, - // }; - // let node1 = Node::new(); - // let node2 = Node::new_with_fields(vec![ - // ("hello".into(), Field::new(Value::String("world".into()))), - // ( - // "someJunk".into(), - // Field::new(Value::Binary(vec![1, 2, 3, 4])), - // ), - // ("age".into(), Field::new(Value::Float(30.0))), - // ("nothing".into(), Field::new(Value::None)), - // ("anotherNode".into(), Field::new(Value::Node(Ulid::new()))), - // ]); - - // rod.put("node1", node1).await?; - // rod.put("node2", node2).await?; - // } - - dbg!(rod.get("node2").await?); - - // Just prevent the process from exiting - let mut interval = async_timer::interval(Duration::from_secs(1)); - loop { - interval.wait().await; + let rod = &Rod::new().await?; + + { + use rod::graph::Node; + let mut mary = Node::new(); + mary.set("name", "Mary".to_string()); + + let mut john = Node::new(); + john.set("name", "John".to_string()); + john.set("wife", &mary); + + rod.put("users/john", john).await?; + rod.put("users/mary", mary).await?; } + + let node2 = dbg!(rod.get("users/john").await?).unwrap(); + + dbg!(node2.get("name")); + let wife = node2.get("wife").unwrap().follow(rod).await?.unwrap(); + + dbg!(wife); + + Ok(()) } fn install_tracing() { diff --git a/src/engine.rs b/src/engine.rs index 0a4ebca..750e224 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use tracing as trc; +use ulid::Ulid; use crate::{ graph::Node, @@ -47,10 +48,16 @@ impl Rod { } /// Get a node from the database - pub async fn get(&self, key: &str) -> Result, StoreError> { + pub async fn get<'a, K: Into>>(&self, key: K) -> Result, StoreError> { let this = &self.inner; + let key = key.into(); - let id = if let Some(id) = this.store.get_id(key).await?.flatten() { + let ulid = match key { + DbIndex::Str(s) => this.store.get_id(s).await?.flatten(), + DbIndex::Ulid(id) => Some(id.clone()), + }; + + let id = if let Some(id) = ulid { id } else { return Ok(None); @@ -73,3 +80,20 @@ impl Rod { Ok(()) } } + +pub enum DbIndex<'a> { + Str(&'a str), + Ulid(&'a Ulid), +} + +impl<'a> From<&'a str> for DbIndex<'a> { + fn from(s: &'a str) -> Self { + DbIndex::Str(s) + } +} + +impl<'a> From<&'a Ulid> for DbIndex<'a> { + fn from(id: &'a Ulid) -> Self { + DbIndex::Ulid(id) + } +} diff --git a/src/graph.rs b/src/graph.rs index f5f9770..b206061 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -30,7 +30,7 @@ pub struct Field { } /// A value represents the different data types that a field value can take -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub enum Value { /// An empty value None, @@ -49,8 +49,14 @@ pub enum Value { } mod impls { + use crate::{engine::Rod, store::StoreError}; + use super::*; + // + // Node + // + impl Default for Node { fn default() -> Self { Self { @@ -71,8 +77,20 @@ mod impls { fields: fields.into_iter().collect(), } } + + pub fn get(&self, key: &str) -> Option<&Value> { + self.fields.get(key).map(|x| &x.value) + } + + pub fn set>(&mut self, key: &str, value: V) { + self.fields.insert(key.into(), Field::new(value.into())); + } } + // + // Field + // + impl Deref for Field { type Target = Value; @@ -80,4 +98,47 @@ mod impls { &self.value } } + + // + // Value + // + + impl Value { + /// Return whether or not the value is [`Value::None`] + pub fn is_none(&self) -> bool { + self == &Value::None + } + + /// Get the value as an ID + pub fn as_id(&self) -> Option<&Ulid> { + if let Value::Node(id) = self { + Some(id) + } else { + None + } + } + + /// If this value is an ID, get the node associated to the ID + pub async fn follow(&self, rod: &Rod) -> Result, StoreError> { + let id = if let Value::Node(id) = self { + id + } else { + return Ok(None); + }; + + rod.get(id).await + } + } + + impl From for Value { + fn from(s: String) -> Self { + Self::String(s) + } + } + + impl From<&Node> for Value { + fn from(n: &Node) -> Self { + Self::Node(n.id.clone()) + } + } } diff --git a/src/store.rs b/src/store.rs index 214c979..439fca7 100644 --- a/src/store.rs +++ b/src/store.rs @@ -27,10 +27,14 @@ pub async fn get_default_store() -> Result { store } -/// A simple, [`Node`] storage interface +/// A [`Node`] storage interface /// /// [`Store`] is designed to be implemented over any persistant storage interface such as the -/// filesystem, S3, browser LocalStorage or IndexedDB, etc. +/// filesystem, S3, browser LocalStorage or IndexedDB, etc. but is also responsible for any +/// in-memory caching or buffering that may need to be done to optimize for performance. +/// +/// Callers will not be expected to have to manually flush do disk or other permanant storage, but +/// it will be the responsibility of the store. #[async_trait::async_trait] pub trait Store { /// Get a node from the store using it's ULID diff --git a/src/store/native.rs b/src/store/native.rs index bc970a4..2a91822 100644 --- a/src/store/native.rs +++ b/src/store/native.rs @@ -58,8 +58,8 @@ impl SimpleFsStore { self.node_dir.join(key.to_string()) } - fn id_path(&self, id: &str) -> PathBuf { - self.id_dir.join(id.to_string()) + fn id_path(&self, key: &str) -> PathBuf { + self.id_dir.join(base64::encode(key)) } } From c9448507d58d9a6de18e91664dee8a814e6f367e Mon Sep 17 00:00:00 2001 From: Zicklag Date: Sat, 25 Sep 2021 13:22:34 -0500 Subject: [PATCH 10/12] Further Improve API Ergonomics --- Cargo.lock | 7 +++ Cargo.toml | 1 + playground/src/main.rs | 47 ++++++++------- src/crdt.rs | 2 +- src/engine.rs | 131 ++++++++++++++++++++++++++++++++++++++--- src/graph.rs | 91 ++++++++++++---------------- src/lib.rs | 6 ++ 7 files changed, 203 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7324e7f..f32de9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -622,6 +622,7 @@ dependencies = [ "scc", "serde", "serde_json", + "tap", "thiserror", "tracing", "ulid", @@ -719,6 +720,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "thiserror" version = "1.0.29" diff --git a/Cargo.toml b/Cargo.toml index 5dba4cc..4099fce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ base64 = { version = "0.13.0", optional = true } # Async async-trait = "0.1.51" +tap = "1.0.1" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Async diff --git a/playground/src/main.rs b/playground/src/main.rs index e043146..dc46414 100644 --- a/playground/src/main.rs +++ b/playground/src/main.rs @@ -1,7 +1,7 @@ use std::process; use tracing as trc; -use rod::engine::Rod; +use rod::prelude::*; use futures_lite::future; @@ -20,25 +20,32 @@ async fn start() -> anyhow::Result<()> { let rod = &Rod::new().await?; - { - use rod::graph::Node; - let mut mary = Node::new(); - mary.set("name", "Mary".to_string()); - - let mut john = Node::new(); - john.set("name", "John".to_string()); - john.set("wife", &mary); - - rod.put("users/john", john).await?; - rod.put("users/mary", mary).await?; - } - - let node2 = dbg!(rod.get("users/john").await?).unwrap(); - - dbg!(node2.get("name")); - let wife = node2.get("wife").unwrap().follow(rod).await?.unwrap(); - - dbg!(wife); + let mary = rod + .get("users/mary") + .await? + .tap_mut(|x| x.set("name", "Mary".to_string())) + .tap_mut(|x| x.set("age", 32)); + rod.put("users/mary", &mary).await?; + + rod.get("users/john") + .await? + .tap_mut(|x| x.set("name", "John".to_string())) + .tap_mut(|x| x.set("wife", &mary)) + .pipe(|x| rod.put("users/john", x)) + .await?; + + let wife_name = rod + .get("users/john") + .await? + .get("wife") + .unwrap() + .follow() + .await? + .get("name") + .unwrap() + .owned(); + + dbg!(wife_name); Ok(()) } diff --git a/src/crdt.rs b/src/crdt.rs index c33a358..436ae7d 100644 --- a/src/crdt.rs +++ b/src/crdt.rs @@ -46,7 +46,7 @@ impl Field { // If the new field has the same timestamp if field.updated_at == self.updated_at { - match self.lexical_cmp(field) { + match self.value.lexical_cmp(&field.value) { // Totally equal, do nothing cmp::Ordering::Equal => return, // Keep our value, do nothing diff --git a/src/engine.rs b/src/engine.rs index 750e224..029bba0 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -6,7 +6,7 @@ use tracing as trc; use ulid::Ulid; use crate::{ - graph::Node, + graph::{Field, Node, Value}, store::{get_default_store, Store, StoreError}, }; @@ -48,7 +48,7 @@ impl Rod { } /// Get a node from the database - pub async fn get<'a, K: Into>>(&self, key: K) -> Result, StoreError> { + pub async fn get<'a, K: Into>>(&self, key: K) -> Result { let this = &self.inner; let key = key.into(); @@ -60,27 +60,140 @@ impl Rod { let id = if let Some(id) = ulid { id } else { - return Ok(None); + return Ok(NodeProxy::new(self, Node::new()).await?); }; if let Some(node) = this.store.get_node(&id).await? { - return Ok(Some(node)); + return Ok(NodeProxy::new(self, node).await?); } else { - return Ok(None); + return Ok(NodeProxy::new(self, Node::new()).await?); } } - pub async fn put(&self, key: &str, node: Node) -> Result<(), StoreError> { + /// Put a node into the database + pub async fn put>(&self, key: &str, node: N) -> Result<(), StoreError> { let this = &self.inner; + let node: &Node = node.as_ref(); - let id = node.id.clone(); - this.store.set_id(key, Some(id)).await?; - this.store.put_node(node).await?; + this.store.set_id(key, Some(node.id.clone())).await?; + this.store.put_node(node.clone()).await?; Ok(()) } } +/// A node loaded from the database with mutators that can be used to modify the node and +/// synchronize it back to the database +/// +/// Having a [`NodeProxy`] does **not** represent exclusive access to the node data. This means that +/// there is nothing stopping another thread from modifying the node while you have a [`NodeProxy`] +pub struct NodeProxy { + rod: Rod, + node: Node, +} + +impl AsRef for NodeProxy { + fn as_ref(&self) -> &Node { + &self.node + } +} + +impl std::fmt::Debug for NodeProxy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NodeProxy") + .field("rod", &"Rod") + .field("node", &self.node) + .finish() + } +} + +impl NodeProxy { + async fn new(rod: &Rod, node: Node) -> Result { + rod.inner.store.put_node(node.clone()).await?; + + Ok(Self { + rod: rod.clone(), + node, + }) + } + + /// Get a node field + pub fn get(&self, key: &str) -> Option { + self.node + .fields + .get(key) + .map(|field| ValueRef::new(&self.rod, &field.value)) + } + + /// Set a node field + /// + /// > **Note:** The changes to the node are not persisted or synchronized until you call + /// > [`NodeRef::save()`] + pub fn set>(&mut self, key: &str, value: V) { + self.node + .fields + .insert(key.to_string(), Field::new(value.into())); + } +} + +impl Into for &NodeProxy { + fn into(self) -> Value { + Value::Node(self.node.id.clone()) + } +} + +pub struct ValueRef<'a> { + rod: Rod, + value: &'a Value, +} + +impl<'a> std::fmt::Debug for ValueRef<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ValueRef") + .field("rod", &"Rod") + .field("value", &self.value) + .finish() + } +} + +impl<'a> ValueRef<'a> { + fn new(rod: &Rod, value: &'a Value) -> Self { + Self { + rod: rod.clone(), + value, + } + } + + /// If this value is a reference to another node, get the node that it references from the + /// database + pub async fn follow(&self) -> Result { + let id = if let Value::Node(id) = self.value { + id + } else { + return Ok(NodeProxy::new(&self.rod, Node::new()).await?); + }; + + if let Some(node) = self.rod.inner.store.get_node(id).await? { + Ok(NodeProxy::new(&self.rod, node).await?) + } else { + Ok(NodeProxy::new(&self.rod, Node::new()).await?) + } + } + + /// Clone the referenced [`Value`] and return it + pub fn owned(&self) -> Value { + self.value.clone() + } +} + +impl<'a> std::ops::Deref for ValueRef<'a> { + type Target = Value; + + fn deref(&self) -> &Self::Target { + &self.value + } +} + pub enum DbIndex<'a> { Str(&'a str), Ulid(&'a Ulid), diff --git a/src/graph.rs b/src/graph.rs index b206061..20b2bf7 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -1,6 +1,6 @@ //! Data structures used to build the rod data graph -use std::{collections::HashMap, ops::Deref}; +use std::collections::HashMap; use ulid::Ulid; @@ -8,7 +8,7 @@ pub mod repr; /// [`Node`] is the core data structure in the data graph /// -/// A grpah is made up of a collection of nodes +/// A graph is made up of a collection of nodes #[derive(Debug, Clone)] pub struct Node { /// The node's universally unique identifier @@ -49,8 +49,6 @@ pub enum Value { } mod impls { - use crate::{engine::Rod, store::StoreError}; - use super::*; // @@ -70,72 +68,61 @@ mod impls { pub fn new() -> Self { Self::default() } - - pub fn new_with_fields(fields: Vec<(String, Field)>) -> Self { - Self { - id: Ulid::new(), - fields: fields.into_iter().collect(), - } - } - - pub fn get(&self, key: &str) -> Option<&Value> { - self.fields.get(key).map(|x| &x.value) - } - - pub fn set>(&mut self, key: &str, value: V) { - self.fields.insert(key.into(), Field::new(value.into())); - } - } - - // - // Field - // - - impl Deref for Field { - type Target = Value; - - fn deref(&self) -> &Self::Target { - &self.value - } } // // Value // - impl Value { - /// Return whether or not the value is [`Value::None`] - pub fn is_none(&self) -> bool { - self == &Value::None + impl From<()> for Value { + fn from(_: ()) -> Self { + Self::None } + } - /// Get the value as an ID - pub fn as_id(&self) -> Option<&Ulid> { - if let Value::Node(id) = self { - Some(id) - } else { - None + macro_rules! from_int { + ($int:ident) => { + impl From<$int> for Value { + fn from(i: $int) -> Self { + Self::Int(i as i64) + } } - } - - /// If this value is an ID, get the node associated to the ID - pub async fn follow(&self, rod: &Rod) -> Result, StoreError> { - let id = if let Value::Node(id) = self { - id - } else { - return Ok(None); - }; + }; + } - rod.get(id).await - } + macro_rules! from_float { + ($float:ident) => { + impl From<$float> for Value { + fn from(f: $float) -> Self { + Self::Float(f as f64) + } + } + }; } + from_int!(i8); + from_int!(i16); + from_int!(i32); + from_int!(i64); + from_int!(u8); + from_int!(u16); + from_int!(u32); + from_int!(u64); + from_float!(f32); + from_float!(f64); + impl From for Value { fn from(s: String) -> Self { Self::String(s) } } + impl From> for Value { + fn from(b: Vec) -> Self { + Self::Binary(b) + } + } + impl From<&Node> for Value { fn from(n: &Node) -> Self { Self::Node(n.id.clone()) diff --git a/src/lib.rs b/src/lib.rs index a028aab..efd8cc1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,4 +12,10 @@ pub mod graph; pub mod protocol; pub mod store; +pub use tap; pub use ulid::Ulid; + +pub mod prelude { + pub use crate::engine::*; + pub use tap::prelude::*; +} From 6f9dc652726cfc94a789e841ac53b0a80e992fbb Mon Sep 17 00:00:00 2001 From: Zicklag Date: Sat, 25 Sep 2021 13:28:37 -0500 Subject: [PATCH 11/12] Fix Some Docs Problems --- src/engine.rs | 3 +-- src/lib.rs | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/engine.rs b/src/engine.rs index 029bba0..0a55e8d 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -127,8 +127,7 @@ impl NodeProxy { /// Set a node field /// - /// > **Note:** The changes to the node are not persisted or synchronized until you call - /// > [`NodeRef::save()`] + /// > **Note:** The changes to the node are not persisted or synchronized unless you call [`Rod::put()`] pub fn set>(&mut self, key: &str, value: V) { self.node .fields diff --git a/src/lib.rs b/src/lib.rs index efd8cc1..d57f7b1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,9 +12,11 @@ pub mod graph; pub mod protocol; pub mod store; +#[doc(inline)] pub use tap; pub use ulid::Ulid; +/// The Rod prelude pub mod prelude { pub use crate::engine::*; pub use tap::prelude::*; From 18a2c71c6ce9886fee8dbf36e90936af021c7163 Mon Sep 17 00:00:00 2001 From: Zicklag Date: Sat, 25 Sep 2021 14:30:37 -0500 Subject: [PATCH 12/12] Fix The CRDT --- playground/src/main.rs | 8 +++++++- src/crdt.rs | 40 ++++++++++++++++++++++++++++++---------- src/engine.rs | 17 ++++++++++++++--- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/playground/src/main.rs b/playground/src/main.rs index dc46414..1cc0587 100644 --- a/playground/src/main.rs +++ b/playground/src/main.rs @@ -23,7 +23,11 @@ async fn start() -> anyhow::Result<()> { let mary = rod .get("users/mary") .await? - .tap_mut(|x| x.set("name", "Mary".to_string())) + .tap_mut(|x| { + if x.get("name").is_none() { + x.set("name", "Mary".to_string()) + } + }) .tap_mut(|x| x.set("age", 32)); rod.put("users/mary", &mary).await?; @@ -47,6 +51,8 @@ async fn start() -> anyhow::Result<()> { dbg!(wife_name); + dbg!(rod.get("users/mary").await?); + Ok(()) } diff --git a/src/crdt.rs b/src/crdt.rs index 436ae7d..d26ba38 100644 --- a/src/crdt.rs +++ b/src/crdt.rs @@ -3,7 +3,7 @@ use std::{cmp, mem, time::SystemTime}; -use crate::graph::{Field, Value}; +use crate::graph::{Field, Node, Value}; /// Trait implemented by structs that can be lexically sorted pub trait LexicalCmp { @@ -11,6 +11,18 @@ pub trait LexicalCmp { fn lexical_cmp(&self, other: &Self) -> cmp::Ordering; } +impl Node { + pub fn merge_into(self, other_node: &mut Node) { + for (key, field_value) in self.fields { + if let Some(other_field_value) = other_node.fields.get_mut(&key) { + other_field_value.merge_with(&field_value); + } else { + other_node.fields.insert(key, field_value); + } + } + } +} + /// If an update comes in that is more than this amount of time in the future, we will assume that /// the node that sent the update is lying and trying to make it's update take precedence over the /// current value of the field. @@ -19,6 +31,7 @@ const FUTURE_UPDATE_THREASHOLD: f64 = 600.0; impl Field { pub fn new(value: Value) -> Self { Self { + // TODO(perf): Worry about avoiding the syscalls involved with getting the current time. updated_at: SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .expect("Could not get system time") @@ -55,22 +68,29 @@ impl Field { cmp::Ordering::Greater => self.value = field.value.clone(), } - // If the other field is and older update than the one we have, just ignore it + // If the other field is an older update than the one we have, just ignore it } else if field.updated_at < self.updated_at { return; - // If the other field is in the future - } else if field.updated_at > current_time { + // If the other field is later than our current value + } else if field.updated_at > self.updated_at { + // If the field was updated before or at the current time + if field.updated_at <= current_time { + *self = field.clone(); + // If the field is too far in the future, ignore it - if field.updated_at - current_time > FUTURE_UPDATE_THREASHOLD { + } else if field.updated_at - current_time > FUTURE_UPDATE_THREASHOLD { return; - } - // Wait to apply this update until later - unimplemented!( - "Use async logic to apply this update once our system clock \ + // If field was updated in the fugure, but not too far in the future, schedule a task to + // update the field at that time in the future + } else { + // Wait to apply this update until later + unimplemented!( + "Use async logic to apply this update once our system clock \ reaches the future time" - ); + ); + } } else { unreachable!() } diff --git a/src/engine.rs b/src/engine.rs index 0a55e8d..a8c61dd 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -73,10 +73,20 @@ impl Rod { /// Put a node into the database pub async fn put>(&self, key: &str, node: N) -> Result<(), StoreError> { let this = &self.inner; - let node: &Node = node.as_ref(); + let new_node = node.as_ref().clone(); + let node_id = new_node.id.clone(); - this.store.set_id(key, Some(node.id.clone())).await?; - this.store.put_node(node.clone()).await?; + let node_to_update = this.store.get_node(&new_node.id).await?; + + let new_node = if let Some(mut node) = node_to_update { + new_node.merge_into(&mut node); + node + } else { + new_node.clone() + }; + + this.store.put_node(new_node).await?; + this.store.set_id(key, Some(node_id.clone())).await?; Ok(()) } @@ -193,6 +203,7 @@ impl<'a> std::ops::Deref for ValueRef<'a> { } } +/// Can be used in [`Rod::get()`], but isn't usually needed by users directly pub enum DbIndex<'a> { Str(&'a str), Ulid(&'a Ulid),