mirror of
https://github.com/loro-dev/loro.git
synced 2024-11-24 04:09:42 +00:00
feat: Stable JSON representation for history (#368)
--------- Co-authored-by: Zixuan Chen <remch183@outlook.com>
This commit is contained in:
Leon Zhao
GitHub
parent
d2973df859
commit
2df2a52b05
40 changed files with 2627 additions and 373 deletions
183
Cargo.lock
generated
183
Cargo.lock
generated
|
|
@ -46,9 +46,9 @@ checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc"
|
|||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.83"
|
||||
version = "1.0.86"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3"
|
||||
checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
|
||||
|
||||
[[package]]
|
||||
name = "append-only-bytes"
|
||||
|
|
@ -638,17 +638,6 @@ version = "1.0.7"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
||||
|
||||
[[package]]
|
||||
name = "fractional_index"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=cd04b27d65128420f6daaf792be9ef511483de99#cd04b27d65128420f6daaf792be9ef511483de99"
|
||||
dependencies = [
|
||||
"imbl",
|
||||
"rand",
|
||||
"serde",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fractional_index"
|
||||
version = "2.0.1"
|
||||
|
|
@ -672,9 +661,10 @@ dependencies = [
|
|||
"fxhash",
|
||||
"itertools 0.12.1",
|
||||
"loro 0.16.2",
|
||||
"loro 0.5.1",
|
||||
"loro-common 0.5.1",
|
||||
"loro 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"loro-common 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"rand",
|
||||
"serde_json",
|
||||
"tabled 0.10.0",
|
||||
"tracing",
|
||||
"tracing-chrome",
|
||||
|
|
@ -946,9 +936,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.63"
|
||||
version = "0.3.69"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790"
|
||||
checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
|
||||
dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
|
@ -990,19 +980,6 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "loro"
|
||||
version = "0.5.1"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=cd04b27d65128420f6daaf792be9ef511483de99#cd04b27d65128420f6daaf792be9ef511483de99"
|
||||
dependencies = [
|
||||
"either",
|
||||
"enum-as-inner 0.6.0",
|
||||
"generic-btree",
|
||||
"loro-delta 0.5.1",
|
||||
"loro-internal 0.5.1",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "loro"
|
||||
version = "0.16.2"
|
||||
|
|
@ -1020,19 +997,16 @@ dependencies = [
|
|||
]
|
||||
|
||||
[[package]]
|
||||
name = "loro-common"
|
||||
version = "0.5.1"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=cd04b27d65128420f6daaf792be9ef511483de99#cd04b27d65128420f6daaf792be9ef511483de99"
|
||||
name = "loro"
|
||||
version = "0.16.2"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9"
|
||||
dependencies = [
|
||||
"arbitrary",
|
||||
"either",
|
||||
"enum-as-inner 0.6.0",
|
||||
"fxhash",
|
||||
"loro-rle 0.5.1",
|
||||
"nonmax",
|
||||
"serde",
|
||||
"serde_columnar",
|
||||
"string_cache",
|
||||
"thiserror",
|
||||
"generic-btree",
|
||||
"loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"loro-internal 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1053,15 +1027,19 @@ dependencies = [
|
|||
]
|
||||
|
||||
[[package]]
|
||||
name = "loro-delta"
|
||||
version = "0.5.1"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=cd04b27d65128420f6daaf792be9ef511483de99#cd04b27d65128420f6daaf792be9ef511483de99"
|
||||
name = "loro-common"
|
||||
version = "0.16.2"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"enum-as-inner 0.5.1",
|
||||
"generic-btree",
|
||||
"heapless 0.8.0",
|
||||
"tracing",
|
||||
"arbitrary",
|
||||
"enum-as-inner 0.6.0",
|
||||
"fxhash",
|
||||
"loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"nonmax",
|
||||
"serde",
|
||||
"serde_columnar",
|
||||
"string_cache",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1081,37 +1059,14 @@ dependencies = [
|
|||
]
|
||||
|
||||
[[package]]
|
||||
name = "loro-internal"
|
||||
version = "0.5.1"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=cd04b27d65128420f6daaf792be9ef511483de99#cd04b27d65128420f6daaf792be9ef511483de99"
|
||||
name = "loro-delta"
|
||||
version = "0.16.2"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9"
|
||||
dependencies = [
|
||||
"append-only-bytes",
|
||||
"arref",
|
||||
"either",
|
||||
"arrayvec",
|
||||
"enum-as-inner 0.5.1",
|
||||
"enum_dispatch",
|
||||
"fractional_index 0.1.0",
|
||||
"fxhash",
|
||||
"generic-btree",
|
||||
"getrandom",
|
||||
"im",
|
||||
"itertools 0.12.1",
|
||||
"leb128",
|
||||
"loro-common 0.5.1",
|
||||
"loro-delta 0.5.1",
|
||||
"loro-rle 0.5.1",
|
||||
"md5",
|
||||
"num",
|
||||
"num-derive",
|
||||
"num-traits",
|
||||
"once_cell",
|
||||
"postcard",
|
||||
"rand",
|
||||
"serde",
|
||||
"serde_columnar",
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
"thiserror",
|
||||
"heapless 0.8.0",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
|
|
@ -1142,7 +1097,7 @@ dependencies = [
|
|||
"loro-common 0.16.2",
|
||||
"loro-delta 0.16.2",
|
||||
"loro-rle 0.16.2",
|
||||
"loro_fractional_index",
|
||||
"loro_fractional_index 0.16.2",
|
||||
"md5",
|
||||
"miniz_oxide 0.7.1",
|
||||
"num",
|
||||
|
|
@ -1154,7 +1109,7 @@ dependencies = [
|
|||
"proptest-derive",
|
||||
"rand",
|
||||
"serde",
|
||||
"serde-wasm-bindgen",
|
||||
"serde-wasm-bindgen 0.5.0",
|
||||
"serde_columnar",
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
|
|
@ -1167,16 +1122,38 @@ dependencies = [
|
|||
]
|
||||
|
||||
[[package]]
|
||||
name = "loro-rle"
|
||||
version = "0.5.1"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=cd04b27d65128420f6daaf792be9ef511483de99#cd04b27d65128420f6daaf792be9ef511483de99"
|
||||
name = "loro-internal"
|
||||
version = "0.16.2"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9"
|
||||
dependencies = [
|
||||
"append-only-bytes",
|
||||
"arref",
|
||||
"enum-as-inner 0.6.0",
|
||||
"either",
|
||||
"enum-as-inner 0.5.1",
|
||||
"enum_dispatch",
|
||||
"fxhash",
|
||||
"generic-btree",
|
||||
"getrandom",
|
||||
"im",
|
||||
"itertools 0.12.1",
|
||||
"leb128",
|
||||
"loro-common 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"loro_fractional_index 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"md5",
|
||||
"num",
|
||||
"num-derive",
|
||||
"num-traits",
|
||||
"once_cell",
|
||||
"postcard",
|
||||
"rand",
|
||||
"serde",
|
||||
"serde_columnar",
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
"thiserror",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1196,6 +1173,19 @@ dependencies = [
|
|||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "loro-rle"
|
||||
version = "0.16.2"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9"
|
||||
dependencies = [
|
||||
"append-only-bytes",
|
||||
"arref",
|
||||
"enum-as-inner 0.6.0",
|
||||
"fxhash",
|
||||
"num",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "loro-thunderdome"
|
||||
version = "0.6.2"
|
||||
|
|
@ -1212,7 +1202,8 @@ dependencies = [
|
|||
"loro-internal 0.16.2",
|
||||
"loro-rle 0.16.2",
|
||||
"serde",
|
||||
"serde-wasm-bindgen",
|
||||
"serde-wasm-bindgen 0.6.5",
|
||||
"serde_json",
|
||||
"tracing",
|
||||
"tracing-wasm",
|
||||
"wasm-bindgen",
|
||||
|
|
@ -1224,7 +1215,18 @@ name = "loro_fractional_index"
|
|||
version = "0.16.2"
|
||||
dependencies = [
|
||||
"criterion 0.5.1",
|
||||
"fractional_index 2.0.1",
|
||||
"fractional_index",
|
||||
"imbl",
|
||||
"rand",
|
||||
"serde",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "loro_fractional_index"
|
||||
version = "0.16.2"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9"
|
||||
dependencies = [
|
||||
"imbl",
|
||||
"rand",
|
||||
"serde",
|
||||
|
|
@ -1844,6 +1846,17 @@ dependencies = [
|
|||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde-wasm-bindgen"
|
||||
version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"serde",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_columnar"
|
||||
version = "0.3.4"
|
||||
|
|
|
|||
|
|
@ -16,12 +16,14 @@ resolver = "2"
|
|||
|
||||
[workspace.dependencies]
|
||||
enum_dispatch = "0.3.11"
|
||||
debug-log = { version = "0.3.1", features = [] }
|
||||
enum-as-inner = "0.5.1"
|
||||
fxhash = "0.2.1"
|
||||
tracing = { version = "0.1", features = [
|
||||
"max_level_debug",
|
||||
"release_max_level_warn",
|
||||
] }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
serde_columnar = { version = "0.3.4" }
|
||||
itertools = "0.12.1"
|
||||
smallvec = { version = "1.8.0", features = ["serde"] }
|
||||
|
|
|
|||
|
|
@ -13,4 +13,4 @@ loro-common = { path = "../loro-common" }
|
|||
enum-as-inner = "0.5.1"
|
||||
flate2 = "1.0.25"
|
||||
rand = "0.8.5"
|
||||
serde_json = "1.0.89"
|
||||
serde_json = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ bench-utils = { path = "../bench-utils" }
|
|||
loro = { path = "../loro" }
|
||||
tabled = "0.15.0"
|
||||
arbitrary = { version = "1.3.0", features = ["derive"] }
|
||||
serde_json = "1.0.111"
|
||||
serde_json = { workspace = true }
|
||||
tracing = "0.1.40"
|
||||
criterion = "0.4.0"
|
||||
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@ keywords = ["crdt", "local-first"]
|
|||
|
||||
[dependencies]
|
||||
imbl = "^3.0"
|
||||
smallvec = "^1.13"
|
||||
serde = { version = "^1.0", features = ["derive", "rc"], optional = true }
|
||||
smallvec = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive", "rc"], optional = true }
|
||||
rand = { version = "^0.8" }
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
use std::{fmt::Display, sync::Arc};
|
||||
use std::{
|
||||
fmt::{Display, Write},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
#[cfg(feature = "serde")]
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
|
@ -27,6 +30,16 @@ impl FractionalIndex {
|
|||
FractionalIndex(Arc::new(bytes))
|
||||
}
|
||||
|
||||
pub fn from_hex_string<T: AsRef<str>>(str: T) -> Self {
|
||||
let s = str.as_ref();
|
||||
let mut bytes = Vec::with_capacity(s.len() / 2);
|
||||
for i in 0..s.len() / 2 {
|
||||
let byte = u8::from_str_radix(&s[i * 2..i * 2 + 2], 16).unwrap();
|
||||
bytes.push(byte);
|
||||
}
|
||||
FractionalIndex::from_bytes(bytes)
|
||||
}
|
||||
|
||||
pub fn as_bytes(&self) -> &[u8] {
|
||||
&self.0
|
||||
}
|
||||
|
|
@ -61,7 +74,7 @@ pub(crate) fn new_after(bytes: &[u8]) -> Vec<u8> {
|
|||
}
|
||||
|
||||
pub(crate) fn new_between(left: &[u8], right: &[u8], extra_capacity: usize) -> Option<Vec<u8>> {
|
||||
let shorter_len = left.len().min(right.len()) - 1;
|
||||
let shorter_len = left.len().min(right.len());
|
||||
for i in 0..shorter_len {
|
||||
if left[i] < right[i] - 1 {
|
||||
let mut ans: Vec<u8> = left[0..=i].into();
|
||||
|
|
@ -183,19 +196,9 @@ impl Display for FractionalIndex {
|
|||
}
|
||||
}
|
||||
|
||||
const HEX_CHARS: &[u8] = b"0123456789abcdef";
|
||||
|
||||
pub fn byte_to_hex(byte: u8) -> String {
|
||||
let mut s = String::new();
|
||||
s.push(HEX_CHARS[(byte >> 4) as usize] as char);
|
||||
s.push(HEX_CHARS[(byte & 0xf) as usize] as char);
|
||||
s
|
||||
}
|
||||
|
||||
pub fn bytes_to_hex(bytes: &[u8]) -> String {
|
||||
let mut s = String::with_capacity(bytes.len() * 2);
|
||||
for byte in bytes {
|
||||
s.push_str(&byte_to_hex(*byte));
|
||||
}
|
||||
s
|
||||
bytes.iter().fold(String::new(), |mut output, b| {
|
||||
let _ = write!(output, "{b:02X}");
|
||||
output
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,13 +10,15 @@ publish = false
|
|||
loro-without-counter = { path = "../loro", package = "loro" }
|
||||
loro = { git = "https://github.com/loro-dev/loro.git", features = [
|
||||
"counter",
|
||||
], rev = "cd04b27d65128420f6daaf792be9ef511483de99" }
|
||||
], rev = "83938290ab2666d85c0c72169127611585a05cf9" }
|
||||
loro-common = { git = "https://github.com/loro-dev/loro.git", features = [
|
||||
"counter",
|
||||
], rev = "cd04b27d65128420f6daaf792be9ef511483de99" }
|
||||
], rev = "83938290ab2666d85c0c72169127611585a05cf9" }
|
||||
# loro = { path = "../loro", package = "loro", features = ["counter"] }
|
||||
# loro-common = { path = "../loro-common", features = ["counter"] }
|
||||
# loro-without-counter = { git = "https://github.com/loro-dev/loro.git", branch = "zxch3n/loro-560-undoredo", package = "loro" }
|
||||
# loro-common = { path = "../loro-common", package = "loro-common", features = [
|
||||
# "counter",
|
||||
# ] }
|
||||
# loro-without-counter = { git = "https://github.com/loro-dev/loro.git", rev = "eb6daf4f064238cbc5c3d357615f5ed73767e98c", package = "loro" }
|
||||
fxhash = { workspace = true }
|
||||
enum_dispatch = { workspace = true }
|
||||
enum-as-inner = { workspace = true }
|
||||
|
|
@ -33,3 +35,4 @@ dev-utils = { path = "../dev-utils" }
|
|||
tracing-subscriber = "0.3.18"
|
||||
tracing-chrome = "0.7.1"
|
||||
color-backtrace = "0.6.1"
|
||||
serde_json = "1"
|
||||
|
|
|
|||
106
crates/fuzz/fuzz/Cargo.lock
generated
106
crates/fuzz/fuzz/Cargo.lock
generated
|
|
@ -206,27 +206,6 @@ version = "1.0.7"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
||||
|
||||
[[package]]
|
||||
name = "fractional_index"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"imbl",
|
||||
"rand",
|
||||
"serde",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fractional_index"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125#0dade6bc0fb574a8190db2aa80c83a479f62e125"
|
||||
dependencies = [
|
||||
"imbl",
|
||||
"rand",
|
||||
"serde",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fuzz"
|
||||
version = "0.1.0"
|
||||
|
|
@ -236,9 +215,9 @@ dependencies = [
|
|||
"enum_dispatch",
|
||||
"fxhash",
|
||||
"itertools 0.12.1",
|
||||
"loro 0.5.1",
|
||||
"loro 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)",
|
||||
"loro-common 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)",
|
||||
"loro 0.16.2",
|
||||
"loro 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"loro-common 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"rand",
|
||||
"tabled",
|
||||
"tracing",
|
||||
|
|
@ -444,37 +423,37 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "loro"
|
||||
version = "0.5.1"
|
||||
version = "0.16.2"
|
||||
dependencies = [
|
||||
"either",
|
||||
"enum-as-inner 0.6.0",
|
||||
"generic-btree",
|
||||
"loro-delta 0.5.1",
|
||||
"loro-internal 0.5.1",
|
||||
"loro-delta 0.16.2",
|
||||
"loro-internal 0.16.2",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "loro"
|
||||
version = "0.5.1"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125#0dade6bc0fb574a8190db2aa80c83a479f62e125"
|
||||
version = "0.16.2"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9"
|
||||
dependencies = [
|
||||
"either",
|
||||
"enum-as-inner 0.6.0",
|
||||
"generic-btree",
|
||||
"loro-delta 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)",
|
||||
"loro-internal 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)",
|
||||
"loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"loro-internal 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "loro-common"
|
||||
version = "0.5.1"
|
||||
version = "0.16.2"
|
||||
dependencies = [
|
||||
"arbitrary",
|
||||
"enum-as-inner 0.6.0",
|
||||
"fxhash",
|
||||
"loro-rle 0.5.1",
|
||||
"loro-rle 0.16.2",
|
||||
"nonmax",
|
||||
"serde",
|
||||
"serde_columnar",
|
||||
|
|
@ -484,13 +463,13 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "loro-common"
|
||||
version = "0.5.1"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125#0dade6bc0fb574a8190db2aa80c83a479f62e125"
|
||||
version = "0.16.2"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9"
|
||||
dependencies = [
|
||||
"arbitrary",
|
||||
"enum-as-inner 0.6.0",
|
||||
"fxhash",
|
||||
"loro-rle 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)",
|
||||
"loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"nonmax",
|
||||
"serde",
|
||||
"serde_columnar",
|
||||
|
|
@ -500,7 +479,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "loro-delta"
|
||||
version = "0.5.1"
|
||||
version = "0.16.2"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"enum-as-inner 0.5.1",
|
||||
|
|
@ -511,8 +490,8 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "loro-delta"
|
||||
version = "0.5.1"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125#0dade6bc0fb574a8190db2aa80c83a479f62e125"
|
||||
version = "0.16.2"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"enum-as-inner 0.5.1",
|
||||
|
|
@ -523,23 +502,23 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "loro-internal"
|
||||
version = "0.5.1"
|
||||
version = "0.16.2"
|
||||
dependencies = [
|
||||
"append-only-bytes",
|
||||
"arref",
|
||||
"either",
|
||||
"enum-as-inner 0.5.1",
|
||||
"enum_dispatch",
|
||||
"fractional_index 0.1.0",
|
||||
"fxhash",
|
||||
"generic-btree",
|
||||
"getrandom",
|
||||
"im",
|
||||
"itertools 0.12.1",
|
||||
"leb128",
|
||||
"loro-common 0.5.1",
|
||||
"loro-delta 0.5.1",
|
||||
"loro-rle 0.5.1",
|
||||
"loro-common 0.16.2",
|
||||
"loro-delta 0.16.2",
|
||||
"loro-rle 0.16.2",
|
||||
"loro_fractional_index 0.16.2",
|
||||
"md5",
|
||||
"num",
|
||||
"num-derive",
|
||||
|
|
@ -557,24 +536,24 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "loro-internal"
|
||||
version = "0.5.1"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125#0dade6bc0fb574a8190db2aa80c83a479f62e125"
|
||||
version = "0.16.2"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9"
|
||||
dependencies = [
|
||||
"append-only-bytes",
|
||||
"arref",
|
||||
"either",
|
||||
"enum-as-inner 0.5.1",
|
||||
"enum_dispatch",
|
||||
"fractional_index 0.1.0 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)",
|
||||
"fxhash",
|
||||
"generic-btree",
|
||||
"getrandom",
|
||||
"im",
|
||||
"itertools 0.12.1",
|
||||
"leb128",
|
||||
"loro-common 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)",
|
||||
"loro-delta 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)",
|
||||
"loro-rle 0.5.1 (git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125)",
|
||||
"loro-common 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"loro_fractional_index 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9)",
|
||||
"md5",
|
||||
"num",
|
||||
"num-derive",
|
||||
|
|
@ -592,7 +571,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "loro-rle"
|
||||
version = "0.5.1"
|
||||
version = "0.16.2"
|
||||
dependencies = [
|
||||
"append-only-bytes",
|
||||
"arref",
|
||||
|
|
@ -604,8 +583,8 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "loro-rle"
|
||||
version = "0.5.1"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=0dade6bc0fb574a8190db2aa80c83a479f62e125#0dade6bc0fb574a8190db2aa80c83a479f62e125"
|
||||
version = "0.16.2"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9"
|
||||
dependencies = [
|
||||
"append-only-bytes",
|
||||
"arref",
|
||||
|
|
@ -621,6 +600,27 @@ version = "0.6.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f3d053a135388e6b1df14e8af1212af5064746e9b87a06a345a7a779ee9695a"
|
||||
|
||||
[[package]]
|
||||
name = "loro_fractional_index"
|
||||
version = "0.16.2"
|
||||
dependencies = [
|
||||
"imbl",
|
||||
"rand",
|
||||
"serde",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "loro_fractional_index"
|
||||
version = "0.16.2"
|
||||
source = "git+https://github.com/loro-dev/loro.git?rev=83938290ab2666d85c0c72169127611585a05cf9#83938290ab2666d85c0c72169127611585a05cf9"
|
||||
dependencies = [
|
||||
"imbl",
|
||||
"rand",
|
||||
"serde",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "md5"
|
||||
version = "0.7.0"
|
||||
|
|
|
|||
|
|
@ -197,20 +197,33 @@ impl CRDTFuzzer {
|
|||
info_span!("Attach", peer = j).in_scope(|| {
|
||||
b_doc.attach();
|
||||
});
|
||||
if (i + j) % 2 == 0 {
|
||||
info_span!("Updates", from = j, to = i).in_scope(|| {
|
||||
a_doc.import(&b_doc.export_from(&a_doc.oplog_vv())).unwrap();
|
||||
});
|
||||
info_span!("Updates", from = i, to = j).in_scope(|| {
|
||||
b_doc.import(&a_doc.export_from(&b_doc.oplog_vv())).unwrap();
|
||||
});
|
||||
} else {
|
||||
info_span!("Snapshot", from = i, to = j).in_scope(|| {
|
||||
b_doc.import(&a_doc.export_snapshot()).unwrap();
|
||||
});
|
||||
info_span!("Snapshot", from = j, to = i).in_scope(|| {
|
||||
a_doc.import(&b_doc.export_snapshot()).unwrap();
|
||||
});
|
||||
match (i + j) % 3 {
|
||||
0 => {
|
||||
info_span!("Updates", from = j, to = i).in_scope(|| {
|
||||
a_doc.import(&b_doc.export_from(&a_doc.oplog_vv())).unwrap();
|
||||
});
|
||||
info_span!("Updates", from = i, to = j).in_scope(|| {
|
||||
b_doc.import(&a_doc.export_from(&b_doc.oplog_vv())).unwrap();
|
||||
});
|
||||
}
|
||||
1 => {
|
||||
info_span!("Snapshot", from = i, to = j).in_scope(|| {
|
||||
b_doc.import(&a_doc.export_snapshot()).unwrap();
|
||||
});
|
||||
info_span!("Snapshot", from = j, to = i).in_scope(|| {
|
||||
a_doc.import(&b_doc.export_snapshot()).unwrap();
|
||||
});
|
||||
}
|
||||
_ => {
|
||||
info_span!("JsonFormat", from = i, to = j).in_scope(|| {
|
||||
let a_json = a_doc.export_json_updates(&b_doc.oplog_vv());
|
||||
b_doc.import_json_updates(a_json).unwrap();
|
||||
});
|
||||
info_span!("JsonFormat", from = j, to = i).in_scope(|| {
|
||||
let b_json = b_doc.export_json_updates(&a_doc.oplog_vv());
|
||||
a_doc.import_json_updates(b_json).unwrap();
|
||||
});
|
||||
}
|
||||
}
|
||||
a.check_eq(b);
|
||||
a.record_history();
|
||||
|
|
|
|||
91
crates/fuzz/tests/json.rs
Normal file
91
crates/fuzz/tests/json.rs
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
use fuzz::{
|
||||
actions::{ActionWrapper::*, GenericAction},
|
||||
crdt_fuzzer::{test_multi_sites, Action::*, FuzzTarget, FuzzValue::*},
|
||||
};
|
||||
use loro::ContainerType::*;
|
||||
|
||||
#[ctor::ctor]
|
||||
fn init() {
|
||||
dev_utils::setup_test_log();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_json() {
|
||||
let doc = loro::LoroDoc::new();
|
||||
let doc_with_unknown = loro_without_counter::LoroDoc::new();
|
||||
let counter = doc.get_counter("counter");
|
||||
counter.increment(5).unwrap();
|
||||
counter.increment(1).unwrap();
|
||||
// json format with counter
|
||||
let json = doc.export_json_updates(&Default::default());
|
||||
// Test1: old version import newer version json
|
||||
if doc_with_unknown
|
||||
.import_json_updates(serde_json::to_string(&json).unwrap())
|
||||
.is_ok()
|
||||
{
|
||||
panic!("json schema don't support forward compatibility");
|
||||
}
|
||||
|
||||
let snapshot_with_counter = doc.export_snapshot();
|
||||
let doc3_without_counter = loro_without_counter::LoroDoc::new();
|
||||
// Test2: older version import newer version snapshot with counter
|
||||
doc3_without_counter.import(&snapshot_with_counter).unwrap();
|
||||
let unknown_json_from_snapshot = doc3_without_counter.export_json_updates(&Default::default());
|
||||
// {
|
||||
// "container": "cid:root-counter:Unknown(5)",
|
||||
// "content": {
|
||||
// "type": "unknown",
|
||||
// "value_type": "unknown",
|
||||
// "value": {"kind":16,"data":[]},
|
||||
// "prop": 5
|
||||
// },
|
||||
// "counter": 0
|
||||
// }
|
||||
// Test3: older version export json with binary unknown
|
||||
let _json_with_binary_unknown = doc3_without_counter.export_json_updates(&Default::default());
|
||||
let new_doc = loro::LoroDoc::new();
|
||||
// Test4: newer version import older version json with binary unknown
|
||||
if new_doc
|
||||
.import_json_updates(serde_json::to_string(&unknown_json_from_snapshot).unwrap())
|
||||
.is_ok()
|
||||
{
|
||||
panic!("json schema don't support forward compatibility");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sub_container() {
|
||||
test_multi_sites(
|
||||
5,
|
||||
vec![FuzzTarget::All],
|
||||
&mut [
|
||||
Handle {
|
||||
site: 0,
|
||||
target: 1,
|
||||
container: 0,
|
||||
action: Generic(GenericAction {
|
||||
value: Container(Text),
|
||||
bool: true,
|
||||
key: 4293853225,
|
||||
pos: 18446744073709551615,
|
||||
length: 4625477192774582511,
|
||||
prop: 18446744073428216116,
|
||||
}),
|
||||
},
|
||||
Sync { from: 0, to: 1 },
|
||||
Handle {
|
||||
site: 0,
|
||||
target: 0,
|
||||
container: 0,
|
||||
action: Generic(GenericAction {
|
||||
value: I32(0),
|
||||
bool: false,
|
||||
key: 0,
|
||||
pos: 0,
|
||||
length: 0,
|
||||
prop: 0,
|
||||
}),
|
||||
},
|
||||
],
|
||||
)
|
||||
}
|
||||
|
|
@ -15,7 +15,7 @@ keywords = ["crdt", "local-first"]
|
|||
|
||||
[dependencies]
|
||||
rle = { path = "../rle", version = "0.16.2", package = "loro-rle" }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde = { workspace = true }
|
||||
thiserror = "1.0.43"
|
||||
wasm-bindgen = { version = "=0.2.92", optional = true }
|
||||
fxhash = "0.2.1"
|
||||
|
|
|
|||
|
|
@ -62,6 +62,8 @@ pub enum LoroError {
|
|||
UndoInvalidIdSpan(ID),
|
||||
#[error("PeerID cannot be changed. Expected: {expected:?}, Actual: {actual:?}")]
|
||||
UndoWithDifferentPeerId { expected: PeerID, actual: PeerID },
|
||||
#[error("The input JSON schema is invalid")]
|
||||
InvalidJsonSchema,
|
||||
}
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
|
|
|
|||
|
|
@ -57,6 +57,32 @@ impl TryFrom<&str> for ID {
|
|||
}
|
||||
}
|
||||
|
||||
impl TryFrom<&str> for IdLp {
|
||||
type Error = LoroError;
|
||||
|
||||
fn try_from(value: &str) -> Result<Self, Self::Error> {
|
||||
if value.split('@').count() != 2 || !value.starts_with('L') {
|
||||
return Err(LoroError::DecodeError("Invalid ID format".into()));
|
||||
}
|
||||
|
||||
let mut iter = value[1..].split('@');
|
||||
let lamport = iter
|
||||
.next()
|
||||
.unwrap()
|
||||
.parse::<Lamport>()
|
||||
.map_err(|_| LoroError::DecodeError("Invalid ID format".into()))?;
|
||||
let client_id = iter
|
||||
.next()
|
||||
.unwrap()
|
||||
.parse::<u64>()
|
||||
.map_err(|_| LoroError::DecodeError("Invalid ID format".into()))?;
|
||||
Ok(IdLp {
|
||||
peer: client_id,
|
||||
lamport,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialOrd for ID {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
|
||||
Some(self.cmp(other))
|
||||
|
|
|
|||
|
|
@ -386,9 +386,28 @@ mod container {
|
|||
"Text" | "text" => Ok(ContainerType::Text),
|
||||
"Tree" | "tree" => Ok(ContainerType::Tree),
|
||||
"MovableList" | "movableList" => Ok(ContainerType::MovableList),
|
||||
_ => Err(LoroError::DecodeError(
|
||||
a => {
|
||||
if a.ends_with(')') {
|
||||
let start = a.find('(').ok_or_else(|| {
|
||||
LoroError::DecodeError(
|
||||
format!("Invalid container type string \"{}\"", value).into(),
|
||||
)
|
||||
})?;
|
||||
let k = a[start+1..a.len() - 1].parse().map_err(|_| {
|
||||
LoroError::DecodeError(
|
||||
format!("Unknown container type \"{}\". The valid options are Map|List|Text|Tree|MovableList.", value).into(),
|
||||
)),
|
||||
)
|
||||
})?;
|
||||
match ContainerType::try_from_u8(k) {
|
||||
Ok(k) => Ok(k),
|
||||
Err(_) => Ok(ContainerType::Unknown(k)),
|
||||
}
|
||||
} else {
|
||||
Err(LoroError::DecodeError(
|
||||
format!("Unknown container type \"{}\". The valid options are Map|List|Text|Tree|MovableList.", value).into(),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -527,5 +546,6 @@ mod test {
|
|||
assert!(ContainerID::try_from("cid:@:Map").is_err());
|
||||
assert!(ContainerID::try_from("cid:x@0:Map").is_err());
|
||||
assert!(ContainerID::try_from("id:0@0:Map").is_err());
|
||||
assert!(ContainerID::try_from("cid:0@0:Unknown(6)").is_ok());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ use std::{collections::HashMap, hash::Hash, ops::Index, sync::Arc};
|
|||
|
||||
use enum_as_inner::EnumAsInner;
|
||||
use fxhash::FxHashMap;
|
||||
use serde::{de::VariantAccess, ser::SerializeStruct, Deserialize, Serialize};
|
||||
use serde::{de::VariantAccess, Deserialize, Serialize};
|
||||
|
||||
use crate::ContainerID;
|
||||
|
||||
|
|
@ -481,6 +481,8 @@ pub mod wasm {
|
|||
}
|
||||
}
|
||||
|
||||
const LORO_CONTAINER_ID_PREFIX: &str = "🦜:";
|
||||
|
||||
impl Serialize for LoroValue {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
|
|
@ -498,9 +500,7 @@ impl Serialize for LoroValue {
|
|||
LoroValue::List(l) => serializer.collect_seq(l.iter()),
|
||||
LoroValue::Map(m) => serializer.collect_map(m.iter()),
|
||||
LoroValue::Container(id) => {
|
||||
let mut state = serializer.serialize_struct("Container", 1)?;
|
||||
state.serialize_field("Container", id)?;
|
||||
state.end()
|
||||
serializer.serialize_str(&format!("{}{}", LORO_CONTAINER_ID_PREFIX, id))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
|
@ -610,6 +610,12 @@ impl<'de> serde::de::Visitor<'de> for LoroValueVisitor {
|
|||
where
|
||||
E: serde::de::Error,
|
||||
{
|
||||
if let Some(id) = v.strip_prefix(LORO_CONTAINER_ID_PREFIX) {
|
||||
return Ok(LoroValue::Container(
|
||||
ContainerID::try_from(id)
|
||||
.map_err(|_| serde::de::Error::custom("Invalid container id"))?,
|
||||
));
|
||||
}
|
||||
Ok(LoroValue::String(Arc::new(v.to_owned())))
|
||||
}
|
||||
|
||||
|
|
@ -617,6 +623,13 @@ impl<'de> serde::de::Visitor<'de> for LoroValueVisitor {
|
|||
where
|
||||
E: serde::de::Error,
|
||||
{
|
||||
if let Some(id) = v.strip_prefix(LORO_CONTAINER_ID_PREFIX) {
|
||||
return Ok(LoroValue::Container(
|
||||
ContainerID::try_from(id)
|
||||
.map_err(|_| serde::de::Error::custom("Invalid container id"))?,
|
||||
));
|
||||
}
|
||||
|
||||
Ok(LoroValue::String(v.into()))
|
||||
}
|
||||
|
||||
|
|
@ -652,9 +665,7 @@ impl<'de> serde::de::Visitor<'de> for LoroValueVisitor {
|
|||
A: serde::de::MapAccess<'de>,
|
||||
{
|
||||
let mut ans: FxHashMap<String, _> = FxHashMap::default();
|
||||
let mut last_key = None;
|
||||
while let Some((key, value)) = map.next_entry::<String, _>()? {
|
||||
last_key.get_or_insert_with(|| key.clone());
|
||||
ans.insert(key, value);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,16 +14,16 @@ keywords = ["crdt", "local-first"]
|
|||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
smallvec = { workspace = true }
|
||||
loro-delta = { path = "../delta", version = "0.16.2", package = "loro-delta" }
|
||||
rle = { path = "../rle", version = "0.16.2", package = "loro-rle" }
|
||||
loro-common = { path = "../loro-common", version = "0.16.2" }
|
||||
fractional_index = { path = "../fractional_index", features = [
|
||||
"serde",
|
||||
], version = "0.16.2", package = "loro_fractional_index" }
|
||||
smallvec = { version = "1.8.0", features = ["serde"] }
|
||||
postcard = "1"
|
||||
fxhash = { workspace = true }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde = { workspace = true }
|
||||
thiserror = "1"
|
||||
enum-as-inner = { workspace = true }
|
||||
num = "0.4.0"
|
||||
|
|
@ -33,7 +33,7 @@ tabled = { version = "0.10.0", optional = true }
|
|||
wasm-bindgen = { version = "=0.2.92", optional = true }
|
||||
serde-wasm-bindgen = { version = "0.5.0", optional = true }
|
||||
js-sys = { version = "0.3.60", optional = true }
|
||||
serde_json = { version = "1" }
|
||||
serde_json = { workspace = true }
|
||||
arref = "0.1.0"
|
||||
serde_columnar = { workspace = true }
|
||||
append-only-bytes = { version = "0.1.12", features = ["u32_range"] }
|
||||
|
|
|
|||
|
|
@ -107,6 +107,21 @@ mod run {
|
|||
store2.import(&buf).unwrap();
|
||||
})
|
||||
});
|
||||
|
||||
b.bench_function("B4_encode_json_update", |b| {
|
||||
ensure_ran();
|
||||
b.iter(|| {
|
||||
let _ = loro.export_json_updates(&Default::default());
|
||||
})
|
||||
});
|
||||
b.bench_function("B4_decode_json_update", |b| {
|
||||
ensure_ran();
|
||||
let json = loro.export_json_updates(&Default::default());
|
||||
b.iter(|| {
|
||||
let store2 = LoroDoc::default();
|
||||
store2.import_json_updates(json.clone()).unwrap();
|
||||
})
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -62,6 +62,15 @@ fn main() {
|
|||
output.len(),
|
||||
);
|
||||
|
||||
let json_updates =
|
||||
serde_json::to_string(&loro.export_json_updates(&Default::default())).unwrap();
|
||||
let output = miniz_oxide::deflate::compress_to_vec(json_updates.as_bytes(), 6);
|
||||
println!(
|
||||
"json updates size {} after compression {}",
|
||||
json_updates.len(),
|
||||
output.len(),
|
||||
);
|
||||
|
||||
// {
|
||||
// // Delta encoding
|
||||
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@ use criterion::black_box;
|
|||
use loro_internal::loro::LoroDoc;
|
||||
|
||||
fn main() {
|
||||
// log_size();
|
||||
bench_decode();
|
||||
log_size();
|
||||
// bench_decode();
|
||||
// bench_decode_updates();
|
||||
}
|
||||
|
||||
|
|
@ -23,9 +23,12 @@ fn log_size() {
|
|||
txn.commit().unwrap();
|
||||
let snapshot = loro.export_snapshot();
|
||||
let updates = loro.export_from(&Default::default());
|
||||
let json_updates =
|
||||
serde_json::to_string(&loro.export_json_updates(&Default::default())).unwrap();
|
||||
println!("\n");
|
||||
println!("Snapshot size={}", snapshot.len());
|
||||
println!("Updates size={}", updates.len());
|
||||
println!("Json Updates size={}", json_updates.as_bytes().len());
|
||||
println!("\n");
|
||||
loro.diagnose_size();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,26 +12,45 @@ use crate::state::TreeParentId;
|
|||
/// - **Move**: move target tree node a child node of the specified parent node.
|
||||
/// - **Delete**: move target tree node to [`loro_common::DELETED_TREE_ROOT`].
|
||||
///
|
||||
|
||||
///
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct TreeOp {
|
||||
pub(crate) target: TreeID,
|
||||
pub(crate) parent: Option<TreeID>,
|
||||
// If the op is `delete`, the position is None
|
||||
pub(crate) position: Option<FractionalIndex>,
|
||||
pub enum TreeOp {
|
||||
Create {
|
||||
target: TreeID,
|
||||
parent: Option<TreeID>,
|
||||
position: FractionalIndex,
|
||||
},
|
||||
Move {
|
||||
target: TreeID,
|
||||
parent: Option<TreeID>,
|
||||
position: FractionalIndex,
|
||||
},
|
||||
Delete {
|
||||
target: TreeID,
|
||||
},
|
||||
}
|
||||
|
||||
impl TreeOp {
|
||||
pub(crate) fn target(&self) -> TreeID {
|
||||
match self {
|
||||
TreeOp::Create { target, .. } => *target,
|
||||
TreeOp::Move { target, .. } => *target,
|
||||
TreeOp::Delete { target, .. } => *target,
|
||||
}
|
||||
}
|
||||
pub(crate) fn parent_id(&self) -> TreeParentId {
|
||||
match self.parent {
|
||||
Some(parent) => {
|
||||
if TreeID::is_deleted_root(&parent) {
|
||||
TreeParentId::Deleted
|
||||
} else {
|
||||
TreeParentId::Node(parent)
|
||||
}
|
||||
match self {
|
||||
TreeOp::Create { parent, .. } => TreeParentId::from(*parent),
|
||||
TreeOp::Move { parent, .. } => TreeParentId::from(*parent),
|
||||
TreeOp::Delete { .. } => TreeParentId::Deleted,
|
||||
}
|
||||
}
|
||||
pub(crate) fn fractional_index(&self) -> Option<FractionalIndex> {
|
||||
match self {
|
||||
TreeOp::Create { position, .. } | TreeOp::Move { position, .. } => {
|
||||
Some(position.clone())
|
||||
}
|
||||
None => TreeParentId::Root,
|
||||
TreeOp::Delete { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -126,9 +126,9 @@ impl TreeDiffCalculator {
|
|||
tracing::info!("forward ops {:?}", forward_ops);
|
||||
for (lamport, op) in forward_ops {
|
||||
let op = MoveLamportAndID {
|
||||
target: op.value.target,
|
||||
target: op.value.target(),
|
||||
parent: op.value.parent_id(),
|
||||
position: op.value.position.clone(),
|
||||
position: op.value.fractional_index(),
|
||||
id: op.id_start(),
|
||||
lamport,
|
||||
effected: false,
|
||||
|
|
@ -239,9 +239,9 @@ impl TreeDiffCalculator {
|
|||
&& to.includes_id(op.id_start())
|
||||
{
|
||||
let op = MoveLamportAndID {
|
||||
target: op.value.target,
|
||||
target: op.value.target(),
|
||||
parent: op.value.parent_id(),
|
||||
position: op.value.position.clone(),
|
||||
position: op.value.fractional_index(),
|
||||
id: op.id_start(),
|
||||
lamport: *lamport,
|
||||
effected: false,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
mod arena;
|
||||
mod encode_reordered;
|
||||
pub(crate) mod json_schema;
|
||||
mod value;
|
||||
pub(crate) use value::OwnedValue;
|
||||
|
||||
|
|
|
|||
|
|
@ -207,7 +207,7 @@ pub fn decode_import_blob_meta(bytes: &[u8]) -> LoroResult<ImportBlobMetadata> {
|
|||
})
|
||||
}
|
||||
|
||||
fn import_changes_to_oplog(
|
||||
pub(crate) fn import_changes_to_oplog(
|
||||
changes: Vec<Change>,
|
||||
oplog: &mut OpLog,
|
||||
) -> Result<(Vec<ID>, Vec<Change>), LoroError> {
|
||||
|
|
@ -356,13 +356,7 @@ fn extract_ops(
|
|||
let peer = arenas.peer_ids[peer_idx as usize];
|
||||
let cid = &containers[container_index as usize];
|
||||
let kind = ValueKind::from_u8(value_type);
|
||||
let value = Value::decode(
|
||||
kind,
|
||||
&mut value_reader,
|
||||
arenas,
|
||||
ID::new(peer, counter),
|
||||
prop,
|
||||
)?;
|
||||
let value = Value::decode(kind, &mut value_reader, arenas, ID::new(peer, counter))?;
|
||||
|
||||
let content = decode_op(
|
||||
cid,
|
||||
|
|
@ -372,6 +366,7 @@ fn extract_ops(
|
|||
arenas,
|
||||
&positions,
|
||||
prop,
|
||||
ID::new(peer, counter),
|
||||
)?;
|
||||
|
||||
let container = shared_arena.register_container(cid);
|
||||
|
|
@ -886,7 +881,7 @@ mod encode {
|
|||
use crate::{
|
||||
arena::SharedArena,
|
||||
change::{Change, Lamport},
|
||||
container::idx::ContainerIdx,
|
||||
container::{idx::ContainerIdx, tree::tree_op::TreeOp},
|
||||
encoding::value::{EncodedTreeMove, MarkStart, Value, ValueKind, ValueWriter},
|
||||
op::{FutureInnerContent, Op},
|
||||
};
|
||||
|
|
@ -1178,18 +1173,16 @@ mod encode {
|
|||
let key = registers.key.register(&map.key);
|
||||
key as i32
|
||||
}
|
||||
crate::op::InnerContent::Tree(op) => {
|
||||
if let Some(position) = &op.position {
|
||||
if let either::Either::Left(position_register) = &mut registers.position {
|
||||
position_register.insert(position.as_bytes());
|
||||
} else {
|
||||
crate::op::InnerContent::Tree(op) => match op {
|
||||
TreeOp::Create { position, .. } | TreeOp::Move { position, .. } => {
|
||||
let either::Either::Left(position_register) = &mut registers.position else {
|
||||
unreachable!()
|
||||
}
|
||||
};
|
||||
position_register.insert(position.as_bytes());
|
||||
0
|
||||
} else {
|
||||
-1
|
||||
}
|
||||
}
|
||||
TreeOp::Delete { .. } => 0,
|
||||
},
|
||||
crate::op::InnerContent::Future(f) => match f {
|
||||
#[cfg(feature = "counter")]
|
||||
FutureInnerContent::Counter(_) => 0,
|
||||
|
|
@ -1286,6 +1279,7 @@ mod encode {
|
|||
}
|
||||
|
||||
#[inline]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn decode_op(
|
||||
cid: &ContainerID,
|
||||
value: Value<'_>,
|
||||
|
|
@ -1294,6 +1288,7 @@ fn decode_op(
|
|||
arenas: &DecodedArenas<'_>,
|
||||
positions: &[Vec<u8>],
|
||||
prop: i32,
|
||||
op_id: ID,
|
||||
) -> LoroResult<crate::op::InnerContent> {
|
||||
let content = match cid.container_type() {
|
||||
ContainerType::Text => match value {
|
||||
|
|
@ -1390,6 +1385,7 @@ fn decode_op(
|
|||
&arenas.peer_ids,
|
||||
positions,
|
||||
&arenas.tree_ids.tree_ids,
|
||||
op_id,
|
||||
)?),
|
||||
_ => {
|
||||
unreachable!()
|
||||
|
|
@ -1450,7 +1446,7 @@ fn decode_op(
|
|||
ContainerType::Counter => {
|
||||
crate::op::InnerContent::Future(FutureInnerContent::Counter(prop as i64))
|
||||
}
|
||||
|
||||
// NOTE: The future container type need also try to parse the unknown type
|
||||
ContainerType::Unknown(_) => crate::op::InnerContent::Future(FutureInnerContent::Unknown {
|
||||
prop,
|
||||
value: value.into_owned(),
|
||||
|
|
|
|||
1175
crates/loro-internal/src/encoding/json_schema.rs
Normal file
1175
crates/loro-internal/src/encoding/json_schema.rs
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -180,32 +180,20 @@ pub enum FutureValue<'a> {
|
|||
// The future value cannot depend on the arena for encoding.
|
||||
Unknown {
|
||||
kind: u8,
|
||||
prop: i32,
|
||||
data: &'a [u8],
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub enum OwnedFutureValue {
|
||||
#[cfg(feature = "counter")]
|
||||
Counter,
|
||||
// The future value cannot depend on the arena for encoding.
|
||||
Unknown {
|
||||
kind: u8,
|
||||
prop: i32,
|
||||
data: Vec<u8>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(tag = "value_type", content = "value", rename_all = "snake_case")]
|
||||
pub enum OwnedValue {
|
||||
Null,
|
||||
True,
|
||||
False,
|
||||
I64(i64),
|
||||
F64(f64),
|
||||
Str(String),
|
||||
Binary(Vec<u8>),
|
||||
Str(Arc<String>),
|
||||
Binary(Arc<Vec<u8>>),
|
||||
ContainerIdx(usize),
|
||||
DeleteOnce,
|
||||
DeleteSeq,
|
||||
|
|
@ -223,9 +211,22 @@ pub enum OwnedValue {
|
|||
lamport: Lamport,
|
||||
value: LoroValue,
|
||||
},
|
||||
#[serde(untagged)]
|
||||
Future(OwnedFutureValue),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(tag = "value_type", content = "value")]
|
||||
pub enum OwnedFutureValue {
|
||||
#[cfg(feature = "counter")]
|
||||
Counter,
|
||||
// The future value cannot depend on the arena for encoding.
|
||||
Unknown {
|
||||
kind: u8,
|
||||
data: Arc<Vec<u8>>,
|
||||
},
|
||||
}
|
||||
|
||||
impl<'a> Value<'a> {
|
||||
pub fn from_owned(owned_value: &'a OwnedValue) -> Self {
|
||||
match owned_value {
|
||||
|
|
@ -264,13 +265,10 @@ impl<'a> Value<'a> {
|
|||
OwnedValue::Future(value) => match value {
|
||||
#[cfg(feature = "counter")]
|
||||
OwnedFutureValue::Counter => Value::Future(FutureValue::Counter),
|
||||
OwnedFutureValue::Unknown { kind, prop, data } => {
|
||||
Value::Future(FutureValue::Unknown {
|
||||
kind: *kind,
|
||||
prop: *prop,
|
||||
data: data.as_slice(),
|
||||
})
|
||||
}
|
||||
OwnedFutureValue::Unknown { kind, data } => Value::Future(FutureValue::Unknown {
|
||||
kind: *kind,
|
||||
data: data.as_slice(),
|
||||
}),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
@ -284,12 +282,12 @@ impl<'a> Value<'a> {
|
|||
Value::I64(x) => OwnedValue::I64(x),
|
||||
Value::ContainerIdx(x) => OwnedValue::ContainerIdx(x),
|
||||
Value::F64(x) => OwnedValue::F64(x),
|
||||
Value::Str(x) => OwnedValue::Str(x.to_owned()),
|
||||
Value::Str(x) => OwnedValue::Str(Arc::new(x.to_owned())),
|
||||
Value::DeleteSeq => OwnedValue::DeleteSeq,
|
||||
Value::DeltaInt(x) => OwnedValue::DeltaInt(x),
|
||||
Value::LoroValue(x) => OwnedValue::LoroValue(x),
|
||||
Value::MarkStart(x) => OwnedValue::MarkStart(x),
|
||||
Value::Binary(x) => OwnedValue::Binary(x.to_owned()),
|
||||
Value::Binary(x) => OwnedValue::Binary(Arc::new(x.to_owned())),
|
||||
Value::TreeMove(x) => OwnedValue::TreeMove(x),
|
||||
Value::ListMove {
|
||||
from,
|
||||
|
|
@ -312,11 +310,10 @@ impl<'a> Value<'a> {
|
|||
Value::Future(value) => match value {
|
||||
#[cfg(feature = "counter")]
|
||||
FutureValue::Counter => OwnedValue::Future(OwnedFutureValue::Counter),
|
||||
FutureValue::Unknown { kind, prop, data } => {
|
||||
FutureValue::Unknown { kind, data } => {
|
||||
OwnedValue::Future(OwnedFutureValue::Unknown {
|
||||
kind,
|
||||
prop,
|
||||
data: data.to_owned(),
|
||||
data: Arc::new(data.to_owned()),
|
||||
})
|
||||
}
|
||||
},
|
||||
|
|
@ -326,17 +323,12 @@ impl<'a> Value<'a> {
|
|||
fn decode_without_arena<'r: 'a>(
|
||||
future_kind: FutureValueKind,
|
||||
value_reader: &'r mut ValueReader,
|
||||
prop: i32,
|
||||
) -> LoroResult<Self> {
|
||||
let bytes_length = value_reader.read_usize()?;
|
||||
let bytes = value_reader.read_binary()?;
|
||||
let value = match future_kind {
|
||||
#[cfg(feature = "counter")]
|
||||
FutureValueKind::Counter => FutureValue::Counter,
|
||||
FutureValueKind::Unknown(kind) => FutureValue::Unknown {
|
||||
kind,
|
||||
prop,
|
||||
data: value_reader.take_bytes(bytes_length),
|
||||
},
|
||||
FutureValueKind::Unknown(kind) => FutureValue::Unknown { kind, data: bytes },
|
||||
};
|
||||
Ok(Value::Future(value))
|
||||
}
|
||||
|
|
@ -346,7 +338,6 @@ impl<'a> Value<'a> {
|
|||
value_reader: &'r mut ValueReader,
|
||||
arenas: &'a DecodedArenas<'a>,
|
||||
id: ID,
|
||||
prop: i32,
|
||||
) -> LoroResult<Self> {
|
||||
Ok(match kind {
|
||||
ValueKind::Null => Value::Null,
|
||||
|
|
@ -388,7 +379,7 @@ impl<'a> Value<'a> {
|
|||
}
|
||||
}
|
||||
ValueKind::Future(future_kind) => {
|
||||
Self::decode_without_arena(future_kind, value_reader, prop)?
|
||||
Self::decode_without_arena(future_kind, value_reader)?
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -397,18 +388,18 @@ impl<'a> Value<'a> {
|
|||
value: FutureValue,
|
||||
value_writer: &mut ValueWriter,
|
||||
) -> (FutureValueKind, usize) {
|
||||
// Note: we should encode FutureValue as binary data.
|
||||
// [binary data length, binary data]
|
||||
// when decoding, we will use reader.read_binary() to read the binary data.
|
||||
// So such as FutureValue::Counter, we should write 0 as the length of binary data first.
|
||||
match value {
|
||||
#[cfg(feature = "counter")]
|
||||
FutureValue::Counter => {
|
||||
// write bytes length
|
||||
value_writer.write_u8(0);
|
||||
(FutureValueKind::Counter, 0)
|
||||
(FutureValueKind::Counter, 1)
|
||||
}
|
||||
FutureValue::Unknown {
|
||||
kind,
|
||||
prop: _,
|
||||
data,
|
||||
} => (
|
||||
FutureValue::Unknown { kind, data } => (
|
||||
FutureValueKind::Unknown(kind),
|
||||
value_writer.write_binary(data),
|
||||
),
|
||||
|
|
@ -476,7 +467,7 @@ pub struct MarkStart {
|
|||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct EncodedTreeMove {
|
||||
pub subject_idx: usize,
|
||||
pub target_idx: usize,
|
||||
pub is_parent_null: bool,
|
||||
pub parent_idx: usize,
|
||||
pub position: usize,
|
||||
|
|
@ -488,6 +479,7 @@ impl EncodedTreeMove {
|
|||
peer_ids: &[u64],
|
||||
positions: &[Vec<u8>],
|
||||
tree_ids: &[EncodedTreeID],
|
||||
op_id: ID,
|
||||
) -> LoroResult<TreeOp> {
|
||||
let parent = if self.is_parent_null {
|
||||
None
|
||||
|
|
@ -500,55 +492,91 @@ impl EncodedTreeMove {
|
|||
counter as Counter,
|
||||
))
|
||||
};
|
||||
let position = if parent.is_some_and(|x| TreeID::is_deleted_root(&x)) {
|
||||
let is_delete = parent.is_some_and(|p| TreeID::is_deleted_root(&p));
|
||||
let position = if is_delete {
|
||||
None
|
||||
} else {
|
||||
let bytes = &positions[self.position];
|
||||
Some(FractionalIndex::from_bytes(bytes.clone()))
|
||||
};
|
||||
let EncodedTreeID { peer_idx, counter } = tree_ids[self.subject_idx];
|
||||
Ok(TreeOp {
|
||||
target: TreeID::new(
|
||||
*(peer_ids
|
||||
.get(peer_idx)
|
||||
.ok_or(LoroError::DecodeDataCorruptionError)?),
|
||||
counter as Counter,
|
||||
),
|
||||
parent,
|
||||
position,
|
||||
})
|
||||
let EncodedTreeID { peer_idx, counter } = tree_ids[self.target_idx];
|
||||
let target = TreeID::new(
|
||||
*(peer_ids
|
||||
.get(peer_idx)
|
||||
.ok_or(LoroError::DecodeDataCorruptionError)?),
|
||||
counter as Counter,
|
||||
);
|
||||
|
||||
let is_create = target.id() == op_id;
|
||||
let op = if is_delete {
|
||||
TreeOp::Delete { target }
|
||||
} else if is_create {
|
||||
TreeOp::Create {
|
||||
target,
|
||||
parent,
|
||||
position: position.unwrap(),
|
||||
}
|
||||
} else {
|
||||
TreeOp::Move {
|
||||
target,
|
||||
parent,
|
||||
position: position.unwrap(),
|
||||
}
|
||||
};
|
||||
Ok(op)
|
||||
}
|
||||
|
||||
pub fn from_tree_op<'p, 'a: 'p>(op: &'a TreeOp, registers: &mut EncodedRegisters) -> Self {
|
||||
let position = if let Some(position) = &op.position {
|
||||
let bytes = position.as_bytes();
|
||||
let either::Right(position_register) = &mut registers.position else {
|
||||
unreachable!()
|
||||
};
|
||||
position_register.get(&bytes).unwrap()
|
||||
} else {
|
||||
debug_assert!(op.parent.is_some_and(|x| TreeID::is_deleted_root(&x)));
|
||||
// placeholder
|
||||
0
|
||||
};
|
||||
match op {
|
||||
TreeOp::Create {
|
||||
target,
|
||||
parent,
|
||||
position,
|
||||
}
|
||||
| TreeOp::Move {
|
||||
target,
|
||||
parent,
|
||||
position,
|
||||
} => {
|
||||
let bytes = position.as_bytes();
|
||||
let either::Right(position_register) = &mut registers.position else {
|
||||
unreachable!()
|
||||
};
|
||||
let position = position_register.get(&bytes).unwrap();
|
||||
let target_idx = registers.tree_id.register(&EncodedTreeID {
|
||||
peer_idx: registers.peer.register(&target.peer),
|
||||
counter: target.counter,
|
||||
});
|
||||
|
||||
let target_idx = registers.tree_id.register(&EncodedTreeID {
|
||||
peer_idx: registers.peer.register(&op.target.peer),
|
||||
counter: op.target.counter,
|
||||
});
|
||||
|
||||
let parent_idx = op.parent.map(|x| {
|
||||
registers.tree_id.register(&EncodedTreeID {
|
||||
peer_idx: registers.peer.register(&x.peer),
|
||||
counter: x.counter,
|
||||
})
|
||||
});
|
||||
|
||||
EncodedTreeMove {
|
||||
subject_idx: target_idx,
|
||||
is_parent_null: op.parent.is_none(),
|
||||
parent_idx: parent_idx.unwrap_or(0),
|
||||
position,
|
||||
let parent_idx = parent.map(|x| {
|
||||
registers.tree_id.register(&EncodedTreeID {
|
||||
peer_idx: registers.peer.register(&x.peer),
|
||||
counter: x.counter,
|
||||
})
|
||||
});
|
||||
EncodedTreeMove {
|
||||
target_idx,
|
||||
is_parent_null: parent.is_none(),
|
||||
parent_idx: parent_idx.unwrap_or(0),
|
||||
position,
|
||||
}
|
||||
}
|
||||
TreeOp::Delete { target } => {
|
||||
let target_idx = registers.tree_id.register(&EncodedTreeID {
|
||||
peer_idx: registers.peer.register(&target.peer),
|
||||
counter: target.counter,
|
||||
});
|
||||
let parent_idx = registers.tree_id.register(&EncodedTreeID {
|
||||
peer_idx: registers.peer.register(&TreeID::delete_root().peer),
|
||||
counter: TreeID::delete_root().counter,
|
||||
});
|
||||
EncodedTreeMove {
|
||||
target_idx,
|
||||
is_parent_null: false,
|
||||
parent_idx,
|
||||
position: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -900,12 +928,6 @@ impl<'a> ValueReader<'a> {
|
|||
})
|
||||
}
|
||||
|
||||
pub fn take_bytes(&mut self, len: usize) -> &'a [u8] {
|
||||
let ans = &self.raw[..len];
|
||||
self.raw = &self.raw[len..];
|
||||
ans
|
||||
}
|
||||
|
||||
pub fn read_tree_move(&mut self) -> LoroResult<EncodedTreeMove> {
|
||||
let subject_idx = self.read_usize()?;
|
||||
let is_parent_null = self.read_u8()? != 0;
|
||||
|
|
@ -915,7 +937,7 @@ impl<'a> ValueReader<'a> {
|
|||
parent_idx = self.read_usize()?;
|
||||
}
|
||||
Ok(EncodedTreeMove {
|
||||
subject_idx,
|
||||
target_idx: subject_idx,
|
||||
is_parent_null,
|
||||
parent_idx,
|
||||
position,
|
||||
|
|
@ -1039,7 +1061,7 @@ impl ValueWriter {
|
|||
|
||||
fn write_tree_move(&mut self, op: &EncodedTreeMove) -> usize {
|
||||
let len = self.buffer.len();
|
||||
self.write_usize(op.subject_idx);
|
||||
self.write_usize(op.target_idx);
|
||||
self.write_u8(op.is_parent_null as u8);
|
||||
self.write_usize(op.position);
|
||||
if op.is_parent_null {
|
||||
|
|
|
|||
|
|
@ -300,11 +300,7 @@ impl TreeHandler {
|
|||
let inner = self.inner.try_attached_state()?;
|
||||
txn.apply_local_op(
|
||||
inner.container_idx,
|
||||
crate::op::RawOpContent::Tree(TreeOp {
|
||||
target,
|
||||
parent: Some(TreeID::delete_root()),
|
||||
position: None,
|
||||
}),
|
||||
crate::op::RawOpContent::Tree(TreeOp::Delete { target }),
|
||||
EventHint::Tree(TreeDiffItem {
|
||||
target,
|
||||
action: TreeExternalDiff::Delete {
|
||||
|
|
@ -528,10 +524,10 @@ impl TreeHandler {
|
|||
) -> LoroResult<TreeID> {
|
||||
txn.apply_local_op(
|
||||
inner.container_idx,
|
||||
crate::op::RawOpContent::Tree(TreeOp {
|
||||
crate::op::RawOpContent::Tree(TreeOp::Create {
|
||||
target: tree_id,
|
||||
parent,
|
||||
position: Some(position.clone()),
|
||||
position: position.clone(),
|
||||
}),
|
||||
EventHint::Tree(TreeDiffItem {
|
||||
target: tree_id,
|
||||
|
|
@ -557,10 +553,10 @@ impl TreeHandler {
|
|||
) -> LoroResult<()> {
|
||||
txn.apply_local_op(
|
||||
inner.container_idx,
|
||||
crate::op::RawOpContent::Tree(TreeOp {
|
||||
crate::op::RawOpContent::Tree(TreeOp::Move {
|
||||
target,
|
||||
parent,
|
||||
position: Some(position.clone()),
|
||||
position: position.clone(),
|
||||
}),
|
||||
EventHint::Tree(TreeDiffItem {
|
||||
target,
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ pub(crate) use id::{PeerID, ID};
|
|||
pub(crate) use loro_common::InternalString;
|
||||
|
||||
pub use container::ContainerType;
|
||||
pub use encoding::json_schema::op::*;
|
||||
pub use loro_common::{loro_value, to_value};
|
||||
#[cfg(feature = "wasm")]
|
||||
pub use value::wasm;
|
||||
|
|
|
|||
|
|
@ -28,7 +28,8 @@ use crate::{
|
|||
cursor::{AbsolutePosition, CannotFindRelativePosition, Cursor, PosQueryResult},
|
||||
dag::DagUtils,
|
||||
encoding::{
|
||||
decode_snapshot, export_snapshot, parse_header_and_body, EncodeMode, ParsedHeaderAndBody,
|
||||
decode_snapshot, export_snapshot, json_schema::op::JsonSchema, parse_header_and_body,
|
||||
EncodeMode, ParsedHeaderAndBody,
|
||||
},
|
||||
event::{str_to_path, EventTriggerKind, Index},
|
||||
handler::{Handler, MovableListHandler, TextHandler, TreeHandler, ValueOrHandler},
|
||||
|
|
@ -570,6 +571,30 @@ impl LoroDoc {
|
|||
ans
|
||||
}
|
||||
|
||||
/// Import the json schema updates.
|
||||
///
|
||||
/// only supports backward compatibility but not forward compatibility.
|
||||
pub fn import_json_updates<T: TryInto<JsonSchema>>(&self, json: T) -> LoroResult<()> {
|
||||
let json = json.try_into().map_err(|_| LoroError::InvalidJsonSchema)?;
|
||||
self.commit_then_stop();
|
||||
self.update_oplog_and_apply_delta_to_state_if_needed(
|
||||
|oplog| crate::encoding::json_schema::import_json(oplog, json),
|
||||
Default::default(),
|
||||
)?;
|
||||
self.emit_events();
|
||||
self.renew_txn_if_auto_commit();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn export_json_updates(&self, vv: &VersionVector) -> JsonSchema {
|
||||
self.commit_then_stop();
|
||||
let oplog = self.oplog.lock().unwrap();
|
||||
let json = crate::encoding::json_schema::export_json(&oplog, vv);
|
||||
drop(oplog);
|
||||
self.renew_txn_if_auto_commit();
|
||||
json
|
||||
}
|
||||
|
||||
/// Get the version vector of the current OpLog
|
||||
#[inline]
|
||||
pub fn oplog_vv(&self) -> VersionVector {
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ impl InnerContent {
|
|||
}
|
||||
}
|
||||
crate::op::InnerContent::Tree(t) => {
|
||||
let id = t.target.associated_meta_container();
|
||||
let id = t.target().associated_meta_container();
|
||||
f(&id);
|
||||
}
|
||||
crate::op::InnerContent::Future(f) => match &f {
|
||||
|
|
|
|||
|
|
@ -8,11 +8,7 @@ use loro_common::LoroValue;
|
|||
|
||||
use crate::{
|
||||
change::Change,
|
||||
container::{
|
||||
list::list_op::{ListOp},
|
||||
map::MapSet,
|
||||
tree::tree_op::TreeOp,
|
||||
},
|
||||
container::{list::list_op::ListOp, map::MapSet},
|
||||
op::{ListSlice, RawOp, RawOpContent},
|
||||
DocState, OpLog,
|
||||
};
|
||||
|
|
@ -71,7 +67,8 @@ impl DocState {
|
|||
self.arena.set_parent(idx, Some(container));
|
||||
}
|
||||
}
|
||||
RawOpContent::Tree(TreeOp { target, .. }) => {
|
||||
RawOpContent::Tree(tree) => {
|
||||
let target = tree.target();
|
||||
// create associated metadata container
|
||||
// TODO: maybe we could create map container only when setting metadata
|
||||
let container_id = target.associated_meta_container();
|
||||
|
|
|
|||
|
|
@ -945,15 +945,31 @@ impl ContainerState for TreeState {
|
|||
|
||||
fn apply_local_op(&mut self, raw_op: &RawOp, _op: &Op) -> LoroResult<()> {
|
||||
match &raw_op.content {
|
||||
crate::op::RawOpContent::Tree(tree) => {
|
||||
let TreeOp {
|
||||
crate::op::RawOpContent::Tree(tree) => match tree {
|
||||
TreeOp::Create {
|
||||
target,
|
||||
parent,
|
||||
position,
|
||||
} = tree;
|
||||
let parent = TreeParentId::from(*parent);
|
||||
self.mov(*target, parent, raw_op.id_full(), position.clone(), true)
|
||||
}
|
||||
}
|
||||
| TreeOp::Move {
|
||||
target,
|
||||
parent,
|
||||
position,
|
||||
} => {
|
||||
let parent = TreeParentId::from(*parent);
|
||||
self.mov(
|
||||
*target,
|
||||
parent,
|
||||
raw_op.id_full(),
|
||||
Some(position.clone()),
|
||||
true,
|
||||
)
|
||||
}
|
||||
TreeOp::Delete { target } => {
|
||||
let parent = TreeParentId::Deleted;
|
||||
self.mov(*target, parent, raw_op.id_full(), None, true)
|
||||
}
|
||||
},
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
|
@ -1053,12 +1069,27 @@ impl ContainerState for TreeState {
|
|||
for op in ctx.ops {
|
||||
assert_eq!(op.op.atom_len(), 1);
|
||||
let content = op.op.content.as_tree().unwrap();
|
||||
let target = content.target;
|
||||
let parent = content.parent;
|
||||
let position = content.position.clone();
|
||||
let parent = TreeParentId::from(parent);
|
||||
self.mov(target, parent, op.id_full(), position, false)
|
||||
.unwrap();
|
||||
match content {
|
||||
TreeOp::Create {
|
||||
target,
|
||||
parent,
|
||||
position,
|
||||
}
|
||||
| TreeOp::Move {
|
||||
target,
|
||||
parent,
|
||||
position,
|
||||
} => {
|
||||
let parent = TreeParentId::from(*parent);
|
||||
self.mov(*target, parent, op.id_full(), Some(position.clone()), false)
|
||||
.unwrap()
|
||||
}
|
||||
TreeOp::Delete { target } => {
|
||||
let parent = TreeParentId::Deleted;
|
||||
self.mov(*target, parent, op.id_full(), None, false)
|
||||
.unwrap()
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -760,19 +760,3 @@ pub mod wasm {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(feature = "json")]
|
||||
mod json_test {
|
||||
use crate::{fx_map, value::ToJson, LoroValue};
|
||||
|
||||
#[test]
|
||||
fn list() {
|
||||
let list = LoroValue::List(
|
||||
vec![12.into(), "123".into(), fx_map!("kk" => 123.into()).into()].into(),
|
||||
);
|
||||
let json = list.to_json();
|
||||
println!("{}", json);
|
||||
assert_eq!(LoroValue::from_json(&json), list);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,14 +12,15 @@ crate-type = ["cdylib", "rlib"]
|
|||
js-sys = "0.3.60"
|
||||
loro-internal = { path = "../loro-internal", features = ["wasm"] }
|
||||
wasm-bindgen = "=0.2.92"
|
||||
serde-wasm-bindgen = { version = "0.5.0" }
|
||||
serde-wasm-bindgen = { version = "^0.6.5" }
|
||||
wasm-bindgen-derive = "0.2.1"
|
||||
console_error_panic_hook = { version = "0.1.6", optional = true }
|
||||
getrandom = { version = "0.2.10", features = ["js"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde = { workspace = true }
|
||||
rle = { path = "../rle", package = "loro-rle" }
|
||||
tracing-wasm = "0.2.1"
|
||||
tracing = { version = "0.1" }
|
||||
serde_json = "1"
|
||||
|
||||
[features]
|
||||
default = ["console_error_panic_hook"]
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ use loro_internal::{
|
|||
obs::SubID,
|
||||
undo::{UndoItemMeta, UndoOrRedo},
|
||||
version::Frontiers,
|
||||
ContainerType, DiffEvent, HandlerTrait, LoroDoc, LoroValue, MovableListHandler,
|
||||
ContainerType, DiffEvent, HandlerTrait, JsonSchema, LoroDoc, LoroValue, MovableListHandler,
|
||||
UndoManager as InnerUndoManager, VersionVector as InternalVersionVector,
|
||||
};
|
||||
use rle::HasLength;
|
||||
|
|
@ -159,6 +159,10 @@ extern "C" {
|
|||
pub type JsCursorQueryAns;
|
||||
#[wasm_bindgen(typescript_type = "UndoConfig")]
|
||||
pub type JsUndoConfig;
|
||||
#[wasm_bindgen(typescript_type = "JsonSchema")]
|
||||
pub type JsJsonSchema;
|
||||
#[wasm_bindgen(typescript_type = "string | JsonSchema")]
|
||||
pub type JsJsonSchemaOrString;
|
||||
}
|
||||
|
||||
mod observer {
|
||||
|
|
@ -873,6 +877,39 @@ impl Loro {
|
|||
}
|
||||
}
|
||||
|
||||
/// Export updates from the specific version to the current version with JSON format.
|
||||
#[wasm_bindgen(js_name = "exportJsonUpdates")]
|
||||
pub fn export_json_updates(&self, vv: Option<VersionVector>) -> JsResult<JsJsonSchema> {
|
||||
let mut json_vv = Default::default();
|
||||
if let Some(vv) = vv {
|
||||
json_vv = vv.0;
|
||||
}
|
||||
let json_schema = self.0.export_json_updates(&json_vv);
|
||||
let s = serde_wasm_bindgen::Serializer::new();
|
||||
let v = json_schema
|
||||
.serialize(&s)
|
||||
.map_err(std::convert::Into::<JsValue>::into)?;
|
||||
Ok(v.into())
|
||||
}
|
||||
|
||||
/// Import updates from the JSON format.
|
||||
///
|
||||
/// only supports backward compatibility but not forward compatibility.
|
||||
#[wasm_bindgen(js_name = "importJsonUpdates")]
|
||||
pub fn import_json_updates(&self, json: JsJsonSchemaOrString) -> JsResult<()> {
|
||||
let json: JsValue = json.into();
|
||||
if JsValue::is_string(&json) {
|
||||
let json_str = json.as_string().unwrap();
|
||||
return self
|
||||
.0
|
||||
.import_json_updates(json_str.as_str())
|
||||
.map_err(|e| e.into());
|
||||
}
|
||||
let json_schema: JsonSchema = serde_wasm_bindgen::from_value(json)?;
|
||||
self.0.import_json_updates(json_schema)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Import a snapshot or a update to current doc.
|
||||
///
|
||||
/// Note:
|
||||
|
|
@ -2726,7 +2763,7 @@ impl LoroTreeNode {
|
|||
/// // / \
|
||||
/// // node2 node
|
||||
/// ```
|
||||
#[wasm_bindgen(js_name = "createNode")]
|
||||
#[wasm_bindgen(js_name = "createNode", skip_typescript)]
|
||||
pub fn create_node(&self, index: Option<usize>) -> JsResult<LoroTreeNode> {
|
||||
let id = if let Some(index) = index {
|
||||
self.tree.create_at(Some(self.id), index)?
|
||||
|
|
@ -2864,7 +2901,7 @@ impl LoroTreeNode {
|
|||
///
|
||||
/// The objects returned are new js objects each time because they need to cross
|
||||
/// the WASM boundary.
|
||||
#[wasm_bindgen]
|
||||
#[wasm_bindgen(skip_typescript)]
|
||||
pub fn children(&self) -> Array {
|
||||
let children = self.tree.children(Some(self.id));
|
||||
let children = children.into_iter().map(|c| {
|
||||
|
|
@ -2912,7 +2949,7 @@ impl LoroTree {
|
|||
/// // / \
|
||||
/// // node root
|
||||
/// ```
|
||||
#[wasm_bindgen(js_name = "createNode")]
|
||||
#[wasm_bindgen(js_name = "createNode", skip_typescript)]
|
||||
pub fn create_node(
|
||||
&mut self,
|
||||
parent: &JsParentTreeID,
|
||||
|
|
@ -2983,7 +3020,7 @@ impl LoroTree {
|
|||
}
|
||||
|
||||
/// Get LoroTreeNode by the TreeID.
|
||||
#[wasm_bindgen(js_name = "getNodeByID")]
|
||||
#[wasm_bindgen(js_name = "getNodeByID", skip_typescript)]
|
||||
pub fn get_node_by_id(&self, target: &JsTreeID) -> Option<LoroTreeNode> {
|
||||
let target: JsValue = target.into();
|
||||
let target = TreeID::try_from(target).ok()?;
|
||||
|
|
@ -4020,3 +4057,125 @@ interface LoroMovableList {
|
|||
|
||||
export type Side = -1 | 0 | 1;
|
||||
"#;
|
||||
|
||||
#[wasm_bindgen(typescript_custom_section)]
|
||||
const JSON_SCHEMA_TYPES: &'static str = r#"
|
||||
export type JsonOpID = `${number}@${PeerID}`;
|
||||
export type JsonContainerID = `🦜:${ContainerID}` ;
|
||||
export type JsonValue =
|
||||
| JsonContainerID
|
||||
| string
|
||||
| number
|
||||
| boolean
|
||||
| null
|
||||
| { [key: string]: JsonValue }
|
||||
| Uint8Array
|
||||
| JsonValue[];
|
||||
|
||||
export type JsonSchema = {
|
||||
schema_version: number;
|
||||
start_version: Map<string, number>,
|
||||
peers: PeerID[],
|
||||
changes: JsonChange[]
|
||||
};
|
||||
|
||||
export type JsonChange = {
|
||||
id: JsonOpID
|
||||
timestamp: number,
|
||||
deps: JsonOpID[],
|
||||
lamport: number,
|
||||
msg: string | null,
|
||||
ops: JsonOp[]
|
||||
}
|
||||
|
||||
export type JsonOp = {
|
||||
container: ContainerID,
|
||||
counter: number,
|
||||
content: ListOp | TextOp | MapOp | TreeOp | MovableListOp | UnknownOp
|
||||
}
|
||||
|
||||
export type ListOp = {
|
||||
type: "insert",
|
||||
pos: number,
|
||||
value: JsonValue
|
||||
} | {
|
||||
type: "delete",
|
||||
pos: number,
|
||||
len: number,
|
||||
start_id: JsonOpID,
|
||||
};
|
||||
|
||||
export type MovableListOp = {
|
||||
type: "insert",
|
||||
pos: number,
|
||||
value: JsonValue
|
||||
} | {
|
||||
type: "delete",
|
||||
pos: number,
|
||||
len: number,
|
||||
start_id: JsonOpID,
|
||||
}| {
|
||||
type: "move",
|
||||
from: number,
|
||||
to: number,
|
||||
elem_id: JsonOpID,
|
||||
}|{
|
||||
type: "set",
|
||||
elem_id: JsonOpID,
|
||||
value: JsonValue
|
||||
}
|
||||
|
||||
export type TextOp = {
|
||||
type: "insert",
|
||||
pos: number,
|
||||
text: string
|
||||
} | {
|
||||
type: "delete",
|
||||
pos: number,
|
||||
len: number,
|
||||
start_id: JsonOpID,
|
||||
} | {
|
||||
type: "mark",
|
||||
start: number,
|
||||
end: number,
|
||||
style_key: string,
|
||||
style_value: JsonValue,
|
||||
info: number
|
||||
}|{
|
||||
type: "mark_end"
|
||||
};
|
||||
|
||||
export type MapOp = {
|
||||
type: "insert",
|
||||
key: string,
|
||||
value: JsonValue
|
||||
} | {
|
||||
type: "delete",
|
||||
key: string,
|
||||
};
|
||||
|
||||
export type TreeOp = {
|
||||
type: "create",
|
||||
target: TreeID,
|
||||
parent: TreeID | undefined,
|
||||
fractional_index: string
|
||||
}|{
|
||||
type: "move",
|
||||
target: TreeID,
|
||||
parent: TreeID | undefined,
|
||||
fractional_index: string
|
||||
}|{
|
||||
type: "delete",
|
||||
target: TreeID
|
||||
};
|
||||
|
||||
export type UnknownOp = {
|
||||
type: "unknown"
|
||||
prop: number,
|
||||
value_type: "unknown",
|
||||
value: {
|
||||
kind: number,
|
||||
data: Uint8Array
|
||||
}
|
||||
};
|
||||
"#;
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ use loro_internal::handler::HandlerTrait;
|
|||
use loro_internal::handler::ValueOrHandler;
|
||||
use loro_internal::loro::CommitOptions;
|
||||
use loro_internal::undo::{OnPop, OnPush};
|
||||
use loro_internal::JsonSchema;
|
||||
use loro_internal::LoroDoc as InnerLoroDoc;
|
||||
use loro_internal::OpLog;
|
||||
|
||||
|
|
@ -289,6 +290,18 @@ impl LoroDoc {
|
|||
self.doc.import_with(bytes, origin.into())
|
||||
}
|
||||
|
||||
/// Import the json schema updates.
|
||||
///
|
||||
/// only supports backward compatibility but not forward compatibility.
|
||||
pub fn import_json_updates<T: TryInto<JsonSchema>>(&self, json: T) -> Result<(), LoroError> {
|
||||
self.doc.import_json_updates(json)
|
||||
}
|
||||
|
||||
/// Export the current state with json-string format of the document.
|
||||
pub fn export_json_updates(&self, vv: &VersionVector) -> JsonSchema {
|
||||
self.doc.export_json_updates(vv)
|
||||
}
|
||||
|
||||
/// Export all the ops not included in the given `VersionVector`
|
||||
pub fn export_from(&self, vv: &VersionVector) -> Vec<u8> {
|
||||
self.doc.export_from(vv)
|
||||
|
|
|
|||
|
|
@ -357,6 +357,24 @@ fn map() -> LoroResult<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tree() {
|
||||
use loro::{LoroDoc, ToJson};
|
||||
|
||||
let doc = LoroDoc::new();
|
||||
doc.set_peer_id(1).unwrap();
|
||||
let tree = doc.get_tree("tree");
|
||||
let root = tree.create(None).unwrap();
|
||||
let root2 = tree.create(None).unwrap();
|
||||
tree.mov(root2, root).unwrap();
|
||||
let root_meta = tree.get_meta(root).unwrap();
|
||||
root_meta.insert("color", "red").unwrap();
|
||||
assert_eq!(
|
||||
tree.get_value_with_meta().to_json(),
|
||||
r#"[{"parent":null,"meta":{"color":"red"},"id":"0@1","index":0,"position":"80"},{"parent":"0@1","meta":{},"id":"1@1","index":0,"position":"80"}]"#
|
||||
)
|
||||
}
|
||||
|
||||
fn check_sync_send(_doc: impl Sync + Send) {}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ keywords = ["crdt", "local-first"]
|
|||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
smallvec = "1.8.0"
|
||||
smallvec = {workspace=true}
|
||||
num = "0.4.0"
|
||||
enum-as-inner = "0.6.0"
|
||||
arref = "0.1.0"
|
||||
|
|
|
|||
391
docs/JsonSchema.md
Normal file
391
docs/JsonSchema.md
Normal file
|
|
@ -0,0 +1,391 @@
|
|||
# JSON Schema for Loro's OpLog
|
||||
|
||||
## Introduction
|
||||
|
||||
Loro supports multiple data structures and introduces many new concepts. Having only binary export formats would make it difficult for developers to understand the underlying processes. Better transparency leads to better developer experience. A human-readable JSON representation enables users to better understand and operate the document and to develop related tools.
|
||||
|
||||
To better understand this document, you may first need to understand how Loro stores historical editing data:
|
||||
|
||||
- [OpLog](https://www.loro.dev/docs/advanced/doc_state_and_oplog)
|
||||
- [`Change`, `Operation`](https://www.loro.dev/docs/advanced/op_and_change)
|
||||
- [`Replayable Event Graph (REG)`](https://www.loro.dev/docs/advanced/replayable_event_graph)
|
||||
|
||||
It should be noted that considering the usage scenario, JSON Schema only supports backward compatibility but not forward compatibility.
|
||||
|
||||
# Specification
|
||||
|
||||
## Root object
|
||||
|
||||
The root object contains all `Change`s, `Op`s, and critical metadata like start/end versions and schema version.
|
||||
|
||||
We will also extract the 64-bit integer PeerID to the beginning of the document and replace it internally with incrementing numbers starting from zero: 0, 1, 2, 3... This significantly reduces the document size and enhances readability.
|
||||
|
||||
```ts
|
||||
{
|
||||
"schema_version": number,
|
||||
"start_version": Map<string, number>,
|
||||
"peers": string[],
|
||||
"changes": Change[],
|
||||
}
|
||||
```
|
||||
|
||||
- `schema_version`: the version of the schema that the document is encoded with. It's 1 for the current specification.
|
||||
- `start_version`: the start `Frontiers` version of the document. They are represented as a map from the decimal string representation of `PeerID` to `Counter`.
|
||||
- `peers`: the list of peers in the document. We represent all PeerIDs as decimal strings to avoid exceeding JavaScript's number limit.
|
||||
- `changes`: the list of changes in the document.
|
||||
|
||||
## Changes
|
||||
|
||||
`Change`s are crucial in the OpLog. A REG([Replay event graph](https://www.loro.dev/docs/advanced/replayable_event_graph)) is a directed acyclic graph where each node is a `Change`, and each edge is a causal dependency between `Change`s. The metadata of the `Change`s helps us reconstruct the graph.
|
||||
|
||||
You can also attach a commit message to a `Change` like you usually do with Git's commit.
|
||||
|
||||
```ts
|
||||
{
|
||||
"id": string,
|
||||
"timestamp": number,
|
||||
"deps": OpID[],
|
||||
"lamport": number,
|
||||
"msg": string,
|
||||
"ops": Op[]
|
||||
}
|
||||
|
||||
type OpID = `${number}@${PeerID}`;
|
||||
```
|
||||
|
||||
- `id`: the string representation of the unique `ID` of each `Change`, in the form of `{Counter}@{PeerID}` which is the `@` character connecting `Counter` and `PeerID`. Of course, This `PeerID` is the index of peers in the global context.
|
||||
- `timestamp`: the number of Unix timestamp when the change is committed. [Timestamp is not recorded by default](https://loro.dev/docs/advanced/timestamp)
|
||||
- `deps`: a list of causal dependency of this `Change`, each item is the `ID` represented by a string.
|
||||
- `lamport`: the lamport timestamp of the `Change`.
|
||||
- `msg`: the commit message.
|
||||
- `ops`: all of the `Op` in the `Change`.
|
||||
|
||||
## Operations
|
||||
|
||||
Operation (abbreviated as `Op`) is the most complex part of the document. Loro currently supports multiple containers `List`, `Map`, `RichText`, `Movable List` and `Movable Tree`. Each data structure has several different `Op`s.
|
||||
|
||||
But in general, each `Op` is composed of the `ContainerID` of the container that created it, a counter, and the corresponding content of the `Op`.
|
||||
|
||||
```ts
|
||||
type Op = {
|
||||
"container": ContainerID,
|
||||
"counter": number,
|
||||
"content": OpContent // Its detailed definition is elaborated below, with different types for different Containers.
|
||||
};
|
||||
|
||||
type OpContent = ListOp | TextOp | MapOp | TreeOp | MovableListOp | UnknownOp;
|
||||
type ContainerID =
|
||||
| `cid:root-${string}:${ContainerType}`
|
||||
| `cid:${number}@${PeerID}:${ContainerType}`;
|
||||
```
|
||||
|
||||
- `container`: the `ContainerID` of the container that created this `Op`, represented by a string starts with `cid:`.
|
||||
- `counter`: the counter part of the OpID
|
||||
- `content`: the semantic content of the `Op`, it is different for each field depending on the `Container`.
|
||||
|
||||
The following is the **content** of each container。
|
||||
|
||||
### List
|
||||
|
||||
```ts
|
||||
type ListOp = ListInsertOp | ListDeleteOp;
|
||||
```
|
||||
|
||||
#### Insert
|
||||
|
||||
```ts
|
||||
type ListInsertOp = {
|
||||
"type": "insert",
|
||||
"pos": number,
|
||||
"value": LoroValue
|
||||
}
|
||||
```
|
||||
|
||||
- `type`: `insert`.
|
||||
- `pos`: the index of the insert operation.
|
||||
- `value`: the insert content which is a list of `LoroValue`
|
||||
|
||||
#### Delete
|
||||
|
||||
```ts
|
||||
type ListDeleteOp = {
|
||||
"type": "delete",
|
||||
"pos": number,
|
||||
"len": number,
|
||||
"start_id": OpID
|
||||
}
|
||||
```
|
||||
|
||||
- `type`: `delete`.
|
||||
- `pos`: the start index of the deletion.
|
||||
- `len`: the length of deleted content.
|
||||
- `start_id`: the string id of start element deleted.
|
||||
|
||||
### MovableList
|
||||
|
||||
```ts
|
||||
type MovableListOp = ListInsertOp | ListDeleteOp | MovableListMoveOp | MovableListSetOp;
|
||||
```
|
||||
|
||||
#### Insert
|
||||
|
||||
```ts
|
||||
type ListInsertOp = {
|
||||
"type": "insert",
|
||||
"pos": number,
|
||||
"value": LoroValue
|
||||
}
|
||||
```
|
||||
|
||||
- `type`: `insert`,
|
||||
- `pos`: the index of the insert operation.
|
||||
- `value`: the insert content which is a list of `LoroValue`
|
||||
|
||||
#### Delete
|
||||
|
||||
```ts
|
||||
type ListDeleteOp = {
|
||||
"type": "delete",
|
||||
"pos": number,
|
||||
"len": number,
|
||||
"start_id": OpID
|
||||
}
|
||||
```
|
||||
|
||||
- `type`: `delete`
|
||||
- `pos`: the start index of the deletion.
|
||||
- `len`: the length of deleted content.
|
||||
- `start_id`: the string id of start element deleted.
|
||||
|
||||
#### Move
|
||||
|
||||
```ts
|
||||
type MovableListMoveOp = {
|
||||
"type": "move",
|
||||
"from": number,
|
||||
"to": number,
|
||||
"elem_id": ElemID
|
||||
}
|
||||
|
||||
type ElemID = `L${number}@${PeerID}`
|
||||
```
|
||||
|
||||
- `type`:`insert`, `delete`, `move` or `set`.
|
||||
- `from`: the index of the element before is moved.
|
||||
- `to`: the index of the index moved to after moving out the element
|
||||
- `elem_id`: the ID (described by lamport@peer) of the element moved.
|
||||
|
||||
#### Set
|
||||
|
||||
```ts
|
||||
type MovableListSetOp = {
|
||||
"type": "set",
|
||||
"elem_id": ElemID,
|
||||
"value": LoroValue
|
||||
}
|
||||
|
||||
type ElemID = `L${number}@${PeerID}`
|
||||
```
|
||||
|
||||
- `type`:`insert`, `delete`, `move` or `set`.
|
||||
- `elem_id`: the ID (described by lamport@peer) of the element replaced.
|
||||
- `value`: the value set.
|
||||
|
||||
### Map
|
||||
|
||||
```ts
|
||||
type MapOp = MapInsertOp | MapDeleteOp;
|
||||
```
|
||||
|
||||
#### Insert
|
||||
|
||||
```ts
|
||||
type MapInsertOp = {
|
||||
"type": "insert",
|
||||
"key": string,
|
||||
"value": LoroValue
|
||||
}
|
||||
```
|
||||
|
||||
- `type`: `insert`.
|
||||
- `key`: the key of the insertion.
|
||||
- `value`: the value of the insertion.
|
||||
|
||||
#### Delete
|
||||
|
||||
```ts
|
||||
type MapDeleteOp = {
|
||||
"type": "delete",
|
||||
"key": string
|
||||
}
|
||||
```
|
||||
|
||||
- `type`: `delete`.
|
||||
- `key`: the key of the deletion
|
||||
|
||||
### Text
|
||||
|
||||
```ts
|
||||
type TextOp = TextInsertOp | TextDeleteOp | TextMarkOp | TextMarkEndOp;
|
||||
```
|
||||
|
||||
#### Insert
|
||||
|
||||
```ts
|
||||
type TextInsertOp = {
|
||||
"type": "insert",
|
||||
"pos": number,
|
||||
"text": string
|
||||
}
|
||||
```
|
||||
|
||||
`type`: `insert`.
|
||||
`pos`: the index of the insert operation. The position is based on the Unicode code point length.
|
||||
`text`: the string of the insertion.
|
||||
|
||||
#### Delete
|
||||
|
||||
```ts
|
||||
type TextDeleteOp = {
|
||||
"type": "delete",
|
||||
"pos": number,
|
||||
"len": number,
|
||||
"start_id": OpID
|
||||
}
|
||||
```
|
||||
|
||||
`type`: `delete`.
|
||||
`pos`: the index of the deletion. The position is based on the Unicode code point length.
|
||||
`len`: the length of the text deleted.
|
||||
`start_id`: the string id of the beginning element deleted.
|
||||
|
||||
|
||||
#### Mark
|
||||
|
||||
```ts
|
||||
type TextMarkOp = {
|
||||
"type": "mark",
|
||||
"start": number,
|
||||
"end": number,
|
||||
"style_key": string,
|
||||
"style_value": LoroValue,
|
||||
"info": number
|
||||
}
|
||||
```
|
||||
|
||||
`type`: `mark`
|
||||
`start`: the start index of text need to mark. The position is based on the Unicode code point length.
|
||||
`end`: the end index of text need to mark. The position is based on the Unicode code point length.
|
||||
`style_key`: the key of style, it is customizable.
|
||||
`style_value`: the value of style, it is customizable.
|
||||
`info`: the config of the style, whether to expand the style when inserting new text around it.
|
||||
|
||||
#### MarkEnd
|
||||
|
||||
```ts
|
||||
type TextMarkEndOp = {
|
||||
"type": "mark_end"
|
||||
}
|
||||
```
|
||||
|
||||
`type`: `mark_end`.
|
||||
|
||||
### Tree
|
||||
|
||||
```ts
|
||||
type TreeOp = TreeCreateOp | TreeMoveOp | TreeDeleteOp;
|
||||
```
|
||||
|
||||
#### Create
|
||||
|
||||
```ts
|
||||
type TreeCreateOp = {
|
||||
"type": "create",
|
||||
"target": TreeID,
|
||||
"parent": TreeID | null,
|
||||
"fractional_index": string
|
||||
}
|
||||
|
||||
type TreeID = `${number}@${PeerID}`
|
||||
```
|
||||
|
||||
- `type`: `create`.
|
||||
- `target`: the string format of target `TreeID` moved.
|
||||
- `parent`: the string format of `TreeID` or `null`. If it is `null`, the target node will be a root node.
|
||||
- `fractional_index`: the fractional index with hex string format of the target node.
|
||||
|
||||
#### Move
|
||||
|
||||
```ts
|
||||
type TreeMoveOp = {
|
||||
"type": "move",
|
||||
"target": TreeID,
|
||||
"parent": TreeID | null,
|
||||
"fractional_index": string
|
||||
}
|
||||
|
||||
type TreeID = `${number}@${PeerID}`
|
||||
```
|
||||
|
||||
- `type`: `move`.
|
||||
- `target`: the string format of target `TreeID` moved.
|
||||
- `parent`: the string format of `TreeID` or `null`. If it is `null`, the target node will be a root node.
|
||||
- `fractional_index`: the fractional index with hex string format of the target node.
|
||||
|
||||
#### Delete
|
||||
|
||||
```ts
|
||||
type TreeDeleteOp = {
|
||||
"type": "delete",
|
||||
"target": TreeID
|
||||
}
|
||||
|
||||
type TreeID = `${number}@${PeerID}`
|
||||
```
|
||||
|
||||
- `type`: `delete`.
|
||||
- `target`: the string format of target `TreeID` deleted.
|
||||
|
||||
### Unknown
|
||||
|
||||
To support forward compatibility, we have an unknown type. When an `Op` with a newly supported Container from a newer version is decoded into the older version, it will be treated as an unknown type in a more general form, such as binary and string. When the new version decodes an unknown `Op`, the newer version of Loro will know its true type and decode correctly.
|
||||
|
||||
```ts
|
||||
type UnknownOp = {
|
||||
"type": "unknown",
|
||||
"prop": number,
|
||||
"value_type": string,
|
||||
"value": `${EncodeValue}`
|
||||
}
|
||||
```
|
||||
|
||||
- `type`: just an unknown type.
|
||||
- `prop`: a property of the encoded op, it's a number.
|
||||
- `value_type`: the type of `EncodeValue`.
|
||||
- `value`: common data types used in encoding with json string format.
|
||||
|
||||
## Value
|
||||
|
||||
In this section, we will introduction two *Value* in Loro. One is `LoroValue`, it's an enum of data types supported by Loro, such as the value inserted by `List` or `Map`.
|
||||
|
||||
The another is `EncodedValue`, it's just used in encoding module for unknown type.
|
||||
|
||||
### LoroValue
|
||||
|
||||
These are data types supported by Loro and its json format:
|
||||
|
||||
- `null`: `null`
|
||||
- `Bool`: `true` or `false`
|
||||
- `F64`: `number`(float)
|
||||
- `I64`: `number` or `bigint` (signed)
|
||||
- `Binary`: `UInt8Array`
|
||||
- `String`: `string`
|
||||
- `List`: `Array<LoroValue>`
|
||||
- `Map`: `Map<string, LoroValue>`
|
||||
- `Container`: the id of container. `🦜:cid:{Counter}@{PeerID}:{ContainerType}` or `🦜:cid:root-{Name}:{ContainerType}`
|
||||
|
||||
Note: Compared with the string format, we add a prefix `🦜:` when encoding the json format of `ContainerID` to prevent users from saving the string format of `ContainerID` and misinterpreting it as `ContainerID` when decoding.
|
||||
|
||||
### EncodedValue
|
||||
|
||||
The `EncodedValue` is the specific type used by Loro when encoding, it's an internal value, users do not need to get it clear. It is specially designed to handle the schema mismatch due to forward and backward compatibility. In JSON encoding schema, the `EncodedValue` will be encoded as an object.
|
||||
|
|
@ -564,11 +564,34 @@ declare module "loro-wasm" {
|
|||
T extends Record<string, unknown> = Record<string, unknown>,
|
||||
> {
|
||||
new (): LoroTree<T>;
|
||||
/**
|
||||
* Create a new tree node as the child of parent and return a `LoroTreeNode` instance.
|
||||
* If the parent is undefined, the tree node will be a root node.
|
||||
*
|
||||
* If the index is not provided, the new node will be appended to the end.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* import { Loro } from "loro-crdt";
|
||||
*
|
||||
* const doc = new Loro();
|
||||
* const tree = doc.getTree("tree");
|
||||
* const root = tree.createNode();
|
||||
* const node = tree.createNode(undefined, 0);
|
||||
*
|
||||
* // undefined
|
||||
* // / \
|
||||
* // node root
|
||||
* ```
|
||||
*/
|
||||
createNode(parent?: TreeID, index?: number): LoroTreeNode<T>;
|
||||
move(target: TreeID, parent?: TreeID, index?: number): void;
|
||||
delete(target: TreeID): void;
|
||||
has(target: TreeID): boolean;
|
||||
getNodeByID(target: TreeID): LoroTreeNode;
|
||||
/**
|
||||
* Get LoroTreeNode by the TreeID.
|
||||
*/
|
||||
getNodeByID(target: TreeID): LoroTreeNode<T>;
|
||||
subscribe(listener: Listener): number;
|
||||
}
|
||||
|
||||
|
|
@ -579,9 +602,35 @@ declare module "loro-wasm" {
|
|||
* Get the associated metadata map container of a tree node.
|
||||
*/
|
||||
readonly data: LoroMap<T>;
|
||||
/**
|
||||
* Create a new node as the child of the current node and
|
||||
* return an instance of `LoroTreeNode`.
|
||||
*
|
||||
* If the index is not provided, the new node will be appended to the end.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { Loro } from "loro-crdt";
|
||||
*
|
||||
* let doc = new Loro();
|
||||
* let tree = doc.getTree("tree");
|
||||
* let root = tree.createNode();
|
||||
* let node = root.createNode();
|
||||
* let node2 = root.createNode(0);
|
||||
* // root
|
||||
* // / \
|
||||
* // node2 node
|
||||
* ```
|
||||
*/
|
||||
createNode(index?: number): LoroTreeNode<T>;
|
||||
move(parent?: LoroTreeNode<T>, index?: number): void;
|
||||
parent(): LoroTreeNode<T> | undefined;
|
||||
/**
|
||||
* Get the children of this node.
|
||||
*
|
||||
* The objects returned are new js objects each time because they need to cross
|
||||
* the WASM boundary.
|
||||
*/
|
||||
children(): Array<LoroTreeNode<T>>;
|
||||
}
|
||||
|
||||
|
|
|
|||
165
loro-js/tests/json.test.ts
Normal file
165
loro-js/tests/json.test.ts
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
import { expect, it } from "vitest";
|
||||
import {
|
||||
Loro,
|
||||
LoroMap,
|
||||
TextOp,
|
||||
} from "../src";
|
||||
|
||||
it("json encoding", () => {
|
||||
const doc = new Loro();
|
||||
const text = doc.getText("text");
|
||||
text.insert(0, "123");
|
||||
const map = doc.getMap("map");
|
||||
const list = doc.getList("list");
|
||||
const movableList = doc.getMovableList("movableList");
|
||||
const tree = doc.getTree("tree");
|
||||
const subMap = map.setContainer("subMap", new LoroMap());
|
||||
subMap.set("foo", "bar");
|
||||
list.push("foo");
|
||||
list.push("🦜");
|
||||
movableList.push("move list");
|
||||
movableList.push("🦜");
|
||||
movableList.move(1, 0);
|
||||
const root = tree.createNode(undefined);
|
||||
const child = tree.createNode(root.id);
|
||||
child.data.set("tree", "abc");
|
||||
text.mark({ start: 0, end: 3 }, "bold", true);
|
||||
const json = doc.exportJsonUpdates();
|
||||
// console.log(json.changes[0].ops);
|
||||
const doc2 = new Loro();
|
||||
doc2.importJsonUpdates(json);
|
||||
});
|
||||
|
||||
it("json decoding", () => {
|
||||
const v15Json = `{
|
||||
"schema_version": 1,
|
||||
"start_version": {},
|
||||
"peers": [
|
||||
"14944917281143706156"
|
||||
],
|
||||
"changes": [
|
||||
{
|
||||
"id": "0@0",
|
||||
"timestamp": 0,
|
||||
"deps": [],
|
||||
"lamport": 0,
|
||||
"msg": null,
|
||||
"ops": [
|
||||
{
|
||||
"container": "cid:root-text:Text",
|
||||
"content": {
|
||||
"type": "insert",
|
||||
"pos": 0,
|
||||
"text": "123"
|
||||
},
|
||||
"counter": 0
|
||||
},
|
||||
{
|
||||
"container": "cid:root-map:Map",
|
||||
"content": {
|
||||
"type": "insert",
|
||||
"key": "subMap",
|
||||
"value": "🦜:cid:3@0:Map"
|
||||
},
|
||||
"counter": 3
|
||||
},
|
||||
{
|
||||
"container": "cid:3@0:Map",
|
||||
"content": {
|
||||
"type": "insert",
|
||||
"key": "foo",
|
||||
"value": "bar"
|
||||
},
|
||||
"counter": 4
|
||||
},
|
||||
{
|
||||
"container": "cid:root-list:List",
|
||||
"content": {
|
||||
"type": "insert",
|
||||
"pos": 0,
|
||||
"value": [
|
||||
"foo",
|
||||
"🦜"
|
||||
]
|
||||
},
|
||||
"counter": 5
|
||||
},
|
||||
{
|
||||
"container": "cid:root-tree:Tree",
|
||||
"content": {
|
||||
"type": "move",
|
||||
"target": "7@0",
|
||||
"parent": null
|
||||
},
|
||||
"counter": 7
|
||||
},
|
||||
{
|
||||
"container": "cid:root-tree:Tree",
|
||||
"content": {
|
||||
"type": "move",
|
||||
"target": "8@0",
|
||||
"parent": "7@0"
|
||||
},
|
||||
"counter": 8
|
||||
},
|
||||
{
|
||||
"container": "cid:8@0:Map",
|
||||
"content": {
|
||||
"type": "insert",
|
||||
"key": "tree",
|
||||
"value": "abc"
|
||||
},
|
||||
"counter": 9
|
||||
},
|
||||
{
|
||||
"container": "cid:root-text:Text",
|
||||
"content": {
|
||||
"type": "mark",
|
||||
"start": 0,
|
||||
"end": 3,
|
||||
"style_key": "bold",
|
||||
"style_value": true,
|
||||
"info": 132
|
||||
},
|
||||
"counter": 10
|
||||
},
|
||||
{
|
||||
"container": "cid:root-text:Text",
|
||||
"content": {
|
||||
"type": "mark_end"
|
||||
},
|
||||
"counter": 11
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}`;
|
||||
const doc = new Loro();
|
||||
doc.importJsonUpdates(v15Json);
|
||||
// console.log(doc.exportJsonUpdates());
|
||||
});
|
||||
|
||||
it("test some type correctness", () => {
|
||||
const doc = new Loro();
|
||||
doc.setPeerId(0);
|
||||
doc.getText("text").insert(0, "123");
|
||||
doc.commit();
|
||||
doc.getText("text").delete(2, 1);
|
||||
doc.getText("text").delete(1, 1);
|
||||
doc.getText("text").delete(0, 1);
|
||||
doc.commit();
|
||||
const updates = doc.exportJsonUpdates();
|
||||
expect(updates.start_version).toBeDefined();
|
||||
expect(updates.changes.length).toBe(1);
|
||||
expect(updates.changes[0].ops[0].content).toStrictEqual({
|
||||
type: "insert",
|
||||
pos: 0,
|
||||
text: "123",
|
||||
} as TextOp);
|
||||
expect(updates.changes[0].ops[1].content).toStrictEqual({
|
||||
type: "delete",
|
||||
pos: 2,
|
||||
len: -3,
|
||||
start_id: "0@0",
|
||||
} as TextOp);
|
||||
});
|
||||
Loading…
Reference in a new issue