mirrors/loro
1
0
Fork 0
mirror of https://github.com/loro-dev/loro.git synced 2025-02-02 02:59:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge main

Browse source
This commit is contained in:
Zixuan Chen 2024-07-12 16:15:54 +08:00
parent 23fbae1f80
commit 46000420e8
No known key found for this signature in database
60 changed files with 8850 additions and 627 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/release_wasm.yml vendored
View file

@ -13,7 +13,7 @@ jobs:
- name: Checkout Repo
uses: actions/checkout@v4
- uses: pnpm/action-setup@v2
- uses: pnpm/action-setup@v4
with:
version: 8

2
.github/workflows/rust.yml vendored
View file

@ -26,7 +26,7 @@ jobs:
- uses: actions/setup-node@v3
with:
node-version: 20
- uses: pnpm/action-setup@v2
- uses: pnpm/action-setup@v4
with:
version: 8
- name: Install nextest

29
Cargo.lock generated
View file

@ -673,8 +673,7 @@ dependencies = [
"fxhash",
"itertools 0.12.1",
"loro 0.16.2",
"loro 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro-common 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"rand",
"serde_json",
"tabled 0.10.0",
@ -1011,13 +1010,13 @@ dependencies = [
[[package]]
name = "loro"
version = "0.16.2"
source = "git+https://github.com/loro-dev/loro.git?branch=main#afac34755f3f9a099c2985ff22dc9f2534d72290"
source = "git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761#90470658435ec4c62b5af59ebb82fe9e1f5aa761"
dependencies = [
"either",
"enum-as-inner 0.6.0",
"generic-btree",
"loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro-internal 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"loro-internal 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"tracing",
]
@ -1042,12 +1041,12 @@ dependencies = [
[[package]]
name = "loro-common"
version = "0.16.2"
source = "git+https://github.com/loro-dev/loro.git?branch=main#afac34755f3f9a099c2985ff22dc9f2534d72290"
source = "git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761#90470658435ec4c62b5af59ebb82fe9e1f5aa761"
dependencies = [
"arbitrary",
"enum-as-inner 0.6.0",
"fxhash",
"loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"nonmax",
"serde",
"serde_columnar",
@ -1074,7 +1073,7 @@ dependencies = [
[[package]]
name = "loro-delta"
version = "0.16.2"
source = "git+https://github.com/loro-dev/loro.git?branch=main#afac34755f3f9a099c2985ff22dc9f2534d72290"
source = "git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761#90470658435ec4c62b5af59ebb82fe9e1f5aa761"
dependencies = [
"arrayvec",
"enum-as-inner 0.5.1",
@ -1138,7 +1137,7 @@ dependencies = [
[[package]]
name = "loro-internal"
version = "0.16.2"
source = "git+https://github.com/loro-dev/loro.git?branch=main#afac34755f3f9a099c2985ff22dc9f2534d72290"
source = "git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761#90470658435ec4c62b5af59ebb82fe9e1f5aa761"
dependencies = [
"append-only-bytes",
"arref",
@ -1151,10 +1150,10 @@ dependencies = [
"im",
"itertools 0.12.1",
"leb128",
"loro-common 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro_fractional_index 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro-common 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"loro_fractional_index 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"md5",
"num",
"num-derive",
@ -1190,7 +1189,7 @@ dependencies = [
[[package]]
name = "loro-rle"
version = "0.16.2"
source = "git+https://github.com/loro-dev/loro.git?branch=main#afac34755f3f9a099c2985ff22dc9f2534d72290"
source = "git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761#90470658435ec4c62b5af59ebb82fe9e1f5aa761"
dependencies = [
"append-only-bytes",
"arref",
@ -1239,7 +1238,7 @@ dependencies = [
[[package]]
name = "loro_fractional_index"
version = "0.16.2"
source = "git+https://github.com/loro-dev/loro.git?branch=main#afac34755f3f9a099c2985ff22dc9f2534d72290"
source = "git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761#90470658435ec4c62b5af59ebb82fe9e1f5aa761"
dependencies = [
"imbl",
"rand",

2
crates/fractional_index/src/lib.rs
View file

@ -74,7 +74,7 @@ pub(crate) fn new_after(bytes: &[u8]) -> Vec<u8> {
}
pub(crate) fn new_between(left: &[u8], right: &[u8], extra_capacity: usize) -> Option<Vec<u8>> {
let shorter_len = left.len().min(right.len());
let shorter_len = left.len().min(right.len()) - 1;
for i in 0..shorter_len {
if left[i] < right[i] - 1 {
let mut ans: Vec<u8> = left[0..=i].into();

15
crates/fuzz/Cargo.toml
View file

@ -7,18 +7,8 @@ publish = false
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
loro-without-counter = { path = "../loro", package = "loro" }
loro = { git = "https://github.com/loro-dev/loro.git", features = [
"counter",
], branch = "main" }
loro-common = { git = "https://github.com/loro-dev/loro.git", features = [
"counter",
], branch = "main" }
# loro = { path = "../loro", package = "loro", features = ["counter"] }
# loro-common = { path = "../loro-common", package = "loro-common", features = [
# "counter",
# ] }
# loro-without-counter = { git = "https://github.com/loro-dev/loro.git", branch = "main", package = "loro" }
loro = { path = "../loro", features = ["counter"], package = "loro" }
loro-without-counter = { git = "https://github.com/loro-dev/loro.git", rev = "90470658435ec4c62b5af59ebb82fe9e1f5aa761", package = "loro", default-features = false }
fxhash = { workspace = true }
enum_dispatch = { workspace = true }
enum-as-inner = { workspace = true }
@ -27,6 +17,7 @@ itertools = { workspace = true }
arbitrary = "1"
tabled = "0.10"
rand = "0.8.5"
serde_json = "1"
[dev-dependencies]
ctor = "0.2"

30
crates/fuzz/fuzz/Cargo.lock generated
View file

@ -222,9 +222,9 @@ dependencies = [
"fxhash",
"itertools 0.12.1",
"loro 0.16.2",
"loro 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro-common 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"rand",
"serde_json",
"tabled",
"tracing",
]
@ -442,13 +442,13 @@ dependencies = [
[[package]]
name = "loro"
version = "0.16.2"
source = "git+https://github.com/loro-dev/loro.git?branch=main#2940029b637328af2c8592070c5e36689b7df367"
source = "git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761#90470658435ec4c62b5af59ebb82fe9e1f5aa761"
dependencies = [
"either",
"enum-as-inner 0.6.0",
"generic-btree",
"loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro-internal 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"loro-internal 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"tracing",
]
@ -471,12 +471,12 @@ dependencies = [
[[package]]
name = "loro-common"
version = "0.16.2"
source = "git+https://github.com/loro-dev/loro.git?branch=main#2940029b637328af2c8592070c5e36689b7df367"
source = "git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761#90470658435ec4c62b5af59ebb82fe9e1f5aa761"
dependencies = [
"arbitrary",
"enum-as-inner 0.6.0",
"fxhash",
"loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"nonmax",
"serde",
"serde_columnar",
@ -498,7 +498,7 @@ dependencies = [
[[package]]
name = "loro-delta"
version = "0.16.2"
source = "git+https://github.com/loro-dev/loro.git?branch=main#2940029b637328af2c8592070c5e36689b7df367"
source = "git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761#90470658435ec4c62b5af59ebb82fe9e1f5aa761"
dependencies = [
"arrayvec",
"enum-as-inner 0.5.1",
@ -545,7 +545,7 @@ dependencies = [
[[package]]
name = "loro-internal"
version = "0.16.2"
source = "git+https://github.com/loro-dev/loro.git?branch=main#2940029b637328af2c8592070c5e36689b7df367"
source = "git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761#90470658435ec4c62b5af59ebb82fe9e1f5aa761"
dependencies = [
"append-only-bytes",
"arref",
@ -558,10 +558,10 @@ dependencies = [
"im",
"itertools 0.12.1",
"leb128",
"loro-common 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro_fractional_index 0.16.2 (git+https://github.com/loro-dev/loro.git?branch=main)",
"loro-common 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"loro-delta 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"loro-rle 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"loro_fractional_index 0.16.2 (git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761)",
"md5",
"num",
"num-derive",
@ -592,7 +592,7 @@ dependencies = [
[[package]]
name = "loro-rle"
version = "0.16.2"
source = "git+https://github.com/loro-dev/loro.git?branch=main#2940029b637328af2c8592070c5e36689b7df367"
source = "git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761#90470658435ec4c62b5af59ebb82fe9e1f5aa761"
dependencies = [
"append-only-bytes",
"arref",
@ -621,7 +621,7 @@ dependencies = [
[[package]]
name = "loro_fractional_index"
version = "0.16.2"
source = "git+https://github.com/loro-dev/loro.git?branch=main#2940029b637328af2c8592070c5e36689b7df367"
source = "git+https://github.com/loro-dev/loro.git?rev=90470658435ec4c62b5af59ebb82fe9e1f5aa761#90470658435ec4c62b5af59ebb82fe9e1f5aa761"
dependencies = [
"imbl",
"rand",

187
crates/fuzz/src/actor.rs
View file

@ -1,11 +1,13 @@
use std::{
collections::VecDeque,
fmt::{Debug, Formatter},
sync::{Arc, Mutex},
};
use enum_as_inner::EnumAsInner;
use enum_dispatch::enum_dispatch;
use fxhash::FxHashMap;
use fxhash::{FxHashMap, FxHashSet};
use itertools::Itertools;
use loro::{
Container, ContainerID, ContainerType, Frontiers, LoroDoc, LoroValue, PeerID, UndoManager, ID,
};
@ -52,7 +54,7 @@ impl Actor {
info_span!("ApplyDiff", id = id).in_scope(|| {
let mut tracker = cb_tracker.lock().unwrap();
tracker.apply_diff(e)
})
});
}));
let mut default_history = FxHashMap::default();
default_history.insert(Vec::new(), loro.get_deep_value());
@ -123,28 +125,20 @@ impl Actor {
pub fn undo(&mut self, undo_length: u32) {
self.loro.attach();
let mut before_undo = self.loro.get_deep_value();
let before_undo = self.loro.get_deep_value();
// println!("\n\nstart undo\n");
for _ in 0..undo_length {
self.undo_manager.undo.undo(&self.loro).unwrap();
}
// println!("\n\nstart redo\n");
for _ in 0..undo_length {
self.undo_manager.undo.redo(&self.loro).unwrap();
}
let mut after_undo = self.loro.get_deep_value();
Self::patch_tree_undo_position(&mut before_undo);
Self::patch_tree_undo_position(&mut after_undo);
assert_value_eq(&before_undo, &after_undo);
}
fn patch_tree_undo_position(a: &mut LoroValue) {
let root = Arc::make_mut(a.as_map_mut().unwrap());
let tree = root.get_mut("tree").unwrap();
let nodes = Arc::make_mut(tree.as_list_mut().unwrap());
for node in nodes.iter_mut() {
let node = Arc::make_mut(node.as_map_mut().unwrap());
node.remove("position");
}
let after_undo = self.loro.get_deep_value();
assert_value_eq(&before_undo, &after_undo);
}
pub fn check_tracker(&self) {
@ -184,6 +178,7 @@ impl Actor {
}
self.loro.checkout(&f).unwrap();
dbg!(&f);
self.loro.check_state_correctness_slow();
// check snapshot correctness after checkout
self.loro.checkout_to_latest();
@ -385,6 +380,14 @@ pub fn assert_value_eq(a: &LoroValue, b: &LoroValue) {
true
}
(LoroValue::List(a_list), LoroValue::List(b_list)) => {
if is_tree_values(a_list.as_ref()) {
assert_tree_value_eq(a_list, b_list);
true
} else {
a_list.iter().zip(b_list.iter()).all(|(a, b)| eq(a, b))
}
}
(a, b) => a == b,
}
}
@ -395,3 +398,155 @@ pub fn assert_value_eq(a: &LoroValue, b: &LoroValue) {
b
);
}
pub fn is_tree_values(value: &[LoroValue]) -> bool {
if let Some(LoroValue::Map(map)) = value.first() {
let map_keys = map.as_ref().keys().cloned().collect::<FxHashSet<_>>();
return map_keys.contains("id")
&& map_keys.contains("parent")
&& map_keys.contains("meta")
&& map_keys.contains("fractional_index");
}
false
}
#[derive(Debug, Clone)]
struct Node {
children: Vec<Node>,
meta: FxHashMap<String, LoroValue>,
position: String,
}
struct FlatNode {
id: String,
parent: Option<String>,
meta: FxHashMap<String, LoroValue>,
index: usize,
position: String,
}
impl FlatNode {
fn from_loro_value(value: &LoroValue) -> Self {
let map = value.as_map().unwrap();
let id = map.get("id").unwrap().as_string().unwrap().to_string();
let parent = map
.get("parent")
.unwrap()
.as_string()
.map(|x| x.to_string());
let meta = map.get("meta").unwrap().as_map().unwrap().as_ref().clone();
let index = *map.get("index").unwrap().as_i64().unwrap() as usize;
let position = map
.get("fractional_index")
.unwrap()
.as_string()
.unwrap()
.to_string();
FlatNode {
id,
parent,
meta,
index,
position,
}
}
}
impl Node {
fn from_loro_value(value: &[LoroValue]) -> Vec<Self> {
let mut node_map = FxHashMap::default();
let mut parent_child_map = FxHashMap::default();
for flat_node in value.iter() {
let flat_node = FlatNode::from_loro_value(flat_node);
let tree_node = Node {
// id: flat_node.id.clone(),
// parent: flat_node.parent.clone(),
children: vec![],
meta: flat_node.meta,
// index: flat_node.index,
position: flat_node.position,
};
node_map.insert(flat_node.id.clone(), tree_node);
parent_child_map
.entry(flat_node.parent)
.or_insert_with(Vec::new)
.push((flat_node.index, flat_node.id));
}
let mut node_map_clone = node_map.clone();
for (parent_id, child_ids) in parent_child_map.iter() {
if let Some(parent_id) = parent_id {
if let Some(parent_node) = node_map.get_mut(parent_id) {
for (_, child_id) in child_ids.into_iter().sorted_by_key(|x| x.0) {
if let Some(child_node) = node_map_clone.remove(child_id) {
parent_node.children.push(child_node);
}
}
}
}
}
parent_child_map.get(&None).map_or(vec![], |root_ids| {
root_ids
.iter()
.filter_map(|(_i, id)| node_map.remove(id))
.collect::<Vec<_>>()
})
}
}
pub fn assert_tree_value_eq(a: &[LoroValue], b: &[LoroValue]) {
// println!("\n\na = {:#?}", a);
// println!("b = {:#?}", b);
let a_tree = Node::from_loro_value(a);
let b_tree = Node::from_loro_value(b);
let mut a_q = VecDeque::from_iter([a_tree]);
let mut b_q = VecDeque::from_iter([b_tree]);
while let (Some(a_node), Some(b_node)) = (a_q.pop_front(), b_q.pop_front()) {
let mut children_a = vec![];
let mut children_b = vec![];
let a_meta = a_node
.into_iter()
.map(|x| {
children_a.extend(x.children);
let mut meta = x
.meta
.into_iter()
.sorted_by_key(|(k, _)| k.clone())
.map(|(mut k, v)| {
k.push_str(v.as_string().map_or("", |f| f.as_str()));
k
})
.collect::<String>();
meta.push_str(&x.position);
meta
})
.collect::<FxHashSet<_>>();
let b_meta = b_node
.into_iter()
.map(|x| {
children_b.extend(x.children);
let mut meta = x
.meta
.into_iter()
.sorted_by_key(|(k, _)| k.clone())
.map(|(mut k, v)| {
k.push_str(v.as_string().map_or("", |f| f.as_str()));
k
})
.collect::<String>();
meta.push_str(&x.position);
meta
})
.collect::<FxHashSet<_>>();
assert!(a_meta.difference(&b_meta).count() == 0);
assert_eq!(children_a.len(), children_b.len());
if children_a.is_empty() {
continue;
}
a_q.push_back(children_a);
b_q.push_back(children_b);
}
}

7
crates/fuzz/src/container/text.rs
View file

@ -2,7 +2,6 @@ use std::sync::{Arc, Mutex};
use loro::{Container, ContainerID, ContainerType, LoroDoc, LoroText};
use crate::{
actions::{Actionable, FromGenericAction, GenericAction},
actor::{ActionExecutor, ActorTrait},
@ -13,9 +12,9 @@ const STYLES_NAME: [&str; 4] = ["bold", "comment", "link", "highlight"];
#[derive(Debug, Clone)]
pub struct TextAction {
pos: usize,
len: usize,
action: TextActionInner,
pub pos: usize,
pub len: usize,
pub action: TextActionInner,
}
#[derive(Debug, Clone)]

8
crates/fuzz/src/container/tree.rs
View file

@ -168,11 +168,11 @@ impl Actionable for TreeAction {
*target = (id.peer, id.counter);
}
TreeActionInner::Delete => {
let target_index = target.1 as usize % node_num;
let target_index = target.0 as usize % node_num;
*target = (nodes[target_index].peer, nodes[target_index].counter);
}
TreeActionInner::Move { parent, index } => {
let target_index = target.1 as usize % node_num;
let target_index = target.0 as usize % node_num;
*target = (nodes[target_index].peer, nodes[target_index].counter);
let mut parent_idx = parent.0 as usize % node_num;
while target_index == parent_idx {
@ -202,7 +202,7 @@ impl Actionable for TreeAction {
*c = nodes[other_idx].counter;
}
TreeActionInner::Meta { meta: (_, v) } => {
let target_index = target.1 as usize % node_num;
let target_index = target.0 as usize % node_num;
*target = (nodes[target_index].peer, nodes[target_index].counter);
if matches!(v, FuzzValue::Container(_)) {
*v = FuzzValue::I32(0);
@ -507,7 +507,7 @@ impl TreeNode {
None => LoroValue::Null,
},
);
map.insert("position".to_string(), self.position.clone().into());
map.insert("fractional_index".to_string(), self.position.clone().into());
map.insert("index".to_string(), (index as i64).into());
map
}

176
crates/fuzz/src/crdt_fuzzer.rs
View file

@ -1,4 +1,7 @@
use std::fmt::{Debug, Display};
use std::{
fmt::{Debug, Display},
time::Instant,
};
use arbitrary::Arbitrary;
use fxhash::FxHashSet;
@ -241,9 +244,10 @@ impl CRDTFuzzer {
}
fn check_history(&mut self) {
for actor in self.actors.iter_mut() {
actor.check_history();
}
self.actors[0].check_history();
// for actor in self.actors.iter_mut() {
// actor.check_history();
// }
}
fn site_num(&self) -> usize {
@ -302,16 +306,178 @@ pub fn test_multi_sites(site_num: u8, fuzz_targets: Vec<FuzzTarget>, actions: &m
let mut applied = Vec::new();
for action in actions.iter_mut() {
fuzzer.pre_process(action);
info_span!("ApplyAction", ?action).in_scope(|| {
applied.push(action.clone());
info!("OptionsTable \n{}", (&applied).table());
// info!("Apply Action {:?}", applied);
fuzzer.apply_action(action);
});
}
let span = &info_span!("check synced");
let _g = span.enter();
fuzzer.check_equal();
fuzzer.check_tracker();
fuzzer.check_history();
}
pub fn minify_error<T, F, N>(site_num: u8, f: F, normalize: N, actions: Vec<T>)
where
F: Fn(u8, &mut [T]),
N: Fn(u8, &mut [T]) -> Vec<T>,
T: Clone + Debug,
{
std::panic::set_hook(Box::new(|_info| {
// ignore panic output
// println!("{:?}", _info);
}));
let f_ref: *const _ = &f;
let f_ref: usize = f_ref as usize;
#[allow(clippy::redundant_clone)]
let mut actions_clone = actions.clone();
let action_ref: usize = (&mut actions_clone) as *mut _ as usize;
#[allow(clippy::blocks_in_conditions)]
if std::panic::catch_unwind(|| {
// SAFETY: test
let f = unsafe { &*(f_ref as *const F) };
// SAFETY: test
let actions_ref = unsafe { &mut *(action_ref as *mut Vec<T>) };
f(site_num, actions_ref);
})
.is_ok()
{
println!("No Error Found");
return;
}
let mut minified = actions.clone();
let mut candidates = Vec::new();
println!("Setup candidates...");
for i in 0..actions.len() {
let mut new = actions.clone();
new.remove(i);
candidates.push(new);
}
println!("Minifying...");
let start = Instant::now();
while let Some(candidate) = candidates.pop() {
let f_ref: *const _ = &f;
let f_ref: usize = f_ref as usize;
let mut actions_clone = candidate.clone();
let action_ref: usize = (&mut actions_clone) as *mut _ as usize;
#[allow(clippy::blocks_in_conditions)]
if std::panic::catch_unwind(|| {
// SAFETY: test
let f = unsafe { &*(f_ref as *const F) };
// SAFETY: test
let actions_ref = unsafe { &mut *(action_ref as *mut Vec<T>) };
f(site_num, actions_ref);
})
.is_err()
{
for i in 0..candidate.len() {
let mut new = candidate.clone();
new.remove(i);
candidates.push(new);
}
if candidate.len() < minified.len() {
minified = candidate;
println!("New min len={}", minified.len());
}
if candidates.len() > 40 {
candidates.drain(0..30);
}
}
if start.elapsed().as_secs() > 10 && minified.len() <= 4 {
break;
}
if start.elapsed().as_secs() > 60 {
break;
}
}
let minified = normalize(site_num, &mut minified);
println!(
"Old Length {}, New Length {}",
actions.len(),
minified.len()
);
dbg!(&minified);
if actions.len() > minified.len() {
minify_error(site_num, f, normalize, minified);
}
}
pub fn minify_simple<T, F, N>(site_num: u8, f: F, normalize: N, actions: Vec<T>)
where
F: Fn(u8, &mut [T]),
N: Fn(u8, &mut [T]) -> Vec<T>,
T: Clone + Debug,
{
std::panic::set_hook(Box::new(|_info| {
// ignore panic output
// println!("{:?}", _info);
}));
let f_ref: *const _ = &f;
let f_ref: usize = f_ref as usize;
#[allow(clippy::redundant_clone)]
let mut actions_clone = actions.clone();
let action_ref: usize = (&mut actions_clone) as *mut _ as usize;
#[allow(clippy::blocks_in_conditions)]
if std::panic::catch_unwind(|| {
// SAFETY: test
let f = unsafe { &*(f_ref as *const F) };
// SAFETY: test
let actions_ref = unsafe { &mut *(action_ref as *mut Vec<T>) };
f(site_num, actions_ref);
})
.is_ok()
{
println!("No Error Found");
return;
}
let mut minified = actions.clone();
let mut current_index = minified.len() as i64 - 1;
while current_index > 0 {
let a = minified.remove(current_index as usize);
let f_ref: *const _ = &f;
let f_ref: usize = f_ref as usize;
let mut actions_clone = minified.clone();
let action_ref: usize = (&mut actions_clone) as *mut _ as usize;
let mut re = false;
#[allow(clippy::blocks_in_conditions)]
if std::panic::catch_unwind(|| {
// SAFETY: test
let f = unsafe { &*(f_ref as *const F) };
// SAFETY: test
let actions_ref = unsafe { &mut *(action_ref as *mut Vec<T>) };
f(site_num, actions_ref);
})
.is_err()
{
re = true;
} else {
minified.insert(current_index as usize, a);
}
println!(
"{}/{} {}",
actions.len() as i64 - current_index,
actions.len(),
re
);
current_index -= 1;
}
let minified = normalize(site_num, &mut minified);
println!("{:?}", &minified);
println!(
"Old Length {}, New Length {}",
actions.len(),
minified.len()
);
if actions.len() > minified.len() {
minify_simple(site_num, f, normalize, minified);
}
}

2385
crates/fuzz/tests/test.rs
View file

File diff suppressed because it is too large Load diff

4192
crates/fuzz/tests/undo.rs
View file

File diff suppressed because it is too large Load diff

12
crates/loro-common/src/error.rs
View file

@ -32,8 +32,12 @@ pub enum LoroError {
NotFoundError(Box<str>),
#[error("Transaction error ({0})")]
TransactionError(Box<str>),
#[error("Index out of bound. The given pos is {pos}, but the length is {len}")]
OutOfBound { pos: usize, len: usize },
#[error("Index out of bound. The given pos is {pos}, but the length is {len}. {info}")]
OutOfBound {
pos: usize,
len: usize,
info: Box<str>,
},
#[error("Every op id should be unique. ID {id} has been used. You should use a new PeerID to edit the content. ")]
UsedOpID { id: ID },
#[error("Movable Tree Error: {0}")]
@ -64,6 +68,10 @@ pub enum LoroError {
UndoWithDifferentPeerId { expected: PeerID, actual: PeerID },
#[error("The input JSON schema is invalid")]
InvalidJsonSchema,
#[error("Cannot insert or delete utf-8 in the middle of the codepoint in Unicode.")]
UTF8InUnicodeCodePoint { pos: usize },
#[error("Cannot insert or delete utf-16 in the middle of the codepoint in Unicode.")]
UTF16InUnicodeCodePoint { pos: usize },
}
#[derive(Error, Debug)]

34
crates/loro-internal/benches/tree.rs
View file

@ -61,20 +61,14 @@ mod tree {
let mut versions = vec![];
let size = 1000;
for _ in 0..size {
ids.push(
loro.with_txn(|txn| tree.create_with_txn(txn, None, 0))
.unwrap(),
)
ids.push(tree.create(None).unwrap())
}
let mut rng: StdRng = rand::SeedableRng::seed_from_u64(0);
let mut n = 1000;
while n > 0 {
let i = rng.gen::<usize>() % size;
let j = rng.gen::<usize>() % size;
if loro
.with_txn(|txn| tree.mov_with_txn(txn, ids[i], ids[j], 0))
.is_ok()
{
if tree.mov(ids[i], ids[j]).is_ok() {
versions.push(loro.oplog_frontiers());
n -= 1;
};
@ -94,15 +88,11 @@ mod tree {
let tree = loro.get_tree("tree");
let mut ids = vec![];
let mut versions = vec![];
let id1 = loro
.with_txn(|txn| tree.create_with_txn(txn, None, 0))
.unwrap();
let id1 = tree.create(None).unwrap();
ids.push(id1);
versions.push(loro.oplog_frontiers());
for _ in 1..depth {
let id = loro
.with_txn(|txn| tree.create_with_txn(txn, *ids.last().unwrap(), 0))
.unwrap();
let id = tree.create(*ids.last().unwrap()).unwrap();
ids.push(id);
versions.push(loro.oplog_frontiers());
}
@ -124,11 +114,7 @@ mod tree {
let mut ids = vec![];
let size = 1000;
for _ in 0..size {
ids.push(
doc_a
.with_txn(|txn| tree_a.create_with_txn(txn, None, 0))
.unwrap(),
)
ids.push(tree_a.create(None).unwrap())
}
doc_b.import(&doc_a.export_snapshot()).unwrap();
let mut rng: StdRng = rand::SeedableRng::seed_from_u64(0);
@ -138,16 +124,10 @@ mod tree {
let i = rng.gen::<usize>() % size;
let j = rng.gen::<usize>() % size;
if t % 2 == 0 {
let mut txn = doc_a.txn().unwrap();
tree_a
.mov_with_txn(&mut txn, ids[i], ids[j], 0)
.unwrap_or_default();
tree_a.mov(ids[i], ids[j]).unwrap_or_default();
doc_b.import(&doc_a.export_from(&doc_b.oplog_vv())).unwrap();
} else {
let mut txn = doc_b.txn().unwrap();
tree_b
.mov_with_txn(&mut txn, ids[i], ids[j], 0)
.unwrap_or_default();
tree_b.mov(ids[i], ids[j]).unwrap_or_default();
doc_a.import(&doc_b.export_from(&doc_a.oplog_vv())).unwrap();
}
}

13
crates/loro-internal/examples/tree.rs
View file

@ -6,19 +6,15 @@ use rand::{rngs::StdRng, Rng};
#[allow(unused)]
fn checkout() {
let depth = 300;
let loro = LoroDoc::default();
let loro = LoroDoc::new_auto_commit();
let tree = loro.get_tree("tree");
let mut ids = vec![];
let mut versions = vec![];
let id1 = loro
.with_txn(|txn| tree.create_with_txn(txn, None, 0))
.unwrap();
let id1 = tree.create_at(None, 0).unwrap();
ids.push(id1);
versions.push(loro.oplog_frontiers());
for _ in 1..depth {
let id = loro
.with_txn(|txn| tree.create_with_txn(txn, *ids.last().unwrap(), 0))
.unwrap();
let id = tree.create_at(*ids.last().unwrap(), 0).unwrap();
ids.push(id);
versions.push(loro.oplog_frontiers());
}
@ -63,8 +59,7 @@ fn create() {
let loro = LoroDoc::default();
let tree = loro.get_tree("tree");
for _ in 0..size {
loro.with_txn(|txn| tree.create_with_txn(txn, None, 0))
.unwrap();
tree.create_at(None, 0).unwrap();
}
println!("encode snapshot size {:?}\n", loro.export_snapshot().len());
println!(

18
crates/loro-internal/src/arena.rs
View file

@ -63,6 +63,24 @@ impl SharedArena {
}
}
pub fn fork(&self) -> Self {
Self {
inner: Arc::new(InnerSharedArena {
container_idx_to_id: Mutex::new(
self.inner.container_idx_to_id.lock().unwrap().clone(),
),
depth: Mutex::new(self.inner.depth.lock().unwrap().clone()),
container_id_to_idx: Mutex::new(
self.inner.container_id_to_idx.lock().unwrap().clone(),
),
parents: Mutex::new(self.inner.parents.lock().unwrap().clone()),
values: Mutex::new(self.inner.values.lock().unwrap().clone()),
root_c_idx: Mutex::new(self.inner.root_c_idx.lock().unwrap().clone()),
str: Mutex::new(self.inner.str.lock().unwrap().clone()),
}),
}
}
pub fn register_container(&self, id: &ContainerID) -> ContainerIdx {
let mut container_id_to_idx = self.inner.container_id_to_idx.lock().unwrap();
if let Some(&idx) = container_id_to_idx.get(id) {

2
crates/loro-internal/src/arena/str_arena.rs
View file

@ -5,7 +5,7 @@ use append_only_bytes::{AppendOnlyBytes, BytesSlice};
use crate::container::richtext::richtext_state::unicode_to_utf8_index;
const INDEX_INTERVAL: u32 = 128;
#[derive(Default, Debug)]
#[derive(Default, Debug, Clone)]
pub(crate) struct StrArena {
bytes: AppendOnlyBytes,
unicode_indexes: Vec<Index>,

20
crates/loro-internal/src/configure.rs
View file

@ -21,6 +21,26 @@ impl Default for Configure {
}
impl Configure {
pub fn fork(&self) -> Self {
Self {
text_style_config: Arc::new(RwLock::new(
self.text_style_config.read().unwrap().clone(),
)),
record_timestamp: Arc::new(AtomicBool::new(
self.record_timestamp
.load(std::sync::atomic::Ordering::Relaxed),
)),
merge_interval: Arc::new(AtomicI64::new(
self.merge_interval
.load(std::sync::atomic::Ordering::Relaxed),
)),
tree_position_jitter: Arc::new(AtomicU8::new(
self.tree_position_jitter
.load(std::sync::atomic::Ordering::Relaxed),
)),
}
}
pub fn text_style_config(&self) -> &Arc<RwLock<StyleConfigMap>> {
&self.text_style_config
}

9
crates/loro-internal/src/container.rs
View file

@ -6,6 +6,10 @@
//!
use crate::{arena::SharedArena, InternalString, ID};
pub mod list;
pub mod map;
pub mod richtext;
pub mod tree;
pub mod idx {
use super::super::ContainerType;
@ -86,11 +90,6 @@ pub mod idx {
}
}
}
pub mod list;
pub mod map;
pub mod richtext;
pub mod tree;
use idx::ContainerIdx;
pub use loro_common::ContainerType;

191
crates/loro-internal/src/container/richtext/richtext_state.rs
View file

@ -4,7 +4,10 @@ use generic_btree::{
rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert},
BTree, BTreeTrait, Cursor,
};
use loro_common::{Counter, IdFull, IdLpSpan, IdSpan, Lamport, LoroValue, ID};
use loro_common::{
Counter, IdFull, IdLpSpan, IdSpan, Lamport, LoroError, LoroResult, LoroValue, ID,
};
use query::{ByteQuery, ByteQueryT};
use serde::{ser::SerializeStruct, Serialize};
use std::{
fmt::{Display, Formatter},
@ -124,6 +127,11 @@ mod text_chunk {
self.unicode_len
}
#[inline]
pub fn utf8_len(&self) -> i32 {
self.bytes.len() as i32
}
#[inline]
pub fn unicode_len(&self) -> i32 {
self.unicode_len
@ -642,8 +650,7 @@ pub(crate) fn utf16_to_unicode_index(s: &str, utf16_index: usize) -> Result<usiz
let mut current_utf16_index = 0;
let mut current_unicode_index = 0;
for (i, c) in s.chars().enumerate() {
let len = c.len_utf16();
current_utf16_index += len;
current_utf16_index += c.len_utf16();
if current_utf16_index == utf16_index {
return Ok(i + 1);
}
@ -658,9 +665,38 @@ pub(crate) fn utf16_to_unicode_index(s: &str, utf16_index: usize) -> Result<usiz
Err(current_unicode_index)
}
pub(crate) fn utf8_to_unicode_index(s: &str, utf8_index: usize) -> Result<usize, usize> {
if utf8_index == 0 {
return Ok(0);
}
let mut current_utf8_index = 0;
let mut current_unicode_index = 0;
for (i, c) in s.chars().enumerate() {
let char_start = current_utf8_index;
current_utf8_index += c.len_utf8();
if utf8_index == char_start {
return Ok(i);
}
if utf8_index < current_utf8_index {
tracing::info!("WARNING: UTF-8 index is in the middle of a codepoint!");
return Err(i);
}
current_unicode_index = i + 1;
}
if current_utf8_index == utf8_index {
Ok(current_unicode_index)
} else {
Err(current_unicode_index)
}
}
fn pos_to_unicode_index(s: &str, pos: usize, kind: PosType) -> Option<usize> {
match kind {
PosType::Bytes => todo!(),
PosType::Bytes => utf8_to_unicode_index(s, pos).ok(),
PosType::Unicode => Some(pos),
PosType::Utf16 => utf16_to_unicode_index(s, pos).ok(),
PosType::Entity => Some(pos),
@ -916,6 +952,7 @@ mod query {
// Allow left to not at the correct utf16 boundary. If so fallback to the last position.
// TODO: if we remove the use of query(pos-1), we won't need this fallback behavior
// WARNING: Unable to report error!!!
let offset = utf16_to_unicode_index(s.as_str(), left).unwrap_or_else(|e| e);
(offset, true)
}
@ -969,13 +1006,55 @@ mod query {
cache.entity_len as usize
}
}
pub(super) struct ByteQueryT;
pub(super) type ByteQuery = IndexQuery<ByteQueryT, RichtextTreeTrait>;
impl QueryByLen<RichtextTreeTrait> for ByteQueryT {
fn get_cache_len(cache: &<RichtextTreeTrait as BTreeTrait>::Cache) -> usize {
cache.bytes as usize
}
fn get_elem_len(elem: &<RichtextTreeTrait as BTreeTrait>::Elem) -> usize {
match elem {
RichtextStateChunk::Text(s) => s.utf8_len() as usize,
RichtextStateChunk::Style { .. } => 0,
}
}
fn get_offset_and_found(
left: usize,
elem: &<RichtextTreeTrait as BTreeTrait>::Elem,
) -> (usize, bool) {
match elem {
RichtextStateChunk::Text(s) => {
if left == 0 {
return (0, true);
}
// Allow left to not at the correct utf16 boundary. If so fallback to the last position.
// TODO: if we remove the use of query(pos-1), we won't need this fallback behavior
// WARNING: Unable to report error!!!
let offset = utf8_to_unicode_index(s.as_str(), left).unwrap_or_else(|e| e);
(offset, true)
}
RichtextStateChunk::Style { .. } => (1, false),
}
}
fn get_cache_entity_len(cache: &<RichtextTreeTrait as BTreeTrait>::Cache) -> usize {
cache.entity_len as usize
}
}
}
mod cursor_cache {
use std::sync::atomic::AtomicUsize;
use super::{pos_to_unicode_index, unicode_to_utf16_index, PosType, RichtextTreeTrait};
use super::{
pos_to_unicode_index, unicode_to_utf16_index, unicode_to_utf8_index, PosType,
RichtextTreeTrait,
};
use generic_btree::{rle::HasLength, BTree, Cursor, LeafIndex};
use loro_common::LoroError;
#[derive(Debug, Clone)]
struct CursorCacheItem {
@ -1044,9 +1123,34 @@ mod cursor_cache {
entity_index: usize,
cursor: Cursor,
tree: &BTree<RichtextTreeTrait>,
) {
) -> Result<(), usize> {
match kind {
PosType::Bytes => todo!(),
PosType::Bytes => {
if cursor.offset == 0 {
self.entity = Some(EntityIndexCacheItem {
pos,
pos_type: kind,
entity_index,
leaf: cursor.leaf,
});
} else {
let elem = tree.get_elem(cursor.leaf).unwrap();
let Some(s) = elem.as_str() else {
return Ok(());
};
let utf8offset = unicode_to_utf8_index(s, cursor.offset).unwrap();
if pos < utf8offset {
return Err(pos);
}
self.entity = Some(EntityIndexCacheItem {
pos: pos - utf8offset,
pos_type: kind,
entity_index: entity_index - cursor.offset,
leaf: cursor.leaf,
});
}
Ok(())
}
PosType::Unicode | PosType::Entity => {
self.entity = Some(EntityIndexCacheItem {
pos: pos - cursor.offset,
@ -1054,6 +1158,7 @@ mod cursor_cache {
entity_index: entity_index - cursor.offset,
leaf: cursor.leaf,
});
Ok(())
}
PosType::Event if cfg!(not(feature = "wasm")) => {
self.entity = Some(EntityIndexCacheItem {
@ -1062,6 +1167,7 @@ mod cursor_cache {
entity_index: entity_index - cursor.offset,
leaf: cursor.leaf,
});
Ok(())
}
_ => {
// utf16
@ -1074,8 +1180,13 @@ mod cursor_cache {
});
} else {
let elem = tree.get_elem(cursor.leaf).unwrap();
let Some(s) = elem.as_str() else { return };
let Some(s) = elem.as_str() else {
return Ok(());
};
let utf16offset = unicode_to_utf16_index(s, cursor.offset).unwrap();
if pos < utf16offset {
return Err(pos);
}
self.entity = Some(EntityIndexCacheItem {
pos: pos - utf16offset,
pos_type: kind,
@ -1083,6 +1194,7 @@ mod cursor_cache {
leaf: cursor.leaf,
});
}
Ok(())
}
}
}
@ -1202,9 +1314,9 @@ impl RichtextState {
&mut self,
pos: usize,
pos_type: PosType,
) -> usize {
) -> Result<usize, LoroError> {
if self.tree.is_empty() {
return 0;
return Ok(0);
}
if let Some(pos) =
@ -1217,11 +1329,11 @@ impl RichtextState {
&self.tree,
&self.cursor_cache
);
return pos;
return Ok(pos);
}
let (c, entity_index) = match pos_type {
PosType::Bytes => todo!(),
PosType::Bytes => self.find_best_insert_pos::<ByteQueryT>(pos),
PosType::Unicode => self.find_best_insert_pos::<UnicodeQueryT>(pos),
PosType::Utf16 => self.find_best_insert_pos::<Utf16QueryT>(pos),
PosType::Entity => self.find_best_insert_pos::<EntityQueryT>(pos),
@ -1233,12 +1345,23 @@ impl RichtextState {
self.cursor_cache
.record_cursor(entity_index, PosType::Entity, c, &self.tree);
if !self.has_styles() {
self.cursor_cache
.record_entity_index(pos, pos_type, entity_index, c, &self.tree);
if let Err(pos) = self.cursor_cache.record_entity_index(
pos,
pos_type,
entity_index,
c,
&self.tree,
) {
return match pos_type {
PosType::Bytes => Err(LoroError::UTF8InUnicodeCodePoint { pos: pos }),
PosType::Utf16 => Err(LoroError::UTF16InUnicodeCodePoint { pos: pos }),
_ => unreachable!(),
};
}
}
}
entity_index
Ok(entity_index)
}
fn has_styles(&self) -> bool {
@ -1257,8 +1380,12 @@ impl RichtextState {
return (0..0, None);
}
let start = self.get_entity_index_for_text_insert(range.start, pos_type);
let end = self.get_entity_index_for_text_insert(range.end, pos_type);
let start = self
.get_entity_index_for_text_insert(range.start, pos_type)
.unwrap();
let end = self
.get_entity_index_for_text_insert(range.end, pos_type)
.unwrap();
if self.has_styles() {
(
start..end,
@ -1662,22 +1789,25 @@ impl RichtextState {
pos: usize,
len: usize,
pos_type: PosType,
) -> Vec<EntityRangeInfo> {
) -> LoroResult<Vec<EntityRangeInfo>> {
if self.tree.is_empty() {
return Vec::new();
return Ok(Vec::new());
}
if len == 0 {
return Vec::new();
return Ok(Vec::new());
}
if pos + len > self.len(pos_type) {
return Vec::new();
return Ok(Vec::new());
}
let mut ans: Vec<EntityRangeInfo> = Vec::new();
let (start, end) = match pos_type {
PosType::Bytes => todo!(),
PosType::Bytes => (
self.tree.query::<ByteQuery>(&pos).unwrap().cursor,
self.tree.query::<ByteQuery>(&(pos + len)).unwrap().cursor,
),
PosType::Unicode => (
self.tree.query::<UnicodeQuery>(&pos).unwrap().cursor,
self.tree
@ -1741,7 +1871,7 @@ impl RichtextState {
}
}
ans
Ok(ans)
}
// PERF: can be splitted into two methods. One is without cursor_to_event_index
@ -2278,7 +2408,7 @@ impl RichtextState {
pos: usize,
kind: PosType,
) -> Option<ID> {
let v = &self.get_text_entity_ranges(pos, 1, kind);
let v = &self.get_text_entity_ranges(pos, 1, kind).unwrap();
let a = v.first()?;
Some(a.id_start)
}
@ -2401,7 +2531,9 @@ mod test {
{
let state = &mut self.state;
let text = self.bytes.slice(start..);
let entity_index = state.get_entity_index_for_text_insert(pos, PosType::Unicode);
let entity_index = state
.get_entity_index_for_text_insert(pos, PosType::Unicode)
.unwrap();
state.insert_at_entity_index(entity_index, text, IdFull::new(0, 0, 0));
};
}
@ -2409,7 +2541,8 @@ mod test {
fn delete(&mut self, pos: usize, len: usize) {
let ranges = self
.state
.get_text_entity_ranges(pos, len, PosType::Unicode);
.get_text_entity_ranges(pos, len, PosType::Unicode)
.unwrap();
for range in ranges.into_iter().rev() {
self.state.drain_by_entity_index(
range.entity_start,
@ -2422,10 +2555,12 @@ mod test {
fn mark(&mut self, range: Range<usize>, style: Arc<StyleOp>) {
let start = self
.state
.get_entity_index_for_text_insert(range.start, PosType::Unicode);
.get_entity_index_for_text_insert(range.start, PosType::Unicode)
.unwrap();
let end = self
.state
.get_entity_index_for_text_insert(range.end, PosType::Unicode);
.get_entity_index_for_text_insert(range.end, PosType::Unicode)
.unwrap();
self.state.mark_with_entity_index(start..end, style);
}
}

1
crates/loro-internal/src/container/richtext/tracker.rs
View file

@ -598,6 +598,7 @@ impl Tracker {
self._checkout(from, false);
self._checkout(to, true);
// self.id_to_cursor.diagnose();
tracing::trace!("Trace::diff {:#?}, ", &self);
self.rope.get_diff()
}

155
crates/loro-internal/src/delta/tree.rs
View file

@ -1,5 +1,5 @@
use fractional_index::FractionalIndex;
use fxhash::FxHashMap;
use fxhash::{FxHashMap, FxHashSet};
use itertools::Itertools;
use loro_common::{IdFull, TreeID};
use std::fmt::Debug;
@ -29,19 +29,14 @@ pub enum TreeExternalDiff {
parent: Option<TreeID>,
index: usize,
position: FractionalIndex,
old_parent: TreeParentId,
old_index: usize,
},
Delete {
old_parent: TreeParentId,
old_index: usize,
},
Delete,
}
impl TreeDiff {
pub(crate) fn compose(mut self, other: Self) -> Self {
// TODO: better compose
self.diff.extend(other.diff);
// self = compose_tree_diff(&self);
self
}
@ -50,125 +45,38 @@ impl TreeDiff {
self
}
fn to_hash_map_mut(&mut self) -> FxHashMap<TreeID, usize> {
let mut ans = FxHashSet::default();
for index in (0..self.diff.len()).rev() {
let target = self.diff[index].target;
if ans.contains(&target) {
self.diff.remove(index);
continue;
}
ans.insert(target);
}
self.iter()
.map(|x| x.target)
.enumerate()
.map(|(i, x)| (x, i))
.collect()
}
pub(crate) fn transform(&mut self, b: &TreeDiff, left_prior: bool) {
// println!("\ntransform prior {:?} {:?} \nb {:?}", left_prior, self, b);
if b.is_empty() || self.is_empty() {
return;
}
let b_update: FxHashMap<_, _> = b.diff.iter().map(|d| (d.target, &d.action)).collect();
let mut self_update: FxHashMap<_, _> = self
.diff
.iter()
.enumerate()
.map(|(i, d)| (d.target, (&d.action, i)))
.collect();
let mut removes = Vec::new();
for (target, diff) in b_update {
if self_update.contains_key(&target) && diff == self_update.get(&target).unwrap().0 {
let (_, i) = self_update.remove(&target).unwrap();
removes.push(i);
continue;
}
if !left_prior {
if let Some((_, i)) = self_update.remove(&target) {
removes.push(i);
}
}
}
for i in removes.into_iter().sorted().rev() {
self.diff.remove(i);
}
let mut b_parent = FxHashMap::default();
fn reset_index(
b_parent: &FxHashMap<TreeParentId, Vec<i32>>,
index: &mut usize,
parent: &TreeParentId,
left_priority: bool,
) {
if let Some(b_indices) = b_parent.get(parent) {
for i in b_indices.iter() {
if (i.unsigned_abs() as usize) < *index
|| (i.unsigned_abs() as usize == *index && !left_priority)
{
if i > &0 {
*index += 1;
} else if *index > (i.unsigned_abs() as usize) {
*index = index.saturating_sub(1);
}
} else {
break;
}
}
}
}
for diff in b.diff.iter() {
match &diff.action {
TreeExternalDiff::Create {
parent,
index,
position: _,
} => {
b_parent
.entry(TreeParentId::from(*parent))
.or_insert_with(Vec::new)
.push(*index as i32);
}
TreeExternalDiff::Move {
parent,
index,
position: _,
old_parent,
old_index,
} => {
b_parent
.entry(*old_parent)
.or_insert_with(Vec::new)
.push(-(*old_index as i32));
b_parent
.entry(TreeParentId::from(*parent))
.or_insert_with(Vec::new)
.push(*index as i32);
}
TreeExternalDiff::Delete {
old_index,
old_parent,
} => {
b_parent
.entry(*old_parent)
.or_insert_with(Vec::new)
.push(-(*old_index as i32));
}
}
}
b_parent
.iter_mut()
.for_each(|(_, v)| v.sort_unstable_by_key(|i| i.abs()));
for diff in self.iter_mut() {
match &mut diff.action {
TreeExternalDiff::Create {
parent,
index,
position: _,
} => reset_index(&b_parent, index, &TreeParentId::from(*parent), left_prior),
TreeExternalDiff::Move {
parent,
index,
position: _,
old_parent,
old_index,
} => {
reset_index(&b_parent, index, &TreeParentId::from(*parent), left_prior);
reset_index(&b_parent, old_index, old_parent, left_prior);
}
TreeExternalDiff::Delete {
old_index,
old_parent,
} => {
reset_index(&b_parent, old_index, old_parent, left_prior);
}
if !left_prior {
let mut self_update = self.to_hash_map_mut();
for i in b
.iter()
.map(|x| x.target)
.filter_map(|x| self_update.remove(&x))
.sorted()
.rev()
{
self.remove(i);
}
}
}
@ -239,7 +147,6 @@ impl TreeDeltaItem {
is_old_parent_deleted: bool,
position: Option<FractionalIndex>,
) -> Self {
// TODO: check op id
let action = if matches!(parent, TreeParentId::Unexist) {
TreeInternalDiff::UnCreate
} else {

1
crates/loro-internal/src/diff_calc.rs
View file

@ -778,6 +778,7 @@ impl DiffCalculatorTrait for RichtextDiffCalculator {
to: &crate::VersionVector,
_: impl FnMut(&ContainerID),
) -> InternalDiff {
tracing::debug!("CalcDiff {:?} {:?}", from, to);
let mut delta = Delta::new();
for item in self.tracker.diff(from, to) {
match item {

4
crates/loro-internal/src/diff_calc/tree.rs
View file

@ -50,7 +50,6 @@ impl DiffCalculatorTrait for TreeDiffCalculator {
on_new_container(&d.target.associated_meta_container())
}
});
tracing::info!("\ndiff {:?}", diff);
InternalDiff::Tree(diff)
@ -70,7 +69,6 @@ impl TreeDiffCalculator {
fn checkout(&mut self, to: &VersionVector, oplog: &OpLog) {
let tree_ops = oplog.op_groups.get_tree(&self.container).unwrap();
let mut tree_cache = tree_ops.tree_for_diff.lock().unwrap();
let s = format!("checkout current {:?} to {:?}", &tree_cache.current_vv, &to);
let s = tracing::span!(tracing::Level::INFO, "checkout", s = s);
let _e = s.enter();
@ -451,7 +449,7 @@ impl TreeCacheForDiff {
ans.push((*tree_id, op.position.clone(), op.id_full()));
}
}
ans.sort_by(|a, b| a.1.cmp(&b.1));
ans
}
}

8
crates/loro-internal/src/event.rs
View file

@ -216,6 +216,14 @@ impl DiffVariant {
(a, _) => Err(a),
}
}
pub fn is_empty(&self) -> bool {
match self {
DiffVariant::Internal(diff) => diff.is_empty(),
DiffVariant::External(diff) => diff.is_empty(),
DiffVariant::None => true,
}
}
}
#[non_exhaustive]

28
crates/loro-internal/src/group.rs
View file

@ -19,13 +19,22 @@ use crate::{
VersionVector,
};
#[derive(Debug, Clone)]
#[derive(Debug)]
pub(crate) struct OpGroups {
arena: SharedArena,
groups: FxHashMap<ContainerIdx, OpGroup>,
}
impl OpGroups {
pub(crate) fn fork(&self, arena: SharedArena) -> Self {
let mut groups = FxHashMap::with_capacity_and_hasher(self.groups.len(), Default::default());
for (container_idx, group) in self.groups.iter() {
groups.insert(*container_idx, group.fork(&arena));
}
Self { arena, groups }
}
pub(crate) fn new(arena: SharedArena) -> Self {
Self {
arena,
@ -108,6 +117,23 @@ pub(crate) enum OpGroup {
MovableList(MovableListOpGroup),
}
impl OpGroup {
fn fork(&self, a: &SharedArena) -> Self {
match self {
OpGroup::Map(m) => OpGroup::Map(m.clone()),
OpGroup::Tree(t) => OpGroup::Tree(TreeOpGroup {
ops: t.ops.clone(),
tree_for_diff: Arc::new(Mutex::new(Default::default())),
}),
OpGroup::MovableList(m) => OpGroup::MovableList(MovableListOpGroup {
arena: a.clone(),
elem_mappings: m.elem_mappings.clone(),
pos_to_elem: m.pos_to_elem.clone(),
}),
}
}
}
#[enum_dispatch]
trait OpGroupTrait {
fn insert(&mut self, op: &RichOp);

457
crates/loro-internal/src/handler.rs
View file

@ -7,7 +7,7 @@ use crate::{
richtext::{richtext_state::PosType, RichtextState, StyleOp, TextStyleInfoFlag},
},
cursor::{Cursor, Side},
delta::{DeltaItem, StyleMeta, TreeExternalDiff},
delta::{DeltaItem, Meta, StyleMeta, TreeExternalDiff},
event::{Diff, TextDiffItem},
op::ListSlice,
state::{ContainerState, IndexType, State},
@ -19,16 +19,20 @@ use enum_as_inner::EnumAsInner;
use fxhash::FxHashMap;
use generic_btree::rle::HasLength;
use loro_common::{
ContainerID, ContainerType, IdFull, InternalString, LoroError, LoroResult, LoroValue, ID,
ContainerID, ContainerType, IdFull, InternalString, LoroError, LoroResult, LoroValue, TreeID,
ID,
};
use serde::{Deserialize, Serialize};
use std::{
borrow::Cow,
cmp::Reverse,
collections::BinaryHeap,
fmt::Debug,
ops::Deref,
sync::{Arc, Mutex, Weak},
};
use tracing::{error, info, instrument};
use tracing::{debug, error, info, instrument, Event};
mod tree;
pub use tree::TreeHandler;
@ -1060,8 +1064,11 @@ impl Handler {
pub(crate) fn apply_diff(
&self,
diff: Diff,
on_container_remap: &mut dyn FnMut(ContainerID, ContainerID),
container_remap: &mut FxHashMap<ContainerID, ContainerID>,
) -> LoroResult<()> {
let on_container_remap = &mut |old_id, new_id| {
container_remap.insert(old_id, new_id);
};
match self {
Self::Map(x) => {
let diff = diff.into_map().unwrap();
@ -1098,20 +1105,79 @@ impl Handler {
x.apply_delta(delta, on_container_remap)?;
}
Self::Tree(x) => {
fn remap_tree_id(
id: &mut TreeID,
container_remap: &FxHashMap<ContainerID, ContainerID>,
) {
let mut remapped = false;
let mut map_id = id.associated_meta_container();
while let Some(rid) = container_remap.get(&map_id) {
remapped = true;
map_id = rid.clone();
}
if remapped {
*id = TreeID::new(
*map_id.as_normal().unwrap().0,
*map_id.as_normal().unwrap().1,
)
}
}
for diff in diff.into_tree().unwrap().diff {
let target = diff.target;
let mut target = diff.target;
match diff.action {
TreeExternalDiff::Create {
parent,
index,
position: _,
mut parent,
index: _,
position,
} => {
x.create_at_with_target(parent, index, target)?;
// create map event
if let Some(p) = parent.as_mut() {
remap_tree_id(p, container_remap)
}
remap_tree_id(&mut target, container_remap);
if x.contains(target) {
// 1@0 is the parent of 2@1
// ┌────┐ ┌───────────────┐
// │xxxx│◀───│Move 2@1 to 0@0◀┐
// └────┘ └───────────────┘│
// ┌───────┐ │ ┌────────┐
// │Del 1@0│◀─────────────────┴─│Meta 2@1│ ◀─── undo 2 ops redo 2 ops
// └───────┘ └────────┘
//
// When we undo the delete operation, we should not create a new tree node and its child.
// However, the concurrent operation has moved the child to another parent. It's still alive.
// So when we redo the delete operation, we should check if the target is still alive.
// If it's alive, we should move it back instead of creating new one.
x.move_at_with_target_for_apply_diff(parent, position, target)?;
} else {
let new_target = x.__internal__next_tree_id();
if x.create_at_with_target_for_apply_diff(
parent, position, new_target,
)? {
container_remap.insert(
target.associated_meta_container(),
new_target.associated_meta_container(),
);
}
}
}
TreeExternalDiff::Delete { .. } => x.delete(target)?,
TreeExternalDiff::Move { parent, index, .. } => {
x.move_to(target, parent, index)?
TreeExternalDiff::Move {
mut parent,
index: _,
position,
} => {
if let Some(p) = parent.as_mut() {
remap_tree_id(p, container_remap)
}
remap_tree_id(&mut target, container_remap);
x.move_at_with_target_for_apply_diff(parent, position, target)?;
}
TreeExternalDiff::Delete => {
remap_tree_id(&mut target, container_remap);
if x.contains(target) {
x.delete(target)?;
}
}
}
}
@ -1291,7 +1357,8 @@ impl TextHandler {
let mut t = t.try_lock().unwrap();
let index = t
.value
.get_entity_index_for_text_insert(pos, PosType::Event);
.get_entity_index_for_text_insert(pos, PosType::Event)
.unwrap();
t.value.insert_at_entity_index(
index,
BytesSlice::from_bytes(s.as_bytes()),
@ -1303,16 +1370,89 @@ impl TextHandler {
}
}
pub fn insert_utf8(&self, pos: usize, s: &str) -> LoroResult<()> {
match &self.inner {
MaybeDetached::Detached(t) => {
let mut t = t.try_lock().unwrap();
let index = t
.value
.get_entity_index_for_text_insert(pos, PosType::Bytes)
.unwrap();
t.value.insert_at_entity_index(
index,
BytesSlice::from_bytes(s.as_bytes()),
IdFull::NONE_ID,
);
Ok(())
}
MaybeDetached::Attached(a) => a.with_txn(|txn| self.insert_with_txn_utf8(txn, pos, s)),
}
}
/// `pos` is a Event Index:
///
/// - if feature="wasm", pos is a UTF-16 index
/// - if feature!="wasm", pos is a Unicode index
pub fn insert_with_txn(&self, txn: &mut Transaction, pos: usize, s: &str) -> LoroResult<()> {
self.insert_with_txn_and_attr(txn, pos, s, None)?;
self.insert_with_txn_and_attr(txn, pos, s, None, PosType::Event)?;
Ok(())
}
/// If attr is specified, it will be used as the attribute of the inserted text.
pub fn insert_with_txn_utf8(
&self,
txn: &mut Transaction,
pos: usize,
s: &str,
) -> LoroResult<()> {
self.insert_with_txn_and_attr(txn, pos, s, None, PosType::Bytes)?;
Ok(())
}
/// `pos` is a Event Index:
///
/// - if feature="wasm", pos is a UTF-16 index
/// - if feature!="wasm", pos is a Unicode index
///
/// This method requires auto_commit to be enabled.
pub fn delete(&self, pos: usize, len: usize) -> LoroResult<()> {
match &self.inner {
MaybeDetached::Detached(t) => {
let mut t = t.try_lock().unwrap();
let ranges = t
.value
.get_text_entity_ranges(pos, len, PosType::Event)
.unwrap();
for range in ranges.iter().rev() {
t.value
.drain_by_entity_index(range.entity_start, range.entity_len(), None);
}
Ok(())
}
MaybeDetached::Attached(a) => a.with_txn(|txn| self.delete_with_txn(txn, pos, len)),
}
}
pub fn delete_utf8(&self, pos: usize, len: usize) -> LoroResult<()> {
match &self.inner {
MaybeDetached::Detached(t) => {
let mut t = t.try_lock().unwrap();
let ranges = match t.value.get_text_entity_ranges(pos, len, PosType::Bytes) {
Err(x) => return Err(x),
Ok(x) => x,
};
for range in ranges.iter().rev() {
t.value
.drain_by_entity_index(range.entity_start, range.entity_len(), None);
}
Ok(())
}
MaybeDetached::Attached(a) => {
a.with_txn(|txn| self.delete_with_txn_utf8(txn, pos, len))
}
}
}
/// If attr is specified, it will be used as the at tribute of the inserted text.
/// It will override the existing attribute of the text.
fn insert_with_txn_and_attr(
&self,
@ -1320,26 +1460,51 @@ impl TextHandler {
pos: usize,
s: &str,
attr: Option<&FxHashMap<String, LoroValue>>,
pos_type: PosType,
) -> Result<Vec<(InternalString, LoroValue)>, LoroError> {
if s.is_empty() {
return Ok(Vec::new());
}
if pos > self.len_event() {
return Err(LoroError::OutOfBound {
pos,
len: self.len_event(),
});
match pos_type {
PosType::Event => {
if pos > self.len_event() {
return Err(LoroError::OutOfBound {
pos,
len: self.len_event(),
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
});
}
}
PosType::Bytes => {
if pos > self.len_utf8() {
return Err(LoroError::OutOfBound {
pos,
len: self.len_utf8(),
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
});
}
}
_ => (),
}
let inner = self.inner.try_attached_state()?;
let (entity_index, styles) = inner.with_state(|state| {
let richtext_state = state.as_richtext_state_mut().unwrap();
let pos = richtext_state.get_entity_index_for_text_insert(pos);
let pos = richtext_state.get_entity_index_for_text_insert(pos, pos_type);
let pos = match pos {
Err(_) => return (pos, StyleMeta::empty()),
Ok(x) => x,
};
let styles = richtext_state.get_styles_at_entity_index(pos);
(pos, styles)
(Ok(pos), styles)
});
let entity_index = match entity_index {
Err(x) => return Err(x),
_ => entity_index.unwrap(),
};
let mut override_styles = Vec::new();
if let Some(attr) = attr {
// current styles
@ -1395,49 +1560,66 @@ impl TextHandler {
///
/// - if feature="wasm", pos is a UTF-16 index
/// - if feature!="wasm", pos is a Unicode index
///
/// This method requires auto_commit to be enabled.
pub fn delete(&self, pos: usize, len: usize) -> LoroResult<()> {
match &self.inner {
MaybeDetached::Detached(t) => {
let mut t = t.try_lock().unwrap();
let ranges = t.value.get_text_entity_ranges(pos, len, PosType::Event);
for range in ranges.iter().rev() {
t.value
.drain_by_entity_index(range.entity_start, range.entity_len(), None);
}
Ok(())
}
MaybeDetached::Attached(a) => a.with_txn(|txn| self.delete_with_txn(txn, pos, len)),
}
pub fn delete_with_txn(&self, txn: &mut Transaction, pos: usize, len: usize) -> LoroResult<()> {
self.delete_with_txn_inline(txn, pos, len, PosType::Event)
}
/// `pos` is a Event Index:
///
/// - if feature="wasm", pos is a UTF-16 index
/// - if feature!="wasm", pos is a Unicode index
pub fn delete_with_txn(&self, txn: &mut Transaction, pos: usize, len: usize) -> LoroResult<()> {
pub fn delete_with_txn_utf8(
&self,
txn: &mut Transaction,
pos: usize,
len: usize,
) -> LoroResult<()> {
self.delete_with_txn_inline(txn, pos, len, PosType::Bytes)
}
fn delete_with_txn_inline(
&self,
txn: &mut Transaction,
pos: usize,
len: usize,
pos_type: PosType,
) -> LoroResult<()> {
if len == 0 {
return Ok(());
}
if pos + len > self.len_event() {
error!("pos={} len={} len_event={}", pos, len, self.len_event());
return Err(LoroError::OutOfBound {
pos: pos + len,
len: self.len_event(),
});
match pos_type {
PosType::Event => {
if pos + len > self.len_event() {
error!("pos={} len={} len_event={}", pos, len, self.len_event());
return Err(LoroError::OutOfBound {
pos: pos + len,
len: self.len_event(),
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
});
}
}
PosType::Bytes => {
if pos + len > self.len_utf8() {
error!("pos={} len={} len_event={}", pos, len, self.len_event());
return Err(LoroError::OutOfBound {
pos: pos + len,
len: self.len_event(),
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
});
}
}
_ => (),
}
let inner = self.inner.try_attached_state()?;
let s = tracing::span!(tracing::Level::INFO, "delete", "pos={} len={}", pos, len);
let _e = s.enter();
let ranges = inner.with_state(|state| {
let ranges = match inner.with_state(|state| {
let richtext_state = state.as_richtext_state_mut().unwrap();
richtext_state.get_text_entity_ranges_in_event_index_range(pos, len)
});
richtext_state.get_text_entity_ranges_in_event_index_range(pos, len, pos_type)
}) {
Err(x) => return Err(x),
Ok(x) => x,
};
debug_assert_eq!(ranges.iter().map(|x| x.event_len).sum::<usize>(), len);
//debug_assert_eq!(ranges.iter().map(|x| x.event_len).sum::<usize>(), len);
let mut event_end = (pos + len) as isize;
for range in ranges.iter().rev() {
let event_start = event_end - range.event_len as isize;
@ -1503,7 +1685,11 @@ impl TextHandler {
));
}
if end > len {
return Err(LoroError::OutOfBound { pos: end, len });
return Err(LoroError::OutOfBound {
pos: end,
len,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
});
}
let (entity_range, styles) =
state.get_entity_range_and_text_styles_at_range(start..end, PosType::Event);
@ -1579,7 +1765,11 @@ impl TextHandler {
let len = self.len_event();
if end > len {
return Err(LoroError::OutOfBound { pos: end, len });
return Err(LoroError::OutOfBound {
pos: end,
len,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
});
}
let inner = self.inner.try_attached_state()?;
@ -1693,6 +1883,7 @@ impl TextHandler {
index,
insert.as_str(),
Some(attributes.as_ref().unwrap_or(&Default::default())),
PosType::Event,
)?;
for (key, value) in override_styles {
@ -1873,6 +2064,7 @@ impl ListHandler {
if pos > self.len() {
return Err(LoroError::OutOfBound {
pos,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: self.len(),
});
}
@ -1957,6 +2149,7 @@ impl ListHandler {
if pos > self.len() {
return Err(LoroError::OutOfBound {
pos,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: self.len(),
});
}
@ -1997,6 +2190,7 @@ impl ListHandler {
if pos + len > self.len() {
return Err(LoroError::OutOfBound {
pos: pos + len,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: self.len(),
});
}
@ -2031,6 +2225,7 @@ impl ListHandler {
let list = l.try_lock().unwrap();
let value = list.value.get(index).ok_or(LoroError::OutOfBound {
pos: index,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: list.value.len(),
})?;
match value {
@ -2050,6 +2245,7 @@ impl ListHandler {
}) else {
return Err(LoroError::OutOfBound {
pos: index,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: a.with_state(|state| state.as_list_state().unwrap().len()),
});
};
@ -2249,6 +2445,7 @@ impl MovableListHandler {
if pos > d.value.len() {
return Err(LoroError::OutOfBound {
pos,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: d.value.len(),
});
}
@ -2271,6 +2468,7 @@ impl MovableListHandler {
if pos > self.len() {
return Err(LoroError::OutOfBound {
pos,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: self.len(),
});
}
@ -2310,12 +2508,14 @@ impl MovableListHandler {
if from >= d.value.len() {
return Err(LoroError::OutOfBound {
pos: from,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: d.value.len(),
});
}
if to >= d.value.len() {
return Err(LoroError::OutOfBound {
pos: to,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: d.value.len(),
});
}
@ -2337,6 +2537,7 @@ impl MovableListHandler {
if from >= self.len() {
return Err(LoroError::OutOfBound {
pos: from,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: self.len(),
});
}
@ -2344,6 +2545,7 @@ impl MovableListHandler {
if to >= self.len() {
return Err(LoroError::OutOfBound {
pos: to,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: self.len(),
});
}
@ -2439,6 +2641,7 @@ impl MovableListHandler {
if pos > d.value.len() {
return Err(LoroError::OutOfBound {
pos,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: d.value.len(),
});
}
@ -2461,6 +2664,7 @@ impl MovableListHandler {
if pos > self.len() {
return Err(LoroError::OutOfBound {
pos,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: self.len(),
});
}
@ -2495,6 +2699,7 @@ impl MovableListHandler {
if index >= d.value.len() {
return Err(LoroError::OutOfBound {
pos: index,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: d.value.len(),
});
}
@ -2516,6 +2721,7 @@ impl MovableListHandler {
if index >= self.len() {
return Err(LoroError::OutOfBound {
pos: index,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: self.len(),
});
}
@ -2604,6 +2810,7 @@ impl MovableListHandler {
if pos + len > self.len() {
return Err(LoroError::OutOfBound {
pos: pos + len,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: self.len(),
});
}
@ -2651,6 +2858,7 @@ impl MovableListHandler {
let list = l.try_lock().unwrap();
let value = list.value.get(index).ok_or(LoroError::OutOfBound {
pos: index,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: list.value.len(),
})?;
match value {
@ -2675,6 +2883,7 @@ impl MovableListHandler {
}) else {
return Err(LoroError::OutOfBound {
pos: index,
info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(),
len: a.with_state(|state| state.as_list_state().unwrap().len()),
});
};
@ -2872,7 +3081,54 @@ impl MovableListHandler {
unimplemented!();
}
MaybeDetached::Attached(_) => {
debug!("movable list apply_delta {:#?}", &delta);
// preprocess all deletions. They will be used to infer the move ops
let mut index = 0;
let mut to_delete = FxHashMap::default();
for d in delta.iter() {
match d {
loro_delta::DeltaItem::Retain { len, .. } => {
index += len;
}
loro_delta::DeltaItem::Replace { delete, .. } => {
if *delete > 0 {
for i in index..index + *delete {
if let Some(LoroValue::Container(c)) = self.get(i) {
to_delete.insert(c, i);
}
}
index += *delete;
}
}
}
}
fn update_on_insert(
d: &mut FxHashMap<ContainerID, usize>,
index: usize,
len: usize,
) {
for pos in d.values_mut() {
if *pos >= index {
*pos += len;
}
}
}
fn update_on_delete(d: &mut FxHashMap<ContainerID, usize>, index: usize) {
for pos in d.values_mut() {
if *pos >= index {
*pos -= 1;
}
}
}
// process all insertions and moves
let mut index = 0;
let mut deleted = Vec::new();
let mut next_deleted = BinaryHeap::new();
let mut index_shift = 0;
for d in delta.iter() {
match d {
loro_delta::DeltaItem::Retain { len, .. } => {
@ -2881,28 +3137,93 @@ impl MovableListHandler {
loro_delta::DeltaItem::Replace {
value,
delete,
attr: _attr,
attr,
} => {
// TODO: handle move error
self.delete(index, *delete)?;
if *delete > 0 {
// skip the deletion if it is already processed by moving
let mut d = *delete;
while let Some(Reverse(old_index)) = next_deleted.peek() {
if *old_index + index_shift < index + d {
assert!(index <= *old_index + index_shift);
assert!(d > 0);
next_deleted.pop();
d -= 1;
} else {
break;
}
}
index += d;
}
for v in value.iter() {
match v {
ValueOrHandler::Value(v) => {
self.insert(index, v.clone())?;
update_on_insert(&mut to_delete, index, 1);
index += 1;
index_shift += 1;
}
ValueOrHandler::Handler(h) => {
let old_id = h.id();
let new_h = self.insert_container(
index,
Handler::new_unattached(old_id.container_type()),
)?;
let new_id = new_h.id();
on_container_remap(old_id, new_id);
if let Some(old_index) = to_delete.remove(&old_id) {
if old_index > index {
self.mov(old_index, index)?;
next_deleted.push(Reverse(old_index));
index += 1;
index_shift += 1;
} else {
// we need to sub 1 because old_index < index, and index means the position before the move
// but the param is the position after the move
self.mov(old_index, index - 1)?;
}
deleted.push(old_index);
update_on_delete(&mut to_delete, old_index);
update_on_insert(&mut to_delete, index, 1);
} else {
let new_h = self.insert_container(
index,
Handler::new_unattached(old_id.container_type()),
)?;
let new_id = new_h.id();
on_container_remap(old_id, new_id);
update_on_insert(&mut to_delete, index, 1);
index += 1;
index_shift += 1;
}
}
}
}
}
}
}
// apply the rest of the deletions
// sort deleted indexes from large to small
deleted.sort_by_key(|x| -(*x as i32));
let mut index = 0;
for d in delta.iter() {
match d {
loro_delta::DeltaItem::Retain { len, .. } => {
index += len;
}
loro_delta::DeltaItem::Replace { delete, value, .. } => {
if *delete > 0 {
let mut d = *delete;
while let Some(last) = deleted.last() {
if *last < index + d {
deleted.pop();
d -= 1;
} else {
break;
}
}
index += 1;
self.delete(index, d)?;
}
index += value.len();
}
}
}
@ -3372,14 +3693,14 @@ pub mod counter {
#[cfg(test)]
mod test {
use super::{HandlerTrait, TextDelta};
use crate::container::richtext::richtext_state::PosType;
use crate::loro::LoroDoc;
use crate::version::Frontiers;
use crate::{fx_map, ToJson};
use loro_common::ID;
use serde_json::json;
use super::{HandlerTrait, TextDelta};
#[test]
fn import() {
let loro = LoroDoc::new();
@ -3555,7 +3876,7 @@ mod test {
.unwrap();
assert_eq!(meta, 123.into());
assert_eq!(
r#"[{"parent":null,"meta":{"a":123},"id":"0@1","index":0,"position":"80"}]"#,
r#"[{"parent":null,"meta":{"a":123},"id":"0@1","index":0,"fractional_index":"80"}]"#,
tree.get_deep_value().to_json()
);
let bytes = loro.export_snapshot();

245
crates/loro-internal/src/handler/tree.rs
View file

@ -3,13 +3,14 @@ use std::collections::VecDeque;
use fractional_index::FractionalIndex;
use fxhash::FxHashMap;
use loro_common::{
ContainerID, ContainerType, Counter, LoroResult, LoroTreeError, LoroValue, PeerID, TreeID,
ContainerID, ContainerType, Counter, IdLp, LoroResult, LoroTreeError, LoroValue, PeerID, TreeID,
};
use smallvec::smallvec;
use crate::{
container::tree::tree_op::TreeOp,
delta::{TreeDiffItem, TreeExternalDiff},
state::{FractionalIndexGenResult, TreeParentId},
state::{FractionalIndexGenResult, NodePosition, TreeParentId},
txn::{EventHint, Transaction},
BasicHandler, HandlerTrait, MapHandler,
};
@ -49,19 +50,6 @@ impl TreeInner {
id
}
fn create_with_target(
&mut self,
parent: Option<TreeID>,
index: usize,
target: TreeID,
) -> TreeID {
self.map.insert(target, MapHandler::new_detached());
self.parent_links.insert(target, parent);
let children = self.children_links.entry(parent).or_default();
children.insert(index, target);
target
}
fn mov(&mut self, target: TreeID, new_parent: Option<TreeID>, index: usize) -> LoroResult<()> {
let old_parent = self
.parent_links
@ -267,7 +255,7 @@ impl HandlerTrait for TreeHandler {
impl std::fmt::Debug for TreeHandler {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match &self.inner {
MaybeDetached::Detached(_) => write!(f, "TreeHandler Dettached"),
MaybeDetached::Detached(_) => write!(f, "TreeHandler Detached"),
MaybeDetached::Attached(a) => write!(f, "TreeHandler {}", a.id),
}
}
@ -296,21 +284,15 @@ impl TreeHandler {
}
}
pub fn delete_with_txn(&self, txn: &mut Transaction, target: TreeID) -> LoroResult<()> {
pub(crate) fn delete_with_txn(&self, txn: &mut Transaction, target: TreeID) -> LoroResult<()> {
let inner = self.inner.try_attached_state()?;
txn.apply_local_op(
inner.container_idx,
crate::op::RawOpContent::Tree(TreeOp::Delete { target }),
EventHint::Tree(TreeDiffItem {
EventHint::Tree(smallvec![TreeDiffItem {
target,
action: TreeExternalDiff::Delete {
old_parent: self
.get_node_parent(&target)
.map(TreeParentId::from)
.unwrap_or(TreeParentId::Unexist),
old_index: self.get_index_by_tree_id(&target).unwrap_or(0),
},
}),
action: TreeExternalDiff::Delete,
}]),
&inner.state,
)
}
@ -338,45 +320,141 @@ impl TreeHandler {
}
/// For undo/redo, Specify the TreeID of the created node
pub(crate) fn create_at_with_target(
pub(crate) fn create_at_with_target_for_apply_diff(
&self,
parent: Option<TreeID>,
index: usize,
position: FractionalIndex,
target: TreeID,
) -> LoroResult<()> {
if let Some(p) = parent {
if !self.contains(p) {
return Ok(());
) -> LoroResult<bool> {
let MaybeDetached::Attached(a) = &self.inner else {
unreachable!();
};
if let Some(p) = self.get_node_parent(&target) {
if p == parent {
return Ok(false);
// If parent is deleted, we need to create the node, so this op from move_apply_diff
} else if !p.is_some_and(|p| !self.contains(p)) {
return self.move_at_with_target_for_apply_diff(parent, position, target);
}
}
match &self.inner {
MaybeDetached::Detached(t) => {
let t = &mut t.try_lock().unwrap().value;
t.create_with_target(parent, index, target);
Ok(())
}
MaybeDetached::Attached(a) => a.with_txn(|txn| {
let inner = self.inner.try_attached_state()?;
match self.generate_position_at(&target, parent, index) {
FractionalIndexGenResult::Ok(position) => {
self.create_with_position(inner, txn, target, parent, index, position)?;
}
FractionalIndexGenResult::Rearrange(ids) => {
for (i, (id, position)) in ids.into_iter().enumerate() {
if i == 0 {
self.create_with_position(inner, txn, id, parent, index, position)?;
continue;
}
self.mov_with_position(inner, txn, id, parent, index + i, position)?;
}
}
};
Ok(())
}),
let with_event = !parent.is_some_and(|p| !self.contains(p));
if !with_event {
return Ok(false);
}
// println!(
// "create_at_with_target_for_apply_diff: {:?} {:?}",
// target, parent
// );
let index = self
.get_index_by_fractional_index(
parent,
&NodePosition {
position: position.clone(),
idlp: self.next_idlp(),
},
)
// TODO: parent has deleted？
.unwrap_or(0);
let children = a.with_txn(|txn| {
let inner = self.inner.try_attached_state()?;
txn.apply_local_op(
inner.container_idx,
crate::op::RawOpContent::Tree(TreeOp::Create {
target,
parent,
position: position.clone(),
}),
EventHint::Tree(smallvec![TreeDiffItem {
target,
action: TreeExternalDiff::Create {
parent,
index,
position: position.clone(),
},
}]),
&inner.state,
)?;
Ok(self.children(Some(target)).unwrap_or_default())
})?;
for child in children {
let position = self.get_position_by_tree_id(&child).unwrap();
self.create_at_with_target_for_apply_diff(Some(target), position, child)?;
}
Ok(true)
}
pub fn create_with_txn<T: Into<Option<TreeID>>>(
/// For undo/redo, Specify the TreeID of the created node
pub(crate) fn move_at_with_target_for_apply_diff(
&self,
parent: Option<TreeID>,
position: FractionalIndex,
target: TreeID,
) -> LoroResult<bool> {
let MaybeDetached::Attached(a) = &self.inner else {
unreachable!();
};
// the move node does not exist, create it
if !self.contains(target) {
return self.create_at_with_target_for_apply_diff(parent, position, target);
}
if let Some(p) = self.get_node_parent(&target) {
if p == parent {
return Ok(false);
}
}
let index = self
.get_index_by_fractional_index(
parent,
&NodePosition {
position: position.clone(),
idlp: self.next_idlp(),
},
)
.unwrap_or(0);
let with_event = !parent.is_some_and(|p| !self.contains(p));
if !with_event {
return Ok(false);
}
// println!(
// "move_at_with_target_for_apply_diff: {:?} {:?}",
// target, parent
// );
a.with_txn(|txn| {
let inner = self.inner.try_attached_state()?;
txn.apply_local_op(
inner.container_idx,
crate::op::RawOpContent::Tree(TreeOp::Move {
target,
parent,
position: position.clone(),
}),
EventHint::Tree(smallvec![TreeDiffItem {
target,
action: TreeExternalDiff::Move {
parent,
index,
position: position.clone(),
},
}]),
&inner.state,
)
})?;
Ok(true)
}
pub(crate) fn create_with_txn<T: Into<Option<TreeID>>>(
&self,
txn: &mut Transaction,
parent: T,
@ -465,7 +543,7 @@ impl TreeHandler {
}
}
pub fn mov_with_txn<T: Into<Option<TreeID>>>(
pub(crate) fn mov_with_txn<T: Into<Option<TreeID>>>(
&self,
txn: &mut Transaction,
target: TreeID,
@ -513,6 +591,7 @@ impl TreeHandler {
}
}
#[allow(clippy::too_many_arguments)]
fn create_with_position(
&self,
inner: &BasicHandler,
@ -529,19 +608,20 @@ impl TreeHandler {
parent,
position: position.clone(),
}),
EventHint::Tree(TreeDiffItem {
EventHint::Tree(smallvec![TreeDiffItem {
target: tree_id,
action: TreeExternalDiff::Create {
parent,
index,
position,
},
}),
}]),
&inner.state,
)?;
Ok(tree_id)
}
#[allow(clippy::too_many_arguments)]
fn mov_with_position(
&self,
inner: &BasicHandler,
@ -558,19 +638,14 @@ impl TreeHandler {
parent,
position: position.clone(),
}),
EventHint::Tree(TreeDiffItem {
EventHint::Tree(smallvec![TreeDiffItem {
target,
action: TreeExternalDiff::Move {
parent,
index,
position,
old_parent: self
.get_node_parent(&target)
.map(TreeParentId::from)
.unwrap_or(TreeParentId::Unexist),
old_index: self.get_index_by_tree_id(&target).unwrap_or(0),
},
}),
}]),
&inner.state,
)
}
@ -615,17 +690,15 @@ impl TreeHandler {
}
// TODO: iterator
pub fn children(&self, parent: Option<TreeID>) -> Vec<TreeID> {
pub fn children(&self, parent: Option<TreeID>) -> Option<Vec<TreeID>> {
match &self.inner {
MaybeDetached::Detached(t) => {
let t = t.try_lock().unwrap();
t.value.get_children(parent).unwrap()
t.value.get_children(parent)
}
MaybeDetached::Attached(a) => a.with_state(|state| {
let a = state.as_tree_state().unwrap();
a.get_children(&TreeParentId::from(parent))
.unwrap()
.collect()
a.children(&TreeParentId::from(parent))
}),
}
}
@ -696,7 +769,7 @@ impl TreeHandler {
}
pub fn roots(&self) -> Vec<TreeID> {
self.children(None)
self.children(None).unwrap_or_default()
}
#[allow(non_snake_case)]
@ -762,4 +835,30 @@ impl TreeHandler {
a.delete_position(&TreeParentId::from(parent), target)
})
}
// use for apply diff
pub(crate) fn get_index_by_fractional_index(
&self,
parent: Option<TreeID>,
node_position: &NodePosition,
) -> Option<usize> {
match &self.inner {
MaybeDetached::Detached(_) => {
unreachable!();
}
MaybeDetached::Attached(a) => a.with_state(|state| {
let a = state.as_tree_state().unwrap();
a.get_index_by_position(&TreeParentId::from(parent), node_position)
}),
}
}
pub(crate) fn next_idlp(&self) -> IdLp {
match &self.inner {
MaybeDetached::Detached(_) => {
unreachable!()
}
MaybeDetached::Attached(a) => a.with_txn(|txn| Ok(txn.next_idlp())).unwrap(),
}
}
}

12
crates/loro-internal/src/kv_store.rs
View file

@ -1,5 +1,8 @@
use bytes::Bytes;
use std::ops::Bound;
use std::{
ops::Bound,
sync::{Arc, Mutex},
};
pub type CompareFn<'a> = &'a mut dyn FnMut(&Bytes, &Bytes) -> std::cmp::Ordering;
pub trait KvStore: std::fmt::Debug + Send + Sync {
@ -28,6 +31,7 @@ pub trait KvStore: std::fmt::Debug + Send + Sync {
) -> Option<(Bytes, Bytes)>;
fn export_all(&self) -> Bytes;
fn import_all(&mut self, bytes: Bytes) -> Result<(), String>;
fn clone_store(&self) -> Arc<Mutex<dyn KvStore>>;
}
pub trait KvEntry {
@ -127,7 +131,7 @@ mod default_binary_format {
mod mem {
use super::*;
use std::collections::BTreeMap;
use std::{collections::BTreeMap, sync::Arc};
pub type MemKvStore = BTreeMap<Bytes, Bytes>;
impl KvStore for MemKvStore {
@ -212,6 +216,10 @@ mod mem {
None
}
fn clone_store(&self) -> Arc<Mutex<dyn KvStore>> {
Arc::new(Mutex::new(self.clone()))
}
}
#[cfg(test)]

53
crates/loro-internal/src/loro.rs
View file

@ -118,6 +118,41 @@ impl LoroDoc {
}
}
pub fn fork(&self) -> Self {
self.commit_then_stop();
let arena = self.arena.fork();
let config = self.config.fork();
let txn = Arc::new(Mutex::new(None));
let new_state =
self.state
.lock()
.unwrap()
.fork(arena.clone(), Arc::downgrade(&txn), config.clone());
let doc = LoroDoc {
oplog: Arc::new(Mutex::new(
self.oplog()
.lock()
.unwrap()
.fork(arena.clone(), config.clone()),
)),
state: new_state,
arena,
config,
observer: Arc::new(Observer::new(self.arena.clone())),
diff_calculator: Arc::new(Mutex::new(DiffCalculator::new())),
txn,
auto_commit: AtomicBool::new(false),
detached: AtomicBool::new(self.detached.load(std::sync::atomic::Ordering::Relaxed)),
};
if self.auto_commit.load(std::sync::atomic::Ordering::Relaxed) {
doc.start_auto_commit();
}
self.renew_txn_if_auto_commit();
doc
}
/// Set whether to record the timestamp of each change. Default is `false`.
///
/// If enabled, the Unix timestamp will be recorded for each change automatically.
@ -819,6 +854,9 @@ impl LoroDoc {
before_diff,
);
// println!("\nundo_internal: diff: {:?}", diff);
// println!("container remap: {:?}", container_remap);
self.checkout_without_emitting(&latest_frontiers)?;
self.detached.store(false, Release);
if was_recording {
@ -915,10 +953,7 @@ impl LoroDoc {
}
let h = self.get_handler(id);
h.apply_diff(diff, &mut |old_id, new_id| {
container_remap.insert(old_id, new_id);
})
.unwrap();
h.apply_diff(diff, container_remap).unwrap();
}
Ok(())
@ -1054,6 +1089,7 @@ impl LoroDoc {
#[instrument(level = "info", skip(self))]
fn checkout_without_emitting(&self, frontiers: &Frontiers) -> Result<(), LoroError> {
tracing::debug!("Checkout from {:?}", self.state_frontiers());
self.commit_then_stop();
let oplog = self.oplog.lock().unwrap();
let mut state = self.state.lock().unwrap();
@ -1071,7 +1107,8 @@ impl LoroDoc {
format!("Cannot find the specified version {:?}", frontiers).into_boxed_str(),
));
};
tracing::trace!("before: {:?}", before);
tracing::trace!("after: {:?}", after);
let diff = calc.calc_diff_internal(
&oplog,
before,
@ -1080,6 +1117,7 @@ impl LoroDoc {
Some(frontiers),
None,
);
tracing::debug!("diff: {:?}", &diff);
state.apply_diff(InternalDocDiff {
origin: "checkout".into(),
diff: Cow::Owned(diff),
@ -1149,6 +1187,11 @@ impl LoroDoc {
let doc = Self::new();
doc.detach();
doc.import(&bytes).unwrap();
dbg!(
self.state_frontiers(),
self.oplog_frontiers(),
self.oplog_vv()
);
doc.checkout(&self.state_frontiers()).unwrap();
let mut calculated_state = doc.app_state().try_lock().unwrap();
let mut current_state = self.app_state().try_lock().unwrap();

1
crates/loro-internal/src/obs.rs
View file

@ -100,7 +100,6 @@ impl Observer {
self.inner.lock().unwrap().event_queue.push(doc_diff);
return;
}
let mut inner = self.take_inner();
self.emit_inner(&doc_diff, &mut inner);
self.reset_inner(inner);

14
crates/loro-internal/src/oplog.rs
View file

@ -81,18 +81,20 @@ pub struct AppDagNode {
pub(crate) len: usize,
}
impl Clone for OpLog {
fn clone(&self) -> Self {
impl OpLog {
pub(crate) fn fork(&self, arena: SharedArena, configure: Configure) -> Self {
Self {
change_store: self
.change_store
.fork(arena.clone(), configure.merge_interval.clone()),
dag: self.dag.clone(),
arena: self.arena.clone(),
op_groups: self.op_groups.clone(),
change_store: self.change_store.clone(),
op_groups: self.op_groups.fork(arena.clone()),
next_lamport: self.next_lamport,
latest_timestamp: self.latest_timestamp,
pending_changes: Default::default(),
batch_importing: false,
configure: self.configure.clone(),
configure,
}
}
}
@ -475,7 +477,7 @@ impl OpLog {
b: &VersionVector,
mut f: impl FnMut(&Change),
) {
let spans = b.sub_iter(a);
let spans = b.iter_between(a);
for span in spans {
for c in self.change_store.iter_changes(span) {
f(&c);

12
crates/loro-internal/src/oplog/change_store.rs
View file

@ -369,6 +369,18 @@ impl ChangeStore {
let mut kv = self.mem_parsed_kv.lock().unwrap();
kv.iter_mut().map(|(_, block)| block.change_num()).sum()
}
pub fn fork(&self, arena: SharedArena, merge_interval: Arc<AtomicI64>) -> Self {
Self {
arena,
vv: self.vv.clone(),
start_vv: self.start_vv.clone(),
start_frontiers: self.start_frontiers.clone(),
mem_parsed_kv: Arc::new(Mutex::new(BTreeMap::new())),
external_kv: self.external_kv.lock().unwrap().clone_store(),
merge_interval,
}
}
}
#[derive(Clone, Debug)]

109
crates/loro-internal/src/state.rs
View file

@ -13,12 +13,13 @@ use enum_dispatch::enum_dispatch;
use fxhash::{FxHashMap, FxHashSet};
use loro_common::{ContainerID, LoroError, LoroResult};
use loro_delta::DeltaItem;
use tracing::{info, instrument};
use tracing::instrument;
use crate::{
configure::{Configure, DefaultRandom, SecureRandomGenerator},
container::{idx::ContainerIdx, richtext::config::StyleConfigMap, ContainerIdRaw},
cursor::Cursor,
delta::TreeExternalDiff,
diff_calc::DiffCalculator,
encoding::{StateSnapshotDecodeContext, StateSnapshotEncoder},
event::{Diff, EventTriggerKind, Index, InternalContainerDiff, InternalDiff},
@ -45,7 +46,9 @@ pub(crate) use self::movable_list_state::{IndexType, MovableListState};
pub(crate) use list_state::ListState;
pub(crate) use map_state::MapState;
pub(crate) use richtext_state::RichtextState;
pub(crate) use tree_state::{get_meta_value, FractionalIndexGenResult, TreeParentId, TreeState};
pub(crate) use tree_state::{
get_meta_value, FractionalIndexGenResult, NodePosition, TreeParentId, TreeState,
};
use self::{container_store::ContainerWrapper, unknown_state::UnknownState};
@ -67,7 +70,6 @@ macro_rules! get_or_create {
}};
}
#[derive(Clone)]
pub struct DocState {
pub(super) peer: Arc<AtomicU64>,
@ -383,6 +385,29 @@ impl DocState {
})
}
pub fn fork(
&self,
arena: SharedArena,
global_txn: Weak<Mutex<Option<Transaction>>>,
config: Configure,
) -> Arc<Mutex<Self>> {
Arc::new_cyclic(|weak| {
let peer = Arc::new(AtomicU64::new(DefaultRandom.next_u64()));
Mutex::new(Self {
peer: peer.clone(),
frontiers: self.frontiers.clone(),
store: self.store.fork(arena.clone(), peer, config.clone()),
arena,
config,
weak_state: weak.clone(),
global_txn,
in_txn: false,
changed_idx_in_txn: FxHashSet::default(),
event_recorder: Default::default(),
})
})
}
pub fn start_recording(&mut self) {
if self.is_recording() {
return;
@ -527,7 +552,6 @@ impl DocState {
// We need to ensure diff is processed in order
diffs.sort_by_cached_key(|diff| self.arena.get_depth(diff.idx).unwrap());
let mut to_revive_in_next_layer: FxHashSet<ContainerIdx> = FxHashSet::default();
let mut to_revive_in_this_layer: FxHashSet<ContainerIdx> = FxHashSet::default();
let mut last_depth = 0;
@ -547,9 +571,13 @@ impl DocState {
let external_diff =
state.to_diff(&self.arena, &self.global_txn, &self.weak_state);
trigger_on_new_container(&external_diff, |cid| {
to_revive_in_this_layer.insert(cid);
});
trigger_on_new_container(
&external_diff,
|cid| {
to_revive_in_this_layer.insert(cid);
},
&self.arena,
);
diffs.push(InternalContainerDiff {
idx: new,
@ -570,9 +598,13 @@ impl DocState {
let state = get_or_create!(self, diff.idx);
let extern_diff =
state.to_diff(&self.arena, &self.global_txn, &self.weak_state);
trigger_on_new_container(&extern_diff, |cid| {
to_revive_in_next_layer.insert(cid);
});
trigger_on_new_container(
&extern_diff,
|cid| {
to_revive_in_next_layer.insert(cid);
},
&self.arena,
);
diff.diff = extern_diff.into();
}
}
@ -600,9 +632,13 @@ impl DocState {
&self.weak_state,
)
};
trigger_on_new_container(&external_diff, |cid| {
to_revive_in_next_layer.insert(cid);
});
trigger_on_new_container(
&external_diff,
|cid| {
to_revive_in_next_layer.insert(cid);
},
&self.arena,
);
diff.diff = external_diff.into();
} else {
state.apply_diff(
@ -617,7 +653,9 @@ impl DocState {
}
to_revive_in_this_layer.remove(&idx);
diffs.push(diff);
if !diff.diff.is_empty() {
diffs.push(diff);
}
}
// Revive the last several layers
@ -630,16 +668,22 @@ impl DocState {
}
let external_diff = state.to_diff(&self.arena, &self.global_txn, &self.weak_state);
trigger_on_new_container(&external_diff, |cid| {
to_revive_in_next_layer.insert(cid);
});
trigger_on_new_container(
&external_diff,
|cid| {
to_revive_in_next_layer.insert(cid);
},
&self.arena,
);
diffs.push(InternalContainerDiff {
idx: new,
bring_back: true,
is_container_deleted: false,
diff: external_diff.into(),
});
if !external_diff.is_empty() {
diffs.push(InternalContainerDiff {
idx: new,
bring_back: true,
is_container_deleted: false,
diff: external_diff.into(),
});
}
}
to_revive_in_this_layer = std::mem::take(&mut to_revive_in_next_layer);
@ -1086,8 +1130,9 @@ impl DocState {
// if we cannot find the path to the container, the container must be overwritten afterwards.
// So we can ignore the diff from it.
tracing::warn!(
"⚠️ WARNING: ignore event because cannot find its path {:#?}",
"⚠️ WARNING: ignore event because cannot find its path {:#?} container id:{}",
&container_diff,
self.arena.idx_to_id(container_diff.idx).unwrap()
);
}
@ -1171,7 +1216,7 @@ impl DocState {
// this container may be deleted
let Ok(prop) = id.clone().into_root() else {
let id = format!("{}", &id);
info!(?id, "Missing parent - container is deleted");
tracing::info!(?id, "Missing parent - container is deleted");
return None;
};
ans.push((id, Index::Key(prop.0)));
@ -1428,7 +1473,11 @@ fn create_state_(idx: ContainerIdx, config: &Configure, peer: u64) -> State {
}
}
fn trigger_on_new_container(state_diff: &Diff, mut listener: impl FnMut(ContainerIdx)) {
fn trigger_on_new_container(
state_diff: &Diff,
mut listener: impl FnMut(ContainerIdx),
arena: &SharedArena,
) {
match state_diff {
Diff::List(list) => {
for delta in list.iter() {
@ -1459,6 +1508,14 @@ fn trigger_on_new_container(state_diff: &Diff, mut listener: impl FnMut(Containe
}
}
}
Diff::Tree(tree) => {
for item in tree.iter() {
if matches!(item.action, TreeExternalDiff::Create { .. }) {
let id = item.target.associated_meta_container();
listener(arena.id_to_idx(&id).unwrap());
}
}
}
_ => {}
};
}

23
crates/loro-internal/src/state/container_store.rs
View file

@ -1,6 +1,4 @@
use std::{
sync::{atomic::AtomicU64, Arc},
};
use std::sync::{atomic::AtomicU64, Arc};
use crate::{
arena::SharedArena,
@ -207,6 +205,25 @@ impl ContainerStore {
pub(super) fn insert(&mut self, idx: ContainerIdx, state: ContainerWrapper) {
self.store.insert(idx, state);
}
pub(crate) fn fork(
&self,
arena: SharedArena,
peer: Arc<AtomicU64>,
config: Configure,
) -> ContainerStore {
let mut store = FxHashMap::default();
for (idx, container) in self.store.iter() {
store.insert(*idx, container.clone());
}
ContainerStore {
arena,
store,
conf: config,
peer,
}
}
}
#[derive(Clone, Debug)]

15
crates/loro-internal/src/state/richtext_state.rs
View file

@ -5,7 +5,7 @@ use std::{
use fxhash::{FxHashMap, FxHashSet};
use generic_btree::rle::HasLength;
use loro_common::{ContainerID, InternalString, LoroResult, LoroValue, ID};
use loro_common::{ContainerID, InternalString, LoroError, LoroResult, LoroValue, ID};
use loro_delta::DeltaRopeBuilder;
use crate::{
@ -743,10 +743,14 @@ impl RichtextState {
}
#[inline]
pub(crate) fn get_entity_index_for_text_insert(&mut self, event_index: usize) -> usize {
pub(crate) fn get_entity_index_for_text_insert(
&mut self,
event_index: usize,
pos_type: PosType,
) -> Result<usize, LoroError> {
self.state
.get_mut()
.get_entity_index_for_text_insert(event_index, PosType::Event)
.get_entity_index_for_text_insert(event_index, pos_type)
}
pub(crate) fn get_entity_range_and_styles_at_range(
@ -771,10 +775,11 @@ impl RichtextState {
&mut self,
pos: usize,
len: usize,
) -> Vec<EntityRangeInfo> {
pos_type: PosType,
) -> LoroResult<Vec<EntityRangeInfo>> {
self.state
.get_mut()
.get_text_entity_ranges(pos, len, PosType::Event)
.get_text_entity_ranges(pos, len, pos_type)
}
#[inline]

120
crates/loro-internal/src/state/tree_state.rs
View file

@ -57,7 +57,7 @@ impl From<Option<TreeID>> for TreeParentId {
}
}
#[derive(Clone)]
#[derive(Debug, Clone)]
enum NodeChildren {
Vec(Vec<(NodePosition, TreeID)>),
BTree(btree::ChildTree),
@ -77,6 +77,16 @@ impl NodeChildren {
}
}
fn get_last_insert_index_by_position(
&self,
node_position: &NodePosition,
) -> Result<usize, usize> {
match self {
NodeChildren::Vec(v) => v.binary_search_by_key(&node_position, |x| &x.0),
NodeChildren::BTree(btree) => btree.get_index_by_node_position(node_position),
}
}
fn get_node_position_at(&self, pos: usize) -> Option<&NodePosition> {
match self {
NodeChildren::Vec(v) => v.get(pos).map(|x| &x.0),
@ -322,6 +332,33 @@ mod btree {
Some(ans)
}
pub(super) fn get_index_by_node_position(
&self,
node_position: &NodePosition,
) -> Result<usize, usize> {
let Some(res) = self.tree.query::<KeyQuery>(node_position) else {
return Ok(0);
};
let mut ans = 0;
self.tree
.visit_previous_caches(res.cursor, |prev| match prev {
generic_btree::PreviousCache::NodeCache(c) => {
ans += c.len;
}
generic_btree::PreviousCache::PrevSiblingElem(_) => {
ans += 1;
}
generic_btree::PreviousCache::ThisElemAndOffset { elem: _, offset } => {
ans += offset;
}
});
if res.found {
Ok(ans)
} else {
Err(ans)
}
}
}
#[derive(Clone, Debug)]
@ -525,13 +562,13 @@ pub struct TreeState {
jitter: u8,
}
#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord)]
struct NodePosition {
position: FractionalIndex,
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub(crate) struct NodePosition {
pub(crate) position: FractionalIndex,
// different nodes created by a peer may have the same position
// when we merge updates that cause cycles.
// for example [::fuzz::test::test_tree::same_peer_have_same_position()]
idlp: IdLp,
pub(crate) idlp: IdLp,
}
impl NodePosition {
@ -584,22 +621,10 @@ impl TreeState {
self.delete_position(&old_parent, target);
}
if !parent.is_deleted() {
let entry = self.children.entry(parent).or_default();
let node_position = NodePosition::new(position.clone().unwrap(), id.idlp());
debug_assert!(!entry.has_child(&node_position));
entry.insert_child(node_position, target);
} else {
// clean the cache recursively, otherwise the index of event will be calculated incorrectly
let mut q = vec![target];
while let Some(id) = q.pop() {
let parent = TreeParentId::from(Some(id));
if let Some(children) = self.children.get(&parent) {
q.extend(children.iter().map(|x| x.1));
}
self.children.remove(&parent);
}
}
let entry = self.children.entry(parent).or_default();
let node_position = NodePosition::new(position.clone().unwrap_or_default(), id.idlp());
debug_assert!(!entry.has_child(&node_position));
entry.insert_child(node_position, target);
self.trees.insert(
target,
@ -652,8 +677,7 @@ impl TreeState {
.unwrap_or(TreeParentId::Unexist)
}
/// If the node is not deleted or does not exist, return false.
/// only the node is deleted and exists, return true
/// If the node exists and is not deleted, return false.
fn is_node_deleted(&self, target: &TreeID) -> bool {
match self.trees.get(target) {
Some(x) => match x.parent {
@ -662,7 +686,7 @@ impl TreeState {
TreeParentId::Node(p) => self.is_node_deleted(&p),
TreeParentId::Unexist => unreachable!(),
},
None => false,
None => true,
}
}
@ -733,11 +757,10 @@ impl TreeState {
self.children.get(parent).map(|x| x.len())
}
pub fn children(&self, parent: &TreeParentId) -> Vec<TreeID> {
pub fn children(&self, parent: &TreeParentId) -> Option<Vec<TreeID>> {
self.children
.get(parent)
.map(|x| x.iter().map(|x| *x.1).collect())
.unwrap_or_default()
}
/// Determine whether the target is the child of the node
@ -790,6 +813,19 @@ impl TreeState {
.flatten()
}
pub(crate) fn get_index_by_position(
&self,
parent: &TreeParentId,
node_position: &NodePosition,
) -> Option<usize> {
self.children.get(parent).map(|c| {
match c.get_last_insert_index_by_position(node_position) {
Ok(i) => i,
Err(i) => i,
}
})
}
pub(crate) fn get_id_by_index(&self, parent: &TreeParentId, index: usize) -> Option<TreeID> {
(!parent.is_deleted())
.then(|| self.children.get(parent).and_then(|x| x.get_id_at(index)))
@ -846,8 +882,6 @@ impl ContainerState for TreeState {
});
}
TreeInternalDiff::Move { parent, position } => {
let old_parent = self.trees.get(&target).unwrap().parent;
let old_index = self.get_index_by_tree_id(&target).unwrap();
self.mov(target, *parent, last_move_op, Some(position.clone()), false)
.unwrap();
let index = self.get_index_by_tree_id(&target).unwrap();
@ -857,22 +891,15 @@ impl ContainerState for TreeState {
parent: parent.into_node().ok(),
index,
position: position.clone(),
old_parent,
old_index,
},
});
}
TreeInternalDiff::Delete { parent, position } => {
let old_parent = self.trees.get(&target).unwrap().parent;
let old_index = self.get_index_by_tree_id(&target).unwrap();
self.mov(target, *parent, last_move_op, position.clone(), false)
.unwrap();
ans.push(TreeDiffItem {
target,
action: TreeExternalDiff::Delete {
old_parent,
old_index,
},
action: TreeExternalDiff::Delete,
});
}
TreeInternalDiff::MoveInDelete { parent, position } => {
@ -880,15 +907,13 @@ impl ContainerState for TreeState {
.unwrap();
}
TreeInternalDiff::UnCreate => {
let old_parent = self.trees.get(&target).unwrap().parent;
let old_index = self.get_index_by_tree_id(&target).unwrap();
ans.push(TreeDiffItem {
target,
action: TreeExternalDiff::Delete {
old_parent,
old_index,
},
});
// maybe the node created and moved to the parent deleted
if !self.is_node_deleted(&target) {
ans.push(TreeDiffItem {
target,
action: TreeExternalDiff::Delete,
});
}
// delete it from state
let parent = self.trees.remove(&target);
if let Some(parent) = parent {
@ -1139,7 +1164,10 @@ impl TreeNode {
self.id.associated_meta_container().into(),
);
t.insert("index".to_string(), (self.index as i64).into());
t.insert("position".to_string(), self.position.to_string().into());
t.insert(
"fractional_index".to_string(),
self.position.to_string().into(),
);
t.into()
}
}

19
crates/loro-internal/src/txn.rs
View file

@ -123,7 +123,8 @@ pub(super) enum EventHint {
key: InternalString,
value: Option<LoroValue>,
},
Tree(TreeDiffItem),
// use vec because we could bring back some node that has children
Tree(SmallVec<[TreeDiffItem; 1]>),
MarkEnd,
#[cfg(feature = "counter")]
Counter(f64),
@ -403,6 +404,7 @@ impl Transaction {
let op = self.arena.convert_raw_op(&raw_op);
state.apply_local_op(&raw_op, &op)?;
drop(state);
debug_assert_eq!(
event.rle_len(),
op.atom_len(),
@ -410,6 +412,7 @@ impl Transaction {
&event,
&op
);
match self.event_hints.last_mut() {
Some(last) if last.can_merge(&event) => {
last.merge_right(&event);
@ -486,6 +489,17 @@ impl Transaction {
counter: self.next_counter,
}
}
pub fn next_idlp(&self) -> IdLp {
IdLp {
peer: self.peer,
lamport: self.next_lamport,
}
}
pub fn is_empty(&self) -> bool {
self.local_ops.is_empty()
}
}
impl Drop for Transaction {
@ -650,7 +664,7 @@ fn change_to_diff(
}),
EventHint::Tree(tree_diff) => {
let mut diff = TreeDiff::default();
diff.push(tree_diff);
diff.diff.extend(tree_diff.into_iter());
ans.push(TxnContainerDiff {
idx: op.container,
diff: Diff::Tree(diff),
@ -709,6 +723,5 @@ fn change_to_diff(
.map(|x| x.content_len() as Lamport)
.sum::<Lamport>();
}
ans
}

62
crates/loro-internal/src/undo.rs
View file

@ -40,9 +40,11 @@ impl DiffBatch {
return;
}
for (idx, diff) in self.0.iter_mut() {
if let Some(b_diff) = other.0.get(idx) {
diff.compose_ref(b_diff);
for (idx, diff) in other.0.iter() {
if let Some(this_diff) = self.0.get_mut(idx) {
this_diff.compose_ref(diff);
} else {
self.0.insert(idx.clone(), diff.clone());
}
}
}
@ -146,7 +148,7 @@ pub type OnPush = Box<dyn Fn(UndoOrRedo, CounterSpan) -> UndoItemMeta + Send + S
pub type OnPop = Box<dyn Fn(UndoOrRedo, CounterSpan, UndoItemMeta) + Send + Sync>;
struct UndoManagerInner {
latest_counter: Counter,
latest_counter: Option<Counter>,
undo_stack: Stack,
redo_stack: Stack,
processing_undo: bool,
@ -180,7 +182,7 @@ struct Stack {
size: usize,
}
#[derive(Debug)]
#[derive(Debug, Clone)]
struct StackItem {
span: CounterSpan,
meta: UndoItemMeta,
@ -210,7 +212,7 @@ impl UndoItemMeta {
}
}
/// It's assumed that the cursor is just acqured before the ops that
/// It's assumed that the cursor is just acquired before the ops that
/// need to be undo/redo.
///
/// We need to rely on the validity of the original pos value
@ -271,19 +273,14 @@ impl Stack {
pub fn push_with_merge(&mut self, span: CounterSpan, meta: UndoItemMeta, can_merge: bool) {
let last = self.stack.back_mut().unwrap();
let mut last_remote_diff = last.1.try_lock().unwrap();
let last_remote_diff = last.1.try_lock().unwrap();
if !last_remote_diff.0.is_empty() {
// If the remote diff is not empty, we cannot merge
if last.0.is_empty() {
last.0.push_back(StackItem { span, meta });
last_remote_diff.clear();
} else {
drop(last_remote_diff);
let mut v = VecDeque::new();
v.push_back(StackItem { span, meta });
self.stack
.push_back((v, Arc::new(Mutex::new(DiffBatch::default()))));
}
drop(last_remote_diff);
let mut v = VecDeque::new();
v.push_back(StackItem { span, meta });
self.stack
.push_back((v, Arc::new(Mutex::new(DiffBatch::default()))));
self.size += 1;
} else {
@ -322,7 +319,6 @@ impl Stack {
if self.is_empty() {
return;
}
let remote_diff = &mut self.stack.back_mut().unwrap().1;
remote_diff.try_lock().unwrap().transform(diff, false);
}
@ -365,7 +361,7 @@ impl Default for Stack {
impl UndoManagerInner {
fn new(last_counter: Counter) -> Self {
UndoManagerInner {
latest_counter: last_counter,
latest_counter: Some(last_counter),
undo_stack: Default::default(),
redo_stack: Default::default(),
processing_undo: false,
@ -380,13 +376,18 @@ impl UndoManagerInner {
}
fn record_checkpoint(&mut self, latest_counter: Counter) {
if latest_counter == self.latest_counter {
if Some(latest_counter) == self.latest_counter {
return;
}
assert!(self.latest_counter < latest_counter);
if self.latest_counter.is_none() {
self.latest_counter = Some(latest_counter);
return;
}
assert!(self.latest_counter.unwrap() < latest_counter);
let now = get_sys_timestamp();
let span = CounterSpan::new(self.latest_counter, latest_counter);
let span = CounterSpan::new(self.latest_counter.unwrap(), latest_counter);
let meta = self
.on_push
.as_ref()
@ -400,7 +401,7 @@ impl UndoManagerInner {
self.undo_stack.push(span, meta);
}
self.latest_counter = latest_counter;
self.latest_counter = Some(latest_counter);
self.redo_stack.clear();
while self.undo_stack.len() > self.max_stack_size {
self.undo_stack.pop_front();
@ -446,7 +447,7 @@ impl UndoManager {
// a remote event.
inner.undo_stack.compose_remote_event(event.events);
inner.redo_stack.compose_remote_event(event.events);
inner.latest_counter = id.counter + 1;
inner.latest_counter = Some(id.counter + 1);
} else {
inner.record_checkpoint(id.counter + 1);
}
@ -457,7 +458,12 @@ impl UndoManager {
inner.undo_stack.compose_remote_event(event.events);
inner.redo_stack.compose_remote_event(event.events);
}
EventTriggerKind::Checkout => {}
EventTriggerKind::Checkout => {
let mut inner = inner_clone.try_lock().unwrap();
inner.undo_stack.clear();
inner.redo_stack.clear();
inner.latest_counter = None;
}
}));
UndoManager {
@ -649,7 +655,7 @@ impl UndoManager {
}
get_opposite(&mut inner).push(CounterSpan::new(end_counter, new_counter), meta);
inner.latest_counter = new_counter;
inner.latest_counter = Some(new_counter);
executed = true;
break;
} else {
@ -750,7 +756,6 @@ pub(crate) fn undo(
// ------------------------------------------------------------------------------
// 1.b Transform and apply Ci-1 based on Ai, call it A'i
// ------------------------------------------------------------------------------
last_ci.transform(&event_a_i, true);
event_a_i.compose(&last_ci);
@ -761,13 +766,12 @@ pub(crate) fn undo(
if i == spans.len() - 1 {
on_last_event_a(&event_a_prime);
}
// --------------------------------------------------
// 3. Transform event A'_i based on B_i, call it C_i
// --------------------------------------------------
event_a_prime.transform(event_b_i, true);
let c_i = event_a_prime;
let c_i = event_a_prime;
last_ci = Some(c_i);
});
}

4
crates/loro-internal/src/value.rs
View file

@ -563,7 +563,7 @@ pub mod wasm {
js_sys::Reflect::set(&obj, &"index".into(), &(*index).into()).unwrap();
js_sys::Reflect::set(
&obj,
&"position".into(),
&"fractional_index".into(),
&position.to_string().into(),
)
.unwrap();
@ -583,7 +583,7 @@ pub mod wasm {
js_sys::Reflect::set(&obj, &"index".into(), &(*index).into()).unwrap();
js_sys::Reflect::set(
&obj,
&"position".into(),
&"fractional_index".into(),
&position.to_string().into(),
)
.unwrap();

6
crates/loro-internal/src/version.rs
View file

@ -477,6 +477,12 @@ impl VersionVector {
})
}
/// Iter all span from a -> b and b -> a
pub fn iter_between<'a>(&'a self, other: &'a Self) -> impl Iterator<Item = IdSpan> + 'a {
// PERF: can be optimized a little
self.sub_iter(other).chain(other.sub_iter(self))
}
pub fn sub_vec(&self, rhs: &Self) -> IdSpanVector {
self.sub_iter(rhs).map(|x| (x.peer, x.counter)).collect()
}

169
crates/loro-internal/tests/test.rs
View file

@ -720,26 +720,18 @@ fn map_concurrent_checkout() {
#[test]
fn tree_checkout() {
let doc_a = LoroDoc::new();
let doc_a = LoroDoc::new_auto_commit();
doc_a.subscribe_root(Arc::new(|_e| {}));
doc_a.set_peer_id(1).unwrap();
let tree = doc_a.get_tree("root");
let id1 = doc_a
.with_txn(|txn| tree.create_with_txn(txn, None, 0))
.unwrap();
let id2 = doc_a
.with_txn(|txn| tree.create_with_txn(txn, id1, 0))
.unwrap();
let id1 = tree.create(None).unwrap();
let id2 = tree.create(id1).unwrap();
let v1_state = tree.get_deep_value();
let v1 = doc_a.oplog_frontiers();
let _id3 = doc_a
.with_txn(|txn| tree.create_with_txn(txn, id2, 0))
.unwrap();
let _id3 = tree.create(id2).unwrap();
let v2_state = tree.get_deep_value();
let v2 = doc_a.oplog_frontiers();
doc_a
.with_txn(|txn| tree.delete_with_txn(txn, id2))
.unwrap();
tree.delete(id2).unwrap();
let v3_state = tree.get_deep_value();
let v3 = doc_a.oplog_frontiers();
doc_a.checkout(&v1).unwrap();
@ -765,12 +757,7 @@ fn tree_checkout() {
);
doc_a.attach();
doc_a
.with_txn(|txn| {
tree.create_with_txn(txn, None, 0)
//tree.insert_meta(txn, id1, "a", 1.into())
})
.unwrap();
tree.create(None).unwrap();
}
#[test]
@ -973,3 +960,147 @@ fn counter() {
let doc2 = LoroDoc::new_auto_commit();
doc2.import_json_updates(json).unwrap();
}
#[test]
fn test_insert_utf8() {
let doc = LoroDoc::new_auto_commit();
let text = doc.get_text("text");
text.insert_utf8(0, "Hello ").unwrap();
text.insert_utf8(6, "World").unwrap();
assert_eq!(
text.get_richtext_value().to_json_value(),
json!([{"insert":"Hello World"}])
)
}
#[test]
fn test_insert_utf8_cross_unicode_1() {
let doc = LoroDoc::new_auto_commit();
let text = doc.get_text("text");
text.insert_utf8(0, "你好").unwrap();
text.insert_utf8(3, "World").unwrap();
assert_eq!(
text.get_richtext_value().to_json_value(),
json!([{"insert":"你World好"}])
)
}
#[test]
fn test_insert_utf8_cross_unicode_2() {
let doc = LoroDoc::new_auto_commit();
let text = doc.get_text("text");
text.insert_utf8(0, "你好").unwrap();
text.insert_utf8(6, "World").unwrap();
assert_eq!(
text.get_richtext_value().to_json_value(),
json!([{"insert":"你好World"}])
)
}
#[test]
fn test_insert_utf8_detached() {
let text = TextHandler::new_detached();
text.insert_utf8(0, "Hello ").unwrap();
text.insert_utf8(6, "World").unwrap();
assert_eq!(
text.get_richtext_value().to_json_value(),
json!([{"insert":"Hello World"}])
)
}
#[test]
#[should_panic]
fn test_insert_utf8_panic_cross_unicode() {
let doc = LoroDoc::new_auto_commit();
let text = doc.get_text("text");
text.insert_utf8(0, "你好").unwrap();
text.insert_utf8(1, "World").unwrap();
}
#[test]
#[should_panic]
fn test_insert_utf8_panic_out_bound() {
let doc = LoroDoc::new_auto_commit();
let text = doc.get_text("text");
text.insert_utf8(0, "Hello ").unwrap();
text.insert_utf8(7, "World").unwrap();
}
// println!("{}", text.get_richtext_value().to_json_value().to_string());
#[test]
fn test_delete_utf8() {
let doc = LoroDoc::new_auto_commit();
let text = doc.get_text("text");
text.insert_utf8(0, "Hello").unwrap();
text.delete_utf8(1, 3).unwrap();
assert_eq!(
text.get_richtext_value().to_json_value(),
json!([{"insert":"Ho"}])
)
}
#[test]
fn test_delete_utf8_with_zero_len() {
let doc = LoroDoc::new_auto_commit();
let text = doc.get_text("text");
text.insert_utf8(0, "Hello").unwrap();
text.delete_utf8(1, 0).unwrap();
assert_eq!(
text.get_richtext_value().to_json_value(),
json!([{"insert":"Hello"}])
)
}
#[test]
fn test_delete_utf8_cross_unicode() {
let doc = LoroDoc::new_auto_commit();
let text = doc.get_text("text");
text.insert_utf8(0, "你好").unwrap();
text.delete_utf8(0, 3).unwrap();
assert_eq!(
text.get_richtext_value().to_json_value(),
json!([{"insert":"好"}])
)
}
#[test]
fn test_delete_utf8_detached() {
let text = TextHandler::new_detached();
text.insert_utf8(0, "Hello").unwrap();
text.delete_utf8(1, 3).unwrap();
assert_eq!(
text.get_richtext_value().to_json_value(),
json!([{"insert":"Ho"}])
)
}
// WARNING:
// Due to the current inability to report an error on
// get_offset_and_found on BTree, this test won't be ok.
// #[test]
// #[should_panic]
// fn test_delete_utf8_panic_cross_unicode() {
// let doc = LoroDoc::new_auto_commit();
// let text = doc.get_text("text");
// text.insert_utf8(0, "你好").unwrap();
// text.delete_utf8(0, 2).unwrap();
// }
#[test]
#[should_panic]
fn test_delete_utf8_panic_out_bound_pos() {
let doc = LoroDoc::new_auto_commit();
let text = doc.get_text("text");
text.insert(0, "Hello").unwrap();
text.delete_utf8(10, 1).unwrap();
}
#[test]
#[should_panic]
fn test_delete_utf8_panic_out_bound_len() {
let doc = LoroDoc::new_auto_commit();
let text = doc.get_text("text");
text.insert(0, "Hello").unwrap();
text.delete_utf8(1, 10).unwrap();
}

12
crates/loro-wasm/CHANGELOG.md
View file

@ -1,5 +1,17 @@
# Changelog
## 0.16.6
### Patch Changes
- 1e94248: Add `.fork()` to duplicate the doc
## 0.16.5
### Patch Changes
- 439e4e9: Update pkg desc
## 0.16.4
### Patch Changes

3
crates/loro-wasm/Cargo.toml
View file

@ -10,7 +10,7 @@ crate-type = ["cdylib", "rlib"]
[dependencies]
js-sys = "0.3.60"
loro-internal = { path = "../loro-internal", features = ["wasm"] }
loro-internal = { path = "../loro-internal", features = ["wasm", "counter"] }
wasm-bindgen = "=0.2.92"
serde-wasm-bindgen = { version = "^0.6.5" }
wasm-bindgen-derive = "0.2.1"
@ -24,4 +24,3 @@ serde_json = "1"
[features]
default = ["console_error_panic_hook"]
counter = ["loro-internal/counter"]

142
crates/loro-wasm/README.md
View file

@ -1,3 +1,141 @@
# Loro WASM
<p align="center">
<a href="https://loro.dev">
<picture>
<img src="./docs/Loro.svg" width="200"/>
</picture>
</a>
</p>
<h1 align="center">
<a href="https://loro.dev" alt="loro-site">Loro</a>
</h1>
<p align="center">
<b>Reimagine state management with CRDTs 🦜</b><br/>
Make your app state synchronized and collaborative effortlessly.
</p>
<p align="center">
<a href="https://trendshift.io/repositories/4964" target="_blank"><img src="https://trendshift.io/api/badge/repositories/4964" alt="loro-dev%2Floro | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
</p>
<p align="center">
<a href="https://loro.dev/docs">
<b>Documentation</b>
</a>
|
<a href="https://loro.dev/docs/tutorial/get_started">
<b>Getting Started</b>
</a>
|
<a href="https://docs.rs/loro">
<b>Rust Doc</b>
</a>
</p>
<p align="center">
<a aria-label="X" href="https://x.com/loro_dev" target="_blank">
<img alt="" src="https://img.shields.io/badge/Twitter-black?style=for-the-badge&logo=Twitter">
</a>
<a aria-label="Discord-Link" href="https://discord.gg/tUsBSVfqzf" target="_blank">
<img alt="" src="https://img.shields.io/badge/Discord-black?style=for-the-badge&logo=discord">
</a>
</p>
Loro WASM is a WASM package of Loro. Learn more at https://loro.dev
https://github.com/loro-dev/loro/assets/18425020/fe246c47-a120-44b3-91d4-1e7232a5b4ac
> ⚠️ **Notice**: The current API and encoding schema of Loro are **experimental** and **subject to change**. You should not use it in production.
Loro is a [CRDTs(Conflict-free Replicated Data Types)](https://crdt.tech/) library that makes building [local-first apps][local-first] easier. It is currently available for JavaScript (via WASM) and Rust developers.
Explore our vision in our blog: [**✨ Reimagine State Management with CRDTs**](https://loro.dev/blog/loro-now-open-source).
# Features
**Basic Features Provided by CRDTs**
- P2P Synchronization
- Automatic Merging
- Local Availability
- Scalability
- Delta Updates
**Supported CRDT Algorithms**
- 📝 Text Editing with [Fugue]
- 📙 [Peritext-like Rich Text CRDT](https://loro.dev/blog/loro-richtext)
- 🌲 [Moveable Tree](https://loro.dev/docs/tutorial/tree)
- 🚗 [Moveable List](https://loro.dev/docs/tutorial/list)
- 🗺️ [Last-Write-Wins Map](https://loro.dev/docs/tutorial/map)
- 🔄 [Replayable Event Graph](https://loro.dev/docs/advanced/replayable_event_graph)
**Advanced Features in Loro**
- 📖 Preserve Editing History in a [Replayable Event Graph](https://loro.dev/docs/advanced/replayable_event_graph)
- ⏱️ Fast [Time Travel](https://loro.dev/docs/tutorial/time_travel) Through History
https://github.com/loro-dev/loro/assets/18425020/ec2d20a3-3d8c-4483-a601-b200243c9792
# Example
[![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/edit/loro-basic-test?file=test%2Floro-sync.test.ts)
```ts
import { expect, test } from 'vitest';
import { Loro, LoroList } from 'loro-crdt';
/**
* Demonstrates synchronization of two documents with two rounds of exchanges.
*/
// Initialize document A
const docA = new Loro();
const listA: LoroList = docA.getList('list');
listA.insert(0, 'A');
listA.insert(1, 'B');
listA.insert(2, 'C');
// Export the state of document A as a byte array
const bytes: Uint8Array = docA.exportFrom();
// Simulate sending `bytes` across the network to another peer, B
const docB = new Loro();
// Peer B imports the updates from A
docB.import(bytes);
// Verify that B's state matches A's state
expect(docB.toJSON()).toStrictEqual({
list: ['A', 'B', 'C'],
});
// Get the current operation log version of document B
const version = docB.oplogVersion();
// Simulate editing at B: delete item 'B'
const listB: LoroList = docB.getList('list');
listB.delete(1, 1);
// Export the updates from B since the last synchronization point
const bytesB: Uint8Array = docB.exportFrom(version);
// Simulate sending `bytesB` back across the network to A
// A imports the updates from B
docA.import(bytesB);
// Verify that the list at A now matches the list at B after merging
expect(docA.toJSON()).toStrictEqual({
list: ['A', 'C'],
});
```
# Credits
Loro draws inspiration from the innovative work of the following projects and individuals:
- [Ink & Switch](https://inkandswitch.com/): The principles of Local-first Software have greatly influenced this project. The [Peritext](https://www.inkandswitch.com/peritext/) project has also shaped our approach to rich text CRDTs.
- [Diamond-types](https://github.com/josephg/diamond-types): The [Replayable Event Graph (REG)](https://loro.dev/docs/advanced/replayable_event_graph) algorithm from @josephg has been adapted to reduce the computation and space usage of CRDTs.
- [Automerge](https://github.com/automerge/automerge): Their use of columnar encoding for CRDTs has informed our strategies for efficient data encoding.
- [Yjs](https://github.com/yjs/yjs): We have incorporated a similar algorithm for effectively merging collaborative editing operations, thanks to their pioneering works.
- [Matthew Weidner](https://mattweidner.com/): His work on the [Fugue](https://arxiv.org/abs/2305.00583) algorithm has been invaluable, enhancing our text editing capabilities.
- [Martin Kleppmann](https://martin.kleppmann.com/): His work on CRDTs has significantly influenced our comprehension of the field.
[local-first]: https://www.inkandswitch.com/local-first/
[Fugue]: https://arxiv.org/abs/2305.00583
[Peritext]: https://www.inkandswitch.com/peritext/

6
crates/loro-wasm/package.json
View file

@ -1,6 +1,6 @@
{
"name": "loro-wasm",
"version": "0.16.4",
"version": "0.16.6",
"description": "Loro CRDTs is a high-performance CRDT framework that makes your app state synchronized, collaborative and maintainable effortlessly.",
"keywords": [
"crdt",
@ -10,6 +10,10 @@
"sync",
"p2p"
],
"repository": {
"type": "git",
"url": "git+https://github.com/loro-dev/loro.git"
},
"main": "nodejs/loro_wasm.js",
"module": "bundler/loro_wasm.js",
"scripts": {

2
crates/loro-wasm/scripts/build.ts
View file

@ -59,7 +59,7 @@ async function build() {
async function cargoBuild() {
const cmd =
`cargo build --features counter --target wasm32-unknown-unknown --profile ${profile}`;
`cargo build --target wasm32-unknown-unknown --profile ${profile}`;
console.log(cmd);
const status = await Deno.run({
cmd: cmd.split(" "),

9
crates/loro-wasm/src/convert.rs
View file

@ -9,15 +9,12 @@ use loro_internal::{ListDiffItem, LoroDoc, LoroValue};
use wasm_bindgen::JsValue;
use crate::{
frontiers_to_ids, Container, Cursor, JsContainer, JsImportBlobMetadata, LoroList, LoroMap,
LoroMovableList, LoroText, LoroTree,
frontiers_to_ids, Container, Cursor, JsContainer, JsImportBlobMetadata, LoroCounter, LoroList,
LoroMap, LoroMovableList, LoroText, LoroTree,
};
use wasm_bindgen::__rt::IntoJsResult;
use wasm_bindgen::convert::RefFromWasmAbi;
#[cfg(feature = "counter")]
use crate::LoroCounter;
/// Convert a `JsValue` to `T` by constructor's name.
///
/// more details can be found in https://github.com/rustwasm/wasm-bindgen/issues/2231#issuecomment-656293288
@ -137,7 +134,6 @@ pub(crate) fn resolved_diff_to_js(value: &Diff, doc: &Arc<LoroDoc>) -> JsValue {
.unwrap();
}
#[cfg(feature = "counter")]
Diff::Counter(v) => {
js_sys::Reflect::set(
&obj,
@ -345,7 +341,6 @@ pub(crate) fn handler_to_js_value(handler: Handler, doc: Option<Arc<LoroDoc>>) -
Handler::List(l) => LoroList { handler: l, doc }.into(),
Handler::Tree(t) => LoroTree { handler: t, doc }.into(),
Handler::MovableList(m) => LoroMovableList { handler: m, doc }.into(),
#[cfg(feature = "counter")]
Handler::Counter(c) => LoroCounter { handler: c, doc }.into(),
Handler::Unknown(_) => unreachable!(),
}

146
crates/loro-wasm/src/lib.rs
View file

@ -29,9 +29,7 @@ use std::{cell::RefCell, cmp::Ordering, rc::Rc, sync::Arc};
use wasm_bindgen::{__rt::IntoJsResult, prelude::*, throw_val};
use wasm_bindgen_derive::TryFromJsValue;
#[cfg(feature = "counter")]
mod counter;
#[cfg(feature = "counter")]
pub use counter::LoroCounter;
mod awareness;
mod log;
@ -314,7 +312,7 @@ impl Loro {
/// If enabled, the Unix timestamp will be recorded for each change automatically.
///
/// You can also set each timestamp manually when you commit a change.
/// The timstamp manually set will override the automatic one.
/// The timestamp manually set will override the automatic one.
///
/// NOTE: Timestamps are forced to be in ascending order.
/// If you commit a new change with a timestamp that is less than the existing one,
@ -481,6 +479,30 @@ impl Loro {
self.0.is_detached()
}
/// Detach the document state from the latest known version.
///
/// After detaching, all import operations will be recorded in the `OpLog` without being applied to the `DocState`.
/// When `detached`, the document is not editable.
///
/// @example
/// ```ts
/// import { Loro } from "loro-crdt";
///
/// const doc = new Loro();
/// doc.detach();
/// console.log(doc.is_detached()); // true
/// ```
pub fn detach(&self) {
self.0.detach()
}
/// Duplicate the document with a different PeerID
///
/// The time complexity and space complexity of this operation are both O(n),
pub fn fork(&self) -> Self {
Self(Arc::new(self.0.fork()))
}
/// Checkout the `DocState` to the latest version of `OpLog`.
///
/// > The document becomes detached during a `checkout` operation.
@ -670,7 +692,6 @@ impl Loro {
}
/// Get a LoroCounter by container id
#[cfg(feature = "counter")]
#[wasm_bindgen(js_name = "getCounter")]
pub fn get_counter(&self, cid: &JsIntoContainerID) -> JsResult<LoroCounter> {
let counter = self
@ -763,7 +784,6 @@ impl Loro {
}
.into()
}
#[cfg(feature = "counter")]
ContainerType::Counter => {
let counter = self.0.get_counter(container_id);
LoroCounter {
@ -1418,8 +1438,37 @@ fn convert_container_path_to_js_value(path: &[(ContainerID, Index)]) -> JsValue
path
}
/// The handler of a text or richtext container.
/// The handler of a text container. It supports rich text CRDT.
///
/// ## Updating Text Content Using a Diff Algorithm
///
/// A common requirement is to update the current text to a target text.
/// You can implement this using a text diff algorithm of your choice.
/// Below is a sample you can directly copy into your code, which uses the
/// [fast-diff](https://www.npmjs.com/package/fast-diff) package.
///
/// ```ts
/// import { diff } from "fast-diff";
/// import { LoroText } from "loro-crdt";
///
/// function updateText(text: LoroText, newText: string) {
/// const src = text.toString();
/// const delta = diff(src, newText);
/// let index = 0;
/// for (const [op, text] of delta) {
/// if (op === 0) {
/// index += text.length;
/// } else if (op === 1) {
/// text.insert(index, text);
/// index += text.length;
/// } else {
/// text.delete(index, text.length);
/// }
/// }
/// ```
///
///
/// Learn more at https://loro.dev/docs/tutorial/text
#[derive(Clone)]
#[wasm_bindgen]
pub struct LoroText {
@ -1469,6 +1518,22 @@ impl LoroText {
Ok(())
}
/// Insert some string at utf-8 index.
///
/// @example
/// ```ts
/// import { Loro } from "loro-crdt";
///
/// const doc = new Loro();
/// const text = doc.getText("text");
/// text.insertUtf8(0, "Hello");
/// ```
#[wasm_bindgen(js_name = "insertUtf8")]
pub fn insert_utf8(&mut self, index: usize, content: &str) -> JsResult<()> {
self.handler.insert_utf8(index, content)?;
Ok(())
}
/// Delete elements from index to index + len
///
/// @example
@ -1487,6 +1552,25 @@ impl LoroText {
Ok(())
}
/// Delete elements from index to utf-8 index + len
///
/// @example
/// ```ts
/// import { Loro } from "loro-crdt";
///
/// const doc = new Loro();
/// const text = doc.getText("text");
/// text.insertUtf8(0, "Hello");
/// text.deleteUtf8(1, 3);
/// const s = text.toString();
/// console.log(s); // "Ho"
/// ```
#[wasm_bindgen(js_name = "deleteUtf8")]
pub fn delete_utf8(&mut self, index: usize, len: usize) -> JsResult<()> {
self.handler.delete_utf8(index, len)?;
Ok(())
}
/// Mark a range of text with a key and a value.
///
/// > You should call `configTextStyle` before using `mark` and `unmark`.
@ -1662,7 +1746,7 @@ impl LoroText {
}
}
/// Whether the container is attached to a docuemnt.
/// Whether the container is attached to a document.
///
/// If it's detached, the operations on the container will not be persisted.
#[wasm_bindgen(js_name = "isAttached")]
@ -1708,6 +1792,8 @@ impl Default for LoroText {
}
/// The handler of a map container.
///
/// Learn more at https://loro.dev/docs/tutorial/map
#[derive(Clone)]
#[wasm_bindgen]
pub struct LoroMap {
@ -2017,7 +2103,7 @@ impl LoroMap {
}
}
/// Whether the container is attached to a docuemnt.
/// Whether the container is attached to a document.
///
/// If it's detached, the operations on the container will not be persisted.
#[wasm_bindgen(js_name = "isAttached")]
@ -2049,6 +2135,8 @@ impl Default for LoroMap {
}
/// The handler of a list container.
///
/// Learn more at https://loro.dev/docs/tutorial/list
#[derive(Clone)]
#[wasm_bindgen]
pub struct LoroList {
@ -2303,7 +2391,7 @@ impl LoroList {
}
}
/// Whether the container is attached to a docuemnt.
/// Whether the container is attached to a document.
///
/// If it's detached, the operations on the container will not be persisted.
#[wasm_bindgen(js_name = "isAttached")]
@ -2368,6 +2456,8 @@ impl Default for LoroList {
}
/// The handler of a list container.
///
/// Learn more at https://loro.dev/docs/tutorial/list
#[derive(Clone)]
#[wasm_bindgen]
pub struct LoroMovableList {
@ -2628,7 +2718,7 @@ impl LoroMovableList {
}
}
/// Whether the container is attached to a docuemnt.
/// Whether the container is attached to a document.
///
/// If it's detached, the operations on the container will not be persisted.
#[wasm_bindgen(js_name = "isAttached")]
@ -2725,6 +2815,8 @@ impl LoroMovableList {
}
/// The handler of a tree(forest) container.
///
/// Learn more at https://loro.dev/docs/tutorial/tree
#[derive(Clone)]
#[wasm_bindgen]
pub struct LoroTree {
@ -2939,13 +3031,15 @@ impl LoroTreeNode {
/// The objects returned are new js objects each time because they need to cross
/// the WASM boundary.
#[wasm_bindgen(skip_typescript)]
pub fn children(&self) -> Array {
let children = self.tree.children(Some(self.id));
pub fn children(&self) -> JsValue {
let Some(children) = self.tree.children(Some(self.id)) else {
return JsValue::undefined();
};
let children = children.into_iter().map(|c| {
let node = LoroTreeNode::from_tree(c, self.tree.clone(), self.doc.clone());
JsValue::from(node)
});
Array::from_iter(children)
Array::from_iter(children).into()
}
}
@ -3092,7 +3186,7 @@ impl LoroTree {
/// but also the metadata, you should use `toJson()`.
///
// TODO: perf
#[wasm_bindgen(js_name = "toArray")]
#[wasm_bindgen(js_name = "toArray", skip_typescript)]
pub fn to_array(&mut self) -> JsResult<Array> {
let value = self.handler.get_value().into_list().unwrap();
let ans = Array::new();
@ -3112,13 +3206,17 @@ impl LoroTree {
.unwrap_or(JsValue::undefined())
.into();
let index = *v["index"].as_i64().unwrap() as u32;
let position = v["position"].as_string().unwrap();
let position = v["fractional_index"].as_string().unwrap();
let map: LoroMap = self.get_node_by_id(&id).unwrap().data()?;
let obj = Object::new();
js_sys::Reflect::set(&obj, &"id".into(), &id)?;
js_sys::Reflect::set(&obj, &"parent".into(), &parent)?;
js_sys::Reflect::set(&obj, &"index".into(), &JsValue::from(index))?;
js_sys::Reflect::set(&obj, &"position".into(), &JsValue::from_str(position))?;
js_sys::Reflect::set(
&obj,
&"fractional_index".into(),
&JsValue::from_str(position),
)?;
js_sys::Reflect::set(&obj, &"meta".into(), &map.into())?;
ans.push(&obj);
}
@ -3256,7 +3354,7 @@ impl LoroTree {
}
}
/// Whether the container is attached to a docuemnt.
/// Whether the container is attached to a document.
///
/// If it's detached, the operations on the container will not be persisted.
#[wasm_bindgen(js_name = "isAttached")]
@ -3335,7 +3433,7 @@ impl Cursor {
/// Get the ID that represents the position.
///
/// It can be undefined if it's not binded into a specific ID.
/// It can be undefined if it's not bind into a specific ID.
pub fn pos(&self) -> Option<JsID> {
match self.pos.id {
Some(id) => {
@ -4052,6 +4150,18 @@ interface LoroList {
getCursor(pos: number, side?: Side): Cursor | undefined;
}
export type TreeNodeValue = {
id: TreeID,
parent: TreeID | undefined,
index: number,
fractionalIndex: string,
meta: LoroMap,
}
interface LoroTree{
toArray(): TreeNodeValue[];
}
interface LoroMovableList {
/**
* Get the cursor position at the given pos.

27
crates/loro/src/lib.rs
View file

@ -11,9 +11,7 @@ use loro_internal::cursor::Side;
use loro_internal::encoding::ImportBlobMetadata;
use loro_internal::handler::HandlerTrait;
use loro_internal::handler::ValueOrHandler;
use loro_internal::loro::CommitOptions;
use loro_internal::undo::{OnPop, OnPush};
use loro_internal::JsonSchema;
use loro_internal::LoroDoc as InnerLoroDoc;
use loro_internal::OpLog;
@ -40,10 +38,13 @@ pub use loro_internal::delta::{TreeDeltaItem, TreeDiff, TreeExternalDiff};
pub use loro_internal::event::Index;
pub use loro_internal::handler::TextDelta;
pub use loro_internal::id::{PeerID, TreeID, ID};
pub use loro_internal::loro::CommitOptions;
pub use loro_internal::obs::SubID;
pub use loro_internal::oplog::FrontiersNotIncluded;
pub use loro_internal::undo;
pub use loro_internal::version::{Frontiers, VersionVector};
pub use loro_internal::ApplyDiff;
pub use loro_internal::JsonSchema;
pub use loro_internal::UndoManager as InnerUndoManager;
pub use loro_internal::{loro_value, to_value};
pub use loro_internal::{LoroError, LoroResult, LoroValue, ToJson};
@ -76,6 +77,14 @@ impl LoroDoc {
LoroDoc { doc }
}
/// Duplicate the document with a different PeerID
///
/// The time complexity and space complexity of this operation are both O(n),
pub fn fork(&self) -> Self {
let doc = self.doc.fork();
LoroDoc { doc }
}
/// Get the configureations of the document.
pub fn config(&self) -> &Configure {
self.doc.config()
@ -974,11 +983,21 @@ impl LoroText {
self.handler.insert(pos, s)
}
/// Insert a string at the given utf-8 position.
pub fn insert_utf8(&self, pos: usize, s: &str) -> LoroResult<()> {
self.handler.insert_utf8(pos, s)
}
/// Delete a range of text at the given unicode position with unicode length.
pub fn delete(&self, pos: usize, len: usize) -> LoroResult<()> {
self.handler.delete(pos, len)
}
/// Delete a range of text at the given utf-8 position with utf-8 length.
pub fn delete_utf8(&self, pos: usize, len: usize) -> LoroResult<()> {
self.handler.delete_utf8(pos, len)
}
/// Whether the text container is empty.
pub fn is_empty(&self) -> bool {
self.handler.is_empty()
@ -1368,7 +1387,9 @@ impl LoroTree {
}
/// Return all children of the target node.
pub fn children(&self, parent: Option<TreeID>) -> Vec<TreeID> {
///
/// If the parent node does not exist, return `None`.
pub fn children(&self, parent: Option<TreeID>) -> Option<Vec<TreeID>> {
self.handler.children(parent)
}

3
crates/loro/tests/integration_test/undo_test.rs
View file

@ -1209,7 +1209,8 @@ fn undo_tree_concurrent_delete2() -> LoroResult<()> {
.get("id")
.unwrap()
.to_json_value(),
json!("1@1")
// create a new node
json!("1@2")
);
Ok(())
}

35
crates/loro/tests/loro_rust_test.rs
View file

@ -1,4 +1,7 @@
use std::{cmp::Ordering, sync::Arc};
use std::{
cmp::Ordering,
sync::{atomic::AtomicBool, Arc},
};
use loro::{
awareness::Awareness, FrontiersNotIncluded, LoroDoc, LoroError, LoroList, LoroMap, LoroText,
@ -42,6 +45,34 @@ fn insert_an_inserted_movable_handler() -> Result<(), LoroError> {
Ok(())
}
#[test]
fn fork_doc() -> anyhow::Result<()> {
let doc0 = LoroDoc::new();
let text = doc0.get_text("123");
text.insert(0, "123")?;
let triggered = Arc::new(AtomicBool::new(false));
let trigger_cloned = triggered.clone();
doc0.commit();
doc0.subscribe_root(Arc::new(move |e| {
for e in e.events {
let _t = e.diff.as_text().unwrap();
triggered.store(true, std::sync::atomic::Ordering::Release);
}
}));
let doc1 = doc0.fork();
let text1 = doc1.get_text("123");
assert_eq!(&text1.to_string(), "123");
text1.insert(3, "456")?;
assert_eq!(&text.to_string(), "123");
assert_eq!(&text1.to_string(), "123456");
assert!(!trigger_cloned.load(std::sync::atomic::Ordering::Acquire),);
doc0.import(&doc1.export_from(&Default::default()))?;
assert!(trigger_cloned.load(std::sync::atomic::Ordering::Acquire),);
assert_eq!(text.to_string(), text1.to_string());
assert_ne!(doc0.peer_id(), doc1.peer_id());
Ok(())
}
#[test]
fn movable_list() -> Result<(), LoroError> {
let doc = LoroDoc::new();
@ -371,7 +402,7 @@ fn tree() {
root_meta.insert("color", "red").unwrap();
assert_eq!(
tree.get_value_with_meta().to_json(),
r#"[{"parent":null,"meta":{"color":"red"},"id":"0@1","index":0,"position":"80"},{"parent":"0@1","meta":{},"id":"1@1","index":0,"position":"80"}]"#
r#"[{"parent":null,"meta":{"color":"red"},"id":"0@1","index":0,"fractional_index":"80"},{"parent":"0@1","meta":{},"id":"1@1","index":0,"fractional_index":"80"}]"#
)
}

16
loro-js/CHANGELOG.md
View file

@ -1,5 +1,21 @@
# Changelog
## 0.16.6
### Patch Changes
- 1e94248: Add `.fork()` to duplicate the doc
- Updated dependencies [1e94248]
- loro-wasm@0.16.6
## 0.16.5
### Patch Changes
- 439e4e9: Update pkg desc
- Updated dependencies [439e4e9]
- loro-wasm@0.16.5
## 0.16.4
### Patch Changes

141
loro-js/README.md
View file

@ -1,8 +1,141 @@
# loro-crdt
<p align="center">
<a href="https://loro.dev">
<picture>
<img src="./docs/Loro.svg" width="200"/>
</picture>
</a>
</p>
<h1 align="center">
<a href="https://loro.dev" alt="loro-site">Loro</a>
</h1>
<p align="center">
<b>Reimagine state management with CRDTs 🦜</b><br/>
Make your app state synchronized and collaborative effortlessly.
</p>
<p align="center">
<a href="https://trendshift.io/repositories/4964" target="_blank"><img src="https://trendshift.io/api/badge/repositories/4964" alt="loro-dev%2Floro | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
</p>
<p align="center">
<a href="https://loro.dev/docs">
<b>Documentation</b>
</a>
|
<a href="https://loro.dev/docs/tutorial/get_started">
<b>Getting Started</b>
</a>
|
<a href="https://docs.rs/loro">
<b>Rust Doc</b>
</a>
</p>
<p align="center">
<a aria-label="X" href="https://x.com/loro_dev" target="_blank">
<img alt="" src="https://img.shields.io/badge/Twitter-black?style=for-the-badge&logo=Twitter">
</a>
<a aria-label="Discord-Link" href="https://discord.gg/tUsBSVfqzf" target="_blank">
<img alt="" src="https://img.shields.io/badge/Discord-black?style=for-the-badge&logo=discord">
</a>
</p>
Loro CRDTs is a high-performance CRDT framework.
It makes your app state synchronized, collaborative and maintainable effortlessly.
https://github.com/loro-dev/loro/assets/18425020/fe246c47-a120-44b3-91d4-1e7232a5b4ac
Learn more at https://loro.dev
> ⚠️ **Notice**: The current API and encoding schema of Loro are **experimental** and **subject to change**. You should not use it in production.
Loro is a [CRDTs(Conflict-free Replicated Data Types)](https://crdt.tech/) library that makes building [local-first apps][local-first] easier. It is currently available for JavaScript (via WASM) and Rust developers.
Explore our vision in our blog: [**✨ Reimagine State Management with CRDTs**](https://loro.dev/blog/loro-now-open-source).
# Features
**Basic Features Provided by CRDTs**
- P2P Synchronization
- Automatic Merging
- Local Availability
- Scalability
- Delta Updates
**Supported CRDT Algorithms**
- 📝 Text Editing with [Fugue]
- 📙 [Peritext-like Rich Text CRDT](https://loro.dev/blog/loro-richtext)
- 🌲 [Moveable Tree](https://loro.dev/docs/tutorial/tree)
- 🚗 [Moveable List](https://loro.dev/docs/tutorial/list)
- 🗺️ [Last-Write-Wins Map](https://loro.dev/docs/tutorial/map)
- 🔄 [Replayable Event Graph](https://loro.dev/docs/advanced/replayable_event_graph)
**Advanced Features in Loro**
- 📖 Preserve Editing History in a [Replayable Event Graph](https://loro.dev/docs/advanced/replayable_event_graph)
- ⏱️ Fast [Time Travel](https://loro.dev/docs/tutorial/time_travel) Through History
https://github.com/loro-dev/loro/assets/18425020/ec2d20a3-3d8c-4483-a601-b200243c9792
# Example
[![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/edit/loro-basic-test?file=test%2Floro-sync.test.ts)
```ts
import { expect, test } from 'vitest';
import { Loro, LoroList } from 'loro-crdt';
/**
* Demonstrates synchronization of two documents with two rounds of exchanges.
*/
// Initialize document A
const docA = new Loro();
const listA: LoroList = docA.getList('list');
listA.insert(0, 'A');
listA.insert(1, 'B');
listA.insert(2, 'C');
// Export the state of document A as a byte array
const bytes: Uint8Array = docA.exportFrom();
// Simulate sending `bytes` across the network to another peer, B
const docB = new Loro();
// Peer B imports the updates from A
docB.import(bytes);
// Verify that B's state matches A's state
expect(docB.toJSON()).toStrictEqual({
list: ['A', 'B', 'C'],
});
// Get the current operation log version of document B
const version = docB.oplogVersion();
// Simulate editing at B: delete item 'B'
const listB: LoroList = docB.getList('list');
listB.delete(1, 1);
// Export the updates from B since the last synchronization point
const bytesB: Uint8Array = docB.exportFrom(version);
// Simulate sending `bytesB` back across the network to A
// A imports the updates from B
docA.import(bytesB);
// Verify that the list at A now matches the list at B after merging
expect(docA.toJSON()).toStrictEqual({
list: ['A', 'C'],
});
```
# Credits
Loro draws inspiration from the innovative work of the following projects and individuals:
- [Ink & Switch](https://inkandswitch.com/): The principles of Local-first Software have greatly influenced this project. The [Peritext](https://www.inkandswitch.com/peritext/) project has also shaped our approach to rich text CRDTs.
- [Diamond-types](https://github.com/josephg/diamond-types): The [Replayable Event Graph (REG)](https://loro.dev/docs/advanced/replayable_event_graph) algorithm from @josephg has been adapted to reduce the computation and space usage of CRDTs.
- [Automerge](https://github.com/automerge/automerge): Their use of columnar encoding for CRDTs has informed our strategies for efficient data encoding.
- [Yjs](https://github.com/yjs/yjs): We have incorporated a similar algorithm for effectively merging collaborative editing operations, thanks to their pioneering works.
- [Matthew Weidner](https://mattweidner.com/): His work on the [Fugue](https://arxiv.org/abs/2305.00583) algorithm has been invaluable, enhancing our text editing capabilities.
- [Martin Kleppmann](https://martin.kleppmann.com/): His work on CRDTs has significantly influenced our comprehension of the field.
[local-first]: https://www.inkandswitch.com/local-first/
[Fugue]: https://arxiv.org/abs/2305.00583
[Peritext]: https://www.inkandswitch.com/peritext/

6
loro-js/package.json
View file

@ -1,6 +1,6 @@
{
"name": "loro-crdt",
"version": "0.16.4",
"version": "0.16.6",
"description": "Loro CRDTs is a high-performance CRDT framework that makes your app state synchronized, collaborative and maintainable effortlessly.",
"keywords": [
"crdt",
@ -10,6 +10,10 @@
"sync",
"p2p"
],
"repository": {
"type": "git",
"url": "git+https://github.com/loro-dev/loro.git"
},
"main": "dist/loro.js",
"module": "dist/loro.mjs",
"typings": "dist/loro.d.ts",

2
loro-js/src/index.ts
View file

@ -638,7 +638,7 @@ declare module "loro-wasm" {
* The objects returned are new js objects each time because they need to cross
* the WASM boundary.
*/
children(): Array<LoroTreeNode<T>>;
children(): Array<LoroTreeNode<T>> | undefined;
}
interface AwarenessWasm<T extends Value = Value> {

15
loro-js/tests/basic.test.ts
View file

@ -461,3 +461,18 @@ it("get elem by path", () => {
map1.set("key1", 1);
expect(doc.getByPath("map/key1")).toBe(1);
});
it("fork", () => {
const doc = new Loro();
const map = doc.getMap("map");
map.set("key", 1);
const doc2 = doc.fork();
const map2 = doc2.getMap("map");
expect(map2.get("key")).toBe(1);
expect(doc2.toJSON()).toStrictEqual({ map: { key: 1 } });
map2.set("key", 2);
expect(doc.toJSON()).toStrictEqual({ map: { key: 1 } });
expect(doc2.toJSON()).toStrictEqual({ map: { key: 2 } });
doc.import(doc2.exportSnapshot());
expect(doc.toJSON()).toStrictEqual({ map: { key: 2 } });
});

26
loro-js/tests/compatibility.test.ts
View file

@ -23,9 +23,10 @@ describe("compatibility", () => {
docA.getMap("map").set("key", "123");
docA.getList("list").insert(0, 1);
docA.getList("list").insert(0, "1");
const t = docA.getTree("tree");
const node = t.createNode();
t.createNode(node.id, 0);
// TODO: rename
// const t = docA.getTree("tree");
// const node = t.createNode();
// t.createNode(node.id, 0);
const bytes = docA.exportFrom();
const docB = new OLD.Loro();
@ -40,9 +41,9 @@ describe("compatibility", () => {
docA.getMap("map").set("key", "123");
docA.getList("list").insert(0, 1);
docA.getList("list").insert(0, "1");
const t = docA.getTree("tree");
const node = t.createNode();
t.createNode(node.id, 0);
// const t = docA.getTree("tree");
// const node = t.createNode();
// t.createNode(node.id, 0);
const bytes = docA.exportSnapshot();
const docB = new OLD.Loro();
@ -57,9 +58,9 @@ describe("compatibility", () => {
docA.getMap("map").set("key", "123");
docA.getList("list").insert(0, 1);
docA.getList("list").insert(0, "1");
const t = docA.getTree("tree");
const node = t.createNode();
t.createNode(node.id);
// const t = docA.getTree("tree");
// const node = t.createNode();
// t.createNode(node.id);
const bytes = docA.exportSnapshot();
const docB = new Loro();
@ -74,9 +75,10 @@ describe("compatibility", () => {
docA.getMap("map").set("key", "123");
docA.getList("list").insert(0, 1);
docA.getList("list").insert(0, "1");
const t = docA.getTree("tree");
const node = t.createNode();
t.createNode(node.id);
// const t = docA.getTree("tree");
// const node = t.createNode();
// t.createNode(node.id);
const bytes = docA.exportSnapshot();
const docB = new Loro();

16
loro-js/tests/richtext.test.ts
View file

@ -286,4 +286,20 @@ describe("richtext", () => {
const text = doc.getText("text");
text.insert(0, `“aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`);
});
it("Insert/delete by utf8 index", () => {
const doc = new Loro();
const text = doc.getText('t');
text.insert(0, "你好");
text.insertUtf8(3, "a");
text.insertUtf8(7, "b");
expect(text.toDelta()).toStrictEqual([
{ insert: "你a好b" },
]);
text.deleteUtf8(3, 4);
expect(text.toDelta()).toStrictEqual([
{ insert: "你b"},
]);
});
});

18
loro-js/tests/tree.test.ts
View file

@ -1,4 +1,4 @@
import { describe, expect, it} from "vitest";
import { assert, describe, expect, it} from "vitest";
import { Loro, LoroTree, LoroTreeNode } from "../src";
function assertEquals(a: any, b: any) {
@ -34,7 +34,7 @@ describe("loro tree", () => {
assertEquals(child2.parent()!.id, root.id);
tree.move(child2.id, child.id);
assertEquals(child2.parent()!.id, child.id);
assertEquals(child.children()[0].id, child2.id);
assertEquals(child.children()![0].id, child2.id);
expect(()=>tree.move(child2.id, child.id, 1)).toThrowError();
});
@ -70,9 +70,9 @@ describe("loro tree", () => {
const root = tree.createNode();
const child = tree.createNode(root.id);
const child2 = tree.createNode(root.id);
assertEquals(root.children().length, 2);
assertEquals(root.children()[0].id, child.id);
assertEquals(root.children()[1].id, child2.id);
assertEquals(root.children()!.length, 2);
assertEquals(root.children()![0].id, child.id);
assertEquals(root.children()![1].id, child2.id);
});
it("toArray", ()=>{
@ -83,6 +83,12 @@ describe("loro tree", () => {
tree2.createNode(root.id);
const arr = tree2.toArray();
assertEquals(arr.length, 3);
const keys = Object.keys(arr[0]);
assert(keys.includes("id"));
assert(keys.includes("parent"));
assert(keys.includes("index"));
assert(keys.includes("fractional_index"));
assert(keys.includes("meta"));
});
it("subscribe", async () => {
@ -141,7 +147,7 @@ describe("loro tree node", ()=>{
assertEquals(child2.parent()!.id, root.id);
child2.move(child);
assertEquals(child2.parent()!.id, child.id);
assertEquals(child.children()[0].id, child2.id);
assertEquals(child.children()![0].id, child2.id);
expect(()=>child2.move(child, 1)).toThrowError();
});