From d3a0d10b126a684b8e3d2719f0cbc70c19faee42 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Nov 2022 18:31:57 +0800 Subject: [PATCH 01/13] feat: encode updates --- Cargo.lock | 78 +-------- crates/loro-core/Cargo.toml | 7 +- crates/loro-core/src/container.rs | 2 +- .../loro-core/src/container/list/list_op.rs | 5 +- .../src/container/map/map_content.rs | 3 +- crates/loro-core/src/container/registry.rs | 2 +- .../src/container/text/text_content.rs | 5 +- .../loro-core/src/container/text/tracker.rs | 3 +- crates/loro-core/src/dag.rs | 2 - crates/loro-core/src/event.rs | 3 +- crates/loro-core/src/log_store.rs | 18 +- .../loro-core/src/log_store/encode_updates.rs | 162 ++++++++++++++++++ crates/loro-core/src/log_store/encoding.rs | 4 +- crates/loro-core/src/loro.rs | 10 +- crates/loro-core/src/op/content.rs | 9 +- crates/loro-core/src/smstring.rs | 18 ++ crates/loro-core/tests/test.rs | 1 - crates/rle/src/rle_vec.rs | 25 +++ crates/rle/src/rle_vec_old.rs | 15 +- 19 files changed, 246 insertions(+), 126 deletions(-) create mode 100644 crates/loro-core/src/log_store/encode_updates.rs diff --git a/Cargo.lock b/Cargo.lock index 4ba8d76e..d838a35b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,15 +157,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - [[package]] name = "block-buffer" version = "0.10.3" @@ -276,17 +267,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "colored" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" -dependencies = [ - "atty", - "lazy_static", - "winapi", -] - [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -671,20 +651,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "im" -version = "15.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" -dependencies = [ - "bitmaps", - "rand_core", - "rand_xoshiro", - "sized-chunks", - "typenum", - "version_check", -] - [[package]] name = "indexmap" version = "1.9.1" @@ -766,9 +732,7 @@ dependencies = [ "arbitrary", "arbtest", "arref", - "bit-vec", "color-backtrace", - "colored", "crdt-list", "criterion", "ctor", @@ -777,11 +741,10 @@ dependencies = [ "enum-as-inner", "flate2", "fxhash", - "im", "js-sys", "num", "owning_ref", - "pin-project", + "postcard", "proptest", "proptest-derive", "rand", @@ -1055,26 +1018,6 @@ dependencies = [ "siphasher", ] -[[package]] -name = "pin-project" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" -dependencies = [ - "proc-macro2 1.0.47", - "quote 1.0.21", - "syn 1.0.103", -] - [[package]] name = "plotters" version = "0.3.4" @@ -1268,15 +1211,6 @@ dependencies = [ "rand_core", ] -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core", -] - [[package]] name = "rayon" version = "1.5.3" @@ -1561,16 +1495,6 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" -[[package]] -name = "sized-chunks" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" -dependencies = [ - "bitmaps", - "typenum", -] - [[package]] name = "smallvec" version = "1.10.0" diff --git a/crates/loro-core/Cargo.toml b/crates/loro-core/Cargo.toml index 6e4108cd..3c9cda54 100644 --- a/crates/loro-core/Cargo.toml +++ b/crates/loro-core/Cargo.toml @@ -9,22 +9,19 @@ edition = "2021" string_cache = "0.8.3" rle = { path = "../rle" } smallvec = "1.8.0" -smartstring = "1.0.1" +smartstring = { version = "1.0.1" } fxhash = "0.2.1" ring = "0.16.20" -pin-project = "1.0.10" serde = { version = "1.0.140", features = ["derive"] } thiserror = "1.0.31" -im = "15.1.0" enum-as-inner = "0.5.1" num = "0.4.0" crdt-list = { version = "0.3.0" } owning_ref = "0.4.1" +postcard = "1.0.2" rand = { version = "0.8.5", optional = true } arbitrary = { version = "1.1.7", optional = true } tabled = { version = "0.10.0", optional = true } -colored = "2.0.0" -bit-vec = "0.6.3" wasm-bindgen = { version = "0.2.83", optional = true } serde-wasm-bindgen = { version = "0.4.5", optional = true } js-sys = { version = "0.3.60", optional = true } diff --git a/crates/loro-core/src/container.rs b/crates/loro-core/src/container.rs index 995102c3..b6d373d5 100644 --- a/crates/loro-core/src/container.rs +++ b/crates/loro-core/src/container.rs @@ -10,7 +10,7 @@ use crate::{ log_store::ImportContext, op::{InnerContent, RemoteContent, RichOp}, version::{IdSpanVector, VersionVector}, - InternalString, LogStore, LoroValue, ID, + InternalString, LoroValue, ID, }; use serde::{Deserialize, Serialize}; diff --git a/crates/loro-core/src/container/list/list_op.rs b/crates/loro-core/src/container/list/list_op.rs index e2201fe0..52ab32e2 100644 --- a/crates/loro-core/src/container/list/list_op.rs +++ b/crates/loro-core/src/container/list/list_op.rs @@ -2,10 +2,11 @@ use std::ops::Range; use enum_as_inner::EnumAsInner; use rle::{HasLength, Mergable, Sliceable}; +use serde::{Deserialize, Serialize}; use crate::container::text::text_content::{ListSlice, SliceRange}; -#[derive(EnumAsInner, Debug, Clone)] +#[derive(EnumAsInner, Debug, Clone, Serialize, Deserialize)] pub enum ListOp { Insert { slice: ListSlice, pos: usize }, Delete(DeleteSpan), @@ -23,7 +24,7 @@ pub enum InnerListOp { /// len cannot be zero; /// /// pos: 5, len: -3 eq a range of (2, 5] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct DeleteSpan { pub pos: isize, pub len: isize, diff --git a/crates/loro-core/src/container/map/map_content.rs b/crates/loro-core/src/container/map/map_content.rs index 9e67bfaa..e237a044 100644 --- a/crates/loro-core/src/container/map/map_content.rs +++ b/crates/loro-core/src/container/map/map_content.rs @@ -1,8 +1,9 @@ use rle::{HasLength, Mergable, Sliceable}; +use serde::{Deserialize, Serialize}; use crate::{ContentType, InsertContentTrait, InternalString, LoroValue}; -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct MapSet { pub(crate) key: InternalString, pub(crate) value: LoroValue, diff --git a/crates/loro-core/src/container/registry.rs b/crates/loro-core/src/container/registry.rs index b98b394f..b19ea7e5 100644 --- a/crates/loro-core/src/container/registry.rs +++ b/crates/loro-core/src/container/registry.rs @@ -17,7 +17,7 @@ use crate::{ log_store::ImportContext, op::{RemoteContent, RichOp}, version::IdSpanVector, - LogStore, LoroError, LoroValue, + LoroError, LoroValue, }; use super::{ diff --git a/crates/loro-core/src/container/text/text_content.rs b/crates/loro-core/src/container/text/text_content.rs index 6e52ebbf..cd81ec07 100644 --- a/crates/loro-core/src/container/text/text_content.rs +++ b/crates/loro-core/src/container/text/text_content.rs @@ -2,10 +2,11 @@ use std::ops::Range; use enum_as_inner::EnumAsInner; use rle::{HasLength, Mergable, Sliceable}; +use serde::{Deserialize, Serialize}; use crate::{smstring::SmString, LoroValue}; -#[derive(PartialEq, Debug, EnumAsInner, Clone)] +#[derive(PartialEq, Debug, EnumAsInner, Clone, Serialize, Deserialize)] pub enum ListSlice { // TODO: use Box<[LoroValue]> ? RawData(Vec), @@ -101,7 +102,7 @@ impl HasLength for ListSlice { impl Sliceable for ListSlice { fn slice(&self, from: usize, to: usize) -> Self { match self { - ListSlice::RawStr(s) => ListSlice::RawStr(s.0[from..to].into()), + ListSlice::RawStr(s) => ListSlice::RawStr(s[from..to].into()), ListSlice::Unknown(_) => ListSlice::Unknown(to - from), ListSlice::RawData(x) => ListSlice::RawData(x[from..to].to_vec()), } diff --git a/crates/loro-core/src/container/text/tracker.rs b/crates/loro-core/src/container/text/tracker.rs index f85e6ef5..db8970e5 100644 --- a/crates/loro-core/src/container/text/tracker.rs +++ b/crates/loro-core/src/container/text/tracker.rs @@ -1,4 +1,3 @@ -use colored::Colorize; use debug_log::debug_log; use rle::{rle_tree::UnsafeCursor, HasLength, Sliceable}; use smallvec::SmallVec; @@ -326,7 +325,7 @@ impl Tracker { let mut spans = self .content .get_active_id_spans(span.start() as usize, span.atom_len()); - debug_log!("DELETED SPANS={}", format!("{:#?}", &spans).red()); + debug_log!("DELETED SPANS={}", format!("{:#?}", &spans)); self.update_spans(&spans, StatusChange::Delete); if span.is_reversed() && span.atom_len() > 1 { diff --git a/crates/loro-core/src/dag.rs b/crates/loro-core/src/dag.rs index 8e476431..eed49f7c 100644 --- a/crates/loro-core/src/dag.rs +++ b/crates/loro-core/src/dag.rs @@ -12,8 +12,6 @@ use std::{ fmt::Debug, }; -#[allow(unused)] -use colored::Colorize; use fxhash::{FxHashMap, FxHashSet}; use rle::{HasLength, Sliceable}; use smallvec::{smallvec, SmallVec}; diff --git a/crates/loro-core/src/event.rs b/crates/loro-core/src/event.rs index 35e61489..d8e51955 100644 --- a/crates/loro-core/src/event.rs +++ b/crates/loro-core/src/event.rs @@ -1,5 +1,6 @@ use enum_as_inner::EnumAsInner; use fxhash::{FxHashMap, FxHashSet}; +use serde::{Deserialize, Serialize}; use crate::{container::ContainerID, delta::Delta, version::Frontiers, InternalString, LoroValue}; @@ -27,7 +28,7 @@ pub struct Event { pub type Path = Vec; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum Index { Key(InternalString), Seq(usize), diff --git a/crates/loro-core/src/log_store.rs b/crates/loro-core/src/log_store.rs index 9289a19a..8d1b7442 100644 --- a/crates/loro-core/src/log_store.rs +++ b/crates/loro-core/src/log_store.rs @@ -1,9 +1,11 @@ //! [LogStore] stores all the [Change]s and [Op]s. It's also a [DAG][crate::dag]; //! //! +mod encode_updates; mod encoding; mod import; mod iter; + use crate::LoroValue; pub(crate) use import::ImportContext; use std::{ @@ -13,7 +15,6 @@ use std::{ use fxhash::FxHashMap; -use crate::context::Context; use rle::{HasLength, RleVec, RleVecWithIndex, Sliceable}; use smallvec::SmallVec; @@ -51,7 +52,7 @@ impl Default for GcConfig { } type ClientChanges = FxHashMap>; -type RemoteClientChanges = FxHashMap, ChangeMergeCfg>>; +type RemoteClientChanges = FxHashMap>>; #[derive(Debug)] /// LogStore stores the full history of Loro @@ -101,21 +102,14 @@ impl LogStore { .map(|changes| changes.get(id.counter as usize).unwrap().element) } - pub fn export( - &self, - remote_vv: &VersionVector, - ) -> FxHashMap, ChangeMergeCfg>> { - let mut ans: FxHashMap, ChangeMergeCfg>> = - Default::default(); + pub fn export(&self, remote_vv: &VersionVector) -> FxHashMap>> { + let mut ans: FxHashMap>> = Default::default(); let self_vv = self.vv(); let diff = self_vv.diff(remote_vv); for span in diff.left.iter() { let changes = self.get_changes_slice(span.id_span()); for change in changes.iter() { - let vec = ans - .entry(change.id.client_id) - .or_insert_with(|| RleVecWithIndex::new_cfg(self.get_change_merge_cfg())); - + let vec = ans.entry(change.id.client_id).or_insert_with(|| Vec::new()); vec.push(self.change_to_export_format(change)); } } diff --git a/crates/loro-core/src/log_store/encode_updates.rs b/crates/loro-core/src/log_store/encode_updates.rs new file mode 100644 index 00000000..1f7ffe4f --- /dev/null +++ b/crates/loro-core/src/log_store/encode_updates.rs @@ -0,0 +1,162 @@ +use rle::{HasLength, RleVec}; +use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; + +use crate::{ + change::{Change, Lamport, Timestamp}, + container::ContainerID, + id::{ClientID, Counter, ID}, + op::{RemoteContent, RemoteOp}, + LogStore, VersionVector, +}; + +use super::RemoteClientChanges; + +#[derive(Serialize, Deserialize)] +struct Updates { + changes: Vec, +} + +/// the continuous changes from the same client +#[derive(Serialize, Deserialize)] +struct EncodedClientChanges { + meta: FirstChangeInfo, + data: Vec, +} + +#[derive(Serialize, Deserialize)] +struct FirstChangeInfo { + pub(crate) client: ClientID, + pub(crate) counter: Counter, + pub(crate) lamport: Lamport, + pub(crate) timestamp: Timestamp, +} + +#[derive(Serialize, Deserialize)] +struct EncodedOp { + pub(crate) container: ContainerID, + pub(crate) contents: Vec, +} + +#[derive(Serialize, Deserialize)] +struct EncodedChange { + pub(crate) ops: Vec, + pub(crate) deps_except_self: Vec, + pub(crate) lamport_delta: u32, + pub(crate) timestamp_delta: i64, +} + +impl LogStore { + pub fn encode_updates(&self, from: &VersionVector) -> Result, postcard::Error> { + let changes = self.export(from); + let mut updates = Updates { + changes: Vec::with_capacity(changes.len()), + }; + for (_, changes) in changes { + let encoded = convert_changes_to_encoded(changes.into_iter()); + updates.changes.push(encoded); + } + + postcard::to_allocvec(&updates) + } + + pub fn decode_updates(&mut self, input: &[u8]) -> Result<(), postcard::Error> { + let updates: Updates = postcard::from_bytes(input)?; + let mut changes: RemoteClientChanges = Default::default(); + for encoded in updates.changes { + changes.insert(encoded.meta.client, convert_encoded_to_changes(encoded)); + } + + self.import(changes); + Ok(()) + } +} + +fn convert_changes_to_encoded(mut changes: I) -> EncodedClientChanges +where + I: Iterator>, +{ + let first_change = changes.next().unwrap(); + let this_client_id = first_change.id.client_id; + let mut data = Vec::with_capacity(changes.size_hint().0 + 1); + let mut last_change = first_change.clone(); + for change in changes { + data.push(EncodedChange { + ops: change + .ops + .iter() + .map(|op| EncodedOp { + container: op.container.clone(), + contents: op.contents.iter().cloned().collect(), + }) + .collect(), + deps_except_self: change + .deps + .iter() + .filter(|x| x.client_id != this_client_id) + .copied() + .collect(), + lamport_delta: change.lamport - last_change.lamport, + timestamp_delta: change.timestamp - last_change.timestamp, + }); + last_change = change; + } + + EncodedClientChanges { + meta: FirstChangeInfo { + client: this_client_id, + counter: first_change.id.counter, + lamport: first_change.lamport, + timestamp: first_change.timestamp, + }, + data, + } +} + +fn convert_encoded_to_changes(changes: EncodedClientChanges) -> Vec> { + let mut result = Vec::with_capacity(changes.data.len() + 1); + let mut last_lamport = changes.meta.lamport; + let mut last_timestamp = changes.meta.timestamp; + let mut counter: Counter = changes.meta.counter; + for encoded in changes.data { + let start_counter = counter; + let mut deps = SmallVec::with_capacity(encoded.deps_except_self.len() + 1); + if counter > 0 { + deps.push(ID { + client_id: changes.meta.client, + counter: changes.meta.counter - 1, + }); + } + + for dep in encoded.deps_except_self { + deps.push(dep); + } + + let mut ops = RleVec::with_capacity(encoded.ops.len()); + for op in encoded.ops { + let len: usize = op.contents.iter().map(|x| x.atom_len()).sum(); + ops.push(RemoteOp { + counter, + container: op.container, + contents: op.contents.into_iter().collect(), + }); + counter += len as Counter; + } + + let change = Change { + id: ID { + client_id: changes.meta.client, + counter: start_counter, + }, + lamport: last_lamport + encoded.lamport_delta, + timestamp: last_timestamp + encoded.timestamp_delta, + ops, + deps, + }; + last_lamport = change.lamport; + last_timestamp = change.timestamp; + result.push(change); + } + + result +} diff --git a/crates/loro-core/src/log_store/encoding.rs b/crates/loro-core/src/log_store/encoding.rs index e7a06683..4c45c8cc 100644 --- a/crates/loro-core/src/log_store/encoding.rs +++ b/crates/loro-core/src/log_store/encoding.rs @@ -19,7 +19,6 @@ use crate::{ dag::remove_included_frontiers, id::{ClientID, Counter, ID}, op::{Op, RemoteContent, RemoteOp}, - smstring::SmString, span::{HasIdSpan, HasLamportSpan}, ContainerType, InternalString, LogStore, LoroValue, VersionVector, }; @@ -154,7 +153,6 @@ fn encode_changes(store: &LogStore) -> Encoded { (span.pos as usize, 0, LoroValue::I32(span.len as i32)) } }, - crate::op::RemoteContent::Dyn(_) => unreachable!(), }; op_len += 1; ops.push(OpEncoding { @@ -271,7 +269,7 @@ fn decode_changes( }, _ => { let slice = match value { - LoroValue::String(s) => ListSlice::RawStr(SmString::from(&*s)), + LoroValue::String(s) => ListSlice::RawStr(s.into()), LoroValue::List(v) => ListSlice::RawData(*v), _ => unreachable!(), }; diff --git a/crates/loro-core/src/loro.rs b/crates/loro-core/src/loro.rs index 4c9e2309..a0e78360 100644 --- a/crates/loro-core/src/loro.rs +++ b/crates/loro-core/src/loro.rs @@ -72,18 +72,12 @@ impl LoroCore { Text::from_instance(instance, cid) } - pub fn export( - &self, - remote_vv: VersionVector, - ) -> FxHashMap, ChangeMergeCfg>> { + pub fn export(&self, remote_vv: VersionVector) -> FxHashMap>> { let store = self.log_store.read().unwrap(); store.export(&remote_vv) } - pub fn import( - &mut self, - changes: FxHashMap, ChangeMergeCfg>>, - ) { + pub fn import(&mut self, changes: FxHashMap>>) { let mut store = self.log_store.write().unwrap(); store.import(changes) } diff --git a/crates/loro-core/src/op/content.rs b/crates/loro-core/src/op/content.rs index 0c09a554..44d5bf35 100644 --- a/crates/loro-core/src/op/content.rs +++ b/crates/loro-core/src/op/content.rs @@ -2,6 +2,7 @@ use std::any::{Any, TypeId}; use enum_as_inner::EnumAsInner; use rle::{HasLength, Mergable, Sliceable}; +use serde::{Deserialize, Serialize}; use crate::container::{ list::list_op::{InnerListOp, ListOp}, @@ -24,11 +25,10 @@ pub enum InnerContent { Map(InnerMapSet), } -#[derive(EnumAsInner, Debug)] +#[derive(EnumAsInner, Debug, Serialize, Deserialize)] pub enum RemoteContent { Map(MapSet), List(ListOp), - Dyn(Box), } impl Clone for RemoteContent { @@ -36,7 +36,6 @@ impl Clone for RemoteContent { match self { Self::Map(arg0) => Self::Map(arg0.clone()), Self::List(arg0) => Self::List(arg0.clone()), - Self::Dyn(arg0) => Self::Dyn(arg0.clone_content()), } } } @@ -93,7 +92,6 @@ impl HasLength for RemoteContent { fn content_len(&self) -> usize { match self { RemoteContent::Map(x) => x.content_len(), - RemoteContent::Dyn(x) => x.content_len(), RemoteContent::List(x) => x.content_len(), } } @@ -103,7 +101,6 @@ impl Sliceable for RemoteContent { fn slice(&self, from: usize, to: usize) -> Self { match self { RemoteContent::Map(x) => RemoteContent::Map(x.slice(from, to)), - RemoteContent::Dyn(x) => RemoteContent::Dyn(x.slice_content(from, to)), RemoteContent::List(x) => RemoteContent::List(x.slice(from, to)), } } @@ -117,7 +114,6 @@ impl Mergable for RemoteContent { match (self, other) { (RemoteContent::Map(x), RemoteContent::Map(y)) => x.is_mergable(y, &()), (RemoteContent::List(x), RemoteContent::List(y)) => x.is_mergable(y, &()), - (RemoteContent::Dyn(x), RemoteContent::Dyn(y)) => x.is_mergable_content(&**y), _ => false, } } @@ -135,7 +131,6 @@ impl Mergable for RemoteContent { RemoteContent::List(y) => x.merge(y, &()), _ => unreachable!(), }, - RemoteContent::Dyn(x) => x.merge_content(&**_other.as_dyn().unwrap()), } } } diff --git a/crates/loro-core/src/smstring.rs b/crates/loro-core/src/smstring.rs index efbffba6..e42a5106 100644 --- a/crates/loro-core/src/smstring.rs +++ b/crates/loro-core/src/smstring.rs @@ -3,6 +3,7 @@ use std::ops::DerefMut; use std::ops::Deref; use rle::Mergable; +use serde::Deserialize; use serde::Serialize; use smartstring::LazyCompact; @@ -63,6 +64,13 @@ impl From for SmString { } } +impl From> for SmString { + fn from(s: Box) -> Self { + let s: &str = &s; + SmString(s.into()) + } +} + impl From<&str> for SmString { fn from(s: &str) -> Self { SmString(s.into()) @@ -77,3 +85,13 @@ impl Serialize for SmString { serializer.serialize_str(&self.0) } } + +impl<'de> Deserialize<'de> for SmString { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Ok(s.into()) + } +} diff --git a/crates/loro-core/tests/test.rs b/crates/loro-core/tests/test.rs index 99621ac6..dece3bda 100644 --- a/crates/loro-core/tests/test.rs +++ b/crates/loro-core/tests/test.rs @@ -1,4 +1,3 @@ -use std::borrow::{Borrow, BorrowMut}; use std::cell::RefCell; use std::rc::Rc; diff --git a/crates/rle/src/rle_vec.rs b/crates/rle/src/rle_vec.rs index 22f53670..89550472 100644 --- a/crates/rle/src/rle_vec.rs +++ b/crates/rle/src/rle_vec.rs @@ -555,6 +555,31 @@ impl Deref for RleVecWithLen { } } +impl Sliceable for Vec { + fn slice(&self, start: usize, end: usize) -> Self { + if start >= end || self.is_empty() { + return Vec::new(); + } + + let mut ans = Vec::new(); + let mut index = 0; + for item in self { + if index >= end { + break; + } + + let len = item.atom_len(); + if start < index + len { + ans.push(item.slice(start.saturating_sub(index), (end - index).min(len))) + } + + index += len; + } + + ans + } +} + #[cfg(test)] mod test { use super::*; diff --git a/crates/rle/src/rle_vec_old.rs b/crates/rle/src/rle_vec_old.rs index 05b4cc94..090c8254 100644 --- a/crates/rle/src/rle_vec_old.rs +++ b/crates/rle/src/rle_vec_old.rs @@ -1,4 +1,7 @@ -use std::ops::{Deref, Range}; +use std::{ + ops::{Deref, Range}, + vec, +}; use num::Integer; @@ -213,6 +216,16 @@ impl RleVecWithIndex { } } +impl IntoIterator for RleVecWithIndex { + type Item = T; + + type IntoIter = vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.vec.into_iter() + } +} + impl Default for RleVecWithIndex { fn default() -> Self { Self::new() From 94b9e95fc1c8f9059f1bc88d2f05df06233f6e2a Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Nov 2022 18:35:48 +0800 Subject: [PATCH 02/13] chore: cargo fix --- crates/loro-core/src/container/text/text_container.rs | 2 +- crates/loro-core/src/lib.rs | 1 - crates/loro-core/src/log_store.rs | 2 +- crates/loro-core/src/loro.rs | 4 ++-- crates/loro-core/src/value.rs | 1 - 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/loro-core/src/container/text/text_container.rs b/crates/loro-core/src/container/text/text_container.rs index b2b34c5e..5b24079c 100644 --- a/crates/loro-core/src/container/text/text_container.rs +++ b/crates/loro-core/src/container/text/text_container.rs @@ -336,7 +336,7 @@ impl Container for TextContainer { &mut self, _: &mut Hierarchy, rich_op: &RichOp, - import_context: &mut ImportContext, + _import_context: &mut ImportContext, ) { self.tracker.track_apply(rich_op); } diff --git a/crates/loro-core/src/lib.rs b/crates/loro-core/src/lib.rs index 22e5c03c..bddd5bea 100644 --- a/crates/loro-core/src/lib.rs +++ b/crates/loro-core/src/lib.rs @@ -40,7 +40,6 @@ pub(crate) use op::{ContentType, InsertContentTrait, Op}; // TODO: rename as Key? pub(crate) type InternalString = DefaultAtom; pub(crate) use container::Container; -pub(crate) use log_store::ImportContext; pub use container::{list::List, map::Map, text::Text, ContainerType}; pub use log_store::LogStore; diff --git a/crates/loro-core/src/log_store.rs b/crates/loro-core/src/log_store.rs index 8d1b7442..1ab88fab 100644 --- a/crates/loro-core/src/log_store.rs +++ b/crates/loro-core/src/log_store.rs @@ -109,7 +109,7 @@ impl LogStore { for span in diff.left.iter() { let changes = self.get_changes_slice(span.id_span()); for change in changes.iter() { - let vec = ans.entry(change.id.client_id).or_insert_with(|| Vec::new()); + let vec = ans.entry(change.id.client_id).or_insert_with(Vec::new); vec.push(self.change_to_export_format(change)); } } diff --git a/crates/loro-core/src/loro.rs b/crates/loro-core/src/loro.rs index a0e78360..85d65c48 100644 --- a/crates/loro-core/src/loro.rs +++ b/crates/loro-core/src/loro.rs @@ -2,10 +2,10 @@ use std::sync::{Arc, RwLock}; use crate::LoroValue; use fxhash::FxHashMap; -use rle::RleVecWithIndex; + use crate::{ - change::{Change, ChangeMergeCfg}, + change::{Change}, configure::Configure, container::{list::List, map::Map, text::Text, ContainerIdRaw, ContainerType}, event::{Observer, SubscriptionID}, diff --git a/crates/loro-core/src/value.rs b/crates/loro-core/src/value.rs index d1ddc501..4039765d 100644 --- a/crates/loro-core/src/value.rs +++ b/crates/loro-core/src/value.rs @@ -6,7 +6,6 @@ use serde::{de::VariantAccess, ser::SerializeStruct, Deserialize, Serialize}; use crate::{ container::{registry::ContainerRegistry, ContainerID}, - context::Context, delta::DeltaItem, event::{Diff, Index, Path}, Container, From 45c1a2e791d5c1de0bcbe4505f552149138a0334 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Nov 2022 19:32:33 +0800 Subject: [PATCH 03/13] fix: decode deps --- crates/loro-core/fuzz/Cargo.lock | 110 +----------------- crates/loro-core/src/fuzz.rs | 62 ++++++++-- .../loro-core/src/log_store/encode_updates.rs | 47 ++++++-- crates/loro-core/src/log_store/import.rs | 5 +- crates/rle/src/lib.rs | 2 +- crates/rle/src/rle_vec.rs | 29 +++++ 6 files changed, 124 insertions(+), 131 deletions(-) diff --git a/crates/loro-core/fuzz/Cargo.lock b/crates/loro-core/fuzz/Cargo.lock index 8a6bcb9d..1f9c8955 100644 --- a/crates/loro-core/fuzz/Cargo.lock +++ b/crates/loro-core/fuzz/Cargo.lock @@ -53,17 +53,6 @@ dependencies = [ "critical-section", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -85,12 +74,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fe8f5a8a398345e52358e18ff07cc17a568fbca5c6f73873d3a62056309603" -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - [[package]] name = "bit_field" version = "0.10.1" @@ -109,15 +92,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - [[package]] name = "bumpalo" version = "3.11.1" @@ -157,17 +131,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" -[[package]] -name = "colored" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" -dependencies = [ - "atty", - "lazy_static", - "winapi", -] - [[package]] name = "cortex-m" version = "0.7.6" @@ -357,35 +320,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "ident_case" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "im" -version = "15.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" -dependencies = [ - "bitmaps", - "rand_core", - "rand_xoshiro", - "sized-chunks", - "typenum", - "version_check", -] - [[package]] name = "itertools" version = "0.10.5" @@ -467,16 +407,13 @@ version = "0.1.0" dependencies = [ "arbitrary", "arref", - "bit-vec", - "colored", "crdt-list", "debug-log", "enum-as-inner", "fxhash", - "im", "num", "owning_ref", - "pin-project", + "postcard", "rand", "ring", "rle", @@ -692,26 +629,6 @@ dependencies = [ "siphasher", ] -[[package]] -name = "pin-project" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "postcard" version = "1.0.2" @@ -807,15 +724,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core", -] - [[package]] name = "redox_syscall" version = "0.2.16" @@ -1006,16 +914,6 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" -[[package]] -name = "sized-chunks" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" -dependencies = [ - "bitmaps", - "typenum", -] - [[package]] name = "smallvec" version = "1.10.0" @@ -1135,12 +1033,6 @@ dependencies = [ "syn", ] -[[package]] -name = "typenum" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" - [[package]] name = "unicode-ident" version = "1.0.4" diff --git a/crates/loro-core/src/fuzz.rs b/crates/loro-core/src/fuzz.rs index f509602b..53a44ca4 100644 --- a/crates/loro-core/src/fuzz.rs +++ b/crates/loro-core/src/fuzz.rs @@ -217,14 +217,22 @@ fn check_eq(site_a: &mut LoroCore, site_b: &mut LoroCore) { fn check_synced(sites: &mut [LoroCore]) { for i in 0..sites.len() - 1 { for j in i + 1..sites.len() { - debug_log!("-------------------------------"); - debug_log!("checking {} with {}", i, j); - debug_log!("-------------------------------"); - + debug_log::group!("checking {} with {}", i, j); let (a, b) = array_mut_ref!(sites, [i, j]); - a.import(b.export(a.vv())); - b.import(a.export(b.vv())); - check_eq(a, b) + { + debug_log::group!("Import {}", i); + a.import_updates(&b.export_updates(&a.vv()).unwrap()) + .unwrap(); + debug_log::group_end!(); + } + { + debug_log::group!("Import {}", j); + b.import_updates(&a.export_updates(&b.vv()).unwrap()) + .unwrap(); + debug_log::group_end!(); + } + check_eq(a, b); + debug_log::group_end!(); } } } @@ -432,9 +440,10 @@ pub fn test_multi_sites(site_num: u8, actions: &mut [Action]) { sites.apply_action(action); } - debug_log!("================================="); + debug_log::group!("CheckSynced"); // println!("{}", actions.table()); check_synced(&mut sites); + debug_log::group_end!(); } #[cfg(test)] @@ -653,6 +662,43 @@ mod test { ) } + #[test] + fn simplest() { + test_multi_sites( + 2, + &mut [Ins { + content: 1, + pos: 0, + site: 0, + }], + ) + } + + #[test] + fn encode() { + test_multi_sites( + 8, + &mut [ + Ins { + content: 0, + pos: 1840611456097844714, + site: 0, + }, + Ins { + content: 0, + pos: 2825745054957034, + site: 10, + }, + Sync { from: 10, to: 0 }, + Del { + pos: 1125899890065408, + len: 1840611456097844714, + site: 0, + }, + ], + ) + } + #[test] fn case_two() { test_multi_sites( diff --git a/crates/loro-core/src/log_store/encode_updates.rs b/crates/loro-core/src/log_store/encode_updates.rs index 1f7ffe4f..a1a57f44 100644 --- a/crates/loro-core/src/log_store/encode_updates.rs +++ b/crates/loro-core/src/log_store/encode_updates.rs @@ -7,24 +7,24 @@ use crate::{ container::ContainerID, id::{ClientID, Counter, ID}, op::{RemoteContent, RemoteOp}, - LogStore, VersionVector, + LogStore, LoroCore, VersionVector, }; use super::RemoteClientChanges; -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] struct Updates { changes: Vec, } /// the continuous changes from the same client -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] struct EncodedClientChanges { meta: FirstChangeInfo, data: Vec, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] struct FirstChangeInfo { pub(crate) client: ClientID, pub(crate) counter: Counter, @@ -32,13 +32,13 @@ struct FirstChangeInfo { pub(crate) timestamp: Timestamp, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] struct EncodedOp { pub(crate) container: ContainerID, pub(crate) contents: Vec, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] struct EncodedChange { pub(crate) ops: Vec, pub(crate) deps_except_self: Vec, @@ -60,7 +60,7 @@ impl LogStore { postcard::to_allocvec(&updates) } - pub fn decode_updates(&mut self, input: &[u8]) -> Result<(), postcard::Error> { + pub fn import_updates(&mut self, input: &[u8]) -> Result<(), postcard::Error> { let updates: Updates = postcard::from_bytes(input)?; let mut changes: RemoteClientChanges = Default::default(); for encoded in updates.changes { @@ -80,6 +80,24 @@ where let this_client_id = first_change.id.client_id; let mut data = Vec::with_capacity(changes.size_hint().0 + 1); let mut last_change = first_change.clone(); + data.push(EncodedChange { + ops: first_change + .ops + .iter() + .map(|op| EncodedOp { + container: op.container.clone(), + contents: op.contents.iter().cloned().collect(), + }) + .collect(), + deps_except_self: first_change + .deps + .iter() + .filter(|x| x.client_id != this_client_id) + .copied() + .collect(), + lamport_delta: 0, + timestamp_delta: 0, + }); for change in changes { data.push(EncodedChange { ops: change @@ -121,10 +139,10 @@ fn convert_encoded_to_changes(changes: EncodedClientChanges) -> Vec 0 { + if start_counter > 0 { deps.push(ID { client_id: changes.meta.client, - counter: changes.meta.counter - 1, + counter: start_counter - 1, }); } @@ -160,3 +178,14 @@ fn convert_encoded_to_changes(changes: EncodedClientChanges) -> Vec Result, postcard::Error> { + self.log_store.read().unwrap().encode_updates(from) + } + + pub fn import_updates(&mut self, input: &[u8]) -> Result<(), postcard::Error> { + let ans = self.log_store.write().unwrap().import_updates(input); + ans + } +} diff --git a/crates/loro-core/src/log_store/import.rs b/crates/loro-core/src/log_store/import.rs index f080986b..f9a00c8b 100644 --- a/crates/loro-core/src/log_store/import.rs +++ b/crates/loro-core/src/log_store/import.rs @@ -325,10 +325,7 @@ impl LogStore { let other_start_ctr = changes.first().unwrap().ctr_start(); match other_start_ctr.cmp(&self_end_ctr) { std::cmp::Ordering::Less => { - *changes = changes.slice( - (self_end_ctr - other_start_ctr) as usize, - changes.atom_len(), - ); + *changes = changes.slice((self_end_ctr - other_start_ctr) as usize, usize::MAX); } std::cmp::Ordering::Equal => {} std::cmp::Ordering::Greater => { diff --git a/crates/rle/src/lib.rs b/crates/rle/src/lib.rs index 19a19a49..30f4ed00 100644 --- a/crates/rle/src/lib.rs +++ b/crates/rle/src/lib.rs @@ -27,7 +27,7 @@ pub mod rle_tree; mod rle_vec; mod rle_vec_old; pub use crate::rle_trait::{HasIndex, HasLength, Mergable, Rle, Slice, Sliceable, ZeroElement}; -pub use crate::rle_vec::{RleVec, RleVecWithLen}; +pub use crate::rle_vec::{slice_vec_by, RleVec, RleVecWithLen}; pub use crate::rle_vec_old::{RleVecWithIndex, SearchResult, SliceIterator}; pub mod rle_impl; pub use rle_tree::tree_trait::RleTreeTrait; diff --git a/crates/rle/src/rle_vec.rs b/crates/rle/src/rle_vec.rs index 89550472..145ed056 100644 --- a/crates/rle/src/rle_vec.rs +++ b/crates/rle/src/rle_vec.rs @@ -580,6 +580,35 @@ impl Sliceable for Vec { } } +pub fn slice_vec_by(vec: &Vec, index: F, start: usize, end: usize) -> Vec +where + F: Fn(&T) -> usize, + T: Sliceable + HasLength, +{ + if start >= end || vec.is_empty() { + return Vec::new(); + } + + let start = start - index(&vec[0]); + let end = end - index(&vec[0]); + let mut ans = Vec::new(); + let mut index = 0; + for i in 0..vec.len() { + if index >= end { + break; + } + + let len = vec[i].atom_len(); + if start < index + len { + ans.push(vec[i].slice(start.saturating_sub(index), (end - index).min(len))) + } + + index += len; + } + + ans +} + #[cfg(test)] mod test { use super::*; From 7adc399605660d8d5b7ad8030ee0a7d21e30590e Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Nov 2022 20:39:00 +0800 Subject: [PATCH 04/13] test: add recursive test for update encoding --- crates/loro-core/src/fuzz/recursive.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/loro-core/src/fuzz/recursive.rs b/crates/loro-core/src/fuzz/recursive.rs index 6dcd7864..e35fa1ae 100644 --- a/crates/loro-core/src/fuzz/recursive.rs +++ b/crates/loro-core/src/fuzz/recursive.rs @@ -612,8 +612,12 @@ fn check_synced(sites: &mut [Actor]) { let (a, b) = array_mut_ref!(sites, [i, j]); let a_doc = &mut a.loro; let b_doc = &mut b.loro; - a_doc.import(b_doc.export(a_doc.vv())); - b_doc.import(a_doc.export(b_doc.vv())); + a_doc + .import_updates(&b_doc.export_updates(&a_doc.vv()).unwrap()) + .unwrap(); + b_doc + .import_updates(&a_doc.export_updates(&b_doc.vv()).unwrap()) + .unwrap(); check_eq(a, b); debug_log::group_end!(); } From 89ae18e87c6c88308a115dd40bb1736265b2d1e8 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Nov 2022 20:48:08 +0800 Subject: [PATCH 05/13] test: add size test for update encoding --- crates/loro-core/examples/encoding.rs | 29 ++++++++++----------------- crates/loro-core/src/log_store.rs | 2 +- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/crates/loro-core/examples/encoding.rs b/crates/loro-core/examples/encoding.rs index feec667d..75193610 100644 --- a/crates/loro-core/examples/encoding.rs +++ b/crates/loro-core/examples/encoding.rs @@ -1,6 +1,9 @@ -use std::{io::Read, time::Instant}; +use std::{ + io::{Read, Write}, + time::Instant, +}; -use flate2::read::GzDecoder; +use flate2::{read::GzDecoder, write::GzEncoder}; use loro_core::{configure::Configure, container::registry::ContainerWrapper, LoroCore}; use serde_json::Value; const RAW_DATA: &[u8; 901823] = include_bytes!("../benches/automerge-paper.json.gz"); @@ -47,20 +50,10 @@ fn main() { assert_eq!(buf, buf2); let json2 = loro.to_json(); assert_eq!(json1, json2); - let mut last = 100; - let mut count = 0; - let mut max_count = 0; - for &byte in buf.iter() { - if byte == last { - count += 1; - if count > max_count { - max_count = count; - } - } else { - count = 0; - } - last = byte; - } - - println!("Longest continuous bytes length {}", max_count); + let update_buf = loro.export_updates(&Default::default()).unwrap(); + println!("Updates have {} bytes", update_buf.len()); + let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(&update_buf).unwrap(); + let data = encoder.finish().unwrap(); + println!("After compress updates have {} bytes", data.len()); } diff --git a/crates/loro-core/src/log_store.rs b/crates/loro-core/src/log_store.rs index 1ab88fab..d5818b15 100644 --- a/crates/loro-core/src/log_store.rs +++ b/crates/loro-core/src/log_store.rs @@ -45,7 +45,7 @@ pub struct GcConfig { impl Default for GcConfig { fn default() -> Self { GcConfig { - gc: true, + gc: false, snapshot_interval: 6 * MONTH, } } From e3b420e41c59c9f2ebbdb016ab32ea7f1ed3c327 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Nov 2022 23:23:19 +0800 Subject: [PATCH 06/13] chore: add tracing dep --- Cargo.lock | 39 +++++++++++++++++++++++++++++++++++++ crates/loro-core/Cargo.toml | 1 + 2 files changed, 40 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index d838a35b..4b84cc51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -760,6 +760,7 @@ dependencies = [ "string_cache", "tabled", "thiserror", + "tracing", "wasm-bindgen", ] @@ -1018,6 +1019,12 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + [[package]] name = "plotters" version = "0.3.4" @@ -1670,6 +1677,38 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2 1.0.47", + "quote 1.0.21", + "syn 1.0.103", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", +] + [[package]] name = "typenum" version = "1.15.0" diff --git a/crates/loro-core/Cargo.toml b/crates/loro-core/Cargo.toml index 3c9cda54..ba47a2a6 100644 --- a/crates/loro-core/Cargo.toml +++ b/crates/loro-core/Cargo.toml @@ -29,6 +29,7 @@ serde_json = { version = "1.0.87", optional = true } arref = "0.1.0" serde_columnar = { version = "0.1.0" } debug-log = "0.1.4" +tracing = { version = "0.1.37" } [dev-dependencies] serde_json = "1.0.87" From d718ed386fc93385d2911a45bcfb4afb2f353a32 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Nov 2022 00:23:30 +0800 Subject: [PATCH 07/13] feat: add tracing spans --- crates/loro-core/src/container/registry.rs | 10 ++++++++++ crates/loro-core/src/container/text/text_container.rs | 7 +++++++ crates/loro-core/src/log_store/encoding.rs | 3 +++ crates/loro-core/src/log_store/import.rs | 6 +++++- 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/loro-core/src/container/registry.rs b/crates/loro-core/src/container/registry.rs index b19ea7e5..2cdbf724 100644 --- a/crates/loro-core/src/container/registry.rs +++ b/crates/loro-core/src/container/registry.rs @@ -8,6 +8,7 @@ use enum_as_inner::EnumAsInner; use fxhash::FxHashMap; use owning_ref::OwningRefMut; use smallvec::SmallVec; +use tracing::instrument; use crate::{ context::Context, @@ -62,6 +63,7 @@ impl Container for ContainerInstance { } } + #[instrument(skip_all)] fn tracker_checkout(&mut self, vv: &crate::VersionVector) { match self { ContainerInstance::Map(x) => x.tracker_checkout(vv), @@ -71,6 +73,7 @@ impl Container for ContainerInstance { } } + #[instrument(skip_all)] fn get_value(&self) -> crate::LoroValue { match self { ContainerInstance::Map(x) => x.get_value(), @@ -80,6 +83,7 @@ impl Container for ContainerInstance { } } + #[instrument(skip_all)] fn update_state_directly( &mut self, hierarchy: &mut Hierarchy, @@ -94,6 +98,7 @@ impl Container for ContainerInstance { } } + #[instrument(skip_all)] fn track_retreat(&mut self, op: &IdSpanVector) { match self { ContainerInstance::Map(x) => x.track_retreat(op), @@ -103,6 +108,7 @@ impl Container for ContainerInstance { } } + #[instrument(skip_all)] fn track_forward(&mut self, op: &IdSpanVector) { match self { ContainerInstance::Map(x) => x.track_forward(op), @@ -112,6 +118,7 @@ impl Container for ContainerInstance { } } + #[instrument(skip_all)] fn track_apply(&mut self, hierarchy: &mut Hierarchy, op: &RichOp, ctx: &mut ImportContext) { match self { ContainerInstance::Map(x) => x.track_apply(hierarchy, op, ctx), @@ -121,6 +128,7 @@ impl Container for ContainerInstance { } } + #[instrument(skip_all)] fn apply_tracked_effects_from( &mut self, h: &mut Hierarchy, @@ -134,6 +142,7 @@ impl Container for ContainerInstance { } } + #[instrument(skip_all)] fn to_export( &mut self, content: crate::op::InnerContent, @@ -147,6 +156,7 @@ impl Container for ContainerInstance { } } + #[instrument(skip_all)] fn to_import(&mut self, content: crate::op::RemoteContent) -> crate::op::InnerContent { match self { ContainerInstance::Map(x) => x.to_import(content), diff --git a/crates/loro-core/src/container/text/text_container.rs b/crates/loro-core/src/container/text/text_container.rs index 5b24079c..eeff4983 100644 --- a/crates/loro-core/src/container/text/text_container.rs +++ b/crates/loro-core/src/container/text/text_container.rs @@ -5,6 +5,7 @@ use rle::{ HasLength, RleTree, }; use smallvec::SmallVec; +use tracing::instrument; use crate::{ container::{ @@ -48,6 +49,7 @@ impl TextContainer { } } + #[instrument(skip_all)] pub fn insert(&mut self, ctx: &C, pos: usize, text: &str) -> Option { if text.is_empty() { return None; @@ -88,6 +90,7 @@ impl TextContainer { Some(id) } + #[instrument(skip_all)] pub fn delete(&mut self, ctx: &C, pos: usize, len: usize) -> Option { if len == 0 { return None; @@ -273,6 +276,7 @@ impl Container for TextContainer { } } + #[instrument(skip_all)] fn update_state_directly( &mut self, hierarchy: &mut Hierarchy, @@ -321,6 +325,7 @@ impl Container for TextContainer { self.tracker.forward(spans); } + #[instrument(skip_all)] fn tracker_checkout(&mut self, vv: &crate::VersionVector) { if (!vv.is_empty() || self.tracker.start_vv().is_empty()) && self.tracker.all_vv() >= vv @@ -332,6 +337,7 @@ impl Container for TextContainer { } } + #[instrument(skip_all)] fn track_apply( &mut self, _: &mut Hierarchy, @@ -341,6 +347,7 @@ impl Container for TextContainer { self.tracker.track_apply(rich_op); } + #[instrument(skip_all)] fn apply_tracked_effects_from( &mut self, hierarchy: &mut Hierarchy, diff --git a/crates/loro-core/src/log_store/encoding.rs b/crates/loro-core/src/log_store/encoding.rs index 4c45c8cc..5a81f6a5 100644 --- a/crates/loro-core/src/log_store/encoding.rs +++ b/crates/loro-core/src/log_store/encoding.rs @@ -5,6 +5,7 @@ use rle::{HasLength, RleVec, RleVecWithIndex}; use serde::{Deserialize, Serialize}; use serde_columnar::{columnar, compress, decompress, from_bytes, to_vec, CompressConfig}; use smallvec::smallvec; +use tracing::instrument; use crate::{ change::{Change, ChangeMergeCfg, Lamport, Timestamp}, @@ -91,6 +92,7 @@ struct Encoded { keys: Vec, } +#[instrument(skip_all)] fn encode_changes(store: &LogStore) -> Encoded { let mut client_id_to_idx: FxHashMap = FxHashMap::default(); let mut clients = Vec::with_capacity(store.changes.len()); @@ -185,6 +187,7 @@ fn encode_changes(store: &LogStore) -> Encoded { } } +#[instrument(skip_all)] fn decode_changes( encoded: Encoded, client_id: Option, diff --git a/crates/loro-core/src/log_store/import.rs b/crates/loro-core/src/log_store/import.rs index f9a00c8b..8a867fda 100644 --- a/crates/loro-core/src/log_store/import.rs +++ b/crates/loro-core/src/log_store/import.rs @@ -1,10 +1,11 @@ +use crate::LogStore; use crate::{ container::registry::ContainerIdx, event::{Diff, RawEvent}, version::{Frontiers, IdSpanVector}, - LogStore, }; use std::{collections::VecDeque, ops::ControlFlow, sync::MutexGuard}; +use tracing::instrument; use fxhash::FxHashMap; @@ -72,6 +73,8 @@ impl LogStore { /// - Stage 1: we iterate over the new changes by causal order, and record them to the tracker. /// - Stage 2: we calculate the effects of the new changes, and apply them to the state. /// - Update the rest of the log store state. + /// + #[instrument(skip_all)] pub fn import(&mut self, mut changes: RemoteClientChanges) { if let ControlFlow::Break(_) = self.tailor_changes(&mut changes) { return; @@ -178,6 +181,7 @@ impl LogStore { (next_vv, next_frontiers) } + #[instrument(skip_all)] pub(crate) fn apply( &mut self, mut container_map: FxHashMap>, From 1829a9ebcaef2f36dc41db32136f638c9fb7ab59 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Sun, 4 Dec 2022 11:09:49 +0800 Subject: [PATCH 08/13] bench: fix ignore parse time in benching --- crates/loro-core/benches/text.rs | 79 ++++++++++++++------------------ 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/crates/loro-core/benches/text.rs b/crates/loro-core/benches/text.rs index 56d3e215..677602ae 100644 --- a/crates/loro-core/benches/text.rs +++ b/crates/loro-core/benches/text.rs @@ -41,11 +41,36 @@ mod run { } pub fn b4(c: &mut Criterion) { + struct Action { + pos: usize, + ins: String, + del: usize, + } + let mut actions = Vec::new(); let mut d = GzDecoder::new(&RAW_DATA[..]); let mut s = String::new(); d.read_to_string(&mut s).unwrap(); let json: Value = serde_json::from_str(&s).unwrap(); let txns = json.as_object().unwrap().get("txns"); + for txn in txns.unwrap().as_array().unwrap() { + let patches = txn + .as_object() + .unwrap() + .get("patches") + .unwrap() + .as_array() + .unwrap(); + for patch in patches { + let pos = patch[0].as_u64().unwrap() as usize; + let del_here = patch[1].as_u64().unwrap() as usize; + let ins_content = patch[2].as_str().unwrap(); + actions.push(Action { + pos, + ins: ins_content.to_string(), + del: del_here, + }); + } + } println!("{}", txns.unwrap().as_array().unwrap().len()); let mut b = c.benchmark_group("direct_apply"); b.bench_function("B4", |b| { @@ -53,21 +78,9 @@ mod run { let mut loro = LoroCore::default(); let text = loro.get_text("text"); text.with_container(|text| { - for txn in txns.unwrap().as_array().unwrap() { - let patches = txn - .as_object() - .unwrap() - .get("patches") - .unwrap() - .as_array() - .unwrap(); - for patch in patches { - let pos = patch[0].as_u64().unwrap() as usize; - let del_here = patch[1].as_u64().unwrap() as usize; - let ins_content = patch[2].as_str().unwrap(); - text.delete(&loro, pos, del_here); - text.insert(&loro, pos, ins_content); - } + for Action { pos, ins, del } in actions.iter() { + text.delete(&loro, *pos, *del); + text.insert(&loro, *pos, ins); } }) }) @@ -79,21 +92,9 @@ mod run { loro.subscribe_deep(Box::new(|_| {})); let text = loro.get_text("text"); text.with_container(|text| { - for txn in txns.unwrap().as_array().unwrap() { - let patches = txn - .as_object() - .unwrap() - .get("patches") - .unwrap() - .as_array() - .unwrap(); - for patch in patches { - let pos = patch[0].as_u64().unwrap() as usize; - let del_here = patch[1].as_u64().unwrap() as usize; - let ins_content = patch[2].as_str().unwrap(); - text.delete(&loro, pos, del_here); - text.insert(&loro, pos, ins_content); - } + for Action { pos, ins, del } in actions.iter() { + text.delete(&loro, *pos, *del); + text.insert(&loro, *pos, ins); } }) }) @@ -104,23 +105,11 @@ mod run { b.iter(|| { let mut loro = LoroCore::default(); let mut loro_b = LoroCore::default(); - for txn in txns.unwrap().as_array().unwrap() { + for Action { pos, ins, del } in actions.iter() { let text = loro.get_text("text"); text.with_container(|text| { - let patches = txn - .as_object() - .unwrap() - .get("patches") - .unwrap() - .as_array() - .unwrap(); - for patch in patches { - let pos = patch[0].as_u64().unwrap() as usize; - let del_here = patch[1].as_u64().unwrap() as usize; - let ins_content = patch[2].as_str().unwrap(); - text.delete(&loro, pos, del_here); - text.insert(&loro, pos, ins_content); - } + text.delete(&loro, *pos, *del); + text.insert(&loro, *pos, ins); }); loro_b.import(loro.export(loro_b.vv())); From 5a225e1da05cff287733480e52a0ebaaa09a50d0 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 5 Dec 2022 16:34:44 +0800 Subject: [PATCH 09/13] perf: turn on lto for release mode --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 6f6c1572..ef1402cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,3 +3,4 @@ members = ["crates/*"] [profile.release] debug = 1 +lto = true From 65465774efe6d8bcc9c67e1b5b21a9b69a1d3b6a Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 6 Dec 2022 15:40:07 +0800 Subject: [PATCH 10/13] fix: cap --- crates/loro-core/src/log_store/encode_updates.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/loro-core/src/log_store/encode_updates.rs b/crates/loro-core/src/log_store/encode_updates.rs index a1a57f44..1e49888e 100644 --- a/crates/loro-core/src/log_store/encode_updates.rs +++ b/crates/loro-core/src/log_store/encode_updates.rs @@ -132,7 +132,7 @@ where } fn convert_encoded_to_changes(changes: EncodedClientChanges) -> Vec> { - let mut result = Vec::with_capacity(changes.data.len() + 1); + let mut result = Vec::with_capacity(changes.data.len()); let mut last_lamport = changes.meta.lamport; let mut last_timestamp = changes.meta.timestamp; let mut counter: Counter = changes.meta.counter; From 134866bf3dcd22f0fb07cbff793a98da90b0f386 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 6 Dec 2022 15:40:56 +0800 Subject: [PATCH 11/13] chore: rm unused fn --- crates/rle/src/lib.rs | 2 +- crates/rle/src/rle_vec.rs | 29 ----------------------------- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/crates/rle/src/lib.rs b/crates/rle/src/lib.rs index 30f4ed00..19a19a49 100644 --- a/crates/rle/src/lib.rs +++ b/crates/rle/src/lib.rs @@ -27,7 +27,7 @@ pub mod rle_tree; mod rle_vec; mod rle_vec_old; pub use crate::rle_trait::{HasIndex, HasLength, Mergable, Rle, Slice, Sliceable, ZeroElement}; -pub use crate::rle_vec::{slice_vec_by, RleVec, RleVecWithLen}; +pub use crate::rle_vec::{RleVec, RleVecWithLen}; pub use crate::rle_vec_old::{RleVecWithIndex, SearchResult, SliceIterator}; pub mod rle_impl; pub use rle_tree::tree_trait::RleTreeTrait; diff --git a/crates/rle/src/rle_vec.rs b/crates/rle/src/rle_vec.rs index 145ed056..89550472 100644 --- a/crates/rle/src/rle_vec.rs +++ b/crates/rle/src/rle_vec.rs @@ -580,35 +580,6 @@ impl Sliceable for Vec { } } -pub fn slice_vec_by(vec: &Vec, index: F, start: usize, end: usize) -> Vec -where - F: Fn(&T) -> usize, - T: Sliceable + HasLength, -{ - if start >= end || vec.is_empty() { - return Vec::new(); - } - - let start = start - index(&vec[0]); - let end = end - index(&vec[0]); - let mut ans = Vec::new(); - let mut index = 0; - for i in 0..vec.len() { - if index >= end { - break; - } - - let len = vec[i].atom_len(); - if start < index + len { - ans.push(vec[i].slice(start.saturating_sub(index), (end - index).min(len))) - } - - index += len; - } - - ans -} - #[cfg(test)] mod test { use super::*; From 610a651b5c02b654ad2ebe28bb4524c5a2481a8e Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 6 Dec 2022 15:50:35 +0800 Subject: [PATCH 12/13] fix: vec slice is ill defined --- crates/loro-core/fuzz/Cargo.lock | 39 +++++++++++++++++++ crates/loro-core/src/log_store/import.rs | 9 ++++- crates/rle/src/lib.rs | 2 +- crates/rle/src/rle_vec.rs | 48 +++++++++++++----------- 4 files changed, 73 insertions(+), 25 deletions(-) diff --git a/crates/loro-core/fuzz/Cargo.lock b/crates/loro-core/fuzz/Cargo.lock index 1f9c8955..353e2d74 100644 --- a/crates/loro-core/fuzz/Cargo.lock +++ b/crates/loro-core/fuzz/Cargo.lock @@ -425,6 +425,7 @@ dependencies = [ "string_cache", "tabled", "thiserror", + "tracing", ] [[package]] @@ -629,6 +630,12 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + [[package]] name = "postcard" version = "1.0.2" @@ -1033,6 +1040,38 @@ dependencies = [ "syn", ] +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", +] + [[package]] name = "unicode-ident" version = "1.0.4" diff --git a/crates/loro-core/src/log_store/import.rs b/crates/loro-core/src/log_store/import.rs index 8a867fda..91701195 100644 --- a/crates/loro-core/src/log_store/import.rs +++ b/crates/loro-core/src/log_store/import.rs @@ -9,7 +9,7 @@ use tracing::instrument; use fxhash::FxHashMap; -use rle::{HasLength, RleVecWithIndex, Sliceable}; +use rle::{slice_vec_by, HasLength, RleVecWithIndex, Sliceable}; use crate::{ container::{registry::ContainerInstance, Container, ContainerID}, @@ -329,7 +329,12 @@ impl LogStore { let other_start_ctr = changes.first().unwrap().ctr_start(); match other_start_ctr.cmp(&self_end_ctr) { std::cmp::Ordering::Less => { - *changes = changes.slice((self_end_ctr - other_start_ctr) as usize, usize::MAX); + *changes = slice_vec_by( + changes, + |x| x.id.counter as usize, + self_end_ctr as usize, + usize::MAX, + ); } std::cmp::Ordering::Equal => {} std::cmp::Ordering::Greater => { diff --git a/crates/rle/src/lib.rs b/crates/rle/src/lib.rs index 19a19a49..30f4ed00 100644 --- a/crates/rle/src/lib.rs +++ b/crates/rle/src/lib.rs @@ -27,7 +27,7 @@ pub mod rle_tree; mod rle_vec; mod rle_vec_old; pub use crate::rle_trait::{HasIndex, HasLength, Mergable, Rle, Slice, Sliceable, ZeroElement}; -pub use crate::rle_vec::{RleVec, RleVecWithLen}; +pub use crate::rle_vec::{slice_vec_by, RleVec, RleVecWithLen}; pub use crate::rle_vec_old::{RleVecWithIndex, SearchResult, SliceIterator}; pub mod rle_impl; pub use rle_tree::tree_trait::RleTreeTrait; diff --git a/crates/rle/src/rle_vec.rs b/crates/rle/src/rle_vec.rs index 89550472..0884976a 100644 --- a/crates/rle/src/rle_vec.rs +++ b/crates/rle/src/rle_vec.rs @@ -555,29 +555,33 @@ impl Deref for RleVecWithLen { } } -impl Sliceable for Vec { - fn slice(&self, start: usize, end: usize) -> Self { - if start >= end || self.is_empty() { - return Vec::new(); - } - - let mut ans = Vec::new(); - let mut index = 0; - for item in self { - if index >= end { - break; - } - - let len = item.atom_len(); - if start < index + len { - ans.push(item.slice(start.saturating_sub(index), (end - index).min(len))) - } - - index += len; - } - - ans +pub fn slice_vec_by(vec: &Vec, index: F, start: usize, end: usize) -> Vec +where + F: Fn(&T) -> usize, + T: Sliceable + HasLength, +{ + if start >= end || vec.is_empty() { + return Vec::new(); } + + let start = start - index(&vec[0]); + let end = end - index(&vec[0]); + let mut ans = Vec::new(); + let mut index = 0; + for i in 0..vec.len() { + if index >= end { + break; + } + + let len = vec[i].atom_len(); + if start < index + len { + ans.push(vec[i].slice(start.saturating_sub(index), (end - index).min(len))) + } + + index += len; + } + + ans } #[cfg(test)] From ff187d967e0d19f3c9481434b6c219651aa95871 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 6 Dec 2022 15:59:14 +0800 Subject: [PATCH 13/13] chore: fix lockfile --- Cargo.lock | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d2f7fa99..4b84cc51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1025,12 +1025,6 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" -[[package]] -name = "pin-project-lite" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" - [[package]] name = "plotters" version = "0.3.4"