diff --git a/.vscode/settings.json b/.vscode/settings.json index 3a40cccf..2d28f9e7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -10,6 +10,7 @@ "LOGSTORE", "napi", "nextest", + "oplog", "peekable", "Peritext", "RUSTFLAGS", @@ -19,12 +20,10 @@ "yspan" ], "rust-analyzer.runnableEnv": { - "RUST_BACKTRACE": "full" + "RUST_BACKTRACE": "full", + "DEBUG": "*" }, - "rust-analyzer.cargo.features": [ - "loro-internal/test_utils", - "loro-internal/wasm" - ], + "rust-analyzer.cargo.features": ["test_utils"], "editor.defaultFormatter": "rust-lang.rust-analyzer", "editor.formatOnSave": true, "todo-tree.general.tags": [ diff --git a/Cargo.lock b/Cargo.lock index 6bee49ce..be128fcd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1033,9 +1033,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "oorandom" diff --git a/crates/loro-internal/Cargo.toml b/crates/loro-internal/Cargo.toml index da58ed5b..6014bbed 100644 --- a/crates/loro-internal/Cargo.toml +++ b/crates/loro-internal/Cargo.toml @@ -26,7 +26,7 @@ serde-wasm-bindgen = { version = "0.5.0", optional = true } js-sys = { version = "0.3.60", optional = true } serde_json = { version = "1.0.87", optional = true } arref = "0.1.0" -debug-log = "0.1.4" +debug-log = { version = "0.1.4", features = [] } serde_columnar = { version = "0.2.5" } tracing = { version = "0.1.37" } append-only-bytes = { version = "0.1.4", features = ["u32_range"] } diff --git a/crates/loro-internal/deno.json b/crates/loro-internal/deno.json index 2b7b1ef8..50762c63 100644 --- a/crates/loro-internal/deno.json +++ b/crates/loro-internal/deno.json @@ -2,7 +2,7 @@ "tasks": { "test": "cargo nextest run", "deny": "cargo deny check", - "fuzz": "cargo fuzz run", + "fuzz": "cargo +nightly fuzz run", "quick-fuzz": "deno run -A ./scripts/fuzz.ts text recursive encoding recursive_txn", "mem": "deno run -A ./scripts/run_mem.ts", "flame": "CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph --example automerge_x100 --root", diff --git a/crates/loro-internal/src/container.rs b/crates/loro-internal/src/container.rs index f19e9071..249c58da 100644 --- a/crates/loro-internal/src/container.rs +++ b/crates/loro-internal/src/container.rs @@ -10,7 +10,7 @@ use crate::{ log_store::ImportContext, op::{InnerContent, RemoteContent, RichOp}, version::PatchedVersionVector, - InternalString, LoroError, LoroValue, ID, + InternalString, LoroError, LoroValue, VersionVector, ID, }; use serde::{Deserialize, Serialize}; @@ -103,10 +103,10 @@ pub trait ContainerTrait: Debug + Any + Unpin + Send + Sync { fn to_import(&mut self, content: RemoteContent) -> InnerContent; /// Initialize tracker at the target version - fn tracker_init(&mut self, vv: &PatchedVersionVector); + fn tracker_init(&mut self, vv: &VersionVector); /// Tracker need to checkout to target version in order to apply the op. - fn tracker_checkout(&mut self, vv: &PatchedVersionVector); + fn tracker_checkout(&mut self, vv: &VersionVector); /// Apply the op to the tracker. /// diff --git a/crates/loro-internal/src/container/list/list_container.rs b/crates/loro-internal/src/container/list/list_container.rs index 5eb683e7..e3ab94cd 100644 --- a/crates/loro-internal/src/container/list/list_container.rs +++ b/crates/loro-internal/src/container/list/list_container.rs @@ -28,7 +28,7 @@ use crate::{ transaction::Transaction, value::LoroValue, version::PatchedVersionVector, - LoroError, Transact, + LoroError, Transact, VersionVector, }; use super::list_op::InnerListOp; @@ -379,7 +379,7 @@ impl ContainerTrait for ListContainer { } } - fn tracker_init(&mut self, vv: &PatchedVersionVector) { + fn tracker_init(&mut self, vv: &VersionVector) { match &mut self.tracker { Some(tracker) => { if (!vv.is_empty() || tracker.start_vv().is_empty()) @@ -396,7 +396,7 @@ impl ContainerTrait for ListContainer { } } - fn tracker_checkout(&mut self, vv: &PatchedVersionVector) { + fn tracker_checkout(&mut self, vv: &VersionVector) { self.tracker.as_mut().unwrap().checkout(vv) } @@ -411,10 +411,12 @@ impl ContainerTrait for ListContainer { ) { let should_notify = hierarchy.should_notify(&self.id); let mut diff = smallvec![]; - for effect in self.tracker.as_mut().unwrap().iter_effects( - import_context.patched_old_vv.as_ref().unwrap(), - &import_context.spans, - ) { + for effect in self + .tracker + .as_mut() + .unwrap() + .iter_effects(&import_context.old_vv, &import_context.spans) + { match effect { Effect::Del { pos, len } => { if should_notify { diff --git a/crates/loro-internal/src/container/map/map_container.rs b/crates/loro-internal/src/container/map/map_container.rs index 66bab336..5e5fabee 100644 --- a/crates/loro-internal/src/container/map/map_container.rs +++ b/crates/loro-internal/src/container/map/map_container.rs @@ -9,7 +9,7 @@ use crate::{ delta::{MapDiff, ValuePair}, op::OwnedRichOp, transaction::Transaction, - LoroError, Transact, + LoroError, Transact, VersionVector, }; use fxhash::FxHashMap; use smallvec::{smallvec, SmallVec}; @@ -238,9 +238,9 @@ impl ContainerTrait for MapContainer { map.into() } - fn tracker_init(&mut self, _vv: &crate::version::PatchedVersionVector) {} + fn tracker_init(&mut self, _vv: &VersionVector) {} - fn tracker_checkout(&mut self, _vv: &crate::version::PatchedVersionVector) {} + fn tracker_checkout(&mut self, _vv: &VersionVector) {} fn to_export(&mut self, content: InnerContent, _gc: bool) -> SmallVec<[RemoteContent; 1]> { if let Ok(set) = content.into_map() { diff --git a/crates/loro-internal/src/container/registry.rs b/crates/loro-internal/src/container/registry.rs index 79710636..4e984796 100644 --- a/crates/loro-internal/src/container/registry.rs +++ b/crates/loro-internal/src/container/registry.rs @@ -21,7 +21,7 @@ use crate::{ op::{RemoteContent, RichOp}, transaction::Transaction, version::PatchedVersionVector, - LoroError, LoroValue, Transact, + LoroError, LoroValue, Transact, VersionVector, }; use super::{ @@ -84,7 +84,7 @@ impl ContainerTrait for ContainerInstance { } #[instrument(skip_all)] - fn tracker_init(&mut self, vv: &PatchedVersionVector) { + fn tracker_init(&mut self, vv: &VersionVector) { match self { ContainerInstance::Map(x) => x.tracker_init(vv), ContainerInstance::Text(x) => x.tracker_init(vv), @@ -94,7 +94,7 @@ impl ContainerTrait for ContainerInstance { } #[instrument(skip_all)] - fn tracker_checkout(&mut self, vv: &PatchedVersionVector) { + fn tracker_checkout(&mut self, vv: &VersionVector) { match self { ContainerInstance::Map(x) => x.tracker_checkout(vv), ContainerInstance::Text(x) => x.tracker_checkout(vv), diff --git a/crates/loro-internal/src/container/text/text_container.rs b/crates/loro-internal/src/container/text/text_container.rs index a57537a6..d58eef38 100644 --- a/crates/loro-internal/src/container/text/text_container.rs +++ b/crates/loro-internal/src/container/text/text_container.rs @@ -1,6 +1,7 @@ -use std::sync::{Mutex, Weak}; +use std::sync::{Arc, Mutex, Weak}; use append_only_bytes::AppendOnlyBytes; +use debug_log::debug_dbg; use rle::HasLength; use smallvec::{smallvec, SmallVec}; use tracing::instrument; @@ -12,7 +13,7 @@ use crate::{ registry::{ContainerIdx, ContainerInstance, ContainerWrapper}, ContainerID, ContainerTrait, ContainerType, }, - delta::Delta, + delta::{Delta, DeltaItem}, event::{Diff, Utf16Meta}, hierarchy::Hierarchy, id::{ClientID, Counter}, @@ -21,7 +22,7 @@ use crate::{ transaction::Transaction, value::LoroValue, version::PatchedVersionVector, - LoroError, Transact, + LoroError, Transact, VersionVector, }; use super::{ @@ -369,7 +370,7 @@ impl ContainerTrait for TextContainer { } #[instrument(skip_all)] - fn tracker_init(&mut self, vv: &PatchedVersionVector) { + fn tracker_init(&mut self, vv: &VersionVector) { match &mut self.tracker { Some(tracker) => { if (!vv.is_empty() || tracker.start_vv().is_empty()) @@ -386,7 +387,7 @@ impl ContainerTrait for TextContainer { } } - fn tracker_checkout(&mut self, vv: &PatchedVersionVector) { + fn tracker_checkout(&mut self, vv: &VersionVector) { self.tracker.as_mut().unwrap().checkout(vv) } @@ -399,49 +400,96 @@ impl ContainerTrait for TextContainer { hierarchy: &mut Hierarchy, import_context: &mut ImportContext, ) { + debug_log::group!("new diff"); + // let mut state = self.get_value().as_string().unwrap().to_string(); + let delta = self + .tracker + .as_mut() + .unwrap() + .diff(&import_context.old_vv, &import_context.new_vv); + + debug_log::debug_dbg!( + &delta, + self.state.len(), + &import_context.old_vv, + &import_context.new_vv + ); let should_notify = hierarchy.should_notify(&self.id); let mut diff = smallvec![]; - for effect in self.tracker.as_mut().unwrap().iter_effects( - import_context.patched_old_vv.as_ref().unwrap(), - &import_context.spans, - ) { - match effect { - Effect::Del { pos, len } => { - if should_notify { - let utf16_pos = self.state.utf8_to_utf16_with_unknown(pos); - let utf16_end = self.state.utf8_to_utf16_with_unknown(pos + len); - let delta = Delta::new() - .retain_with_meta(pos, Utf16Meta::new(utf16_pos)) - .delete_with_meta(len, Utf16Meta::new(utf16_end - utf16_pos)); - diff.push(Diff::Text(delta)); - } - - self.state.delete_range(Some(pos), Some(pos + len)); + let mut index = 0; + for span in delta.iter() { + match span { + DeltaItem::Retain { len, .. } => { + index += len; } - Effect::Ins { pos, content } => { - // HACK: after lazifying the event, we can avoid this weird hack + DeltaItem::Insert { value: values, .. } => { + for value in values.0.iter() { + // HACK: after lazifying the event, we can avoid this weird hack + if should_notify { + let s = if value.is_unknown() { + unreachable!() + // " ".repeat(value.atom_len()) + } else { + self.raw_str.slice(&value.0).to_owned() + }; + let s_len = Utf16Meta::new(count_utf16_chars(s.as_bytes())); + let delta = Delta::new() + .retain_with_meta( + index, + Utf16Meta::new(self.state.utf8_to_utf16_with_unknown(index)), + ) + .insert_with_meta(s, s_len); + diff.push(Diff::Text(delta)); + } + + self.state.insert( + index, + PoolString::from_slice_range(&self.raw_str, value.clone()), + ); + index += value.atom_len(); + } + } + DeltaItem::Delete { len, .. } => { if should_notify { - let s = if content.is_unknown() { - " ".repeat(content.atom_len()) - } else { - self.raw_str.slice(&content.0).to_owned() - }; - let s_len = Utf16Meta::new(count_utf16_chars(s.as_bytes())); + let utf16_pos = self.state.utf8_to_utf16_with_unknown(index); + let utf16_end = self.state.utf8_to_utf16_with_unknown(index + len); let delta = Delta::new() - .retain_with_meta( - pos, - Utf16Meta::new(self.state.utf8_to_utf16_with_unknown(pos)), - ) - .insert_with_meta(s, s_len); + .retain_with_meta(index, Utf16Meta::new(utf16_pos)) + .delete_with_meta(*len, Utf16Meta::new(utf16_end - utf16_pos)); diff.push(Diff::Text(delta)); } - self.state - .insert(pos, PoolString::from_slice_range(&self.raw_str, content)); + self.state.delete_range(Some(index), Some(index + len)); } } } + debug_log::group_end!(); + // debug_log::group!("old"); + // { + // for effect in self + // .tracker + // .as_mut() + // .unwrap() + // .iter_effects(&import_context.old_vv, &import_context.spans) + // { + // debug_dbg!(&effect); + // match effect { + // Effect::Del { pos, len } => { + // state.drain(pos..pos + len); + // } + // Effect::Ins { pos, content } => { + // state.insert_str( + // pos, + // PoolString::from_slice_range(&self.raw_str, content).as_str_unchecked(), + // ); + // } + // } + // } + // } + + // debug_log::group_end!(); + // assert_eq!(&**self.get_value().as_string().unwrap(), &state); if should_notify { import_context.push_diff_vec(&self.id, diff); } diff --git a/crates/loro-internal/src/container/text/text_content.rs b/crates/loro-internal/src/container/text/text_content.rs index 22264b58..ec9dbdcb 100644 --- a/crates/loro-internal/src/container/text/text_content.rs +++ b/crates/loro-internal/src/container/text/text_content.rs @@ -3,8 +3,13 @@ use std::ops::Range; use enum_as_inner::EnumAsInner; use rle::{HasLength, Mergable, Sliceable}; use serde::{Deserialize, Serialize}; +use smallvec::{smallvec, SmallVec}; -use crate::{smstring::SmString, LoroValue}; +use crate::{ + delta::{DeltaItem, DeltaValue}, + smstring::SmString, + LoroValue, +}; use super::string_pool::PoolString; @@ -139,6 +144,44 @@ impl Mergable for ListSlice { } } +#[derive(Debug, Clone)] +pub struct SliceRanges(pub SmallVec<[SliceRange; 2]>); + +impl From for SliceRanges { + fn from(value: SliceRange) -> Self { + Self(smallvec![value]) + } +} + +impl DeltaValue for SliceRanges { + fn value_extend(&mut self, other: Self) { + self.0.extend(other.0.into_iter()); + } + + fn take(&mut self, target_len: usize) -> Self { + let mut ret = SmallVec::new(); + let mut cur_len = 0; + while cur_len < target_len { + let range = self.0.pop().unwrap(); + let range_len = range.content_len(); + if cur_len + range_len <= target_len { + ret.push(range); + cur_len += range_len; + } else { + let new_range = range.slice(0, target_len - cur_len); + ret.push(new_range); + self.0.push(range.slice(target_len - cur_len, range_len)); + cur_len = target_len; + } + } + SliceRanges(ret) + } + + fn length(&self) -> usize { + self.0.iter().fold(0, |acc, x| acc + x.atom_len()) + } +} + #[cfg(test)] mod test { use crate::LoroValue; diff --git a/crates/loro-internal/src/container/text/tracker.rs b/crates/loro-internal/src/container/text/tracker.rs index 067d87df..d323f2d3 100644 --- a/crates/loro-internal/src/container/text/tracker.rs +++ b/crates/loro-internal/src/container/text/tracker.rs @@ -1,13 +1,15 @@ -use debug_log::debug_log; +use debug_log::debug_dbg; use rle::{rle_tree::UnsafeCursor, HasLength, Sliceable}; use smallvec::SmallVec; use crate::{ container::{list::list_op::InnerListOp, text::tracker::yata_impl::YataImpl}, + delta::Delta, id::{Counter, ID}, op::{InnerContent, RichOp}, span::{HasId, HasIdSpan, IdSpan}, - version::{IdSpanVector, PatchedVersionVector}, + version::IdSpanVector, + VersionVector, }; #[allow(unused)] @@ -22,7 +24,7 @@ use self::{ pub(crate) use effects_iter::Effect; -use super::text_content::ListSlice; +use super::text_content::{ListSlice, SliceRanges}; mod content_map; mod cursor_map; mod effects_iter; @@ -44,11 +46,11 @@ pub struct Tracker { #[cfg(feature = "test_utils")] client_id: ClientID, /// from start_vv to latest vv are applied - start_vv: PatchedVersionVector, + start_vv: VersionVector, /// latest applied ops version vector - all_vv: PatchedVersionVector, + all_vv: VersionVector, /// current content version vector - current_vv: PatchedVersionVector, + current_vv: VersionVector, /// The pretend current content version vector. /// /// Because sometimes we don't actually need to checkout to the version. @@ -69,7 +71,7 @@ impl From for u128 { } impl Tracker { - pub fn new(start_vv: PatchedVersionVector, init_len: Counter) -> Self { + pub fn new(start_vv: VersionVector, init_len: Counter) -> Self { let mut content: ContentMap = Default::default(); let mut id_to_cursor: CursorMap = Default::default(); if init_len > 0 { @@ -80,6 +82,7 @@ impl Tracker { origin_right: None, id: ID::unknown(0), status: Status::new(), + after_status: None, slice: ListSlice::unknown_range(init_len as usize), }, &mut make_notify(&mut id_to_cursor), @@ -97,11 +100,11 @@ impl Tracker { } #[inline] - pub fn start_vv(&self) -> &PatchedVersionVector { + pub fn start_vv(&self) -> &VersionVector { &self.start_vv } - pub fn all_vv(&self) -> &PatchedVersionVector { + pub fn all_vv(&self) -> &VersionVector { &self.all_vv } @@ -135,18 +138,64 @@ impl Tracker { self.id_to_cursor.debug_check(); } - pub fn checkout(&mut self, vv: &PatchedVersionVector) { + pub fn checkout(&mut self, vv: &VersionVector) { + self._checkout(vv, false) + } + + /// for_diff = true should be called after the tracker checkout to A version with for_diff = false. + /// Then we can calculate the diff between A and vv. + fn _checkout(&mut self, vv: &VersionVector, for_diff: bool) { + // clear after_status as it may be outdated + if for_diff { + for mut span in self.content.iter_mut() { + span.as_mut().after_status = None; + } + } + if &self.current_vv == vv { + // we can return here even if in for_diff mode. + // because by default after_status will use the status in the current version return; } + debug_dbg!(&self.current_vv, &vv); let self_vv = std::mem::take(&mut self.current_vv); { let diff = self_vv.diff_iter(vv); - self.retreat(diff.0); - self.forward(diff.1); + self.retreat(diff.0, for_diff); + self.forward(diff.1, for_diff); } - self.current_vv = vv.clone(); + + if for_diff { + // if it's for_diff, current_version is not changed, so it should be reset to its old value + self.current_vv = self_vv; + } else { + self.current_vv = vv.clone(); + } + + debug_dbg!(&self.current_vv, &vv); + } + + pub fn diff(&mut self, from: &VersionVector, to: &VersionVector) -> Delta { + self._checkout(from, false); + self._checkout(to, true); + let mut ans = Delta::new(); + for span in self.content.iter() { + let s = span.as_ref(); + debug_dbg!(&s); + match s.status_diff() { + y_span::StatusDiff::New => { + let v: SliceRanges = s.slice.clone().into(); + ans = ans.insert(v); + } + y_span::StatusDiff::Delete => ans = ans.delete(s.slice.atom_len()), + y_span::StatusDiff::Unchanged => { + ans = ans.retain(s.content_len()); + } + } + } + + ans.chop() } pub fn track_apply(&mut self, rich_op: &RichOp) { @@ -156,14 +205,14 @@ impl Tracker { .all_vv() .includes_id(id.inc(content.atom_len() as Counter - 1)) { - self.forward(std::iter::once(id.to_span(content.atom_len()))); + self.forward(std::iter::once(id.to_span(content.atom_len())), false); return; } if self.all_vv().includes_id(id) { let this_ctr = self.all_vv().get(&id.client_id).unwrap(); let shift = this_ctr - id.counter; - self.forward(std::iter::once(id.to_span(shift as usize))); + self.forward(std::iter::once(id.to_span(shift as usize)), false); if shift as usize >= content.atom_len() { unreachable!(); } @@ -176,12 +225,15 @@ impl Tracker { } } - fn forward(&mut self, spans: impl Iterator) { + fn forward(&mut self, spans: impl Iterator, for_diff: bool) { let mut cursors = Vec::new(); let mut args = Vec::new(); for span in spans { + debug_log::group!("forward {:?}", &span); let end_id = ID::new(span.client_id, span.counter.end); - self.current_vv.set_end(end_id); + if !for_diff { + self.current_vv.set_end(end_id); + } if let Some(all_end_ctr) = self.all_vv.get(&span.client_id) { let all_end = *all_end_ctr; if all_end < span.counter.end { @@ -199,6 +251,7 @@ impl Tracker { let IdSpanQueryResult { inserts, deletes } = self.id_to_cursor.get_cursors_at_id_span( IdSpan::new(span.client_id, span.counter.start, span.counter.end), ); + debug_dbg!(&deletes); for (_, delete) in deletes { for deleted_span in delete.iter() { for span in self @@ -208,6 +261,7 @@ impl Tracker { .into_iter() .map(|x| x.1) { + debug_dbg!(&span); cursors.push(span); args.push(StatusChange::Delete); } @@ -218,24 +272,38 @@ impl Tracker { cursors.push(span); args.push(StatusChange::SetAsCurrent); } + debug_log::group_end!(); } self.content.update_at_cursors_with_args( &cursors, &args, &mut |v: &mut YSpan, arg| { - v.status.apply(*arg); + debug_dbg!(&v); + if !for_diff { + v.status.apply(*arg); + } else { + if v.after_status.is_none() { + v.after_status = Some(v.status); + } + + v.after_status.as_mut().unwrap().apply(*arg); + } + debug_dbg!(&v); }, &mut make_notify(&mut self.id_to_cursor), ) } - fn retreat(&mut self, spans: impl Iterator) { + fn retreat(&mut self, spans: impl Iterator, for_diff: bool) { let mut cursors = Vec::new(); let mut args = Vec::new(); for span in spans { + debug_dbg!("retreat", &span); let span_start = ID::new(span.client_id, span.counter.start); - self.current_vv.set_end(span_start); + if !for_diff { + self.current_vv.set_end(span_start); + } if let Some(all_end_ctr) = self.all_vv.get(&span.client_id) { let all_end = *all_end_ctr; if all_end < span.counter.start { @@ -282,7 +350,15 @@ impl Tracker { &cursors, &args, &mut |v: &mut YSpan, arg| { - v.status.apply(*arg); + if !for_diff { + v.status.apply(*arg); + } else { + if v.after_status.is_none() { + v.after_status = Some(v.status); + } + + v.after_status.as_mut().unwrap().apply(*arg); + } }, &mut make_notify(&mut self.id_to_cursor), ) @@ -309,7 +385,7 @@ impl Tracker { let mut spans = self .content .get_active_id_spans(span.start() as usize, span.atom_len()); - debug_log!("DELETED SPANS={}", format!("{:#?}", &spans)); + debug_log::debug_log!("DELETED SPANS={}", format!("{:?}", &spans)); self.update_spans(&spans, StatusChange::Delete); if span.is_reversed() && span.atom_len() > 1 { @@ -370,11 +446,7 @@ impl Tracker { ) } - pub fn iter_effects( - &mut self, - from: &PatchedVersionVector, - target: &IdSpanVector, - ) -> EffectIter<'_> { + pub fn iter_effects(&mut self, from: &VersionVector, target: &IdSpanVector) -> EffectIter<'_> { self.checkout(from); EffectIter::new(self, target) } @@ -382,4 +454,8 @@ impl Tracker { pub fn check(&mut self) { self.check_consistency(); } + + pub fn len(&self) -> usize { + self.content.len() + } } diff --git a/crates/loro-internal/src/container/text/tracker/content_map.rs b/crates/loro-internal/src/container/text/tracker/content_map.rs index eb96759c..52c6fccb 100644 --- a/crates/loro-internal/src/container/text/tracker/content_map.rs +++ b/crates/loro-internal/src/container/text/tracker/content_map.rs @@ -32,6 +32,7 @@ impl ContentMap { origin_right: right, id, status: Default::default(), + after_status: None, slice, } } diff --git a/crates/loro-internal/src/container/text/tracker/y_span.rs b/crates/loro-internal/src/container/text/tracker/y_span.rs index 0db3a8a3..28f1ca3e 100644 --- a/crates/loro-internal/src/container/text/tracker/y_span.rs +++ b/crates/loro-internal/src/container/text/tracker/y_span.rs @@ -14,22 +14,20 @@ use rle::{ const MAX_CHILDREN_SIZE: usize = 16; pub(super) type YSpanTreeTrait = CumulateTreeTrait; -/// 80 bytes #[derive(Debug, Clone, PartialEq, Eq)] pub struct YSpan { - // 16 bytes pub id: ID, - // 8 bytes + /// The status at the current version pub status: Status, - // 24 bytes + /// The status at the `after` version + /// It's used when calculating diff + pub after_status: Option, pub origin_left: Option, - // 24 bytes pub origin_right: Option, - // 8 bytes pub slice: SliceRange, } -#[derive(Debug, Clone, PartialEq, Eq, Default, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Default, Hash, Copy)] pub struct Status { /// is this span from a future operation pub future: bool, @@ -37,6 +35,12 @@ pub struct Status { pub undo_times: u16, } +pub enum StatusDiff { + New, + Delete, + Unchanged, +} + impl Display for Status { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { if self.is_activated() { @@ -126,12 +130,28 @@ impl YSpan { self.id.counter < id.ctr_end() && self.id.counter + (self.atom_len() as Counter) > id.ctr_start() } + + pub fn status_diff(&self) -> StatusDiff { + if self.after_status.is_none() { + return StatusDiff::Unchanged; + } + + match ( + self.status.is_activated(), + self.after_status.unwrap().is_activated(), + ) { + (true, false) => StatusDiff::Delete, + (false, true) => StatusDiff::New, + _ => StatusDiff::Unchanged, + } + } } impl Mergable for YSpan { fn is_mergable(&self, other: &Self, _: &()) -> bool { other.id.client_id == self.id.client_id && self.status == other.status + && self.after_status == other.after_status && self.id.counter + self.atom_len() as Counter == other.id.counter && self.origin_right == other.origin_right && Some(self.id.inc(self.atom_len() as Counter - 1)) == other.origin_left @@ -162,7 +182,8 @@ impl Sliceable for YSpan { origin_left, origin_right, id: self.id.inc(from as i32), - status: self.status.clone(), + status: self.status, + after_status: self.after_status, slice: self.slice.slice(from, to), } } diff --git a/crates/loro-internal/src/container/text/tracker/yata_impl.rs b/crates/loro-internal/src/container/text/tracker/yata_impl.rs index 81963b86..98f0f21c 100644 --- a/crates/loro-internal/src/container/text/tracker/yata_impl.rs +++ b/crates/loro-internal/src/container/text/tracker/yata_impl.rs @@ -197,6 +197,7 @@ mod test { origin_left: Some(ID::new(0, 1)), origin_right: Some(ID::new(0, 2)), status: Status::new(), + after_status: None, slice: ListSlice::unknown_range(10), }); assert!(set.contain(ID::new(1, 10))); diff --git a/crates/loro-internal/src/delta/seq.rs b/crates/loro-internal/src/delta/seq.rs index d9038eb6..8423b982 100644 --- a/crates/loro-internal/src/delta/seq.rs +++ b/crates/loro-internal/src/delta/seq.rs @@ -544,7 +544,7 @@ impl Delta { self } - fn chop(mut self) -> Self { + pub fn chop(mut self) -> Self { let last_op = self.vec.last(); if let Some(last_op) = last_op { if last_op.is_retain() && last_op.meta().is_empty() { diff --git a/crates/loro-internal/src/fuzz.rs b/crates/loro-internal/src/fuzz.rs index 0b4b78ef..208e5c62 100644 --- a/crates/loro-internal/src/fuzz.rs +++ b/crates/loro-internal/src/fuzz.rs @@ -462,6 +462,31 @@ pub fn test_multi_sites(site_num: u8, actions: &mut [Action]) { mod test { use super::Action::*; use super::*; + + #[test] + fn case2() { + test_multi_sites( + 8, + &mut [ + Ins { + content: 54005, + pos: 4846792390771214546, + site: 67, + }, + Del { + pos: 3261524511316722499, + len: 3111424388986580269, + site: 43, + }, + Ins { + content: 0, + pos: 18446548360639872768, + site: 255, + }, + ], + ) + } + #[test] fn case1() { test_multi_sites( @@ -723,194 +748,75 @@ mod test { } #[test] - fn mini() { - minify_error( - 8, - vec![ + fn case_diff() { + test_multi_sites( + 5, + &mut [ Ins { - content: 35108, + content: 65362, pos: 0, site: 2, }, - Ins { - content: 18218, - pos: 0, - site: 7, - }, - Ins { - content: 35624, - pos: 0, - site: 0, - }, - Ins { - content: 38400, - pos: 0, - site: 6, - }, - Ins { - content: 65280, - pos: 2, - site: 7, - }, - Ins { - content: 4626, - pos: 5, - site: 0, - }, - Ins { - content: 60672, - pos: 0, - site: 1, - }, - Ins { - content: 35072, - pos: 1, - site: 2, - }, - Ins { - content: 15035, - pos: 3, - site: 0, - }, - Ins { - content: 65280, - pos: 0, - site: 7, - }, - Ins { - content: 4626, - pos: 0, - site: 0, - }, - Ins { - content: 201, - pos: 2, - site: 2, - }, - Ins { - content: 65377, - pos: 3, - site: 1, - }, - Ins { - content: 9988, - pos: 0, - site: 0, - }, - Ins { - content: 4626, - pos: 14, - site: 0, - }, - Ins { - content: 4626, - pos: 11, - site: 7, - }, - Ins { - content: 1070, - pos: 0, - site: 5, - }, - Ins { - content: 27421, - pos: 7, - site: 1, - }, - Ins { - content: 65121, - pos: 22, - site: 0, - }, - Ins { - content: 65462, - pos: 1, - site: 0, - }, - Ins { - content: 4626, - pos: 0, - site: 4, - }, - Ins { - content: 4626, - pos: 16, - site: 0, - }, - Ins { - content: 65462, - pos: 11, - site: 2, - }, - Ins { - content: 48009, - pos: 10, - site: 0, - }, - Ins { - content: 23277, - pos: 7, - site: 0, - }, - Ins { - content: 60672, - pos: 13, - site: 1, - }, - Ins { - content: 4626, - pos: 2, - site: 7, - }, - Ins { - content: 4626, - pos: 2, - site: 0, - }, - Ins { - content: 2606, - pos: 0, - site: 3, - }, - Ins { - content: 65270, - pos: 10, - site: 0, - }, SyncAll, Ins { - content: 65462, - pos: 107, - site: 4, + content: 1837, + pos: 2, + site: 2, }, - SyncAll, Ins { - content: 4626, - pos: 98, - site: 0, + content: 2570, + pos: 0, + site: 2, + }, + Ins { + content: 2570, + pos: 8, + site: 2, + }, + Ins { + content: 2570, + pos: 0, + site: 1, }, - SyncAll, Ins { content: 0, + pos: 10, + site: 2, + }, + Ins { + content: 2570, + pos: 1, + site: 2, + }, + Ins { + content: 2570, + pos: 2, + site: 2, + }, + Ins { + content: 2570, pos: 0, site: 0, }, Del { - pos: 0, - len: 147, - site: 0, + pos: 4, + len: 1, + site: 3, }, - Ins { - content: 0, - pos: 146, + Del { + pos: 3, + len: 2, site: 4, }, ], - test_multi_sites, - normalize, ) } + #[test] + fn mini() { + minify_error(8, vec![], test_multi_sites, normalize) + } + #[test] fn simplify_checkout() { test_multi_sites( diff --git a/crates/loro-internal/src/fuzz/recursive_txn.rs b/crates/loro-internal/src/fuzz/recursive_txn.rs index 7eb799a0..63535570 100644 --- a/crates/loro-internal/src/fuzz/recursive_txn.rs +++ b/crates/loro-internal/src/fuzz/recursive_txn.rs @@ -3154,6 +3154,225 @@ mod failed_tests { ) } + #[test] + fn diff() { + test_multi_sites( + 5, + &mut [ + Text { + site: 1, + container_idx: 0, + pos: 0, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 6, + value: 146, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 2, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 13, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 16, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 14, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 26, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 5, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 38, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 24, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 10, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 71, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 64, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 57, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 50, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 43, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 36, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 29, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 22, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 15, + value: 37522, + is_del: false, + }, + Text { + site: 0, + container_idx: 0, + pos: 0, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 8, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 1, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 146, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 146, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 146, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 146, + value: 18, + is_del: true, + }, + Text { + site: 1, + container_idx: 0, + pos: 146, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 146, + value: 37522, + is_del: false, + }, + Text { + site: 1, + container_idx: 0, + pos: 146, + value: 18, + is_del: true, + }, + ], + ) + } + use super::ContainerType as C; #[test] fn to_minify() { diff --git a/crates/loro-internal/src/lib.rs b/crates/loro-internal/src/lib.rs index b614beaf..30d5f5a8 100644 --- a/crates/loro-internal/src/lib.rs +++ b/crates/loro-internal/src/lib.rs @@ -14,6 +14,7 @@ pub mod dag; pub mod id; pub mod log_store; pub mod op; +mod refactor; pub mod version; mod error; diff --git a/crates/loro-internal/src/log_store/encoding/encode_snapshot.rs b/crates/loro-internal/src/log_store/encoding/encode_snapshot.rs index 1ddf8ff2..74a06691 100644 --- a/crates/loro-internal/src/log_store/encoding/encode_snapshot.rs +++ b/crates/loro-internal/src/log_store/encoding/encode_snapshot.rs @@ -549,7 +549,6 @@ fn load_snapshot( spans: vv.diff(&new_store.vv).left, new_vv: vv.clone(), diff: Default::default(), - patched_old_vv: None, }; for (container_id, pool_mapping) in containers.into_iter().zip(container_states.into_iter()) { let state = pool_mapping.into_state(keys, clients); diff --git a/crates/loro-internal/src/log_store/import.rs b/crates/loro-internal/src/log_store/import.rs index 7e802b25..8170f045 100644 --- a/crates/loro-internal/src/log_store/import.rs +++ b/crates/loro-internal/src/log_store/import.rs @@ -37,7 +37,6 @@ pub struct ImportContext { // pub old_frontiers: Frontiers, pub new_frontiers: Frontiers, pub old_vv: VersionVector, - pub patched_old_vv: Option, pub new_vv: VersionVector, pub spans: IdSpanVector, pub diff: Vec<(ContainerID, SmallVec<[Diff; 1]>)>, @@ -110,7 +109,6 @@ impl LogStore { spans: next_vv.diff(&self.vv).left, new_vv: next_vv, diff: Default::default(), - patched_old_vv: None, }; hierarchy.take_deleted(); @@ -260,12 +258,6 @@ impl LogStore { let mut common_ancestors_vv = self.vv.clone(); common_ancestors_vv.retreat(&self.find_path(&common_ancestors, &self.frontiers).right); let iter_targets = context.new_vv.sub_vec(&common_ancestors_vv); - let common_ancestors_vv = Arc::new(common_ancestors_vv); - context.patched_old_vv = Some(PatchedVersionVector::from_version( - &common_ancestors_vv, - &context.old_vv, - )); - let common_ancestors_vv = PatchedVersionVector::new(common_ancestors_vv); for (_, container) in container_map.iter_mut() { container.tracker_init(&common_ancestors_vv); } diff --git a/crates/loro-internal/src/refactor/container.rs b/crates/loro-internal/src/refactor/container.rs new file mode 100644 index 00000000..ac8369e6 --- /dev/null +++ b/crates/loro-internal/src/refactor/container.rs @@ -0,0 +1,7 @@ +use crate::{event::Diff, VersionVector}; + +use super::oplog::OpLog; + +pub trait Container { + fn diff(&self, log: &OpLog, before: &VersionVector, after: &VersionVector) -> Vec; +} diff --git a/crates/loro-internal/src/refactor/mod.rs b/crates/loro-internal/src/refactor/mod.rs new file mode 100644 index 00000000..3b6df870 --- /dev/null +++ b/crates/loro-internal/src/refactor/mod.rs @@ -0,0 +1,3 @@ +mod container; +pub mod oplog; +mod state; diff --git a/crates/loro-internal/src/refactor/oplog.rs b/crates/loro-internal/src/refactor/oplog.rs new file mode 100644 index 00000000..2b09a28d --- /dev/null +++ b/crates/loro-internal/src/refactor/oplog.rs @@ -0,0 +1,4 @@ +/// This store the +pub struct OpLog {} + +pub struct Dag {} diff --git a/crates/loro-internal/src/refactor/state.rs b/crates/loro-internal/src/refactor/state.rs new file mode 100644 index 00000000..c8adbd74 --- /dev/null +++ b/crates/loro-internal/src/refactor/state.rs @@ -0,0 +1,7 @@ +mod list; +mod map; +mod text; + +pub trait ContainerState: Clone { + fn apply_diff(&mut self); +} diff --git a/crates/loro-internal/src/refactor/state/list.rs b/crates/loro-internal/src/refactor/state/list.rs new file mode 100644 index 00000000..e69de29b diff --git a/crates/loro-internal/src/refactor/state/map.rs b/crates/loro-internal/src/refactor/state/map.rs new file mode 100644 index 00000000..e69de29b diff --git a/crates/loro-internal/src/refactor/state/text.rs b/crates/loro-internal/src/refactor/state/text.rs new file mode 100644 index 00000000..e69de29b diff --git a/crates/loro-internal/src/version.rs b/crates/loro-internal/src/version.rs index 2d27d87b..592ac648 100644 --- a/crates/loro-internal/src/version.rs +++ b/crates/loro-internal/src/version.rs @@ -338,7 +338,7 @@ impl VersionVector { } else { None } - } else { + } else if counter > 0 { Some(IdSpan { client_id: *client_id, counter: CounterSpan { @@ -346,6 +346,8 @@ impl VersionVector { end: counter, }, }) + } else { + None } }) } @@ -833,17 +835,17 @@ impl PartialEq for PatchedVersionVector { if Arc::ptr_eq(&self.base, &other.base) { self.patch.eq(&other.patch) } else { - unimplemented!() + self.base == other.base && self.patch == other.patch } } } impl PartialOrd for PatchedVersionVector { fn partial_cmp(&self, other: &Self) -> Option { - if Arc::ptr_eq(&self.base, &other.base) { + if Arc::ptr_eq(&self.base, &other.base) || self.base == other.base { self.patch.partial_cmp(&other.patch) } else { - unimplemented!() + None } } }