diff --git a/Cargo.lock b/Cargo.lock index c19c6eb5..4b84cc51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,15 +157,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - [[package]] name = "block-buffer" version = "0.10.3" @@ -276,17 +267,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "colored" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" -dependencies = [ - "atty", - "lazy_static", - "winapi", -] - [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -671,20 +651,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "im" -version = "15.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" -dependencies = [ - "bitmaps", - "rand_core", - "rand_xoshiro", - "sized-chunks", - "typenum", - "version_check", -] - [[package]] name = "indexmap" version = "1.9.1" @@ -766,9 +732,7 @@ dependencies = [ "arbitrary", "arbtest", "arref", - "bit-vec", "color-backtrace", - "colored", "crdt-list", "criterion", "ctor", @@ -777,11 +741,10 @@ dependencies = [ "enum-as-inner", "flate2", "fxhash", - "im", "js-sys", "num", "owning_ref", - "pin-project", + "postcard", "proptest", "proptest-derive", "rand", @@ -1056,26 +1019,6 @@ dependencies = [ "siphasher", ] -[[package]] -name = "pin-project" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" -dependencies = [ - "proc-macro2 1.0.47", - "quote 1.0.21", - "syn 1.0.103", -] - [[package]] name = "pin-project-lite" version = "0.2.9" @@ -1275,15 +1218,6 @@ dependencies = [ "rand_core", ] -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core", -] - [[package]] name = "rayon" version = "1.5.3" @@ -1568,16 +1502,6 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" -[[package]] -name = "sized-chunks" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" -dependencies = [ - "bitmaps", - "typenum", -] - [[package]] name = "smallvec" version = "1.10.0" diff --git a/crates/loro-core/Cargo.toml b/crates/loro-core/Cargo.toml index ddf7ef35..ba47a2a6 100644 --- a/crates/loro-core/Cargo.toml +++ b/crates/loro-core/Cargo.toml @@ -9,22 +9,19 @@ edition = "2021" string_cache = "0.8.3" rle = { path = "../rle" } smallvec = "1.8.0" -smartstring = "1.0.1" +smartstring = { version = "1.0.1" } fxhash = "0.2.1" ring = "0.16.20" -pin-project = "1.0.10" serde = { version = "1.0.140", features = ["derive"] } thiserror = "1.0.31" -im = "15.1.0" enum-as-inner = "0.5.1" num = "0.4.0" crdt-list = { version = "0.3.0" } owning_ref = "0.4.1" +postcard = "1.0.2" rand = { version = "0.8.5", optional = true } arbitrary = { version = "1.1.7", optional = true } tabled = { version = "0.10.0", optional = true } -colored = "2.0.0" -bit-vec = "0.6.3" wasm-bindgen = { version = "0.2.83", optional = true } serde-wasm-bindgen = { version = "0.4.5", optional = true } js-sys = { version = "0.3.60", optional = true } diff --git a/crates/loro-core/examples/encoding.rs b/crates/loro-core/examples/encoding.rs index feec667d..75193610 100644 --- a/crates/loro-core/examples/encoding.rs +++ b/crates/loro-core/examples/encoding.rs @@ -1,6 +1,9 @@ -use std::{io::Read, time::Instant}; +use std::{ + io::{Read, Write}, + time::Instant, +}; -use flate2::read::GzDecoder; +use flate2::{read::GzDecoder, write::GzEncoder}; use loro_core::{configure::Configure, container::registry::ContainerWrapper, LoroCore}; use serde_json::Value; const RAW_DATA: &[u8; 901823] = include_bytes!("../benches/automerge-paper.json.gz"); @@ -47,20 +50,10 @@ fn main() { assert_eq!(buf, buf2); let json2 = loro.to_json(); assert_eq!(json1, json2); - let mut last = 100; - let mut count = 0; - let mut max_count = 0; - for &byte in buf.iter() { - if byte == last { - count += 1; - if count > max_count { - max_count = count; - } - } else { - count = 0; - } - last = byte; - } - - println!("Longest continuous bytes length {}", max_count); + let update_buf = loro.export_updates(&Default::default()).unwrap(); + println!("Updates have {} bytes", update_buf.len()); + let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(&update_buf).unwrap(); + let data = encoder.finish().unwrap(); + println!("After compress updates have {} bytes", data.len()); } diff --git a/crates/loro-core/fuzz/Cargo.lock b/crates/loro-core/fuzz/Cargo.lock index 8a6bcb9d..353e2d74 100644 --- a/crates/loro-core/fuzz/Cargo.lock +++ b/crates/loro-core/fuzz/Cargo.lock @@ -53,17 +53,6 @@ dependencies = [ "critical-section", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -85,12 +74,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fe8f5a8a398345e52358e18ff07cc17a568fbca5c6f73873d3a62056309603" -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - [[package]] name = "bit_field" version = "0.10.1" @@ -109,15 +92,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - [[package]] name = "bumpalo" version = "3.11.1" @@ -157,17 +131,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" -[[package]] -name = "colored" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" -dependencies = [ - "atty", - "lazy_static", - "winapi", -] - [[package]] name = "cortex-m" version = "0.7.6" @@ -357,35 +320,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "ident_case" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "im" -version = "15.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" -dependencies = [ - "bitmaps", - "rand_core", - "rand_xoshiro", - "sized-chunks", - "typenum", - "version_check", -] - [[package]] name = "itertools" version = "0.10.5" @@ -467,16 +407,13 @@ version = "0.1.0" dependencies = [ "arbitrary", "arref", - "bit-vec", - "colored", "crdt-list", "debug-log", "enum-as-inner", "fxhash", - "im", "num", "owning_ref", - "pin-project", + "postcard", "rand", "ring", "rle", @@ -488,6 +425,7 @@ dependencies = [ "string_cache", "tabled", "thiserror", + "tracing", ] [[package]] @@ -693,24 +631,10 @@ dependencies = [ ] [[package]] -name = "pin-project" -version = "1.0.12" +name = "pin-project-lite" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "postcard" @@ -807,15 +731,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core", -] - [[package]] name = "redox_syscall" version = "0.2.16" @@ -1006,16 +921,6 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" -[[package]] -name = "sized-chunks" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" -dependencies = [ - "bitmaps", - "typenum", -] - [[package]] name = "smallvec" version = "1.10.0" @@ -1136,10 +1041,36 @@ dependencies = [ ] [[package]] -name = "typenum" -version = "1.15.0" +name = "tracing" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", +] [[package]] name = "unicode-ident" diff --git a/crates/loro-core/src/container.rs b/crates/loro-core/src/container.rs index 995102c3..b6d373d5 100644 --- a/crates/loro-core/src/container.rs +++ b/crates/loro-core/src/container.rs @@ -10,7 +10,7 @@ use crate::{ log_store::ImportContext, op::{InnerContent, RemoteContent, RichOp}, version::{IdSpanVector, VersionVector}, - InternalString, LogStore, LoroValue, ID, + InternalString, LoroValue, ID, }; use serde::{Deserialize, Serialize}; diff --git a/crates/loro-core/src/container/list/list_op.rs b/crates/loro-core/src/container/list/list_op.rs index e2201fe0..52ab32e2 100644 --- a/crates/loro-core/src/container/list/list_op.rs +++ b/crates/loro-core/src/container/list/list_op.rs @@ -2,10 +2,11 @@ use std::ops::Range; use enum_as_inner::EnumAsInner; use rle::{HasLength, Mergable, Sliceable}; +use serde::{Deserialize, Serialize}; use crate::container::text::text_content::{ListSlice, SliceRange}; -#[derive(EnumAsInner, Debug, Clone)] +#[derive(EnumAsInner, Debug, Clone, Serialize, Deserialize)] pub enum ListOp { Insert { slice: ListSlice, pos: usize }, Delete(DeleteSpan), @@ -23,7 +24,7 @@ pub enum InnerListOp { /// len cannot be zero; /// /// pos: 5, len: -3 eq a range of (2, 5] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct DeleteSpan { pub pos: isize, pub len: isize, diff --git a/crates/loro-core/src/container/map/map_content.rs b/crates/loro-core/src/container/map/map_content.rs index 9e67bfaa..e237a044 100644 --- a/crates/loro-core/src/container/map/map_content.rs +++ b/crates/loro-core/src/container/map/map_content.rs @@ -1,8 +1,9 @@ use rle::{HasLength, Mergable, Sliceable}; +use serde::{Deserialize, Serialize}; use crate::{ContentType, InsertContentTrait, InternalString, LoroValue}; -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct MapSet { pub(crate) key: InternalString, pub(crate) value: LoroValue, diff --git a/crates/loro-core/src/container/registry.rs b/crates/loro-core/src/container/registry.rs index 80c403b2..2cdbf724 100644 --- a/crates/loro-core/src/container/registry.rs +++ b/crates/loro-core/src/container/registry.rs @@ -18,7 +18,7 @@ use crate::{ log_store::ImportContext, op::{RemoteContent, RichOp}, version::IdSpanVector, - LogStore, LoroError, LoroValue, + LoroError, LoroValue, }; use super::{ diff --git a/crates/loro-core/src/container/text/text_container.rs b/crates/loro-core/src/container/text/text_container.rs index 130dfd3a..eeff4983 100644 --- a/crates/loro-core/src/container/text/text_container.rs +++ b/crates/loro-core/src/container/text/text_container.rs @@ -342,7 +342,7 @@ impl Container for TextContainer { &mut self, _: &mut Hierarchy, rich_op: &RichOp, - import_context: &mut ImportContext, + _import_context: &mut ImportContext, ) { self.tracker.track_apply(rich_op); } diff --git a/crates/loro-core/src/container/text/text_content.rs b/crates/loro-core/src/container/text/text_content.rs index 6e52ebbf..cd81ec07 100644 --- a/crates/loro-core/src/container/text/text_content.rs +++ b/crates/loro-core/src/container/text/text_content.rs @@ -2,10 +2,11 @@ use std::ops::Range; use enum_as_inner::EnumAsInner; use rle::{HasLength, Mergable, Sliceable}; +use serde::{Deserialize, Serialize}; use crate::{smstring::SmString, LoroValue}; -#[derive(PartialEq, Debug, EnumAsInner, Clone)] +#[derive(PartialEq, Debug, EnumAsInner, Clone, Serialize, Deserialize)] pub enum ListSlice { // TODO: use Box<[LoroValue]> ? RawData(Vec), @@ -101,7 +102,7 @@ impl HasLength for ListSlice { impl Sliceable for ListSlice { fn slice(&self, from: usize, to: usize) -> Self { match self { - ListSlice::RawStr(s) => ListSlice::RawStr(s.0[from..to].into()), + ListSlice::RawStr(s) => ListSlice::RawStr(s[from..to].into()), ListSlice::Unknown(_) => ListSlice::Unknown(to - from), ListSlice::RawData(x) => ListSlice::RawData(x[from..to].to_vec()), } diff --git a/crates/loro-core/src/container/text/tracker.rs b/crates/loro-core/src/container/text/tracker.rs index f85e6ef5..db8970e5 100644 --- a/crates/loro-core/src/container/text/tracker.rs +++ b/crates/loro-core/src/container/text/tracker.rs @@ -1,4 +1,3 @@ -use colored::Colorize; use debug_log::debug_log; use rle::{rle_tree::UnsafeCursor, HasLength, Sliceable}; use smallvec::SmallVec; @@ -326,7 +325,7 @@ impl Tracker { let mut spans = self .content .get_active_id_spans(span.start() as usize, span.atom_len()); - debug_log!("DELETED SPANS={}", format!("{:#?}", &spans).red()); + debug_log!("DELETED SPANS={}", format!("{:#?}", &spans)); self.update_spans(&spans, StatusChange::Delete); if span.is_reversed() && span.atom_len() > 1 { diff --git a/crates/loro-core/src/dag.rs b/crates/loro-core/src/dag.rs index 8e476431..eed49f7c 100644 --- a/crates/loro-core/src/dag.rs +++ b/crates/loro-core/src/dag.rs @@ -12,8 +12,6 @@ use std::{ fmt::Debug, }; -#[allow(unused)] -use colored::Colorize; use fxhash::{FxHashMap, FxHashSet}; use rle::{HasLength, Sliceable}; use smallvec::{smallvec, SmallVec}; diff --git a/crates/loro-core/src/event.rs b/crates/loro-core/src/event.rs index 35e61489..d8e51955 100644 --- a/crates/loro-core/src/event.rs +++ b/crates/loro-core/src/event.rs @@ -1,5 +1,6 @@ use enum_as_inner::EnumAsInner; use fxhash::{FxHashMap, FxHashSet}; +use serde::{Deserialize, Serialize}; use crate::{container::ContainerID, delta::Delta, version::Frontiers, InternalString, LoroValue}; @@ -27,7 +28,7 @@ pub struct Event { pub type Path = Vec; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum Index { Key(InternalString), Seq(usize), diff --git a/crates/loro-core/src/fuzz.rs b/crates/loro-core/src/fuzz.rs index f509602b..53a44ca4 100644 --- a/crates/loro-core/src/fuzz.rs +++ b/crates/loro-core/src/fuzz.rs @@ -217,14 +217,22 @@ fn check_eq(site_a: &mut LoroCore, site_b: &mut LoroCore) { fn check_synced(sites: &mut [LoroCore]) { for i in 0..sites.len() - 1 { for j in i + 1..sites.len() { - debug_log!("-------------------------------"); - debug_log!("checking {} with {}", i, j); - debug_log!("-------------------------------"); - + debug_log::group!("checking {} with {}", i, j); let (a, b) = array_mut_ref!(sites, [i, j]); - a.import(b.export(a.vv())); - b.import(a.export(b.vv())); - check_eq(a, b) + { + debug_log::group!("Import {}", i); + a.import_updates(&b.export_updates(&a.vv()).unwrap()) + .unwrap(); + debug_log::group_end!(); + } + { + debug_log::group!("Import {}", j); + b.import_updates(&a.export_updates(&b.vv()).unwrap()) + .unwrap(); + debug_log::group_end!(); + } + check_eq(a, b); + debug_log::group_end!(); } } } @@ -432,9 +440,10 @@ pub fn test_multi_sites(site_num: u8, actions: &mut [Action]) { sites.apply_action(action); } - debug_log!("================================="); + debug_log::group!("CheckSynced"); // println!("{}", actions.table()); check_synced(&mut sites); + debug_log::group_end!(); } #[cfg(test)] @@ -653,6 +662,43 @@ mod test { ) } + #[test] + fn simplest() { + test_multi_sites( + 2, + &mut [Ins { + content: 1, + pos: 0, + site: 0, + }], + ) + } + + #[test] + fn encode() { + test_multi_sites( + 8, + &mut [ + Ins { + content: 0, + pos: 1840611456097844714, + site: 0, + }, + Ins { + content: 0, + pos: 2825745054957034, + site: 10, + }, + Sync { from: 10, to: 0 }, + Del { + pos: 1125899890065408, + len: 1840611456097844714, + site: 0, + }, + ], + ) + } + #[test] fn case_two() { test_multi_sites( diff --git a/crates/loro-core/src/fuzz/recursive.rs b/crates/loro-core/src/fuzz/recursive.rs index 6dcd7864..e35fa1ae 100644 --- a/crates/loro-core/src/fuzz/recursive.rs +++ b/crates/loro-core/src/fuzz/recursive.rs @@ -612,8 +612,12 @@ fn check_synced(sites: &mut [Actor]) { let (a, b) = array_mut_ref!(sites, [i, j]); let a_doc = &mut a.loro; let b_doc = &mut b.loro; - a_doc.import(b_doc.export(a_doc.vv())); - b_doc.import(a_doc.export(b_doc.vv())); + a_doc + .import_updates(&b_doc.export_updates(&a_doc.vv()).unwrap()) + .unwrap(); + b_doc + .import_updates(&a_doc.export_updates(&b_doc.vv()).unwrap()) + .unwrap(); check_eq(a, b); debug_log::group_end!(); } diff --git a/crates/loro-core/src/lib.rs b/crates/loro-core/src/lib.rs index 22e5c03c..bddd5bea 100644 --- a/crates/loro-core/src/lib.rs +++ b/crates/loro-core/src/lib.rs @@ -40,7 +40,6 @@ pub(crate) use op::{ContentType, InsertContentTrait, Op}; // TODO: rename as Key? pub(crate) type InternalString = DefaultAtom; pub(crate) use container::Container; -pub(crate) use log_store::ImportContext; pub use container::{list::List, map::Map, text::Text, ContainerType}; pub use log_store::LogStore; diff --git a/crates/loro-core/src/log_store.rs b/crates/loro-core/src/log_store.rs index 9289a19a..d5818b15 100644 --- a/crates/loro-core/src/log_store.rs +++ b/crates/loro-core/src/log_store.rs @@ -1,9 +1,11 @@ //! [LogStore] stores all the [Change]s and [Op]s. It's also a [DAG][crate::dag]; //! //! +mod encode_updates; mod encoding; mod import; mod iter; + use crate::LoroValue; pub(crate) use import::ImportContext; use std::{ @@ -13,7 +15,6 @@ use std::{ use fxhash::FxHashMap; -use crate::context::Context; use rle::{HasLength, RleVec, RleVecWithIndex, Sliceable}; use smallvec::SmallVec; @@ -44,14 +45,14 @@ pub struct GcConfig { impl Default for GcConfig { fn default() -> Self { GcConfig { - gc: true, + gc: false, snapshot_interval: 6 * MONTH, } } } type ClientChanges = FxHashMap>; -type RemoteClientChanges = FxHashMap, ChangeMergeCfg>>; +type RemoteClientChanges = FxHashMap>>; #[derive(Debug)] /// LogStore stores the full history of Loro @@ -101,21 +102,14 @@ impl LogStore { .map(|changes| changes.get(id.counter as usize).unwrap().element) } - pub fn export( - &self, - remote_vv: &VersionVector, - ) -> FxHashMap, ChangeMergeCfg>> { - let mut ans: FxHashMap, ChangeMergeCfg>> = - Default::default(); + pub fn export(&self, remote_vv: &VersionVector) -> FxHashMap>> { + let mut ans: FxHashMap>> = Default::default(); let self_vv = self.vv(); let diff = self_vv.diff(remote_vv); for span in diff.left.iter() { let changes = self.get_changes_slice(span.id_span()); for change in changes.iter() { - let vec = ans - .entry(change.id.client_id) - .or_insert_with(|| RleVecWithIndex::new_cfg(self.get_change_merge_cfg())); - + let vec = ans.entry(change.id.client_id).or_insert_with(Vec::new); vec.push(self.change_to_export_format(change)); } } diff --git a/crates/loro-core/src/log_store/encode_updates.rs b/crates/loro-core/src/log_store/encode_updates.rs new file mode 100644 index 00000000..1e49888e --- /dev/null +++ b/crates/loro-core/src/log_store/encode_updates.rs @@ -0,0 +1,191 @@ +use rle::{HasLength, RleVec}; +use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; + +use crate::{ + change::{Change, Lamport, Timestamp}, + container::ContainerID, + id::{ClientID, Counter, ID}, + op::{RemoteContent, RemoteOp}, + LogStore, LoroCore, VersionVector, +}; + +use super::RemoteClientChanges; + +#[derive(Serialize, Deserialize, Debug)] +struct Updates { + changes: Vec, +} + +/// the continuous changes from the same client +#[derive(Serialize, Deserialize, Debug)] +struct EncodedClientChanges { + meta: FirstChangeInfo, + data: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct FirstChangeInfo { + pub(crate) client: ClientID, + pub(crate) counter: Counter, + pub(crate) lamport: Lamport, + pub(crate) timestamp: Timestamp, +} + +#[derive(Serialize, Deserialize, Debug)] +struct EncodedOp { + pub(crate) container: ContainerID, + pub(crate) contents: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct EncodedChange { + pub(crate) ops: Vec, + pub(crate) deps_except_self: Vec, + pub(crate) lamport_delta: u32, + pub(crate) timestamp_delta: i64, +} + +impl LogStore { + pub fn encode_updates(&self, from: &VersionVector) -> Result, postcard::Error> { + let changes = self.export(from); + let mut updates = Updates { + changes: Vec::with_capacity(changes.len()), + }; + for (_, changes) in changes { + let encoded = convert_changes_to_encoded(changes.into_iter()); + updates.changes.push(encoded); + } + + postcard::to_allocvec(&updates) + } + + pub fn import_updates(&mut self, input: &[u8]) -> Result<(), postcard::Error> { + let updates: Updates = postcard::from_bytes(input)?; + let mut changes: RemoteClientChanges = Default::default(); + for encoded in updates.changes { + changes.insert(encoded.meta.client, convert_encoded_to_changes(encoded)); + } + + self.import(changes); + Ok(()) + } +} + +fn convert_changes_to_encoded(mut changes: I) -> EncodedClientChanges +where + I: Iterator>, +{ + let first_change = changes.next().unwrap(); + let this_client_id = first_change.id.client_id; + let mut data = Vec::with_capacity(changes.size_hint().0 + 1); + let mut last_change = first_change.clone(); + data.push(EncodedChange { + ops: first_change + .ops + .iter() + .map(|op| EncodedOp { + container: op.container.clone(), + contents: op.contents.iter().cloned().collect(), + }) + .collect(), + deps_except_self: first_change + .deps + .iter() + .filter(|x| x.client_id != this_client_id) + .copied() + .collect(), + lamport_delta: 0, + timestamp_delta: 0, + }); + for change in changes { + data.push(EncodedChange { + ops: change + .ops + .iter() + .map(|op| EncodedOp { + container: op.container.clone(), + contents: op.contents.iter().cloned().collect(), + }) + .collect(), + deps_except_self: change + .deps + .iter() + .filter(|x| x.client_id != this_client_id) + .copied() + .collect(), + lamport_delta: change.lamport - last_change.lamport, + timestamp_delta: change.timestamp - last_change.timestamp, + }); + last_change = change; + } + + EncodedClientChanges { + meta: FirstChangeInfo { + client: this_client_id, + counter: first_change.id.counter, + lamport: first_change.lamport, + timestamp: first_change.timestamp, + }, + data, + } +} + +fn convert_encoded_to_changes(changes: EncodedClientChanges) -> Vec> { + let mut result = Vec::with_capacity(changes.data.len()); + let mut last_lamport = changes.meta.lamport; + let mut last_timestamp = changes.meta.timestamp; + let mut counter: Counter = changes.meta.counter; + for encoded in changes.data { + let start_counter = counter; + let mut deps = SmallVec::with_capacity(encoded.deps_except_self.len() + 1); + if start_counter > 0 { + deps.push(ID { + client_id: changes.meta.client, + counter: start_counter - 1, + }); + } + + for dep in encoded.deps_except_self { + deps.push(dep); + } + + let mut ops = RleVec::with_capacity(encoded.ops.len()); + for op in encoded.ops { + let len: usize = op.contents.iter().map(|x| x.atom_len()).sum(); + ops.push(RemoteOp { + counter, + container: op.container, + contents: op.contents.into_iter().collect(), + }); + counter += len as Counter; + } + + let change = Change { + id: ID { + client_id: changes.meta.client, + counter: start_counter, + }, + lamport: last_lamport + encoded.lamport_delta, + timestamp: last_timestamp + encoded.timestamp_delta, + ops, + deps, + }; + last_lamport = change.lamport; + last_timestamp = change.timestamp; + result.push(change); + } + + result +} + +impl LoroCore { + pub fn export_updates(&self, from: &VersionVector) -> Result, postcard::Error> { + self.log_store.read().unwrap().encode_updates(from) + } + + pub fn import_updates(&mut self, input: &[u8]) -> Result<(), postcard::Error> { + let ans = self.log_store.write().unwrap().import_updates(input); + ans + } +} diff --git a/crates/loro-core/src/log_store/encoding.rs b/crates/loro-core/src/log_store/encoding.rs index 33b22758..5a81f6a5 100644 --- a/crates/loro-core/src/log_store/encoding.rs +++ b/crates/loro-core/src/log_store/encoding.rs @@ -20,7 +20,6 @@ use crate::{ dag::remove_included_frontiers, id::{ClientID, Counter, ID}, op::{Op, RemoteContent, RemoteOp}, - smstring::SmString, span::{HasIdSpan, HasLamportSpan}, ContainerType, InternalString, LogStore, LoroValue, VersionVector, }; @@ -156,7 +155,6 @@ fn encode_changes(store: &LogStore) -> Encoded { (span.pos as usize, 0, LoroValue::I32(span.len as i32)) } }, - crate::op::RemoteContent::Dyn(_) => unreachable!(), }; op_len += 1; ops.push(OpEncoding { @@ -274,7 +272,7 @@ fn decode_changes( }, _ => { let slice = match value { - LoroValue::String(s) => ListSlice::RawStr(SmString::from(&*s)), + LoroValue::String(s) => ListSlice::RawStr(s.into()), LoroValue::List(v) => ListSlice::RawData(*v), _ => unreachable!(), }; diff --git a/crates/loro-core/src/log_store/import.rs b/crates/loro-core/src/log_store/import.rs index a3031bb2..91701195 100644 --- a/crates/loro-core/src/log_store/import.rs +++ b/crates/loro-core/src/log_store/import.rs @@ -9,7 +9,7 @@ use tracing::instrument; use fxhash::FxHashMap; -use rle::{HasLength, RleVecWithIndex, Sliceable}; +use rle::{slice_vec_by, HasLength, RleVecWithIndex, Sliceable}; use crate::{ container::{registry::ContainerInstance, Container, ContainerID}, @@ -329,9 +329,11 @@ impl LogStore { let other_start_ctr = changes.first().unwrap().ctr_start(); match other_start_ctr.cmp(&self_end_ctr) { std::cmp::Ordering::Less => { - *changes = changes.slice( - (self_end_ctr - other_start_ctr) as usize, - changes.atom_len(), + *changes = slice_vec_by( + changes, + |x| x.id.counter as usize, + self_end_ctr as usize, + usize::MAX, ); } std::cmp::Ordering::Equal => {} diff --git a/crates/loro-core/src/loro.rs b/crates/loro-core/src/loro.rs index 4c9e2309..85d65c48 100644 --- a/crates/loro-core/src/loro.rs +++ b/crates/loro-core/src/loro.rs @@ -2,10 +2,10 @@ use std::sync::{Arc, RwLock}; use crate::LoroValue; use fxhash::FxHashMap; -use rle::RleVecWithIndex; + use crate::{ - change::{Change, ChangeMergeCfg}, + change::{Change}, configure::Configure, container::{list::List, map::Map, text::Text, ContainerIdRaw, ContainerType}, event::{Observer, SubscriptionID}, @@ -72,18 +72,12 @@ impl LoroCore { Text::from_instance(instance, cid) } - pub fn export( - &self, - remote_vv: VersionVector, - ) -> FxHashMap, ChangeMergeCfg>> { + pub fn export(&self, remote_vv: VersionVector) -> FxHashMap>> { let store = self.log_store.read().unwrap(); store.export(&remote_vv) } - pub fn import( - &mut self, - changes: FxHashMap, ChangeMergeCfg>>, - ) { + pub fn import(&mut self, changes: FxHashMap>>) { let mut store = self.log_store.write().unwrap(); store.import(changes) } diff --git a/crates/loro-core/src/op/content.rs b/crates/loro-core/src/op/content.rs index 0c09a554..44d5bf35 100644 --- a/crates/loro-core/src/op/content.rs +++ b/crates/loro-core/src/op/content.rs @@ -2,6 +2,7 @@ use std::any::{Any, TypeId}; use enum_as_inner::EnumAsInner; use rle::{HasLength, Mergable, Sliceable}; +use serde::{Deserialize, Serialize}; use crate::container::{ list::list_op::{InnerListOp, ListOp}, @@ -24,11 +25,10 @@ pub enum InnerContent { Map(InnerMapSet), } -#[derive(EnumAsInner, Debug)] +#[derive(EnumAsInner, Debug, Serialize, Deserialize)] pub enum RemoteContent { Map(MapSet), List(ListOp), - Dyn(Box), } impl Clone for RemoteContent { @@ -36,7 +36,6 @@ impl Clone for RemoteContent { match self { Self::Map(arg0) => Self::Map(arg0.clone()), Self::List(arg0) => Self::List(arg0.clone()), - Self::Dyn(arg0) => Self::Dyn(arg0.clone_content()), } } } @@ -93,7 +92,6 @@ impl HasLength for RemoteContent { fn content_len(&self) -> usize { match self { RemoteContent::Map(x) => x.content_len(), - RemoteContent::Dyn(x) => x.content_len(), RemoteContent::List(x) => x.content_len(), } } @@ -103,7 +101,6 @@ impl Sliceable for RemoteContent { fn slice(&self, from: usize, to: usize) -> Self { match self { RemoteContent::Map(x) => RemoteContent::Map(x.slice(from, to)), - RemoteContent::Dyn(x) => RemoteContent::Dyn(x.slice_content(from, to)), RemoteContent::List(x) => RemoteContent::List(x.slice(from, to)), } } @@ -117,7 +114,6 @@ impl Mergable for RemoteContent { match (self, other) { (RemoteContent::Map(x), RemoteContent::Map(y)) => x.is_mergable(y, &()), (RemoteContent::List(x), RemoteContent::List(y)) => x.is_mergable(y, &()), - (RemoteContent::Dyn(x), RemoteContent::Dyn(y)) => x.is_mergable_content(&**y), _ => false, } } @@ -135,7 +131,6 @@ impl Mergable for RemoteContent { RemoteContent::List(y) => x.merge(y, &()), _ => unreachable!(), }, - RemoteContent::Dyn(x) => x.merge_content(&**_other.as_dyn().unwrap()), } } } diff --git a/crates/loro-core/src/smstring.rs b/crates/loro-core/src/smstring.rs index efbffba6..e42a5106 100644 --- a/crates/loro-core/src/smstring.rs +++ b/crates/loro-core/src/smstring.rs @@ -3,6 +3,7 @@ use std::ops::DerefMut; use std::ops::Deref; use rle::Mergable; +use serde::Deserialize; use serde::Serialize; use smartstring::LazyCompact; @@ -63,6 +64,13 @@ impl From for SmString { } } +impl From> for SmString { + fn from(s: Box) -> Self { + let s: &str = &s; + SmString(s.into()) + } +} + impl From<&str> for SmString { fn from(s: &str) -> Self { SmString(s.into()) @@ -77,3 +85,13 @@ impl Serialize for SmString { serializer.serialize_str(&self.0) } } + +impl<'de> Deserialize<'de> for SmString { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Ok(s.into()) + } +} diff --git a/crates/loro-core/src/value.rs b/crates/loro-core/src/value.rs index d1ddc501..4039765d 100644 --- a/crates/loro-core/src/value.rs +++ b/crates/loro-core/src/value.rs @@ -6,7 +6,6 @@ use serde::{de::VariantAccess, ser::SerializeStruct, Deserialize, Serialize}; use crate::{ container::{registry::ContainerRegistry, ContainerID}, - context::Context, delta::DeltaItem, event::{Diff, Index, Path}, Container, diff --git a/crates/loro-core/tests/test.rs b/crates/loro-core/tests/test.rs index 99621ac6..dece3bda 100644 --- a/crates/loro-core/tests/test.rs +++ b/crates/loro-core/tests/test.rs @@ -1,4 +1,3 @@ -use std::borrow::{Borrow, BorrowMut}; use std::cell::RefCell; use std::rc::Rc; diff --git a/crates/rle/src/lib.rs b/crates/rle/src/lib.rs index 19a19a49..30f4ed00 100644 --- a/crates/rle/src/lib.rs +++ b/crates/rle/src/lib.rs @@ -27,7 +27,7 @@ pub mod rle_tree; mod rle_vec; mod rle_vec_old; pub use crate::rle_trait::{HasIndex, HasLength, Mergable, Rle, Slice, Sliceable, ZeroElement}; -pub use crate::rle_vec::{RleVec, RleVecWithLen}; +pub use crate::rle_vec::{slice_vec_by, RleVec, RleVecWithLen}; pub use crate::rle_vec_old::{RleVecWithIndex, SearchResult, SliceIterator}; pub mod rle_impl; pub use rle_tree::tree_trait::RleTreeTrait; diff --git a/crates/rle/src/rle_vec.rs b/crates/rle/src/rle_vec.rs index 22f53670..0884976a 100644 --- a/crates/rle/src/rle_vec.rs +++ b/crates/rle/src/rle_vec.rs @@ -555,6 +555,35 @@ impl Deref for RleVecWithLen { } } +pub fn slice_vec_by(vec: &Vec, index: F, start: usize, end: usize) -> Vec +where + F: Fn(&T) -> usize, + T: Sliceable + HasLength, +{ + if start >= end || vec.is_empty() { + return Vec::new(); + } + + let start = start - index(&vec[0]); + let end = end - index(&vec[0]); + let mut ans = Vec::new(); + let mut index = 0; + for i in 0..vec.len() { + if index >= end { + break; + } + + let len = vec[i].atom_len(); + if start < index + len { + ans.push(vec[i].slice(start.saturating_sub(index), (end - index).min(len))) + } + + index += len; + } + + ans +} + #[cfg(test)] mod test { use super::*; diff --git a/crates/rle/src/rle_vec_old.rs b/crates/rle/src/rle_vec_old.rs index 05b4cc94..090c8254 100644 --- a/crates/rle/src/rle_vec_old.rs +++ b/crates/rle/src/rle_vec_old.rs @@ -1,4 +1,7 @@ -use std::ops::{Deref, Range}; +use std::{ + ops::{Deref, Range}, + vec, +}; use num::Integer; @@ -213,6 +216,16 @@ impl RleVecWithIndex { } } +impl IntoIterator for RleVecWithIndex { + type Item = T; + + type IntoIter = vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.vec.into_iter() + } +} + impl Default for RleVecWithIndex { fn default() -> Self { Self::new()