Merge pull request #31 from loro-dev/feat-encode-update

Feat: encode/decode update

Cargo.lock

@@ -157,15 +157,6 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
-[[package]]
-name = "bitmaps"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2"
-dependencies = [
- "typenum",
-]
-
 [[package]]
 name = "block-buffer"
 version = "0.10.3"
@@ -276,17 +267,6 @@ dependencies = [
  "termcolor",
 ]
 
-[[package]]
-name = "colored"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd"
-dependencies = [
- "atty",
- "lazy_static",
- "winapi",
-]
-
 [[package]]
 name = "console_error_panic_hook"
 version = "0.1.7"
@@ -671,20 +651,6 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
-[[package]]
-name = "im"
-version = "15.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9"
-dependencies = [
- "bitmaps",
- "rand_core",
- "rand_xoshiro",
- "sized-chunks",
- "typenum",
- "version_check",
-]
-
 [[package]]
 name = "indexmap"
 version = "1.9.1"
@@ -766,9 +732,7 @@ dependencies = [
  "arbitrary",
  "arbtest",
  "arref",
- "bit-vec",
  "color-backtrace",
- "colored",
  "crdt-list",
  "criterion",
  "ctor",
@@ -777,11 +741,10 @@ dependencies = [
  "enum-as-inner",
  "flate2",
  "fxhash",
- "im",
  "js-sys",
  "num",
  "owning_ref",
- "pin-project",
+ "postcard",
  "proptest",
  "proptest-derive",
  "rand",
@@ -1056,26 +1019,6 @@ dependencies = [
  "siphasher",
 ]
 
-[[package]]
-name = "pin-project"
-version = "1.0.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc"
-dependencies = [
- "pin-project-internal",
-]
-
-[[package]]
-name = "pin-project-internal"
-version = "1.0.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55"
-dependencies = [
- "proc-macro2 1.0.47",
- "quote 1.0.21",
- "syn 1.0.103",
-]
-
 [[package]]
 name = "pin-project-lite"
 version = "0.2.9"
@@ -1275,15 +1218,6 @@ dependencies = [
  "rand_core",
 ]
 
-[[package]]
-name = "rand_xoshiro"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa"
-dependencies = [
- "rand_core",
-]
-
 [[package]]
 name = "rayon"
 version = "1.5.3"
@@ -1568,16 +1502,6 @@ version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
 
-[[package]]
-name = "sized-chunks"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e"
-dependencies = [
- "bitmaps",
- "typenum",
-]
-
 [[package]]
 name = "smallvec"
 version = "1.10.0"

crates/loro-core/Cargo.toml

@@ -9,22 +9,19 @@ edition = "2021"
 string_cache = "0.8.3"
 rle = { path = "../rle" }
 smallvec = "1.8.0"
-smartstring = "1.0.1"
+smartstring = { version = "1.0.1" }
 fxhash = "0.2.1"
 ring = "0.16.20"
-pin-project = "1.0.10"
 serde = { version = "1.0.140", features = ["derive"] }
 thiserror = "1.0.31"
-im = "15.1.0"
 enum-as-inner = "0.5.1"
 num = "0.4.0"
 crdt-list = { version = "0.3.0" }
 owning_ref = "0.4.1"
+postcard = "1.0.2"
 rand = { version = "0.8.5", optional = true }
 arbitrary = { version = "1.1.7", optional = true }
 tabled = { version = "0.10.0", optional = true }
-colored = "2.0.0"
-bit-vec = "0.6.3"
 wasm-bindgen = { version = "0.2.83", optional = true }
 serde-wasm-bindgen = { version = "0.4.5", optional = true }
 js-sys = { version = "0.3.60", optional = true }

crates/loro-core/examples/encoding.rs

@@ -1,6 +1,9 @@
-use std::{io::Read, time::Instant};
+use std::{
+    io::{Read, Write},
+    time::Instant,
+};
 
-use flate2::read::GzDecoder;
+use flate2::{read::GzDecoder, write::GzEncoder};
 use loro_core::{configure::Configure, container::registry::ContainerWrapper, LoroCore};
 use serde_json::Value;
 const RAW_DATA: &[u8; 901823] = include_bytes!("../benches/automerge-paper.json.gz");
@@ -47,20 +50,10 @@ fn main() {
     assert_eq!(buf, buf2);
     let json2 = loro.to_json();
     assert_eq!(json1, json2);
-    let mut last = 100;
+    let update_buf = loro.export_updates(&Default::default()).unwrap();
-    let mut count = 0;
+    println!("Updates have {} bytes", update_buf.len());
-    let mut max_count = 0;
+    let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default());
-    for &byte in buf.iter() {
+    encoder.write_all(&update_buf).unwrap();
-        if byte == last {
+    let data = encoder.finish().unwrap();
-            count += 1;
+    println!("After compress updates have {} bytes", data.len());
-            if count > max_count {
-                max_count = count;
-            }
-        } else {
-            count = 0;
-        }
-        last = byte;
-    }
-
-    println!("Longest continuous bytes length {}", max_count);
 }

crates/loro-core/fuzz/Cargo.lock

@@ -53,17 +53,6 @@ dependencies = [
  "critical-section",
 ]
 
-[[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi",
- "libc",
- "winapi",
-]
-
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -85,12 +74,6 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8fe8f5a8a398345e52358e18ff07cc17a568fbca5c6f73873d3a62056309603"
 
-[[package]]
-name = "bit-vec"
-version = "0.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
-
 [[package]]
 name = "bit_field"
 version = "0.10.1"
@@ -109,15 +92,6 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
-[[package]]
-name = "bitmaps"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2"
-dependencies = [
- "typenum",
-]
-
 [[package]]
 name = "bumpalo"
 version = "3.11.1"
@@ -157,17 +131,6 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15"
 
-[[package]]
-name = "colored"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd"
-dependencies = [
- "atty",
- "lazy_static",
- "winapi",
-]
-
 [[package]]
 name = "cortex-m"
 version = "0.7.6"
@@ -357,35 +320,12 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
 
-[[package]]
-name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "ident_case"
 version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
-[[package]]
-name = "im"
-version = "15.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9"
-dependencies = [
- "bitmaps",
- "rand_core",
- "rand_xoshiro",
- "sized-chunks",
- "typenum",
- "version_check",
-]
-
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -467,16 +407,13 @@ version = "0.1.0"
 dependencies = [
  "arbitrary",
  "arref",
- "bit-vec",
- "colored",
  "crdt-list",
  "debug-log",
  "enum-as-inner",
  "fxhash",
- "im",
  "num",
  "owning_ref",
- "pin-project",
+ "postcard",
  "rand",
  "ring",
  "rle",
@@ -488,6 +425,7 @@ dependencies = [
  "string_cache",
  "tabled",
  "thiserror",
+ "tracing",
 ]
 
 [[package]]
@@ -693,24 +631,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "pin-project"
+name = "pin-project-lite"
-version = "1.0.12"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc"
+checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
-dependencies = [
- "pin-project-internal",
-]
-
-[[package]]
-name = "pin-project-internal"
-version = "1.0.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
 
 [[package]]
 name = "postcard"
@@ -807,15 +731,6 @@ dependencies = [
  "getrandom",
 ]
 
-[[package]]
-name = "rand_xoshiro"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa"
-dependencies = [
- "rand_core",
-]
-
 [[package]]
 name = "redox_syscall"
 version = "0.2.16"
@@ -1006,16 +921,6 @@ version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
 
-[[package]]
-name = "sized-chunks"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e"
-dependencies = [
- "bitmaps",
- "typenum",
-]
-
 [[package]]
 name = "smallvec"
 version = "1.10.0"
@@ -1136,10 +1041,36 @@ dependencies = [
 ]
 
 [[package]]
-name = "typenum"
+name = "tracing"
-version = "1.15.0"
+version = "0.1.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
+checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
+dependencies = [
+ "cfg-if",
+ "pin-project-lite",
+ "tracing-attributes",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-attributes"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a"
+dependencies = [
+ "once_cell",
+]
 
 [[package]]
 name = "unicode-ident"

crates/loro-core/src/container.rs

@@ -10,7 +10,7 @@ use crate::{
     log_store::ImportContext,
     op::{InnerContent, RemoteContent, RichOp},
     version::{IdSpanVector, VersionVector},
-    InternalString, LogStore, LoroValue, ID,
+    InternalString, LoroValue, ID,
 };
 
 use serde::{Deserialize, Serialize};

crates/loro-core/src/container/list/list_op.rs

@@ -2,10 +2,11 @@ use std::ops::Range;
 
 use enum_as_inner::EnumAsInner;
 use rle::{HasLength, Mergable, Sliceable};
+use serde::{Deserialize, Serialize};
 
 use crate::container::text::text_content::{ListSlice, SliceRange};
 
-#[derive(EnumAsInner, Debug, Clone)]
+#[derive(EnumAsInner, Debug, Clone, Serialize, Deserialize)]
 pub enum ListOp {
     Insert { slice: ListSlice, pos: usize },
     Delete(DeleteSpan),
@@ -23,7 +24,7 @@ pub enum InnerListOp {
 /// len cannot be zero;
 ///
 /// pos: 5, len: -3 eq a range of (2, 5]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
 pub struct DeleteSpan {
     pub pos: isize,
     pub len: isize,

crates/loro-core/src/container/map/map_content.rs

@@ -1,8 +1,9 @@
 use rle::{HasLength, Mergable, Sliceable};
+use serde::{Deserialize, Serialize};
 
 use crate::{ContentType, InsertContentTrait, InternalString, LoroValue};
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct MapSet {
     pub(crate) key: InternalString,
     pub(crate) value: LoroValue,

crates/loro-core/src/container/registry.rs

@@ -18,7 +18,7 @@ use crate::{
     log_store::ImportContext,
     op::{RemoteContent, RichOp},
     version::IdSpanVector,
-    LogStore, LoroError, LoroValue,
+    LoroError, LoroValue,
 };
 
 use super::{

crates/loro-core/src/container/text/text_container.rs

@@ -342,7 +342,7 @@ impl Container for TextContainer {
         &mut self,
         _: &mut Hierarchy,
         rich_op: &RichOp,
-        import_context: &mut ImportContext,
+        _import_context: &mut ImportContext,
     ) {
         self.tracker.track_apply(rich_op);
     }

crates/loro-core/src/container/text/text_content.rs

@@ -2,10 +2,11 @@ use std::ops::Range;
 
 use enum_as_inner::EnumAsInner;
 use rle::{HasLength, Mergable, Sliceable};
+use serde::{Deserialize, Serialize};
 
 use crate::{smstring::SmString, LoroValue};
 
-#[derive(PartialEq, Debug, EnumAsInner, Clone)]
+#[derive(PartialEq, Debug, EnumAsInner, Clone, Serialize, Deserialize)]
 pub enum ListSlice {
     // TODO: use Box<[LoroValue]> ?
     RawData(Vec<LoroValue>),
@@ -101,7 +102,7 @@ impl HasLength for ListSlice {
 impl Sliceable for ListSlice {
     fn slice(&self, from: usize, to: usize) -> Self {
         match self {
-            ListSlice::RawStr(s) => ListSlice::RawStr(s.0[from..to].into()),
+            ListSlice::RawStr(s) => ListSlice::RawStr(s[from..to].into()),
             ListSlice::Unknown(_) => ListSlice::Unknown(to - from),
             ListSlice::RawData(x) => ListSlice::RawData(x[from..to].to_vec()),
         }

crates/loro-core/src/container/text/tracker.rs

@@ -1,4 +1,3 @@
-use colored::Colorize;
 use debug_log::debug_log;
 use rle::{rle_tree::UnsafeCursor, HasLength, Sliceable};
 use smallvec::SmallVec;
@@ -326,7 +325,7 @@ impl Tracker {
                 let mut spans = self
                     .content
                     .get_active_id_spans(span.start() as usize, span.atom_len());
-                debug_log!("DELETED SPANS={}", format!("{:#?}", &spans).red());
+                debug_log!("DELETED SPANS={}", format!("{:#?}", &spans));
                 self.update_spans(&spans, StatusChange::Delete);
 
                 if span.is_reversed() && span.atom_len() > 1 {

crates/loro-core/src/dag.rs

@@ -12,8 +12,6 @@ use std::{
     fmt::Debug,
 };
 
-#[allow(unused)]
-use colored::Colorize;
 use fxhash::{FxHashMap, FxHashSet};
 use rle::{HasLength, Sliceable};
 use smallvec::{smallvec, SmallVec};

crates/loro-core/src/event.rs

@@ -1,5 +1,6 @@
 use enum_as_inner::EnumAsInner;
 use fxhash::{FxHashMap, FxHashSet};
+use serde::{Deserialize, Serialize};
 
 use crate::{container::ContainerID, delta::Delta, version::Frontiers, InternalString, LoroValue};
 
@@ -27,7 +28,7 @@ pub struct Event {
 
 pub type Path = Vec<Index>;
 
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub enum Index {
     Key(InternalString),
     Seq(usize),

crates/loro-core/src/fuzz.rs

@@ -217,14 +217,22 @@ fn check_eq(site_a: &mut LoroCore, site_b: &mut LoroCore) {
 fn check_synced(sites: &mut [LoroCore]) {
     for i in 0..sites.len() - 1 {
         for j in i + 1..sites.len() {
-            debug_log!("-------------------------------");
+            debug_log::group!("checking {} with {}", i, j);
-            debug_log!("checking {} with {}", i, j);
-            debug_log!("-------------------------------");
-
             let (a, b) = array_mut_ref!(sites, [i, j]);
-            a.import(b.export(a.vv()));
+            {
-            b.import(a.export(b.vv()));
+                debug_log::group!("Import {}", i);
-            check_eq(a, b)
+                a.import_updates(&b.export_updates(&a.vv()).unwrap())
+                    .unwrap();
+                debug_log::group_end!();
+            }
+            {
+                debug_log::group!("Import {}", j);
+                b.import_updates(&a.export_updates(&b.vv()).unwrap())
+                    .unwrap();
+                debug_log::group_end!();
+            }
+            check_eq(a, b);
+            debug_log::group_end!();
         }
     }
 }
@@ -432,9 +440,10 @@ pub fn test_multi_sites(site_num: u8, actions: &mut [Action]) {
         sites.apply_action(action);
     }
 
-    debug_log!("=================================");
+    debug_log::group!("CheckSynced");
     // println!("{}", actions.table());
     check_synced(&mut sites);
+    debug_log::group_end!();
 }
 
 #[cfg(test)]
@@ -653,6 +662,43 @@ mod test {
         )
     }
 
+    #[test]
+    fn simplest() {
+        test_multi_sites(
+            2,
+            &mut [Ins {
+                content: 1,
+                pos: 0,
+                site: 0,
+            }],
+        )
+    }
+
+    #[test]
+    fn encode() {
+        test_multi_sites(
+            8,
+            &mut [
+                Ins {
+                    content: 0,
+                    pos: 1840611456097844714,
+                    site: 0,
+                },
+                Ins {
+                    content: 0,
+                    pos: 2825745054957034,
+                    site: 10,
+                },
+                Sync { from: 10, to: 0 },
+                Del {
+                    pos: 1125899890065408,
+                    len: 1840611456097844714,
+                    site: 0,
+                },
+            ],
+        )
+    }
+
     #[test]
     fn case_two() {
         test_multi_sites(

crates/loro-core/src/fuzz/recursive.rs

@@ -612,8 +612,12 @@ fn check_synced(sites: &mut [Actor]) {
             let (a, b) = array_mut_ref!(sites, [i, j]);
             let a_doc = &mut a.loro;
             let b_doc = &mut b.loro;
-            a_doc.import(b_doc.export(a_doc.vv()));
+            a_doc
-            b_doc.import(a_doc.export(b_doc.vv()));
+                .import_updates(&b_doc.export_updates(&a_doc.vv()).unwrap())
+                .unwrap();
+            b_doc
+                .import_updates(&a_doc.export_updates(&b_doc.vv()).unwrap())
+                .unwrap();
             check_eq(a, b);
             debug_log::group_end!();
         }

crates/loro-core/src/lib.rs

@@ -40,7 +40,6 @@ pub(crate) use op::{ContentType, InsertContentTrait, Op};
 // TODO: rename as Key?
 pub(crate) type InternalString = DefaultAtom;
 pub(crate) use container::Container;
-pub(crate) use log_store::ImportContext;
 
 pub use container::{list::List, map::Map, text::Text, ContainerType};
 pub use log_store::LogStore;

crates/loro-core/src/log_store.rs

@@ -1,9 +1,11 @@
 //! [LogStore] stores all the [Change]s and [Op]s. It's also a [DAG][crate::dag];
 //!
 //!
+mod encode_updates;
 mod encoding;
 mod import;
 mod iter;
+
 use crate::LoroValue;
 pub(crate) use import::ImportContext;
 use std::{
@@ -13,7 +15,6 @@ use std::{
 
 use fxhash::FxHashMap;
 
-use crate::context::Context;
 use rle::{HasLength, RleVec, RleVecWithIndex, Sliceable};
 use smallvec::SmallVec;
 
@@ -44,14 +45,14 @@ pub struct GcConfig {
 impl Default for GcConfig {
     fn default() -> Self {
         GcConfig {
-            gc: true,
+            gc: false,
             snapshot_interval: 6 * MONTH,
         }
     }
 }
 
 type ClientChanges = FxHashMap<ClientID, RleVecWithIndex<Change, ChangeMergeCfg>>;
-type RemoteClientChanges = FxHashMap<ClientID, RleVecWithIndex<Change<RemoteOp>, ChangeMergeCfg>>;
+type RemoteClientChanges = FxHashMap<ClientID, Vec<Change<RemoteOp>>>;
 
 #[derive(Debug)]
 /// LogStore stores the full history of Loro
@@ -101,21 +102,14 @@ impl LogStore {
             .map(|changes| changes.get(id.counter as usize).unwrap().element)
     }
 
-    pub fn export(
+    pub fn export(&self, remote_vv: &VersionVector) -> FxHashMap<ClientID, Vec<Change<RemoteOp>>> {
-        &self,
+        let mut ans: FxHashMap<ClientID, Vec<Change<RemoteOp>>> = Default::default();
-        remote_vv: &VersionVector,
-    ) -> FxHashMap<ClientID, RleVecWithIndex<Change<RemoteOp>, ChangeMergeCfg>> {
-        let mut ans: FxHashMap<ClientID, RleVecWithIndex<Change<RemoteOp>, ChangeMergeCfg>> =
-            Default::default();
         let self_vv = self.vv();
         let diff = self_vv.diff(remote_vv);
         for span in diff.left.iter() {
             let changes = self.get_changes_slice(span.id_span());
             for change in changes.iter() {
-                let vec = ans
+                let vec = ans.entry(change.id.client_id).or_insert_with(Vec::new);
-                    .entry(change.id.client_id)
-                    .or_insert_with(|| RleVecWithIndex::new_cfg(self.get_change_merge_cfg()));
-
                 vec.push(self.change_to_export_format(change));
             }
         }

crates/loro-core/src/log_store/encode_updates.rs

@@ -0,0 +1,191 @@
+use rle::{HasLength, RleVec};
+use serde::{Deserialize, Serialize};
+use smallvec::SmallVec;
+
+use crate::{
+    change::{Change, Lamport, Timestamp},
+    container::ContainerID,
+    id::{ClientID, Counter, ID},
+    op::{RemoteContent, RemoteOp},
+    LogStore, LoroCore, VersionVector,
+};
+
+use super::RemoteClientChanges;
+
+#[derive(Serialize, Deserialize, Debug)]
+struct Updates {
+    changes: Vec<EncodedClientChanges>,
+}
+
+/// the continuous changes from the same client
+#[derive(Serialize, Deserialize, Debug)]
+struct EncodedClientChanges {
+    meta: FirstChangeInfo,
+    data: Vec<EncodedChange>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct FirstChangeInfo {
+    pub(crate) client: ClientID,
+    pub(crate) counter: Counter,
+    pub(crate) lamport: Lamport,
+    pub(crate) timestamp: Timestamp,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct EncodedOp {
+    pub(crate) container: ContainerID,
+    pub(crate) contents: Vec<RemoteContent>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct EncodedChange {
+    pub(crate) ops: Vec<EncodedOp>,
+    pub(crate) deps_except_self: Vec<ID>,
+    pub(crate) lamport_delta: u32,
+    pub(crate) timestamp_delta: i64,
+}
+
+impl LogStore {
+    pub fn encode_updates(&self, from: &VersionVector) -> Result<Vec<u8>, postcard::Error> {
+        let changes = self.export(from);
+        let mut updates = Updates {
+            changes: Vec::with_capacity(changes.len()),
+        };
+        for (_, changes) in changes {
+            let encoded = convert_changes_to_encoded(changes.into_iter());
+            updates.changes.push(encoded);
+        }
+
+        postcard::to_allocvec(&updates)
+    }
+
+    pub fn import_updates(&mut self, input: &[u8]) -> Result<(), postcard::Error> {
+        let updates: Updates = postcard::from_bytes(input)?;
+        let mut changes: RemoteClientChanges = Default::default();
+        for encoded in updates.changes {
+            changes.insert(encoded.meta.client, convert_encoded_to_changes(encoded));
+        }
+
+        self.import(changes);
+        Ok(())
+    }
+}
+
+fn convert_changes_to_encoded<I>(mut changes: I) -> EncodedClientChanges
+where
+    I: Iterator<Item = Change<RemoteOp>>,
+{
+    let first_change = changes.next().unwrap();
+    let this_client_id = first_change.id.client_id;
+    let mut data = Vec::with_capacity(changes.size_hint().0 + 1);
+    let mut last_change = first_change.clone();
+    data.push(EncodedChange {
+        ops: first_change
+            .ops
+            .iter()
+            .map(|op| EncodedOp {
+                container: op.container.clone(),
+                contents: op.contents.iter().cloned().collect(),
+            })
+            .collect(),
+        deps_except_self: first_change
+            .deps
+            .iter()
+            .filter(|x| x.client_id != this_client_id)
+            .copied()
+            .collect(),
+        lamport_delta: 0,
+        timestamp_delta: 0,
+    });
+    for change in changes {
+        data.push(EncodedChange {
+            ops: change
+                .ops
+                .iter()
+                .map(|op| EncodedOp {
+                    container: op.container.clone(),
+                    contents: op.contents.iter().cloned().collect(),
+                })
+                .collect(),
+            deps_except_self: change
+                .deps
+                .iter()
+                .filter(|x| x.client_id != this_client_id)
+                .copied()
+                .collect(),
+            lamport_delta: change.lamport - last_change.lamport,
+            timestamp_delta: change.timestamp - last_change.timestamp,
+        });
+        last_change = change;
+    }
+
+    EncodedClientChanges {
+        meta: FirstChangeInfo {
+            client: this_client_id,
+            counter: first_change.id.counter,
+            lamport: first_change.lamport,
+            timestamp: first_change.timestamp,
+        },
+        data,
+    }
+}
+
+fn convert_encoded_to_changes(changes: EncodedClientChanges) -> Vec<Change<RemoteOp>> {
+    let mut result = Vec::with_capacity(changes.data.len());
+    let mut last_lamport = changes.meta.lamport;
+    let mut last_timestamp = changes.meta.timestamp;
+    let mut counter: Counter = changes.meta.counter;
+    for encoded in changes.data {
+        let start_counter = counter;
+        let mut deps = SmallVec::with_capacity(encoded.deps_except_self.len() + 1);
+        if start_counter > 0 {
+            deps.push(ID {
+                client_id: changes.meta.client,
+                counter: start_counter - 1,
+            });
+        }
+
+        for dep in encoded.deps_except_self {
+            deps.push(dep);
+        }
+
+        let mut ops = RleVec::with_capacity(encoded.ops.len());
+        for op in encoded.ops {
+            let len: usize = op.contents.iter().map(|x| x.atom_len()).sum();
+            ops.push(RemoteOp {
+                counter,
+                container: op.container,
+                contents: op.contents.into_iter().collect(),
+            });
+            counter += len as Counter;
+        }
+
+        let change = Change {
+            id: ID {
+                client_id: changes.meta.client,
+                counter: start_counter,
+            },
+            lamport: last_lamport + encoded.lamport_delta,
+            timestamp: last_timestamp + encoded.timestamp_delta,
+            ops,
+            deps,
+        };
+        last_lamport = change.lamport;
+        last_timestamp = change.timestamp;
+        result.push(change);
+    }
+
+    result
+}
+
+impl LoroCore {
+    pub fn export_updates(&self, from: &VersionVector) -> Result<Vec<u8>, postcard::Error> {
+        self.log_store.read().unwrap().encode_updates(from)
+    }
+
+    pub fn import_updates(&mut self, input: &[u8]) -> Result<(), postcard::Error> {
+        let ans = self.log_store.write().unwrap().import_updates(input);
+        ans
+    }
+}

crates/loro-core/src/log_store/encoding.rs

@@ -20,7 +20,6 @@ use crate::{
     dag::remove_included_frontiers,
     id::{ClientID, Counter, ID},
     op::{Op, RemoteContent, RemoteOp},
-    smstring::SmString,
     span::{HasIdSpan, HasLamportSpan},
     ContainerType, InternalString, LogStore, LoroValue, VersionVector,
 };
@@ -156,7 +155,6 @@ fn encode_changes(store: &LogStore) -> Encoded {
                                 (span.pos as usize, 0, LoroValue::I32(span.len as i32))
                             }
                         },
-                        crate::op::RemoteContent::Dyn(_) => unreachable!(),
                     };
                     op_len += 1;
                     ops.push(OpEncoding {
@@ -274,7 +272,7 @@ fn decode_changes(
                         },
                         _ => {
                             let slice = match value {
-                                LoroValue::String(s) => ListSlice::RawStr(SmString::from(&*s)),
+                                LoroValue::String(s) => ListSlice::RawStr(s.into()),
                                 LoroValue::List(v) => ListSlice::RawData(*v),
                                 _ => unreachable!(),
                             };

crates/loro-core/src/log_store/import.rs

@@ -9,7 +9,7 @@ use tracing::instrument;
 
 use fxhash::FxHashMap;
 
-use rle::{HasLength, RleVecWithIndex, Sliceable};
+use rle::{slice_vec_by, HasLength, RleVecWithIndex, Sliceable};
 
 use crate::{
     container::{registry::ContainerInstance, Container, ContainerID},
@@ -329,9 +329,11 @@ impl LogStore {
             let other_start_ctr = changes.first().unwrap().ctr_start();
             match other_start_ctr.cmp(&self_end_ctr) {
                 std::cmp::Ordering::Less => {
-                    *changes = changes.slice(
+                    *changes = slice_vec_by(
-                        (self_end_ctr - other_start_ctr) as usize,
+                        changes,
-                        changes.atom_len(),
+                        |x| x.id.counter as usize,
+                        self_end_ctr as usize,
+                        usize::MAX,
                     );
                 }
                 std::cmp::Ordering::Equal => {}

crates/loro-core/src/loro.rs

@@ -2,10 +2,10 @@ use std::sync::{Arc, RwLock};
 
 use crate::LoroValue;
 use fxhash::FxHashMap;
-use rle::RleVecWithIndex;
+
 
 use crate::{
-    change::{Change, ChangeMergeCfg},
+    change::{Change},
     configure::Configure,
     container::{list::List, map::Map, text::Text, ContainerIdRaw, ContainerType},
     event::{Observer, SubscriptionID},
@@ -72,18 +72,12 @@ impl LoroCore {
         Text::from_instance(instance, cid)
     }
 
-    pub fn export(
+    pub fn export(&self, remote_vv: VersionVector) -> FxHashMap<u64, Vec<Change<RemoteOp>>> {
-        &self,
-        remote_vv: VersionVector,
-    ) -> FxHashMap<u64, RleVecWithIndex<Change<RemoteOp>, ChangeMergeCfg>> {
         let store = self.log_store.read().unwrap();
         store.export(&remote_vv)
     }
 
-    pub fn import(
+    pub fn import(&mut self, changes: FxHashMap<u64, Vec<Change<RemoteOp>>>) {
-        &mut self,
-        changes: FxHashMap<u64, RleVecWithIndex<Change<RemoteOp>, ChangeMergeCfg>>,
-    ) {
         let mut store = self.log_store.write().unwrap();
         store.import(changes)
     }

crates/loro-core/src/op/content.rs

@@ -2,6 +2,7 @@ use std::any::{Any, TypeId};
 
 use enum_as_inner::EnumAsInner;
 use rle::{HasLength, Mergable, Sliceable};
+use serde::{Deserialize, Serialize};
 
 use crate::container::{
     list::list_op::{InnerListOp, ListOp},
@@ -24,11 +25,10 @@ pub enum InnerContent {
     Map(InnerMapSet),
 }
 
-#[derive(EnumAsInner, Debug)]
+#[derive(EnumAsInner, Debug, Serialize, Deserialize)]
 pub enum RemoteContent {
     Map(MapSet),
     List(ListOp),
-    Dyn(Box<dyn InsertContentTrait>),
 }
 
 impl Clone for RemoteContent {
@@ -36,7 +36,6 @@ impl Clone for RemoteContent {
         match self {
             Self::Map(arg0) => Self::Map(arg0.clone()),
             Self::List(arg0) => Self::List(arg0.clone()),
-            Self::Dyn(arg0) => Self::Dyn(arg0.clone_content()),
         }
     }
 }
@@ -93,7 +92,6 @@ impl HasLength for RemoteContent {
     fn content_len(&self) -> usize {
         match self {
             RemoteContent::Map(x) => x.content_len(),
-            RemoteContent::Dyn(x) => x.content_len(),
             RemoteContent::List(x) => x.content_len(),
         }
     }
@@ -103,7 +101,6 @@ impl Sliceable for RemoteContent {
     fn slice(&self, from: usize, to: usize) -> Self {
         match self {
             RemoteContent::Map(x) => RemoteContent::Map(x.slice(from, to)),
-            RemoteContent::Dyn(x) => RemoteContent::Dyn(x.slice_content(from, to)),
             RemoteContent::List(x) => RemoteContent::List(x.slice(from, to)),
         }
     }
@@ -117,7 +114,6 @@ impl Mergable for RemoteContent {
         match (self, other) {
             (RemoteContent::Map(x), RemoteContent::Map(y)) => x.is_mergable(y, &()),
             (RemoteContent::List(x), RemoteContent::List(y)) => x.is_mergable(y, &()),
-            (RemoteContent::Dyn(x), RemoteContent::Dyn(y)) => x.is_mergable_content(&**y),
             _ => false,
         }
     }
@@ -135,7 +131,6 @@ impl Mergable for RemoteContent {
                 RemoteContent::List(y) => x.merge(y, &()),
                 _ => unreachable!(),
             },
-            RemoteContent::Dyn(x) => x.merge_content(&**_other.as_dyn().unwrap()),
         }
     }
 }

crates/loro-core/src/smstring.rs

@@ -3,6 +3,7 @@ use std::ops::DerefMut;
 use std::ops::Deref;
 
 use rle::Mergable;
+use serde::Deserialize;
 use serde::Serialize;
 use smartstring::LazyCompact;
 
@@ -63,6 +64,13 @@ impl From<String> for SmString {
     }
 }
 
+impl From<Box<str>> for SmString {
+    fn from(s: Box<str>) -> Self {
+        let s: &str = &s;
+        SmString(s.into())
+    }
+}
+
 impl From<&str> for SmString {
     fn from(s: &str) -> Self {
         SmString(s.into())
@@ -77,3 +85,13 @@ impl Serialize for SmString {
         serializer.serialize_str(&self.0)
     }
 }
+
+impl<'de> Deserialize<'de> for SmString {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        Ok(s.into())
+    }
+}

crates/loro-core/src/value.rs

@@ -6,7 +6,6 @@ use serde::{de::VariantAccess, ser::SerializeStruct, Deserialize, Serialize};
 
 use crate::{
     container::{registry::ContainerRegistry, ContainerID},
-    context::Context,
     delta::DeltaItem,
     event::{Diff, Index, Path},
     Container,

crates/loro-core/tests/test.rs

@@ -1,4 +1,3 @@
-use std::borrow::{Borrow, BorrowMut};
 use std::cell::RefCell;
 use std::rc::Rc;
 

crates/rle/src/lib.rs

@@ -27,7 +27,7 @@ pub mod rle_tree;
 mod rle_vec;
 mod rle_vec_old;
 pub use crate::rle_trait::{HasIndex, HasLength, Mergable, Rle, Slice, Sliceable, ZeroElement};
-pub use crate::rle_vec::{RleVec, RleVecWithLen};
+pub use crate::rle_vec::{slice_vec_by, RleVec, RleVecWithLen};
 pub use crate::rle_vec_old::{RleVecWithIndex, SearchResult, SliceIterator};
 pub mod rle_impl;
 pub use rle_tree::tree_trait::RleTreeTrait;

crates/rle/src/rle_vec.rs

@@ -555,6 +555,35 @@ impl<A: Array> Deref for RleVecWithLen<A> {
     }
 }
 
+pub fn slice_vec_by<T, F>(vec: &Vec<T>, index: F, start: usize, end: usize) -> Vec<T>
+where
+    F: Fn(&T) -> usize,
+    T: Sliceable + HasLength,
+{
+    if start >= end || vec.is_empty() {
+        return Vec::new();
+    }
+
+    let start = start - index(&vec[0]);
+    let end = end - index(&vec[0]);
+    let mut ans = Vec::new();
+    let mut index = 0;
+    for i in 0..vec.len() {
+        if index >= end {
+            break;
+        }
+
+        let len = vec[i].atom_len();
+        if start < index + len {
+            ans.push(vec[i].slice(start.saturating_sub(index), (end - index).min(len)))
+        }
+
+        index += len;
+    }
+
+    ans
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

crates/rle/src/rle_vec_old.rs

@@ -1,4 +1,7 @@
-use std::ops::{Deref, Range};
+use std::{
+    ops::{Deref, Range},
+    vec,
+};
 
 use num::Integer;
 
@@ -213,6 +216,16 @@ impl<T, Conf> RleVecWithIndex<T, Conf> {
     }
 }
 
+impl<T, Cfg> IntoIterator for RleVecWithIndex<T, Cfg> {
+    type Item = T;
+
+    type IntoIter = vec::IntoIter<T>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.vec.into_iter()
+    }
+}
+
 impl<T> Default for RleVecWithIndex<T> {
     fn default() -> Self {
         Self::new()