From a941cddfb7b36456c0312222370e811bfe44abaa Mon Sep 17 00:00:00 2001 From: Leon Zhao Date: Tue, 11 Jun 2024 20:08:03 +0800 Subject: [PATCH] feat: export any range version with json schema (#383) --- crates/fuzz/tests/json.rs | 6 +- crates/loro-internal/benches/encode.rs | 4 +- crates/loro-internal/examples/encoding.rs | 3 +- .../examples/encoding_refactored.rs | 3 +- .../loro-internal/src/encoding/json_schema.rs | 83 ++++++++++++++----- crates/loro-internal/src/loro.rs | 8 +- crates/loro-wasm/src/lib.rs | 18 ++-- crates/loro/src/lib.rs | 8 +- 8 files changed, 97 insertions(+), 36 deletions(-) diff --git a/crates/fuzz/tests/json.rs b/crates/fuzz/tests/json.rs index 99a2f8c0..70158ec3 100644 --- a/crates/fuzz/tests/json.rs +++ b/crates/fuzz/tests/json.rs @@ -30,7 +30,8 @@ fn unknown_json() { let doc3_without_counter = loro_without_counter::LoroDoc::new(); // Test2: older version import newer version snapshot with counter doc3_without_counter.import(&snapshot_with_counter).unwrap(); - let unknown_json_from_snapshot = doc3_without_counter.export_json_updates(&Default::default()); + let unknown_json_from_snapshot = doc3_without_counter + .export_json_updates(&Default::default(), &doc3_without_counter.oplog_vv()); // { // "container": "cid:root-counter:Unknown(5)", // "content": { @@ -42,7 +43,8 @@ fn unknown_json() { // "counter": 0 // } // Test3: older version export json with binary unknown - let _json_with_binary_unknown = doc3_without_counter.export_json_updates(&Default::default()); + let _json_with_binary_unknown = doc3_without_counter + .export_json_updates(&Default::default(), &doc3_without_counter.oplog_vv()); let new_doc = loro::LoroDoc::new(); // Test4: newer version import older version json with binary unknown if new_doc diff --git a/crates/loro-internal/benches/encode.rs b/crates/loro-internal/benches/encode.rs index 62e2a90b..42509fb9 100644 --- a/crates/loro-internal/benches/encode.rs +++ b/crates/loro-internal/benches/encode.rs @@ -111,12 +111,12 @@ mod run { b.bench_function("B4_encode_json_update", |b| { ensure_ran(); b.iter(|| { - let _ = loro.export_json_updates(&Default::default()); + let _ = loro.export_json_updates(&Default::default(), &loro.oplog_vv()); }) }); b.bench_function("B4_decode_json_update", |b| { ensure_ran(); - let json = loro.export_json_updates(&Default::default()); + let json = loro.export_json_updates(&Default::default(), &loro.oplog_vv()); b.iter(|| { let store2 = LoroDoc::default(); store2.import_json_updates(json.clone()).unwrap(); diff --git a/crates/loro-internal/examples/encoding.rs b/crates/loro-internal/examples/encoding.rs index b4952d74..0c8c560c 100644 --- a/crates/loro-internal/examples/encoding.rs +++ b/crates/loro-internal/examples/encoding.rs @@ -63,7 +63,8 @@ fn main() { ); let json_updates = - serde_json::to_string(&loro.export_json_updates(&Default::default())).unwrap(); + serde_json::to_string(&loro.export_json_updates(&Default::default(), &loro.oplog_vv())) + .unwrap(); let output = miniz_oxide::deflate::compress_to_vec(json_updates.as_bytes(), 6); println!( "json updates size {} after compression {}", diff --git a/crates/loro-internal/examples/encoding_refactored.rs b/crates/loro-internal/examples/encoding_refactored.rs index 30225aa4..15650642 100644 --- a/crates/loro-internal/examples/encoding_refactored.rs +++ b/crates/loro-internal/examples/encoding_refactored.rs @@ -24,7 +24,8 @@ fn log_size() { let snapshot = loro.export_snapshot(); let updates = loro.export_from(&Default::default()); let json_updates = - serde_json::to_string(&loro.export_json_updates(&Default::default())).unwrap(); + serde_json::to_string(&loro.export_json_updates(&Default::default(), &loro.oplog_vv())) + .unwrap(); println!("\n"); println!("Snapshot size={}", snapshot.len()); println!("Updates size={}", updates.len()); diff --git a/crates/loro-internal/src/encoding/json_schema.rs b/crates/loro-internal/src/encoding/json_schema.rs index 4cbfaa1e..8ba56943 100644 --- a/crates/loro-internal/src/encoding/json_schema.rs +++ b/crates/loro-internal/src/encoding/json_schema.rs @@ -22,27 +22,34 @@ use op::{JsonOpContent, JsonSchema}; const SCHEMA_VERSION: u8 = 1; -pub(crate) fn export_json<'a, 'c: 'a>(oplog: &'c OpLog, vv: &VersionVector) -> JsonSchema { - let actual_start_vv: VersionVector = vv - .iter() - .filter_map(|(&peer, &end_counter)| { - if end_counter == 0 { - return None; - } +fn refine_vv(vv: &VersionVector, oplog: &OpLog) -> VersionVector { + let mut refined = VersionVector::new(); + for (peer, counter) in vv.iter() { + if counter == &0 { + continue; + } + let end = oplog.vv().get(peer).copied().unwrap_or(0); + if end <= *counter { + refined.insert(*peer, end); + } else { + refined.insert(*peer, *counter); + } + } + refined +} - let this_end = oplog.vv().get(&peer).cloned().unwrap_or(0); - if this_end <= end_counter { - return Some((peer, this_end)); - } - - Some((peer, end_counter)) - }) - .collect(); +pub(crate) fn export_json<'a, 'c: 'a>( + oplog: &'c OpLog, + start_vv: &VersionVector, + end_vv: &VersionVector, +) -> JsonSchema { + let actual_start_vv = refine_vv(start_vv, oplog); + let actual_end_vv = refine_vv(end_vv, oplog); let frontiers = oplog.dag.vv_to_frontiers(&actual_start_vv); let mut peer_register = ValueRegister::::new(); - let diff_changes = init_encode(oplog, &actual_start_vv); + let diff_changes = init_encode(oplog, &actual_start_vv, &actual_end_vv); let changes = encode_changes(&diff_changes, &oplog.arena, &mut peer_register); JsonSchema { changes, @@ -62,15 +69,24 @@ pub(crate) fn import_json(oplog: &mut OpLog, json: JsonSchema) -> LoroResult<()> Ok(()) } -fn init_encode<'s, 'a: 's>(oplog: &'a OpLog, vv: &'_ VersionVector) -> Vec> { - let self_vv = oplog.vv(); - let start_vv = vv.trim(oplog.vv()); +fn init_encode<'s, 'a: 's>( + oplog: &'a OpLog, + start_vv: &VersionVector, + end_vv: &VersionVector, +) -> Vec> { let mut diff_changes: Vec> = Vec::new(); - for change in oplog.iter_changes_peer_by_peer(&start_vv, self_vv) { + for change in oplog.iter_changes_peer_by_peer(start_vv, end_vv) { let start_cnt = start_vv.get(&change.id.peer).copied().unwrap_or(0); + let end_cnt = end_vv.get(&change.id.peer).copied().unwrap_or(0); if change.id.counter < start_cnt { let offset = start_cnt - change.id.counter; - diff_changes.push(Cow::Owned(change.slice(offset as usize, change.atom_len()))); + let to = change + .atom_len() + .min((end_cnt - change.id.counter) as usize); + diff_changes.push(Cow::Owned(change.slice(offset as usize, to))); + } else if change.id.counter + change.atom_len() as i32 > end_cnt { + let len = end_cnt - change.id.counter; + diff_changes.push(Cow::Owned(change.slice(0, len as usize))); } else { diff_changes.push(Cow::Borrowed(change)); } @@ -1173,3 +1189,28 @@ pub mod op { } } } + +#[cfg(test)] +mod tests { + use crate::{LoroDoc, VersionVector}; + + #[test] + fn json_range_version() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0).unwrap(); + let list = doc.get_list("list"); + list.insert(0, "a").unwrap(); + list.insert(0, "b").unwrap(); + list.insert(0, "c").unwrap(); + let json = doc.export_json_updates( + &VersionVector::from_iter(vec![(0, 1)]), + &VersionVector::from_iter(vec![(0, 2)]), + ); + assert_eq!(json.changes[0].ops.len(), 1); + let json = doc.export_json_updates( + &VersionVector::from_iter(vec![(0, 0)]), + &VersionVector::from_iter(vec![(0, 2)]), + ); + assert_eq!(json.changes[0].ops.len(), 2); + } +} diff --git a/crates/loro-internal/src/loro.rs b/crates/loro-internal/src/loro.rs index c25f2cdd..67dfefa2 100644 --- a/crates/loro-internal/src/loro.rs +++ b/crates/loro-internal/src/loro.rs @@ -586,10 +586,14 @@ impl LoroDoc { Ok(()) } - pub fn export_json_updates(&self, vv: &VersionVector) -> JsonSchema { + pub fn export_json_updates( + &self, + start_vv: &VersionVector, + end_vv: &VersionVector, + ) -> JsonSchema { self.commit_then_stop(); let oplog = self.oplog.lock().unwrap(); - let json = crate::encoding::json_schema::export_json(&oplog, vv); + let json = crate::encoding::json_schema::export_json(&oplog, start_vv, end_vv); drop(oplog); self.renew_txn_if_auto_commit(); json diff --git a/crates/loro-wasm/src/lib.rs b/crates/loro-wasm/src/lib.rs index 7835f657..2a05df1c 100644 --- a/crates/loro-wasm/src/lib.rs +++ b/crates/loro-wasm/src/lib.rs @@ -879,12 +879,20 @@ impl Loro { /// Export updates from the specific version to the current version with JSON format. #[wasm_bindgen(js_name = "exportJsonUpdates")] - pub fn export_json_updates(&self, vv: Option) -> JsResult { - let mut json_vv = Default::default(); - if let Some(vv) = vv { - json_vv = vv.0; + pub fn export_json_updates( + &self, + start_vv: Option, + end_vv: Option, + ) -> JsResult { + let mut json_start_vv = Default::default(); + if let Some(vv) = start_vv { + json_start_vv = vv.0; } - let json_schema = self.0.export_json_updates(&json_vv); + let mut json_end_vv = self.oplog_version().0; + if let Some(vv) = end_vv { + json_end_vv = vv.0; + } + let json_schema = self.0.export_json_updates(&json_start_vv, &json_end_vv); let s = serde_wasm_bindgen::Serializer::new(); let v = json_schema .serialize(&s) diff --git a/crates/loro/src/lib.rs b/crates/loro/src/lib.rs index 1f2202e7..34796674 100644 --- a/crates/loro/src/lib.rs +++ b/crates/loro/src/lib.rs @@ -298,8 +298,12 @@ impl LoroDoc { } /// Export the current state with json-string format of the document. - pub fn export_json_updates(&self, vv: &VersionVector) -> JsonSchema { - self.doc.export_json_updates(vv) + pub fn export_json_updates( + &self, + start_vv: &VersionVector, + end_vv: &VersionVector, + ) -> JsonSchema { + self.doc.export_json_updates(start_vv, end_vv) } /// Export all the ops not included in the given `VersionVector`