mirrors/loro
1
0
Fork 0
mirror of https://github.com/loro-dev/loro.git synced 2024-11-28 09:25:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

feat: export any range version with json schema (#383)

Browse source
This commit is contained in:
Leon Zhao 2024-06-11 20:08:03 +08:00 committed by GitHub
parent 881167b18e
commit a941cddfb7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 97 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
crates/fuzz/tests/json.rs
View file

@ -30,7 +30,8 @@ fn unknown_json() {
let doc3_without_counter = loro_without_counter::LoroDoc::new();
// Test2: older version import newer version snapshot with counter
doc3_without_counter.import(&snapshot_with_counter).unwrap();
let unknown_json_from_snapshot = doc3_without_counter.export_json_updates(&Default::default());
let unknown_json_from_snapshot = doc3_without_counter
.export_json_updates(&Default::default(), &doc3_without_counter.oplog_vv());
// {
// "container": "cid:root-counter:Unknown(5)",
// "content": {
@ -42,7 +43,8 @@ fn unknown_json() {
// "counter": 0
// }
// Test3: older version export json with binary unknown
let _json_with_binary_unknown = doc3_without_counter.export_json_updates(&Default::default());
let _json_with_binary_unknown = doc3_without_counter
.export_json_updates(&Default::default(), &doc3_without_counter.oplog_vv());
let new_doc = loro::LoroDoc::new();
// Test4: newer version import older version json with binary unknown
if new_doc

4
crates/loro-internal/benches/encode.rs
View file

@ -111,12 +111,12 @@ mod run {
b.bench_function("B4_encode_json_update", |b| {
ensure_ran();
b.iter(|| {
let _ = loro.export_json_updates(&Default::default());
let _ = loro.export_json_updates(&Default::default(), &loro.oplog_vv());
})
});
b.bench_function("B4_decode_json_update", |b| {
ensure_ran();
let json = loro.export_json_updates(&Default::default());
let json = loro.export_json_updates(&Default::default(), &loro.oplog_vv());
b.iter(|| {
let store2 = LoroDoc::default();
store2.import_json_updates(json.clone()).unwrap();

3
crates/loro-internal/examples/encoding.rs
View file

@ -63,7 +63,8 @@ fn main() {
);
let json_updates =
serde_json::to_string(&loro.export_json_updates(&Default::default())).unwrap();
serde_json::to_string(&loro.export_json_updates(&Default::default(), &loro.oplog_vv()))
.unwrap();
let output = miniz_oxide::deflate::compress_to_vec(json_updates.as_bytes(), 6);
println!(
"json updates size {} after compression {}",

3
crates/loro-internal/examples/encoding_refactored.rs
View file

@ -24,7 +24,8 @@ fn log_size() {
let snapshot = loro.export_snapshot();
let updates = loro.export_from(&Default::default());
let json_updates =
serde_json::to_string(&loro.export_json_updates(&Default::default())).unwrap();
serde_json::to_string(&loro.export_json_updates(&Default::default(), &loro.oplog_vv()))
.unwrap();
println!("\n");
println!("Snapshot size={}", snapshot.len());
println!("Updates size={}", updates.len());

83
crates/loro-internal/src/encoding/json_schema.rs
View file

@ -22,27 +22,34 @@ use op::{JsonOpContent, JsonSchema};
const SCHEMA_VERSION: u8 = 1;
pub(crate) fn export_json<'a, 'c: 'a>(oplog: &'c OpLog, vv: &VersionVector) -> JsonSchema {
let actual_start_vv: VersionVector = vv
.iter()
.filter_map(|(&peer, &end_counter)| {
if end_counter == 0 {
return None;
}
fn refine_vv(vv: &VersionVector, oplog: &OpLog) -> VersionVector {
let mut refined = VersionVector::new();
for (peer, counter) in vv.iter() {
if counter == &0 {
continue;
}
let end = oplog.vv().get(peer).copied().unwrap_or(0);
if end <= *counter {
refined.insert(*peer, end);
} else {
refined.insert(*peer, *counter);
}
}
refined
}
let this_end = oplog.vv().get(&peer).cloned().unwrap_or(0);
if this_end <= end_counter {
return Some((peer, this_end));
}
Some((peer, end_counter))
})
.collect();
pub(crate) fn export_json<'a, 'c: 'a>(
oplog: &'c OpLog,
start_vv: &VersionVector,
end_vv: &VersionVector,
) -> JsonSchema {
let actual_start_vv = refine_vv(start_vv, oplog);
let actual_end_vv = refine_vv(end_vv, oplog);
let frontiers = oplog.dag.vv_to_frontiers(&actual_start_vv);
let mut peer_register = ValueRegister::<PeerID>::new();
let diff_changes = init_encode(oplog, &actual_start_vv);
let diff_changes = init_encode(oplog, &actual_start_vv, &actual_end_vv);
let changes = encode_changes(&diff_changes, &oplog.arena, &mut peer_register);
JsonSchema {
changes,
@ -62,15 +69,24 @@ pub(crate) fn import_json(oplog: &mut OpLog, json: JsonSchema) -> LoroResult<()>
Ok(())
}
fn init_encode<'s, 'a: 's>(oplog: &'a OpLog, vv: &'_ VersionVector) -> Vec<Cow<'s, Change>> {
let self_vv = oplog.vv();
let start_vv = vv.trim(oplog.vv());
fn init_encode<'s, 'a: 's>(
oplog: &'a OpLog,
start_vv: &VersionVector,
end_vv: &VersionVector,
) -> Vec<Cow<'s, Change>> {
let mut diff_changes: Vec<Cow<'a, Change>> = Vec::new();
for change in oplog.iter_changes_peer_by_peer(&start_vv, self_vv) {
for change in oplog.iter_changes_peer_by_peer(start_vv, end_vv) {
let start_cnt = start_vv.get(&change.id.peer).copied().unwrap_or(0);
let end_cnt = end_vv.get(&change.id.peer).copied().unwrap_or(0);
if change.id.counter < start_cnt {
let offset = start_cnt - change.id.counter;
diff_changes.push(Cow::Owned(change.slice(offset as usize, change.atom_len())));
let to = change
.atom_len()
.min((end_cnt - change.id.counter) as usize);
diff_changes.push(Cow::Owned(change.slice(offset as usize, to)));
} else if change.id.counter + change.atom_len() as i32 > end_cnt {
let len = end_cnt - change.id.counter;
diff_changes.push(Cow::Owned(change.slice(0, len as usize)));
} else {
diff_changes.push(Cow::Borrowed(change));
}
@ -1173,3 +1189,28 @@ pub mod op {
}
}
}
#[cfg(test)]
mod tests {
use crate::{LoroDoc, VersionVector};
#[test]
fn json_range_version() {
let doc = LoroDoc::new_auto_commit();
doc.set_peer_id(0).unwrap();
let list = doc.get_list("list");
list.insert(0, "a").unwrap();
list.insert(0, "b").unwrap();
list.insert(0, "c").unwrap();
let json = doc.export_json_updates(
&VersionVector::from_iter(vec![(0, 1)]),
&VersionVector::from_iter(vec![(0, 2)]),
);
assert_eq!(json.changes[0].ops.len(), 1);
let json = doc.export_json_updates(
&VersionVector::from_iter(vec![(0, 0)]),
&VersionVector::from_iter(vec![(0, 2)]),
);
assert_eq!(json.changes[0].ops.len(), 2);
}
}

8
crates/loro-internal/src/loro.rs
View file

@ -586,10 +586,14 @@ impl LoroDoc {
Ok(())
}
pub fn export_json_updates(&self, vv: &VersionVector) -> JsonSchema {
pub fn export_json_updates(
&self,
start_vv: &VersionVector,
end_vv: &VersionVector,
) -> JsonSchema {
self.commit_then_stop();
let oplog = self.oplog.lock().unwrap();
let json = crate::encoding::json_schema::export_json(&oplog, vv);
let json = crate::encoding::json_schema::export_json(&oplog, start_vv, end_vv);
drop(oplog);
self.renew_txn_if_auto_commit();
json

18
crates/loro-wasm/src/lib.rs
View file

@ -879,12 +879,20 @@ impl Loro {
/// Export updates from the specific version to the current version with JSON format.
#[wasm_bindgen(js_name = "exportJsonUpdates")]
pub fn export_json_updates(&self, vv: Option<VersionVector>) -> JsResult<JsJsonSchema> {
let mut json_vv = Default::default();
if let Some(vv) = vv {
json_vv = vv.0;
pub fn export_json_updates(
&self,
start_vv: Option<VersionVector>,
end_vv: Option<VersionVector>,
) -> JsResult<JsJsonSchema> {
let mut json_start_vv = Default::default();
if let Some(vv) = start_vv {
json_start_vv = vv.0;
}
let json_schema = self.0.export_json_updates(&json_vv);
let mut json_end_vv = self.oplog_version().0;
if let Some(vv) = end_vv {
json_end_vv = vv.0;
}
let json_schema = self.0.export_json_updates(&json_start_vv, &json_end_vv);
let s = serde_wasm_bindgen::Serializer::new();
let v = json_schema
.serialize(&s)

8
crates/loro/src/lib.rs
View file

@ -298,8 +298,12 @@ impl LoroDoc {
}
/// Export the current state with json-string format of the document.
pub fn export_json_updates(&self, vv: &VersionVector) -> JsonSchema {
self.doc.export_json_updates(vv)
pub fn export_json_updates(
&self,
start_vv: &VersionVector,
end_vv: &VersionVector,
) -> JsonSchema {
self.doc.export_json_updates(start_vv, end_vv)
}
/// Export all the ops not included in the given `VersionVector`