mirrors/loro
1
0
Fork 0
mirror of https://github.com/loro-dev/loro.git synced 2025-01-22 12:57:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Test fuzz oom (#508)

Browse source 
* test: avoid fuzz oom

* fix: get pending dag node err
This commit is contained in:
Zixuan Chen 2024-10-12 00:49:23 +08:00 committed by GitHub
parent b5e153a33d
commit 23a99f53c9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 332 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

256
crates/fuzz/examples/oom.rs Normal file
View file

@ -0,0 +1,256 @@
use fuzz::{
actions::{ActionWrapper::*, GenericAction},
crdt_fuzzer::{Action::*, FuzzValue::*},
test_multi_sites_on_one_doc,
};
use loro::ContainerType::*;
pub fn main() {
test_multi_sites_on_one_doc(
5,
&mut [
Handle {
site: 1,
target: 0,
container: 0,
action: Generic(GenericAction {
value: I32(53423135),
bool: true,
key: 3271672771,
pos: 72391498414670731,
length: 2534382878240390,
prop: 12587145347420061696,
}),
},
Handle {
site: 0,
target: 0,
container: 0,
action: Generic(GenericAction {
value: Container(Tree),
bool: true,
key: 2248212783,
pos: 10886371984050945,
length: 3393509701437167640,
prop: 2821266740684986392,
}),
},
Handle {
site: 39,
target: 39,
container: 126,
action: Generic(GenericAction {
value: I32(-132),
bool: true,
key: 4281597952,
pos: 9899933171711,
length: 2821266740684990247,
prop: 2821266740684990247,
}),
},
SyncAll,
Sync { from: 191, to: 195 },
Checkout {
site: 1,
to: 783189505,
},
Handle {
site: 195,
target: 195,
container: 195,
action: Generic(GenericAction {
value: I32(842347833),
bool: false,
key: 3351758643,
pos: 13382947428572317639,
length: 14395678941249124793,
prop: 13386888078246266823,
}),
},
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 199 },
Sync { from: 199, to: 199 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 52, to: 56 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
Checkout {
site: 199,
to: 3351758791,
},
Sync { from: 81, to: 199 },
Sync { from: 54, to: 57 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 199, to: 199 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
Checkout {
site: 199,
to: 3351758791,
},
Sync { from: 81, to: 199 },
Sync { from: 54, to: 57 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 199, to: 199 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 199 },
Sync { from: 199, to: 199 },
Sync { from: 185, to: 185 },
Sync { from: 199, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 199 },
Sync { from: 199, to: 199 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 52, to: 56 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
Checkout {
site: 199,
to: 3351758791,
},
Sync { from: 81, to: 199 },
Sync { from: 54, to: 57 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 199, to: 199 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
Checkout {
site: 199,
to: 3351758791,
},
Sync { from: 81, to: 199 },
Sync { from: 54, to: 57 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 199, to: 199 },
Sync { from: 185, to: 185 },
Checkout {
site: 185,
to: 3115956665,
},
Checkout {
site: 185,
to: 3115956665,
},
Sync { from: 185, to: 185 },
Sync { from: 185, to: 199 },
Sync { from: 199, to: 199 },
Sync { from: 52, to: 53 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
Checkout {
site: 199,
to: 3351758791,
},
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 199 },
Sync { from: 199, to: 199 },
Sync { from: 52, to: 53 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
Checkout {
site: 199,
to: 3351758791,
},
Sync { from: 199, to: 199 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Sync { from: 185, to: 199 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
Checkout {
site: 199,
to: 3351758791,
},
Sync { from: 185, to: 185 },
Sync { from: 185, to: 185 },
Handle {
site: 3,
target: 3,
container: 3,
action: Generic(GenericAction {
value: I32(50529027),
bool: true,
key: 50529027,
pos: 217020518514230019,
length: 217020518514230019,
prop: 217020518514230019,
}),
},
Sync { from: 185, to: 185 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
Sync { from: 199, to: 199 },
],
);
}

8
crates/fuzz/src/one_doc_fuzzer.rs
View file

@ -1,6 +1,6 @@
use loro::{ContainerType, Frontiers, LoroDoc, LoroError, TreeID};
use tabled::TableIteratorExt;
use tracing::{info, info_span};
use tracing::{info, info_span, trace};
use crate::{actions::ActionWrapper, crdt_fuzzer::FuzzValue, Action};
@ -365,8 +365,10 @@ impl OneDocFuzzer {
}
}
Action::Sync { from, to } => {
let a = self.branches[*from as usize].frontiers.clone();
self.branches[*to as usize].frontiers.extend_from_slice(&a);
trace!("vv={:?}", self.doc.oplog_vv());
let mut f = self.branches[*from as usize].frontiers.clone();
f.extend_from_slice(&self.branches[*to as usize].frontiers);
self.branches[*to as usize].frontiers = self.doc.minimize_frontiers(&f).unwrap();
}
Action::SyncAll => {
let f = self.doc.oplog_frontiers();

28
crates/fuzz/tests/test.rs
View file

@ -11727,7 +11727,7 @@ fn tree_event_parent_not_found() {
}
#[test]
fn unknown() {
fn movable_list_undo() {
test_multi_sites(
5,
vec![FuzzTarget::All],
@ -12583,6 +12583,32 @@ fn unknown() {
)
}
#[test]
fn one_doc_checkout_err() {
test_multi_sites_on_one_doc(
5,
&mut [
Handle {
site: 80,
target: 244,
container: 230,
action: Generic(GenericAction {
value: Container(Text),
bool: false,
key: 1495330430,
pos: 8708481874743882534,
length: 12040763259495246688,
prop: 503168992986692129,
}),
},
SyncAll,
Sync { from: 132, to: 248 },
SyncAll,
SyncAll,
],
)
}
#[test]
fn minify() {
minify_error(

3
crates/loro-internal/src/loro.rs
View file

@ -1068,7 +1068,8 @@ impl LoroDoc {
return Err(LoroError::SwitchToVersionBeforeShallowRoot);
}
let frontiers = shrink_frontiers(frontiers, &oplog.dag);
let frontiers = shrink_frontiers(frontiers, &oplog.dag)
.map_err(|_| LoroError::SwitchToVersionBeforeShallowRoot)?;
if from_frontiers == frontiers {
drop(oplog);
self.renew_txn_if_auto_commit();

30
crates/loro-internal/src/oplog/loro_dag.rs
View file

@ -815,20 +815,22 @@ impl Dag for AppDag {
fn get(&self, id: ID) -> Option<Self::Node> {
self.ensure_lazy_load_node(id);
let binding = self.map.try_lock().unwrap();
let x = binding.range(..=id).next_back()?;
if x.1.contains_id(id) {
// PERF: do we need to optimize clone like this?
// by adding another layer of Arc?
Some(x.1.clone())
} else {
if let Some(node) = &self.pending_txn_node {
if node.peer == id.peer && node.cnt <= id.counter {
assert!(node.cnt + node.len as Counter > id.counter);
return Some(node.clone());
}
if let Some(x) = binding.range(..=id).next_back() {
if x.1.contains_id(id) {
// PERF: do we need to optimize clone like this?
// by adding another layer of Arc?
return Some(x.1.clone());
}
None
}
if let Some(node) = &self.pending_txn_node {
if node.peer == id.peer && node.cnt <= id.counter {
assert!(node.cnt + node.len as Counter > id.counter);
return Some(node.clone());
}
}
None
}
fn vv(&self) -> VersionVector {
@ -1013,7 +1015,7 @@ impl AppDag {
return self.shallow_since_frontiers.clone();
}
shrink_frontiers(&last_ids, self)
shrink_frontiers(&last_ids, self).unwrap()
}
pub fn vv_to_frontiers(&self, vv: &VersionVector) -> Frontiers {
@ -1044,7 +1046,7 @@ impl AppDag {
return self.shallow_since_frontiers.clone();
}
shrink_frontiers(&last_ids, self)
shrink_frontiers(&last_ids, self).unwrap()
}
pub(crate) fn frontiers_to_next_lamport(&self, frontiers: &Frontiers) -> Lamport {

2
crates/loro-internal/src/txn.rs
View file

@ -398,6 +398,7 @@ impl Transaction {
self._commit()
}
#[tracing::instrument(level = "debug", skip(self))]
fn _commit(&mut self) -> Result<(), LoroError> {
if self.finished {
return Ok(());
@ -645,6 +646,7 @@ impl Transaction {
}
impl Drop for Transaction {
#[tracing::instrument(level = "debug", skip(self))]
fn drop(&mut self) {
if !self.finished {
// TODO: should we abort here or commit here?

28
crates/loro-internal/src/version.rs
View file

@ -1,4 +1,3 @@
use itertools::Itertools;
use loro_common::{HasCounter, HasCounterSpan, HasIdSpan, HasLamportSpan, IdFull, IdSpanVector};
use smallvec::smallvec;
use std::{
@ -1017,26 +1016,35 @@ impl VersionVector {
}
/// Use minimal set of ids to represent the frontiers
pub fn shrink_frontiers(last_ids: &[ID], dag: &AppDag) -> Frontiers {
#[tracing::instrument(skip(dag))]
pub fn shrink_frontiers(last_ids: &[ID], dag: &AppDag) -> Result<Frontiers, ID> {
// it only keep the ids of ops that are concurrent to each other
let mut frontiers = Frontiers::default();
if last_ids.is_empty() {
return frontiers;
return Ok(frontiers);
}
if last_ids.len() == 1 {
frontiers.push(last_ids[0]);
return frontiers;
return Ok(frontiers);
}
let mut last_ids = filter_duplicated_peer_id(last_ids)
.into_iter()
.map(|x| IdFull::new(x.peer, x.counter, dag.get_lamport(&x).unwrap()))
.collect_vec();
let mut last_ids = {
let ids = filter_duplicated_peer_id(last_ids);
let mut last_ids = Vec::with_capacity(ids.len());
for id in ids {
let Some(lamport) = dag.get_lamport(&id) else {
return Err(id);
};
last_ids.push(IdFull::new(id.peer, id.counter, lamport))
}
last_ids
};
if last_ids.len() == 1 {
frontiers.push(last_ids[0].id());
return frontiers;
return Ok(frontiers);
}
// Iterate from the greatest lamport to the smallest
@ -1065,7 +1073,7 @@ pub fn shrink_frontiers(last_ids: &[ID], dag: &AppDag) -> Frontiers {
}
}
frontiers
Ok(frontiers)
}
fn filter_duplicated_peer_id(last_ids: &[ID]) -> Vec<ID> {

6
crates/loro/src/lib.rs
View file

@ -12,6 +12,7 @@ use loro_internal::handler::HandlerTrait;
use loro_internal::handler::ValueOrHandler;
use loro_internal::loro::ChangeTravelError;
use loro_internal::undo::{OnPop, OnPush};
use loro_internal::version::shrink_frontiers;
pub use loro_internal::version::ImVersionVector;
use loro_internal::DocState;
use loro_internal::LoroDoc as InnerLoroDoc;
@ -449,6 +450,11 @@ impl LoroDoc {
self.doc.frontiers_to_vv(frontiers)
}
/// Minimize the frontiers by removing the unnecessary entries.
pub fn minimize_frontiers(&self, frontiers: &[ID]) -> Result<Frontiers, ID> {
self.with_oplog(|oplog| shrink_frontiers(frontiers, oplog.dag()))
}
/// Convert `VersionVector` into `Frontiers`
#[inline]
pub fn vv_to_frontiers(&self, vv: &VersionVector) -> Frontiers {