mirrors/loro
1
0
Fork 0
mirror of https://github.com/loro-dev/loro.git synced 2025-02-02 02:59:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

fix: gc snapshot error (#481)

Browse source 
it should retain all containers created after the `from` version
This commit is contained in:
Zixuan Chen 2024-09-25 22:35:29 +08:00 committed by GitHub
parent f1bb4f1020
commit 672062c9d1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 168 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
crates/fuzz/src/actor.rs
View file

@ -190,7 +190,7 @@ impl Actor {
let from = &self.loro.state_frontiers();
let to = &f;
let peer = self.peer;
tracing::info_span!("Checkout", ?from, ?to, ?peer).in_scope(|| {
tracing::info_span!("FuzzCheckout", ?from, ?to, ?peer).in_scope(|| {
match self.loro.checkout(&f) {
Ok(_) => {}
Err(LoroError::SwitchToTrimmedVersion) => {
@ -230,11 +230,16 @@ impl Actor {
self.loro.check_state_correctness_slow();
self.loro.checkout_to_latest();
let new_doc = LoroDoc::new();
new_doc
.import(&self.loro.export(loro::ExportMode::Snapshot))
.unwrap();
new_doc.checkout(&f).unwrap();
new_doc.check_state_correctness_slow();
info_span!("FuzzCheckoutCreatingNewSnapshotDoc",).in_scope(|| {
new_doc
.import(&self.loro.export(loro::ExportMode::Snapshot))
.unwrap();
assert_eq!(new_doc.get_deep_value(), self.loro.get_deep_value());
});
info_span!("FuzzCheckoutOnNewSnapshotDoc",).in_scope(|| {
new_doc.checkout(&f).unwrap();
new_doc.check_state_correctness_slow();
});
}
Err(LoroError::SwitchToTrimmedVersion) => {}
Err(e) => panic!("{}", e),

128
crates/fuzz/tests/test.rs
View file

@ -11047,6 +11047,132 @@ fn gc_fuzz_22() {
)
}
#[test]
fn gc_fuzz_24() {
test_multi_sites_with_gc(
5,
vec![FuzzTarget::All],
&mut [
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
SyncAll,
Handle {
site: 11,
target: 11,
container: 11,
action: Generic(GenericAction {
value: I32(185273099),
bool: true,
key: 185273099,
pos: 795741901218843403,
length: 5764338190115343115,
prop: 814957259628957519,
}),
},
Handle {
site: 0,
target: 49,
container: 0,
action: Generic(GenericAction {
value: Container(Counter),
bool: true,
key: 3351758791,
pos: 3010594536784644039,
length: 14395694394777257927,
prop: 795741901231212487,
}),
},
Handle {
site: 5,
target: 165,
container: 165,
action: Generic(GenericAction {
value: I32(84215045),
bool: true,
key: 5,
pos: 1280,
length: 361700864190383360,
prop: 361701409651229957,
}),
},
Handle {
site: 5,
target: 5,
container: 0,
action: Generic(GenericAction {
value: I32(131072),
bool: false,
key: 4294907136,
pos: 361701942142959381,
length: 14395694391509714181,
prop: 14395694394777257927,
}),
},
Handle {
site: 0,
target: 0,
container: 0,
action: Generic(GenericAction {
value: Container(Counter),
bool: true,
key: 3351758791,
pos: 14395520671940069319,
length: 14395694394777257927,
prop: 795741901218843591,
}),
},
SyncAllUndo {
site: 165,
op_len: 100663295,
},
Handle {
site: 5,
target: 0,
container: 0,
action: Generic(GenericAction {
value: I32(0),
bool: false,
key: 84215040,
pos: 361700864190383365,
length: 361700864190383492,
prop: 11936128518282651045,
}),
},
Handle {
site: 0,
target: 50,
container: 0,
action: Generic(GenericAction {
value: I32(-60160),
bool: true,
key: 4294967288,
pos: 361700864192480517,
length: 18446744073706405637,
prop: 795741901218843403,
}),
},
Handle {
site: 5,
target: 5,
container: 0,
action: Generic(GenericAction {
value: I32(0),
bool: false,
key: 83886139,
pos: 361700864190383365,
length: 361700864198706437,
prop: 11936128518282609925,
}),
},
],
)
}
#[test]
fn detached_editing_arb_test() {
fn prop(u: &mut Unstructured<'_>, site_num: u8) -> arbitrary::Result<()> {
@ -11126,6 +11252,8 @@ fn detached_editing_failed_case_0() {
],
)
}
#[test]
fn gc_fuzz_23() {
test_multi_sites_with_gc(
5,

18
crates/loro-internal/src/encoding/gc.rs
View file

@ -1,7 +1,7 @@
use rle::HasLength;
use std::collections::BTreeSet;
use loro_common::LoroResult;
use loro_common::{ContainerID, LoroResult, ID};
use tracing::{debug, trace};
use crate::{
@ -65,7 +65,8 @@ pub(crate) fn export_gc_snapshot<W: std::io::Write>(
doc.checkout_without_emitting(&start_from)?;
let mut state = doc.app_state().lock().unwrap();
let alive_containers = state.ensure_all_alive_containers();
let alive_c_bytes: BTreeSet<Vec<u8>> = alive_containers.iter().map(|x| x.to_bytes()).collect();
let mut alive_c_bytes: BTreeSet<Vec<u8>> =
alive_containers.iter().map(|x| x.to_bytes()).collect();
state.store.flush();
let gc_state_kv = state.store.get_kv().clone();
drop(state);
@ -74,6 +75,19 @@ pub(crate) fn export_gc_snapshot<W: std::io::Write>(
let mut state = doc.app_state().lock().unwrap();
state.ensure_all_alive_containers();
state.store.encode();
// All the containers that are created after start_from need to be encoded
for cid in state.store.iter_all_container_ids() {
if let ContainerID::Normal { peer, counter, .. } = cid {
let temp_id = ID::new(peer, counter);
if !start_from.contains(&temp_id) {
trace!("Retain Container {:?}", temp_id);
alive_c_bytes.insert(cid.to_bytes());
}
} else {
alive_c_bytes.insert(cid.to_bytes());
}
}
let new_kv = state.store.get_kv().clone();
new_kv.remove_same(&gc_state_kv);
new_kv.retain_keys(&alive_c_bytes);

6
crates/loro-internal/src/state/container_store.rs
View file

@ -5,7 +5,7 @@ use crate::{
};
use bytes::Bytes;
use inner_store::InnerStore;
use loro_common::{LoroResult, LoroValue};
use loro_common::{ContainerID, LoroResult, LoroValue};
use std::sync::{atomic::AtomicU64, Arc, Mutex};
pub(crate) use container_wrapper::ContainerWrapper;
@ -190,6 +190,10 @@ impl ContainerStore {
self.store.iter_all_containers_mut()
}
pub fn iter_all_container_ids(&mut self) -> impl Iterator<Item = ContainerID> + '_ {
self.store.iter_all_container_ids()
}
pub(super) fn get_or_create_mut(&mut self, idx: ContainerIdx) -> &mut State {
self.store
.get_or_insert_with(idx, || {

8
crates/loro-internal/src/state/container_store/inner_store.rs
View file

@ -93,6 +93,14 @@ impl InnerStore {
self.store.iter_mut()
}
pub(crate) fn iter_all_container_ids(&mut self) -> impl Iterator<Item = ContainerID> + '_ {
// PERF: we don't need to load all the containers here
self.load_all();
self.store
.keys()
.map(|idx| self.arena.get_container_id(*idx).unwrap())
}
pub(crate) fn encode(&mut self) -> Bytes {
self.flush();
self.kv.export()