feat: movable list tree gc (#439)

* feat: movable list tree gc

* fix: record value in movable history when gc

* fix: movable list gc

* fix: remove value from pos

* refactor: tree init

* test: add tree gc checkout test
This commit is contained in:
Leon Zhao 2024-08-29 09:53:32 +08:00 committed by GitHub
parent 1a80cb7572
commit c1f0a40f4b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 423 additions and 37 deletions

View file

@ -506,6 +506,14 @@ impl TreeCacheForDiff {
effected
}
pub(crate) fn init_tree_with_trimmed_version(&mut self, nodes: Vec<MoveLamportAndID>) {
debug_assert!(self.tree.is_empty());
self.current_vv.set_last(nodes.last().unwrap().id.id());
for node in nodes.into_iter() {
self.tree.entry(node.op.target()).or_default().insert(node);
}
}
fn is_parent_deleted(&self, parent: TreeParentId) -> bool {
match parent {
TreeParentId::Deleted => true,

View file

@ -18,7 +18,7 @@ use crate::{
change::{Change, Lamport},
container::{idx::ContainerIdx, list::list_op::InnerListOp, tree::tree_op::TreeOp},
delta::MapValue,
diff_calc::tree::TreeCacheForDiff,
diff_calc::tree::{MoveLamportAndID, TreeCacheForDiff},
encoding::value_register::ValueRegister,
op::{InnerContent, RichOp},
oplog::ChangeStore,
@ -227,8 +227,41 @@ impl ContainerHistoryCache {
}
}
}
crate::state::State::MovableListState(_) => todo!(),
crate::state::State::TreeState(_) => todo!(),
crate::state::State::MovableListState(l) => {
for (idlp, elem) in l.elements() {
if for_checkout {
let c = self.for_checkout.as_mut().unwrap();
let item = l.get_list_item(elem.pos).unwrap();
c.movable_list.record_gc_state(
item.id,
idlp.peer,
idlp.lamport.into(),
elem.value.clone(),
);
}
}
}
crate::state::State::TreeState(t) => {
if for_importing {
let c = self.for_importing.as_mut().unwrap();
if let Some(HistoryCacheForImporting::Tree(tree)) = c.get_mut(idx) {
tree.record_gc_state(
t.tree_nodes()
.into_iter()
.map(|node| MoveLamportAndID {
id: node.last_move_op,
op: Arc::new(TreeOp::Create {
target: node.id,
parent: node.parent,
position: node.position.clone(),
}),
effected: true,
})
.collect(),
);
}
}
}
_ => unreachable!(),
}
}
@ -500,15 +533,29 @@ impl TreeOpGroup {
pub fn tree(&self) -> &Mutex<TreeCacheForDiff> {
&self.tree_for_diff
}
pub(crate) fn record_gc_state(&mut self, nodes: Vec<MoveLamportAndID>) {
let mut tree = self.tree_for_diff.lock().unwrap();
for node in nodes.iter() {
self.ops.insert(
node.id.idlp(),
GroupedTreeOpInfo {
counter: node.id.counter,
value: node.op.clone(),
},
);
}
tree.init_tree_with_trimmed_version(nodes);
}
}
#[derive(Debug, Default)]
pub(crate) struct MovableListHistoryCache {
move_set: BTreeSet<MovableListInnerDeltaEntry>,
set_set: BTreeSet<MovableListInnerDeltaEntry>,
set_set: BTreeSet<MovableListSetDeltaEntry>,
}
#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone)]
#[derive(Debug, Eq, PartialEq, Clone)]
struct MovableListInnerDeltaEntry {
element_lamport: Lamport,
element_peer: PeerID,
@ -517,6 +564,47 @@ struct MovableListInnerDeltaEntry {
counter: Counter,
}
#[derive(Debug, Eq, PartialEq, Clone)]
struct MovableListSetDeltaEntry {
element_lamport: Lamport,
element_peer: PeerID,
lamport: Lamport,
peer: PeerID,
counter_or_value: Either<Counter, Box<LoroValue>>,
}
impl Ord for MovableListInnerDeltaEntry {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.element_lamport
.cmp(&other.element_lamport)
.then_with(|| self.element_peer.cmp(&other.element_peer))
.then_with(|| self.lamport.cmp(&other.lamport))
.then_with(|| self.peer.cmp(&other.peer))
}
}
impl PartialOrd for MovableListInnerDeltaEntry {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl PartialOrd for MovableListSetDeltaEntry {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for MovableListSetDeltaEntry {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.element_lamport
.cmp(&other.element_lamport)
.then_with(|| self.element_peer.cmp(&other.element_peer))
.then_with(|| self.lamport.cmp(&other.lamport))
.then_with(|| self.peer.cmp(&other.peer))
}
}
impl HistoryCacheTrait for MovableListHistoryCache {
fn insert(&mut self, op: &RichOp) {
let cur_id = op.id_full();
@ -532,12 +620,12 @@ impl HistoryCacheTrait for MovableListHistoryCache {
});
}
crate::container::list::list_op::InnerListOp::Set { elem_id, .. } => {
self.set_set.insert(MovableListInnerDeltaEntry {
self.set_set.insert(MovableListSetDeltaEntry {
element_lamport: elem_id.lamport,
element_peer: elem_id.peer,
lamport: cur_id.lamport,
peer: cur_id.peer,
counter: cur_id.counter,
counter_or_value: Either::Left(cur_id.counter),
});
}
_ => {}
@ -548,6 +636,29 @@ impl HistoryCacheTrait for MovableListHistoryCache {
}
impl MovableListHistoryCache {
pub(crate) fn record_gc_state(
&mut self,
id: IdFull,
elem_peer: PeerID,
elem_lamport: Lamport,
value: LoroValue,
) {
self.set_set.insert(MovableListSetDeltaEntry {
element_lamport: elem_lamport,
element_peer: elem_peer,
lamport: id.lamport,
peer: id.peer,
counter_or_value: Either::Right(Box::new(value.clone())),
});
self.move_set.insert(MovableListInnerDeltaEntry {
element_lamport: elem_lamport,
element_peer: elem_peer,
lamport: id.lamport,
peer: id.peer,
counter: id.counter,
});
}
pub(crate) fn last_value(
&self,
key: IdLp,
@ -557,30 +668,35 @@ impl MovableListHistoryCache {
) -> Option<GroupedMapOpInfo<LoroValue>> {
self.set_set
.range((
Bound::Included(MovableListInnerDeltaEntry {
Bound::Included(MovableListSetDeltaEntry {
element_lamport: key.lamport,
element_peer: key.peer,
lamport: 0,
peer: 0,
counter: 0,
counter_or_value: Either::Left(0),
}),
Bound::Excluded(MovableListInnerDeltaEntry {
Bound::Excluded(MovableListSetDeltaEntry {
element_lamport: key.lamport,
element_peer: key.peer,
lamport: max_lamport,
peer: PeerID::MAX,
counter: Counter::MAX,
counter_or_value: Either::Left(Counter::MAX),
}),
))
.rev()
.find(|e| vv.get(&e.peer).copied().unwrap_or(0) > e.counter)
.find(|e| {
let counter = match &e.counter_or_value {
Either::Left(c) => *c,
Either::Right(v) => -1,
};
vv.get(&e.peer).copied().unwrap_or(0) > counter
})
.map_or_else(
|| {
let id = oplog.idlp_to_id(key).unwrap();
if vv.get(&id.peer).copied().unwrap_or(0) <= id.counter {
return None;
}
let op = oplog.get_op_that_includes(id).unwrap();
let offset = id.counter - op.counter;
match &op.content {
@ -601,22 +717,28 @@ impl MovableListHistoryCache {
}
},
|e| {
let id = ID::new(e.peer, e.counter);
let op = oplog.get_op_that_includes(id).unwrap();
debug_assert_eq!(op.atom_len(), 1);
let lamport = op.lamport();
match &op.content {
InnerContent::List(InnerListOp::Set { value, .. }) => {
Some(GroupedMapOpInfo {
value: value.clone(),
lamport,
peer: id.peer,
})
let (lamport, value) = match &e.counter_or_value {
Either::Left(c) => {
let id = ID::new(e.peer, *c);
let op = oplog.get_op_that_includes(id).unwrap();
debug_assert_eq!(op.atom_len(), 1);
let lamport = op.lamport();
match &op.content {
InnerContent::List(InnerListOp::Set { value, .. }) => {
(lamport, value.clone())
}
_ => {
unreachable!()
}
}
}
_ => {
unreachable!()
}
}
Either::Right(v) => (e.lamport, v.as_ref().clone()),
};
Some(GroupedMapOpInfo {
value: value.clone(),
lamport,
peer: e.peer,
})
},
)
}
@ -658,7 +780,7 @@ impl MovableListHistoryCache {
},
|e| {
let id = ID::new(e.peer, e.counter);
let lamport = oplog.get_lamport_at(id).unwrap();
let lamport = oplog.get_lamport_at(id).unwrap_or(e.lamport);
Some(IdFull::new(e.peer, e.counter, lamport))
},
)

View file

@ -38,14 +38,14 @@ pub struct MovableListState {
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ListItem {
pointed_by: Option<CompactIdLp>,
id: IdFull,
pub(crate) id: IdFull,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct Element {
value: LoroValue,
value_id: IdLp,
pos: IdLp,
pub(crate) value: LoroValue,
pub(crate) value_id: IdLp,
pub(crate) pos: IdLp,
}
impl Element {
@ -748,12 +748,12 @@ impl MovableListState {
}
#[inline]
fn list(&self) -> &BTree<MovableListTreeTrait> {
pub(crate) fn list(&self) -> &BTree<MovableListTreeTrait> {
self.inner.list()
}
#[inline]
fn elements(&self) -> &FxHashMap<CompactIdLp, Element> {
pub(crate) fn elements(&self) -> &FxHashMap<CompactIdLp, Element> {
self.inner.elements()
}
@ -871,7 +871,7 @@ impl MovableListState {
Some(self.inner.get_index_of(c.cursor.leaf, to) as usize)
}
fn get_list_item(&self, id: IdLp) -> Option<&ListItem> {
pub(crate) fn get_list_item(&self, id: IdLp) -> Option<&ListItem> {
self.inner.get_list_item_by_id(id)
}

View file

@ -726,6 +726,7 @@ impl TreeState {
parent,
position: position.position.clone(),
index,
last_move_op: self.trees.get(&target).map(|x| x.last_move_op).unwrap(),
});
if let Some(children) = self.children.get(&TreeParentId::Node(target)) {
q.extend(
@ -757,6 +758,7 @@ impl TreeState {
parent: root.id(),
position: position.position.clone(),
index,
last_move_op: self.trees.get(target).map(|x| x.last_move_op).unwrap(),
});
}
@ -1238,6 +1240,7 @@ pub(crate) struct TreeNode {
pub(crate) parent: Option<TreeID>,
pub(crate) position: FractionalIndex,
pub(crate) index: usize,
pub(crate) last_move_op: IdFull,
}
impl TreeNode {

View file

@ -961,7 +961,7 @@ fn new_update_encode_mode() {
fn apply_random_ops(doc: &LoroDoc, seed: u64, mut op_len: usize) {
let mut rng = rand::rngs::StdRng::seed_from_u64(seed);
while op_len > 0 {
match rng.gen_range(0..4) {
match rng.gen_range(0..6) {
0 => {
// Insert text
let text = doc.get_text("text");
@ -994,6 +994,19 @@ fn apply_random_ops(doc: &LoroDoc, seed: u64, mut op_len: usize) {
list.push(item).unwrap();
op_len -= 1;
}
4 => {
// Create node in tree
let tree = doc.get_tree("tree");
tree.create(None).unwrap();
op_len -= 1;
}
5 => {
// Push to movable list
let list = doc.get_movable_list("movable_list");
let item = format!("item{}", rng.gen::<u32>());
list.push(item).unwrap();
op_len -= 1;
}
_ => unreachable!(),
}
}
@ -1196,3 +1209,243 @@ fn test_map_checkout_on_trimmed_doc() {
.unwrap_err();
assert_eq!(err, LoroError::SwitchToTrimmedVersion);
}
#[test]
fn test_movable_list_checkout_on_trimmed_doc() -> LoroResult<()> {
let doc = LoroDoc::new();
let list = doc.get_movable_list("list");
list.insert(0, 0)?;
list.set(0, 1)?;
list.set(0, 3)?;
list.insert(1, 2)?;
list.mov(1, 0)?;
list.delete(0, 1)?;
list.set(0, 0)?;
let new_doc_bytes = doc.export(loro::ExportMode::GcSnapshot(
&ID::new(doc.peer_id(), 2).into(),
));
let new_doc = LoroDoc::new();
new_doc.import(&new_doc_bytes).unwrap();
assert_eq!(
new_doc.get_deep_value(),
loro_value!({
"list": [0]
})
);
new_doc.checkout(&ID::new(doc.peer_id(), 2).into()).unwrap();
assert_eq!(
new_doc.get_deep_value(),
loro_value!({
"list": [3]
})
);
new_doc.checkout_to_latest();
assert_eq!(
new_doc.get_deep_value(),
loro_value!({
"list": [0]
})
);
let err = new_doc
.checkout(&ID::new(doc.peer_id(), 1).into())
.unwrap_err();
assert_eq!(err, LoroError::SwitchToTrimmedVersion);
Ok(())
}
#[test]
fn test_tree_checkout_on_trimmed_doc() -> LoroResult<()> {
let doc = LoroDoc::new();
doc.set_peer_id(0)?;
let tree = doc.get_tree("tree");
let root = tree.create(None)?;
let child1 = tree.create(None)?;
tree.mov(child1, root)?;
let child2 = tree.create(None).unwrap();
tree.mov(child2, root)?;
let new_doc_bytes = doc.export(loro::ExportMode::GcSnapshot(
&ID::new(doc.peer_id(), 1).into(),
));
let new_doc = LoroDoc::new();
new_doc.import(&new_doc_bytes).unwrap();
assert_eq!(
new_doc.get_deep_value(),
loro_value!({
"tree": [
{
"parent": null,
"meta":{},
"id": "0@0",
"index": 0,
"fractional_index": "80",
},
{
"parent": "0@0",
"meta":{},
"id": "1@0",
"index": 0,
"fractional_index": "80",
},
{
"parent": "0@0",
"meta":{},
"id": "3@0",
"index": 1,
"fractional_index": "8180",
},
]
})
);
new_doc.checkout(&ID::new(doc.peer_id(), 2).into()).unwrap();
assert_eq!(
new_doc.get_deep_value(),
loro_value!({
"tree": [
{
"parent": null,
"meta":{},
"id": "0@0",
"index": 0,
"fractional_index": "80",
},
{
"parent": "0@0",
"meta":{},
"id": "1@0",
"index": 0,
"fractional_index": "80",
},
]
})
);
new_doc.checkout(&ID::new(doc.peer_id(), 1).into()).unwrap();
assert_eq!(
new_doc.get_deep_value(),
loro_value!({
"tree": [
{
"parent": null,
"meta":{},
"id": "0@0",
"index": 0,
"fractional_index": "80",
},
{
"parent": null,
"meta":{},
"id": "1@0",
"index": 1,
"fractional_index": "8180",
},
]
})
);
new_doc.checkout_to_latest();
assert_eq!(
new_doc.get_deep_value(),
loro_value!({
"tree": [
{
"parent": null,
"meta":{},
"id": "0@0",
"index": 0,
"fractional_index": "80",
},
{
"parent": "0@0",
"meta":{},
"id": "1@0",
"index": 0,
"fractional_index": "80",
},
{
"parent": "0@0",
"meta":{},
"id": "3@0",
"index": 1,
"fractional_index": "8180",
},
]
})
);
let err = new_doc
.checkout(&ID::new(doc.peer_id(), 0).into())
.unwrap_err();
assert_eq!(err, LoroError::SwitchToTrimmedVersion);
Ok(())
}
#[test]
fn test_tree_with_other_ops_checkout_on_trimmed_doc() -> LoroResult<()> {
let doc = LoroDoc::new();
doc.set_peer_id(0)?;
let tree = doc.get_tree("tree");
let root = tree.create(None)?;
let child1 = tree.create(None)?;
tree.mov(child1, root)?;
let child2 = tree.create(None).unwrap();
tree.mov(child2, root)?;
let map = doc.get_map("map");
map.insert("0", 0)?;
map.insert("1", 1)?;
doc.commit();
let gc_frontiers = doc.oplog_frontiers();
map.insert("2", 2)?;
tree.mov(child2, child1)?;
tree.delete(child1)?;
let new_doc_bytes = doc.export(loro::ExportMode::GcSnapshot(&gc_frontiers));
let new_doc = LoroDoc::new();
new_doc.import(&new_doc_bytes).unwrap();
new_doc.checkout(&gc_frontiers)?;
let value = new_doc.get_deep_value();
assert_eq!(
value,
loro_value!(
{
"map":{
"0":0,
"1":1,
},
"tree":[
{
"parent": null,
"meta":{},
"id": "0@0",
"index": 0,
"fractional_index": "80",
},
{
"parent": "0@0",
"meta":{},
"id": "1@0",
"index": 0,
"fractional_index": "80",
},
{
"parent": "0@0",
"meta":{},
"id": "3@0",
"index": 1,
"fractional_index": "8180",
},
]
}
)
);
let err = new_doc
.checkout(&ID::new(doc.peer_id(), 0).into())
.unwrap_err();
assert_eq!(err, LoroError::SwitchToTrimmedVersion);
Ok(())
}