mirrors/loro
1
0
Fork 0
mirror of https://github.com/loro-dev/loro.git synced 2025-02-02 11:06:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

perf: reduce mem use of tree history cache

Browse source
This commit is contained in:
Zixuan Chen 2024-08-13 20:30:20 +08:00
parent c8f505539e
commit 2881b45bfe
No known key found for this signature in database
3 changed files with 77 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
crates/examples/examples/outliner.rs
View file

@ -86,6 +86,11 @@ pub fn main() {
"Memory usage after freeing diff calculator: {}", "Memory usage after freeing diff calculator: {}",
get_mem_usage() get_mem_usage()
); );
doc.free_history_cache();
println!(
"Memory usage after freeing history cache: {}",
get_mem_usage()
);
println!( println!(
"Updates size: {}", "Updates size: {}",

10
crates/loro-internal/src/container/tree/tree_op.rs
View file

@ -38,6 +38,15 @@ impl TreeOp {
TreeOp::Delete { target, .. } => *target, TreeOp::Delete { target, .. } => *target,
} }
} }
pub(crate) fn parent(&self) -> Option<TreeID> {
match self {
TreeOp::Create { parent, .. } => *parent,
TreeOp::Move { parent, .. } => *parent,
TreeOp::Delete { .. } => Some(TreeID::delete_root()),
}
}
pub(crate) fn parent_id(&self) -> TreeParentId { pub(crate) fn parent_id(&self) -> TreeParentId {
match self { match self {
TreeOp::Create { parent, .. } => TreeParentId::from(*parent), TreeOp::Create { parent, .. } => TreeParentId::from(*parent),
@ -45,6 +54,7 @@ impl TreeOp {
TreeOp::Delete { .. } => TreeParentId::Deleted, TreeOp::Delete { .. } => TreeParentId::Deleted,
} }
} }
pub(crate) fn fractional_index(&self) -> Option<FractionalIndex> { pub(crate) fn fractional_index(&self) -> Option<FractionalIndex> {
match self { match self {
TreeOp::Create { position, .. } | TreeOp::Move { position, .. } => { TreeOp::Create { position, .. } | TreeOp::Move { position, .. } => {

109
crates/loro-internal/src/diff_calc/tree.rs
View file

@ -1,4 +1,4 @@
use std::collections::BTreeSet; use std::{collections::BTreeSet, sync::Arc};
use fractional_index::FractionalIndex; use fractional_index::FractionalIndex;
use fxhash::FxHashMap; use fxhash::FxHashMap;
@ -6,7 +6,7 @@ use itertools::Itertools;
use loro_common::{ContainerID, IdFull, IdLp, IdSpan, Lamport, PeerID, TreeID, ID}; use loro_common::{ContainerID, IdFull, IdLp, IdSpan, Lamport, PeerID, TreeID, ID};
use crate::{ use crate::{
container::idx::ContainerIdx, container::{idx::ContainerIdx, tree::tree_op::TreeOp},
dag::DagUtils, dag::DagUtils,
delta::{TreeDelta, TreeDeltaItem, TreeInternalDiff}, delta::{TreeDelta, TreeDeltaItem, TreeInternalDiff},
event::InternalDiff, event::InternalDiff,
@ -178,17 +178,21 @@ impl TreeDiffCalculator {
let mut retreat_ops = vec![]; let mut retreat_ops = vec![];
for (_target, ops) in tree_cache.tree.iter() { for (_target, ops) in tree_cache.tree.iter() {
for op in ops.iter().rev() { for op in ops.iter().rev() {
if op.lamport < min_lamport { if op.id.lamport < min_lamport {
break; break;
} }
if !to.includes_id(op.id) { if !to.includes_id(op.id.id()) {
retreat_ops.push(op.clone()); retreat_ops.push(op.clone());
} }
} }
} }
tracing::info!(msg="retreat ops", retreat_ops=?retreat_ops); tracing::info!(msg="retreat ops", retreat_ops=?retreat_ops);
for op in retreat_ops { for op in retreat_ops {
tree_cache.tree.get_mut(&op.target).unwrap().remove(&op); tree_cache
.tree
.get_mut(&op.op.target())
.unwrap()
.remove(&op);
tree_cache.current_vv.shrink_to_exclude(IdSpan::new( tree_cache.current_vv.shrink_to_exclude(IdSpan::new(
op.id.peer, op.id.peer,
op.id.counter, op.id.counter,
@ -228,11 +232,8 @@ impl TreeDiffCalculator {
// tracing::info!("forward ops {:?}", forward_ops); // tracing::info!("forward ops {:?}", forward_ops);
for (id, op) in forward_ops { for (id, op) in forward_ops {
let op = MoveLamportAndID { let op = MoveLamportAndID {
target: op.value.target(), id,
parent: op.value.parent_id(), op: op.value.clone(),
position: op.value.fractional_index(),
id: id.id(),
lamport: id.lamport,
effected: false, effected: false,
}; };
tree_cache.apply(op); tree_cache.apply(op);
@ -277,10 +278,10 @@ impl TreeDiffCalculator {
let mut retreat_ops = vec![]; let mut retreat_ops = vec![];
for (_target, ops) in tree_cache.tree.iter() { for (_target, ops) in tree_cache.tree.iter() {
for op in ops.iter().rev() { for op in ops.iter().rev() {
if op.lamport < lca_min_lamport { if op.id.lamport < lca_min_lamport {
break; break;
} }
if !lca_vv.includes_id(op.id) { if !lca_vv.includes_id(op.id.id()) {
retreat_ops.push(op.clone()); retreat_ops.push(op.clone());
} }
} }
@ -288,22 +289,26 @@ impl TreeDiffCalculator {
// tracing::info!("retreat ops {:?}", retreat_ops); // tracing::info!("retreat ops {:?}", retreat_ops);
for op in retreat_ops.into_iter().sorted().rev() { for op in retreat_ops.into_iter().sorted().rev() {
tree_cache.tree.get_mut(&op.target).unwrap().remove(&op); tree_cache
.tree
.get_mut(&op.op.target())
.unwrap()
.remove(&op);
tree_cache.current_vv.shrink_to_exclude(IdSpan::new( tree_cache.current_vv.shrink_to_exclude(IdSpan::new(
op.id.peer, op.id.peer,
op.id.counter, op.id.counter,
op.id.counter + 1, op.id.counter + 1,
)); ));
let (old_parent, position, last_effective_move_op_id) = let (old_parent, position, last_effective_move_op_id) =
tree_cache.get_parent_with_id(op.target); tree_cache.get_parent_with_id(op.op.target());
if op.effected { if op.effected {
// we need to know whether old_parent is deleted // we need to know whether old_parent is deleted
let is_parent_deleted = tree_cache.is_parent_deleted(op.parent); let is_parent_deleted = tree_cache.is_parent_deleted(op.op.parent_id());
let is_old_parent_deleted = tree_cache.is_parent_deleted(old_parent); let is_old_parent_deleted = tree_cache.is_parent_deleted(old_parent);
let this_diff = TreeDeltaItem::new( let this_diff = TreeDeltaItem::new(
op.target, op.op.target(),
old_parent, old_parent,
op.parent, op.op.parent_id(),
last_effective_move_op_id, last_effective_move_op_id,
is_old_parent_deleted, is_old_parent_deleted,
is_parent_deleted, is_parent_deleted,
@ -312,7 +317,7 @@ impl TreeDiffCalculator {
let is_create = matches!(this_diff.action, TreeInternalDiff::Create { .. }); let is_create = matches!(this_diff.action, TreeInternalDiff::Create { .. });
diffs.push(this_diff); diffs.push(this_diff);
if is_create { if is_create {
let mut s = vec![op.target]; let mut s = vec![op.op.target()];
while let Some(t) = s.pop() { while let Some(t) = s.pop() {
let children = tree_cache.get_children_with_id(TreeParentId::Node(t)); let children = tree_cache.get_children_with_id(TreeParentId::Node(t));
children.iter().for_each(|c| { children.iter().for_each(|c| {
@ -350,32 +355,34 @@ impl TreeDiffCalculator {
let id = ID::new(idlp.peer, op.counter); let id = ID::new(idlp.peer, op.counter);
if !tree_cache.current_vv.includes_id(id) && to.includes_id(id) { if !tree_cache.current_vv.includes_id(id) && to.includes_id(id) {
let op = MoveLamportAndID { let op = MoveLamportAndID {
target: op.value.target(), id: IdFull {
parent: op.value.parent_id(), peer: id.peer,
position: op.value.fractional_index(),
id,
lamport: idlp.lamport, lamport: idlp.lamport,
counter: id.counter,
},
op: op.value.clone(),
effected: false, effected: false,
}; };
let (old_parent, _position, _id) = tree_cache.get_parent_with_id(op.target); let (old_parent, _position, _id) =
let is_parent_deleted = tree_cache.is_parent_deleted(op.parent); tree_cache.get_parent_with_id(op.op.target());
let is_parent_deleted = tree_cache.is_parent_deleted(op.op.parent_id());
let is_old_parent_deleted = tree_cache.is_parent_deleted(old_parent); let is_old_parent_deleted = tree_cache.is_parent_deleted(old_parent);
let effected = tree_cache.apply(op.clone()); let effected = tree_cache.apply(op.clone());
if effected { if effected {
let this_diff = TreeDeltaItem::new( let this_diff = TreeDeltaItem::new(
op.target, op.op.target(),
op.parent, op.op.parent_id(),
old_parent, old_parent,
op.id_full(), op.id_full(),
is_parent_deleted, is_parent_deleted,
is_old_parent_deleted, is_old_parent_deleted,
op.position, op.op.fractional_index(),
); );
let is_create = matches!(this_diff.action, TreeInternalDiff::Create { .. }); let is_create = matches!(this_diff.action, TreeInternalDiff::Create { .. });
diffs.push(this_diff); diffs.push(this_diff);
if is_create { if is_create {
// TODO: per // TODO: per
let mut s = vec![op.target]; let mut s = vec![op.op.target()];
while let Some(t) = s.pop() { while let Some(t) = s.pop() {
let children = let children =
tree_cache.get_children_with_id(TreeParentId::Node(t)); tree_cache.get_children_with_id(TreeParentId::Node(t));
@ -417,13 +424,10 @@ impl TreeDiffCalculator {
} }
/// All information of an operation for diff calculating of movable tree. /// All information of an operation for diff calculating of movable tree.
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone)]
pub struct MoveLamportAndID { pub struct MoveLamportAndID {
pub(crate) lamport: Lamport, pub(crate) id: IdFull,
pub(crate) id: ID, pub(crate) op: Arc<TreeOp>,
pub(crate) target: TreeID,
pub(crate) parent: TreeParentId,
pub(crate) position: Option<FractionalIndex>,
/// Whether this action is applied in the current version. /// Whether this action is applied in the current version.
/// If this action will cause a circular reference, then this action will not be applied. /// If this action will cause a circular reference, then this action will not be applied.
pub(crate) effected: bool, pub(crate) effected: bool,
@ -431,14 +435,18 @@ pub struct MoveLamportAndID {
impl MoveLamportAndID { impl MoveLamportAndID {
fn id_full(&self) -> IdFull { fn id_full(&self) -> IdFull {
IdFull { self.id
peer: self.id.peer,
lamport: self.lamport,
counter: self.id.counter,
}
} }
} }
impl PartialEq for MoveLamportAndID {
fn eq(&self, other: &Self) -> bool {
self.id == other.id
}
}
impl Eq for MoveLamportAndID {}
impl PartialOrd for MoveLamportAndID { impl PartialOrd for MoveLamportAndID {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other)) Some(self.cmp(other))
@ -447,9 +455,10 @@ impl PartialOrd for MoveLamportAndID {
impl Ord for MoveLamportAndID { impl Ord for MoveLamportAndID {
fn cmp(&self, other: &Self) -> std::cmp::Ordering { fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.lamport self.id
.cmp(&other.lamport) .lamport
.then_with(|| self.id.cmp(&other.id)) .cmp(&other.id.lamport)
.then_with(|| self.id.peer.cmp(&other.id.peer))
} }
} }
@ -484,12 +493,12 @@ impl TreeCacheForDiff {
fn apply(&mut self, mut node: MoveLamportAndID) -> bool { fn apply(&mut self, mut node: MoveLamportAndID) -> bool {
let mut effected = true; let mut effected = true;
if self.is_ancestor_of(&node.target, &node.parent) { if self.is_ancestor_of(&node.op.target(), &node.op.parent_id()) {
effected = false; effected = false;
} }
node.effected = effected; node.effected = effected;
self.current_vv.set_last(node.id); self.current_vv.set_last(node.id.id());
self.tree.entry(node.target).or_default().insert(node); self.tree.entry(node.op.target()).or_default().insert(node);
effected effected
} }
@ -511,7 +520,11 @@ impl TreeCacheForDiff {
if let Some(cache) = self.tree.get(&tree_id) { if let Some(cache) = self.tree.get(&tree_id) {
for op in cache.iter().rev() { for op in cache.iter().rev() {
if op.effected { if op.effected {
ans = (op.parent, op.position.clone(), op.id_full()); ans = (
op.op.parent_id(),
op.op.fractional_index().clone(),
op.id_full(),
);
break; break;
} }
} }
@ -548,8 +561,8 @@ impl TreeCacheForDiff {
continue; continue;
}; };
if op.parent == parent { if op.op.parent_id() == parent {
ans.push((*tree_id, op.position.clone(), op.id_full())); ans.push((*tree_id, op.op.fractional_index().clone(), op.id_full()));
} }
} }
// The children should be sorted by the position. // The children should be sorted by the position.