mirrors/loro
1
0
Fork 0
mirror of https://github.com/loro-dev/loro.git synced 2025-01-23 13:39:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #23 from loro-dev/refactor-content

Browse source 
Refactor: differentiate inner content & remote content
This commit is contained in:
Zixuan Chen 2022-11-22 20:37:59 +08:00 committed by GitHub
commit 51a28129a9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 395 additions and 358 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
Cargo.lock generated
View file

@ -474,6 +474,12 @@ dependencies = [
"syn 1.0.103",
]
[[package]]
name = "debug-log"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd2cce8e65a25b144661b7c661eefd89d06c5329ed9a30da8ec98c4f3fefc60b"
[[package]]
name = "derive_arbitrary"
version = "1.2.0"
@ -756,6 +762,7 @@ dependencies = [
"crdt-list",
"criterion",
"ctor",
"debug-log",
"dhat",
"enum-as-inner",
"flate2",

1
crates/loro-core/Cargo.toml
View file

@ -44,6 +44,7 @@ ctor = "0.1.23"
criterion = "0.4.0"
flate2 = "1.0.24"
arbtest = "0.2.0"
debug-log = "0.1.1"
# See https://matklad.github.io/2021/02/27/delete-cargo-integration-tests.html

7
crates/loro-core/src/container.rs
View file

@ -5,12 +5,13 @@
//! Every [Container] can take a [Snapshot], which contains [crate::LoroValue] that describes the state.
//!
use crate::{
op::{RemoteOp, RichOp},
op::{InnerContent, RemoteContent, RichOp},
version::{IdSpanVector, VersionVector},
InternalString, LoroValue, ID,
};
use serde::{Deserialize, Serialize};
use smallvec::SmallVec;
use std::{any::Any, fmt::Debug};
@ -40,10 +41,10 @@ pub trait Container: Debug + Any + Unpin {
// fn serialize(&self) -> Vec<u8>;
/// convert an op content to exported format that includes the raw data
fn to_export(&mut self, op: &mut RemoteOp, gc: bool);
fn to_export(&mut self, content: InnerContent, gc: bool) -> SmallVec<[RemoteContent; 1]>;
/// convert an op content to compact imported format
fn to_import(&mut self, op: &mut RemoteOp);
fn to_import(&mut self, content: RemoteContent) -> InnerContent;
/// Apply the effect of the op directly to the state.
fn update_state_directly(&mut self, op: &RichOp);

75
crates/loro-core/src/container/list/list_container.rs
View file

@ -5,6 +5,7 @@ use rle::{
rle_tree::{tree_trait::CumulateTreeTrait, HeapMode},
RleTree,
};
use smallvec::SmallVec;
use crate::{
container::{
@ -19,11 +20,13 @@ use crate::{
},
context::Context,
id::{Counter, ID},
op::{Content, Op, RemoteOp, RichOp},
op::{InnerContent, Op, RemoteContent, RichOp},
value::LoroValue,
version::IdSpanVector,
};
use super::list_op::InnerListOp;
#[derive(Debug)]
pub struct ListContainer {
id: ContainerID,
@ -54,7 +57,7 @@ impl ListContainer {
self.state.insert(pos, slice.clone().into());
let op = Op::new(
id,
Content::List(ListOp::Insert {
InnerContent::List(InnerListOp::Insert {
slice: slice.into(),
pos,
}),
@ -76,7 +79,7 @@ impl ListContainer {
self.state.insert(pos, slice.clone().into());
let op = Op::new(
id,
Content::List(ListOp::Insert {
InnerContent::List(InnerListOp::Insert {
slice: slice.into(),
pos,
}),
@ -101,7 +104,7 @@ impl ListContainer {
let id = store.next_id();
let op = Op::new(
id,
Content::List(ListOp::new_del(pos, len)),
InnerContent::List(InnerListOp::new_del(pos, len)),
store.get_or_create_container_idx(&self.id),
);
@ -170,40 +173,46 @@ impl Container for ListContainer {
values.into()
}
fn to_export(&mut self, op: &mut RemoteOp, _gc: bool) {
for content in op.contents.iter_mut() {
if let Some((slice, _pos)) = content.as_list_mut().and_then(|x| x.as_insert_mut()) {
if let Some(change) = if let ListSlice::Slice(ranges) = slice {
Some(self.raw_data.slice(&ranges.0))
} else {
None
} {
*slice = ListSlice::RawData(change.to_vec());
fn to_export(&mut self, content: InnerContent, _gc: bool) -> SmallVec<[RemoteContent; 1]> {
match content {
InnerContent::List(list) => match list {
InnerListOp::Insert { slice, pos } => {
let data = self.raw_data.slice(&slice.0);
smallvec::smallvec![RemoteContent::List(ListOp::Insert {
pos,
slice: ListSlice::RawData(data.to_vec()),
})]
}
}
InnerListOp::Delete(del) => {
smallvec::smallvec![RemoteContent::List(ListOp::Delete(del))]
}
},
InnerContent::Map(_) => unreachable!(),
}
}
fn to_import(&mut self, op: &mut RemoteOp) {
for content in op.contents.iter_mut() {
if let Some((slice, _pos)) = content.as_list_mut().and_then(|x| x.as_insert_mut()) {
if let Some(slice_range) = match std::mem::take(slice) {
ListSlice::RawData(data) => Some(self.raw_data.alloc_arr(data)),
fn to_import(&mut self, content: RemoteContent) -> InnerContent {
match content {
RemoteContent::List(list) => match list {
ListOp::Insert { slice, pos } => match slice {
ListSlice::RawData(data) => {
let slice_range = self.raw_data.alloc_arr(data);
let slice: SliceRange = slice_range.into();
InnerContent::List(InnerListOp::Insert { slice, pos })
}
_ => unreachable!(),
} {
*slice = slice_range.into();
}
}
},
ListOp::Delete(del) => InnerContent::List(InnerListOp::Delete(del)),
},
_ => unreachable!(),
}
}
fn update_state_directly(&mut self, op: &RichOp) {
match &op.get_sliced().content {
Content::List(op) => match op {
ListOp::Insert { slice, pos } => {
self.state.insert(*pos, slice.as_slice().unwrap().clone())
}
ListOp::Delete(span) => self
InnerContent::List(op) => match op {
InnerListOp::Insert { slice, pos } => self.state.insert(*pos, slice.clone()),
InnerListOp::Delete(span) => self
.state
.delete_range(Some(span.start() as usize), Some(span.end() as usize)),
},
@ -242,15 +251,7 @@ impl Container for ListContainer {
for effect in self.tracker.iter_effects(from, effect_spans) {
match effect {
Effect::Del { pos, len } => self.state.delete_range(Some(pos), Some(pos + len)),
Effect::Ins { pos, content } => {
let v = match content {
ListSlice::Slice(slice) => slice.clone(),
ListSlice::Unknown(u) => ListSlice::unknown_range(u),
_ => unreachable!(),
};
self.state.insert(pos, v)
}
Effect::Ins { pos, content } => self.state.insert(pos, content),
}
}
}

80
crates/loro-core/src/container/list/list_op.rs
View file

@ -3,7 +3,7 @@ use std::ops::Range;
use enum_as_inner::EnumAsInner;
use rle::{HasLength, Mergable, Sliceable};
use crate::container::text::text_content::ListSlice;
use crate::container::text::text_content::{ListSlice, SliceRange};
#[derive(EnumAsInner, Debug, Clone)]
pub enum ListOp {
@ -11,6 +11,12 @@ pub enum ListOp {
Delete(DeleteSpan),
}
#[derive(EnumAsInner, Debug, Clone)]
pub enum InnerListOp {
Insert { slice: SliceRange, pos: usize },
Delete(DeleteSpan),
}
/// `len` can be negative so that we can merge text deletions efficiently.
/// It looks like [crate::span::CounterSpan], but how should they merge ([Mergable] impl) and slice ([Sliceable] impl) are very different
///
@ -33,6 +39,16 @@ impl ListOp {
}
}
impl InnerListOp {
pub fn new_del(pos: usize, len: usize) -> Self {
assert!(len != 0);
Self::Delete(DeleteSpan {
pos: pos as isize,
len: len as isize,
})
}
}
impl HasLength for DeleteSpan {
fn content_len(&self) -> usize {
self.len.unsigned_abs()
@ -230,6 +246,68 @@ impl Sliceable for ListOp {
}
}
impl Mergable for InnerListOp {
fn is_mergable(&self, _other: &Self, _conf: &()) -> bool
where
Self: Sized,
{
match self {
InnerListOp::Insert { pos, slice } => match _other {
InnerListOp::Insert {
pos: other_pos,
slice: other_slice,
} => pos + slice.content_len() == *other_pos && slice.is_mergable(other_slice, &()),
_ => false,
},
&InnerListOp::Delete(span) => match _other {
InnerListOp::Delete(other_span) => span.is_mergable(other_span, &()),
_ => false,
},
}
}
fn merge(&mut self, _other: &Self, _conf: &())
where
Self: Sized,
{
match self {
InnerListOp::Insert { slice, .. } => match _other {
InnerListOp::Insert {
slice: other_slice, ..
} => {
slice.merge(other_slice, &());
}
_ => unreachable!(),
},
InnerListOp::Delete(span) => match _other {
InnerListOp::Delete(other_span) => span.merge(other_span, &()),
_ => unreachable!(),
},
}
}
}
impl HasLength for InnerListOp {
fn content_len(&self) -> usize {
match self {
InnerListOp::Insert { slice, .. } => slice.content_len(),
InnerListOp::Delete(span) => span.atom_len(),
}
}
}
impl Sliceable for InnerListOp {
fn slice(&self, from: usize, to: usize) -> Self {
match self {
InnerListOp::Insert { slice, pos } => InnerListOp::Insert {
slice: slice.slice(from, to),
pos: *pos + from,
},
InnerListOp::Delete(span) => InnerListOp::Delete(span.slice(from, to)),
}
}
}
#[cfg(test)]
mod test {
use rle::{Mergable, Sliceable};

62
crates/loro-core/src/container/map/map_container.rs
View file

@ -1,7 +1,8 @@
use std::sync::{Arc, Mutex};
use super::super::pool::Pool;
use super::{super::pool::Pool, InnerMapSet};
use fxhash::FxHashMap;
use smallvec::{smallvec, SmallVec};
use crate::{
container::{
@ -9,8 +10,7 @@ use crate::{
Container, ContainerID, ContainerType,
},
context::Context,
op::RemoteOp,
op::{Content, Op, RichOp},
op::{InnerContent, Op, RemoteContent, RichOp},
span::HasLamport,
value::LoroValue,
version::{IdSpanVector, TotalOrderStamp},
@ -31,7 +31,7 @@ pub struct MapContainer {
#[derive(Debug)]
struct ValueSlot {
value: LoroValue,
value: u32,
order: TotalOrderStamp,
}
@ -53,8 +53,8 @@ impl MapContainer {
value: V,
) {
let value = value.into();
let value_index = self.pool.alloc(value).start as i32;
let value = LoroValue::I32(value_index);
let value_index = self.pool.alloc(value).start;
let value = value_index;
let self_id = &self.id;
let m = ctx.log_store();
let mut store = m.write().unwrap();
@ -70,9 +70,9 @@ impl MapContainer {
store.append_local_ops(&[Op {
counter: id.counter,
container,
content: Content::Map(MapSet {
content: InnerContent::Map(InnerMapSet {
key: key.clone(),
value: value.clone(),
value,
}),
}]);
@ -90,8 +90,8 @@ impl MapContainer {
let mut store = m.write().unwrap();
let client_id = store.this_client_id;
let container_id = store.create_container(obj);
let value_index = self.pool.alloc(container_id.clone()).start as i32;
let value = LoroValue::I32(value_index);
let value_index = self.pool.alloc(container_id.clone()).start;
let value = value_index;
// TODO: store this value?
let id = store.next_id_for(client_id);
let container = store.get_container_idx(self_id).unwrap();
@ -103,9 +103,9 @@ impl MapContainer {
store.append_local_ops(&[Op {
counter: id.counter,
container,
content: Content::Map(MapSet {
content: InnerContent::Map(InnerMapSet {
value,
key: key.clone(),
value: value.clone(),
}),
}]);
self.state.insert(key, ValueSlot { value, order });
@ -131,7 +131,7 @@ impl Container for MapContainer {
fn get_value(&self) -> LoroValue {
let mut map = FxHashMap::default();
for (key, value) in self.state.iter() {
let index = *value.value.as_i32().unwrap() as u32;
let index = value.value;
let value = self.pool.slice(&(index..index + 1))[0].clone();
if let Some(container_id) = value.as_unresolved() {
map.insert(
@ -149,41 +149,47 @@ impl Container for MapContainer {
fn tracker_checkout(&mut self, _vv: &crate::version::VersionVector) {}
fn to_export(&mut self, op: &mut RemoteOp, _gc: bool) {
for content in op.contents.iter_mut() {
if let Some(set) = content.as_map_mut() {
let index = *set.value.as_i32().unwrap() as u32;
set.value = self.pool.slice(&(index..index + 1))[0].clone();
}
fn to_export(&mut self, content: InnerContent, _gc: bool) -> SmallVec<[RemoteContent; 1]> {
if let Ok(set) = content.into_map() {
let index = set.value;
let value = self.pool.slice(&(index..index + 1))[0].clone();
return smallvec![RemoteContent::Map(MapSet {
key: set.key,
value,
})];
}
unreachable!()
}
fn to_import(&mut self, op: &mut RemoteOp) {
for content in op.contents.iter_mut() {
if let Some(set) = content.as_map_mut() {
let index = self.pool.alloc(std::mem::take(&mut set.value));
set.value = LoroValue::I32(index.start as i32);
}
fn to_import(&mut self, mut content: RemoteContent) -> InnerContent {
if let Some(set) = content.as_map_mut() {
let index = self.pool.alloc(std::mem::take(&mut set.value));
return InnerContent::Map(InnerMapSet {
key: set.key.clone(),
value: index.start,
});
}
unreachable!()
}
fn update_state_directly(&mut self, op: &RichOp) {
let content = op.get_sliced().content;
let v: &MapSet = content.as_map().unwrap();
let v: &InnerMapSet = content.as_map().unwrap();
let order = TotalOrderStamp {
lamport: op.lamport(),
client_id: op.client_id(),
};
if let Some(slot) = self.state.get_mut(&v.key) {
if slot.order < order {
slot.value = v.value.clone();
slot.value = v.value;
slot.order = order;
}
} else {
self.state.insert(
v.key.to_owned(),
ValueSlot {
value: v.value.clone(),
value: v.value,
order,
},
);

20
crates/loro-core/src/container/map/map_content.rs
View file

@ -2,13 +2,18 @@ use rle::{HasLength, Mergable, Sliceable};
use crate::{ContentType, InsertContentTrait, InternalString, LoroValue};
// TODO: use imported and exported format to save the space
#[derive(Clone, Debug, PartialEq)]
pub struct MapSet {
pub(crate) key: InternalString,
pub(crate) value: LoroValue,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct InnerMapSet {
pub(crate) key: InternalString,
pub(crate) value: u32,
}
impl Mergable for MapSet {}
impl Sliceable for MapSet {
fn slice(&self, from: usize, to: usize) -> Self {
@ -22,6 +27,19 @@ impl HasLength for MapSet {
}
}
impl Mergable for InnerMapSet {}
impl Sliceable for InnerMapSet {
fn slice(&self, from: usize, to: usize) -> Self {
assert!(from == 0 && to == 1);
self.clone()
}
}
impl HasLength for InnerMapSet {
fn content_len(&self) -> usize {
1
}
}
impl InsertContentTrait for MapSet {
fn id(&self) -> ContentType {
ContentType::Map

2
crates/loro-core/src/container/map/mod.rs
View file

@ -4,4 +4,4 @@ mod tests;
pub use map_container::Map;
pub(crate) use map_container::MapContainer;
pub(crate) use map_content::MapSet;
pub(crate) use map_content::{InnerMapSet, MapSet};

42
crates/loro-core/src/container/registry.rs
View file

@ -7,11 +7,12 @@ use enum_as_inner::EnumAsInner;
use fxhash::FxHashMap;
use owning_ref::OwningRefMut;
use smallvec::SmallVec;
use crate::{
context::Context,
id::ContainerIdx,
op::{RemoteOp, RichOp},
op::{RemoteContent, RichOp},
version::IdSpanVector,
LoroValue, VersionVector,
};
@ -66,23 +67,6 @@ impl Container for ContainerInstance {
ContainerInstance::List(x) => x.get_value(),
}
}
fn to_export(&mut self, op: &mut RemoteOp, gc: bool) {
match self {
ContainerInstance::Map(x) => x.to_export(op, gc),
ContainerInstance::Text(x) => x.to_export(op, gc),
ContainerInstance::Dyn(x) => x.to_export(op, gc),
ContainerInstance::List(x) => x.to_export(op, gc),
}
}
fn to_import(&mut self, op: &mut RemoteOp) {
match self {
ContainerInstance::Map(x) => x.to_import(op),
ContainerInstance::Text(x) => x.to_import(op),
ContainerInstance::Dyn(x) => x.to_import(op),
ContainerInstance::List(x) => x.to_import(op),
}
}
fn update_state_directly(&mut self, op: &RichOp) {
match self {
@ -128,6 +112,28 @@ impl Container for ContainerInstance {
ContainerInstance::List(x) => x.apply_tracked_effects_from(from, effect_spans),
}
}
fn to_export(
&mut self,
content: crate::op::InnerContent,
gc: bool,
) -> SmallVec<[RemoteContent; 1]> {
match self {
ContainerInstance::Map(x) => x.to_export(content, gc),
ContainerInstance::Text(x) => x.to_export(content, gc),
ContainerInstance::Dyn(x) => x.to_export(content, gc),
ContainerInstance::List(x) => x.to_export(content, gc),
}
}
fn to_import(&mut self, content: crate::op::RemoteContent) -> crate::op::InnerContent {
match self {
ContainerInstance::Map(x) => x.to_import(content),
ContainerInstance::Text(x) => x.to_import(content),
ContainerInstance::Dyn(x) => x.to_import(content),
ContainerInstance::List(x) => x.to_import(content),
}
}
}
// TODO: containers snapshot: we need to resolve each container's parent even

119
crates/loro-core/src/container/text/text_container.rs
View file

@ -2,19 +2,20 @@ use std::sync::{Arc, Mutex};
use rle::{
rle_tree::{tree_trait::CumulateTreeTrait, HeapMode},
HasLength, RleTree, RleVec,
HasLength, RleTree,
};
use smallvec::SmallVec;
use crate::{
container::{
list::list_op::ListOp,
list::list_op::{InnerListOp, ListOp},
registry::{ContainerInstance, ContainerWrapper},
Container, ContainerID, ContainerType,
},
context::Context,
debug_log,
id::{Counter, ID},
op::{Content, Op, RemoteOp, RichOp},
op::{InnerContent, Op, RemoteContent, RichOp},
value::LoroValue,
version::IdSpanVector,
};
@ -59,7 +60,7 @@ impl TextContainer {
self.state.insert(pos, slice.clone().into());
let op = Op::new(
id,
Content::List(ListOp::Insert {
InnerContent::List(InnerListOp::Insert {
slice: slice.into(),
pos,
}),
@ -84,7 +85,7 @@ impl TextContainer {
let id = store.next_id();
let op = Op::new(
id,
Content::List(ListOp::new_del(pos, len)),
InnerContent::List(InnerListOp::new_del(pos, len)),
store.get_or_create_container_idx(&self.id),
);
@ -136,39 +137,42 @@ impl Container for TextContainer {
LoroValue::String(ans_str.into_boxed_str())
}
fn to_export(&mut self, op: &mut RemoteOp, gc: bool) {
fn to_export(&mut self, content: InnerContent, gc: bool) -> SmallVec<[RemoteContent; 1]> {
if gc && self.raw_str.should_update_aliveness(self.text_len()) {
self.raw_str
.update_aliveness(self.state.iter().map(|x| x.as_ref().0.clone()))
}
let mut contents: RleVec<[Content; 1]> = RleVec::new();
for content in op.contents.iter_mut() {
if let Some((slice, pos)) = content.as_list_mut().and_then(|x| x.as_insert_mut()) {
match slice {
ListSlice::Slice(r) => {
if r.is_unknown() {
panic!("Unknown range in state");
}
let mut ans = SmallVec::new();
match content {
InnerContent::List(list) => match list {
InnerListOp::Insert { slice, pos } => {
let r = slice;
if r.is_unknown() {
let v = RemoteContent::List(ListOp::Insert {
slice: ListSlice::Unknown(r.atom_len()),
pos,
});
ans.push(v);
} else {
let s = self.raw_str.get_str(&r.0);
if gc {
let mut start = 0;
let mut pos_start = *pos;
let mut pos_start = pos;
for span in self.raw_str.get_aliveness(&r.0) {
match span {
Alive::True(span) => {
contents.push(Content::List(ListOp::Insert {
ans.push(RemoteContent::List(ListOp::Insert {
slice: ListSlice::RawStr(s[start..start + span].into()),
pos: pos_start,
}));
}
Alive::False(span) => {
let v = Content::List(ListOp::Insert {
let v = RemoteContent::List(ListOp::Insert {
slice: ListSlice::Unknown(span),
pos: pos_start,
});
contents.push(v);
ans.push(v);
}
}
@ -177,60 +181,49 @@ impl Container for TextContainer {
}
assert_eq!(start, r.atom_len());
} else {
contents.push(Content::List(ListOp::Insert {
ans.push(RemoteContent::List(ListOp::Insert {
slice: ListSlice::RawStr(s),
pos: *pos,
}));
pos,
}))
}
}
this => {
contents.push(Content::List(ListOp::Insert {
slice: this.clone(),
pos: *pos,
}));
}
}
} else {
contents.push(content.clone());
}
InnerListOp::Delete(del) => ans.push(RemoteContent::List(ListOp::Delete(del))),
},
InnerContent::Map(_) => unreachable!(),
}
op.contents = contents;
assert!(!ans.is_empty());
ans
}
fn to_import(&mut self, op: &mut RemoteOp) {
debug_log!("IMPORT {:#?}", &op);
for content in op.contents.iter_mut() {
if let Some((slice, _pos)) = content.as_list_mut().and_then(|x| x.as_insert_mut()) {
if let Some(slice_range) = match slice {
fn to_import(&mut self, content: RemoteContent) -> InnerContent {
debug_log!("IMPORT {:#?}", &content);
match content {
RemoteContent::List(list) => match list {
ListOp::Insert { slice, pos } => match slice {
ListSlice::RawStr(s) => {
let range = self.raw_str.alloc(s);
Some(range)
let range = self.raw_str.alloc(&s);
let slice: SliceRange = range.into();
InnerContent::List(InnerListOp::Insert { slice, pos })
}
ListSlice::Unknown(_) => None,
ListSlice::Slice(_) => unreachable!(),
ListSlice::RawData(_) => unreachable!(),
} {
*slice = slice_range.into();
}
}
ListSlice::Unknown(u) => InnerContent::List(InnerListOp::Insert {
slice: SliceRange::new_unknown(u as u32),
pos,
}),
_ => unreachable!(),
},
ListOp::Delete(del) => InnerContent::List(InnerListOp::Delete(del)),
},
_ => unreachable!(),
}
debug_log!("IMPORTED {:#?}", &op);
}
fn update_state_directly(&mut self, op: &RichOp) {
match &op.get_sliced().content {
Content::List(op) => match op {
ListOp::Insert { slice, pos } => {
let v = match slice {
ListSlice::Slice(slice) => slice.clone(),
ListSlice::Unknown(u) => ListSlice::unknown_range(*u),
_ => unreachable!(),
};
self.state.insert(*pos, v)
}
ListOp::Delete(span) => self
InnerContent::List(op) => match op {
InnerListOp::Insert { slice, pos } => self.state.insert(*pos, slice.clone()),
InnerListOp::Delete(span) => self
.state
.delete_range(Some(span.start() as usize), Some(span.end() as usize)),
},
@ -276,15 +269,7 @@ impl Container for TextContainer {
debug_log!("APPLY EFFECT {:?}", &effect);
match effect {
Effect::Del { pos, len } => self.state.delete_range(Some(pos), Some(pos + len)),
Effect::Ins { pos, content } => {
let v = match content {
ListSlice::Slice(slice) => slice.clone(),
ListSlice::Unknown(u) => ListSlice::unknown_range(u),
_ => unreachable!(),
};
self.state.insert(pos, v)
}
Effect::Ins { pos, content } => self.state.insert(pos, content),
}
}
debug_log!("AFTER APPLY EFFECT {:?}", self.get_value());

25
crates/loro-core/src/container/text/text_content.rs
View file

@ -10,7 +10,6 @@ pub enum ListSlice {
// TODO: use Box<[LoroValue]> ?
RawData(Vec<LoroValue>),
RawStr(SmString),
Slice(SliceRange),
Unknown(usize),
}
@ -36,18 +35,6 @@ impl Default for ListSlice {
}
}
impl From<Range<u32>> for ListSlice {
fn from(a: Range<u32>) -> Self {
ListSlice::Slice(a.into())
}
}
impl From<SliceRange> for ListSlice {
fn from(a: SliceRange) -> Self {
ListSlice::Slice(a)
}
}
impl From<Range<u32>> for SliceRange {
fn from(a: Range<u32>) -> Self {
SliceRange(a)
@ -99,21 +86,12 @@ impl ListSlice {
pub fn is_unknown(range: &SliceRange) -> bool {
range.is_unknown()
}
pub fn to_range(&self) -> SliceRange {
match self {
ListSlice::Slice(slice) => slice.clone(),
ListSlice::Unknown(u) => SliceRange::new_unknown(*u as u32),
_ => unreachable!(),
}
}
}
impl HasLength for ListSlice {
fn content_len(&self) -> usize {
match self {
ListSlice::RawStr(s) => s.len(),
ListSlice::Slice(x) => rle::HasLength::content_len(&x),
ListSlice::Unknown(x) => *x,
ListSlice::RawData(x) => x.len(),
}
@ -124,7 +102,6 @@ impl Sliceable for ListSlice {
fn slice(&self, from: usize, to: usize) -> Self {
match self {
ListSlice::RawStr(s) => ListSlice::RawStr(s.0[from..to].into()),
ListSlice::Slice(x) => ListSlice::Slice(x.slice(from, to)),
ListSlice::Unknown(_) => ListSlice::Unknown(to - from),
ListSlice::RawData(x) => ListSlice::RawData(x[from..to].to_vec()),
}
@ -134,7 +111,6 @@ impl Sliceable for ListSlice {
impl Mergable for ListSlice {
fn is_mergable(&self, other: &Self, _: &()) -> bool {
match (self, other) {
(ListSlice::Slice(x), ListSlice::Slice(y)) => x.is_mergable(y, &()),
(ListSlice::Unknown(_), ListSlice::Unknown(_)) => true,
(ListSlice::RawStr(a), ListSlice::RawStr(b)) => a.is_mergable(b, &()),
_ => false,
@ -143,7 +119,6 @@ impl Mergable for ListSlice {
fn merge(&mut self, other: &Self, _: &()) {
match (self, other) {
(ListSlice::Slice(x), ListSlice::Slice(y)) => x.merge(y, &()),
(ListSlice::Unknown(x), ListSlice::Unknown(y)) => {
*x += y;
}

12
crates/loro-core/src/container/text/tracker.rs
View file

@ -2,10 +2,10 @@ use rle::{rle_tree::UnsafeCursor, HasLength, Sliceable};
use smallvec::SmallVec;
use crate::{
container::{list::list_op::ListOp, text::tracker::yata_impl::YataImpl},
container::{list::list_op::InnerListOp, text::tracker::yata_impl::YataImpl},
debug_log,
id::{Counter, ID},
op::{Content, RichOp},
op::{InnerContent, RichOp},
span::{HasId, HasIdSpan, IdSpan},
version::IdSpanVector,
VersionVector,
@ -303,7 +303,7 @@ impl Tracker {
}
/// apply an operation directly to the current tracker
fn apply(&mut self, id: ID, content: &Content) {
fn apply(&mut self, id: ID, content: &InnerContent) {
self.real_checkout();
assert!(*self.current_vv.get(&id.client_id).unwrap_or(&0) <= id.counter);
assert!(*self.all_vv.get(&id.client_id).unwrap_or(&0) <= id.counter);
@ -314,14 +314,14 @@ impl Tracker {
self.all_vv.set_end(id.inc(content.content_len() as i32));
let text_content = content.as_list().expect("Content is not for list");
match text_content {
ListOp::Insert { slice, pos } => {
InnerListOp::Insert { slice, pos } => {
let yspan =
self.content
.get_yspan_at_pos(id, *pos, slice.content_len(), slice.to_range());
.get_yspan_at_pos(id, *pos, slice.content_len(), slice.clone());
// SAFETY: we know this is safe because in [YataImpl::insert_after] there is no access to shared elements
unsafe { crdt_list::yata::integrate::<YataImpl>(self, yspan) };
}
ListOp::Delete(span) => {
InnerListOp::Delete(span) => {
let mut spans = self
.content
.get_active_id_spans(span.start() as usize, span.atom_len());

6
crates/loro-core/src/container/text/tracker/effects_iter.rs
View file

@ -1,7 +1,7 @@
use rle::HasLength;
use crate::{
container::text::text_content::ListSlice,
container::text::text_content::SliceRange,
id::{Counter, ID},
span::{CounterSpan, HasId, HasIdSpan, IdSpan},
version::IdSpanVector,
@ -35,7 +35,7 @@ impl<'a> EffectIter<'a> {
#[derive(Debug)]
pub enum Effect {
Del { pos: usize, len: usize },
Ins { pos: usize, content: ListSlice },
Ins { pos: usize, content: SliceRange },
}
impl<'a> Iterator for EffectIter<'a> {
@ -122,7 +122,7 @@ impl<'a> Iterator for EffectIter<'a> {
debug_assert_eq!(length_diff, len as i32);
return Some(Effect::Ins {
pos: index,
content: content.into(),
content,
});
}
}

75
crates/loro-core/src/container/text/tracker/y_span.rs
View file

@ -189,78 +189,3 @@ impl HasLength for YSpan {
self.slice.atom_len()
}
}
#[cfg(any(test, features = "test_utils"))]
pub mod test {
use crate::{container::text::text_content::ListSlice, op::Content, ContentType, Op, ID};
use rle::{HasLength, RleVecWithIndex};
use super::YSpan;
#[test]
fn test_merge() {
let mut vec: RleVecWithIndex<Op> = RleVecWithIndex::new();
vec.push(Op::new(
ID::new(0, 1),
Content::Dyn(Box::new(YSpan {
origin_left: Some(ID::new(0, 0)),
origin_right: None,
id: ID::new(0, 1),
status: Default::default(),
slice: ListSlice::unknown_range(1),
})),
5,
));
vec.push(Op::new(
ID::new(0, 2),
Content::Dyn(Box::new(YSpan {
origin_left: Some(ID::new(0, 1)),
origin_right: None,
id: ID::new(0, 2),
status: Default::default(),
slice: ListSlice::unknown_range(1),
})),
5,
));
assert_eq!(vec.merged_len(), 1);
let merged = vec.get_merged(0).unwrap();
assert_eq!(merged.content.id(), ContentType::List);
let text_content = merged.content.as_dyn().unwrap();
dbg!(&merged);
assert_eq!(text_content.content_len(), 2);
}
#[test]
fn slice() {
let mut vec: RleVecWithIndex<Op> = RleVecWithIndex::new();
vec.push(Op::new(
ID::new(0, 1),
Content::Dyn(Box::new(YSpan {
origin_left: Some(ID::new(0, 0)),
origin_right: None,
id: ID::new(0, 1),
status: Default::default(),
slice: ListSlice::unknown_range(4),
})),
5,
));
vec.push(Op::new(
ID::new(0, 2),
Content::Dyn(Box::new(YSpan {
origin_left: Some(ID::new(0, 0)),
origin_right: Some(ID::new(0, 1)),
id: ID::new(0, 5),
status: Default::default(),
slice: ListSlice::unknown_range(4),
})),
5,
));
assert_eq!(vec.merged_len(), 2);
assert_eq!(
vec.slice_iter(2, 6)
.map(|x| x.into_inner().content.content_len())
.collect::<Vec<usize>>(),
vec![2, 2]
)
}
}

10
crates/loro-core/src/log_store.rs
View file

@ -145,10 +145,8 @@ impl LogStore {
let mut new_ops = RleVec::new();
for op in change.ops.iter() {
let container = containers.get_mut(&op.container).unwrap();
// TODO: avoid this clone
let mut op = op.clone();
container.to_import(&mut op);
for op in op.convert(self) {
let container_idx = self.get_container_idx(&op.container).unwrap();
for op in op.clone().convert(container, container_idx) {
new_ops.push(op);
}
}
@ -180,9 +178,7 @@ impl LogStore {
fn to_remote_op(&self, op: &Op) -> RemoteOp {
let container = self.reg.get_by_idx(op.container).unwrap();
let mut container = container.lock().unwrap();
let mut op = op.clone().convert(self);
container.to_export(&mut op, self.cfg.gc.gc);
op
op.clone().convert(&mut container, self.cfg.gc.gc)
}
pub(crate) fn create_container(&mut self, container_type: ContainerType) -> ContainerID {

73
crates/loro-core/src/log_store/encoding.rs
View file

@ -1,7 +1,4 @@
use std::{
marker::PhantomPinned,
sync::{Arc, RwLock},
};
use std::sync::{Arc, RwLock};
use fxhash::FxHashMap;
use rle::{HasLength, RleVec, RleVecWithIndex};
@ -14,13 +11,12 @@ use crate::{
container::{
list::list_op::{DeleteSpan, ListOp},
map::MapSet,
registry::ContainerRegistry,
text::text_content::ListSlice,
ContainerID,
Container, ContainerID,
},
dag::remove_included_frontiers,
id::{ClientID, ContainerIdx, Counter, ID},
op::{Content, Op, RemoteOp},
op::{Op, RemoteContent, RemoteOp},
smstring::SmString,
span::{HasIdSpan, HasLamportSpan},
ContainerType, InternalString, LogStore, LoroValue, VersionVector,
@ -31,7 +27,7 @@ type Clients = Vec<ClientID>;
type Containers = Vec<ContainerID>;
#[columnar(vec, ser, de)]
#[derive(Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize)]
struct ChangeEncoding {
#[columnar(strategy = "DeltaRle", original_type = "u32")]
client_idx: ClientIdx,
@ -47,7 +43,7 @@ struct ChangeEncoding {
}
#[columnar(vec, ser, de)]
#[derive(Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize)]
struct OpEncoding {
#[columnar(strategy = "Rle", original_type = "u32")]
container: ContainerIdx,
@ -61,7 +57,7 @@ struct OpEncoding {
}
#[columnar(vec, ser, de)]
#[derive(Copy, Clone, Serialize, Deserialize)]
#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
struct DepsEncoding {
#[columnar(strategy = "Rle", original_type = "u32")]
client_idx: ClientIdx,
@ -79,7 +75,7 @@ impl DepsEncoding {
}
#[columnar(ser, de)]
#[derive(Serialize, Deserialize)]
#[derive(Debug, Serialize, Deserialize)]
struct Encoded {
#[columnar(type = "vec")]
changes: Vec<ChangeEncoding>,
@ -129,7 +125,7 @@ fn encode_changes(store: &LogStore) -> Encoded {
for (op, container) in remote_ops.into_iter().zip(containers.into_iter()) {
for content in op.contents.into_iter() {
let (prop, gc, value) = match content {
crate::op::Content::Map(MapSet { key, value }) => (
crate::op::RemoteContent::Map(MapSet { key, value }) => (
*key_to_idx.entry(key.clone()).or_insert_with(|| {
keys.push(key);
keys.len() - 1
@ -137,7 +133,7 @@ fn encode_changes(store: &LogStore) -> Encoded {
0,
value,
),
crate::op::Content::List(list) => match list {
crate::op::RemoteContent::List(list) => match list {
ListOp::Insert { slice, pos } => (
pos,
match &slice {
@ -155,7 +151,7 @@ fn encode_changes(store: &LogStore) -> Encoded {
(span.pos as usize, 0, LoroValue::I32(span.len as i32))
}
},
crate::op::Content::Dyn(_) => unreachable!(),
crate::op::RemoteContent::Dyn(_) => unreachable!(),
};
op_len += 1;
ops.push(OpEncoding {
@ -191,9 +187,8 @@ fn encode_changes(store: &LogStore) -> Encoded {
fn decode_changes(
encoded: Encoded,
client_id: Option<ClientID>,
mut cfg: Configure,
cfg: Configure,
) -> Arc<RwLock<LogStore>> {
let this_client_id = client_id.unwrap_or_else(|| cfg.rand.next_u64());
let Encoded {
changes: change_encodings,
ops,
@ -215,12 +210,13 @@ fn decode_changes(
return store;
}
let mut container_reg = ContainerRegistry::new();
let mut op_iter = ops.into_iter();
let mut changes = FxHashMap::default();
let mut deps_iter = deps.into_iter();
let log_store = LogStore::new(cfg, client_id);
let mut store = log_store.write().unwrap();
for container in containers.iter() {
container_reg.register(container);
store.reg.register(container);
}
for change_encoding in change_encodings {
@ -245,18 +241,18 @@ fn decode_changes(
let mut op_counter = counter;
for op in op_iter.by_ref().take(op_len as usize) {
let OpEncoding {
container,
container: container_idx,
prop,
value,
gc,
} = op;
let container_id = containers[container as usize].clone();
let container_id = containers[container_idx as usize].clone();
let container_type = container_id.container_type();
let content = match container_type {
ContainerType::Map => {
let key = keys[prop].clone();
Content::Map(MapSet { key, value })
RemoteContent::Map(MapSet { key, value })
}
ContainerType::List | ContainerType::Text => {
let pos = prop;
@ -278,14 +274,17 @@ fn decode_changes(
ListOp::Insert { slice, pos }
}
};
Content::List(list_op)
RemoteContent::List(list_op)
}
};
// TODO: can make this faster
let container_idx = store.get_container_idx(&container_id).unwrap();
let container = store.get_container(&container_id).unwrap();
let op = Op {
counter: op_counter,
container,
content,
container: container_idx,
content: container.lock().unwrap().to_import(content),
};
op_counter += op.content_len() as i32;
@ -311,34 +310,30 @@ fn decode_changes(
.map(|changes| changes.last().unwrap().id_last())
.collect();
let mut frontier = vv.clone();
let mut frontiers = vv.clone();
for (_, changes) in changes.iter() {
for change in changes.iter() {
remove_included_frontiers(&mut frontier, &change.deps);
remove_included_frontiers(&mut frontiers, &change.deps);
}
}
let latest_lamport = changes
store.latest_lamport = changes
.values()
.map(|changes| changes.last().unwrap().lamport_last())
.max()
.unwrap();
let latest_timestamp = changes
store.latest_timestamp = changes
.values()
.map(|changes| changes.last().unwrap().timestamp)
.max()
.unwrap();
Arc::new(RwLock::new(LogStore {
changes,
vv,
cfg,
latest_lamport,
latest_timestamp,
this_client_id,
frontiers: frontier.get_frontiers(),
reg: container_reg,
_pin: PhantomPinned,
}))
store.changes = changes;
store.vv = vv;
store.frontiers = frontiers.get_frontiers();
drop(store);
// FIXME: set all
log_store
}
impl LogStore {

31
crates/loro-core/src/op.rs
View file

@ -1,15 +1,14 @@
use crate::{
change::{Change, Lamport, Timestamp},
container::ContainerID,
container::{registry::ContainerInstance, Container, ContainerID},
id::{ClientID, ContainerIdx, Counter, ID},
span::{HasCounter, HasId, HasLamport},
LogStore,
};
use rle::{HasIndex, HasLength, Mergable, RleVec, Sliceable};
mod content;
pub use content::*;
use smallvec::{smallvec, SmallVec};
use smallvec::SmallVec;
/// Operation is a unit of change.
///
@ -23,14 +22,14 @@ use smallvec::{smallvec, SmallVec};
pub struct Op {
pub(crate) counter: Counter,
pub(crate) container: ContainerIdx,
pub(crate) content: Content,
pub(crate) content: InnerContent,
}
#[derive(Debug, Clone)]
pub struct RemoteOp {
pub(crate) counter: Counter,
pub(crate) container: ContainerID,
pub(crate) contents: RleVec<[Content; 1]>,
pub(crate) contents: RleVec<[RemoteContent; 1]>,
}
/// RichOp includes lamport and timestamp info, which is used for conflict resolution.
@ -46,7 +45,7 @@ pub struct RichOp<'a> {
impl Op {
#[inline]
pub(crate) fn new(id: ID, content: Content, container: u32) -> Self {
pub(crate) fn new(id: ID, content: InnerContent, container: u32) -> Self {
Op {
counter: id.counter,
content,
@ -54,27 +53,29 @@ impl Op {
}
}
pub(crate) fn convert(self, log: &LogStore) -> RemoteOp {
let container = log.reg.get_id(self.container).unwrap().clone();
pub(crate) fn convert(self, container: &mut ContainerInstance, gc: bool) -> RemoteOp {
RemoteOp {
counter: self.counter,
container,
contents: RleVec::from(smallvec![self.content]),
container: container.id().clone(),
contents: RleVec::from(container.to_export(self.content, gc)),
}
}
}
impl RemoteOp {
pub(crate) fn convert(self, log: &mut LogStore) -> SmallVec<[Op; 1]> {
let container = log.get_or_create_container_idx(&self.container);
pub(crate) fn convert(
self,
container: &mut ContainerInstance,
container_idx: u32,
) -> SmallVec<[Op; 1]> {
let mut counter = self.counter;
self.contents
.into_iter()
.map(|content| {
let ans = Op {
counter,
container,
content,
container: container_idx,
content: container.to_import(content),
};
counter += ans.atom_len() as Counter;
ans
@ -104,7 +105,7 @@ impl HasLength for Op {
impl Sliceable for Op {
fn slice(&self, from: usize, to: usize) -> Self {
assert!(to > from);
let content: Content = self.content.slice(from, to);
let content: InnerContent = self.content.slice(from, to);
Op {
counter: (self.counter + from as Counter),
content,

102
crates/loro-core/src/op/content.rs
View file

@ -3,7 +3,10 @@ use std::any::{Any, TypeId};
use enum_as_inner::EnumAsInner;
use rle::{HasLength, Mergable, Sliceable};
use crate::container::{list::list_op::ListOp, map::MapSet};
use crate::container::{
list::list_op::{InnerListOp, ListOp},
map::{InnerMapSet, MapSet},
};
#[derive(PartialEq, Eq, Debug, Clone, Copy)]
pub enum ContentType {
@ -15,14 +18,20 @@ pub enum ContentType {
Custom(u16),
}
#[derive(EnumAsInner, Debug, Clone)]
pub enum InnerContent {
List(InnerListOp),
Map(InnerMapSet),
}
#[derive(EnumAsInner, Debug)]
pub enum Content {
pub enum RemoteContent {
Map(MapSet),
List(ListOp),
Dyn(Box<dyn InsertContentTrait>),
}
impl Clone for Content {
impl Clone for RemoteContent {
fn clone(&self) -> Self {
match self {
Self::Map(arg0) => Self::Map(arg0.clone()),
@ -32,16 +41,6 @@ impl Clone for Content {
}
}
impl Content {
pub fn id(&self) -> ContentType {
match self {
Self::Map(_) => ContentType::Map,
Self::List(_) => ContentType::List,
Self::Dyn(arg0) => arg0.id(),
}
}
}
pub trait MergeableContent {
fn is_mergable_content(&self, other: &dyn InsertContentTrait) -> bool;
fn merge_content(&mut self, other: &dyn InsertContentTrait);
@ -90,35 +89,35 @@ impl<T: Mergable + Any> MergeableContent for T {
}
}
impl HasLength for Content {
impl HasLength for RemoteContent {
fn content_len(&self) -> usize {
match self {
Content::Map(x) => x.content_len(),
Content::Dyn(x) => x.content_len(),
Content::List(x) => x.content_len(),
RemoteContent::Map(x) => x.content_len(),
RemoteContent::Dyn(x) => x.content_len(),
RemoteContent::List(x) => x.content_len(),
}
}
}
impl Sliceable for Content {
impl Sliceable for RemoteContent {
fn slice(&self, from: usize, to: usize) -> Self {
match self {
Content::Map(x) => Content::Map(x.slice(from, to)),
Content::Dyn(x) => Content::Dyn(x.slice_content(from, to)),
Content::List(x) => Content::List(x.slice(from, to)),
RemoteContent::Map(x) => RemoteContent::Map(x.slice(from, to)),
RemoteContent::Dyn(x) => RemoteContent::Dyn(x.slice_content(from, to)),
RemoteContent::List(x) => RemoteContent::List(x.slice(from, to)),
}
}
}
impl Mergable for Content {
impl Mergable for RemoteContent {
fn is_mergable(&self, other: &Self, _conf: &()) -> bool
where
Self: Sized,
{
match (self, other) {
(Content::Map(x), Content::Map(y)) => x.is_mergable(y, &()),
(Content::List(x), Content::List(y)) => x.is_mergable(y, &()),
(Content::Dyn(x), Content::Dyn(y)) => x.is_mergable_content(&**y),
(RemoteContent::Map(x), RemoteContent::Map(y)) => x.is_mergable(y, &()),
(RemoteContent::List(x), RemoteContent::List(y)) => x.is_mergable(y, &()),
(RemoteContent::Dyn(x), RemoteContent::Dyn(y)) => x.is_mergable_content(&**y),
_ => false,
}
}
@ -128,15 +127,58 @@ impl Mergable for Content {
Self: Sized,
{
match self {
Content::Map(x) => match _other {
Content::Map(y) => x.merge(y, &()),
RemoteContent::Map(x) => match _other {
RemoteContent::Map(y) => x.merge(y, &()),
_ => unreachable!(),
},
Content::List(x) => match _other {
Content::List(y) => x.merge(y, &()),
RemoteContent::List(x) => match _other {
RemoteContent::List(y) => x.merge(y, &()),
_ => unreachable!(),
},
Content::Dyn(x) => x.merge_content(&**_other.as_dyn().unwrap()),
RemoteContent::Dyn(x) => x.merge_content(&**_other.as_dyn().unwrap()),
}
}
}
impl HasLength for InnerContent {
fn content_len(&self) -> usize {
match self {
InnerContent::List(list) => list.atom_len(),
InnerContent::Map(_) => 1,
}
}
}
impl Sliceable for InnerContent {
fn slice(&self, from: usize, to: usize) -> Self {
match self {
a @ InnerContent::Map(_) => a.clone(),
InnerContent::List(x) => InnerContent::List(x.slice(from, to)),
}
}
}
impl Mergable for InnerContent {
fn is_mergable(&self, other: &Self, _conf: &()) -> bool
where
Self: Sized,
{
match (self, other) {
(InnerContent::List(x), InnerContent::List(y)) => x.is_mergable(y, &()),
_ => false,
}
}
fn merge(&mut self, _other: &Self, _conf: &())
where
Self: Sized,
{
match self {
InnerContent::List(x) => match _other {
InnerContent::List(y) => x.merge(y, &()),
_ => unreachable!(),
},
InnerContent::Map(_) => unreachable!(),
}
}
}

4
crates/loro-core/src/tests/mod.rs
View file

@ -18,7 +18,7 @@ fn size_of() {
use crate::{
container::{map::MapSet, text::text_content::ListSlice, ContainerID},
id::ID,
op::{Content, Op},
op::{Op, RemoteContent},
span::IdSpan,
Container, InternalString,
};
@ -28,7 +28,7 @@ fn size_of() {
println!("Change {}", std::mem::size_of::<Change>());
println!("Op {}", std::mem::size_of::<Op>());
println!("InsertContent {}", std::mem::size_of::<Content>());
println!("InsertContent {}", std::mem::size_of::<RemoteContent>());
println!("MapSet {}", std::mem::size_of::<MapSet>());
println!("ListSlice {}", std::mem::size_of::<ListSlice>());
println!("Box {}", std::mem::size_of::<Box<dyn Container>>());