mirrors/loro
1
0
Fork 0
mirror of https://github.com/loro-dev/loro.git synced 2025-01-22 21:07:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: decode remove unknown

Browse source
This commit is contained in:
leeeon233 2023-09-12 15:11:53 +08:00
parent dad768049e
commit eb8a07641f
8 changed files with 59 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
crates/loro-internal/examples/encoding.rs
View file

@ -24,7 +24,7 @@ fn main() {
let data = loro.export_from(&Default::default());
let start = Instant::now();
for _ in 0..100 {
for _ in 0..10 {
let mut b = LoroDoc::default();
b.detach();
b.import(&data).unwrap();

8
crates/loro-internal/src/container/list/list_op.rs
View file

@ -54,14 +54,6 @@ impl InnerListOp {
})
}
pub fn new_unknown(pos: usize, len: usize) -> Self {
assert!(len != 0);
Self::Insert {
slice: SliceRange::new_unknown(len as u32),
pos,
}
}
pub fn new_insert(slice: Range<u32>, pos: usize) -> Self {
Self::Insert {
slice: SliceRange(slice),

2
crates/loro-internal/src/container/text/text_content.rs
View file

@ -136,7 +136,7 @@ impl<'a> Sliceable for ListSlice<'a> {
}
impl<'a> Mergable for ListSlice<'a> {
fn is_mergable(&self, other: &Self, _: &()) -> bool {
fn is_mergable(&self, _other: &Self, _: &()) -> bool {
false
}
}

32
crates/loro-internal/src/encoding/encode_enhanced.rs
View file

@ -70,11 +70,11 @@ struct OpEncoding {
/// key index or insert/delete pos
#[columnar(strategy = "DeltaRle")]
prop: usize,
// the length of the deletion
#[columnar(strategy = "BoolRle")]
is_del: bool,
// the length of the deletion or insertion
#[columnar(strategy = "Rle")]
del_len: isize,
#[columnar(strategy = "Rle")]
insert_len: usize,
insert_del_len: isize,
}
#[columnar(vec, ser, de, iterable)]
@ -183,7 +183,7 @@ pub fn encode_oplog_v2(oplog: &OpLog, vv: &VersionVector) -> Vec<u8> {
let container_index = *container_idx2index.get(&container).unwrap();
let op = oplog.local_op_to_remote(op);
for content in op.contents.into_iter() {
let (prop, del_len, insert_len) = match content {
let (prop, is_del, insert_del_len) = match content {
crate::op::RawOpContent::Map(MapSet { key, value }) => {
values.push(value.clone());
(
@ -191,7 +191,7 @@ pub fn encode_oplog_v2(oplog: &OpLog, vv: &VersionVector) -> Vec<u8> {
keys.push(key.clone());
keys.len() - 1
}),
-1,
false,
0,
)
}
@ -210,19 +210,20 @@ pub fn encode_oplog_v2(oplog: &OpLog, vv: &VersionVector) -> Vec<u8> {
string.push_str(str.deref());
}
};
(pos, -1, len)
assert!(len > 0);
(pos, false, len as isize)
}
ListOp::Delete(span) => {
assert!(span.len >= 0);
(span.pos as usize, span.len, 0)
// span.len maybe negative
(span.pos as usize, true, span.len)
}
},
};
op_len += 1;
ops.push(OpEncoding {
prop,
del_len,
insert_len,
is_del,
insert_del_len,
container: container_index,
})
}
@ -414,9 +415,9 @@ pub fn decode_oplog_v2(oplog: &mut OpLog, input: &[u8]) -> Result<(), LoroError>
for op in op_iter.by_ref().take(op_len as usize) {
let OpEncoding {
container: container_idx,
insert_len,
prop,
del_len,
is_del,
insert_del_len,
} = op;
let Some(container_id) = get_container(container_idx) else {
@ -433,14 +434,15 @@ pub fn decode_oplog_v2(oplog: &mut OpLog, input: &[u8]) -> Result<(), LoroError>
}
ContainerType::List | ContainerType::Text => {
let pos = prop;
if del_len >= 0 {
if is_del {
RawOpContent::List(ListOp::Delete(DeleteSpan {
pos: pos as isize,
len: del_len,
len: insert_del_len,
}))
} else {
match container_type {
ContainerType::Text => {
let insert_len = insert_del_len as usize;
let s = &str[str_index..str_index + insert_len];
str_index += insert_len;
RawOpContent::List(ListOp::Insert {

115
crates/loro-internal/src/snapshot_encode.rs
View file

@ -118,11 +118,6 @@ pub fn decode_oplog(
InnerContent::List(InnerListOp::new_del(pos, len)),
container_idx,
),
SnapshotOp::TextOrListUnknown { len, pos } => Op::new(
id,
InnerContent::List(InnerListOp::new_unknown(pos, len)),
container_idx,
),
SnapshotOp::Map { .. } => {
unreachable!()
}
@ -322,11 +317,11 @@ struct EncodedSnapshotOp {
// Map: always 0
#[columnar(strategy = "DeltaRle")]
len: i64,
// List: insert 0 | unkonwn -2 | deletion -1
// Text: insert 0 | unkonwn -2 | deletion -1
// List: insert 0 | deletion -1
// Text: insert 0 | deletion -1
// Map: always 0
#[columnar(strategy = "Rle")]
kind: i64,
#[columnar(strategy = "BoolRle")]
is_del: bool,
// Text: 0
// List: 0 | value index
// Map: value index
@ -338,22 +333,16 @@ enum SnapshotOp {
TextInsert { pos: usize, len: usize },
ListInsert { pos: usize, value_idx: u32 },
TextOrListDelete { pos: usize, len: isize },
TextOrListUnknown { pos: usize, len: usize },
Map { key: usize, value_idx_plus_one: u32 },
}
impl EncodedSnapshotOp {
pub fn get_text(&self) -> SnapshotOp {
if self.kind == -1 {
if self.is_del {
SnapshotOp::TextOrListDelete {
pos: self.prop,
len: self.len as isize,
}
} else if self.kind == -2 {
SnapshotOp::TextOrListUnknown {
pos: self.prop,
len: self.len as usize,
}
} else {
SnapshotOp::TextInsert {
pos: self.prop,
@ -363,16 +352,11 @@ impl EncodedSnapshotOp {
}
pub fn get_list(&self) -> SnapshotOp {
if self.kind == -1 {
if self.is_del {
SnapshotOp::TextOrListDelete {
pos: self.prop,
len: self.len as isize,
}
} else if self.kind == -2 {
SnapshotOp::TextOrListUnknown {
pos: self.prop,
len: self.len as usize,
}
} else {
SnapshotOp::ListInsert {
pos: self.prop,
@ -397,21 +381,14 @@ impl EncodedSnapshotOp {
container,
prop: pos,
len: 0,
kind: 0,
is_del: false,
value: start as usize,
},
SnapshotOp::TextOrListDelete { pos, len } => Self {
container,
prop: pos,
len: len as i64,
kind: -1,
value: 0,
},
SnapshotOp::TextOrListUnknown { pos, len } => Self {
container,
prop: pos,
len: len as i64,
kind: -2,
is_del: true,
value: 0,
},
SnapshotOp::Map {
@ -421,14 +398,14 @@ impl EncodedSnapshotOp {
container,
prop: key,
len: 0,
kind: 0,
is_del: false,
value: value as usize,
},
SnapshotOp::TextInsert { pos, len } => Self {
container,
prop: pos,
len: len as i64,
kind: 0,
is_del: false,
value: 0,
},
}
@ -644,51 +621,39 @@ fn encode_oplog(oplog: &OpLog, state_ref: Option<PreEncodedState>) -> FinalPhase
for op in change.ops.iter() {
match &op.content {
InnerContent::List(list) => match list {
InnerListOp::Insert { slice, pos } => {
if slice.is_unknown() {
encoded_ops.push(EncodedSnapshotOp::from(
SnapshotOp::TextOrListUnknown {
pos: *pos,
len: slice.atom_len(),
},
op.container.to_index(),
));
} else {
match op.container.get_type() {
loro_common::ContainerType::Text => {
let range = slice.0.start as usize..slice.0.end as usize;
let mut pos = *pos;
oplog.arena.with_text_slice(range, |slice| {
encoded_ops.push(record_str(
slice.as_bytes(),
pos,
op.container.to_index(),
));
InnerListOp::Insert { slice, pos } => match op.container.get_type() {
loro_common::ContainerType::Text => {
let range = slice.0.start as usize..slice.0.end as usize;
let mut pos = *pos;
oplog.arena.with_text_slice(range, |slice| {
encoded_ops.push(record_str(
slice.as_bytes(),
pos,
op.container.to_index(),
));
pos += slice.chars().count();
})
}
loro_common::ContainerType::List => {
let values = oplog
.arena
.get_values(slice.0.start as usize..slice.0.end as usize);
let mut pos = *pos;
for value in values {
let idx = record_value(&value);
encoded_ops.push(EncodedSnapshotOp::from(
SnapshotOp::ListInsert {
pos,
value_idx: idx as u32,
},
op.container.to_index(),
));
pos += 1;
}
}
loro_common::ContainerType::Map => unreachable!(),
pos += slice.chars().count();
})
}
loro_common::ContainerType::List => {
let values = oplog
.arena
.get_values(slice.0.start as usize..slice.0.end as usize);
let mut pos = *pos;
for value in values {
let idx = record_value(&value);
encoded_ops.push(EncodedSnapshotOp::from(
SnapshotOp::ListInsert {
pos,
value_idx: idx as u32,
},
op.container.to_index(),
));
pos += 1;
}
}
}
loro_common::ContainerType::Map => unreachable!(),
},
InnerListOp::Delete(del) => {
encoded_ops.push(EncodedSnapshotOp::from(
SnapshotOp::TextOrListDelete {

BIN
crates/loro-internal/tests/history_compressed_rle_updates.dat
View file

Binary file not shown.

BIN
crates/loro-internal/tests/history_snapshot.dat
View file

Binary file not shown.

2
crates/loro-internal/tests/test.rs
View file

@ -1,5 +1,3 @@
use std::fs::File;
use loro_common::ID;
use loro_internal::{version::Frontiers, LoroDoc, ToJson};