2024-03-08 08:40:06 +00:00
|
|
|
use std::{fmt::Debug, time::Instant};
|
|
|
|
|
2022-11-15 07:53:03 +00:00
|
|
|
pub fn minify_error<T, F, N>(site_num: u8, actions: Vec<T>, f: F, normalize: N)
|
|
|
|
where
|
|
|
|
F: Fn(u8, &mut [T]),
|
|
|
|
N: Fn(u8, &mut [T]) -> Vec<T>,
|
|
|
|
T: Clone + Debug,
|
|
|
|
{
|
|
|
|
std::panic::set_hook(Box::new(|_info| {
|
|
|
|
// ignore panic output
|
2023-03-08 12:09:51 +00:00
|
|
|
// println!("{:?}", _info);
|
2022-11-15 07:53:03 +00:00
|
|
|
}));
|
|
|
|
|
|
|
|
let f_ref: *const _ = &f;
|
|
|
|
let f_ref: usize = f_ref as usize;
|
2023-07-11 16:29:23 +00:00
|
|
|
#[allow(clippy::redundant_clone)]
|
feat: stabilizing encoding (#219)
This PR implements a new encode schema that is more extendible and more compact. It’s also simpler and takes less binary size and maintaining effort. It is inspired by the [Automerge Encoding Format](https://automerge.org/automerge-binary-format-spec/).
The main motivation is the extensibility. When we integrate a new CRDT algorithm, we don’t want to make a breaking change to the encoding or keep multiple versions of the encoding schema in the code, as it will make our WASM size much larger. We need a stable and extendible encoding schema for our v1.0 version.
This PR also exposes the ops that compose the current container state. For example, now you can make a query about which operation a certain character quickly. This behavior is required in the new snapshot encoding, so it’s included in this PR.
# Encoding Schema
## Header
The header has 22 bytes.
- (0-4 bytes) Magic Bytes: The encoding starts with `loro` as magic bytes.
- (4-20 bytes) Checksum: MD5 checksum of the encoded data, including the header starting from 20th bytes. The checksum is encoded as a 16-byte array. The `checksum` and `magic bytes` fields are trimmed when calculating the checksum.
- (20-21 bytes) Encoding Method (2 bytes, big endian): Multiple encoding methods are available for a specific encoding version.
## Encode Mode: Updates
In this approach, only ops, specifically their historical record, are encoded, while document states are excluded.
Like Automerge's format, we employ columnar encoding for operations and changes.
Previously, operations were ordered by their Operation ID (OpId) before columnar encoding. However, sorting operations based on their respective containers initially enhance compression potential.
## Encode Mode: Snapshot
This mode simultaneously captures document state and historical data. Upon importing a snapshot into a new document, initialization occurs directly from the snapshot, bypassing the need for CRDT-based recalculations.
Unlike previous snapshot encoding methods, the current binary output in snapshot mode is compatible with the updates mode. This enhances the efficiency of importing snapshots into non-empty documents, where initialization via snapshot is infeasible.
Additionally, when feasible, we leverage the sequence of operations to construct state snapshots. In CRDTs, deducing the specific ops constituting the current container state is feasible. These ops are tagged in relation to the container, facilitating direct state reconstruction from them. This approach, pioneered by Automerge, significantly improves compression efficiency.
2024-01-02 09:03:24 +00:00
|
|
|
let mut actions_clone = actions.clone();
|
|
|
|
let action_ref: usize = (&mut actions_clone) as *mut _ as usize;
|
2024-02-18 09:27:33 +00:00
|
|
|
#[allow(clippy::blocks_in_conditions)]
|
2022-11-15 07:53:03 +00:00
|
|
|
if std::panic::catch_unwind(|| {
|
|
|
|
// SAFETY: test
|
|
|
|
let f = unsafe { &*(f_ref as *const F) };
|
|
|
|
// SAFETY: test
|
|
|
|
let actions_ref = unsafe { &mut *(action_ref as *mut Vec<T>) };
|
|
|
|
f(site_num, actions_ref);
|
|
|
|
})
|
|
|
|
.is_ok()
|
|
|
|
{
|
|
|
|
println!("No Error Found");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut minified = actions.clone();
|
|
|
|
let mut candidates = Vec::new();
|
|
|
|
for i in 0..actions.len() {
|
|
|
|
let mut new = actions.clone();
|
|
|
|
new.remove(i);
|
|
|
|
candidates.push(new);
|
|
|
|
}
|
|
|
|
|
|
|
|
println!("Minifying...");
|
|
|
|
let start = Instant::now();
|
|
|
|
while let Some(candidate) = candidates.pop() {
|
|
|
|
let f_ref: *const _ = &f;
|
|
|
|
let f_ref: usize = f_ref as usize;
|
feat: stabilizing encoding (#219)
This PR implements a new encode schema that is more extendible and more compact. It’s also simpler and takes less binary size and maintaining effort. It is inspired by the [Automerge Encoding Format](https://automerge.org/automerge-binary-format-spec/).
The main motivation is the extensibility. When we integrate a new CRDT algorithm, we don’t want to make a breaking change to the encoding or keep multiple versions of the encoding schema in the code, as it will make our WASM size much larger. We need a stable and extendible encoding schema for our v1.0 version.
This PR also exposes the ops that compose the current container state. For example, now you can make a query about which operation a certain character quickly. This behavior is required in the new snapshot encoding, so it’s included in this PR.
# Encoding Schema
## Header
The header has 22 bytes.
- (0-4 bytes) Magic Bytes: The encoding starts with `loro` as magic bytes.
- (4-20 bytes) Checksum: MD5 checksum of the encoded data, including the header starting from 20th bytes. The checksum is encoded as a 16-byte array. The `checksum` and `magic bytes` fields are trimmed when calculating the checksum.
- (20-21 bytes) Encoding Method (2 bytes, big endian): Multiple encoding methods are available for a specific encoding version.
## Encode Mode: Updates
In this approach, only ops, specifically their historical record, are encoded, while document states are excluded.
Like Automerge's format, we employ columnar encoding for operations and changes.
Previously, operations were ordered by their Operation ID (OpId) before columnar encoding. However, sorting operations based on their respective containers initially enhance compression potential.
## Encode Mode: Snapshot
This mode simultaneously captures document state and historical data. Upon importing a snapshot into a new document, initialization occurs directly from the snapshot, bypassing the need for CRDT-based recalculations.
Unlike previous snapshot encoding methods, the current binary output in snapshot mode is compatible with the updates mode. This enhances the efficiency of importing snapshots into non-empty documents, where initialization via snapshot is infeasible.
Additionally, when feasible, we leverage the sequence of operations to construct state snapshots. In CRDTs, deducing the specific ops constituting the current container state is feasible. These ops are tagged in relation to the container, facilitating direct state reconstruction from them. This approach, pioneered by Automerge, significantly improves compression efficiency.
2024-01-02 09:03:24 +00:00
|
|
|
let mut actions_clone = candidate.clone();
|
|
|
|
let action_ref: usize = (&mut actions_clone) as *mut _ as usize;
|
2024-02-18 09:27:33 +00:00
|
|
|
#[allow(clippy::blocks_in_conditions)]
|
2022-11-15 07:53:03 +00:00
|
|
|
if std::panic::catch_unwind(|| {
|
|
|
|
// SAFETY: test
|
|
|
|
let f = unsafe { &*(f_ref as *const F) };
|
|
|
|
// SAFETY: test
|
|
|
|
let actions_ref = unsafe { &mut *(action_ref as *mut Vec<T>) };
|
|
|
|
f(site_num, actions_ref);
|
|
|
|
})
|
|
|
|
.is_err()
|
|
|
|
{
|
|
|
|
for i in 0..candidate.len() {
|
|
|
|
let mut new = candidate.clone();
|
|
|
|
new.remove(i);
|
|
|
|
candidates.push(new);
|
|
|
|
}
|
|
|
|
if candidate.len() < minified.len() {
|
|
|
|
minified = candidate;
|
|
|
|
println!("New min len={}", minified.len());
|
|
|
|
}
|
|
|
|
if candidates.len() > 40 {
|
|
|
|
candidates.drain(0..30);
|
|
|
|
}
|
|
|
|
}
|
2022-11-18 08:16:09 +00:00
|
|
|
if start.elapsed().as_secs() > 10 && minified.len() <= 4 {
|
|
|
|
break;
|
2022-11-15 07:53:03 +00:00
|
|
|
}
|
|
|
|
if start.elapsed().as_secs() > 60 {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
let minified = normalize(site_num, &mut minified);
|
|
|
|
println!(
|
|
|
|
"Old Length {}, New Length {}",
|
|
|
|
actions.len(),
|
|
|
|
minified.len()
|
|
|
|
);
|
|
|
|
dbg!(&minified);
|
|
|
|
if actions.len() > minified.len() {
|
|
|
|
minify_error(site_num, minified, f, normalize);
|
|
|
|
}
|
|
|
|
}
|