mirrors/loro
1
0
Fork 0
mirror of https://github.com/loro-dev/loro.git synced 2025-01-22 12:57:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(wasm) Extract VersionVector class and fix inconsistent PeerID repr (#249)

Browse source 
* refactor(wasm): extract VersionVector class and fix inconsistent PeerID in wasm

* fix: example type err

* fix: binding err

* fix: peer id repr should be consistent

* test: update tests
This commit is contained in:
Zixuan Chen 2024-01-18 13:28:28 +08:00 committed by GitHub
parent ce1ac36b62
commit 1295ac6d61
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 196 additions and 158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.vscode/settings.json vendored
View file

@ -1,6 +1,7 @@
{
"cSpell.words": [
"arbtest",
"bolds",
"cids",
"clippy",
"collab",
@ -25,9 +26,11 @@
"thiserror",
"tinyvec",
"txns",
"typeparam",
"unbold",
"unexist",
"unmark",
"unmergeable",
"yspan"
],
"rust-analyzer.runnableEnv": {

7
crates/loro-common/src/id.rs
View file

@ -15,7 +15,7 @@ impl Debug for ID {
impl Display for ID {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(format!("{}@{:X}", self.counter, self.peer).as_str())
f.write_str(format!("{}@{}", self.counter, self.peer).as_str())
}
}
@ -33,7 +33,10 @@ impl TryFrom<&str> for ID {
.unwrap()
.parse::<Counter>()
.map_err(|_| LoroError::DecodeError("Invalid ID format".into()))?;
let client_id = u64::from_str_radix(iter.next().unwrap(), 16)
let client_id = iter
.next()
.unwrap()
.parse::<u64>()
.map_err(|_| LoroError::DecodeError("Invalid ID format".into()))?;
Ok(ID {
peer: client_id,

2
crates/loro-common/src/lib.rs
View file

@ -376,7 +376,7 @@ mod test {
container_type: crate::ContainerType::Map,
};
let id_str = id.to_string();
assert_eq!(id_str.as_str(), "cid:10@FF:Map");
assert_eq!(id_str.as_str(), "cid:10@255:Map");
assert_eq!(ContainerID::try_from(id_str.as_str()).unwrap(), id);
let id = ContainerID::try_from("cid:root-a:b:c:Tree").unwrap();

2
crates/loro-internal/src/container.rs
View file

@ -187,7 +187,7 @@ mod test {
fn container_id_convert() {
let container_id = ContainerID::new_normal(ID::new(12, 12), ContainerType::List);
let s = container_id.to_string();
assert_eq!(s, "cid:12@C:List");
assert_eq!(s, "cid:12@12:List");
let actual = ContainerID::try_from(s.as_str()).unwrap();
assert_eq!(actual, container_id);

276
crates/loro-wasm/src/lib.rs
View file

@ -10,7 +10,8 @@ use loro_internal::{
id::{Counter, TreeID, ID},
obs::SubID,
version::Frontiers,
ContainerType, DiffEvent, LoroDoc, LoroError, LoroValue, VersionVector,
ContainerType, DiffEvent, LoroDoc, LoroError, LoroValue,
VersionVector as InternalVersionVector,
};
use rle::HasLength;
use serde::{Deserialize, Serialize};
@ -56,6 +57,8 @@ pub struct Loro(Arc<LoroDoc>);
#[wasm_bindgen]
extern "C" {
#[wasm_bindgen(typescript_type = "number | bigint | string")]
pub type JsIntoPeerID;
#[wasm_bindgen(typescript_type = "ContainerID")]
pub type JsContainerID;
#[wasm_bindgen(typescript_type = "ContainerID | string")]
@ -80,8 +83,10 @@ extern "C" {
pub type JsChanges;
#[wasm_bindgen(typescript_type = "Change")]
pub type JsChange;
#[wasm_bindgen(typescript_type = "Map<PeerID, number> | Uint8Array")]
pub type JsVersionVector;
#[wasm_bindgen(
typescript_type = "Map<PeerID, number> | Uint8Array | VersionVector | undefined | null"
)]
pub type JsIntoVersionVector;
#[wasm_bindgen(typescript_type = "Value | Container")]
pub type JsValueOrContainer;
#[wasm_bindgen(typescript_type = "Value | Container | undefined")]
@ -124,9 +129,7 @@ mod observer {
}
}
// TODO: need to double check whether this is safe
unsafe impl Send for Observer {}
// TODO: need to double check whether this is safe
unsafe impl Sync for Observer {}
}
@ -181,22 +184,6 @@ fn js_value_to_container_id(
Ok(cid)
}
fn js_value_to_version(version: &JsValue) -> Result<VersionVector, JsValue> {
let version: Option<Vec<u8>> = if version.is_null() || version.is_undefined() {
None
} else {
let arr: Uint8Array = Uint8Array::new(version);
Some(arr.to_vec())
};
let vv = match version {
Some(x) => VersionVector::decode(&x)?,
None => Default::default(),
};
Ok(vv)
}
#[derive(Debug, Clone, Serialize)]
struct StringID {
peer: String,
@ -330,7 +317,7 @@ impl Loro {
self.0.is_detached()
}
/// Checkout the `DocState` to the lastest version of `OpLog`.
/// Checkout the `DocState` to the latest version of `OpLog`.
///
/// > The document becomes detached during a `checkout` operation.
/// > Being `detached` implies that the `DocState` is not synchronized with the latest version of the `OpLog`.
@ -365,7 +352,7 @@ impl Loro {
/// > In a detached state, the document is not editable, and any `import` operations will be
/// > recorded in the `OpLog` without being applied to the `DocState`.
///
/// You should call `attach` to attach the `DocState` to the lastest version of `OpLog`.
/// You should call `attach` to attach the `DocState` to the latest version of `OpLog`.
///
/// @param frontiers - the specific frontiers
///
@ -391,10 +378,10 @@ impl Loro {
self.0.peer_id()
}
/// Get peer id in hex string.
/// Get peer id in decimal string.
#[wasm_bindgen(js_name = "peerIdStr", method, getter)]
pub fn peer_id_str(&self) -> String {
format!("{:X}", self.0.peer_id())
format!("{}", self.0.peer_id())
}
/// Set the peer ID of the current writer.
@ -402,12 +389,13 @@ impl Loro {
/// Note: use it with caution. You need to make sure there is not chance that two peers
/// have the same peer ID.
#[wasm_bindgen(js_name = "setPeerId", method)]
pub fn set_peer_id(&self, id: u64) -> JsResult<()> {
pub fn set_peer_id(&self, peer_id: JsIntoPeerID) -> JsResult<()> {
let id = id_value_to_u64(peer_id.into())?;
self.0.set_peer_id(id)?;
Ok(())
}
/// Commit the cumulative auto commit transaction.
/// Commit the cumulative auto committed transaction.
pub fn commit(&self, origin: Option<String>) {
self.0.commit_with(origin.map(|x| x.into()), None, true);
}
@ -548,17 +536,16 @@ impl Loro {
///
/// If you checkout to a specific version, the version vector will change.
#[inline(always)]
pub fn version(&self) -> Vec<u8> {
self.0.state_vv().encode()
pub fn version(&self) -> VersionVector {
VersionVector(self.0.state_vv())
}
/// Get the encoded version vector of the lastest version in OpLog.
/// Get the encoded version vector of the latest version in OpLog.
///
/// If you checkout to a specific version, the version vector will not change.
#[inline(always)]
#[wasm_bindgen(js_name = "oplogVersion")]
pub fn oplog_version(&self) -> Vec<u8> {
self.0.oplog_vv().encode()
pub fn oplog_version(&self) -> VersionVector {
VersionVector(self.0.oplog_vv())
}
/// Get the frontiers of the current document.
@ -569,7 +556,7 @@ impl Loro {
frontiers_to_ids(&self.0.state_frontiers())
}
/// Get the frontiers of the lastest version in OpLog.
/// Get the frontiers of the latest version in OpLog.
///
/// If you checkout to a specific version, this value will not change.
#[inline(always)]
@ -628,10 +615,13 @@ impl Loro {
/// const updates2 = doc.exportFrom(version);
/// ```
#[wasm_bindgen(skip_typescript, js_name = "exportFrom")]
pub fn export_from(&self, version: &JsValue) -> JsResult<Vec<u8>> {
// `version` may be null or undefined
let vv = js_value_to_version(version)?;
Ok(self.0.export_from(&vv))
pub fn export_from(&self, vv: Option<VersionVector>) -> JsResult<Vec<u8>> {
if let Some(vv) = vv {
// `version` may be null or undefined
Ok(self.0.export_from(&vv.0))
} else {
Ok(self.0.export_from(&Default::default()))
}
}
/// Import a snapshot or a update to current doc.
@ -882,17 +872,14 @@ impl Loro {
/// const version = doc.frontiersToVV(frontiers);
/// ```
#[wasm_bindgen(js_name = "frontiersToVV")]
pub fn frontiers_to_vv(&self, frontiers: Vec<JsID>) -> JsResult<JsVersionVectorMap> {
pub fn frontiers_to_vv(&self, frontiers: Vec<JsID>) -> JsResult<VersionVector> {
let frontiers = ids_to_frontiers(frontiers)?;
let borrow_mut = &self.0;
let oplog = borrow_mut.oplog().try_lock().unwrap();
oplog
.dag()
.frontiers_to_vv(&frontiers)
.map(|vv| {
let ans: JsVersionVectorMap = vv_to_js_value(vv).into();
ans
})
.map(VersionVector)
.ok_or_else(|| JsError::new("Frontiers not found").into())
}
@ -909,42 +896,12 @@ impl Loro {
/// const frontiers = doc.vvToFrontiers(version);
/// ```
#[wasm_bindgen(js_name = "vvToFrontiers")]
pub fn vv_to_frontiers(&self, vv: &JsVersionVector) -> JsResult<Vec<JsID>> {
let value: JsValue = vv.into();
let is_bytes = value.is_instance_of::<js_sys::Uint8Array>();
let vv = if is_bytes {
let bytes = js_sys::Uint8Array::from(value.clone());
let bytes = bytes.to_vec();
VersionVector::decode(&bytes)?
} else {
let map = js_sys::Map::from(value);
js_map_to_vv(map)?
};
let f = self.0.oplog().lock().unwrap().dag().vv_to_frontiers(&vv);
pub fn vv_to_frontiers(&self, vv: &VersionVector) -> JsResult<Vec<JsID>> {
let f = self.0.oplog().lock().unwrap().dag().vv_to_frontiers(&vv.0);
Ok(frontiers_to_ids(&f))
}
}
fn js_map_to_vv(map: js_sys::Map) -> JsResult<VersionVector> {
let mut vv = VersionVector::new();
for pair in map.entries() {
let pair = pair.unwrap_throw();
let key = Reflect::get(&pair, &0.into()).unwrap_throw();
let peer_id = key.as_string().expect_throw("PeerID must be string");
let value = Reflect::get(&pair, &1.into()).unwrap_throw();
let counter = value.as_f64().expect_throw("Invalid counter") as Counter;
vv.insert(
peer_id
.parse()
.expect_throw(&format!("{} cannot be parsed as u64", peer_id)),
counter,
);
}
Ok(vv)
}
#[allow(unused)]
fn call_subscriber(ob: observer::Observer, e: DiffEvent, doc: Arc<LoroDoc>) {
// We convert the event to js object here, so that we don't need to worry about GC.
@ -1148,8 +1105,6 @@ impl LoroText {
///
/// *You should make sure that a key is always associated with the same expand type.*
///
/// Note: you cannot delete unmergeable annotations like comments by this method.
///
/// @example
/// ```ts
/// import { Loro } from "loro-crdt";
@ -1849,7 +1804,7 @@ impl LoroTree {
/// const node = tree.create(root);
/// const node2 = tree.create(node);
/// tree.mov(node2, root);
/// // Error wiil be thrown if move operation creates a cycle
/// // Error will be thrown if move operation creates a cycle
/// tree.mov(root, node);
/// ```
pub fn mov(&mut self, target: JsTreeID, parent: Option<JsTreeID>) -> JsResult<()> {
@ -2069,62 +2024,6 @@ impl LoroTree {
}
}
/// Convert a encoded version vector to a readable js Map.
///
/// @example
/// ```ts
/// import { Loro } from "loro-crdt";
///
/// const doc = new Loro();
/// doc.setPeerId('100');
/// doc.getText("t").insert(0, 'a');
/// doc.commit();
/// const version = doc.getVersion();
/// const readableVersion = convertVersionToReadableObj(version);
/// console.log(readableVersion); // Map(1) { 100n => 1 }
/// ```
#[wasm_bindgen(js_name = "toReadableVersion")]
pub fn to_readable_version(version: &[u8]) -> Result<JsVersionVectorMap, JsValue> {
let version_vector = VersionVector::decode(version)?;
let map = vv_to_js_value(version_vector);
Ok(JsVersionVectorMap::from(map))
}
/// Convert a readable js Map to a encoded version vector.
///
/// @example
/// ```ts
/// import { Loro } from "loro-crdt";
///
/// const doc = new Loro();
/// doc.setPeerId('100');
/// doc.getText("t").insert(0, 'a');
/// doc.commit();
/// const version = doc.getVersion();
/// const readableVersion = convertVersionToReadableObj(version);
/// console.log(readableVersion); // Map(1) { 100n => 1 }
/// const encodedVersion = toEncodedVersion(readableVersion);
/// ```
#[wasm_bindgen(js_name = "toEncodedVersion")]
pub fn to_encoded_version(version: JsVersionVectorMap) -> Result<Vec<u8>, JsValue> {
let map: JsValue = version.into();
let map: js_sys::Map = map.into();
let vv = js_map_to_vv(map)?;
let encoded = vv.encode();
Ok(encoded)
}
fn vv_to_js_value(vv: VersionVector) -> JsValue {
let map = js_sys::Map::new();
for (k, v) in vv.iter() {
let k = k.to_string().into();
let v = JsValue::from(*v);
map.set(&k, &v);
}
map.into()
}
fn loro_value_to_js_value_or_container(value: ValueOrContainer, doc: &Arc<LoroDoc>) -> JsValue {
match value {
ValueOrContainer::Value(v) => {
@ -2138,12 +2037,111 @@ fn loro_value_to_js_value_or_container(value: ValueOrContainer, doc: &Arc<LoroDo
}
}
#[wasm_bindgen]
#[derive(Debug, Default)]
pub struct VersionVector(pub(crate) InternalVersionVector);
#[wasm_bindgen]
impl VersionVector {
#[wasm_bindgen(constructor)]
pub fn new(value: JsIntoVersionVector) -> JsResult<VersionVector> {
let value: JsValue = value.into();
if value.is_null() || value.is_undefined() {
return Ok(Self::default());
}
let is_bytes = value.is_instance_of::<js_sys::Uint8Array>();
if is_bytes {
let bytes = js_sys::Uint8Array::from(value.clone());
let bytes = bytes.to_vec();
return VersionVector::decode(&bytes);
}
VersionVector::from_json(JsVersionVectorMap::from(value))
}
#[wasm_bindgen(js_name = "parseJSON", method)]
pub fn from_json(version: JsVersionVectorMap) -> JsResult<VersionVector> {
let map: JsValue = version.into();
let map: js_sys::Map = map.into();
let mut vv = InternalVersionVector::new();
for pair in map.entries() {
let pair = pair.unwrap_throw();
let key = Reflect::get(&pair, &0.into()).unwrap_throw();
let peer_id = key.as_string().expect_throw("PeerID must be string");
let value = Reflect::get(&pair, &1.into()).unwrap_throw();
let counter = value.as_f64().expect_throw("Invalid counter") as Counter;
vv.insert(
peer_id
.parse()
.expect_throw(&format!("{} cannot be parsed as u64", peer_id)),
counter,
);
}
Ok(Self(vv))
}
#[wasm_bindgen(js_name = "toJSON", method)]
pub fn to_json(&self) -> JsVersionVectorMap {
let vv = &self.0;
let map = js_sys::Map::new();
for (k, v) in vv.iter() {
let k = k.to_string().into();
let v = JsValue::from(*v);
map.set(&k, &v);
}
let value: JsValue = map.into();
JsVersionVectorMap::from(value)
}
pub fn encode(&self) -> Vec<u8> {
self.0.encode()
}
pub fn decode(bytes: &[u8]) -> JsResult<VersionVector> {
let vv = InternalVersionVector::decode(bytes)?;
Ok(Self(vv))
}
pub fn get(&self, peer_id: JsIntoPeerID) -> JsResult<Option<Counter>> {
let id = id_value_to_u64(peer_id.into())?;
Ok(self.0.get(&id).copied())
}
pub fn compare(&self, other: &VersionVector) -> Option<i32> {
self.0.partial_cmp(&other.0).map(|o| match o {
std::cmp::Ordering::Less => -1,
std::cmp::Ordering::Equal => 0,
std::cmp::Ordering::Greater => 1,
})
}
}
fn id_value_to_u64(value: JsValue) -> JsResult<u64> {
if value.is_bigint() {
let bigint = js_sys::BigInt::from(value);
let v: u64 = bigint.try_into().unwrap_throw();
Ok(v)
} else if value.is_string() {
let v: u64 = value.as_string().unwrap().parse().unwrap_throw();
Ok(v)
} else if let Some(v) = value.as_f64() {
Ok(v as u64)
} else {
Err(JsValue::from_str(
"id value must be a string, number or bigint",
))
}
}
#[wasm_bindgen(typescript_custom_section)]
const TYPES: &'static str = r#"
/**
* Container types supported by loro.
*
* It is most commonly used to specify the type of subcontainer to be created.
* It is most commonly used to specify the type of sub-container to be created.
* @example
* ```ts
* import { Loro } from "loro-crdt";
@ -2156,6 +2154,7 @@ const TYPES: &'static str = r#"
* ```
*/
export type ContainerType = "Text" | "Map" | "List"| "Tree";
export type PeerID = string;
/**
* The unique id of each container.
@ -2179,11 +2178,10 @@ export type ContainerID =
export type TreeID = `${number}@${string}`;
interface Loro {
exportFrom(version?: Uint8Array): Uint8Array;
exportFromV0(version?: Uint8Array): Uint8Array;
exportFromCompressed(version?: Uint8Array): Uint8Array;
exportFrom(version?: VersionVector): Uint8Array;
getContainerById(id: ContainerID): LoroText | LoroMap | LoroList;
}
/**
* Represents a `Delta` type which is a union of different operations that can be performed.
*
@ -2221,10 +2219,12 @@ export type Delta<T> =
delete?: undefined;
insert?: undefined;
};
/**
* The unique id of each operation.
*/
export type OpId = { peer: PeerID, counter: number };
/**
* Change is a group of continuous operations
*/

4
examples/loro-quill/src/App.vue
View file

@ -5,7 +5,7 @@
import "quill/dist/quill.bubble.css";
import "quill/dist/quill.snow.css";
import { QuillBinding } from "./binding";
import { Loro, toReadableVersion } from "loro-crdt";
import { Loro } from "loro-crdt";
const editor1 = ref<null | HTMLDivElement>(null);
const editor2 = ref<null | HTMLDivElement>(null);
@ -62,7 +62,7 @@
}
Promise.resolve().then(() => {
const version = text.version();
const map = toReadableVersion(version);
const map = version.toJSON();
const versionObj = {};
for (const [key, value] of map) {
versionObj[key.toString()] = value;

19
examples/loro-quill/src/binding.ts
View file

@ -25,6 +25,13 @@ export class QuillBinding {
public doc: Loro,
public quill: Quill,
) {
doc.configTextStyle({
bold: { expand: "after" },
italic: { expand: "after" },
underline: { expand: "after" },
link: { expand: "none" },
header: { expand: "none" },
})
this.quill = quill;
this.richtext = doc.getText("text");
this.richtext.subscribe(doc, (event) => {
@ -94,7 +101,9 @@ export class QuillBinding {
const a = this.richtext.toDelta();
const b = this.quill.getContents().ops;
console.log(this.doc.peerId, "COMPARE AFTER QUILL_EVENT");
assertEqual(a, b as any);
if (!assertEqual(a, b as any)) {
this.quill.setContents(new Delta(a), "this" as any);
}
console.log("SIZE", this.doc.exportFrom().length);
this.doc.debugHistory();
}
@ -113,9 +122,9 @@ export class QuillBinding {
for (const key of Object.keys(op.attributes)) {
let value = op.attributes[key];
if (value == null) {
this.richtext.unmark({ start: index, end, expand: EXPAND_CONFIG[key] }, key)
this.richtext.unmark({ start: index, end, }, key)
} else {
this.richtext.mark({ start: index, end, expand: EXPAND_CONFIG[key] }, key, value,)
this.richtext.mark({ start: index, end }, key, value,)
}
}
}
@ -128,9 +137,9 @@ export class QuillBinding {
for (const key of Object.keys(op.attributes)) {
let value = op.attributes[key];
if (value == null) {
this.richtext.unmark({ start: index, end, expand: EXPAND_CONFIG[key] }, key)
this.richtext.unmark({ start: index, end, }, key)
} else {
this.richtext.mark({ start: index, end, expand: EXPAND_CONFIG[key] }, key, value)
this.richtext.mark({ start: index, end }, key, value)
}
}
}

5
loro-js/tests/basic.test.ts
View file

@ -4,11 +4,12 @@ import {
LoroList,
LoroMap,
isContainer,
toEncodedVersion,
getType,
VersionVector,
} from "../src";
import { Container } from "../dist/loro";
it("basic example", () => {
const doc = new Loro();
const list: LoroList = doc.getList("list");
@ -155,7 +156,7 @@ describe("import", () => {
b.getText("text").insert(1, "b");
b.getList("list").insert(0, [1, 2]);
const updates = b.exportFrom(
toEncodedVersion(b.frontiersToVV(a.frontiers())),
b.frontiersToVV(a.frontiers()),
);
a.import(updates);
expect(a.toJson()).toStrictEqual(b.toJson());

5
loro-js/tests/misc.test.ts
View file

@ -3,6 +3,7 @@ import {
Loro,
LoroList,
LoroMap,
VersionVector,
} from "../src";
import { expectTypeOf } from "vitest";
@ -131,8 +132,8 @@ describe("sync", () => {
it("two insert at beginning", async () => {
const a = new Loro();
const b = new Loro();
let a_version: undefined | Uint8Array = undefined;
let b_version: undefined | Uint8Array = undefined;
let a_version: undefined | VersionVector = undefined;
let b_version: undefined | VersionVector = undefined;
a.subscribe((e: { local: boolean }) => {
if (e.local) {
const exported = a.exportFrom(a_version);

31
loro-js/tests/version.test.ts
View file

@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { Loro, toReadableVersion, setPanicHook, OpId } from "../src";
import { Loro, OpId, VersionVector } from "../src";
describe("Frontiers", () => {
it("two clients", () => {
@ -26,6 +26,25 @@ describe("Frontiers", () => {
});
});
it('peer id repr should be consistent', () => {
const doc = new Loro();
const id = doc.peerIdStr;
doc.getText("text").insert(0, "hello");
doc.commit();
const f = doc.frontiers();
expect(f[0].peer).toBe(id);
const map = doc.getList("list").insertContainer(0, "Map");
const mapId = map.id;
const peerIdInContainerId = mapId.split(":")[1].split("@")[1]
expect(peerIdInContainerId).toBe(id);
doc.commit();
expect(doc.version().get(id)).toBe(6);
expect(doc.version().toJSON().get(id)).toBe(6);
const m = doc.getMap(mapId);
m.set("0", 1);
expect(map.get("0")).toBe(1)
})
describe("Version", () => {
const a = new Loro();
a.setPeerId(0n);
@ -42,10 +61,12 @@ describe("Version", () => {
const vv = new Map();
vv.set("0", 3);
vv.set("1", 2);
expect(toReadableVersion(a.version())).toStrictEqual(vv);
expect(toReadableVersion(a.version())).toStrictEqual(vv);
expect(a.vvToFrontiers(vv)).toStrictEqual(a.frontiers());
expect(a.vvToFrontiers(a.version())).toStrictEqual(a.frontiers());
expect((a.version().toJSON())).toStrictEqual(vv);
expect((a.version().toJSON())).toStrictEqual(vv);
expect(a.vvToFrontiers(new VersionVector(vv))).toStrictEqual(a.frontiers());
const v = a.version();
const temp = a.vvToFrontiers(v);
expect(temp).toStrictEqual(a.frontiers());
expect(a.frontiers()).toStrictEqual([{ peer: "0", counter: 2 }] as OpId[]);
}
});