diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..10efcb2fe --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Debug", + "program": "${workspaceFolder}/", + "args": [], + "cwd": "${workspaceFolder}" + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index e8d2d886a..acba475fe 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -20,6 +20,7 @@ "tinyvec", "txns", "unbold", + "unexist", "unmark", "yspan" ], diff --git a/crates/loro-common/src/lib.rs b/crates/loro-common/src/lib.rs index 52d16de0e..a9b3d5c8c 100644 --- a/crates/loro-common/src/lib.rs +++ b/crates/loro-common/src/lib.rs @@ -3,7 +3,6 @@ use std::{fmt::Display, sync::Arc}; use arbitrary::Arbitrary; use enum_as_inner::EnumAsInner; -use fxhash::FxHashMap; use serde::{Deserialize, Serialize}; mod error; mod id; @@ -95,12 +94,7 @@ impl ContainerType { ContainerType::Map => LoroValue::Map(Arc::new(Default::default())), ContainerType::List => LoroValue::List(Arc::new(Default::default())), ContainerType::Text => LoroValue::String(Arc::new(Default::default())), - ContainerType::Tree => { - let mut map: FxHashMap = FxHashMap::default(); - map.insert("roots".to_string(), LoroValue::List(vec![].into())); - map.insert("deleted".to_string(), LoroValue::List(vec![].into())); - map.into() - } + ContainerType::Tree => LoroValue::List(Arc::new(Default::default())), } } @@ -334,6 +328,10 @@ impl TreeID { counter: self.counter, } } + + pub fn associated_meta_container(&self) -> ContainerID { + ContainerID::new_normal(self.id(), ContainerType::Map) + } } impl Display for TreeID { diff --git a/crates/loro-internal/examples/tree.rs b/crates/loro-internal/examples/tree.rs index 22983fee3..1b576d1e4 100644 --- a/crates/loro-internal/examples/tree.rs +++ b/crates/loro-internal/examples/tree.rs @@ -3,8 +3,33 @@ use std::time::Instant; use loro_internal::LoroDoc; use rand::{rngs::StdRng, Rng}; -fn main() { - let s = Instant::now(); +fn checkout() { + let depth = 300; + let mut loro = LoroDoc::default(); + let tree = loro.get_tree("tree"); + let mut ids = vec![]; + let mut versions = vec![]; + let id1 = loro.with_txn(|txn| tree.create(txn)).unwrap(); + ids.push(id1); + versions.push(loro.oplog_frontiers()); + for _ in 1..depth { + let id = loro + .with_txn(|txn| tree.create_and_mov(txn, *ids.last().unwrap())) + .unwrap(); + ids.push(id); + versions.push(loro.oplog_frontiers()); + } + let mut rng: StdRng = rand::SeedableRng::seed_from_u64(0); + + for _ in 0..1000 { + let i = rng.gen::() % depth; + let f = &versions[i]; + loro.checkout(f).unwrap(); + } +} + +#[allow(unused)] +fn mov() { let loro = LoroDoc::default(); let tree = loro.get_tree("tree"); let mut ids = vec![]; @@ -22,5 +47,12 @@ fn main() { tree.mov(&mut txn, ids[i], ids[j]).unwrap_or_default(); } drop(txn); +} +fn main() { + let s = Instant::now(); + for _ in 0..30 { + checkout(); + } + println!("{} ms", s.elapsed().as_millis()); } diff --git a/crates/loro-internal/src/arena.rs b/crates/loro-internal/src/arena.rs index bd1bb061d..f9da7681a 100644 --- a/crates/loro-internal/src/arena.rs +++ b/crates/loro-internal/src/arena.rs @@ -7,7 +7,6 @@ use std::{ use append_only_bytes::BytesSlice; use fxhash::FxHashMap; -use loro_common::ContainerType; use loro_common::PeerID; use crate::{ @@ -165,7 +164,7 @@ impl<'a> OpConverter<'a> { crate::op::RawOpContent::Tree(tree) => { // we need create every meta container associated with target TreeID let id = tree.target; - let meta_container_id = ContainerID::new_normal(id.id(), ContainerType::Map); + let meta_container_id = id.associated_meta_container(); if self.container_id_to_idx.get(&meta_container_id).is_none() { let container_idx_to_id = &mut self.container_idx_to_id; diff --git a/crates/loro-internal/src/container/richtext/tracker/crdt_rope.rs b/crates/loro-internal/src/container/richtext/tracker/crdt_rope.rs index d9ca25677..9080d5606 100644 --- a/crates/loro-internal/src/container/richtext/tracker/crdt_rope.rs +++ b/crates/loro-internal/src/container/richtext/tracker/crdt_rope.rs @@ -352,6 +352,7 @@ impl CrdtRope { while let Some((index, elem)) = iter.next() { // The elements will not be changed by this method. // This index is current index of the elem (calculated by `status` field rather than `diff_status` field) + debug_log::debug_dbg!(&index, &elem); match elem.diff() { DiffStatus::NotChanged => {} DiffStatus::Created => { diff --git a/crates/loro-internal/src/delta.rs b/crates/loro-internal/src/delta.rs index d78b7be3d..eedb4488d 100644 --- a/crates/loro-internal/src/delta.rs +++ b/crates/loro-internal/src/delta.rs @@ -7,4 +7,7 @@ pub use map_delta::{MapDelta, MapValue}; mod text; pub use text::{StyleMeta, StyleMetaItem}; mod tree; -pub use tree::{TreeDelta, TreeDiff, TreeDiffItem}; +pub(crate) use tree::TreeValue; +pub use tree::{ + TreeDelta, TreeDeltaItem, TreeDiff, TreeDiffItem, TreeExternalDiff, TreeInternalDiff, +}; diff --git a/crates/loro-internal/src/delta/seq.rs b/crates/loro-internal/src/delta/seq.rs index 35aa0a086..457d37ca9 100644 --- a/crates/loro-internal/src/delta/seq.rs +++ b/crates/loro-internal/src/delta/seq.rs @@ -605,7 +605,7 @@ impl Delta { delta.chop() } - fn concat(mut self, mut other: Self) -> Self { + pub(crate) fn concat(mut self, mut other: Self) -> Self { if !other.vec.is_empty() { let other_first = other.vec.remove(0); self.push(other_first); diff --git a/crates/loro-internal/src/delta/tree.rs b/crates/loro-internal/src/delta/tree.rs index bab9850cb..846dc981b 100644 --- a/crates/loro-internal/src/delta/tree.rs +++ b/crates/loro-internal/src/delta/tree.rs @@ -1,43 +1,163 @@ -use loro_common::TreeID; +use std::{ + ops::{Deref, DerefMut}, + sync::Arc, +}; + +use fxhash::{FxHashMap, FxHashSet}; +use loro_common::{ContainerType, LoroValue, TreeID}; use serde::Serialize; +use smallvec::{smallvec, SmallVec}; + +#[derive(Debug, Clone, Default, Serialize)] +pub struct TreeDiff { + pub(crate) diff: Vec, +} + +#[derive(Debug, Clone, Copy, Serialize)] +pub struct TreeDiffItem { + pub target: TreeID, + pub action: TreeExternalDiff, +} + +#[derive(Debug, Clone, Copy, Serialize)] +pub enum TreeExternalDiff { + Create, + Move(Option), + Delete, +} + +impl TreeDiffItem { + pub(crate) fn from_delta_item(item: TreeDeltaItem) -> SmallVec<[TreeDiffItem; 2]> { + let target = item.target; + match item.action { + TreeInternalDiff::Create | TreeInternalDiff::Restore => { + smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Create + }] + } + TreeInternalDiff::AsRoot => { + smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Move(None) + }] + } + TreeInternalDiff::Move(p) => { + smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Move(Some(p)) + }] + } + TreeInternalDiff::CreateMove(p) | TreeInternalDiff::RestoreMove(p) => { + smallvec![ + TreeDiffItem { + target, + action: TreeExternalDiff::Create + }, + TreeDiffItem { + target, + action: TreeExternalDiff::Move(Some(p)) + } + ] + } + TreeInternalDiff::Delete | TreeInternalDiff::UnCreate => { + smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Delete + }] + } + } + } +} + +impl TreeDiff { + pub(crate) fn compose(self, _other: Self) -> Self { + unreachable!("tree compose") + } + + pub(crate) fn extend>(mut self, other: I) -> Self { + self.diff.extend(other); + self + } +} /// Representation of differences in movable tree. It's an ordered list of [`TreeDiff`]. #[derive(Debug, Clone, Default, Serialize)] pub struct TreeDelta { - pub(crate) diff: Vec, + pub(crate) diff: Vec, } /// The semantic action in movable tree. #[derive(Debug, Clone, Copy, Serialize)] -pub struct TreeDiff { +pub struct TreeDeltaItem { pub target: TreeID, - pub action: TreeDiffItem, + pub action: TreeInternalDiff, } /// The action of [`TreeDiff`]. It's the same as [`crate::container::tree::tree_op::TreeOp`], but semantic. #[derive(Debug, Clone, Copy, Serialize)] -pub enum TreeDiffItem { - CreateOrRestore, +pub enum TreeInternalDiff { + /// First create the node, have not seen it before + Create, + /// Recreate the node, the node has been deleted before + Restore, + /// Same as move to `None` and the node is exist + AsRoot, + /// Move the node to the parent, the node is exist Move(TreeID), + /// First create the node and move it to the parent + CreateMove(TreeID), + /// Recreate the node, and move it to the parent + RestoreMove(TreeID), + /// Delete the node Delete, + /// For retreating, if the node is only created, not move it to `DELETED_ROOT` but delete it directly UnCreate, } -impl From<(TreeID, Option)> for TreeDiff { - fn from(value: (TreeID, Option)) -> Self { - let (target, parent) = value; - let action = if let Some(p) = parent { - if TreeID::is_deleted_root(parent) { - TreeDiffItem::Delete - } else if TreeID::is_unexist_root(parent) { - TreeDiffItem::UnCreate - } else { - TreeDiffItem::Move(p) +impl TreeDeltaItem { + pub(crate) fn new( + target: TreeID, + parent: Option, + old_parent: Option, + is_parent_deleted: bool, + is_old_parent_deleted: bool, + ) -> Self { + let action = match (parent, old_parent) { + (Some(p), _) => { + if is_parent_deleted { + TreeInternalDiff::Delete + } else if TreeID::is_unexist_root(parent) { + TreeInternalDiff::UnCreate + } else if TreeID::is_unexist_root(old_parent) { + TreeInternalDiff::CreateMove(p) + } else if is_old_parent_deleted { + TreeInternalDiff::RestoreMove(p) + } else { + TreeInternalDiff::Move(p) + } + } + (None, Some(_)) => { + if TreeID::is_unexist_root(old_parent) { + TreeInternalDiff::Create + } else if is_old_parent_deleted { + TreeInternalDiff::Restore + } else { + TreeInternalDiff::AsRoot + } + } + (None, None) => { + unreachable!() } - } else { - TreeDiffItem::CreateOrRestore }; - TreeDiff { target, action } + TreeDeltaItem { target, action } + } +} + +impl Deref for TreeDelta { + type Target = Vec; + fn deref(&self) -> &Self::Target { + &self.diff } } @@ -46,9 +166,81 @@ impl TreeDelta { pub(crate) fn compose(&self, _x: TreeDelta) -> TreeDelta { unimplemented!("tree compose") } +} - pub(crate) fn push(mut self, diff: TreeDiff) -> Self { - self.diff.push(diff); - self +#[derive(Debug)] +pub(crate) struct TreeValue<'a>(pub(crate) &'a mut Vec); + +impl<'a> TreeValue<'a> { + pub(crate) fn apply_diff(&mut self, diff: &TreeDiff) { + for d in diff.diff.iter() { + let target = d.target; + match d.action { + TreeExternalDiff::Create => self.create_target(target), + TreeExternalDiff::Delete => self.delete_target(target), + TreeExternalDiff::Move(parent) => self.mov(target, parent), + } + } + } + + fn mov(&mut self, target: TreeID, parent: Option) { + let map = self + .0 + .iter_mut() + .find(|x| { + let id = x.as_map().unwrap().get("id").unwrap().as_string().unwrap(); + id.as_ref() == &target.to_string() + }) + .unwrap() + .as_map_mut() + .unwrap(); + let map_mut = Arc::make_mut(map); + let p = if let Some(p) = parent { + p.to_string().into() + } else { + LoroValue::Null + }; + map_mut.insert("parent".to_string(), p); + } + + fn create_target(&mut self, target: TreeID) { + let mut t = FxHashMap::default(); + t.insert("id".to_string(), target.id().to_string().into()); + t.insert("parent".to_string(), LoroValue::Null); + t.insert("meta".to_string(), ContainerType::Map.default_value()); + self.0.push(t.into()); + } + + fn delete_target(&mut self, target: TreeID) { + let mut deleted = FxHashSet::default(); + let mut s = vec![target.to_string()]; + while let Some(delete) = s.pop() { + deleted.insert(delete.clone()); + self.0.retain_mut(|x| { + let id = x.as_map().unwrap().get("id").unwrap().as_string().unwrap(); + !deleted.contains(id.as_ref()) + }); + for node in self.0.iter() { + let node = node.as_map().unwrap().as_ref(); + if let Some(LoroValue::String(parent)) = node.get("parent") { + if parent.as_ref() == &delete { + s.push((*node.get("id").unwrap().as_string().unwrap().clone()).clone()); + } + } + } + } + } +} + +impl Deref for TreeDiff { + type Target = Vec; + fn deref(&self) -> &Self::Target { + &self.diff + } +} + +impl DerefMut for TreeDiff { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.diff } } diff --git a/crates/loro-internal/src/design.excalidraw b/crates/loro-internal/src/design.excalidraw index 0a5ad1540..cc1c5899a 100644 --- a/crates/loro-internal/src/design.excalidraw +++ b/crates/loro-internal/src/design.excalidraw @@ -491,8 +491,8 @@ }, { "type": "text", - "version": 122, - "versionNonce": 1767273320, + "version": 124, + "versionNonce": 1161247789, "isDeleted": false, "id": "o9-gbN3_3SW7l1OMVBHv4", "fillStyle": "hachure", @@ -512,7 +512,7 @@ "frameId": null, "roundness": null, "boundElements": [], - "updated": 1697094746332, + "updated": 1698975088180, "link": null, "locked": false, "fontSize": 28, @@ -527,8 +527,8 @@ }, { "type": "text", - "version": 103, - "versionNonce": 1409996568, + "version": 105, + "versionNonce": 973842723, "isDeleted": false, "id": "sV53JjHOqVdbNK-CZIScM", "fillStyle": "hachure", @@ -548,7 +548,7 @@ "frameId": null, "roundness": null, "boundElements": [], - "updated": 1697094746332, + "updated": 1698975088180, "link": null, "locked": false, "fontSize": 28, @@ -563,8 +563,8 @@ }, { "type": "text", - "version": 266, - "versionNonce": 1745077864, + "version": 268, + "versionNonce": 2092510861, "isDeleted": false, "id": "gn2aMXWdRMAPol-lsYZJB", "fillStyle": "hachure", @@ -584,7 +584,7 @@ "frameId": null, "roundness": null, "boundElements": [], - "updated": 1697094746332, + "updated": 1698975088181, "link": null, "locked": false, "fontSize": 28, @@ -599,8 +599,8 @@ }, { "type": "text", - "version": 173, - "versionNonce": 1607480344, + "version": 175, + "versionNonce": 698387651, "isDeleted": false, "id": "wFeIjJ1VGyOJMI-trmac6", "fillStyle": "hachure", @@ -620,7 +620,7 @@ "frameId": null, "roundness": null, "boundElements": [], - "updated": 1697094746333, + "updated": 1698975088181, "link": null, "locked": false, "fontSize": 20, @@ -757,8 +757,8 @@ }, { "type": "text", - "version": 233, - "versionNonce": 1804515688, + "version": 235, + "versionNonce": 671995117, "isDeleted": false, "id": "R3cAPuLZewGeyAKN504Ax", "fillStyle": "hachure", @@ -778,7 +778,7 @@ "frameId": null, "roundness": null, "boundElements": [], - "updated": 1697094746333, + "updated": 1698975088181, "link": null, "locked": false, "fontSize": 28, @@ -793,8 +793,8 @@ }, { "type": "text", - "version": 37, - "versionNonce": 1692528920, + "version": 39, + "versionNonce": 135789667, "isDeleted": false, "id": "H2UWV51x64jg8zNDe_6qV", "fillStyle": "hachure", @@ -814,7 +814,7 @@ "frameId": null, "roundness": null, "boundElements": [], - "updated": 1697094746333, + "updated": 1698975088181, "link": null, "locked": false, "fontSize": 28, @@ -2525,8 +2525,8 @@ }, { "type": "text", - "version": 371, - "versionNonce": 1195592808, + "version": 373, + "versionNonce": 582922061, "isDeleted": false, "id": "vbnvqLubmb9mSigy7IPzB", "fillStyle": "hachure", @@ -2546,7 +2546,7 @@ "frameId": null, "roundness": null, "boundElements": [], - "updated": 1697094746333, + "updated": 1698975088181, "link": null, "locked": false, "fontSize": 28, @@ -3338,8 +3338,8 @@ }, { "type": "text", - "version": 62, - "versionNonce": 984878616, + "version": 64, + "versionNonce": 1900050435, "isDeleted": false, "id": "esyuXNMFxHhWITBsKcfkg", "fillStyle": "hachure", @@ -3359,7 +3359,7 @@ "frameId": null, "roundness": null, "boundElements": [], - "updated": 1697094746333, + "updated": 1698975088181, "link": null, "locked": false, "fontSize": 20, @@ -3582,8 +3582,8 @@ }, { "type": "text", - "version": 399, - "versionNonce": 82673512, + "version": 401, + "versionNonce": 455425453, "isDeleted": false, "id": "M5ld9FicUuQ134VaZ-Hcl", "fillStyle": "hachure", @@ -3603,7 +3603,7 @@ "frameId": null, "roundness": null, "boundElements": [], - "updated": 1697094746333, + "updated": 1698975088181, "link": null, "locked": false, "fontSize": 20, diff --git a/crates/loro-internal/src/diff_calc.rs b/crates/loro-internal/src/diff_calc.rs index a66d188fa..bfdbcc6ec 100644 --- a/crates/loro-internal/src/diff_calc.rs +++ b/crates/loro-internal/src/diff_calc.rs @@ -1,7 +1,9 @@ use std::sync::Arc; pub(super) mod tree; +use debug_log::debug_dbg; use itertools::Itertools; +pub(crate) use tree::TreeDeletedSetTrait; pub(super) use tree::TreeDiffCache; use enum_dispatch::enum_dispatch; @@ -20,7 +22,7 @@ use crate::{ tree::tree_op::TreeOp, }, dag::DagUtils, - delta::{Delta, MapDelta, MapValue}, + delta::{Delta, MapDelta, MapValue, TreeInternalDiff}, event::InternalDiff, id::Counter, op::RichOp, @@ -75,6 +77,7 @@ impl DiffCalculator { after: &crate::VersionVector, after_frontiers: Option<&Frontiers>, ) -> Vec { + debug_dbg!(&before, &after, &oplog); if self.has_all { let include_before = self.last_vv.includes_vv(before); let include_after = self.last_vv.includes_vv(after); @@ -107,7 +110,6 @@ impl DiffCalculator { self.has_all = true; self.last_vv = Default::default(); } - let (lca, iter) = oplog.iter_from_lca_causally(before, before_frontiers, after, after_frontiers); @@ -208,10 +210,10 @@ impl DiffCalculator { } }; - // Because we need to get correct `reset` value that indicates container is created during this round of diff calc, + // Because we need to get correct `bring_back` value that indicates container is created during this round of diff calc, // we need to iterate from parents to children. i.e. from smaller depth to larger depth. - let mut new_containers: FxHashSet = FxHashSet::default(); - let empty_vv: VersionVector = Default::default(); + let mut new_containers = FxHashSet::default(); + let mut container_id_to_depth = FxHashMap::default(); let mut all: Vec<(u16, ContainerIdx)> = if let Some(set) = affected_set { // only visit the affected containers set.into_iter() @@ -226,7 +228,6 @@ impl DiffCalculator { .map(|(x, (depth, _))| (*depth, *x)) .collect() }; - let mut are_rest_containers_deleted = false; let mut ans = FxHashMap::default(); while !all.is_empty() { @@ -237,81 +238,82 @@ impl DiffCalculator { if ans.contains_key(&idx) { continue; } - let (depth, calc) = self.calculators.get_mut(&idx).unwrap(); if *depth == u16::MAX && !are_rest_containers_deleted { if let Some(d) = oplog.arena.get_depth(idx) { - *depth = d; + if d != *depth { + *depth = d; + all.push((*depth, idx)); + continue; + } } - - all.push((*depth, idx)); - continue; } + let id = oplog.arena.idx_to_id(idx).unwrap(); + let bring_back = new_containers.remove(&id); - let (from, reset) = if new_containers.remove(&oplog.arena.idx_to_id(idx).unwrap()) { - // if the container is new, we need to calculate the diff from the beginning - (&empty_vv, true) - } else { - (before, false) - }; - - let diff = calc.calculate_diff(oplog, from, after, |c| { - new_containers.insert(c.clone()); - let child_idx = oplog.arena.register_container(c); - oplog.arena.set_parent(child_idx, Some(idx)); - }); - if !diff.is_empty() || reset { - ans.insert( - idx, - InternalContainerDiff { - idx, - reset, - is_container_deleted: are_rest_containers_deleted, - diff: diff.into(), - }, - ); - } - } - - // reset left new_containers - while !new_containers.is_empty() { - for id in std::mem::take(&mut new_containers) { - let Some(idx) = oplog.arena.id_to_idx(&id) else { - continue; - }; - let Some((_, calc)) = self.calculators.get_mut(&idx) else { - continue; - }; - let diff = calc.calculate_diff(oplog, &empty_vv, after, |c| { + let diff = calc.calculate_diff(oplog, before, after, |c| { + if !are_rest_containers_deleted { new_containers.insert(c.clone()); + container_id_to_depth.insert(c.clone(), depth.saturating_add(1)); let child_idx = oplog.arena.register_container(c); oplog.arena.set_parent(child_idx, Some(idx)); - }); - // this can override the previous diff with `reset = false` - // otherwise, the diff event will be incorrect + } + }); + if !diff.is_empty() || bring_back { ans.insert( idx, - InternalContainerDiff { - idx, - reset: true, - is_container_deleted: false, - diff: diff.into(), - }, + ( + *depth, + InternalContainerDiff { + idx, + bring_back, + is_container_deleted: are_rest_containers_deleted, + diff: Some(diff.into()), + }, + ), ); } } + debug_log::debug_dbg!(&new_containers); if len == all.len() { - // debug_log::debug_dbg!(&all); - // for (_, idx) in all.iter() { - // debug_log::debug_dbg!(oplog.arena.get_container_id(*idx)); - // } + debug_log::debug_log!("Container might be deleted"); + debug_log::debug_dbg!(&all); + for (_, idx) in all.iter() { + debug_log::debug_dbg!(oplog.arena.get_container_id(*idx)); + } // we still emit the event of deleted container are_rest_containers_deleted = true; } } - - ans.into_iter().map(|x| x.1).collect_vec() + while !new_containers.is_empty() { + for id in std::mem::take(&mut new_containers) { + let Some(idx) = oplog.arena.id_to_idx(&id) else { + continue; + }; + if ans.contains_key(&idx) { + continue; + } + let depth = container_id_to_depth.remove(&id).unwrap(); + ans.insert( + idx, + ( + depth, + InternalContainerDiff { + idx, + bring_back: true, + is_container_deleted: false, + diff: None, + }, + ), + ); + } + } + debug_log::debug_dbg!(&ans); + ans.into_values() + .sorted_by_key(|x| x.0) + .map(|x| x.1) + .collect_vec() } } @@ -428,9 +430,9 @@ impl DiffCalculatorTrait for MapDiffCalculator { value: None, lamport: (0, 0), }); + updated.insert(key, value); } - InternalDiff::Map(MapDelta { updated }) } } @@ -751,9 +753,9 @@ impl DiffCalculatorTrait for TreeDiffCalculator { oplog: &OpLog, from: &crate::VersionVector, to: &crate::VersionVector, - _: impl FnMut(&ContainerID), + mut on_new_container: impl FnMut(&ContainerID), ) -> InternalDiff { - // debug_log::debug_log!("from {:?} to {:?}", from, to); + debug_log::debug_log!("from {:?} to {:?}", from, to); let mut merged_vv = from.clone(); merged_vv.merge(to); let from_frontiers = from.to_frontiers(&oplog.dag); @@ -763,7 +765,7 @@ impl DiffCalculatorTrait for TreeDiffCalculator { .find_common_ancestor(&from_frontiers, &to_frontiers); let lca_vv = oplog.dag.frontiers_to_vv(&common_ancestors).unwrap(); let lca_frontiers = lca_vv.to_frontiers(&oplog.dag); - // debug_log::debug_log!("lca vv {:?}", lca_vv); + debug_log::debug_log!("lca vv {:?}", lca_vv); let mut tree_cache = oplog.tree_parent_cache.lock().unwrap(); let to_max_lamport = self.get_max_lamport_by_frontiers(&to_frontiers, oplog); @@ -779,8 +781,21 @@ impl DiffCalculatorTrait for TreeDiffCalculator { (from_min_lamport, from_max_lamport), ); - // FIXME: inserting new containers - // debug_log::debug_log!("\ndiff {:?}", diff); + diff.diff.iter().for_each(|d| { + // the metadata could be modified before, so (re)create a node need emit the map container diffs + // `Create` here is because maybe in a diff calc uncreate and then create back + if matches!( + d.action, + TreeInternalDiff::Restore + | TreeInternalDiff::RestoreMove(_) + | TreeInternalDiff::Create + | TreeInternalDiff::CreateMove(_) + ) { + on_new_container(&d.target.associated_meta_container()) + } + }); + + debug_log::debug_log!("\ndiff {:?}", diff); InternalDiff::Tree(diff) } diff --git a/crates/loro-internal/src/diff_calc/tree.rs b/crates/loro-internal/src/diff_calc/tree.rs index a47238519..9ea551391 100644 --- a/crates/loro-internal/src/diff_calc/tree.rs +++ b/crates/loro-internal/src/diff_calc/tree.rs @@ -3,11 +3,15 @@ use std::{ ops::{Deref, DerefMut}, }; -use fxhash::FxHashMap; +use fxhash::{FxHashMap, FxHashSet}; use itertools::Itertools; use loro_common::{CounterSpan, IdSpan, TreeID, ID}; -use crate::{change::Lamport, delta::TreeDelta, VersionVector}; +use crate::{ + change::Lamport, + delta::{TreeDelta, TreeDeltaItem, TreeInternalDiff}, + VersionVector, +}; /// All information of an operation for diff calculating of movable tree. #[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Eq, Ord)] @@ -38,6 +42,7 @@ impl core::hash::Hash for MoveLamportAndID { pub struct TreeDiffCache { cache: Cache, pending: BTreeSet, + deleted: FxHashSet, all_version: VersionVector, current_version: VersionVector, } @@ -87,8 +92,11 @@ impl TreeDiffCache { // Because importing the local op must not cause circular references, it has been checked. pub(crate) fn add_node_uncheck(&mut self, node: MoveLamportAndID) { if !self.all_version.includes_id(node.id) { + let old_parent = self.get_parent(node.target); + + self.update_deleted_cache(node.target, node.parent, old_parent); + self.cache.entry(node.target).or_default().insert(node); - // assert len == 1 self.current_version.set_last(node.id); self.all_version.set_last(node.id); } @@ -108,7 +116,6 @@ impl TreeDiffCache { lca_min_lamport: Lamport, from_min_max_lamport: (Lamport, Lamport), ) -> TreeDelta { - // println!("\nFROM {:?} TO {:?} LCA {:?}", from, to, lca); self.checkout(from, from_min_max_lamport.0, from_min_max_lamport.1); // println!( // "current vv {:?} all vv {:?}", @@ -131,13 +138,8 @@ impl TreeDiffCache { lca_min_lamport: Lamport, ) -> TreeDelta { debug_log::group!("tree calc diff"); - let mut diff = Vec::new(); - let revert_ops = self.retreat_for_diff(lca, lca_min_lamport); - for (op, old_parent) in revert_ops.iter().sorted().rev() { - if op.effected { - diff.push((op.target, *old_parent).into()); - } - } + let mut diff = self.retreat_for_diff(lca, lca_min_lamport); + debug_log::debug_log!("revert diff:"); for d in diff.iter() { debug_log::debug_log!(" {:?}", d); @@ -145,11 +147,40 @@ impl TreeDiffCache { let apply_ops = self.forward(to, to_max_lamport); debug_log::debug_log!("apply ops {:?}", apply_ops); for op in apply_ops.into_iter() { + let old_parent = self.get_parent(op.target); + let is_parent_deleted = + op.parent.is_some() && self.is_deleted(op.parent.as_ref().unwrap()); + let is_old_parent_deleted = + old_parent.is_some() && self.is_deleted(old_parent.as_ref().unwrap()); let effected = self.apply(op); if effected { - debug_log::debug_log!(" target {:?} to {:?}", op.target, op.parent); - - diff.push((op.target, op.parent).into()) + // we need to know whether op.parent is deleted + let this_diff = TreeDeltaItem::new( + op.target, + op.parent, + old_parent, + is_parent_deleted, + is_old_parent_deleted, + ); + debug_log::debug_log!(" {:?}", this_diff); + diff.push(this_diff); + if matches!( + this_diff.action, + TreeInternalDiff::Restore | TreeInternalDiff::RestoreMove(_) + ) { + // TODO: perf how to get children faster + let mut s = vec![op.target]; + while let Some(t) = s.pop() { + let children = self.get_children(t); + children.iter().for_each(|c| { + diff.push(TreeDeltaItem { + target: *c, + action: TreeInternalDiff::CreateMove(t), + }) + }); + s.extend(children); + } + } } } debug_log::debug_log!("diff {:?}", diff); @@ -157,6 +188,7 @@ impl TreeDiffCache { } fn checkout(&mut self, vv: &VersionVector, min_lamport: Lamport, max_lamport: Lamport) { + // TODO: use all as max, because from version may contain `meta op`, current version != from version if vv == &self.current_version { return; } @@ -175,6 +207,8 @@ impl TreeDiffCache { ans = false; } node.effected = ans; + let old_parent = self.get_parent(node.target); + self.update_deleted_cache(node.target, node.parent, old_parent); self.cache.entry(node.target).or_default().insert(node); self.current_version.set_last(node.id); ans @@ -215,15 +249,17 @@ impl TreeDiffCache { self.current_version.shrink_to_exclude(IdSpan { client_id: op.id.peer, counter: CounterSpan::new(op.id.counter, op.id.counter + 1), - }) + }); + if op.effected { + // update deleted cache + let old_parent = self.get_parent(op.target); + self.update_deleted_cache(op.target, old_parent, op.parent); + } } } - fn retreat_for_diff( - &mut self, - vv: &VersionVector, - min_lamport: Lamport, - ) -> Vec<(MoveLamportAndID, Option)> { + fn retreat_for_diff(&mut self, vv: &VersionVector, min_lamport: Lamport) -> Vec { + let mut diffs = vec![]; // remove ops from cache, and then insert to pending let mut retreat_ops = Vec::new(); for (_, ops) in self.cache.iter() { @@ -232,11 +268,11 @@ impl TreeDiffCache { break; } if !vv.includes_id(op.id) { - retreat_ops.push((*op, None)) + retreat_ops.push(*op) } } } - for (op, old_parent) in retreat_ops.iter_mut() { + for op in retreat_ops.iter_mut().sorted().rev() { self.cache.get_mut(&op.target).unwrap().remove(op); self.pending.insert(*op); self.current_version.shrink_to_exclude(IdSpan { @@ -244,9 +280,42 @@ impl TreeDiffCache { counter: CounterSpan::new(op.id.counter, op.id.counter + 1), }); // calc old parent - *old_parent = self.get_parent(op.target); + let old_parent = self.get_parent(op.target); + if op.effected { + // we need to know whether old_parent is deleted + let is_parent_deleted = + op.parent.is_some() && self.is_deleted(op.parent.as_ref().unwrap()); + let is_old_parent_deleted = + old_parent.is_some() && self.is_deleted(old_parent.as_ref().unwrap()); + let this_diff = TreeDeltaItem::new( + op.target, + old_parent, + op.parent, + is_old_parent_deleted, + is_parent_deleted, + ); + self.update_deleted_cache(op.target, old_parent, op.parent); + diffs.push(this_diff); + if matches!( + this_diff.action, + TreeInternalDiff::Restore | TreeInternalDiff::RestoreMove(_) + ) { + let mut s = vec![op.target]; + while let Some(t) = s.pop() { + let children = self.get_children(t); + children.iter().for_each(|c| { + diffs.push(TreeDeltaItem { + target: *c, + action: TreeInternalDiff::CreateMove(t), + }) + }); + s.extend(children); + } + } + } } - retreat_ops + + diffs } /// get the parent of the first effected op @@ -255,12 +324,15 @@ impl TreeDiffCache { return None; } let mut ans = TreeID::unexist_root(); - for op in self.cache.get(&tree_id).unwrap().iter().rev() { - if op.effected { - ans = op.parent; - break; + if let Some(cache) = self.cache.get(&tree_id) { + for op in cache.iter().rev() { + if op.effected { + ans = op.parent; + break; + } } } + ans } @@ -282,3 +354,79 @@ impl TreeDiffCache { } } } + +pub(crate) trait TreeDeletedSetTrait { + fn deleted(&self) -> &FxHashSet; + fn deleted_mut(&mut self) -> &mut FxHashSet; + fn get_children(&self, target: TreeID) -> Vec; + fn get_children_recursively(&self, target: TreeID) -> Vec { + let mut ans = vec![]; + let mut s = vec![target]; + while let Some(t) = s.pop() { + let children = self.get_children(t); + ans.extend(children.clone()); + s.extend(children); + } + ans + } + fn is_deleted(&self, target: &TreeID) -> bool { + self.deleted().contains(target) || TreeID::is_deleted_root(Some(*target)) + } + fn update_deleted_cache( + &mut self, + target: TreeID, + parent: Option, + old_parent: Option, + ) { + if parent.is_some() && self.is_deleted(&parent.unwrap()) { + self.update_deleted_cache_inner(target, true); + } else if let Some(old_parent) = old_parent { + if self.is_deleted(&old_parent) { + self.update_deleted_cache_inner(target, false); + } + } + } + fn update_deleted_cache_inner(&mut self, target: TreeID, set_children_deleted: bool) { + if set_children_deleted { + self.deleted_mut().insert(target); + } else { + self.deleted_mut().remove(&target); + } + let mut s = self.get_children(target); + while let Some(child) = s.pop() { + if child == target { + continue; + } + if set_children_deleted { + self.deleted_mut().insert(child); + } else { + self.deleted_mut().remove(&child); + } + s.extend(self.get_children(child)) + } + } +} + +impl TreeDeletedSetTrait for TreeDiffCache { + fn deleted(&self) -> &FxHashSet { + &self.deleted + } + + fn deleted_mut(&mut self) -> &mut FxHashSet { + &mut self.deleted + } + + fn get_children(&self, target: TreeID) -> Vec { + let mut ans = vec![]; + for (tree_id, _) in self.cache.iter() { + if tree_id == &target { + continue; + } + let parent = self.get_parent(*tree_id); + if parent == Some(target) { + ans.push(*tree_id) + } + } + ans + } +} diff --git a/crates/loro-internal/src/encoding/encode_snapshot.rs b/crates/loro-internal/src/encoding/encode_snapshot.rs index 1fdb9ee19..a8cf54be9 100644 --- a/crates/loro-internal/src/encoding/encode_snapshot.rs +++ b/crates/loro-internal/src/encoding/encode_snapshot.rs @@ -244,7 +244,7 @@ pub fn decode_oplog( for mut change in changes { let lamport = oplog.dag.frontiers_to_next_lamport(&change.deps); change.lamport = lamport; - oplog.import_local_change(change)?; + oplog.import_local_change(change, false)?; } Ok(()) } @@ -312,7 +312,7 @@ pub fn decode_state<'b>( richtext.decode_snapshot(*richtext_data, &state_arena, &common, &arena); container_states.insert(idx, State::RichtextState(richtext)); } - loro_preload::EncodedContainerState::Tree(tree_data) => { + loro_preload::EncodedContainerState::Tree((tree_data, deleted)) => { let mut tree = TreeState::new(); for (target, parent) in tree_data { let (peer, counter) = state_arena.tree_ids[target - 1]; @@ -333,6 +333,17 @@ pub fn decode_state<'b>( }; tree.trees.insert(target, parent); } + + for target in deleted { + let (peer, counter) = state_arena.tree_ids[target - 1]; + let target_peer = common.peer_ids[peer as usize]; + let target = TreeID { + peer: target_peer, + counter, + }; + tree.deleted.insert(target); + } + container_states.insert(idx, State::TreeState(tree)); } } @@ -688,9 +699,9 @@ fn encode_app_state(app_state: &DocState) -> PreEncodedState { loro_common::ContainerType::Map => { encoded.states.push(EncodedContainerState::Map(Vec::new())) } - loro_common::ContainerType::Tree => { - encoded.states.push(EncodedContainerState::Tree(Vec::new())) - } + loro_common::ContainerType::Tree => encoded + .states + .push(EncodedContainerState::Tree((Vec::new(), Vec::new()))), loro_common::ContainerType::Text => encoded .states .push(EncodedContainerState::Richtext(Default::default())), @@ -716,7 +727,15 @@ fn encode_app_state(app_state: &DocState) -> PreEncodedState { (t, p) }) .collect::>(); - encoded.states.push(EncodedContainerState::Tree(v)) + let d = tree + .deleted + .iter() + .map(|target| { + let peer_idx = record_peer(target.peer); + record_tree_id(*target, peer_idx) + }) + .collect::>(); + encoded.states.push(EncodedContainerState::Tree((v, d))) } State::ListState(list) => { let v = list.iter().map(&mut record_value).collect(); diff --git a/crates/loro-internal/src/event.rs b/crates/loro-internal/src/event.rs index 408a3c148..3482a0ce6 100644 --- a/crates/loro-internal/src/event.rs +++ b/crates/loro-internal/src/event.rs @@ -4,7 +4,7 @@ use smallvec::SmallVec; use crate::{ container::richtext::richtext_state::RichtextStateChunk, - delta::{Delta, MapDelta, StyleMeta, TreeDelta}, + delta::{Delta, MapDelta, StyleMeta, TreeDelta, TreeDiff}, op::SliceRanges, utils::string_slice::StringSlice, InternalString, LoroValue, @@ -49,9 +49,10 @@ pub struct DocDiff { #[derive(Debug, Clone)] pub(crate) struct InternalContainerDiff { pub(crate) idx: ContainerIdx, - pub(crate) reset: bool, + // If true, this event is created by the container which was resurrected by another container + pub(crate) bring_back: bool, pub(crate) is_container_deleted: bool, - pub(crate) diff: DiffVariant, + pub(crate) diff: Option, } #[derive(Debug, Clone, EnumAsInner)] @@ -175,7 +176,7 @@ pub enum Diff { /// - When feature `wasm` is disabled, it should use unicode indexes. Text(Delta), NewMap(MapDelta), - Tree(TreeDelta), + Tree(TreeDiff), } impl InternalDiff { @@ -184,7 +185,7 @@ impl InternalDiff { InternalDiff::SeqRaw(s) => s.is_empty(), InternalDiff::RichtextRaw(t) => t.is_empty(), InternalDiff::Map(m) => m.updated.is_empty(), - InternalDiff::Tree(t) => t.diff.is_empty(), + InternalDiff::Tree(t) => t.is_empty(), } } @@ -216,4 +217,30 @@ impl Diff { (a, _) => Err(a), } } + + pub(crate) fn is_empty(&self) -> bool { + match self { + Diff::List(s) => s.is_empty(), + Diff::Text(t) => t.is_empty(), + Diff::NewMap(m) => m.updated.is_empty(), + Diff::Tree(t) => t.diff.is_empty(), + } + } + + pub(crate) fn concat(self, diff: Diff) -> Diff { + match (self, diff) { + (Diff::List(a), Diff::List(b)) => Diff::List(a.compose(b)), + (Diff::Text(a), Diff::Text(b)) => Diff::Text(a.compose(b)), + (Diff::NewMap(a), Diff::NewMap(b)) => { + let mut a = a; + for (k, v) in b.updated { + a = a.with_entry(k, v); + } + Diff::NewMap(a) + } + + (Diff::Tree(a), Diff::Tree(b)) => Diff::Tree(a.extend(b.diff)), + _ => unreachable!(), + } + } } diff --git a/crates/loro-internal/src/fuzz/recursive_refactored.rs b/crates/loro-internal/src/fuzz/recursive_refactored.rs index dfb14d909..8f45a1ea6 100644 --- a/crates/loro-internal/src/fuzz/recursive_refactored.rs +++ b/crates/loro-internal/src/fuzz/recursive_refactored.rs @@ -2757,8 +2757,298 @@ mod failed_tests { ) } + #[test] + fn fuzz_map_bring_back() { + test_multi_sites( + 5, + &mut [ + Map { + site: 6, + container_idx: 63, + key: 255, + value: Container(C::Tree), + }, + List { + site: 96, + container_idx: 96, + key: 96, + value: Container(C::Map), + }, + List { + site: 96, + container_idx: 96, + key: 96, + value: Null, + }, + SyncAll, + Map { + site: 223, + container_idx: 255, + key: 96, + value: I32(1616928864), + }, + ], + ) + } + + #[test] + fn fuzz_sub_sub_bring_back() { + test_multi_sites( + 5, + &mut [ + List { + site: 65, + container_idx: 65, + key: 65, + value: Container(C::Map), + }, + Map { + site: 50, + container_idx: 7, + key: 7, + value: Container(C::Text), + }, + Sync { from: 0, to: 112 }, + Text { + site: 0, + container_idx: 13, + pos: 0, + value: 47712, + is_del: false, + }, + Sync { from: 50, to: 50 }, + SyncAll, + Map { + site: 59, + container_idx: 0, + key: 0, + value: Container(C::Map), + }, + List { + site: 65, + container_idx: 65, + key: 112, + value: Null, + }, + List { + site: 67, + container_idx: 65, + key: 65, + value: Null, + }, + Map { + site: 7, + container_idx: 50, + key: 7, + value: Container(C::Text), + }, + ], + ) + } + + #[test] + fn fuzz_bring_back_sub_is_other_bring_back() { + test_multi_sites( + 5, + &mut [ + List { + site: 255, + container_idx: 255, + key: 90, + value: Container(C::List), + }, + List { + site: 255, + container_idx: 255, + key: 199, + value: Container(C::Text), + }, + Map { + site: 0, + container_idx: 0, + key: 0, + value: Null, + }, + SyncAll, + Text { + site: 147, + container_idx: 33, + pos: 251, + value: 37779, + is_del: false, + }, + Map { + site: 0, + container_idx: 68, + key: 68, + value: Null, + }, + List { + site: 75, + container_idx: 0, + key: 75, + value: Null, + }, + Text { + site: 0, + container_idx: 177, + pos: 0, + value: 53883, + is_del: true, + }, + ], + ) + } + + #[test] + fn fuzz_sub_sub_is_has_diff() { + test_multi_sites( + 5, + &mut [ + List { + site: 251, + container_idx: 251, + key: 123, + value: Container(C::List), + }, + List { + site: 255, + container_idx: 251, + key: 123, + value: Container(C::List), + }, + List { + site: 91, + container_idx: 33, + key: 126, + value: Container(C::List), + }, + List { + site: 255, + container_idx: 63, + key: 251, + value: Container(C::Tree), + }, + Map { + site: 255, + container_idx: 148, + key: 255, + value: Container(C::Text), + }, + SyncAll, + Sync { from: 14, to: 140 }, + Map { + site: 126, + container_idx: 0, + key: 58, + value: Null, + }, + List { + site: 251, + container_idx: 63, + key: 255, + value: Container(C::Tree), + }, + List { + site: 56, + container_idx: 40, + key: 255, + value: Null, + }, + ], + ) + } + #[test] fn fuzz_12() { + test_multi_sites( + 5, + &mut [ + List { + site: 69, + container_idx: 69, + key: 69, + value: Container(C::List), + }, + List { + site: 69, + container_idx: 69, + key: 69, + value: Container(C::List), + }, + List { + site: 69, + container_idx: 69, + key: 4, + value: Container(C::Text), + }, + List { + site: 69, + container_idx: 69, + key: 69, + value: Null, + }, + List { + site: 0, + container_idx: 0, + key: 0, + value: Null, + }, + List { + site: 4, + container_idx: 255, + key: 47, + value: Null, + }, + Map { + site: 0, + container_idx: 0, + key: 0, + value: Null, + }, + List { + site: 69, + container_idx: 69, + key: 69, + value: Null, + }, + Map { + site: 47, + container_idx: 38, + key: 250, + value: Null, + }, + List { + site: 69, + container_idx: 69, + key: 69, + value: Container(C::Tree), + }, + Text { + site: 0, + container_idx: 0, + pos: 255, + value: 17919, + is_del: true, + }, + List { + site: 69, + container_idx: 69, + key: 4, + value: I32(553672192), + }, + List { + site: 69, + container_idx: 14, + key: 255, + value: Container(C::Tree), + }, + ], + ) + } + + #[test] + fn fuzz_14() { test_multi_sites( 5, &mut [ @@ -3986,29 +4276,6 @@ mod failed_tests { ) } - #[test] - fn fuzz_14() { - test_multi_sites( - 5, - &mut [ - Text { - site: 111, - container_idx: 238, - pos: 110, - value: 28270, - is_del: false, - }, - Text { - site: 191, - container_idx: 42, - pos: 191, - value: 37186, - is_del: true, - }, - ], - ) - } - #[test] fn to_minify() { minify_error(5, vec![], test_multi_sites, normalize) diff --git a/crates/loro-internal/src/fuzz/tree.rs b/crates/loro-internal/src/fuzz/tree.rs index 02106b3ad..2133e96b4 100644 --- a/crates/loro-internal/src/fuzz/tree.rs +++ b/crates/loro-internal/src/fuzz/tree.rs @@ -17,8 +17,13 @@ use crate::{ ContainerType, LoroValue, }; use crate::{ - delta::TreeDiffItem, handler::TreeHandler, loro::LoroDoc, state::Forest, value::ToJson, - version::Frontiers, ApplyDiff, ListHandler, MapHandler, TextHandler, + delta::TreeValue, + event::Index, + handler::TreeHandler, + loro::LoroDoc, + value::{unresolved_to_collection, ToJson}, + version::Frontiers, + ApplyDiff, ListHandler, MapHandler, TextHandler, }; #[derive(Arbitrary, EnumAsInner, Clone, PartialEq, Eq, Debug)] @@ -60,10 +65,7 @@ struct Actor { peer: PeerID, loro: LoroDoc, value_tracker: Arc>, - map_tracker: Arc>>, - list_tracker: Arc>>, - text_tracker: Arc>, - tree_tracker: Arc>>>, + tree_tracker: Arc>>, map_containers: Vec, list_containers: Vec, text_containers: Vec, @@ -75,16 +77,11 @@ impl Actor { fn new(id: PeerID) -> Self { let app = LoroDoc::new(); app.set_peer_id(id).unwrap(); - let mut default_tree_tracker = FxHashMap::default(); - default_tree_tracker.insert(TreeID::delete_root().unwrap(), None); let mut actor = Actor { peer: id, loro: app, value_tracker: Arc::new(Mutex::new(LoroValue::Map(Default::default()))), - map_tracker: Default::default(), - list_tracker: Default::default(), - text_tracker: Default::default(), - tree_tracker: Arc::new(Mutex::new(default_tree_tracker)), + tree_tracker: Default::default(), map_containers: Default::default(), list_containers: Default::default(), text_containers: Default::default(), @@ -96,10 +93,17 @@ impl Actor { actor.loro.subscribe_root(Arc::new(move |event| { let mut root_value = root_value.lock().unwrap(); debug_dbg!(&event); + // if id == 0 { + // println!("\nbefore {:?}", root_value); + // println!("\ndiff {:?}", event); + // } root_value.apply( &event.container.path.iter().map(|x| x.1.clone()).collect(), &[event.container.diff.clone()], ); + // if id == 0 { + // println!("\nafter {:?}", root_value); + // } })); let tree = Arc::clone(&actor.tree_tracker); @@ -107,24 +111,40 @@ impl Actor { &ContainerID::new_root("tree", ContainerType::Tree), Arc::new(move |event| { if event.from_children { - return; - } - let mut tree = tree.lock().unwrap(); - if let Diff::Tree(tree_delta) = &event.container.diff { - for diff in tree_delta.diff.iter() { - let target = diff.target; - match diff.action { - TreeDiffItem::CreateOrRestore => { - tree.insert(target, None); - } - TreeDiffItem::Move(parent) => { - tree.insert(target, Some(parent)); - } - TreeDiffItem::Delete | TreeDiffItem::UnCreate => { - tree.insert(target, TreeID::delete_root()); + // meta + let Index::Node(target) = event.container.path.last().unwrap().1 else { + unreachable!() + }; + let mut tree = tree.lock().unwrap(); + let Some(map) = tree.iter_mut().find(|x| { + let id = x.as_map().unwrap().get("id").unwrap().as_string().unwrap(); + id.as_ref() == &target.to_string() + }) else { + // maybe delete tree node first + return; + }; + let map = Arc::make_mut(map.as_map_mut().unwrap()); + let meta = map.get_mut("meta").unwrap(); + let meta = Arc::make_mut(meta.as_map_mut().unwrap()); + if let Diff::NewMap(update) = &event.container.diff { + for (key, value) in update.updated.iter() { + match &value.value { + Some(value) => { + meta.insert(key.to_string(), unresolved_to_collection(value)); + } + None => { + meta.remove(&key.to_string()); + } } } } + + return; + } + let mut tree = tree.lock().unwrap(); + if let Diff::Tree(tree_diff) = &event.container.diff { + let mut v = TreeValue(&mut tree); + v.apply_diff(tree_diff); } else { debug_dbg!(&event.container); unreachable!() @@ -149,7 +169,18 @@ impl Actor { fn record_history(&mut self) { let f = self.loro.oplog_frontiers(); - let value = self.loro.get_deep_value(); + let mut value = self.loro.get_deep_value(); + Arc::make_mut( + Arc::make_mut(value.as_map_mut().unwrap()) + .get_mut("tree") + .unwrap() + .as_list_mut() + .unwrap(), + ) + .sort_by_key(|x| { + let id = x.as_map().unwrap().get("id").unwrap(); + id.clone().into_string().unwrap() + }); let mut ids: Vec = f.iter().cloned().collect(); ids.sort_by_key(|x| x.peer); self.history.insert(ids, value); @@ -557,14 +588,7 @@ fn assert_value_eq(a: &LoroValue, b: &LoroValue) { let is_empty = match v { LoroValue::String(s) => s.is_empty(), LoroValue::List(l) => l.is_empty(), - LoroValue::Map(m) => { - m.is_empty() || { - m.get("roots") - .is_some_and(|x| x.as_list().is_some_and(|l| l.is_empty())) - && m.get("deleted") - .is_some_and(|x| x.as_list().is_some_and(|l| l.is_empty())) - } - } + LoroValue::Map(m) => m.is_empty(), _ => false, }; if is_empty { @@ -577,20 +601,12 @@ fn assert_value_eq(a: &LoroValue, b: &LoroValue) { let is_empty = match v { LoroValue::String(s) => s.is_empty(), LoroValue::List(l) => l.is_empty(), - LoroValue::Map(m) => { - m.is_empty() || { - m.get("roots") - .is_some_and(|x| x.as_list().is_some_and(|l| l.is_empty())) - && m.get("deleted") - .is_some_and(|x| x.as_list().is_some_and(|l| l.is_empty())) - } - } + LoroValue::Map(m) => m.is_empty(), _ => false, }; if is_empty { continue; } - assert_value_eq(v, a.get(k).unwrap()); } } @@ -601,35 +617,46 @@ fn assert_value_eq(a: &LoroValue, b: &LoroValue) { fn check_eq(a_actor: &mut Actor, b_actor: &mut Actor) { let a_doc = &mut a_actor.loro; let b_doc = &mut b_actor.loro; - let a_result = a_doc.get_state_deep_value(); + let mut a_result = a_doc.get_state_deep_value(); + let mut b_result = b_doc.get_state_deep_value(); + let mut a_value = a_actor.value_tracker.lock().unwrap(); + + if let Some(tree) = Arc::make_mut(a_result.as_map_mut().unwrap()).get_mut("tree") { + Arc::make_mut(tree.as_list_mut().unwrap()).sort_by_key(|x| { + let id = x.as_map().unwrap().get("id").unwrap(); + id.clone().into_string().unwrap() + }); + } + if let Some(tree) = Arc::make_mut(b_result.as_map_mut().unwrap()).get_mut("tree") { + Arc::make_mut(tree.as_list_mut().unwrap()).sort_by_key(|x| { + let id = x.as_map().unwrap().get("id").unwrap(); + id.clone().into_string().unwrap() + }); + } + if let Some(tree) = Arc::make_mut(a_value.as_map_mut().unwrap()).get_mut("tree") { + Arc::make_mut(tree.as_list_mut().unwrap()).sort_by_key(|x| { + let id = x.as_map().unwrap().get("id").unwrap(); + id.clone().into_string().unwrap() + }); + } debug_log::debug_log!("{}", a_result.to_json_pretty()); - assert_eq!(&a_result, &b_doc.get_state_deep_value()); - assert_value_eq(&a_result, &a_actor.value_tracker.lock().unwrap()); + assert_eq!(&a_result, &b_result); + assert_value_eq(&a_result, &a_value); - let a = a_doc.get_text("text"); - let value_a = a.get_value(); - assert_eq!( - &**value_a.as_string().unwrap(), - &*a_actor.text_tracker.lock().unwrap(), - ); + let a = a_doc.get_tree("tree"); + let mut value_a = a.get_deep_value().into_list().unwrap(); + let mut tracker_a = a_actor.tree_tracker.lock().unwrap(); - let a = a_doc.get_map("map"); - let value_a = a.get_value(); - assert_eq!( - &**value_a.as_map().unwrap(), - &*a_actor.map_tracker.lock().unwrap() - ); + Arc::make_mut(&mut value_a).sort_by_key(|x| { + let id = x.as_map().unwrap().get("id").unwrap(); + id.clone().into_string().unwrap() + }); + tracker_a.sort_by_key(|x| { + let id = x.as_map().unwrap().get("id").unwrap(); + id.clone().into_string().unwrap() + }); - let a = a_doc.get_list("list"); - let value_a = a.get_value(); - assert_eq!( - &**value_a.as_list().unwrap(), - &*a_actor.list_tracker.lock().unwrap(), - ); - let a = a_doc.get_tree("tree"); - let value_a = a.get_value(); - let forest = Forest::from_tree_state(&a_actor.tree_tracker.lock().unwrap()); - assert_eq!(&value_a, &forest.to_value()); + assert_eq!(&*value_a, &*tracker_a); } fn check_synced(sites: &mut [Actor]) { @@ -654,7 +681,6 @@ fn check_synced(sites: &mut [Actor]) { b_doc.import(&a_doc.export_snapshot()).unwrap(); debug_log::group_end!(); } - check_eq(a, b); debug_log::group_end!(); if i == 1 { @@ -674,23 +700,19 @@ fn check_history(actor: &mut Actor) { // println!("\nfrom {:?} checkout {:?}", actor.loro.oplog_vv(), f); // println!("before state {:?}", actor.loro.get_deep_value()); actor.loro.checkout(&f).unwrap(); - let actual = actor.loro.get_deep_value(); - for key in ["tree", "map", "list", "text"] { - let hv = v.as_map().unwrap().get(key).unwrap(); - let av = actual.as_map().unwrap().get(key).unwrap(); - if key == "tree" { - assert_eq!( - hv.as_map().unwrap().get("roots"), - av.as_map().unwrap().get("roots"), - "Version mismatched at {:?}, cnt={}", - f, - c - ) - } else { - assert_eq!(hv, av, "Version mismatched at {:?}, cnt={}", f, c); - } - } - // assert_eq!(v, &actual, "Version mismatched at {:?}, cnt={}", f, c); + let mut actual = actor.loro.get_deep_value(); + Arc::make_mut( + Arc::make_mut(actual.as_map_mut().unwrap()) + .get_mut("tree") + .unwrap() + .as_list_mut() + .unwrap(), + ) + .sort_by_key(|x| { + let id = x.as_map().unwrap().get("id").unwrap(); + id.clone().into_string().unwrap() + }); + assert_eq!(v, &actual, "Version mismatched at {:?}, cnt={}", f, c); } } @@ -1425,6 +1447,438 @@ mod failed_tests { ) } + #[test] + fn tree_meta2() { + test_multi_sites( + 5, + &mut [ + Tree { + site: 68, + container_idx: 68, + action: TreeAction::Move, + target: (4971973958552256511, 1157579844), + parent: (1663823979171038354, 387389207), + }, + Tree { + site: 23, + container_idx: 23, + action: TreeAction::Create, + target: (1663823975275763479, 1513239), + parent: (18446744069802491904, -1157625864), + }, + Tree { + site: 68, + container_idx: 255, + action: TreeAction::Meta, + target: (17457358724263116799, -12257212), + parent: (4941210755937475839, -458940), + }, + ], + ) + } + + #[test] + fn tree_meta3() { + test_multi_sites( + 5, + &mut [ + Tree { + site: 83, + container_idx: 68, + action: TreeAction::Delete, + target: (6144232899428974267, -12303292), + parent: (64457769666740223, 1136376803), + }, + Tree { + site: 83, + container_idx: 126, + action: TreeAction::Create, + target: (4485090715960753726, 1145328467), + parent: (144106391970530482, -134021120), + }, + SyncAll, + SyncAll, + Tree { + site: 83, + container_idx: 198, + action: TreeAction::Delete, + target: (1374463284756593595, 320017171), + parent: (1374463283923456787, 320017171), + }, + Tree { + site: 19, + container_idx: 19, + action: TreeAction::Create, + target: (1374463286960132883, 320017171), + parent: (1374463283923456787, 320017171), + }, + Tree { + site: 19, + container_idx: 19, + action: TreeAction::Create, + target: (1374463902398747411, 320017171), + parent: (1374463283923456787, 320017171), + }, + Tree { + site: 85, + container_idx: 68, + action: TreeAction::Meta, + target: (48946959133704191, 0), + parent: (4485090716314435584, 1044266558), + }, + Tree { + site: 255, + container_idx: 255, + action: TreeAction::Move, + target: (5999845544699159807, 1397969747), + parent: (18446743267408233540, 7602687), + }, + ], + ) + } + + #[test] + fn tree_meta4() { + test_multi_sites( + 5, + &mut [ + Tree { + site: 255, + container_idx: 255, + action: TreeAction::Meta, + target: (18446742974197989375, -1), + parent: (12826251736570199838, 520028164), + }, + Tree { + site: 1, + container_idx: 0, + action: TreeAction::Create, + target: (16625775453143040, 1761552105), + parent: (17654109439859425792, -553647873), + }, + Tree { + site: 128, + container_idx: 125, + action: TreeAction::Meta, + target: (18446744073692774400, 1140849151), + parent: (4846791580151137091, -2147418307), + }, + SyncAll, + Tree { + site: 67, + container_idx: 67, + action: TreeAction::Meta, + target: (18446742974204248064, 150996991), + parent: (18446505380905958145, 1330592767), + }, + Tree { + site: 17, + container_idx: 59, + action: TreeAction::Meta, + target: (1224980236811632639, 255), + parent: (18446743008557662719, 16777460), + }, + SyncAll, + Tree { + site: 104, + container_idx: 104, + action: TreeAction::Move, + target: (65283536480360, -1545651360), + parent: (18446462600351842559, 524287), + }, + Tree { + site: 0, + container_idx: 233, + action: TreeAction::Meta, + target: (1229783205210443579, 652804155), + parent: (291370715578367, 65297), + }, + ], + ) + } + + #[test] + fn tree_meta_container() { + test_multi_sites( + 5, + &mut [ + Tree { + site: 146, + container_idx: 68, + action: TreeAction::Meta, + target: (10539624087947575836, -48060), + parent: (4919337068460520959, 1150436607), + }, + Tree { + site: 255, + container_idx: 255, + action: TreeAction::Create, + target: (4952757824032145407, 1150476543), + parent: (18446736377124224836, -12303292), + }, + SyncAll, + SyncAll, + Tree { + site: 68, + container_idx: 68, + action: TreeAction::Meta, + target: (4941087607480665156, -188), + parent: (4952757824032145407, 1150476543), + }, + Tree { + site: 0, + container_idx: 255, + action: TreeAction::Meta, + target: (2089670193885516356, 1145324546), + parent: (18446743267406136388, -513537), + }, + SyncAll, + Tree { + site: 255, + container_idx: 255, + action: TreeAction::Meta, + target: (10583739794993119239, 1040139332), + parent: (4919132559297282047, -1840971777), + }, + Tree { + site: 187, + container_idx: 187, + action: TreeAction::Delete, + target: (18444773745600971844, 1145324799), + parent: (4919217516204851199, 486539256), + }, + Tree { + site: 68, + container_idx: 41, + action: TreeAction::Move, + target: (18446537660035301188, -117440513), + parent: (10583739794993119239, -48060), + }, + SyncAll, + Tree { + site: 68, + container_idx: 248, + action: TreeAction::Move, + target: (15481123706782866, 1157562368), + parent: (18446537660035301188, -117440513), + }, + ], + ) + } + + #[test] + fn tree_0() { + test_multi_sites( + 5, + &mut [ + Tree { + site: 85, + container_idx: 85, + action: TreeAction::Move, + target: (6148914691236517205, -43691), + parent: (6156420687763341311, 1431655765), + }, + Tree { + site: 85, + container_idx: 85, + action: TreeAction::Move, + target: (6148914691085522261, 1431655765), + parent: (6148914691236517205, 1431655765), + }, + Tree { + site: 85, + container_idx: 85, + action: TreeAction::Move, + target: (6148914691236517205, 1431655765), + parent: (6148914691236517205, 1090475349), + }, + Tree { + site: 122, + container_idx: 0, + action: TreeAction::Create, + target: (4485090715960753726, 943208504), + parent: (4051049678932293688, 943208504), + }, + Tree { + site: 85, + container_idx: 85, + action: TreeAction::Move, + target: (6151166491050202453, 1431655765), + parent: (18295140478440789, 1195839745), + }, + Tree { + site: 71, + container_idx: 71, + action: TreeAction::Move, + target: (5136152271503443783, 122111815), + parent: (5136152271503443783, 1195853639), + }, + Tree { + site: 71, + container_idx: 71, + action: TreeAction::Move, + target: (5128677179139770183, 1195853639), + parent: (5136152271503443783, 1195853639), + }, + Tree { + site: 71, + container_idx: 71, + action: TreeAction::Move, + target: (5136152271503427399, 1195853639), + parent: (5136152271503443783, 1195853639), + }, + Tree { + site: 71, + container_idx: 71, + action: TreeAction::Move, + target: (5136152271503443783, 1195853639), + parent: (5136152271503443783, 1195853639), + }, + Tree { + site: 71, + container_idx: 71, + action: TreeAction::Move, + target: (5136152271503443783, 1195853639), + parent: (5136152271503443783, 1195853639), + }, + Tree { + site: 182, + container_idx: 184, + action: TreeAction::Create, + target: (5497853135693813784, 1195854924), + parent: (5136152271503443783, 1195853639), + }, + Tree { + site: 71, + container_idx: 71, + action: TreeAction::Move, + target: (668581441151911751, 0), + parent: (5136152271498772480, 1195853639), + }, + Tree { + site: 71, + container_idx: 71, + action: TreeAction::Move, + target: (5136152271503443783, 1195853639), + parent: (5136152271503443783, 1195853639), + }, + Tree { + site: 71, + container_idx: 71, + action: TreeAction::Move, + target: (13310589115948287815, 404232236), + parent: (5497853135693827096, 1280068684), + }, + Tree { + site: 24, + container_idx: 255, + action: TreeAction::Create, + target: (7016996347047838720, 895574369), + parent: (9223936088976472370, 65385), + }, + Tree { + site: 0, + container_idx: 1, + action: TreeAction::Create, + target: (3761688987579973632, 3421236), + parent: (72269528138039296, -16384477), + }, + Tree { + site: 76, + container_idx: 76, + action: TreeAction::Create, + target: (2748795787288, 4), + parent: (3617904946535555425, 10377529), + }, + Sync { from: 255, to: 0 }, + Tree { + site: 0, + container_idx: 0, + action: TreeAction::Move, + target: (4485090467895050240, 1044266558), + parent: (4051049678932293694, 943208504), + }, + Tree { + site: 56, + container_idx: 56, + action: TreeAction::Create, + target: (4051049678932293688, 943208504), + parent: (4051049679033350200, 943208504), + }, + Tree { + site: 56, + container_idx: 56, + action: TreeAction::Move, + target: (6148914691236517205, 1431655765), + parent: (6148914691236517205, 1431655765), + }, + Tree { + site: 1, + container_idx: 17, + action: TreeAction::Move, + target: (4485090715960738940, 943210046), + parent: (6124895493227558968, 1090475349), + }, + Tree { + site: 122, + container_idx: 0, + action: TreeAction::Create, + target: (4485090715960753726, 943208504), + parent: (4051049678932293688, 943208504), + }, + Tree { + site: 85, + container_idx: 85, + action: TreeAction::Move, + target: (6148914691236517205, 1431655765), + parent: (6148914324732641365, 9257215), + }, + Tree { + site: 0, + container_idx: 96, + action: TreeAction::Create, + target: (4051056301872922174, 943208504), + parent: (4051049678932293688, 1431655736), + }, + Tree { + site: 85, + container_idx: 85, + action: TreeAction::Move, + target: (6148914691236517205, 138477397), + parent: (323238826388099329, 1044266558), + }, + Tree { + site: 96, + container_idx: 31, + action: TreeAction::Create, + target: (18085043209519168007, -84215046), + parent: (18085043209503048698, 1090592251), + }, + Tree { + site: 1, + container_idx: 1, + action: TreeAction::Create, + target: (4051049678928675073, 943208504), + parent: (4050486728978872376, 1431647799), + }, + Tree { + site: 85, + container_idx: 85, + action: TreeAction::Move, + target: (4051049678932301141, 943208504), + parent: (4051049678932293688, 943208504), + }, + Tree { + site: 56, + container_idx: 56, + action: TreeAction::Create, + target: (4051049678932293688, 943208504), + parent: (87882006846257208, 16843009), + }, + ], + ) + } + #[test] fn to_minify() { minify_error(5, vec![], test_multi_sites, normalize) diff --git a/crates/loro-internal/src/handler.rs b/crates/loro-internal/src/handler.rs index ba0ff7acb..72ae824b3 100644 --- a/crates/loro-internal/src/handler.rs +++ b/crates/loro-internal/src/handler.rs @@ -6,7 +6,7 @@ use crate::{ richtext::TextStyleInfoFlag, tree::tree_op::TreeOp, }, - delta::MapValue, + delta::{MapValue, TreeDiffItem, TreeExternalDiff}, op::ListSlice, state::RichtextState, txn::EventHint, @@ -18,6 +18,7 @@ use loro_common::{ ContainerID, ContainerType, LoroError, LoroResult, LoroTreeError, LoroValue, TreeID, }; use serde::{Deserialize, Serialize}; +use smallvec::smallvec; use std::{ borrow::Cow, sync::{Mutex, Weak}, @@ -952,7 +953,7 @@ impl TreeHandler { pub fn create(&self, txn: &mut Transaction) -> LoroResult { let tree_id = TreeID::from_id(txn.next_id()); - let container_id = self.meta_container_id(tree_id); + let container_id = tree_id.associated_meta_container(); let child_idx = txn.arena.register_container(&container_id); txn.arena.set_parent(child_idx, Some(self.container_idx)); txn.apply_local_op( @@ -961,7 +962,10 @@ impl TreeHandler { target: tree_id, parent: None, }), - EventHint::Tree((tree_id, None).into()), + EventHint::Tree(smallvec![TreeDiffItem { + target: tree_id, + action: TreeExternalDiff::Create, + }]), &self.state, )?; Ok(tree_id) @@ -978,7 +982,10 @@ impl TreeHandler { target, parent: TreeID::delete_root(), }), - EventHint::Tree((target, TreeID::delete_root()).into()), + EventHint::Tree(smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Delete, + }]), &self.state, ) } @@ -989,7 +996,7 @@ impl TreeHandler { pub fn create_and_mov(&self, txn: &mut Transaction, parent: TreeID) -> LoroResult { let tree_id = TreeID::from_id(txn.next_id()); - let container_id = self.meta_container_id(tree_id); + let container_id = tree_id.associated_meta_container(); let child_idx = txn.arena.register_container(&container_id); txn.arena.set_parent(child_idx, Some(self.container_idx)); txn.apply_local_op( @@ -998,7 +1005,16 @@ impl TreeHandler { target: tree_id, parent: Some(parent), }), - EventHint::Tree((tree_id, Some(parent)).into()), + EventHint::Tree(smallvec![ + TreeDiffItem { + target: tree_id, + action: TreeExternalDiff::Create, + }, + TreeDiffItem { + target: tree_id, + action: TreeExternalDiff::Move(Some(parent)), + } + ]), &self.state, )?; Ok(tree_id) @@ -1015,7 +1031,10 @@ impl TreeHandler { target, parent: None, }), - EventHint::Tree((target, None).into()), + EventHint::Tree(smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Move(None), + }]), &self.state, ) } @@ -1031,7 +1050,10 @@ impl TreeHandler { target, parent: Some(parent), }), - EventHint::Tree((target, Some(parent)).into()), + EventHint::Tree(smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Move(Some(parent)), + }]), &self.state, ) } @@ -1040,7 +1062,7 @@ impl TreeHandler { if !self.contains(target) { return Err(LoroTreeError::TreeNodeNotExist(target).into()); } - let map_container_id = self.meta_container_id(target); + let map_container_id = target.associated_meta_container(); let idx = self .state .upgrade() @@ -1118,10 +1140,6 @@ impl TreeHandler { }) } - fn meta_container_id(&self, target: TreeID) -> ContainerID { - ContainerID::new_normal(target.id(), ContainerType::Map) - } - #[cfg(feature = "test_utils")] pub fn max_counter(&self) -> i32 { self.state @@ -1353,7 +1371,7 @@ mod test { .unwrap(); assert_eq!(meta, 123.into()); assert_eq!( - r#"{"roots":[{"parent":null,"meta":{"a":123},"id":"0@1","children":[]}],"deleted":[]}"#, + r#"[{"parent":null,"meta":{"a":123},"id":"0@1"}]"#, tree.get_deep_value().to_json() ); let bytes = loro.export_snapshot(); diff --git a/crates/loro-internal/src/loro.rs b/crates/loro-internal/src/loro.rs index b07cb67cd..8a7c61db7 100644 --- a/crates/loro-internal/src/loro.rs +++ b/crates/loro-internal/src/loro.rs @@ -418,7 +418,6 @@ impl LoroDoc { } EncodeMode::Auto => unreachable!(), }; - self.emit_events(); Ok(()) } diff --git a/crates/loro-internal/src/oplog.rs b/crates/loro-internal/src/oplog.rs index 3990f7915..8da45d1bd 100644 --- a/crates/loro-internal/src/oplog.rs +++ b/crates/loro-internal/src/oplog.rs @@ -228,7 +228,7 @@ impl OpLog { /// /// - Return Err(LoroError::UsedOpID) when the change's id is occupied /// - Return Err(LoroError::DecodeError) when the change's deps are missing - pub fn import_local_change(&mut self, change: Change) -> Result<(), LoroError> { + pub fn import_local_change(&mut self, change: Change, from_txn: bool) -> Result<(), LoroError> { let Some(change) = self.trim_the_known_part_of_change(change) else { return Ok(()); }; @@ -260,8 +260,9 @@ impl OpLog { let mut tree_cache = self.tree_parent_cache.lock().unwrap(); for op in change.ops().iter() { if let crate::op::InnerContent::Tree(tree) = op.content { + let diff = op.counter - change.id.counter; let node = MoveLamportAndID { - lamport: change.lamport, + lamport: change.lamport + diff as Lamport, id: ID { peer: change.id.peer, counter: op.counter, @@ -270,7 +271,11 @@ impl OpLog { parent: tree.parent, effected: true, }; - tree_cache.add_node_uncheck(node); + if from_txn { + tree_cache.add_node_uncheck(node); + } else { + tree_cache.add_node(node); + } } } diff --git a/crates/loro-internal/src/state.rs b/crates/loro-internal/src/state.rs index 9600be955..05a124465 100644 --- a/crates/loro-internal/src/state.rs +++ b/crates/loro-internal/src/state.rs @@ -8,8 +8,7 @@ use loro_common::{ContainerID, LoroResult}; use crate::{ configure::{DefaultRandom, SecureRandomGenerator}, container::{idx::ContainerIdx, ContainerIdRaw}, - delta::Delta, - event::{Diff, DiffVariant, Index}, + event::{Diff, Index}, event::{InternalContainerDiff, InternalDiff}, fx_map, id::PeerID, @@ -26,7 +25,7 @@ mod tree_state; pub(crate) use list_state::ListState; pub(crate) use map_state::MapState; pub(crate) use richtext_state::RichtextState; -pub(crate) use tree_state::{get_meta_value, Forest, TreeState}; +pub(crate) use tree_state::{get_meta_value, TreeState}; use super::{ arena::SharedArena, @@ -235,43 +234,101 @@ impl DocState { if self.in_txn { panic!("apply_diff should not be called in a transaction"); } - + let is_recording = self.is_recording(); self.pre_txn(diff.origin.clone(), diff.local); - let Cow::Owned(inner) = &mut diff.diff else { + let Cow::Owned(inner) = std::mem::take(&mut diff.diff) else { unreachable!() }; - for diff in inner.iter_mut() { - let is_recording = self.is_recording(); - let state = self - .states - .entry(diff.idx) - .or_insert_with(|| create_state(diff.idx)); - if self.in_txn { - state.start_txn(); - self.changed_idx_in_txn.insert(diff.idx); + let mut idx2state_diff = FxHashMap::default(); + let mut diffs = if is_recording { + // To handle the `bring_back`, we need cache the state diff of current version first, + // because the state that is applied diffs could be also set to `bring_back` later. + // We recursively determine one by one whether we need to bring back and push the diff to the queue. + let mut diff_queue = vec![]; + let mut need_bring_back = FxHashSet::default(); + let all_idx: FxHashSet = inner.iter().map(|d| d.idx).collect(); + for mut diff in inner { + let idx = diff.idx; + if need_bring_back.contains(&idx) { + diff.bring_back = true; + } + if diff.bring_back { + let state = self + .states + .entry(diff.idx) + .or_insert_with(|| create_state(idx)); + let state_diff = state.to_diff(); + if diff.diff.is_none() && state_diff.is_empty() { + // empty diff, skip it + continue; + } + diff_queue.push(diff); + if !state_diff.is_empty() { + bring_back_sub_container( + &state_diff, + &mut diff_queue, + &mut need_bring_back, + &all_idx, + &mut self.states, + &mut idx2state_diff, + &self.arena, + ); + idx2state_diff.insert(idx, state_diff); + } + } else { + diff_queue.push(diff); + } } + diff_queue + } else { + inner + }; - let internal_diff = std::mem::replace( - &mut diff.diff, - DiffVariant::External(Diff::List(Delta::default())), - ); + // apply diff + for diff in &mut diffs { + let Some(internal_diff) = std::mem::take(&mut diff.diff) else { + // only bring_back + if is_recording { + if let Some(state_diff) = idx2state_diff.remove(&diff.idx) { + diff.diff = Some(state_diff.into()); + }; + } + continue; + }; + let idx = diff.idx; + let state = self.states.entry(idx).or_insert_with(|| create_state(idx)); - if diff.reset { - *state = create_state(diff.idx); + if self.in_txn { + state.start_txn(); + self.changed_idx_in_txn.insert(idx); } - if is_recording { - let external_diff = state - .apply_diff_and_convert(internal_diff.into_internal().unwrap(), &self.arena); - diff.diff = external_diff.into(); + // process bring_back before apply + let external_diff = if diff.bring_back { + let external_diff = state.apply_diff_and_convert( + internal_diff.into_internal().unwrap(), + &self.arena, + ); + if let Some(state_diff) = idx2state_diff.remove(&idx) { + // use `concat`(hierarchical and relative order) rather than `compose` + state_diff.concat(external_diff) + } else { + // empty state + external_diff + } + } else { + state + .apply_diff_and_convert(internal_diff.into_internal().unwrap(), &self.arena) + }; + diff.diff = Some(external_diff.into()); } else { state.apply_diff(internal_diff.into_internal().unwrap(), &self.arena); } } + diff.diff = diffs.into(); self.frontiers = (*diff.new_version).to_owned(); - if self.is_recording() { self.record_diff(diff) } @@ -288,7 +345,6 @@ impl DocState { self.changed_idx_in_txn.insert(op.container); } - // TODO: make apply_op return a result state.apply_op(raw_op, op, &self.arena)?; Ok(()) } @@ -365,9 +421,9 @@ impl DocState { .iter_mut() .map(|(&idx, state)| InternalContainerDiff { idx, - reset: true, + bring_back: false, is_container_deleted: false, - diff: state.to_diff().into(), + diff: Some(state.to_diff().into()), }) .collect(); self.record_diff(InternalDocDiff { @@ -554,45 +610,44 @@ impl DocState { match value { LoroValue::Container(_) => unreachable!(), LoroValue::List(mut list) => { - if list.iter().all(|x| !x.is_container()) { - return LoroValue::List(list); - } + if container.get_type() == ContainerType::Tree { + // Each tree node has an associated map container to represent + // the metadata of this node. When the user get the deep value, + // we need to add a field named `meta` to the tree node, + // whose value is deep value of map container. + get_meta_value(Arc::make_mut(&mut list), self); + } else { + if list.iter().all(|x| !x.is_container()) { + return LoroValue::List(list); + } - let list_mut = Arc::make_mut(&mut list); - for item in list_mut.iter_mut() { - if item.is_container() { - let container = item.as_container().unwrap(); - let container_idx = self.arena.register_container(container); - let value = self.get_container_deep_value(container_idx); - *item = value; + let list_mut = Arc::make_mut(&mut list); + for item in list_mut.iter_mut() { + if item.is_container() { + let container = item.as_container().unwrap(); + let container_idx = self.arena.register_container(container); + let value = self.get_container_deep_value(container_idx); + *item = value; + } } } - LoroValue::List(list) } LoroValue::Map(mut map) => { - if container.get_type() == ContainerType::Tree { - // get tree's meta - for nodes in Arc::make_mut(&mut map).values_mut() { - get_meta_value(nodes, self); - } - LoroValue::Map(map) - } else { - if map.iter().all(|x| !x.1.is_container()) { - return LoroValue::Map(map); - } + if map.iter().all(|x| !x.1.is_container()) { + return LoroValue::Map(map); + } - let map_mut = Arc::make_mut(&mut map); - for (_key, value) in map_mut.iter_mut() { - if value.is_container() { - let container = value.as_container().unwrap(); - let container_idx = self.arena.register_container(container); - let new_value = self.get_container_deep_value(container_idx); - *value = new_value; - } + let map_mut = Arc::make_mut(&mut map); + for (_key, value) in map_mut.iter_mut() { + if value.is_container() { + let container = value.as_container().unwrap(); + let container_idx = self.arena.register_container(container); + let new_value = self.get_container_deep_value(container_idx); + *value = new_value; } - LoroValue::Map(map) } + LoroValue::Map(map) } _ => value, } @@ -616,10 +671,9 @@ impl DocState { // omit event form deleted container continue; } - let Some((last_container_diff, _)) = containers.get_mut(&container_diff.idx) else { if let Some(path) = self.get_path(container_diff.idx) { - containers.insert(container_diff.idx, (container_diff.diff, path)); + containers.insert(container_diff.idx, (container_diff.diff.unwrap(), path)); } else { // if we cannot find the path to the container, the container must be overwritten afterwards. // So we can ignore the diff from it. @@ -635,7 +689,7 @@ impl DocState { // TODO: PERF avoid this clone *last_container_diff = last_container_diff .clone() - .compose(container_diff.diff) + .compose(container_diff.diff.unwrap()) .unwrap(); } } @@ -695,6 +749,89 @@ impl DocState { } } +fn bring_back_sub_container( + state_diff: &Diff, + queue: &mut Vec, + mark_bring_back: &mut FxHashSet, + all_idx: &FxHashSet, + states: &mut FxHashMap, + idx2state: &mut FxHashMap, + arena: &SharedArena, +) { + match state_diff { + Diff::List(list) => { + for delta in list.iter() { + if delta.is_insert() { + for v in delta.as_insert().unwrap().0.iter() { + if v.is_container() { + let idx = arena.id_to_idx(v.as_container().unwrap()).unwrap(); + if all_idx.contains(&idx) { + // There is one in subsequent elements that require applying the diff + mark_bring_back.insert(idx); + } else if let Some(state) = states.get_mut(&idx) { + // only bring back + // If the state is not empty, add this to queue and check + // whether there are sub-containers created by it recursively + // and finally cache the state + let diff = state.to_diff(); + if !diff.is_empty() { + queue.push(InternalContainerDiff { + idx, + bring_back: true, + is_container_deleted: false, + diff: None, + }); + bring_back_sub_container( + &diff, + queue, + mark_bring_back, + all_idx, + states, + idx2state, + arena, + ); + idx2state.insert(idx, diff); + } + } + } + } + } + } + } + Diff::NewMap(map) => { + for (_, v) in map.updated.iter() { + if let Some(LoroValue::Container(id)) = &v.value { + let idx = arena.id_to_idx(id).unwrap(); + if all_idx.contains(&idx) { + mark_bring_back.insert(idx); + } else if let Some(state) = states.get_mut(&idx) { + let diff = state.to_diff(); + if !diff.is_empty() { + queue.push(InternalContainerDiff { + idx, + bring_back: true, + is_container_deleted: false, + diff: None, + }); + bring_back_sub_container( + &diff, + queue, + mark_bring_back, + all_idx, + states, + idx2state, + arena, + ); + idx2state.insert(idx, diff); + } + } + } + } + } + _ => {} + }; +} + pub fn create_state(idx: ContainerIdx) -> State { match idx.get_type() { ContainerType::Map => State::MapState(MapState::new(idx)), diff --git a/crates/loro-internal/src/state/list_state.rs b/crates/loro-internal/src/state/list_state.rs index 1a6edbaf3..cc0fd399e 100644 --- a/crates/loro-internal/src/state/list_state.rs +++ b/crates/loro-internal/src/state/list_state.rs @@ -202,6 +202,7 @@ impl ListState { pub fn delete(&mut self, index: usize) { let leaf = self.list.query::(&index); + let elem = self.list.remove_leaf(leaf.unwrap().cursor).unwrap(); let value = elem.v; if self.in_txn { diff --git a/crates/loro-internal/src/state/tree_state.rs b/crates/loro-internal/src/state/tree_state.rs index 79520b113..bfd4fc026 100644 --- a/crates/loro-internal/src/state/tree_state.rs +++ b/crates/loro-internal/src/state/tree_state.rs @@ -1,19 +1,18 @@ use fxhash::{FxHashMap, FxHashSet}; use itertools::Itertools; -use loro_common::{ - ContainerID, ContainerType, LoroError, LoroResult, LoroTreeError, LoroValue, TreeID, ID, -}; +use loro_common::{ContainerID, LoroError, LoroResult, LoroTreeError, LoroValue, TreeID}; use serde::{Deserialize, Serialize}; use std::collections::{hash_map::Iter, VecDeque}; use std::sync::Arc; -use crate::delta::{TreeDelta, TreeDiff}; +use crate::delta::{TreeDiff, TreeDiffItem, TreeExternalDiff}; +use crate::diff_calc::TreeDeletedSetTrait; use crate::event::InternalDiff; use crate::DocState; use crate::{ arena::SharedArena, container::tree::tree_op::TreeOp, - delta::TreeDiffItem, + delta::TreeInternalDiff, event::{Diff, Index}, op::RawOp, }; @@ -53,9 +52,12 @@ impl TreeState { } pub fn mov(&mut self, target: TreeID, parent: Option) -> Result<(), LoroError> { - let Some(parent) = parent else{ + let Some(parent) = parent else { // new root node - let old_parent = self.trees.insert(target, None); + let old_parent = self + .trees + .insert(target, None) + .unwrap_or(TreeID::unexist_root()); self.update_deleted_cache(target, None, old_parent); if self.in_txn { self.undo_items.push(TreeUndoItem { @@ -71,18 +73,24 @@ impl TreeState { if self.is_ancestor_of(&target, &parent) { return Err(LoroTreeError::CyclicMoveError.into()); } - if self.trees.get(&target).copied().flatten() == Some(parent) { + if self + .trees + .get(&target) + .copied() + .unwrap_or(TreeID::unexist_root()) + == Some(parent) + { return Ok(()); } // move or delete or create children node - let old_parent = self.trees.insert(target, Some(parent)); + let old_parent = self + .trees + .insert(target, Some(parent)) + .unwrap_or(TreeID::unexist_root()); self.update_deleted_cache(target, Some(parent), old_parent); if self.in_txn { - self.undo_items.push(TreeUndoItem { - target, - old_parent: old_parent.unwrap_or(TreeID::unexist_root()), - }) + self.undo_items.push(TreeUndoItem { target, old_parent }) } Ok(()) @@ -130,50 +138,10 @@ impl TreeState { } } - /// Get the first-level children of the target node - pub fn children(&self, target: TreeID) -> Vec { - let mut ans = Vec::new(); - for (t, parent) in self.trees.iter() { - if let Some(p) = parent { - if p == &target { - ans.push(*t); - } - } - } - ans - } - fn is_deleted(&self, target: &TreeID) -> bool { self.deleted.contains(target) } - fn update_deleted_cache( - &mut self, - target: TreeID, - parent: Option, - old_parent: Option>, - ) { - if parent.is_some() && self.is_deleted(&parent.unwrap()) { - self.update_deleted_cache_inner(target, true); - } else if let Some(old_parent) = old_parent.flatten() { - if self.is_deleted(&old_parent) { - self.update_deleted_cache_inner(target, false); - } - } - } - - fn update_deleted_cache_inner(&mut self, target: TreeID, set_children_deleted: bool) { - let mut s = self.children(target); - while let Some(child) = s.pop() { - if set_children_deleted { - self.deleted.insert(child); - } else { - self.deleted.remove(&child); - } - s.extend(self.children(child)) - } - } - pub fn nodes(&self) -> Vec { self.trees .keys() @@ -204,22 +172,36 @@ impl ContainerState for TreeState { for diff in tree.diff.iter() { let target = diff.target; let parent = match diff.action { - TreeDiffItem::CreateOrRestore => None, - TreeDiffItem::Move(parent) => Some(parent), - TreeDiffItem::Delete => TreeID::delete_root(), - TreeDiffItem::UnCreate => { + TreeInternalDiff::Create + | TreeInternalDiff::Restore + | TreeInternalDiff::AsRoot => None, + TreeInternalDiff::Move(parent) + | TreeInternalDiff::CreateMove(parent) + | TreeInternalDiff::RestoreMove(parent) => Some(parent), + TreeInternalDiff::Delete => TreeID::delete_root(), + TreeInternalDiff::UnCreate => { // delete it from state self.trees.remove(&target); continue; } }; - let old_parent = self.trees.insert(target, parent); - if Some(parent) != old_parent { + let old_parent = self + .trees + .insert(target, parent) + .unwrap_or(TreeID::unexist_root()); + if parent != old_parent { self.update_deleted_cache(target, parent, old_parent); } } } - Diff::Tree(diff.into_tree().unwrap()) + let ans = diff + .into_tree() + .unwrap() + .diff + .into_iter() + .flat_map(TreeDiffItem::from_delta_item) + .collect_vec(); + Diff::Tree(TreeDiff { diff: ans }) } fn apply_op( @@ -244,11 +226,15 @@ impl ContainerState for TreeState { let mut q = VecDeque::from(forest.roots); while let Some(node) = q.pop_front() { let action = if let Some(parent) = node.parent { - TreeDiffItem::Move(parent) + diffs.push(TreeDiffItem { + target: node.id, + action: TreeExternalDiff::Create, + }); + TreeExternalDiff::Move(Some(parent)) } else { - TreeDiffItem::CreateOrRestore + TreeExternalDiff::Create }; - let diff = TreeDiff { + let diff = TreeDiffItem { target: node.id, action, }; @@ -256,22 +242,7 @@ impl ContainerState for TreeState { q.extend(node.children); } - let mut q = VecDeque::from(forest.deleted); - while let Some(node) = q.pop_front() { - let action = if let Some(parent) = node.parent { - TreeDiffItem::Move(parent) - } else { - unreachable!() - }; - let diff = TreeDiff { - target: node.id, - action, - }; - diffs.push(diff); - q.extend(node.children); - } - - Diff::Tree(TreeDelta { diff: diffs }) + Diff::Tree(TreeDiff { diff: diffs }) } fn start_txn(&mut self) { @@ -285,7 +256,10 @@ impl ContainerState for TreeState { if TreeID::is_unexist_root(old_parent) { self.trees.remove(&target); } else { - let parent = self.trees.insert(target, old_parent); + let parent = self + .trees + .insert(target, old_parent) + .unwrap_or(TreeID::unexist_root()); self.update_deleted_cache(target, old_parent, parent); } } @@ -296,11 +270,24 @@ impl ContainerState for TreeState { self.in_txn = false; } - // TODO: whether the node in deleted exists in the current version - // when checkout to a past version, deleted may have some nodes from the future. fn get_value(&mut self) -> LoroValue { - let forest = Forest::from_tree_state(&self.trees); - forest.to_value() + let mut ans = vec![]; + for (target, parent) in self.trees.iter() { + if !self.deleted.contains(target) && !TreeID::is_unexist_root(Some(*target)) { + let mut t = FxHashMap::default(); + t.insert("id".to_string(), target.id().to_string().into()); + let p = parent + .map(|p| p.to_string().into()) + .unwrap_or(LoroValue::Null); + t.insert("parent".to_string(), p); + t.insert( + "meta".to_string(), + target.associated_meta_container().into(), + ); + ans.push(t.into()); + } + } + ans.into() } /// Get the index of the child container @@ -315,11 +302,33 @@ impl ContainerState for TreeState { fn get_child_containers(&self) -> Vec { self.nodes() .into_iter() - .map(|n| ContainerID::new_normal(n.id(), ContainerType::Map)) + .map(|n| n.associated_meta_container()) .collect_vec() } } +impl TreeDeletedSetTrait for TreeState { + fn deleted(&self) -> &FxHashSet { + &self.deleted + } + + fn deleted_mut(&mut self) -> &mut FxHashSet { + &mut self.deleted + } + + fn get_children(&self, target: TreeID) -> Vec { + let mut ans = Vec::new(); + for (t, parent) in self.trees.iter() { + if let Some(p) = parent { + if p == &target { + ans.push(*t); + } + } + } + ans + } +} + /// Convert flatten tree structure to hierarchy for user interface. /// /// ```json @@ -344,79 +353,6 @@ pub struct TreeNode { } impl Forest { - pub(crate) fn from_tree_state_and_meta( - state: FxHashMap, LoroValue)>, - ) -> Self { - let mut forest = Self::default(); - let mut node_to_children = FxHashMap::default(); - let mut node_to_meta = FxHashMap::default(); - let mut node_to_parent = FxHashMap::default(); - - for (id, (parent, meta)) in state.into_iter().sorted_by_key(|(k, _)| *k) { - node_to_meta.insert(id, meta); - node_to_parent.insert(id, parent); - if let Some(parent) = parent { - node_to_children - .entry(parent) - .or_insert_with(Vec::new) - .push(id) - } - } - - for root in node_to_parent - .iter() - .filter(|(_, parent)| parent.is_none()) - .map(|(id, _)| *id) - .sorted() - { - let mut stack = vec![( - root, - TreeNode { - id: root, - parent: None, - meta: node_to_meta.remove(&root).unwrap().clone(), - children: vec![], - }, - )]; - let mut id_to_node = FxHashMap::default(); - while let Some((id, mut node)) = stack.pop() { - if let Some(children) = node_to_children.get(&id) { - let mut children_to_stack = Vec::new(); - for child in children { - if let Some(child_node) = id_to_node.remove(child) { - node.children.push(child_node); - } else { - children_to_stack.push(( - *child, - TreeNode { - id: *child, - parent: Some(id), - meta: node_to_meta.remove(child).unwrap().clone(), - children: vec![], - }, - )); - } - } - if !children_to_stack.is_empty() { - stack.push((id, node)); - stack.extend(children_to_stack); - } else { - id_to_node.insert(id, node); - } - } else { - id_to_node.insert(id, node); - } - } - let root_node = id_to_node.remove(&root).unwrap(); - if root_node.id == TreeID::delete_root().unwrap() { - forest.deleted = root_node.children; - } else { - forest.roots.push(root_node); - } - } - forest - } - pub(crate) fn from_tree_state(state: &FxHashMap>) -> Self { let mut forest = Self::default(); let mut node_to_children = FxHashMap::default(); @@ -444,10 +380,7 @@ impl Forest { TreeNode { id: root, parent: None, - meta: LoroValue::Container(ContainerID::new_normal( - root.id(), - ContainerType::Map, - )), + meta: LoroValue::Container(root.associated_meta_container()), children: vec![], }, )]; @@ -464,10 +397,7 @@ impl Forest { TreeNode { id: *child, parent: Some(id), - meta: LoroValue::Container(ContainerID::new_normal( - child.id(), - ContainerType::Map, - )), + meta: LoroValue::Container(child.associated_meta_container()), children: vec![], }, )); @@ -492,173 +422,16 @@ impl Forest { } forest } - - fn to_state(&self) -> FxHashMap, LoroValue)> { - let mut ans = FxHashMap::default(); - for root in self.roots.iter() { - let mut stack = vec![root]; - while let Some(node) = stack.pop() { - ans.insert(node.id, (node.parent, node.meta.clone())); - stack.extend(node.children.iter()) - } - } - ans.insert(TreeID::delete_root().unwrap(), (None, LoroValue::Null)); - for root in self.deleted.iter() { - let mut stack = vec![root]; - while let Some(node) = stack.pop() { - ans.insert(node.id, (node.parent, node.meta.clone())); - stack.extend(node.children.iter()) - } - } - ans - } - - // for test only - pub(crate) fn apply_diffs(&self, diff: &[Diff]) -> Self { - let mut state = self.to_state(); - for item in diff { - for diff in item.as_tree().unwrap().diff.iter() { - let target = diff.target; - let meta = if let Some((_, meta)) = state.remove(&target) { - meta - } else { - ContainerType::Map.default_value() - }; - match diff.action { - TreeDiffItem::CreateOrRestore => { - state.insert(target, (None, meta)); - } - TreeDiffItem::Move(parent) => { - state.insert(target, (Some(parent), meta)); - } - TreeDiffItem::Delete => { - state.insert(target, (TreeID::delete_root(), meta)); - } - TreeDiffItem::UnCreate => { - // If fuzz test, un exist node move to delete, - // Because it is necessary to record the meta created by these nodes before. - #[cfg(feature = "test_utils")] - state.insert(target, (TreeID::delete_root(), meta)); - #[cfg(not(feature = "test_utils"))] - state.remove(&target); - } - } - } - } - Self::from_tree_state_and_meta(state) - } - - pub(crate) fn to_value(&self) -> LoroValue { - let mut ans = FxHashMap::default(); - ans.insert( - "roots".to_string(), - self.roots.iter().map(|r| r.to_value()).collect_vec().into(), - ); - ans.insert( - "deleted".to_string(), - self.deleted - .iter() - .map(|r| r.to_value()) - .collect_vec() - .into(), - ); - ans.into() - } - - // for test only - pub(crate) fn from_value(value: LoroValue) -> LoroResult { - let mut map = Arc::try_unwrap(value.into_map().unwrap()).unwrap(); - // TODO: perf - let roots = map - .remove("roots") - .unwrap() - .into_list() - .unwrap() - .as_ref() - .iter() - .cloned() - .map(TreeNode::from_value) - .collect_vec(); - let deleted = if let Some(deleted) = map.remove("deleted") { - deleted - .into_list() - .unwrap() - .iter() - .cloned() - .map(TreeNode::from_value) - .collect_vec() - } else { - vec![] - }; - Ok(Self { roots, deleted }) - } -} - -impl TreeNode { - // for test only - fn from_value(value: LoroValue) -> Self { - let map = value.into_map().unwrap(); - let id = map.get("id").unwrap().clone().into_string().unwrap(); - let id = TreeID::from_id(ID::try_from(id.as_str()).unwrap()); - let parent = { - match map.get("parent").unwrap() { - LoroValue::Null => None, - LoroValue::String(str) => { - Some(TreeID::from_id(ID::try_from(str.as_str()).unwrap())) - } - _ => unreachable!(), - } - }; - let meta = map.get("meta").unwrap().clone(); - let children = map - .get("children") - .unwrap() - .clone() - .into_list() - .unwrap() - .iter() - .cloned() - .map(TreeNode::from_value) - .collect_vec(); - Self { - id, - meta, - parent, - children, - } - } - - fn to_value(&self) -> LoroValue { - let mut ans = FxHashMap::default(); - ans.insert("id".to_string(), self.id.id().to_string().into()); - if let Some(p) = &self.parent { - ans.insert("parent".to_string(), p.id().to_string().into()); - } else { - ans.insert("parent".to_string(), LoroValue::Null); - } - ans.insert("meta".to_string(), self.meta.clone()); - - ans.insert( - "children".to_string(), - self.children - .iter() - .map(|c| c.to_value()) - .collect_vec() - .into(), - ); - LoroValue::Map(Arc::new(ans)) - } } // convert map container to LoroValue -pub(crate) fn get_meta_value(nodes: &mut LoroValue, state: &mut DocState) { - for node in Arc::make_mut(nodes.as_list_mut().unwrap()).iter_mut() { +#[allow(clippy::ptr_arg)] +pub(crate) fn get_meta_value(nodes: &mut Vec, state: &mut DocState) { + for node in nodes.iter_mut() { let map = Arc::make_mut(node.as_map_mut().unwrap()); let meta = map.get_mut("meta").unwrap(); let id = meta.as_container().unwrap(); *meta = state.get_container_deep_value(state.arena.id_to_idx(id).unwrap()); - let children = map.get_mut("children").unwrap(); - get_meta_value(children, state) } } diff --git a/crates/loro-internal/src/txn.rs b/crates/loro-internal/src/txn.rs index 286408aed..e5a7bc6ba 100644 --- a/crates/loro-internal/src/txn.rs +++ b/crates/loro-internal/src/txn.rs @@ -8,6 +8,7 @@ use enum_as_inner::EnumAsInner; use generic_btree::rle::{HasLength as RleHasLength, Mergeable, Sliceable as GBSliceable}; use loro_common::{ContainerType, LoroResult}; use rle::{HasLength, Mergable, RleVec, Sliceable}; +use smallvec::SmallVec; use crate::{ change::{get_sys_timestamp, Change, Lamport, Timestamp}, @@ -17,7 +18,7 @@ use crate::{ richtext::{Style, StyleKey, TextStyleInfoFlag}, IntoContainerId, }, - delta::{Delta, MapValue, StyleMeta, StyleMetaItem, TreeDelta, TreeDiff}, + delta::{Delta, MapValue, StyleMeta, StyleMetaItem, TreeDiff, TreeDiffItem}, event::Diff, id::{Counter, PeerID, ID}, op::{Op, RawOp, RawOpContent}, @@ -79,7 +80,7 @@ pub(super) enum EventHint { key: InternalString, value: Option, }, - Tree(TreeDiff), + Tree(SmallVec<[TreeDiffItem; 2]>), MarkEnd, } @@ -279,7 +280,7 @@ impl Transaction { }; let last_id = change.id_last(); - if let Err(err) = oplog.import_local_change(change) { + if let Err(err) = oplog.import_local_change(change, true) { drop(state); drop(oplog); self._abort(); @@ -295,9 +296,9 @@ impl Transaction { arr.into_iter() .map(|x| InternalContainerDiff { idx: x.idx, - reset: false, + bring_back: false, is_container_deleted: false, - diff: x.diff.into(), + diff: Some(x.diff.into()), }) .collect(), ), @@ -537,7 +538,7 @@ fn change_to_diff( }, )) } - EventHint::Tree(tree_diff) => Diff::Tree(TreeDelta::default().push(tree_diff)), + EventHint::Tree(tree_diff) => Diff::Tree(TreeDiff::default().extend(tree_diff)), EventHint::MarkEnd => { // do nothing break 'outer; diff --git a/crates/loro-internal/src/value.rs b/crates/loro-internal/src/value.rs index 745ecfd12..9fef0bc37 100644 --- a/crates/loro-internal/src/value.rs +++ b/crates/loro-internal/src/value.rs @@ -1,15 +1,13 @@ use std::sync::Arc; use crate::{ - delta::{Delta, DeltaItem, Meta, StyleMeta}, + delta::{Delta, DeltaItem, Meta, StyleMeta, TreeValue}, event::{Diff, Index, Path}, - state::Forest, utils::string_slice::StringSlice, }; -use fxhash::FxHashMap; +use loro_common::ContainerType; pub use loro_common::LoroValue; -use loro_common::{ContainerType, TreeID}; // TODO: rename this trait pub trait ToJson { @@ -176,58 +174,64 @@ impl ApplyDiff for LoroValue { *value = Arc::new(s); } LoroValue::List(seq) => { - let seq = Arc::make_mut(seq); - for item in diff.iter() { - let delta = item.as_list().unwrap(); - let mut index = 0; - for delta_item in delta.iter() { - match delta_item { - DeltaItem::Retain { retain: len, .. } => { - index += len; - } - DeltaItem::Insert { insert: value, .. } => { - value.iter().for_each(|v| { - let value = unresolved_to_collection(v); - seq.insert(index, value); - index += 1; - }); + let is_tree = matches!(diff.first(), Some(Diff::Tree(_))); + if !is_tree { + let seq = Arc::make_mut(seq); + for item in diff.iter() { + let delta = item.as_list().unwrap(); + let mut index = 0; + for delta_item in delta.iter() { + match delta_item { + DeltaItem::Retain { retain: len, .. } => { + index += len; + } + DeltaItem::Insert { insert: value, .. } => { + value.iter().for_each(|v| { + let value = unresolved_to_collection(v); + seq.insert(index, value); + index += 1; + }); + } + DeltaItem::Delete { delete: len, .. } => { + seq.drain(index..index + len); + } } - DeltaItem::Delete { delete: len, .. } => { - seq.drain(index..index + len); + } + } + } else { + let seq = Arc::make_mut(seq); + for item in diff.iter() { + match item { + Diff::Tree(tree) => { + let mut v = TreeValue(seq); + v.apply_diff(tree); } + _ => unreachable!(), } } } } LoroValue::Map(map) => { - let is_tree = matches!(diff.first(), Some(Diff::Tree(_))); - if !is_tree { - for item in diff.iter() { - match item { - Diff::NewMap(diff) => { - let map = Arc::make_mut(map); - for (key, value) in diff.updated.iter() { - match &value.value { - Some(value) => { - map.insert( - key.to_string(), - unresolved_to_collection(value), - ); - } - None => { - map.remove(&key.to_string()); - } + for item in diff.iter() { + match item { + Diff::NewMap(diff) => { + let map = Arc::make_mut(map); + for (key, value) in diff.updated.iter() { + match &value.value { + Some(value) => { + map.insert( + key.to_string(), + unresolved_to_collection(value), + ); + } + None => { + map.remove(&key.to_string()); } } } - _ => unreachable!(), } + _ => unreachable!(), } - } else { - // TODO: perf - let forest = Forest::from_value(map.as_ref().clone().into()).unwrap(); - let diff_forest = forest.apply_diffs(diff); - *map = diff_forest.to_value().into_map().unwrap() } } _ => unreachable!(), @@ -264,14 +268,9 @@ impl ApplyDiff for LoroValue { let map = Arc::make_mut(m); value = map.entry(key.to_string()).or_insert_with(|| match hint { TypeHint::Map => LoroValue::Map(Default::default()), - TypeHint::Text => LoroValue::String(Arc::new(String::new())), + TypeHint::Text => LoroValue::String(Default::default()), TypeHint::List => LoroValue::List(Default::default()), - TypeHint::Tree => { - let mut map: FxHashMap = FxHashMap::default(); - map.insert("roots".to_string(), LoroValue::List(vec![].into())); - map.insert("deleted".to_string(), LoroValue::List(vec![].into())); - map.into() - } + TypeHint::Tree => LoroValue::List(Default::default()), }) } Index::Seq(index) => { @@ -279,7 +278,23 @@ impl ApplyDiff for LoroValue { let list = Arc::make_mut(l); value = list.get_mut(*index).unwrap(); } - Index::Node(tree_id) => value = get_meta_from_tree_value(value, *tree_id), + Index::Node(tree_id) => { + let l = value.as_list_mut().unwrap(); + let list = Arc::make_mut(l); + let Some(map) = list.iter_mut().find(|x| { + let id = x.as_map().unwrap().get("id").unwrap().as_string().unwrap(); + id.as_ref() == &tree_id.to_string() + }) else { + // delete node first + return; + }; + let map_mut = Arc::make_mut(map.as_map_mut().unwrap()); + let meta = map_mut.get_mut("meta").unwrap(); + if meta.is_container() { + *meta = ContainerType::Map.default_value(); + } + value = meta + } } } value @@ -288,36 +303,7 @@ impl ApplyDiff for LoroValue { } } -fn get_meta_from_tree_value(value: &mut LoroValue, target: TreeID) -> &mut LoroValue { - // find the meta of `tree_id` - let tree = Arc::make_mut(value.as_map_mut().unwrap()); - let mut map_value = None; - 'out: for (_, nodes) in tree.iter_mut() { - let mut s = vec![]; - let roots = nodes.as_list_mut().unwrap(); - let roots = Arc::make_mut(roots); - s.extend(roots); - while let Some(root) = s.pop() { - let root = Arc::make_mut(root.as_map_mut().unwrap()); - let this_node = - root.get("id").unwrap().as_string().unwrap().as_ref() == &target.to_string(); - if this_node { - let meta = root.get_mut("meta").unwrap(); - if meta.is_container() { - *meta = ContainerType::Map.default_value(); - } - map_value = Some(meta); - break 'out; - } else { - let children = root.get_mut("children").unwrap().as_list_mut().unwrap(); - s.extend(Arc::make_mut(children)); - } - } - } - map_value.unwrap() -} - -fn unresolved_to_collection(v: &LoroValue) -> LoroValue { +pub(crate) fn unresolved_to_collection(v: &LoroValue) -> LoroValue { if let Some(container) = v.as_container() { container.container_type().default_value() } else { @@ -333,7 +319,7 @@ pub mod wasm { use wasm_bindgen::{JsValue, __rt::IntoJsResult}; use crate::{ - delta::{Delta, DeltaItem, MapDelta, MapDiff, Meta, StyleMeta, TreeDelta, TreeDiffItem}, + delta::{Delta, DeltaItem, MapDelta, MapDiff, Meta, StyleMeta, TreeDiff, TreeExternalDiff}, event::{Diff, Index}, utils::string_slice::StringSlice, LoroValue, @@ -450,11 +436,11 @@ pub mod wasm { } } - impl From for JsValue { - fn from(value: TreeDiffItem) -> Self { + impl From for JsValue { + fn from(value: TreeExternalDiff) -> Self { let obj = Object::new(); match value { - TreeDiffItem::Delete | TreeDiffItem::UnCreate => { + TreeExternalDiff::Delete => { js_sys::Reflect::set( &obj, &JsValue::from_str("type"), @@ -462,7 +448,7 @@ pub mod wasm { ) .unwrap(); } - TreeDiffItem::Move(parent) => { + TreeExternalDiff::Move(parent) => { js_sys::Reflect::set( &obj, &JsValue::from_str("type"), @@ -473,7 +459,8 @@ pub mod wasm { js_sys::Reflect::set(&obj, &JsValue::from_str("parent"), &parent.into()) .unwrap(); } - TreeDiffItem::CreateOrRestore => { + + TreeExternalDiff::Create => { js_sys::Reflect::set( &obj, &JsValue::from_str("type"), @@ -486,8 +473,8 @@ pub mod wasm { } } - impl From for JsValue { - fn from(value: TreeDelta) -> Self { + impl From for JsValue { + fn from(value: TreeDiff) -> Self { let obj = Object::new(); for diff in value.diff.into_iter() { js_sys::Reflect::set(&obj, &"target".into(), &diff.target.into()).unwrap(); diff --git a/crates/loro-preload/src/encode.rs b/crates/loro-preload/src/encode.rs index 436b028b5..02db89bb7 100644 --- a/crates/loro-preload/src/encode.rs +++ b/crates/loro-preload/src/encode.rs @@ -135,7 +135,7 @@ pub enum EncodedContainerState<'a> { List(Vec), #[serde(borrow)] Richtext(Box>), - Tree(Vec<(usize, Option)>), + Tree((Vec<(usize, Option)>, Vec)), } #[derive(Debug, Default, Clone, Serialize, Deserialize)]