Merge branch 'main' into doc-wasm

Author: Leeeon233

crates/loro-internal/examples/encoding.rs

@@ -1,43 +1,49 @@
-use std::time::Instant;
-
 use bench_utils::TextAction;
 use loro_internal::LoroDoc;
 
 fn main() {
     let actions = bench_utils::get_automerge_actions();
 
+    // let loro = LoroDoc::default();
+    // let loro_b = LoroDoc::default();
+    // let text = loro.get_text("text");
+    // let mut i = 0;
+    // let start = Instant::now();
+    // for TextAction { pos, ins, del } in actions.iter() {
+    //     {
+    //         let mut txn = loro.txn().unwrap();
+    //         text.delete(&mut txn, *pos, *del).unwrap();
+    //         text.insert(&mut txn, *pos, ins).unwrap();
+    //     }
+
+    //     loro_b
+    //         .import(&loro.export_from(&loro_b.oplog_vv()))
+    //         .unwrap();
+    //     i += 1;
+    //     if i == 30000 {
+    //         break;
+    //     }
+    // }
+
+    // println!("{}ms", start.elapsed().as_millis());
+
     let loro = LoroDoc::default();
-    let loro_b = LoroDoc::default();
     let text = loro.get_text("text");
-    let mut i = 0;
-    let start = Instant::now();
+
     for TextAction { pos, ins, del } in actions.iter() {
-        {
-            let mut txn = loro.txn().unwrap();
-            text.delete(&mut txn, *pos, *del).unwrap();
-            text.insert(&mut txn, *pos, ins).unwrap();
-        }
-
-        loro_b
-            .import(&loro.export_from(&loro_b.oplog_vv()))
-            .unwrap();
-        i += 1;
-        if i == 30000 {
-            break;
-        }
+        let mut txn = loro.txn().unwrap();
+        text.delete(&mut txn, *pos, *del).unwrap();
+        text.insert(&mut txn, *pos, ins).unwrap();
+        txn.commit().unwrap();
     }
 
-    println!("{}ms", start.elapsed().as_millis());
-
-    // let loro = LoroDoc::default();
-    // let text = loro.get_text("text");
-
-    // for TextAction { pos, ins, del } in actions.iter() {
-    //     let mut txn = loro.txn().unwrap();
-    //     text.delete(&mut txn, *pos, *del).unwrap();
-    //     text.insert(&mut txn, *pos, ins).unwrap();
-    //     txn.commit().unwrap();
-    // }
+    let snapshot = loro.export_snapshot();
+    let output = miniz_oxide::deflate::compress_to_vec(&snapshot, 6);
+    println!(
+        "snapshot size {} after compression {}",
+        snapshot.len(),
+        output.len()
+    );
 
     // {
     //     // Delta encoding

crates/loro-internal/src/container/richtext/richtext_state.rs

@@ -1547,7 +1547,7 @@ impl RichtextState {
             "pos: {}, len: {}, self.len(): {}",
             pos,
             len,
-            &self.to_string()
+            &self.len_entity(),
         );
         // PERF: may use cache to speed up
         self.cursor_cache.invalidate();

crates/loro-internal/src/encoding/encode_enhanced.rs

@@ -5,7 +5,6 @@ use loro_common::{HasCounterSpan, HasIdSpan, HasLamportSpan, TreeID};
 use rle::{HasLength, RleVec, Sliceable};
 use serde_columnar::{columnar, iter_from_bytes, to_vec};
 use std::{borrow::Cow, ops::Deref, sync::Arc};
-use zerovec::{vecs::Index32, VarZeroVec};
 
 use crate::{
     change::{Change, Timestamp},
@@ -27,12 +26,9 @@ use crate::{
 
 type PeerIdx = u32;
 
-#[zerovec::make_varule(RootContainerULE)]
-#[zerovec::derive(Serialize, Deserialize)]
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)]
-struct RootContainer<'a> {
-    #[serde(borrow)]
-    name: Cow<'a, str>,
+struct RootContainer {
+    name: InternalString,
     type_: ContainerType,
 }
 
@@ -149,8 +145,7 @@ struct DocEncoding<'a> {
     style_info: Cow<'a, [u8]>,
     style_key: Vec<usize>,
     style_values: Vec<LoroValue>,
-    #[columnar(borrow)]
-    root_containers: VarZeroVec<'a, RootContainerULE, Index32>,
+    root_containers: Vec<RootContainer>,
     start_counter: Vec<Counter>,
     values: Vec<Option<LoroValue>>,
     clients: Vec<PeerID>,
@@ -379,7 +374,7 @@ pub fn encode_oplog_v2(oplog: &OpLog, vv: &VersionVector) -> Vec<u8> {
         clients: peers,
         keys,
         start_counter,
-        root_containers: VarZeroVec::from(&root_containers),
+        root_containers,
         normal_containers,
         values,
         style_key: style_key_idx,
@@ -401,7 +396,7 @@ fn extract_containers(
     peer_id_to_idx: &mut FxHashMap<PeerID, PeerIdx>,
     peers: &mut Vec<PeerID>,
 ) -> (
-    Vec<RootContainer<'static>>,
+    Vec<RootContainer>,
     FxHashMap<ContainerIdx, usize>,
     Vec<NormalContainer>,
 ) {
@@ -427,7 +422,7 @@ fn extract_containers(
                     } => {
                         container_idx2index.insert(container, root_containers.len());
                         root_containers.push(RootContainer {
-                            name: Cow::Owned(name.to_string()),
+                            name,
                             type_: container_type,
                         });
                     }
@@ -505,8 +500,8 @@ pub fn decode_oplog_v2(oplog: &mut OpLog, input: &[u8]) -> Result<(), LoroError>
                 return None;
             };
             Some(ContainerID::Root {
-                name: container.name.into(),
-                container_type: ContainerType::from_u8(container.type_),
+                name: container.name.clone(),
+                container_type: container.type_,
             })
         } else {
             let Some(container) = normal_containers.get(idx - root_containers.len()) else {

crates/loro-internal/src/event.rs

@@ -1,4 +1,5 @@
 use enum_as_inner::EnumAsInner;
+use fxhash::FxHasher64;
 use serde::{Deserialize, Serialize};
 use smallvec::SmallVec;
 
@@ -10,7 +11,10 @@ use crate::{
     InternalString, LoroValue,
 };
 
-use std::borrow::Cow;
+use std::{
+    borrow::Cow,
+    hash::{Hash, Hasher},
+};
 
 use loro_common::{ContainerID, TreeID};
 
@@ -43,9 +47,21 @@ pub struct DocDiff {
     pub to: Frontiers,
     pub origin: InternalString,
     pub local: bool,
+    /// Whether the diff is created from the checkout operation.
+    pub from_checkout: bool,
     pub diff: Vec<ContainerDiff>,
 }
 
+impl DocDiff {
+    /// Get the unique id of the diff.
+    pub fn id(&self) -> u64 {
+        let mut hasher = FxHasher64::default();
+        self.from.hash(&mut hasher);
+        self.to.hash(&mut hasher);
+        hasher.finish()
+    }
+}
+
 #[derive(Debug, Clone)]
 pub(crate) struct InternalContainerDiff {
     pub(crate) idx: ContainerIdx,
@@ -71,6 +87,7 @@ pub(crate) enum DiffVariant {
 pub(crate) struct InternalDocDiff<'a> {
     pub(crate) origin: InternalString,
     pub(crate) local: bool,
+    pub(crate) from_checkout: bool,
     pub(crate) diff: Cow<'a, [InternalContainerDiff]>,
     pub(crate) new_version: Cow<'a, Frontiers>,
 }
@@ -80,6 +97,7 @@ impl<'a> InternalDocDiff<'a> {
         InternalDocDiff {
             origin: self.origin,
             local: self.local,
+            from_checkout: self.from_checkout,
             diff: Cow::Owned((*self.diff).to_owned()),
             new_version: Cow::Owned((*self.new_version).to_owned()),
         }

crates/loro-internal/src/loro.rs

@@ -52,11 +52,18 @@ pub struct LoroDoc {
     arena: SharedArena,
     observer: Arc<Observer>,
     diff_calculator: Arc<Mutex<DiffCalculator>>,
+    // when dropping the doc, the txn will be commited
     txn: Arc<Mutex<Option<Transaction>>>,
     auto_commit: bool,
     detached: bool,
 }
 
+impl Default for LoroDoc {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl LoroDoc {
     pub fn new() -> Self {
         let oplog = OpLog::new();
@@ -397,6 +404,7 @@ impl LoroDoc {
                         origin,
                         local: false,
                         diff: (diff).into(),
+                        from_checkout: false,
                         new_version: Cow::Owned(oplog.frontiers().clone()),
                     });
                 }
@@ -620,6 +628,7 @@ impl LoroDoc {
             origin: "checkout".into(),
             local: true,
             diff: Cow::Owned(diff),
+            from_checkout: true,
             new_version: Cow::Owned(frontiers.clone()),
         });
         let events = state.take_events();
@@ -664,18 +673,6 @@ fn parse_encode_header(bytes: &[u8]) -> Result<(&[u8], EncodeMode), LoroError> {
     Ok((&input[1..], mode))
 }
 
-impl Default for LoroDoc {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl Drop for LoroDoc {
-    fn drop(&mut self) {
-        self.abort_txn();
-    }
-}
-
 #[cfg(test)]
 mod test {
     use loro_common::ID;

crates/loro-internal/src/state.rs

@@ -429,6 +429,7 @@ impl DocState {
             self.record_diff(InternalDocDiff {
                 origin: Default::default(),
                 local: false,
+                from_checkout: false,
                 diff,
                 new_version: Cow::Borrowed(&frontiers),
             });
@@ -664,6 +665,7 @@ impl DocState {
         let to = (*diffs.last().unwrap().new_version).to_owned();
         let origin = diffs[0].origin.clone();
         let local = diffs[0].local;
+        let from_checkout = diffs[0].from_checkout;
         for diff in diffs {
             #[allow(clippy::unnecessary_to_owned)]
             for container_diff in diff.diff.into_owned() {
@@ -714,6 +716,7 @@ impl DocState {
             from,
             to,
             origin,
+            from_checkout,
             local,
             diff,
         }

crates/loro-internal/src/txn.rs

@@ -321,6 +321,7 @@ impl Transaction {
                         })
                         .collect(),
                 ),
+                from_checkout: false,
                 new_version: Cow::Borrowed(oplog.frontiers()),
             }),
         );

crates/loro-internal/tests/test.rs

@@ -1,11 +1,33 @@
-use std::sync::{Arc, Mutex};
+use std::sync::{atomic::AtomicBool, Arc, Mutex};
 
 use loro_common::{ContainerID, ContainerType, LoroValue, ID};
 use loro_internal::{
     container::richtext::TextStyleInfoFlag, version::Frontiers, ApplyDiff, LoroDoc, ToJson,
 };
 use serde_json::json;
 
+#[test]
+fn event_from_checkout() {
+    let mut a = LoroDoc::new_auto_commit();
+    let sub_id = a.subscribe_root(Arc::new(|event| {
+        assert!(!event.doc.from_checkout);
+    }));
+    a.get_text("text").insert_(0, "hello").unwrap();
+    a.commit_then_renew();
+    let version = a.oplog_frontiers();
+    a.get_text("text").insert_(0, "hello").unwrap();
+    a.commit_then_renew();
+    a.unsubscribe(sub_id);
+    let ran = Arc::new(AtomicBool::new(false));
+    let ran_cloned = ran.clone();
+    a.subscribe_root(Arc::new(move |event| {
+        assert!(event.doc.from_checkout);
+        ran.store(true, std::sync::atomic::Ordering::Relaxed);
+    }));
+    a.checkout(&version).unwrap();
+    assert!(ran_cloned.load(std::sync::atomic::Ordering::Relaxed));
+}
+
 #[test]
 fn out_of_bound_test() {
     let a = LoroDoc::new_auto_commit();

crates/loro-wasm/src/lib.rs

@@ -873,6 +873,7 @@ fn call_subscriber(ob: observer::Observer, e: DiffEvent) {
         origin: e.doc.origin.to_string(),
         target: e.container.id.clone(),
         diff: e.container.diff.to_owned(),
+        from_checkout: e.doc.from_checkout,
     }
     // PERF: converting the events into js values may hurt performance
     .into_js();
@@ -890,6 +891,7 @@ fn call_after_micro_task(ob: observer::Observer, e: DiffEvent) {
     let copy = drop_handler.clone();
     let event = Event {
         from_children: e.from_children,
+        from_checkout: e.doc.from_checkout,
         local: e.doc.local,
         origin: e.doc.origin.to_string(),
         target: e.container.id.clone(),
@@ -924,6 +926,7 @@ pub struct Event {
     pub from_children: bool,
     origin: String,
     target: ContainerID,
+    from_checkout: bool,
     diff: Diff,
     path: JsValue,
 }
@@ -932,6 +935,7 @@ impl Event {
     fn into_js(self) -> JsValue {
         let obj = js_sys::Object::new();
         Reflect::set(&obj, &"local".into(), &self.local.into()).unwrap();
+        Reflect::set(&obj, &"fromCheckout".into(), &self.from_checkout.into()).unwrap();
         Reflect::set(&obj, &"fromChildren".into(), &self.from_children.into()).unwrap();
         Reflect::set(&obj, &"origin".into(), &self.origin.into()).unwrap();
         Reflect::set(&obj, &"target".into(), &self.target.to_string().into()).unwrap();

loro-js/src/index.ts

@@ -94,6 +94,13 @@ export type TreeDiff = {
 
 export type Diff = ListDiff | TextDiff | MapDiff | TreeDiff;
 
+export interface LoroEvent {
+  local: boolean;
+  origin?: string;
+  diff: Diff;
+  target: ContainerID;
+  path: Path;
+}
 
 interface Listener {
   (event: LoroEvent): void;