Perf: reduce mem usage when transaction is large (#128)

* perf: reduce mem usage when txn is large

* test: add diagnose info for id_to_cursor

* perf: make id span easier to merge

crates/loro-common/src/span.rs

@@ -67,6 +67,11 @@ impl CounterSpan {
         }
     }
 
+    #[inline(always)]
+    pub fn bidirectional(&self) -> bool {
+        (self.end - self.start).abs() == 1
+    }
+
     #[inline(always)]
     pub fn direction(&self) -> i32 {
         if self.start < self.end {
@@ -141,6 +146,20 @@ impl CounterSpan {
             self.end = new_end.min(self.start);
         }
     }
+
+    /// if we can merge element on the left, this method return the last atom of it
+    fn prev_pos(&self) -> i32 {
+        if self.start < self.end {
+            self.start - 1
+        } else {
+            self.start + 1
+        }
+    }
+
+    /// if we can merge element on the right, this method return the first atom of it
+    fn next_pos(&self) -> i32 {
+        self.end
+    }
 }
 
 impl HasLength for CounterSpan {
@@ -176,13 +195,32 @@ impl Sliceable for CounterSpan {
 impl Mergable for CounterSpan {
     #[inline]
     fn is_mergable(&self, other: &Self, _: &()) -> bool {
-        // TODO: can use the similar logic as [DeleteSpan] to merge
-        self.end == other.start && self.direction() == other.direction()
+        match (self.bidirectional(), other.bidirectional()) {
+            (true, true) => self.start + 1 == other.start || self.start == other.start + 1,
+            (true, false) => self.start == other.prev_pos(),
+            (false, true) => self.next_pos() == other.start,
+            (false, false) => {
+                self.next_pos() == other.start && self.direction() == other.direction()
+            }
+        }
     }
 
     #[inline]
     fn merge(&mut self, other: &Self, _: &()) {
-        self.end = other.end;
+        match (self.bidirectional(), other.bidirectional()) {
+            (true, true) => {
+                if self.start + 1 == other.start {
+                    self.end = self.start + 2;
+                } else if self.start - 1 == other.start {
+                    self.end = self.start - 2;
+                }
+            }
+            (true, false) => self.end = other.end,
+            (false, true) => self.end = self.end + self.direction(),
+            (false, false) => {
+                self.end = other.end;
+            }
+        }
     }
 }
 
@@ -444,4 +482,47 @@ mod test_id_span {
         let slice: Vec<IdSpan> = id_span_vec.slice_iter(5, 14).map(|x| x.into()).collect();
         assert_eq!(slice, id_spans!([0, 95, 90], [2, 2, 4], [2, 8, 6]).to_vec());
     }
+
+    #[test]
+    fn merge() {
+        let mut a = CounterSpan::new(0, 2);
+        let b = CounterSpan::new(2, 1);
+        assert!(a.is_mergable(&b, &()));
+        a.merge(&b, &());
+        assert_eq!(a, CounterSpan::new(0, 3));
+
+        let mut a = CounterSpan::new(3, 2);
+        let b = CounterSpan::new(2, 1);
+        assert!(a.is_mergable(&b, &()));
+        a.merge(&b, &());
+        assert_eq!(a, CounterSpan::new(3, 1));
+
+        let mut a = CounterSpan::new(4, 2);
+        let b = CounterSpan::new(2, 3);
+        assert!(a.is_mergable(&b, &()));
+        a.merge(&b, &());
+        assert_eq!(a, CounterSpan::new(4, 1));
+
+        let mut a = CounterSpan::new(8, 9);
+        let b = CounterSpan::new(9, 8);
+        assert!(a.is_mergable(&b, &()));
+        a.merge(&b, &());
+        assert_eq!(a, CounterSpan::new(8, 10));
+
+        let mut a = CounterSpan::new(8, 9);
+        let b = CounterSpan::new(10, 11);
+        assert!(!a.is_mergable(&b, &()));
+
+        let mut a = CounterSpan::new(0, 2);
+        let b = CounterSpan::new(2, 4);
+        assert!(a.is_mergable(&b, &()));
+        a.merge(&b, &());
+        assert_eq!(a, CounterSpan::new(0, 4));
+
+        let mut a = CounterSpan::new(4, 2);
+        let b = CounterSpan::new(2, 0);
+        assert!(a.is_mergable(&b, &()));
+        a.merge(&b, &());
+        assert_eq!(a, CounterSpan::new(4, 0));
+    }
 }

crates/loro-internal/examples/encoding_refactored.rs

@@ -3,8 +3,8 @@ use criterion::black_box;
 use loro_internal::loro::LoroDoc;
 
 fn main() {
-    log_size();
-    // bench_decode();
+    // log_size();
+    bench_decode();
     // bench_decode_updates();
 }
 
@@ -55,17 +55,18 @@ fn log_size() {
 
 #[allow(unused)]
 fn bench_decode() {
+    println!("Bench decode");
     let actions = bench_utils::get_automerge_actions();
     {
-        let loro = LoroDoc::default();
+        let mut loro = LoroDoc::default();
         let text = loro.get_text("text");
+        loro.start_auto_commit();
 
-        #[allow(warnings)]
-        for TextAction { pos, ins, del } in actions.iter() {
-            let mut txn = loro.txn().unwrap();
-            text.delete(&mut txn, *pos, *del);
-            text.insert(&mut txn, *pos, ins);
-            txn.commit().unwrap();
+        for _ in 0..10 {
+            for TextAction { pos, ins, del } in actions.iter() {
+                text.delete_(*pos, *del);
+                text.insert_(*pos, ins);
+            }
         }
         let snapshot = loro.export_snapshot();
         // for _ in 0..100 {

crates/loro-internal/src/container/list/list_op.rs

@@ -190,6 +190,8 @@ impl Mergable for DeleteSpan {
     where
         Self: Sized,
     {
+        // merge continuous deletions:
+        // note that the previous deletions will affect the position of the later deletions
         match (self.bidirectional(), other.bidirectional()) {
             (true, true) => self.pos == other.pos || self.pos == other.pos + 1,
             (true, false) => self.pos == other.prev_pos(),

crates/loro-internal/src/container/richtext/tracker.rs

@@ -228,6 +228,7 @@ impl Tracker {
         self._checkout(from, false);
         self._checkout(to, true);
         // debug_log::debug_dbg!(from, to, &self);
+        // self.id_to_cursor.diagnose();
         self.rope.get_diff()
     }
 }

crates/loro-internal/src/container/richtext/tracker/id_to_cursor.rs

@@ -162,6 +162,23 @@ impl IdToCursor {
             .cursor
             .get_insert((id.counter - list[index].counter) as usize)
     }
+
+    pub fn diagnose(&self) {
+        let fragment_num = self.map.iter().map(|x| x.1.len()).sum::<usize>();
+        let insert_pieces = self
+            .map
+            .iter()
+            .flat_map(|x| x.1.iter())
+            .map(|x| match &x.cursor {
+                Cursor::Insert { set, len } => set.len(),
+                Cursor::Delete(_) => 0,
+            })
+            .sum::<usize>();
+        eprintln!(
+            "fragments:{}, insert_pieces:{}",
+            fragment_num, insert_pieces
+        );
+    }
 }
 
 #[derive(Debug)]

crates/loro-internal/src/handler.rs

@@ -265,19 +265,21 @@ impl TextHandler {
                     .get_entity_index_for_text_insert_event_index(pos)
             });
 
+        let unicode_len = s.chars().count();
         txn.apply_local_op(
             self.container_idx,
             crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Insert {
                 slice: ListSlice::RawStr {
                     str: Cow::Borrowed(s),
-                    unicode_len: s.chars().count(),
+                    unicode_len,
                 },
                 pos: entity_index,
             }),
             EventHint::InsertText {
-                pos,
+                pos: pos as u32,
                 // FIXME: this is wrong
                 styles: vec![],
+                len: unicode_len as u32,
             },
             &self.state,
         )
@@ -333,7 +335,10 @@ impl TextHandler {
                     signed_len: (range.end - range.start) as isize,
                 })),
                 if is_first {
-                    EventHint::DeleteText { pos, len }
+                    EventHint::DeleteText(DeleteSpan {
+                        pos: pos as isize,
+                        signed_len: len as isize,
+                    })
                 } else {
                     EventHint::None
                 },
@@ -408,8 +413,8 @@ impl TextHandler {
                 info: flag,
             }),
             EventHint::Mark {
-                start,
-                end,
+                start: start as u32,
+                end: end as u32,
                 style: crate::container::richtext::Style {
                     key: key.into(),
                     // FIXME: style meta is incorrect
@@ -541,7 +546,7 @@ impl ListHandler {
                 pos: pos as isize,
                 signed_len: len as isize,
             })),
-            EventHint::DeleteList { pos, len },
+            EventHint::DeleteList(DeleteSpan::new(pos as isize, len as isize)),
             &self.state,
         )
     }