Speed/docs/reliability fixes for recent defrag changes

Author: gfwilliams

src/jsvar.c

@@ -4366,9 +4366,9 @@ int jsvGarbageCollect() {
 }
 
 #ifndef SAVE_ON_FLASH
-static void _jsvDefragment_moveReferences(JsVarRef defragFromRef, JsVarRef defragToRef) {
+static void _jsvDefragment_moveReferences(JsVarRef defragFromRef, JsVarRef defragToRef, unsigned int lastAllocated) {
   // find references!
-  for (JsVarRef vr=1;vr<=jsVarsSize;vr++) {
+  for (JsVarRef vr=1;vr<=lastAllocated;vr++) {
     JsVar *v = _jsvGetAddressOf(vr);
     if ((v->flags&JSV_VARTYPEMASK)!=JSV_UNUSED) {
       if (jsvIsFlatString(v)) {
@@ -4404,17 +4404,29 @@ void jsvDefragment() {
   // garbage collect - removes cruft, also puts free list in order
   jsvGarbageCollect();
   jshInterruptOff();
+  const unsigned int minMove = 20; // don't move vars back less than this or we're just wasting CPU time
+  // find last allocated block of memory - speeds up searches!
+  unsigned int lastAllocated = 0;
+  for (JsVarRef i=1;i<=jsVarsSize;i++) {
+    JsVar *v = _jsvGetAddressOf(i);
+    if ((v->flags&JSV_VARTYPEMASK)!=JSV_UNUSED) {
+      if (jsvIsFlatString(v)) {
+        i += 1+(unsigned int)jsvGetFlatStringBlocks(v); // skip forward
+      }
+      lastAllocated = i;
+    }
+  }
   // the var we're planning on writing to
   JsVarRef defragToRef = 1;
   // now for all blocks in memory...
-  for (JsVarRef defragFromRef=1;defragFromRef<=jsVarsSize;defragFromRef++) {
+  for (JsVarRef defragFromRef=1;defragFromRef<=lastAllocated;defragFromRef++) {
     // First move our destination block on until we find an UNUSED one...
     JsVar *defragTo = _jsvGetAddressOf(defragToRef);
     while ((defragTo->flags&JSV_VARTYPEMASK)!=JSV_UNUSED) {
       if (jsvIsFlatString(defragTo)) {
         defragToRef += 1+(unsigned int)jsvGetFlatStringBlocks(defragTo); // skip forward
       } else defragToRef++;
-      if (defragToRef > jsVarsSize) { // no more free blocks? quit
+      if (defragToRef > lastAllocated) { // no more free blocks? quit
         jsvCreateEmptyVarList();
         jshInterruptOn();
         return;
@@ -4432,11 +4444,11 @@ void jsvDefragment() {
           from defragToRef trying to see how many blocks we can move */
           JsVarRef fsToRef = defragToRef;
           bool isClear = false;
-          while (!isClear && fsToRef<defragFromRef) {
+          while (!isClear && (defragFromRef > fsToRef+minMove)) {
             isClear = true;
             // check area in fsToRef to see if it's clear
-            for (int i=0;i<blocksNeeded;i++) {
-              // TODO: what if we overlap with ourself?? would have to ensure we don't clear overlapping area
+            for (unsigned int i=0;i<blocksNeeded;i++) {
+              // TODO: what if we want to overlap with ourself?? would have to ensure we don't clear overlapping area
               if ((_jsvGetAddressOf(fsToRef+i)->flags&JSV_VARTYPEMASK)!=JSV_UNUSED) {
                 isClear = false; // it's not clear!
                 fsToRef += i; // jump to this used block
@@ -4450,33 +4462,32 @@ void jsvDefragment() {
                 if (jsvIsFlatString(v)) {
                   fsToRef += 1+(unsigned int)jsvGetFlatStringBlocks(v); // skip forward
                 } else fsToRef++;
-                if (fsToRef <= jsVarsSize)
-                  v = _jsvGetAddressOf(defragToRef);
+                if (fsToRef <= lastAllocated)
+                  v = _jsvGetAddressOf(fsToRef);
                 else
                   break; // end of memory
               }
             }
           }
-          if (isClear && (defragFromRef > fsToRef)) {
+          if (isClear && (defragFromRef > fsToRef+minMove)) { // can we move it earlier in memory?
             //jsiConsolePrintf("Move FlatString %d -> %d\n", defragFromRef, fsToRef);
             defragTo = _jsvGetAddressOf(fsToRef);
             // copy data and clear old var
             memmove(defragTo, defragFrom, sizeof(JsVar)*blocksNeeded);
             memset(defragFrom, 0, sizeof(JsVar)*blocksNeeded);
             // copy references
-            _jsvDefragment_moveReferences(defragFromRef, defragToRef);
+            _jsvDefragment_moveReferences(defragFromRef, fsToRef, lastAllocated);
           }
         }
         defragFromRef += blocksNeeded-1; // skip forward (for loop adds 1)
-        // skip flat strings for now
       } else if (canMove) { // moving a single var
-        if (defragFromRef > defragToRef) { // can we move back?
+        if (defragFromRef > defragToRef+minMove) { // can we move it earlier in memory?
           //jsiConsolePrintf("Move JsVar %d -> %d\n", defragFromRef, defragToRef);
           // copy data and clear old var
           *defragTo = *defragFrom;
           memset(defragFrom, 0, sizeof(JsVar)); // set flags to 0=unused
           // copy references
-          _jsvDefragment_moveReferences(defragFromRef, defragToRef);
+          _jsvDefragment_moveReferences(defragFromRef, defragToRef, lastAllocated);
         }
       }
     }

src/jswrap_espruino.c

@@ -1563,16 +1563,25 @@ void jswrap_espruino_dumpFreeList() {
   "ifndef" : "SAVE_ON_FLASH",
   "generate" : "jswrap_e_dumpFragmentation"
 }
-Show fragmentation.
+Show fragmentation. As of 2v27 this stops at the last allocated variable
+so as to avoid outputting blank lines if memory isn't full.
 
 * ` ` is free space
 * `#` is a normal variable
 * `L` is a locked variable (address used, cannot be moved)
 * `=` represents data in a Flat String (must be contiguous)
  */
 void jswrap_e_dumpFragmentation() {
-  int l = 0;
+  // find last allocated
+  unsigned int lastAllocated = 0;
   for (unsigned int i=0;i<jsvGetMemoryTotal();i++) {
+    JsVar *v = _jsvGetAddressOf(i+1);
+    if ((v->flags&JSV_VARTYPEMASK)!=JSV_UNUSED)
+      lastAllocated = i;
+  }
+  // output data as lines
+  int l = 0;
+  for (unsigned int i=0;i<lastAllocated;i++) {
     JsVar *v = _jsvGetAddressOf(i+1);
     if ((v->flags&JSV_VARTYPEMASK)==JSV_UNUSED) {
       jsiConsolePrint(" ");
@@ -1662,7 +1671,16 @@ void jswrap_e_dumpVariables() {
   "name" : "defrag",
   "generate" : "jsvDefragment"
 }
-BETA: defragment memory!
+This defragment's Espruino's memory. 
+
+While Espruino does a lot of work to avoid fragmentation (variables spread over memory)
+and can usually work around it (such as by allocating data in chunks) sometimes
+it is useful to be able to allocate a large contiguous chunk of memory, and
+if memory is low and has been fragmented it may need defragmenting in order to
+find that chunk.
+
+See `E.dumpFragmentation()` to show a map of the arrangement of variables
+within memory.
 */
 
 /*TYPESCRIPT

tests/test_defrag.js

@@ -0,0 +1,15 @@
+var a = [];
+for (var i=0;i<100;i++) {
+  a[i]=E.toFlatString(i+"oh hello world a big flat string -------------------------------------------------------------------------------".substr(0,25+Math.randInt(50)));
+}
+for (var i=0;i<50;i++) { var n = Math.randInt(100); a[n]=n.toString(); }
+var pos = E.getAddressOf(a[99]);
+print("\nBefore:");
+E.dumpFragmentation();
+E.defrag();
+print("\nAfter:");
+E.dumpFragmentation();
+for (var i=0;i<100;i++) {
+  if (!a[i].startsWith(i)) throw new Error("Entry "+i+" is wrong");
+}
+result = E.getAddressOf(a[99]) < pos; // it moved!