line based image serialization

Author: jerch

src/ImageAddon.ts

@@ -318,102 +318,24 @@ export class ImageAddon implements ITerminalAddon {
   /**
    * demo hack for complex terminal buffer serialization
    */
-  private _parts: string[] = [''];
 
-  // example for text/FG/BG serializer
-  private _serText(num: number): number[] {
-    // FIXME: no FG/BG attributes atm
-    const buffer = this._terminal!._core.buffer;
-    const line = buffer.lines.get(num);
-    if (!line) return [];
-    const cell = this._terminal?.buffer.active.getNullCell()!;
-    const cols = this._terminal!.cols;
-    const res: number[] = [];
-    let partPos = 0;
-    let content = '';
-    for (let col = 0; col < cols; ++col) {
-      line?.loadCell(col, cell as any);
-      if (!cell.getChars()) {
-        partPos = 0;
-        if (content) this._parts.push(content);
-        content = '';
-      } else {
-        partPos = this._parts.length;
-        content += cell.getChars();
-      }
-      res.push(partPos);
-    }
-    if (content) this._parts.push(content);
-    return res;
-  }
-
-  private _encodeTile(canvas: HTMLCanvasElement): string {
-    // repack cell tile into a proper cell covering canvas if it is too small
-    const cw = this._renderer!.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width;
-    const ch = this._renderer!.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height;
-    if (canvas.width < cw || canvas.height < ch) {
-      const newCanvas = ImageRenderer.createCanvas(window, Math.ceil(cw), Math.ceil(ch));
-      newCanvas.getContext('2d')?.drawImage(canvas, 0, 0);
-      canvas = newCanvas;
-    }
+  public serializeLine(num: number): string {
+    const canvas = this._storage!.extractLineCanvas(num);
+    if (!canvas) return '';
+    const w = this._terminal!.cols;
     const data = canvas.toDataURL('image/png').slice(22);
-    const iipSeq = `\x1b]1337;File=inline=1;width=1;height=1;preserveAspectRatio=0;size=${atob(data).length}:${data}`;
-    return iipSeq + '\x1b[C'; // + cursor advance by one
-  }
-
-  // example for image serializer
-  private _serImages(num: number): number[] {
-    const res: number[] = [];
-    const cols = this._terminal!.cols;
-    const buffer = this._terminal!._core.buffer;
-    const line = buffer.lines.get(num);
-    if (!line) return [];
-
-    for (let col = 0; col < cols; ++col) {
-      // for simplicity only single cell tile encoding atm
-      const canvas = this.extractTileAtBufferCell(col, num);
-      if (!canvas || !canvas.width || !canvas.height) {
-        res.push(0);
-        continue;
-      }
-      res.push(this._parts.length);
-      this._parts.push(this._encodeTile(canvas));
-    }
-    return res;
+    const iipSeq = `\x1b]1337;File=inline=1;width=${w};height=1;preserveAspectRatio=0;size=${atob(data).length}:${data}`;
+    return '\r' + iipSeq + `\x1b[${w}C`; // CR + IIP sequence + cursor advance by line width
   }
 
   public serialize(start: number, end: number): string[] {
     const lines: string[] = [];
-    const cols = this._terminal!.cols;
+    const buffer = this._terminal!._core.buffer;
     for (let row = start; row < end; ++row) {
-      const indices: number[][] = [];
-      // FIXME: turn next 2 invocations into registered event handlers
-      indices.push(this._serText(row));
-      indices.push(this._serImages(row));
-
-      // fuse logic
-      const entries: string[] = [];
-      let cursorAdjust = 0;
-      for (let i = 0; i < cols; ++i) {
-        let entry = '';
-        let handled = 0;
-        for (let k = 0; k < indices.length; ++k) {
-          handled |= indices[k][i];
-          if (this._parts[indices[k][i]]) {
-            entry += this._parts[indices[k][i]];
-            this._parts[indices[k][i]] = '';
-          }
-        }
-        if (handled && cursorAdjust) {
-          entries.push(`\x1b[${cursorAdjust}C`);
-          cursorAdjust = 0;
-        } else if (!handled) {
-          cursorAdjust++;
-        }
-        entries.push(entry);
-      }
-      lines.push(entries.join(''));
-      this._parts.length = 1;
+      const line = buffer.lines.get(row);
+      if (!line) break;
+      // FIXME: hook into serialize addon instead of translateToString
+      lines.push(line.translateToString(true) + this.serializeLine(row));
     }
     return lines;
   }

src/ImageRenderer.ts

@@ -187,6 +187,39 @@ export class ImageRenderer implements IDisposable {
     );
   }
 
+  // TODO: cleanup, maybe merge with draw?
+  public lineDraw(ctx: CanvasRenderingContext2D, imgSpec: IImageSpec, tileId: number, col: number, row: number, count: number = 1): void {
+    const { width, height } = this.cellSize;
+
+    // Don't try to draw anything, if we cannot get valid renderer metrics.
+    if (width === -1 || height === -1) {
+      return;
+    }
+
+    this._rescaleImage(imgSpec, width, height);
+    const img = imgSpec.actual!;
+    const cols = Math.ceil(img.width / width);
+
+    const sx = (tileId % cols) * width;
+    const sy = Math.floor(tileId / cols) * height;
+    const dx = col * width;
+    // const dy = row * height;
+
+    // safari bug: never access image source out of bounds
+    const finalWidth = count * width + sx > img.width ? img.width - sx : count * width;
+    const finalHeight = sy + height > img.height ? img.height - sy : height;
+
+    // Floor all pixel offsets to get stable tile mapping without any overflows.
+    // Note: For not pixel perfect aligned cells like in the DOM renderer
+    // this will move a tile slightly to the top/left (subpixel range, thus ignore it).
+    // FIX #34: avoid striping on displays with pixelDeviceRatio != 1 by ceiling height and width
+    ctx.drawImage(
+      img,
+      Math.floor(sx), Math.floor(sy), Math.ceil(finalWidth), Math.ceil(finalHeight),
+      Math.floor(dx), 0, Math.ceil(finalWidth), Math.ceil(finalHeight)
+    );
+  }
+
   /**
    * Extract a single tile from an image.
    */

src/ImageStorage.ts

@@ -484,6 +484,57 @@ export class ImageStorage implements IDisposable {
     }
   }
 
+  // TODO: cleanup, merge with render? Extend it to an x*y dim extraction method?
+  public extractLineCanvas(num: number): HTMLCanvasElement | undefined {
+    if (!this._images.size) return;
+
+    const buffer = this._terminal._core.buffer;
+    const cols = this._terminal._core.cols;
+    const line = buffer.lines.get(num) as IBufferLineExt;
+    if (!line) return;
+
+    const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width;
+    const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height;
+    const width = this._renderer.dimensions?.css.canvas.width || cw * this._terminal.cols;
+
+    const canvas = ImageRenderer.createCanvas(window, width, Math.ceil(ch));
+    const ctx = canvas.getContext('2d');
+    if (!ctx) return;
+
+    let hasTiles = false;
+
+    for (let col = 0; col < cols; ++col) {
+      if (line.getBg(col) & BgFlags.HAS_EXTENDED) {
+        let e: IExtendedAttrsImage = line._extendedAttrs[col] || EMPTY_ATTRS;
+        const imageId = e.imageId;
+        if (imageId === undefined || imageId === -1) {
+          continue;
+        }
+        const imgSpec = this._images.get(imageId);
+        if (e.tileId !== -1) {
+          const startTile = e.tileId;
+          const startCol = col;
+          let count = 1;
+          while (
+            ++col < cols
+            && (line.getBg(col) & BgFlags.HAS_EXTENDED)
+            && (e = line._extendedAttrs[col] || EMPTY_ATTRS)
+            && (e.imageId === imageId)
+            && (e.tileId === startTile + count)
+          ) {
+            count++;
+          }
+          col--;
+          if (imgSpec && imgSpec.actual) {
+            this._renderer.lineDraw(ctx, imgSpec, startTile, startCol, num, count);
+            hasTiles = true;
+          }
+        }
+      }
+    }
+    if (hasTiles) return canvas;
+  }
+
   /**
    * Extract active single tile at buffer position.
    */