Merge branch 'utf32_buffer' into utf8_input

.gitignore

@@ -23,7 +23,3 @@ coverage/
 # Keep bundled code out of Git
 dist/
 demo/dist/
-
-# dont pullin other files from .vscode than launch.json
-.vscode/
-!.vscode/launch.json

demo/start.js

@@ -5,7 +5,6 @@
  * This file is the entry point for browserify.
  */
 
-const cp = require('child_process');
 const path = require('path');
 const webpack = require('webpack');
 const startServer = require('./server.js');

src/Buffer.ts

@@ -3,13 +3,15 @@
  * @license MIT
  */
 
-import { CircularList } from './common/CircularList';
+import { CircularList, IInsertEvent, IDeleteEvent } from './common/CircularList';
 import { ITerminal, IBuffer, IBufferLine, BufferIndex, IBufferStringIterator, IBufferStringIteratorResult, ICellData } from './Types';
 import { EventEmitter } from './common/EventEmitter';
 import { IMarker } from 'xterm';
 import { BufferLine, CellData } from './BufferLine';
+import { reflowLargerApplyNewLayout, reflowLargerCreateNewLayout, reflowLargerGetLinesToRemove, reflowSmallerGetNewLineLengths } from './BufferReflow';
 import { DEFAULT_COLOR } from './renderer/atlas/Types';
 
+
 export const DEFAULT_ATTR = (0 << 18) | (DEFAULT_COLOR << 9) | (256 << 0);
 export const CHAR_DATA_ATTR_INDEX = 0;
 export const CHAR_DATA_CHAR_INDEX = 1;
@@ -17,10 +19,20 @@ export const CHAR_DATA_WIDTH_INDEX = 2;
 export const CHAR_DATA_CODE_INDEX = 3;
 export const MAX_BUFFER_SIZE = 4294967295; // 2^32 - 1
 
+/**
+ * Null cell - a real empty cell (containing nothing).
+ * Note that code should always be 0 for a null cell as
+ * several test condition of the buffer line rely on this.
+ */
 export const NULL_CELL_CHAR = '';
 export const NULL_CELL_WIDTH = 1;
 export const NULL_CELL_CODE = 0;
 
+/**
+ * Whilespace cell.
+ * This is meant as a replacement for empty cells when needed
+ * during rendering lines to preserve correct aligment.
+ */
 export const WHITESPACE_CELL_CHAR = ' ';
 export const WHITESPACE_CELL_WIDTH = 1;
 export const WHITESPACE_CELL_CODE = 32;
@@ -47,6 +59,8 @@ export class Buffer implements IBuffer {
   public markers: Marker[] = [];
   private _nullCell: ICellData = CellData.fromCharData([0, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]);
   private _whitespaceCell: ICellData = CellData.fromCharData([0, WHITESPACE_CELL_CHAR, WHITESPACE_CELL_WIDTH, WHITESPACE_CELL_CODE]);
+  private _cols: number;
+  private _rows: number;
 
   /**
    * Create a new Buffer.
@@ -58,6 +72,8 @@ export class Buffer implements IBuffer {
     private _terminal: ITerminal,
     private _hasScrollback: boolean
   ) {
+    this._cols = this._terminal.cols;
+    this._rows = this._terminal.rows;
     this.clear();
   }
 
@@ -78,13 +94,13 @@ export class Buffer implements IBuffer {
   }
 
   public get hasScrollback(): boolean {
-    return this._hasScrollback && this.lines.maxLength > this._terminal.rows;
+    return this._hasScrollback && this.lines.maxLength > this._rows;
   }
 
   public get isCursorInViewport(): boolean {
     const absoluteY = this.ybase + this.y;
     const relativeY = absoluteY - this.ydisp;
-    return (relativeY >= 0 && relativeY < this._terminal.rows);
+    return (relativeY >= 0 && relativeY < this._rows);
   }
 
   /**
@@ -110,7 +126,7 @@ export class Buffer implements IBuffer {
       if (fillAttr === undefined) {
         fillAttr = DEFAULT_ATTR;
       }
-      let i = this._terminal.rows;
+      let i = this._rows;
       while (i--) {
         this.lines.push(this.getBlankLine(fillAttr));
       }
@@ -125,9 +141,9 @@ export class Buffer implements IBuffer {
     this.ybase = 0;
     this.y = 0;
     this.x = 0;
-    this.lines = new CircularList<IBufferLine>(this._getCorrectBufferLength(this._terminal.rows));
+    this.lines = new CircularList<IBufferLine>(this._getCorrectBufferLength(this._rows));
     this.scrollTop = 0;
-    this.scrollBottom = this._terminal.rows - 1;
+    this.scrollBottom = this._rows - 1;
     this.setupTabStops();
   }
 
@@ -137,6 +153,9 @@ export class Buffer implements IBuffer {
    * @param newRows The new number of rows.
    */
   public resize(newCols: number, newRows: number): void {
+    // store reference to null cell with default attrs
+    const nullCell = this.getNullCell(DEFAULT_ATTR);
+
     // Increase max length if needed before adjustments to allow space to fill
     // as required.
     const newMaxLength = this._getCorrectBufferLength(newRows);
@@ -147,18 +166,17 @@ export class Buffer implements IBuffer {
     // The following adjustments should only happen if the buffer has been
     // initialized/filled.
     if (this.lines.length > 0) {
-      // Deal with columns increasing (we don't do anything when columns reduce)
-      if (this._terminal.cols < newCols) {
-        const cell = this.getNullCell(DEFAULT_ATTR);  // does xterm use the default attr?
+      // Deal with columns increasing (reducing needs to happen after reflow)
+      if (this._cols < newCols) {
         for (let i = 0; i < this.lines.length; i++) {
-          this.lines.get(i).resize(newCols, cell);
+          this.lines.get(i).resize(newCols, nullCell);
         }
       }
 
       // Resize rows in both directions as needed
       let addToY = 0;
-      if (this._terminal.rows < newRows) {
-        for (let y = this._terminal.rows; y < newRows; y++) {
+      if (this._rows < newRows) {
+        for (let y = this._rows; y < newRows; y++) {
           if (this.lines.length < newRows + this.ybase) {
             if (this.ybase > 0 && this.lines.length <= this.ybase + this.y + addToY + 1) {
               // There is room above the buffer and there are no empty elements below the line,
@@ -172,12 +190,12 @@ export class Buffer implements IBuffer {
             } else {
               // Add a blank line if there is no buffer left at the top to scroll to, or if there
               // are blank lines after the cursor
-              this.lines.push(new BufferLine(newCols, this.getNullCell(DEFAULT_ATTR)));
+              this.lines.push(new BufferLine(newCols, nullCell));
             }
           }
         }
-      } else { // (this._terminal.rows >= newRows)
-        for (let y = this._terminal.rows; y > newRows; y--) {
+      } else { // (this._rows >= newRows)
+        for (let y = this._rows; y > newRows; y--) {
           if (this.lines.length > newRows + this.ybase) {
             if (this.lines.length > this.ybase + this.y + 1) {
               // The line is a blank line below the cursor, remove it
@@ -217,8 +235,234 @@ export class Buffer implements IBuffer {
     }
 
     this.scrollBottom = newRows - 1;
+
+    if (this._hasScrollback) {
+      this._reflow(newCols, newRows);
+
+      // Trim the end of the line off if cols shrunk
+      if (this._cols > newCols) {
+        for (let i = 0; i < this.lines.length; i++) {
+          this.lines.get(i).resize(newCols, nullCell);
+        }
+      }
+    }
+
+    this._cols = newCols;
+    this._rows = newRows;
   }
 
+  private _reflow(newCols: number, newRows: number): void {
+    if (this._cols === newCols) {
+      return;
+    }
+
+    // Iterate through rows, ignore the last one as it cannot be wrapped
+    if (newCols > this._cols) {
+      this._reflowLarger(newCols);
+    } else {
+      this._reflowSmaller(newCols, newRows);
+    }
+  }
+
+  private _reflowLarger(newCols: number): void {
+    const toRemove: number[] = reflowLargerGetLinesToRemove(this.lines, newCols, this.ybase + this.y);
+    if (toRemove.length > 0) {
+      const newLayoutResult = reflowLargerCreateNewLayout(this.lines, toRemove);
+      reflowLargerApplyNewLayout(this.lines, newLayoutResult.layout);
+      this._reflowLargerAdjustViewport(newCols, newLayoutResult.countRemoved);
+    }
+  }
+
+  private _reflowLargerAdjustViewport(newCols: number, countRemoved: number): void {
+    const nullCell = this.getNullCell(DEFAULT_ATTR);
+    // Adjust viewport based on number of items removed
+    let viewportAdjustments = countRemoved;
+    while (viewportAdjustments-- > 0) {
+      if (this.ybase === 0) {
+        if (this.y > 0) {
+          this.y--;
+        }
+        if (this.lines.length < this._rows) {
+          // Add an extra row at the bottom of the viewport
+          this.lines.push(new BufferLine(newCols, nullCell));
+        }
+      } else {
+        if (this.ydisp === this.ybase) {
+          this.ydisp--;
+        }
+        this.ybase--;
+      }
+    }
+  }
+
+  private _reflowSmaller(newCols: number, newRows: number): void {
+    const nullCell = this.getNullCell(DEFAULT_ATTR);
+    // Gather all BufferLines that need to be inserted into the Buffer here so that they can be
+    // batched up and only committed once
+    const toInsert = [];
+    let countToInsert = 0;
+    // Go backwards as many lines may be trimmed and this will avoid considering them
+    for (let y = this.lines.length - 1; y >= 0; y--) {
+      // Check whether this line is a problem
+      let nextLine = this.lines.get(y) as BufferLine;
+      if (!nextLine.isWrapped && nextLine.getTrimmedLength() <= newCols) {
+        continue;
+      }
+
+      // Gather wrapped lines and adjust y to be the starting line
+      const wrappedLines: BufferLine[] = [nextLine];
+      while (nextLine.isWrapped && y > 0) {
+        nextLine = this.lines.get(--y) as BufferLine;
+        wrappedLines.unshift(nextLine);
+      }
+
+      // If these lines contain the cursor don't touch them, the program will handle fixing up
+      // wrapped lines with the cursor
+      const absoluteY = this.ybase + this.y;
+      if (absoluteY >= y && absoluteY < y + wrappedLines.length) {
+        continue;
+      }
+
+      const lastLineLength = wrappedLines[wrappedLines.length - 1].getTrimmedLength();
+      const destLineLengths = reflowSmallerGetNewLineLengths(wrappedLines, this._cols, newCols);
+      const linesToAdd = destLineLengths.length - wrappedLines.length;
+      let trimmedLines: number;
+      if (this.ybase === 0 && this.y !== this.lines.length - 1) {
+        // If the top section of the buffer is not yet filled
+        trimmedLines = Math.max(0, this.y - this.lines.maxLength + linesToAdd);
+      } else {
+        trimmedLines = Math.max(0, this.lines.length - this.lines.maxLength + linesToAdd);
+      }
+
+      // Add the new lines
+      const newLines: BufferLine[] = [];
+      for (let i = 0; i < linesToAdd; i++) {
+        const newLine = this.getBlankLine(DEFAULT_ATTR, true) as BufferLine;
+        newLines.push(newLine);
+      }
+      if (newLines.length > 0) {
+        toInsert.push({
+          // countToInsert here gets the actual index, taking into account other inserted items.
+          // using this we can iterate through the list forwards
+          start: y + wrappedLines.length + countToInsert,
+          newLines
+        });
+        countToInsert += newLines.length;
+      }
+      wrappedLines.push(...newLines);
+
+      // Copy buffer data to new locations, this needs to happen backwards to do in-place
+      let destLineIndex = destLineLengths.length - 1; // Math.floor(cellsNeeded / newCols);
+      let destCol = destLineLengths[destLineIndex]; // cellsNeeded % newCols;
+      if (destCol === 0) {
+        destLineIndex--;
+        destCol = destLineLengths[destLineIndex];
+      }
+      let srcLineIndex = wrappedLines.length - linesToAdd - 1;
+      let srcCol = lastLineLength;
+      while (srcLineIndex >= 0) {
+        const cellsToCopy = Math.min(srcCol, destCol);
+        wrappedLines[destLineIndex].copyCellsFrom(wrappedLines[srcLineIndex], srcCol - cellsToCopy, destCol - cellsToCopy, cellsToCopy, true);
+        destCol -= cellsToCopy;
+        if (destCol === 0) {
+          destLineIndex--;
+          destCol = destLineLengths[destLineIndex];
+        }
+        srcCol -= cellsToCopy;
+        if (srcCol === 0) {
+          srcLineIndex--;
+          // TODO: srcCol shoudl take trimmed length into account
+          srcCol = wrappedLines[Math.max(srcLineIndex, 0)].getTrimmedLength(); // this._cols;
+        }
+      }
+
+      // Null out the end of the line ends if a wide character wrapped to the following line
+      for (let i = 0; i < wrappedLines.length; i++) {
+        if (destLineLengths[i] < newCols) {
+          wrappedLines[i].setCell(destLineLengths[i], nullCell);
+        }
+      }
+
+      // Adjust viewport as needed
+      let viewportAdjustments = linesToAdd - trimmedLines;
+      while (viewportAdjustments-- > 0) {
+        if (this.ybase === 0) {
+          if (this.y < this._rows - 1) {
+            this.y++;
+            this.lines.pop();
+          } else {
+            this.ybase++;
+            this.ydisp++;
+          }
+        } else {
+          // Ensure ybase does not exceed its maximum value
+          if (this.ybase < Math.min(this.lines.maxLength, this.lines.length + countToInsert) - newRows) {
+            if (this.ybase === this.ydisp) {
+              this.ydisp++;
+            }
+            this.ybase++;
+          }
+        }
+      }
+    }
+
+    // Rearrange lines in the buffer if there are any insertions, this is done at the end rather
+    // than earlier so that it's a single O(n) pass through the buffer, instead of O(n^2) from many
+    // costly calls to CircularList.splice.
+    if (toInsert.length > 0) {
+      // Record buffer insert events and then play them back backwards so that the indexes are
+      // correct
+      const insertEvents: IInsertEvent[] = [];
+
+      // Record original lines so they don't get overridden when we rearrange the list
+      const originalLines: BufferLine[] = [];
+      for (let i = 0; i < this.lines.length; i++) {
+        originalLines.push(this.lines.get(i) as BufferLine);
+      }
+      const originalLinesLength = this.lines.length;
+
+      let originalLineIndex = originalLinesLength - 1;
+      let nextToInsertIndex = 0;
+      let nextToInsert = toInsert[nextToInsertIndex];
+      this.lines.length = Math.min(this.lines.maxLength, this.lines.length + countToInsert);
+      let countInsertedSoFar = 0;
+      for (let i = Math.min(this.lines.maxLength - 1, originalLinesLength + countToInsert - 1); i >= 0; i--) {
+        if (nextToInsert && nextToInsert.start > originalLineIndex + countInsertedSoFar) {
+          // Insert extra lines here, adjusting i as needed
+          for (let nextI = nextToInsert.newLines.length - 1; nextI >= 0; nextI--) {
+            this.lines.set(i--, nextToInsert.newLines[nextI]);
+          }
+          i++;
+
+          // Create insert events for later
+          insertEvents.push({
+            index: originalLineIndex + 1,
+            amount: nextToInsert.newLines.length
+          } as IInsertEvent);
+
+          countInsertedSoFar += nextToInsert.newLines.length;
+          nextToInsert = toInsert[++nextToInsertIndex];
+        } else {
+          this.lines.set(i, originalLines[originalLineIndex--]);
+        }
+      }
+
+      // Update markers
+      let insertCountEmitted = 0;
+      for (let i = insertEvents.length - 1; i >= 0; i--) {
+        insertEvents[i].index += insertCountEmitted;
+        this.lines.emit('insert', insertEvents[i]);
+        insertCountEmitted += insertEvents[i].amount;
+      }
+      const amountToTrim = Math.max(0, originalLinesLength + countToInsert - this.lines.maxLength);
+      if (amountToTrim > 0) {
+        this.lines.emitMayRemoveListeners('trim', amountToTrim);
+      }
+    }
+  }
+
+  // private _reflowSmallerGetLinesNeeded()
+
   /**
    * Translates a string index back to a BufferIndex.
    * To get the correct buffer position the string must start at `startCol` 0
@@ -232,14 +476,19 @@ export class Buffer implements IBuffer {
    * @param stringIndex index within the string
    * @param startCol column offset the string was retrieved from
    */
-  public stringIndexToBufferIndex(lineIndex: number, stringIndex: number): BufferIndex {
+  public stringIndexToBufferIndex(lineIndex: number, stringIndex: number, trimRight: boolean = false): BufferIndex {
     while (stringIndex) {
       const line = this.lines.get(lineIndex);
       if (!line) {
         return [-1, -1];
       }
-      for (let i = 0; i < line.length; ++i) {
-        stringIndex -= line.getString(i).length;
+      const length = (trimRight) ? line.getTrimmedLength() : line.length;
+      for (let i = 0; i < length; ++i) {
+        if (line.get(i)[CHAR_DATA_WIDTH_INDEX]) {
+          // empty cells report a string length of 0, but get replaced
+          // with a whitespace in translateToString, thus replace with 1
+          stringIndex -= line.get(i)[CHAR_DATA_CHAR_INDEX].length || 1;
+        }
         if (stringIndex < 0) {
           return [lineIndex, i];
         }
@@ -295,7 +544,7 @@ export class Buffer implements IBuffer {
       i = 0;
     }
 
-    for (; i < this._terminal.cols; i += this._terminal.options.tabStopWidth) {
+    for (; i < this._cols; i += this._terminal.options.tabStopWidth) {
       this.tabs[i] = true;
     }
   }
@@ -309,7 +558,7 @@ export class Buffer implements IBuffer {
       x = this.x;
     }
     while (!this.tabs[--x] && x > 0);
-    return x >= this._terminal.cols ? this._terminal.cols - 1 : x < 0 ? 0 : x;
+    return x >= this._cols ? this._cols - 1 : x < 0 ? 0 : x;
   }
 
   /**
@@ -320,8 +569,8 @@ export class Buffer implements IBuffer {
     if (x === null || x === undefined) {
       x = this.x;
     }
-    while (!this.tabs[++x] && x < this._terminal.cols);
-    return x >= this._terminal.cols ? this._terminal.cols - 1 : x < 0 ? 0 : x;
+    while (!this.tabs[++x] && x < this._cols);
+    return x >= this._cols ? this._cols - 1 : x < 0 ? 0 : x;
   }
 
   public addMarker(y: number): Marker {
@@ -334,12 +583,27 @@ export class Buffer implements IBuffer {
         marker.dispose();
       }
     }));
+    marker.register(this.lines.addDisposableListener('insert', (event: IInsertEvent) => {
+      if (marker.line >= event.index) {
+        marker.line += event.amount;
+      }
+    }));
+    marker.register(this.lines.addDisposableListener('delete', (event: IDeleteEvent) => {
+      // Delete the marker if it's within the range
+      if (marker.line >= event.index && marker.line < event.index + event.amount) {
+        marker.dispose();
+      }
+
+      // Shift the marker if it's after the deleted range
+      if (marker.line > event.index) {
+        marker.line -= event.amount;
+      }
+    }));
     marker.register(marker.addDisposableListener('dispose', () => this._removeMarker(marker)));
     return marker;
   }
 
   private _removeMarker(marker: Marker): void {
-    // TODO: This could probably be optimized by relying on sort order and trimming the array using .length
     this.markers.splice(this.markers.indexOf(marker), 1);
   }
 

src/BufferLine.test.ts

@@ -9,6 +9,10 @@ import { NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE, DEFAULT_ATTR } from '.
 
 
 class TestBufferLine extends BufferLine {
+  public get combined(): {[index: number]: string} {
+    return this._combined;
+  }
+
   public toArray(): CharData[] {
     const result = [];
     for (let i = 0; i < this.length; ++i) {
@@ -24,23 +28,23 @@ describe('CellData', () => {
     // ASCII
     cell.setFromCharData([123, 'a', 1, 'a'.charCodeAt(0)]);
     chai.assert.deepEqual(cell.asCharData, [123, 'a', 1, 'a'.charCodeAt(0)]);
-    chai.assert.equal(cell.combined, 0);
+    chai.assert.equal(cell.isCombined, 0);
     // combining
     cell.setFromCharData([123, 'e\u0301', 1, '\u0301'.charCodeAt(0)]);
     chai.assert.deepEqual(cell.asCharData, [123, 'e\u0301', 1, '\u0301'.charCodeAt(0)]);
-    chai.assert.equal(cell.combined, Content.IS_COMBINED);
+    chai.assert.equal(cell.isCombined, Content.IS_COMBINED);
     // surrogate
     cell.setFromCharData([123, '𝄞', 1, 0x1D11E]);
     chai.assert.deepEqual(cell.asCharData, [123, '𝄞', 1, 0x1D11E]);
-    chai.assert.equal(cell.combined, 0);
+    chai.assert.equal(cell.isCombined, 0);
     // surrogate + combining
     cell.setFromCharData([123, '𓂀\u0301', 1, '𓂀\u0301'.charCodeAt(2)]);
     chai.assert.deepEqual(cell.asCharData, [123, '𓂀\u0301', 1, '𓂀\u0301'.charCodeAt(2)]);
-    chai.assert.equal(cell.combined, Content.IS_COMBINED);
+    chai.assert.equal(cell.isCombined, Content.IS_COMBINED);
     // wide char
     cell.setFromCharData([123, '１', 2, '１'.charCodeAt(0)]);
     chai.assert.deepEqual(cell.asCharData, [123, '１', 2, '１'.charCodeAt(0)]);
-    chai.assert.equal(cell.combined, 0);
+    chai.assert.equal(cell.isCombined, 0);
   });
 });
 
@@ -167,63 +171,30 @@ describe('BufferLine', function(): void {
     });
     it('enlarge(true)', function(): void {
       const line = new TestBufferLine(5, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      line.resize(10, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), true);
+      line.resize(10, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]));
       chai.expect(line.toArray()).eql(Array(10).fill([1, 'a', 0, 'a'.charCodeAt(0)]));
     });
     it('shrink(true) - should apply new size', function(): void {
       const line = new TestBufferLine(10, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      line.resize(5, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), true);
+      line.resize(5, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]));
       chai.expect(line.toArray()).eql(Array(5).fill([1, 'a', 0, 'a'.charCodeAt(0)]));
     });
-    it('shrink(false) - should not apply new size', function(): void {
-      const line = new TestBufferLine(10, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      line.resize(5, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      chai.expect(line.toArray()).eql(Array(10).fill([1, 'a', 0, 'a'.charCodeAt(0)]));
-    });
-    it('shrink(false) + shrink(false) - should not apply new size', function(): void {
-      const line = new TestBufferLine(20, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      line.resize(10, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      line.resize(5, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      chai.expect(line.toArray()).eql(Array(20).fill([1, 'a', 0, 'a'.charCodeAt(0)]));
-    });
-    it('shrink(false) + enlarge(false) to smaller than before', function(): void {
-      const line = new TestBufferLine(20, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      line.resize(10, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      line.resize(15, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]));
-      chai.expect(line.toArray()).eql(Array(20).fill([1, 'a', 0, 'a'.charCodeAt(0)]));
-    });
-    it('shrink(false) + enlarge(false) to bigger than before', function(): void {
-      const line = new TestBufferLine(20, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      line.resize(10, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      line.resize(25, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]));
-      chai.expect(line.toArray()).eql(Array(25).fill([1, 'a', 0, 'a'.charCodeAt(0)]));
-    });
-    it('shrink(false) + resize shrink=true should enforce shrinking', function(): void {
-      const line = new TestBufferLine(20, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      line.resize(10, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      line.resize(10, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), true);
-      chai.expect(line.toArray()).eql(Array(10).fill([1, 'a', 0, 'a'.charCodeAt(0)]));
-    });
-    it('enlarge from 0 length', function(): void {
-      const line = new TestBufferLine(0, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      line.resize(10, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      chai.expect(line.toArray()).eql(Array(10).fill([1, 'a', 0, 'a'.charCodeAt(0)]));
-    });
     it('shrink to 0 length', function(): void {
       const line = new TestBufferLine(10, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      line.resize(0, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), true);
+      line.resize(0, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]));
       chai.expect(line.toArray()).eql(Array(0).fill([1, 'a', 0, 'a'.charCodeAt(0)]));
     });
-    it('shrink(false) to 0 and enlarge to different sizes', function(): void {
+    it('should remove combining data on replaced cells after shrinking then enlarging', () => {
       const line = new TestBufferLine(10, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      line.resize(0, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      chai.expect(line.toArray()).eql(Array(10).fill([1, 'a', 0, 'a'.charCodeAt(0)]));
-      line.resize(5, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      chai.expect(line.toArray()).eql(Array(10).fill([1, 'a', 0, 'a'.charCodeAt(0)]));
-      line.resize(7, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), false);
-      chai.expect(line.toArray()).eql(Array(10).fill([1, 'a', 0, 'a'.charCodeAt(0)]));
-      line.resize(7, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]), true);
-      chai.expect(line.toArray()).eql(Array(7).fill([1, 'a', 0, 'a'.charCodeAt(0)]));
+      line.set(2, [ null, '😁', 1, '😁'.charCodeAt(0) ]);
+      line.set(9, [ null, '😁', 1, '😁'.charCodeAt(0) ]);
+      chai.expect(line.translateToString()).eql('aa😁aaaaaa😁');
+      chai.expect(Object.keys(line.combined).length).eql(2);
+      line.resize(5, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]));
+      chai.expect(line.translateToString()).eql('aa😁aa');
+      line.resize(10, CellData.fromCharData([1, 'a', 0, 'a'.charCodeAt(0)]));
+      chai.expect(line.translateToString()).eql('aa😁aaaaaaa');
+      chai.expect(Object.keys(line.combined).length).eql(1);
     });
   });
   describe('getTrimLength', function(): void {
@@ -360,39 +331,39 @@ describe('BufferLine', function(): void {
   describe('addCharToCell', () => {
     it('should set width to 1 for empty cell', () => {
       const line = new TestBufferLine(3, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
-      line.addCharToCell(0, '\u0301'.charCodeAt(0));
+      line.addCodepointToCell(0, '\u0301'.charCodeAt(0));
       const cell = line.loadCell(0, new CellData());
       // chars contains single combining char
       // width is set to 1
       chai.assert.deepEqual(cell.asCharData, [DEFAULT_ATTR, '\u0301', 1, 0x0301]);
       // do not account a single combining char as combined
-      chai.assert.equal(cell.combined, 0);
+      chai.assert.equal(cell.isCombined, 0);
     });
     it('should add char to combining string in cell', () => {
       const line = new TestBufferLine(3, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
       const cell = line .loadCell(0, new CellData());
       cell.setFromCharData([123, 'e\u0301', 1, 'e\u0301'.charCodeAt(1)]);
       line.setCell(0, cell);
-      line.addCharToCell(0, '\u0301'.charCodeAt(0));
+      line.addCodepointToCell(0, '\u0301'.charCodeAt(0));
       line.loadCell(0, cell);
       // chars contains 3 chars
       // width is set to 1
       chai.assert.deepEqual(cell.asCharData, [123, 'e\u0301\u0301', 1, 0x0301]);
       // do not account a single combining char as combined
-      chai.assert.equal(cell.combined, Content.IS_COMBINED);
+      chai.assert.equal(cell.isCombined, Content.IS_COMBINED);
     });
     it('should create combining string on taken cell', () => {
       const line = new TestBufferLine(3, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
       const cell = line .loadCell(0, new CellData());
       cell.setFromCharData([123, 'e', 1, 'e'.charCodeAt(1)]);
       line.setCell(0, cell);
-      line.addCharToCell(0, '\u0301'.charCodeAt(0));
+      line.addCodepointToCell(0, '\u0301'.charCodeAt(0));
       line.loadCell(0, cell);
       // chars contains 2 chars
       // width is set to 1
       chai.assert.deepEqual(cell.asCharData, [123, 'e\u0301', 1, 0x0301]);
       // do not account a single combining char as combined
-      chai.assert.equal(cell.combined, Content.IS_COMBINED);
+      chai.assert.equal(cell.isCombined, Content.IS_COMBINED);
     });
   });
 });

src/BufferLine.ts

@@ -60,7 +60,7 @@ export const enum Content {
    *              whether a cell contains anything
    *              read:   `isEmtpy = !(content & Content.hasContent)`
    */
-  HAS_CONTENT = 0x2FFFFF,
+  HAS_CONTENT = 0x3FFFFF,
 
   /**
    * bit 23..24   wcwidth value of cell, takes 2 bits (ranges from 0..2)
@@ -78,8 +78,6 @@ export const enum Content {
 
 /**
  * CellData - represents a single Cell in the terminal buffer.
- *
- * TODO: attr getter
  */
 export class CellData implements ICellData {
 
@@ -97,7 +95,7 @@ export class CellData implements ICellData {
   public combinedData: string = '';
 
   /** Whether cell contains a combined string. */
-  public get combined(): number {
+  public get isCombined(): number {
     return this.content & Content.IS_COMBINED;
   }
 
@@ -117,9 +115,16 @@ export class CellData implements ICellData {
     return '';
   }
 
-  /** Codepoint of cell (or last charCode of combined string) */
+  /**
+   * Codepoint of cell
+   * Note this returns the UTF32 codepoint of single chars,
+   * if content is a combined string it returns the codepoint
+   * of the last char in string to be in line with code in CharData.
+   * */
   public get code(): number {
-    return ((this.combined) ? this.combinedData.charCodeAt(this.combinedData.length - 1) : this.content & Content.CODEPOINT_MASK);
+    return (this.isCombined)
+      ? this.combinedData.charCodeAt(this.combinedData.length - 1)
+      : this.content & Content.CODEPOINT_MASK;
   }
 
   /** Set data from CharData */
@@ -127,10 +132,14 @@ export class CellData implements ICellData {
     this.fg = value[CHAR_DATA_ATTR_INDEX];
     this.bg = 0;
     let combined = false;
+
+    // surrogates and combined strings need special treatment
     if (value[CHAR_DATA_CHAR_INDEX].length > 2) {
       combined = true;
     } else if (value[CHAR_DATA_CHAR_INDEX].length === 2) {
       const code = value[CHAR_DATA_CHAR_INDEX].charCodeAt(0);
+      // if the 2-char string is a surrogate create single codepoint
+      // everything else is combined
       if (0xD800 <= code && code <= 0xDBFF) {
         const second = value[CHAR_DATA_CHAR_INDEX].charCodeAt(1);
         if (0xDC00 <= second && second <= 0xDFFF) {
@@ -159,6 +168,18 @@ export class CellData implements ICellData {
 
 /**
  * Typed array based bufferline implementation.
+ *
+ * There are 2 ways to insert data into the cell buffer:
+ * - `setCellFromCodepoint` + `addCodepointToCell`
+ *   Use these for data that is already UTF32.
+ *   Used during normal input in `InputHandler` for faster buffer access.
+ * - `setCell`
+ *   This method takes a CellData object and stores the data in the buffer.
+ *   Use `CellData.fromCharData` to create the CellData object (e.g. from JS string).
+ *
+ * To retrieve data from the buffer use either one of the primitive methods
+ * (if only one particular value is needed) or `loadCell`. For `loadCell` in a loop
+ * memory allocs / GC pressure can be greatly reduced by reusing the CellData object.
  */
 export class BufferLine implements IBufferLine {
   protected _data: Uint32Array | null = null;
@@ -217,24 +238,36 @@ export class BufferLine implements IBufferLine {
     return this._data[index * CELL_SIZE + Cell.CONTENT] >> Content.WIDTH_SHIFT;
   }
 
+  /** Test whether content has width. */
   public hasWidth(index: number): number {
     return this._data[index * CELL_SIZE + Cell.CONTENT] & Content.WIDTH_MASK;
   }
 
+  /** Get FG cell component. */
   public getFG(index: number): number {
     return this._data[index * CELL_SIZE + Cell.FG];
   }
 
+  /** Get BG cell component. */
   public getBG(index: number): number {
     return this._data[index * CELL_SIZE + Cell.BG];
   }
 
+  /**
+   * Test whether contains any chars.
+   * Basically an empty has no content, but other cells might differ in FG/BG
+   * from real empty cells.
+   * */
   public hasContent(index: number): number {
     return this._data[index * CELL_SIZE + Cell.CONTENT] & Content.HAS_CONTENT;
   }
 
+  /**
+   * Get codepoint of the cell.
+   * To be in line with `code` in CharData this either returns
+   * a single UTF32 codepoint or the last codepoint of a combined string.
+   */
   public getCodePoint(index: number): number {
-    // returns either the single codepoint or the last charCode in combined
     const content = this._data[index * CELL_SIZE + Cell.CONTENT];
     if (content & Content.IS_COMBINED) {
       return this._combined[index].charCodeAt(this._combined[index].length - 1);
@@ -242,10 +275,12 @@ export class BufferLine implements IBufferLine {
     return content & Content.CODEPOINT_MASK;
   }
 
+  /** Test whether the cell contains a combined string. */
   public isCombined(index: number): number {
     return this._data[index * CELL_SIZE + Cell.CONTENT] & Content.IS_COMBINED;
   }
 
+  /** Returns the string content of the cell. */
   public getString(index: number): string {
     const content = this._data[index * CELL_SIZE + Cell.CONTENT];
     if (content & Content.IS_COMBINED) {
@@ -254,7 +289,8 @@ export class BufferLine implements IBufferLine {
     if (content & Content.CODEPOINT_MASK) {
       return stringFromCodePoint(content & Content.CODEPOINT_MASK);
     }
-    return ''; // return empty string for empty cells
+    // return empty string for empty cells
+    return '';
   }
 
   /**
@@ -276,9 +312,6 @@ export class BufferLine implements IBufferLine {
   public setCell(index: number, cell: ICellData): void {
     if (cell.content & Content.IS_COMBINED) {
       this._combined[index] = cell.combinedData;
-      // we also need to clear and set codepoint to index
-      cell.content &= ~Content.CODEPOINT_MASK;
-      cell.content |= index;
     }
     this._data[index * CELL_SIZE + Cell.CONTENT] = cell.content;
     this._data[index * CELL_SIZE + Cell.FG] = cell.fg;
@@ -290,19 +323,19 @@ export class BufferLine implements IBufferLine {
    * Since the input handler see the incoming chars as UTF32 codepoints,
    * it gets an optimized access method.
    */
-  public setDataFromCodePoint(index: number, codePoint: number, width: number, fg: number, bg: number): void {
+  public setCellFromCodePoint(index: number, codePoint: number, width: number, fg: number, bg: number): void {
     this._data[index * CELL_SIZE + Cell.CONTENT] = codePoint | (width << Content.WIDTH_SHIFT);
     this._data[index * CELL_SIZE + Cell.FG] = fg;
     this._data[index * CELL_SIZE + Cell.BG] = bg;
   }
 
   /**
-   * Add a char to a cell from input handler.
+   * Add a codepoint to a cell from input handler.
    * During input stage combining chars with a width of 0 follow and stack
    * onto a leading char. Since we already set the attrs
    * by the previous `setDataFromCodePoint` call, we can omit it here.
    */
-  public addCharToCell(index: number, codePoint: number): void {
+  public addCodepointToCell(index: number, codePoint: number): void {
     let content = this._data[index * CELL_SIZE + Cell.CONTENT];
     if (content & Content.IS_COMBINED) {
       // we already have a combined string, simply add
@@ -311,11 +344,10 @@ export class BufferLine implements IBufferLine {
       if (content & Content.CODEPOINT_MASK) {
         // normal case for combining chars:
         //  - move current leading char + new one into combined string
-        //  - set codepoint in cell buffer to index
         //  - set combined flag
         this._combined[index] = stringFromCodePoint(content & Content.CODEPOINT_MASK) + stringFromCodePoint(codePoint);
-        content &= ~Content.CODEPOINT_MASK;
-        content |= index | Content.IS_COMBINED;
+        content &= ~Content.CODEPOINT_MASK; // set codepoint in buffer to 0
+        content |= Content.IS_COMBINED;
       } else {
         // should not happen - we actually have no data in the cell yet
         // simply set the data in the cell buffer with a width of 1
@@ -365,8 +397,8 @@ export class BufferLine implements IBufferLine {
     }
   }
 
-  public resize(cols: number, fillCellData: ICellData, shrink: boolean = false): void {
-    if (cols === this.length || (!shrink && cols < this.length)) {
+  public resize(cols: number, fillCellData: ICellData): void {
+    if (cols === this.length) {
       return;
     }
     if (cols > this.length) {
@@ -382,13 +414,22 @@ export class BufferLine implements IBufferLine {
       for (let i = this.length; i < cols; ++i) {
         this.setCell(i, fillCellData);
       }
-    } else if (shrink) {
+    } else {
       if (cols) {
         const data = new Uint32Array(cols * CELL_SIZE);
         data.set(this._data.subarray(0, cols * CELL_SIZE));
         this._data = data;
+        // Remove any cut off combined data
+        const keys = Object.keys(this._combined);
+        for (let i = 0; i < keys.length; i++) {
+          const key = parseInt(keys[i], 10);
+          if (key >= cols) {
+            delete this._combined[key];
+          }
+        }
       } else {
         this._data = null;
+        this._combined = {};
       }
     }
     this.length = cols;
@@ -439,6 +480,32 @@ export class BufferLine implements IBufferLine {
     return 0;
   }
 
+  public copyCellsFrom(src: BufferLine, srcCol: number, destCol: number, length: number, applyInReverse: boolean): void {
+    const srcData = src._data;
+    if (applyInReverse) {
+      for (let cell = length - 1; cell >= 0; cell--) {
+        for (let i = 0; i < CELL_SIZE; i++) {
+          this._data[(destCol + cell) * CELL_SIZE + i] = srcData[(srcCol + cell) * CELL_SIZE + i];
+        }
+      }
+    } else {
+      for (let cell = 0; cell < length; cell++) {
+        for (let i = 0; i < CELL_SIZE; i++) {
+          this._data[(destCol + cell) * CELL_SIZE + i] = srcData[(srcCol + cell) * CELL_SIZE + i];
+        }
+      }
+    }
+
+    // Move any combined data over as needed
+    const srcCombinedKeys = Object.keys(src._combined);
+    for (let i = 0; i < srcCombinedKeys.length; i++) {
+      const key = parseInt(srcCombinedKeys[i], 10);
+      if (key >= srcCol) {
+        this._combined[key - srcCol + destCol] = src._combined[key];
+      }
+    }
+  }
+
   public translateToString(trimRight: boolean = false, startCol: number = 0, endCol: number = this.length): string {
     if (trimRight) {
       endCol = Math.min(endCol, this.getTrimmedLength());

src/BufferReflow.test.ts

@@ -0,0 +1,93 @@
+/**
+ * Copyright (c) 2019 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+import { assert } from 'chai';
+import { BufferLine } from './BufferLine';
+import { reflowSmallerGetNewLineLengths } from './BufferReflow';
+import { NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE } from './Buffer';
+
+describe('BufferReflow', () => {
+  describe('reflowSmallerGetNewLineLengths', () => {
+    it('should return correct line lengths for a small line with wide characters', () => {
+      const line = new BufferLine(4);
+      line.set(0, [null, '汉', 2, '汉'.charCodeAt(0)]);
+      line.set(1, [null, '', 0, undefined]);
+      line.set(2, [null, '语', 2, '语'.charCodeAt(0)]);
+      line.set(3, [null, '', 0, undefined]);
+      assert.equal(line.translateToString(true), '汉语');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 4, 3), [2, 2], 'line: 汉, 语');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 4, 2), [2, 2], 'line: 汉, 语');
+    });
+    it('should return correct line lengths for a large line with wide characters', () => {
+      const line = new BufferLine(12);
+      for (let i = 0; i < 12; i += 4) {
+        line.set(i, [null, '汉', 2, '汉'.charCodeAt(0)]);
+        line.set(i + 2, [null, '语', 2, '语'.charCodeAt(0)]);
+      }
+      for (let i = 1; i < 12; i += 2) {
+        line.set(i, [null, '', 0, undefined]);
+        line.set(i, [null, '', 0, undefined]);
+      }
+      assert.equal(line.translateToString(), '汉语汉语汉语');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 12, 11), [10, 2], 'line: 汉语汉语汉, 语');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 12, 10), [10, 2], 'line: 汉语汉语汉, 语');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 12, 9), [8, 4], 'line: 汉语汉语, 汉语');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 12, 8), [8, 4], 'line: 汉语汉语, 汉语');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 12, 7), [6, 6], 'line: 汉语汉, 语汉语');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 12, 6), [6, 6], 'line: 汉语汉, 语汉语');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 12, 5), [4, 4, 4], 'line: 汉语, 汉语, 汉语');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 12, 4), [4, 4, 4], 'line: 汉语, 汉语, 汉语');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 12, 3), [2, 2, 2, 2, 2, 2], 'line: 汉, 语, 汉, 语, 汉, 语');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 12, 2), [2, 2, 2, 2, 2, 2], 'line: 汉, 语, 汉, 语, 汉, 语');
+    });
+    it('should return correct line lengths for a string with wide and single characters', () => {
+      const line = new BufferLine(6);
+      line.set(0, [null, 'a', 1, 'a'.charCodeAt(0)]);
+      line.set(1, [null, '汉', 2, '汉'.charCodeAt(0)]);
+      line.set(2, [null, '', 0, undefined]);
+      line.set(3, [null, '语', 2, '语'.charCodeAt(0)]);
+      line.set(4, [null, '', 0, undefined]);
+      line.set(5, [null, 'b', 1, 'b'.charCodeAt(0)]);
+      assert.equal(line.translateToString(), 'a汉语b');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 6, 5), [5, 1], 'line: a汉语b');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 6, 4), [3, 3], 'line: a汉, 语b');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 6, 3), [3, 3], 'line: a汉, 语b');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 6, 2), [1, 2, 2, 1], 'line: a, 汉, 语, b');
+    });
+    it('should return correct line lengths for a wrapped line with wide and single characters', () => {
+      const line1 = new BufferLine(6);
+      line1.set(0, [null, 'a', 1, 'a'.charCodeAt(0)]);
+      line1.set(1, [null, '汉', 2, '汉'.charCodeAt(0)]);
+      line1.set(2, [null, '', 0, undefined]);
+      line1.set(3, [null, '语', 2, '语'.charCodeAt(0)]);
+      line1.set(4, [null, '', 0, undefined]);
+      line1.set(5, [null, 'b', 1, 'b'.charCodeAt(0)]);
+      const line2 = new BufferLine(6, undefined, true);
+      line2.set(0, [null, 'a', 1, 'a'.charCodeAt(0)]);
+      line2.set(1, [null, '汉', 2, '汉'.charCodeAt(0)]);
+      line2.set(2, [null, '', 0, undefined]);
+      line2.set(3, [null, '语', 2, '语'.charCodeAt(0)]);
+      line2.set(4, [null, '', 0, undefined]);
+      line2.set(5, [null, 'b', 1, 'b'.charCodeAt(0)]);
+      assert.equal(line1.translateToString(), 'a汉语b');
+      assert.equal(line2.translateToString(), 'a汉语b');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line1, line2], 6, 5), [5, 4, 3], 'lines: a汉语, ba汉, 语b');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line1, line2], 6, 4), [3, 4, 4, 1], 'lines: a汉, 语ba, 汉语, b');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line1, line2], 6, 3), [3, 3, 3, 3], 'lines: a汉, 语b, a汉, 语b');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line1, line2], 6, 2), [1, 2, 2, 2, 2, 2, 1], 'lines: a, 汉, 语, ba, 汉, 语, b');
+    });
+    it('should work on lines ending in null space', () => {
+      const line = new BufferLine(5);
+      line.set(0, [null, '汉', 2, '汉'.charCodeAt(0)]);
+      line.set(1, [null, '', 0, undefined]);
+      line.set(2, [null, '语', 2, '语'.charCodeAt(0)]);
+      line.set(3, [null, '', 0, undefined]);
+      line.set(4, [null, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]);
+      assert.equal(line.translateToString(true), '汉语');
+      assert.equal(line.translateToString(false), '汉语 ');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 4, 3), [2, 2], 'line: 汉, 语');
+      assert.deepEqual(reflowSmallerGetNewLineLengths([line], 4, 2), [2, 2], 'line: 汉, 语');
+    });
+  });
+});

src/BufferReflow.ts

@@ -0,0 +1,206 @@
+/**
+ * Copyright (c) 2019 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+
+import { BufferLine, CellData } from './BufferLine';
+import { CircularList, IDeleteEvent } from './common/CircularList';
+import { IBufferLine } from './Types';
+import { NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE, DEFAULT_ATTR } from './Buffer';
+
+export interface INewLayoutResult {
+  layout: number[];
+  countRemoved: number;
+}
+
+/**
+ * Evaluates and returns indexes to be removed after a reflow larger occurs. Lines will be removed
+ * when a wrapped line unwraps.
+ * @param lines The buffer lines.
+ * @param newCols The columns after resize.
+ */
+export function reflowLargerGetLinesToRemove(lines: CircularList<IBufferLine>, newCols: number, bufferAbsoluteY: number): number[] {
+  const nullCell = CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]);
+  // Gather all BufferLines that need to be removed from the Buffer here so that they can be
+  // batched up and only committed once
+  const toRemove: number[] = [];
+
+  for (let y = 0; y < lines.length - 1; y++) {
+    // Check if this row is wrapped
+    let i = y;
+    let nextLine = lines.get(++i) as BufferLine;
+    if (!nextLine.isWrapped) {
+      continue;
+    }
+
+    // Check how many lines it's wrapped for
+    const wrappedLines: BufferLine[] = [lines.get(y) as BufferLine];
+    while (i < lines.length && nextLine.isWrapped) {
+      wrappedLines.push(nextLine);
+      nextLine = lines.get(++i) as BufferLine;
+    }
+
+    // If these lines contain the cursor don't touch them, the program will handle fixing up wrapped
+    // lines with the cursor
+    if (bufferAbsoluteY >= y && bufferAbsoluteY < i) {
+      y += wrappedLines.length - 1;
+      continue;
+    }
+
+    // Copy buffer data to new locations
+    let destLineIndex = 0;
+    let destCol = wrappedLines[destLineIndex].getTrimmedLength();
+    let srcLineIndex = 1;
+    let srcCol = 0;
+    while (srcLineIndex < wrappedLines.length) {
+      const srcTrimmedTineLength = wrappedLines[srcLineIndex].getTrimmedLength();
+      const srcRemainingCells = srcTrimmedTineLength - srcCol;
+      const destRemainingCells = newCols - destCol;
+      const cellsToCopy = Math.min(srcRemainingCells, destRemainingCells);
+
+      wrappedLines[destLineIndex].copyCellsFrom(wrappedLines[srcLineIndex], srcCol, destCol, cellsToCopy, false);
+
+      destCol += cellsToCopy;
+      if (destCol === newCols) {
+        destLineIndex++;
+        destCol = 0;
+      }
+      srcCol += cellsToCopy;
+      if (srcCol === srcTrimmedTineLength) {
+        srcLineIndex++;
+        srcCol = 0;
+      }
+
+      // Make sure the last cell isn't wide, if it is copy it to the current dest
+      if (destCol === 0 && destLineIndex !== 0) {
+        if (wrappedLines[destLineIndex - 1].getWidth(newCols - 1) === 2) {
+          wrappedLines[destLineIndex].copyCellsFrom(wrappedLines[destLineIndex - 1], newCols - 1, destCol++, 1, false);
+          // Null out the end of the last row
+          wrappedLines[destLineIndex - 1].setCell(newCols - 1, nullCell);
+        }
+      }
+    }
+
+    // Clear out remaining cells or fragments could remain;
+    wrappedLines[destLineIndex].replaceCells(destCol, newCols, nullCell);
+
+    // Work backwards and remove any rows at the end that only contain null cells
+    let countToRemove = 0;
+    for (let i = wrappedLines.length - 1; i > 0; i--) {
+      if (i > destLineIndex || wrappedLines[i].getTrimmedLength() === 0) {
+        countToRemove++;
+      } else {
+        break;
+      }
+    }
+
+    if (countToRemove > 0) {
+      toRemove.push(y + wrappedLines.length - countToRemove); // index
+      toRemove.push(countToRemove);
+    }
+
+    y += wrappedLines.length - 1;
+  }
+  return toRemove;
+}
+
+/**
+ * Creates and return the new layout for lines given an array of indexes to be removed.
+ * @param lines The buffer lines.
+ * @param toRemove The indexes to remove.
+ */
+export function reflowLargerCreateNewLayout(lines: CircularList<IBufferLine>, toRemove: number[]): INewLayoutResult {
+  const layout: number[] = [];
+  // First iterate through the list and get the actual indexes to use for rows
+  let nextToRemoveIndex = 0;
+  let nextToRemoveStart = toRemove[nextToRemoveIndex];
+  let countRemovedSoFar = 0;
+  for (let i = 0; i < lines.length; i++) {
+    if (nextToRemoveStart === i) {
+      const countToRemove = toRemove[++nextToRemoveIndex];
+
+      // Tell markers that there was a deletion
+      lines.emit('delete', {
+        index: i - countRemovedSoFar,
+        amount: countToRemove
+      } as IDeleteEvent);
+
+      i += countToRemove - 1;
+      countRemovedSoFar += countToRemove;
+      nextToRemoveStart = toRemove[++nextToRemoveIndex];
+    } else {
+      layout.push(i);
+    }
+  }
+  return {
+    layout,
+    countRemoved: countRemovedSoFar
+  };
+}
+
+/**
+ * Applies a new layout to the buffer. This essentially does the same as many splice calls but it's
+ * done all at once in a single iteration through the list since splice is very expensive.
+ * @param lines The buffer lines.
+ * @param newLayout The new layout to apply.
+ */
+export function reflowLargerApplyNewLayout(lines: CircularList<IBufferLine>, newLayout: number[]): void {
+  // Record original lines so they don't get overridden when we rearrange the list
+  const newLayoutLines: BufferLine[] = [];
+  for (let i = 0; i < newLayout.length; i++) {
+    newLayoutLines.push(lines.get(newLayout[i]) as BufferLine);
+  }
+
+  // Rearrange the list
+  for (let i = 0; i < newLayoutLines.length; i++) {
+    lines.set(i, newLayoutLines[i]);
+  }
+  lines.length = newLayout.length;
+}
+
+/**
+ * Gets the new line lengths for a given wrapped line. The purpose of this function it to pre-
+ * compute the wrapping points since wide characters may need to be wrapped onto the following line.
+ * This function will return an array of numbers of where each line wraps to, the resulting array
+ * will only contain the values `newCols` (when the line does not end with a wide character) and
+ * `newCols - 1` (when the line does end with a wide character), except for the last value which
+ * will contain the remaining items to fill the line.
+ *
+ * Calling this with a `newCols` value of `1` will lock up.
+ *
+ * @param wrappedLines The wrapped lines to evaluate.
+ * @param oldCols The columns before resize.
+ * @param newCols The columns after resize.
+ */
+export function reflowSmallerGetNewLineLengths(wrappedLines: BufferLine[], oldCols: number, newCols: number): number[] {
+  const newLineLengths: number[] = [];
+  const cellsNeeded = wrappedLines.map(l => l.getTrimmedLength()).reduce((p, c) => p + c);
+
+  // Use srcCol and srcLine to find the new wrapping point, use that to get the cellsAvailable and
+  // linesNeeded
+  let srcCol = 0;
+  let srcLine = 0;
+  let cellsAvailable = 0;
+  while (cellsAvailable < cellsNeeded) {
+    if (cellsNeeded - cellsAvailable < newCols) {
+      // Add the final line and exit the loop
+      newLineLengths.push(cellsNeeded - cellsAvailable);
+      break;
+    }
+    srcCol += newCols;
+    const oldTrimmedLength = wrappedLines[srcLine].getTrimmedLength();
+    if (srcCol > oldTrimmedLength) {
+      srcCol -= oldTrimmedLength;
+      srcLine++;
+    }
+    const endsWithWide = wrappedLines[srcLine].getWidth(srcCol - 1) === 2;
+    if (endsWithWide) {
+      srcCol--;
+    }
+    const lineLength = endsWithWide ? newCols - 1 : newCols;
+    newLineLengths.push(lineLength);
+    cellsAvailable += lineLength;
+  }
+
+  return newLineLengths;
+}

src/BufferSet.test.ts

@@ -15,8 +15,8 @@ describe('BufferSet', () => {
 
   beforeEach(() => {
     terminal = new MockTerminal();
-    terminal.cols = 80;
-    terminal.rows = 24;
+    (terminal as any).cols = 80;
+    (terminal as any).rows = 24;
     terminal.options.scrollback = 1000;
     bufferSet = new BufferSet(terminal);
   });

src/EscapeSequenceParser.ts

@@ -6,7 +6,7 @@
 import { ParserState, ParserAction, IParsingState, IDcsHandler, IEscapeSequenceParser } from './Types';
 import { IDisposable } from 'xterm';
 import { Disposable } from './common/Lifecycle';
-import { utf32ToString } from './common/TypedArrayUtils';
+import { utf32ToString } from './core/input/TextDecoder';
 
 interface IHandlerCollection<T> {
   [key: string]: T[];

src/InputHandler.ts

@@ -14,8 +14,8 @@ import { EscapeSequenceParser } from './EscapeSequenceParser';
 import { ICharset } from './core/Types';
 import { IDisposable } from 'xterm';
 import { Disposable } from './common/Lifecycle';
-import { concat, utf32ToString } from './common/TypedArrayUtils';
-import { StringToUtf32, stringFromCodePoint, Utf8ToUtf32 } from './core/input/TextDecoder';
+import { concat } from './common/TypedArrayUtils';
+import { StringToUtf32, stringFromCodePoint, Utf8ToUtf32, utf32ToString } from './core/input/TextDecoder';
 import { CellData } from './BufferLine';
 
 /**
@@ -382,9 +382,9 @@ export class InputHandler extends Disposable implements IInputHandler {
           // found empty cell after fullwidth, need to go 2 cells back
           // it is save to step 2 cells back here
           // since an empty cell is only set by fullwidth chars
-          bufferRow.addCharToCell(buffer.x - 2, code);
+          bufferRow.addCodepointToCell(buffer.x - 2, code);
         } else {
-          bufferRow.addCharToCell(buffer.x - 1, code);
+          bufferRow.addCodepointToCell(buffer.x - 1, code);
         }
         continue;
       }
@@ -428,12 +428,12 @@ export class InputHandler extends Disposable implements IInputHandler {
         // a halfwidth char any fullwidth shifted there is lost
         // and will be set to empty cell
         if (bufferRow.loadCell(cols - 1, this._cell).width === 2) {
-          bufferRow.setDataFromCodePoint(cols - 1, NULL_CELL_CODE, NULL_CELL_WIDTH, curAttr, 0);
+          bufferRow.setCellFromCodePoint(cols - 1, NULL_CELL_CODE, NULL_CELL_WIDTH, curAttr, 0);
         }
       }
 
       // write current char to buffer and advance cursor
-      bufferRow.setDataFromCodePoint(buffer.x++, code, chWidth, curAttr, 0);
+      bufferRow.setCellFromCodePoint(buffer.x++, code, chWidth, curAttr, 0);
 
       // fullwidth char - also set next cell to placeholder stub and advance cursor
       // for graphemes bigger than fullwidth we can simply loop to zero
@@ -441,7 +441,7 @@ export class InputHandler extends Disposable implements IInputHandler {
       if (chWidth > 0) {
         while (--chWidth) {
           // other than a regular empty cell a cell following a wide char has no width
-          bufferRow.setDataFromCodePoint(buffer.x++, 0, 0, curAttr, 0);
+          bufferRow.setCellFromCodePoint(buffer.x++, 0, 0, curAttr, 0);
         }
       }
     }

src/Linkifier.test.ts

@@ -41,8 +41,8 @@ describe('Linkifier', () => {
 
   beforeEach(() => {
     terminal = new MockTerminal();
-    terminal.cols = 100;
-    terminal.rows = 10;
+    (terminal as any).cols = 100;
+    (terminal as any).rows = 10;
     terminal.buffer = new MockBuffer();
     (<MockBuffer>terminal.buffer).setLines(new CircularList<IBufferLine>(20));
     terminal.buffer.ydisp = 0;
@@ -65,7 +65,7 @@ describe('Linkifier', () => {
   function assertLinkifiesRow(rowText: string, linkMatcherRegex: RegExp, links: {x: number, length: number}[], done: MochaDone): void {
     addRow(rowText);
     linkifier.registerLinkMatcher(linkMatcherRegex, () => {});
-    terminal.rows = terminal.buffer.lines.length - 1;
+    (terminal as any).rows = terminal.buffer.lines.length - 1;
     linkifier.linkifyRows();
     // Allow linkify to happen
     setTimeout(() => {
@@ -142,19 +142,19 @@ describe('Linkifier', () => {
       });
       describe('multi-line links', () => {
         it('should match links that start on line 1/2 of a wrapped line and end on the last character of line 1/2', done => {
-          terminal.cols = 4;
+          (terminal as any).cols = 4;
           assertLinkifiesMultiLineLink('12345', /1234/, [{x1: 0, x2: 4, y1: 0, y2: 0}], done);
         });
         it('should match links that start on line 1/2 of a wrapped line and wrap to line 2/2', done => {
-          terminal.cols = 4;
+          (terminal as any).cols = 4;
           assertLinkifiesMultiLineLink('12345', /12345/, [{x1: 0, x2: 1, y1: 0, y2: 1}], done);
         });
         it('should match links that start and end on line 2/2 of a wrapped line', done => {
-          terminal.cols = 4;
+          (terminal as any).cols = 4;
           assertLinkifiesMultiLineLink('12345678', /5678/, [{x1: 0, x2: 4, y1: 1, y2: 1}], done);
         });
         it('should match links that start on line 2/3 of a wrapped line and wrap to line 3/3', done => {
-          terminal.cols = 4;
+          (terminal as any).cols = 4;
           assertLinkifiesMultiLineLink('123456789', /56789/, [{x1: 0, x2: 1, y1: 1, y2: 2}], done);
         });
       });

src/SelectionManager.test.ts

@@ -45,8 +45,8 @@ describe('SelectionManager', () => {
 
   beforeEach(() => {
     terminal = new TestMockTerminal();
-    terminal.cols = 80;
-    terminal.rows = 2;
+    (terminal as any).cols = 80;
+    (terminal as any).rows = 2;
     terminal.options.scrollback = 100;
     terminal.buffers = new BufferSet(terminal);
     terminal.buffer = terminal.buffers.active;

src/SelectionModel.test.ts

@@ -23,8 +23,8 @@ describe('SelectionManager', () => {
 
   beforeEach(() => {
     terminal = new MockTerminal();
-    terminal.cols = 80;
-    terminal.rows = 2;
+    (terminal as any).cols = 80;
+    (terminal as any).rows = 2;
     terminal.options.scrollback = 10;
     terminal.buffers = new BufferSet(terminal);
     terminal.buffer = terminal.buffers.active;

src/Terminal.ts

@@ -69,6 +69,9 @@ const WRITE_BUFFER_PAUSE_THRESHOLD = 5;
  */
 const WRITE_BATCH_SIZE = 300;
 
+const MINIMUM_COLS = 2; // Less than 2 can mess with wide chars
+const MINIMUM_ROWS = 1;
+
 /**
  * The set of options that only have an effect when set in the Terminal constructor.
  */
@@ -262,8 +265,8 @@ export class Terminal extends EventEmitter implements ITerminal, IDisposable, II
     // TODO: WHy not document.body?
     this._parent = document ? document.body : null;
 
-    this.cols = this.options.cols;
-    this.rows = this.options.rows;
+    this.cols = Math.max(this.options.cols, MINIMUM_COLS);
+    this.rows = Math.max(this.options.rows, MINIMUM_ROWS);
 
     if (this.options.handler) {
       this.on('data', this.options.handler);
@@ -1710,8 +1713,8 @@ export class Terminal extends EventEmitter implements ITerminal, IDisposable, II
       return;
     }
 
-    if (x < 1) x = 1;
-    if (y < 1) y = 1;
+    if (x < MINIMUM_COLS) x = MINIMUM_COLS;
+    if (y < MINIMUM_ROWS) y = MINIMUM_ROWS;
 
     this.buffers.resize(x, y);
 

src/Types.ts

@@ -238,7 +238,7 @@ export interface IBufferAccessor {
 }
 
 export interface IElementAccessor {
-  element: HTMLElement;
+  readonly element: HTMLElement;
 }
 
 export interface ILinkifierAccessor {
@@ -455,14 +455,22 @@ export interface IParsingState {
 * DCS handler signature for EscapeSequenceParser.
 * EscapeSequenceParser handles DCS commands via separate
 * subparsers that get hook/unhooked and can handle
-* arbitrary amount of print data.
+* arbitrary amount of data.
+*
 * On entering a DSC sequence `hook` is called by
 * `EscapeSequenceParser`. Use it to initialize or reset
 * states needed to handle the current DCS sequence.
+* Note: A DCS parser is only instantiated once, therefore
+* you cannot rely on the ctor to reinitialize state.
+*
 * EscapeSequenceParser will call `put` several times if the
-* parsed string got splitted, therefore you might have to collect
-* `data` until `unhook` is called. `unhook` marks the end
-* of the current DCS sequence.
+* parsed data got split, therefore you might have to collect
+* `data` until `unhook` is called.
+* Note: `data` is borrowed, if you cannot process the data
+* in chunks you have to copy it, doing otherwise will lead to
+* data losses or corruption.
+*
+* `unhook` marks the end of the current DCS sequence.
 */
 export interface IDcsHandler {
   hook(collect: string, params: number[], flag: number): void;
@@ -520,7 +528,7 @@ export interface ICellData {
   fg: number;
   bg: number;
   combinedData: string;
-  combined: number;
+  isCombined: number;
   width: number;
   chars: string;
   code: number;
@@ -538,12 +546,12 @@ export interface IBufferLine {
   set(index: number, value: CharData): void;
   loadCell(index: number, cell: ICellData): ICellData;
   setCell(index: number, cell: ICellData): void;
-  setDataFromCodePoint(index: number, codePoint: number, width: number, fg: number, bg: number): void;
-  addCharToCell(index: number, codePoint: number): void;
+  setCellFromCodePoint(index: number, codePoint: number, width: number, fg: number, bg: number): void;
+  addCodepointToCell(index: number, codePoint: number): void;
   insertCells(pos: number, n: number, ch: ICellData): void;
   deleteCells(pos: number, n: number, fill: ICellData): void;
   replaceCells(start: number, end: number, fill: ICellData): void;
-  resize(cols: number, fill: ICellData, shrink?: boolean): void;
+  resize(cols: number, fill: ICellData): void;
   fill(fillCellData: ICellData): void;
   copyFrom(line: IBufferLine): void;
   clone(): IBufferLine;

src/common/CircularList.ts

@@ -6,6 +6,16 @@
 import { EventEmitter } from './EventEmitter';
 import { ICircularList } from './Types';
 
+export interface IInsertEvent {
+  index: number;
+  amount: number;
+}
+
+export interface IDeleteEvent {
+  index: number;
+  amount: number;
+}
+
 /**
  * Represents a circular list; a list with a maximum size that wraps around when push is called,
  * overriding values at the start of the list.
@@ -91,7 +101,7 @@ export class CircularList<T> extends EventEmitter implements ICircularList<T> {
     this._array[this._getCyclicIndex(this._length)] = value;
     if (this._length === this._maxLength) {
       this._startIndex = ++this._startIndex % this._maxLength;
-      this.emit('trim', 1);
+      this.emitMayRemoveListeners('trim', 1);
     } else {
       this._length++;
     }
@@ -107,7 +117,7 @@ export class CircularList<T> extends EventEmitter implements ICircularList<T> {
       throw new Error('Can only recycle when the buffer is full');
     }
     this._startIndex = ++this._startIndex % this._maxLength;
-    this.emit('trim', 1);
+    this.emitMayRemoveListeners('trim', 1);
     return this._array[this._getCyclicIndex(this._length - 1)]!;
   }
 
@@ -144,24 +154,22 @@ export class CircularList<T> extends EventEmitter implements ICircularList<T> {
       this._length -= deleteCount;
     }
 
-    if (items && items.length) {
-      // Add items
-      for (let i = this._length - 1; i >= start; i--) {
-        this._array[this._getCyclicIndex(i + items.length)] = this._array[this._getCyclicIndex(i)];
-      }
-      for (let i = 0; i < items.length; i++) {
-        this._array[this._getCyclicIndex(start + i)] = items[i];
-      }
+    // Add items
+    for (let i = this._length - 1; i >= start; i--) {
+      this._array[this._getCyclicIndex(i + items.length)] = this._array[this._getCyclicIndex(i)];
+    }
+    for (let i = 0; i < items.length; i++) {
+      this._array[this._getCyclicIndex(start + i)] = items[i];
+    }
 
-      // Adjust length as needed
-      if (this._length + items.length > this._maxLength) {
-        const countToTrim = (this._length + items.length) - this._maxLength;
-        this._startIndex += countToTrim;
-        this._length = this._maxLength;
-        this.emit('trim', countToTrim);
-      } else {
-        this._length += items.length;
-      }
+    // Adjust length as needed
+    if (this._length + items.length > this._maxLength) {
+      const countToTrim = (this._length + items.length) - this._maxLength;
+      this._startIndex += countToTrim;
+      this._length = this._maxLength;
+      this.emitMayRemoveListeners('trim', countToTrim);
+    } else {
+      this._length += items.length;
     }
   }
 
@@ -175,7 +183,7 @@ export class CircularList<T> extends EventEmitter implements ICircularList<T> {
     }
     this._startIndex += count;
     this._length -= count;
-    this.emit('trim', count);
+    this.emitMayRemoveListeners('trim', count);
   }
 
   public shiftElements(start: number, count: number, offset: number): void {
@@ -199,7 +207,7 @@ export class CircularList<T> extends EventEmitter implements ICircularList<T> {
         while (this._length > this._maxLength) {
           this._length--;
           this._startIndex++;
-          this.emit('trim', 1);
+          this.emitMayRemoveListeners('trim', 1);
         }
       }
     } else {

src/common/EventEmitter.ts

@@ -75,6 +75,19 @@ export class EventEmitter extends Disposable implements IEventEmitter, IDisposab
     }
   }
 
+  public emitMayRemoveListeners(type: string, ...args: any[]): void {
+    if (!this._events[type]) {
+      return;
+    }
+    const obj = this._events[type];
+    let length = obj.length;
+    for (let i = 0; i < obj.length; i++) {
+      obj[i].apply(this, args);
+      i -= length - obj.length;
+      length = obj.length;
+    }
+  }
+
   public listeners(type: string): XtermListener[] {
     return this._events[type] || [];
   }

src/common/TypedArrayUtils.test.ts

@@ -3,7 +3,7 @@
  * @license MIT
  */
 import { assert } from 'chai';
-import { fillFallback, concat, utf32ToString } from './TypedArrayUtils';
+import { fillFallback, concat } from './TypedArrayUtils';
 
 type TypedArray = Uint8Array | Uint16Array | Uint32Array | Uint8ClampedArray
   | Int8Array | Int16Array | Int32Array
@@ -94,12 +94,4 @@ describe('typed array convenience functions', () => {
     const merged = concat(a, b);
     deepEquals(merged, new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]));
   });
-  it('utf16ToString', () => {
-    const s = 'abcdefg';
-    const data = new Uint16Array(s.length);
-    for (let i = 0; i < s.length; ++i) {
-      data[i] = s.charCodeAt(i);
-    }
-    assert.equal(utf32ToString(data), s);
-  });
 });

src/common/TypedArrayUtils.ts

@@ -3,10 +3,11 @@
  * @license MIT
  */
 
-type TypedArray = Uint8Array | Uint16Array | Uint32Array | Uint8ClampedArray
+export type TypedArray = Uint8Array | Uint16Array | Uint32Array | Uint8ClampedArray
   | Int8Array | Int16Array | Int32Array
   | Float32Array | Float64Array;
 
+
 /**
  * polyfill for TypedArray.fill
  * This is needed to support .fill in all safari versions and IE 11.
@@ -49,20 +50,3 @@ export function concat<T extends TypedArray>(a: T, b: T): T {
   result.set(b, a.length);
   return result;
 }
-
-/**
- * Convert UTF32 char codes into JS string.
- */
-export function utf32ToString<T extends TypedArray>(data: T, start: number = 0, end: number = data.length): string {
-  let result = '';
-  let cp;
-  for (let i = start; i < end; ++i) {
-    if ((cp = data[i]) > 0xFFFF) {
-      cp -= 0x10000;
-      result += String.fromCharCode((cp >> 10) + 0xD800) + String.fromCharCode((cp % 0x400) + 0xDC00);
-    } else {
-      result += String.fromCharCode(cp);
-    }
-  }
-  return result;
-}