Merge pull request #1796 from jerch/typedarray_parser

support typed array in parser

src/EscapeSequenceParser.ts

@@ -6,6 +6,7 @@
 import { ParserState, ParserAction, IParsingState, IDcsHandler, IEscapeSequenceParser } from './Types';
 import { IDisposable } from 'xterm';
 import { Disposable } from './common/Lifecycle';
+import { utf32ToString } from './core/input/TextDecoder';
 
 interface IHandlerCollection<T> {
   [key: string]: T[];
@@ -134,6 +135,7 @@ export const VT500_TRANSITION_TABLE = (function (): TransitionTable {
   table.addMany(PRINTABLES, ParserState.SOS_PM_APC_STRING, ParserAction.IGNORE, ParserState.SOS_PM_APC_STRING);
   table.addMany(EXECUTABLES, ParserState.SOS_PM_APC_STRING, ParserAction.IGNORE, ParserState.SOS_PM_APC_STRING);
   table.add(0x9c, ParserState.SOS_PM_APC_STRING, ParserAction.IGNORE, ParserState.GROUND);
+  table.add(0x7f, ParserState.SOS_PM_APC_STRING, ParserAction.IGNORE, ParserState.SOS_PM_APC_STRING);
   // csi entries
   table.add(0x5b, ParserState.ESCAPE, ParserAction.CLEAR, ParserState.CSI_ENTRY);
   table.addMany(r(0x40, 0x7f), ParserState.CSI_ENTRY, ParserAction.CSI_DISPATCH, ParserState.GROUND);
@@ -202,7 +204,7 @@ export const VT500_TRANSITION_TABLE = (function (): TransitionTable {
  */
 class DcsDummy implements IDcsHandler {
   hook(collect: string, params: number[], flag: number): void { }
-  put(data: string, start: number, end: number): void { }
+  put(data: Uint32Array, start: number, end: number): void { }
   unhook(): void { }
 }
 
@@ -228,7 +230,7 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
   protected _collect: string;
 
   // handler lookup containers
-  protected _printHandler: (data: string, start: number, end: number) => void;
+  protected _printHandler: (data: Uint32Array, start: number, end: number) => void;
   protected _executeHandlers: any;
   protected _csiHandlers: IHandlerCollection<CsiHandler>;
   protected _escHandlers: any;
@@ -238,7 +240,7 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
   protected _errorHandler: (state: IParsingState) => IParsingState;
 
   // fallback handlers
-  protected _printHandlerFb: (data: string, start: number, end: number) => void;
+  protected _printHandlerFb: (data: Uint32Array, start: number, end: number) => void;
   protected _executeHandlerFb: (code: number) => void;
   protected _csiHandlerFb: (collect: string, params: number[], flag: number) => void;
   protected _escHandlerFb: (collect: string, flag: number) => void;
@@ -294,7 +296,7 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
     this._errorHandler = null;
   }
 
-  setPrintHandler(callback: (data: string, start: number, end: number) => void): void {
+  setPrintHandler(callback: (data: Uint32Array, start: number, end: number) => void): void {
     this._printHandler = callback;
   }
   clearPrintHandler(): void {
@@ -397,7 +399,7 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
     this._activeDcsHandler = null;
   }
 
-  parse(data: string): void {
+  parse(data: Uint32Array, length: number): void {
     let code = 0;
     let transition = 0;
     let error = false;
@@ -412,15 +414,14 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
     let callback: Function | null = null;
 
     // process input string
-    const l = data.length;
-    for (let i = 0; i < l; ++i) {
-      code = data.charCodeAt(i);
+    for (let i = 0; i < length; ++i) {
+      code = data[i];
 
       // shortcut for most chars (print action)
       if (currentState === ParserState.GROUND && code > 0x1f && code < 0x80) {
         print = (~print) ? print : i;
         do i++;
-        while (i < l && data.charCodeAt(i) > 0x1f && data.charCodeAt(i) < 0x80);
+        while (i < length && data[i] > 0x1f && data[i] < 0x80);
         i--;
         continue;
       }
@@ -563,10 +564,10 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
           break;
         case ParserAction.OSC_PUT:
           for (let j = i + 1; ; j++) {
-            if (j >= l
-                || (code = data.charCodeAt(j)) < 0x20
+            if (j >= length
+                || (code = data[j]) < 0x20
                 || (code > 0x7f && code <= 0x9f)) {
-              osc += data.substring(i, j);
+              osc += utf32ToString(data, i, j);
               i = j - 1;
               break;
             }
@@ -610,9 +611,9 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP
 
     // push leftover pushable buffers to terminal
     if (currentState === ParserState.GROUND && ~print) {
-      this._printHandler(data, print, data.length);
+      this._printHandler(data, print, length);
     } else if (currentState === ParserState.DCS_PASSTHROUGH && ~dcs && dcsHandler) {
-      dcsHandler.put(data, dcs, data.length);
+      dcsHandler.put(data, dcs, length);
     }
 
     // save non pushable buffers

src/InputHandler.test.ts

@@ -337,7 +337,9 @@ describe('InputHandler', () => {
     it('should not cause an infinite loop (regression test)', () => {
       const term = new Terminal();
       const inputHandler = new InputHandler(term);
-      inputHandler.print(String.fromCharCode(0x200B), 0, 1);
+      const container = new Uint32Array(10);
+      container[0] = 0x200B;
+      inputHandler.print(container, 0, 1);
     });
   });
 

src/InputHandler.ts

@@ -14,6 +14,8 @@ import { EscapeSequenceParser } from './EscapeSequenceParser';
 import { ICharset } from './core/Types';
 import { IDisposable } from 'xterm';
 import { Disposable } from './common/Lifecycle';
+import { concat } from './common/TypedArrayUtils';
+import { StringToUtf32, stringFromCodePoint, utf32ToString } from './core/input/TextDecoder';
 
 /**
  * Map collect to glevel. Used in `selectCharset`.
@@ -32,21 +34,22 @@ const GLEVEL: {[key: string]: number} = {'(': 0, ')': 1, '*': 2, '+': 3, '-': 1,
  *   Response: DECRPSS (https://vt100.net/docs/vt510-rm/DECRPSS.html)
  */
 class DECRQSS implements IDcsHandler {
-  private _data: string;
+  private _data: Uint32Array = new Uint32Array(0);
 
   constructor(private _terminal: any) { }
 
   hook(collect: string, params: number[], flag: number): void {
-    // reset data
-    this._data = '';
+    this._data = new Uint32Array(0);
   }
 
-  put(data: string, start: number, end: number): void {
-    this._data += data.substring(start, end);
+  put(data: Uint32Array, start: number, end: number): void {
+    this._data = concat(this._data, data.subarray(start, end));
   }
 
   unhook(): void {
-    switch (this._data) {
+    const data = utf32ToString(this._data);
+    this._data = new Uint32Array(0);
+    switch (data) {
       // valid: DCS 1 $ r Pt ST (xterm)
       case '"q': // DECSCA
         return this._terminal.handler(`${C0.ESC}P1$r0"q${C0.ESC}\\`);
@@ -66,7 +69,7 @@ class DECRQSS implements IDcsHandler {
         return this._terminal.handler(`${C0.ESC}P1$r${style} q${C0.ESC}\\`);
       default:
         // invalid: DCS 0 $ r Pt ST (xterm)
-        this._terminal.error('Unknown DCS $q %s', this._data);
+        this._terminal.error('Unknown DCS $q %s', data);
         this._terminal.handler(`${C0.ESC}P0$r${C0.ESC}\\`);
     }
   }
@@ -78,11 +81,17 @@ class DECRQSS implements IDcsHandler {
  *   not supported
  */
 
- /**
-  * DCS + p Pt ST (xterm)
-  *   Set Terminfo Data
-  *   not supported
-  */
+/**
+ * DCS + q Pt ST (xterm)
+ *   Request Terminfo String
+ *   not implemented
+ */
+
+/**
+ * DCS + p Pt ST (xterm)
+ *   Set Terminfo Data
+ *   not supported
+ */
 
 
 
@@ -94,7 +103,8 @@ class DECRQSS implements IDcsHandler {
  * each function's header comment.
  */
 export class InputHandler extends Disposable implements IInputHandler {
-  private _surrogateFirst: string;
+  private _parseBuffer: Uint32Array = new Uint32Array(4096);
+  private _stringDecoder: StringToUtf32 = new StringToUtf32();
 
   constructor(
       protected _terminal: IInputHandlingTerminal,
@@ -104,8 +114,6 @@ export class InputHandler extends Disposable implements IInputHandler {
 
     this.register(this._parser);
 
-    this._surrogateFirst = '';
-
     /**
      * custom fallback handlers
      */
@@ -290,13 +298,13 @@ export class InputHandler extends Disposable implements IInputHandler {
       this._terminal.log('data: ' + data);
     }
 
-    // apply leftover surrogate high from last write
-    if (this._surrogateFirst) {
-      data = this._surrogateFirst + data;
-      this._surrogateFirst = '';
+    if (this._parseBuffer.length < data.length) {
+      this._parseBuffer = new Uint32Array(data.length);
     }
-
-    this._parser.parse(data);
+    for (let i = 0; i < data.length; ++i) {
+      this._parseBuffer[i] = data.charCodeAt(i);
+    }
+    this._parser.parse(this._parseBuffer, this._stringDecoder.decode(data, this._parseBuffer));
 
     buffer = this._terminal.buffer;
     if (buffer.x !== cursorStartX || buffer.y !== cursorStartY) {
@@ -304,9 +312,9 @@ export class InputHandler extends Disposable implements IInputHandler {
     }
   }
 
-  public print(data: string, start: number, end: number): void {
-    let char: string;
+  public print(data: Uint32Array, start: number, end: number): void {
     let code: number;
+    let char: string;
     let chWidth: number;
     const buffer: IBuffer = this._terminal.buffer;
     const charset: ICharset = this._terminal.charset;
@@ -318,41 +326,23 @@ export class InputHandler extends Disposable implements IInputHandler {
     let bufferRow = buffer.lines.get(buffer.y + buffer.ybase);
 
     this._terminal.updateRange(buffer.y);
-    for (let stringPosition = start; stringPosition < end; ++stringPosition) {
-      char = data.charAt(stringPosition);
-      code = data.charCodeAt(stringPosition);
-
-      // surrogate pair handling
-      if (0xD800 <= code && code <= 0xDBFF) {
-        if (++stringPosition >= end) {
-          // end of input:
-          // handle pairs as true UTF-16 and wait for the second part
-          // since we expect the input comming from a stream there is
-          // a small chance that the surrogate pair got split
-          // therefore we dont process the first char here, instead
-          // it gets added as first char to the next processed chunk
-          this._surrogateFirst = char;
-          continue;
-        }
-        const second = data.charCodeAt(stringPosition);
-        // if the second part is in surrogate pair range create the high codepoint
-        // otherwise fall back to UCS-2 behavior (handle codepoints independently)
-        if (0xDC00 <= second && second <= 0xDFFF) {
-          code = (code - 0xD800) * 0x400 + second - 0xDC00 + 0x10000;
-          char += data.charAt(stringPosition);
-        } else {
-          stringPosition--;
-        }
-      }
+    for (let pos = start; pos < end; ++pos) {
+      code = data[pos];
+      char = stringFromCodePoint(code);
 
       // calculate print space
       // expensive call, therefore we save width in line buffer
       chWidth = wcwidth(code);
 
       // get charset replacement character
-      if (charset) {
-        char = charset[char] || char;
-        code = char.charCodeAt(0);
+      // charset are only defined for ASCII, therefore we only
+      // search for an replacement char if code < 127
+      if (code < 127 && charset) {
+        const ch = charset[char];
+        if (ch) {
+          code = ch.charCodeAt(0);
+          char = ch;
+        }
       }
 
       if (screenReaderMode) {

src/Types.ts

@@ -111,7 +111,7 @@ export interface ICompositionHelper {
  */
 export interface IInputHandler {
   parse(data: string): void;
-  print(data: string, start: number, end: number): void;
+  print(data: Uint32Array, start: number, end: number): void;
 
   /** C0 BEL */ bell(): void;
   /** C0 LF */ lineFeed(): void;
@@ -452,18 +452,26 @@ export interface IParsingState {
 * DCS handler signature for EscapeSequenceParser.
 * EscapeSequenceParser handles DCS commands via separate
 * subparsers that get hook/unhooked and can handle
-* arbitrary amount of print data.
+* arbitrary amount of data.
+*
 * On entering a DSC sequence `hook` is called by
 * `EscapeSequenceParser`. Use it to initialize or reset
 * states needed to handle the current DCS sequence.
+* Note: A DCS parser is only instantiated once, therefore
+* you cannot rely on the ctor to reinitialize state.
+*
 * EscapeSequenceParser will call `put` several times if the
-* parsed string got splitted, therefore you might have to collect
-* `data` until `unhook` is called. `unhook` marks the end
-* of the current DCS sequence.
+* parsed data got split, therefore you might have to collect
+* `data` until `unhook` is called.
+* Note: `data` is borrowed, if you cannot process the data
+* in chunks you have to copy it, doing otherwise will lead to
+* data losses or corruption.
+*
+* `unhook` marks the end of the current DCS sequence.
 */
 export interface IDcsHandler {
   hook(collect: string, params: number[], flag: number): void;
-  put(data: string, start: number, end: number): void;
+  put(data: Uint32Array, start: number, end: number): void;
   unhook(): void;
 }
 
@@ -480,9 +488,9 @@ export interface IEscapeSequenceParser extends IDisposable {
    * Parse string `data`.
    * @param data The data to parse.
    */
-  parse(data: string): void;
+  parse(data: Uint32Array, length: number): void;
 
-  setPrintHandler(callback: (data: string, start: number, end: number) => void): void;
+  setPrintHandler(callback: (data: Uint32Array, start: number, end: number) => void): void;
   clearPrintHandler(): void;
 
   setExecuteHandler(flag: string, callback: () => void): void;

src/common/TypedArrayUtils.test.ts

@@ -3,21 +3,20 @@
  * @license MIT
  */
 import { assert } from 'chai';
-import { fillFallback } from './TypedArrayUtils';
+import { fillFallback, concat } from './TypedArrayUtils';
 
 type TypedArray = Uint8Array | Uint16Array | Uint32Array | Uint8ClampedArray
   | Int8Array | Int16Array | Int32Array
   | Float32Array | Float64Array;
 
-describe('polyfill conformance tests', function(): void {
-
-  function deepEquals(a: TypedArray, b: TypedArray): void {
-    assert.equal(a.length, b.length);
-    for (let i = 0; i < a.length; ++i) {
-      assert.equal(a[i], b[i]);
-    }
+function deepEquals(a: TypedArray, b: TypedArray): void {
+  assert.equal(a.length, b.length);
+  for (let i = 0; i < a.length; ++i) {
+    assert.equal(a[i], b[i]);
   }
+}
 
+describe('polyfill conformance tests', function(): void {
   describe('TypedArray.fill', function(): void {
     it('should work with all typed array types', function(): void {
       const u81 = new Uint8Array(5);
@@ -87,3 +86,12 @@ describe('polyfill conformance tests', function(): void {
     });
   });
 });
+
+describe('typed array convenience functions', () => {
+  it('concat', () => {
+    const a = new Uint8Array([1, 2, 3, 4, 5]);
+    const b = new Uint8Array([6, 7, 8, 9, 0]);
+    const merged = concat(a, b);
+    deepEquals(merged, new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]));
+  });
+});

src/common/TypedArrayUtils.ts

@@ -3,15 +3,15 @@
  * @license MIT
  */
 
+export type TypedArray = Uint8Array | Uint16Array | Uint32Array | Uint8ClampedArray
+  | Int8Array | Int16Array | Int32Array
+  | Float32Array | Float64Array;
+
+
 /**
  * polyfill for TypedArray.fill
  * This is needed to support .fill in all safari versions and IE 11.
  */
-
-type TypedArray = Uint8Array | Uint16Array | Uint32Array | Uint8ClampedArray
-  | Int8Array | Int16Array | Int32Array
-  | Float32Array | Float64Array;
-
 export function fill<T extends TypedArray>(array: T, value: number, start?: number, end?: number): T {
   // all modern engines that support .fill
   if (array.fill) {
@@ -39,3 +39,14 @@ export function fillFallback<T extends TypedArray>(array: T, value: number, star
   }
   return array;
 }
+
+/**
+ * Concat two typed arrays `a` and `b`.
+ * Returns a new typed array.
+ */
+export function concat<T extends TypedArray>(a: T, b: T): T {
+  const result = new (a.constructor as any)(a.length + b.length);
+  result.set(a);
+  result.set(b, a.length);
+  return result;
+}

src/core/input/TextDecoder.test.ts

@@ -0,0 +1,67 @@
+/**
+ * Copyright (c) 2019 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+
+import { assert } from 'chai';
+import { StringToUtf32, stringFromCodePoint, utf32ToString } from './TextDecoder';
+
+describe('text encodings', () => {
+  it('stringFromCodePoint/utf32ToString', () => {
+    const s = 'abcdefg';
+    const data = new Uint32Array(s.length);
+    for (let i = 0; i < s.length; ++i) {
+      data[i] = s.charCodeAt(i);
+      assert.equal(stringFromCodePoint(data[i]), s[i]);
+    }
+    assert.equal(utf32ToString(data), s);
+  });
+
+  describe('StringToUtf32 Decoder', () => {
+    describe('full codepoint test', () => {
+      it('0..65535', () => {
+        const decoder = new StringToUtf32();
+        const target = new Uint32Array(5);
+        for (let i = 0; i < 65536; ++i) {
+          // skip surrogate pairs
+          if (i >= 0xD800 && i <= 0xDFFF) {
+            continue;
+          }
+          const length = decoder.decode(String.fromCharCode(i), target);
+          assert.equal(length, 1);
+          assert.equal(target[0], i);
+          assert.equal(utf32ToString(target, 0, length), String.fromCharCode(i));
+          decoder.clear();
+        }
+      });
+      it('65536..0x10FFFF (surrogates)', function(): void {
+        this.timeout(20000);
+        const decoder = new StringToUtf32();
+        const target = new Uint32Array(5);
+        for (let i = 65536; i < 0x10FFFF; ++i) {
+          const codePoint = i - 0x10000;
+          const s = String.fromCharCode((codePoint >> 10) + 0xD800) + String.fromCharCode((codePoint % 0x400) + 0xDC00);
+          const length = decoder.decode(s, target);
+          assert.equal(length, 1);
+          assert.equal(target[0], i);
+          assert.equal(utf32ToString(target, 0, length), s);
+          decoder.clear();
+        }
+      });
+    });
+
+    describe('stream handling', () => {
+      it('surrogates mixed advance by 1', () => {
+        const decoder = new StringToUtf32();
+        const target = new Uint32Array(5);
+        const input = 'Ä€𝄞Ö𝄞€Ü𝄞€';
+        let decoded = '';
+        for (let i = 0; i < input.length; ++i) {
+          const written = decoder.decode(input[i], target);
+          decoded += utf32ToString(target, written);
+        }
+        assert(decoded, 'Ä€𝄞Ö𝄞€Ü𝄞€');
+      });
+    });
+  });
+});

src/core/input/TextDecoder.ts

@@ -0,0 +1,109 @@
+/**
+ * Copyright (c) 2019 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+
+/**
+ * StringToUtf32 - decodes UTF16 sequences into UTF32 codepoints.
+ * To keep the decoder in line with JS strings it handles single surrogates as UCS2.
+ */
+export class StringToUtf32 {
+  private _interim: number = 0;
+
+  /**
+   * Clears interim and resets decoder to clean state.
+   */
+  public clear(): void {
+    this._interim = 0;
+  }
+
+  /**
+   * Decode JS string to UTF32 codepoints.
+   * The methods assumes stream input and will store partly transmitted
+   * surrogate pairs and decode them with the next data chunk.
+   * Note: The method does no bound checks for target, therefore make sure
+   * the provided input data does not exceed the size of `target`.
+   * Returns the number of written codepoints in `target`.
+   */
+  decode(input: string, target: Uint32Array): number {
+    const length = input.length;
+
+    if (!length) {
+      return 0;
+    }
+
+    let size = 0;
+    let startPos = 0;
+
+    // handle leftover surrogate high
+    if (this._interim) {
+      const second = input.charCodeAt(startPos++);
+      if (0xDC00 <= second && second <= 0xDFFF) {
+        target[size++] = (this._interim - 0xD800) * 0x400 + second - 0xDC00 + 0x10000;
+      } else {
+        // illegal codepoint (USC2 handling)
+        target[size++] = this._interim;
+        target[size++] = second;
+      }
+      this._interim = 0;
+    }
+
+    for (let i = startPos; i < length; ++i) {
+      const code = input.charCodeAt(i);
+      // surrogate pair first
+      if (0xD800 <= code && code <= 0xDBFF) {
+        if (++i >= length) {
+          this._interim = code;
+          return size;
+        }
+        const second = input.charCodeAt(i);
+        if (0xDC00 <= second && second <= 0xDFFF) {
+          target[size++] = (code - 0xD800) * 0x400 + second - 0xDC00 + 0x10000;
+        } else {
+          // illegal codepoint (USC2 handling)
+          target[size++] = code;
+          target[size++] = second;
+        }
+        continue;
+      }
+      target[size++] = code;
+    }
+    return size;
+  }
+}
+
+/**
+ * Convert UTF32 codepoint into JS string.
+ */
+export function stringFromCodePoint(codePoint: number): string {
+  if (codePoint > 0xFFFF) {
+    // UTF32 to UTF16 conversion (see comments in utf32ToString)
+    codePoint -= 0x10000;
+    return String.fromCharCode((codePoint >> 10) + 0xD800) + String.fromCharCode((codePoint % 0x400) + 0xDC00);
+  }
+  return String.fromCharCode(codePoint);
+}
+
+/**
+ * Convert UTF32 char codes into JS string.
+ * Basically the same as `stringFromCodePoint` but for multiple codepoints
+ * in a loop (which is a lot faster).
+ */
+export function utf32ToString(data: Uint32Array, start: number = 0, end: number = data.length): string {
+  let result = '';
+  for (let i = start; i < end; ++i) {
+    let codepoint = data[i];
+    if (codepoint > 0xFFFF) {
+      // JS string are encoded as UTF16, thus a non BMP codepoint gets converted into a surrogate pair
+      // conversion rules:
+      //  - subtract 0x10000 from code point, leaving a 20 bit number
+      //  - add high 10 bits to 0xD800  --> first surrogate
+      //  - add low 10 bits to 0xDC00   --> second surrogate
+      codepoint -= 0x10000;
+      result += String.fromCharCode((codepoint >> 10) + 0xD800) + String.fromCharCode((codepoint % 0x400) + 0xDC00);
+    } else {
+      result += String.fromCharCode(codepoint);
+    }
+  }
+  return result;
+}