From 8fc66a39e32c6465ad5e49f492f0f5530b72983c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= Date: Thu, 3 Jan 2019 00:48:00 +0100 Subject: [PATCH] switch parse buffer to UTF32 --- src/EscapeSequenceParser.test.ts | 27 ++++++------ src/EscapeSequenceParser.ts | 14 +++---- src/InputHandler.test.ts | 2 +- src/InputHandler.ts | 66 ++++++++---------------------- src/Types.ts | 8 ++-- src/common/TypedArrayUtils.test.ts | 4 +- src/common/TypedArrayUtils.ts | 15 +++++-- 7 files changed, 57 insertions(+), 79 deletions(-) diff --git a/src/EscapeSequenceParser.test.ts b/src/EscapeSequenceParser.test.ts index 3c9db585..135cc393 100644 --- a/src/EscapeSequenceParser.test.ts +++ b/src/EscapeSequenceParser.test.ts @@ -6,6 +6,7 @@ import { ParserState, IDcsHandler, IParsingState } from './Types'; import { EscapeSequenceParser, TransitionTable, VT500_TRANSITION_TABLE } from './EscapeSequenceParser'; import * as chai from 'chai'; +import { StringToUtf32, stringFromCodePoint } from './core/input/TextDecoder'; function r(a: number, b: number): string[] { let c = b - a; @@ -50,10 +51,10 @@ const testTerminal: any = { compare: function (value: any): void { chai.expect(this.calls.slice()).eql(value); // weird bug w'o slicing here }, - print: function (data: Uint16Array, start: number, end: number): void { + print: function (data: Uint32Array, start: number, end: number): void { let s = ''; for (let i = start; i < end; ++i) { - s += String.fromCharCode(data[i]); + s += stringFromCodePoint(data[i]); } this.calls.push(['print', s]); }, @@ -72,10 +73,10 @@ const testTerminal: any = { actionDCSHook: function (collect: string, params: number[], flag: string): void { this.calls.push(['dcs hook', collect, params, flag]); }, - actionDCSPrint: function (data: Uint16Array, start: number, end: number): void { + actionDCSPrint: function (data: Uint32Array, start: number, end: number): void { let s = ''; for (let i = start; i < end; ++i) { - s += String.fromCharCode(data[i]); + s += stringFromCodePoint(data[i]); } this.calls.push(['dcs put', s]); }, @@ -89,7 +90,7 @@ class DcsTest implements IDcsHandler { hook(collect: string, params: number[], flag: number): void { testTerminal.actionDCSHook(collect, params, String.fromCharCode(flag)); } - put(data: Uint16Array, start: number, end: number): void { + put(data: Uint32Array, start: number, end: number): void { testTerminal.actionDCSPrint(data, start, end); } unhook(): void { @@ -165,11 +166,9 @@ interface IRun { // translate string based parse calls into typed array based function parse(parser: TestEscapeSequenceParser, data: string): void { - const container = new Uint16Array(data.length); - for (let i = 0; i < data.length; ++i) { - container[i] = data.charCodeAt(i); - } - parser.parse(container, data.length); + const container = new Uint32Array(data.length); + const decoder = new StringToUtf32(); + parser.parse(container, decoder.decode(data, container)); } describe('EscapeSequenceParser', function (): void { @@ -1143,9 +1142,9 @@ describe('EscapeSequenceParser', function (): void { clearAccu(); }); it('print handler', function (): void { - parser2.setPrintHandler(function (data: Uint16Array, start: number, end: number): void { + parser2.setPrintHandler(function (data: Uint32Array, start: number, end: number): void { for (let i = start; i < end; ++i) { - print += String.fromCharCode(data[i]); + print += stringFromCodePoint(data[i]); } }); parse(parser2, INPUT); @@ -1353,10 +1352,10 @@ describe('EscapeSequenceParser', function (): void { hook: function (collect: string, params: number[], flag: number): void { dcs.push(['hook', collect, params, flag]); }, - put: function (data: Uint16Array, start: number, end: number): void { + put: function (data: Uint32Array, start: number, end: number): void { let s = ''; for (let i = start; i < end; ++i) { - s += String.fromCharCode(data[i]); + s += stringFromCodePoint(data[i]); } dcs.push(['put', s]); }, diff --git a/src/EscapeSequenceParser.ts b/src/EscapeSequenceParser.ts index cee9957b..7bd425d4 100644 --- a/src/EscapeSequenceParser.ts +++ b/src/EscapeSequenceParser.ts @@ -6,7 +6,7 @@ import { ParserState, ParserAction, IParsingState, IDcsHandler, IEscapeSequenceParser } from './Types'; import { IDisposable } from 'xterm'; import { Disposable } from './common/Lifecycle'; -import { utf16ToString } from './common/TypedArrayUtils'; +import { utf32ToString } from './common/TypedArrayUtils'; interface IHandlerCollection { [key: string]: T[]; @@ -204,7 +204,7 @@ export const VT500_TRANSITION_TABLE = (function (): TransitionTable { */ class DcsDummy implements IDcsHandler { hook(collect: string, params: number[], flag: number): void { } - put(data: Uint16Array, start: number, end: number): void { } + put(data: Uint32Array, start: number, end: number): void { } unhook(): void { } } @@ -230,7 +230,7 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP protected _collect: string; // handler lookup containers - protected _printHandler: (data: Uint16Array, start: number, end: number) => void; + protected _printHandler: (data: Uint32Array, start: number, end: number) => void; protected _executeHandlers: any; protected _csiHandlers: IHandlerCollection; protected _escHandlers: any; @@ -240,7 +240,7 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP protected _errorHandler: (state: IParsingState) => IParsingState; // fallback handlers - protected _printHandlerFb: (data: Uint16Array, start: number, end: number) => void; + protected _printHandlerFb: (data: Uint32Array, start: number, end: number) => void; protected _executeHandlerFb: (code: number) => void; protected _csiHandlerFb: (collect: string, params: number[], flag: number) => void; protected _escHandlerFb: (collect: string, flag: number) => void; @@ -296,7 +296,7 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP this._errorHandler = null; } - setPrintHandler(callback: (data: Uint16Array, start: number, end: number) => void): void { + setPrintHandler(callback: (data: Uint32Array, start: number, end: number) => void): void { this._printHandler = callback; } clearPrintHandler(): void { @@ -399,7 +399,7 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP this._activeDcsHandler = null; } - parse(data: Uint16Array, length: number): void { + parse(data: Uint32Array, length: number): void { let code = 0; let transition = 0; let error = false; @@ -567,7 +567,7 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP if (j >= length || (code = data[j]) < 0x20 || (code > 0x7f && code <= 0x9f)) { - osc += utf16ToString(data.subarray(i, j)); + osc += utf32ToString(data.subarray(i, j)); i = j - 1; break; } diff --git a/src/InputHandler.test.ts b/src/InputHandler.test.ts index b92d009b..a7963a2c 100644 --- a/src/InputHandler.test.ts +++ b/src/InputHandler.test.ts @@ -337,7 +337,7 @@ describe('InputHandler', () => { it('should not cause an infinite loop (regression test)', () => { const term = new Terminal(); const inputHandler = new InputHandler(term); - const container = new Uint16Array(10); + const container = new Uint32Array(10); container[0] = 0x200B; inputHandler.print(container, 0, 1); }); diff --git a/src/InputHandler.ts b/src/InputHandler.ts index 3a2dab11..e270a5f3 100644 --- a/src/InputHandler.ts +++ b/src/InputHandler.ts @@ -14,7 +14,8 @@ import { EscapeSequenceParser } from './EscapeSequenceParser'; import { ICharset } from './core/Types'; import { IDisposable } from 'xterm'; import { Disposable } from './common/Lifecycle'; -import { concat, utf16ToString } from './common/TypedArrayUtils'; +import { concat, utf32ToString } from './common/TypedArrayUtils'; +import { StringToUtf32, stringFromCodePoint } from './core/input/TextDecoder'; /** * Map collect to glevel. Used in `selectCharset`. @@ -32,17 +33,17 @@ const GLEVEL: {[key: string]: number} = {'(': 0, ')': 1, '*': 2, '+': 3, '-': 1, * not supported */ class RequestTerminfo implements IDcsHandler { - private _data: Uint16Array = new Uint16Array(0); + private _data: Uint32Array = new Uint32Array(0); constructor(private _terminal: any) { } hook(collect: string, params: number[], flag: number): void { - this._data = new Uint16Array(0); + this._data = new Uint32Array(0); } - put(data: Uint16Array, start: number, end: number): void { + put(data: Uint32Array, start: number, end: number): void { this._data = concat(this._data, data.subarray(start, end)); } unhook(): void { - const data = utf16ToString(this._data); - this._data = new Uint16Array(0); + const data = utf32ToString(this._data); + this._data = new Uint32Array(0); // invalid: DCS 0 + r Pt ST this._terminal.handler(`${C0.ESC}P0+r${data}${C0.ESC}\\`); } @@ -55,21 +56,21 @@ const GLEVEL: {[key: string]: number} = {'(': 0, ')': 1, '*': 2, '+': 3, '-': 1, * Response: DECRPSS (https://vt100.net/docs/vt510-rm/DECRPSS.html) */ class DECRQSS implements IDcsHandler { - private _data: Uint16Array = new Uint16Array(0); + private _data: Uint32Array = new Uint32Array(0); constructor(private _terminal: any) { } hook(collect: string, params: number[], flag: number): void { - this._data = new Uint16Array(0); + this._data = new Uint32Array(0); } - put(data: Uint16Array, start: number, end: number): void { + put(data: Uint32Array, start: number, end: number): void { this._data = concat(this._data, data.subarray(start, end)); } unhook(): void { - const data = utf16ToString(this._data); - this._data = new Uint16Array(0); + const data = utf32ToString(this._data); + this._data = new Uint32Array(0); switch (data) { // valid: DCS 1 $ r Pt ST (xterm) case '"q': // DECSCA @@ -118,8 +119,8 @@ class DECRQSS implements IDcsHandler { * each function's header comment. */ export class InputHandler extends Disposable implements IInputHandler { - private _surrogateFirst: string; - private _parseBuffer: Uint16Array = new Uint16Array(4096); + private _parseBuffer: Uint32Array = new Uint32Array(4096); + private _stringDecoder: StringToUtf32 = new StringToUtf32(); constructor( protected _terminal: IInputHandlingTerminal, @@ -129,8 +130,6 @@ export class InputHandler extends Disposable implements IInputHandler { this.register(this._parser); - this._surrogateFirst = ''; - /** * custom fallback handlers */ @@ -316,19 +315,13 @@ export class InputHandler extends Disposable implements IInputHandler { this._terminal.log('data: ' + data); } - // apply leftover surrogate high from last write - if (this._surrogateFirst) { - data = this._surrogateFirst + data; - this._surrogateFirst = ''; - } - if (this._parseBuffer.length < data.length) { - this._parseBuffer = new Uint16Array(data.length); + this._parseBuffer = new Uint32Array(data.length); } for (let i = 0; i < data.length; ++i) { this._parseBuffer[i] = data.charCodeAt(i); } - this._parser.parse(this._parseBuffer, data.length); + this._parser.parse(this._parseBuffer, this._stringDecoder.decode(data, this._parseBuffer)); buffer = this._terminal.buffer; if (buffer.x !== cursorStartX || buffer.y !== cursorStartY) { @@ -336,7 +329,7 @@ export class InputHandler extends Disposable implements IInputHandler { } } - public print(data: Uint16Array, start: number, end: number): void { + public print(data: Uint32Array, start: number, end: number): void { let code: number; let char: string; let chWidth: number; @@ -352,30 +345,7 @@ export class InputHandler extends Disposable implements IInputHandler { this._terminal.updateRange(buffer.y); for (let pos = start; pos < end; ++pos) { code = data[pos]; - char = String.fromCharCode(code); - - // surrogate pair handling - if (0xD800 <= code && code <= 0xDBFF) { - if (++pos >= end) { - // end of input: - // handle pairs as true UTF-16 and wait for the second part - // since we expect the input comming from a stream there is - // a small chance that the surrogate pair got split - // therefore we dont process the first char here, instead - // it gets added as first char to the next processed chunk - this._surrogateFirst = String.fromCharCode(code); - continue; - } - const second = data[pos]; - // if the second part is in surrogate pair range create the high codepoint - // otherwise fall back to UCS-2 behavior (handle codepoints independently) - if (0xDC00 <= second && second <= 0xDFFF) { - code = (code - 0xD800) * 0x400 + second - 0xDC00 + 0x10000; - char += String.fromCharCode(second); - } else { - pos--; - } - } + char = stringFromCodePoint(code); // calculate print space // expensive call, therefore we save width in line buffer diff --git a/src/Types.ts b/src/Types.ts index ca50afd5..60b86de1 100644 --- a/src/Types.ts +++ b/src/Types.ts @@ -111,7 +111,7 @@ export interface ICompositionHelper { */ export interface IInputHandler { parse(data: string): void; - print(data: Uint16Array, start: number, end: number): void; + print(data: Uint32Array, start: number, end: number): void; /** C0 BEL */ bell(): void; /** C0 LF */ lineFeed(): void; @@ -463,7 +463,7 @@ export interface IParsingState { */ export interface IDcsHandler { hook(collect: string, params: number[], flag: number): void; - put(data: Uint16Array, start: number, end: number): void; + put(data: Uint32Array, start: number, end: number): void; unhook(): void; } @@ -480,9 +480,9 @@ export interface IEscapeSequenceParser extends IDisposable { * Parse string `data`. * @param data The data to parse. */ - parse(data: Uint16Array, length: number): void; + parse(data: Uint32Array, length: number): void; - setPrintHandler(callback: (data: Uint16Array, start: number, end: number) => void): void; + setPrintHandler(callback: (data: Uint32Array, start: number, end: number) => void): void; clearPrintHandler(): void; setExecuteHandler(flag: string, callback: () => void): void; diff --git a/src/common/TypedArrayUtils.test.ts b/src/common/TypedArrayUtils.test.ts index b25b9b9a..79546ca9 100644 --- a/src/common/TypedArrayUtils.test.ts +++ b/src/common/TypedArrayUtils.test.ts @@ -3,7 +3,7 @@ * @license MIT */ import { assert } from 'chai'; -import { fillFallback, concat, utf16ToString } from './TypedArrayUtils'; +import { fillFallback, concat, utf32ToString } from './TypedArrayUtils'; type TypedArray = Uint8Array | Uint16Array | Uint32Array | Uint8ClampedArray | Int8Array | Int16Array | Int32Array @@ -100,6 +100,6 @@ describe('typed array convenience functions', () => { for (let i = 0; i < s.length; ++i) { data[i] = s.charCodeAt(i); } - assert.equal(utf16ToString(data), s); + assert.equal(utf32ToString(data), s); }); }); diff --git a/src/common/TypedArrayUtils.ts b/src/common/TypedArrayUtils.ts index 5d3bcb8a..5ec1fd48 100644 --- a/src/common/TypedArrayUtils.ts +++ b/src/common/TypedArrayUtils.ts @@ -1,3 +1,5 @@ +import { stringFromCodePoint } from '../core/input/TextDecoder'; + /** * Copyright (c) 2018 The xterm.js authors. All rights reserved. * @license MIT @@ -51,11 +53,18 @@ export function concat(a: T, b: T): T { } /** - * Convert UTF16 char codes into JS string. + * Convert UTF32 char codes into JS string. * Note the typed array is not limited to Uint16Array, make sure to align * the values to 0-65535 integer for other typed array types, otherwise * the conversion will fail. */ -export function utf16ToString(data: T): string { - return String.fromCharCode.apply(null, data); +export function utf32ToString(data: T): string { + if ((String as any).fromCodePoint) { + return (String as any).fromCodePoint.apply(null, data); + } + let result = ''; + for (let i = 0; i < data.length; ++i) { + result += stringFromCodePoint(data[i]); + } + return result; }