From 9b762854eb2626d420eeb58090332bc7a9f8a9b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= Date: Sun, 18 Nov 2018 23:31:45 +0100 Subject: [PATCH 1/7] faster wcwidth init and runtime; fix slightly wrong surrogate handling --- src/CharWidth.ts | 87 +++++++++++++++++++++++---------------------- src/InputHandler.ts | 41 +++++++++++---------- 2 files changed, 67 insertions(+), 61 deletions(-) diff --git a/src/CharWidth.ts b/src/CharWidth.ts index d54e9392..045a796c 100644 --- a/src/CharWidth.ts +++ b/src/CharWidth.ts @@ -120,50 +120,41 @@ export const wcwidth = (function(opts: {nul: number, control: number}): (ucs: nu return 1; } const control = opts.control | 0; - let table: number[] | Uint32Array = null; - function initTable(): number[] | Uint32Array { - // lookup table for BMP - const CODEPOINTS = 65536; // BMP holds 65536 codepoints - const BITWIDTH = 2; // a codepoint can have a width of 0, 1 or 2 - const ITEMSIZE = 32; // using uint32_t - const CONTAINERSIZE = CODEPOINTS * BITWIDTH / ITEMSIZE; - const CODEPOINTS_PER_ITEM = ITEMSIZE / BITWIDTH; - table = (typeof Uint32Array === 'undefined') - ? new Array(CONTAINERSIZE) - : new Uint32Array(CONTAINERSIZE); - for (let i = 0; i < CONTAINERSIZE; ++i) { - let num = 0; - let pos = CODEPOINTS_PER_ITEM; - while (pos--) { - num = (num << 2) | wcwidthBMP(CODEPOINTS_PER_ITEM * i + pos); - } - table[i] = num; - } - return table; + + // create lookup table for BMP plane + // TODO: make callable/configurable from UnicodeManager + const table = new Uint8Array(65536); + table.fill(1); + table[0] = opts.nul; + // control chars + table.subarray(1, 32).fill(opts.control); + table.subarray(0x7f, 0xa0).fill(opts.control); + // combining 0 + for (let r = 0; r < COMBINING_BMP.length; ++r) { + table.subarray(COMBINING_BMP[r][0], COMBINING_BMP[r][1]).fill(0); } - // get width from lookup table - // position in container : num / CODEPOINTS_PER_ITEM - // ==> n = table[Math.floor(num / 16)] - // ==> n = table[num >> 4] - // 16 codepoints per number: FFEEDDCCBBAA99887766554433221100 - // position in number : (num % CODEPOINTS_PER_ITEM) * BITWIDTH - // ==> m = (n % 16) * 2 - // ==> m = (num & 15) << 1 - // right shift to position m - // ==> n = n >> m e.g. m=12 000000000000FFEEDDCCBBAA99887766 - // we are only interested in 2 LSBs, cut off higher bits - // ==> n = n & 3 e.g. 000000000000000000000000000000XX + // wide chars + table.subarray(0x1100, 0x1160).fill(2); + table[0x2329] = 2; + table[0x232a] = 2; + table.subarray(0x2e80, 0xa4d0).fill(2); + table[0x303f] = 1; // wrongly added before + table.subarray(0xac00, 0xd7a4).fill(2); + table.subarray(0xf900, 0xfb00).fill(2); + table.subarray(0xfe10, 0xfe1a).fill(2); + table.subarray(0xfe30, 0xfe70).fill(2); + table.subarray(0xff00, 0xff61).fill(2); + table.subarray(0xffe0, 0xffe7).fill(2); + return function (num: number): number { - num = num | 0; // get asm.js like optimization under V8 if (num < 32) { return control | 0; } if (num < 127) { return 1; } - const t = table || initTable(); if (num < 65536) { - return t[num >> 4] >> ((num & 15) << 1) & 3; + return table[num]; } // do a full search for high codepoints return wcwidthHigh(num); @@ -175,17 +166,27 @@ export const wcwidth = (function(opts: {nul: number, control: number}): (ucs: nu */ export function getStringCellWidth(s: string): number { let result = 0; - for (let i = 0; i < s.length; ++i) { + const length = s.length; + for (let i = 0; i < length; ++i) { let code = s.charCodeAt(i); + // surrogate pair first if (0xD800 <= code && code <= 0xDBFF) { - const low = s.charCodeAt(i + 1); - if (isNaN(low)) { - return result; + if (++i >= length) { + // this should not happen with strings retrieved from + // Buffer.translateToString as it converts from UTF-32 + // and therefore always should contain the second part + // for any other string we still have to handle it somehow: + // simply treat the lonely surrogate first as a single char (UCS-2 behavior) + return result + wcwidth(code); + } + const second = s.charCodeAt(i); + // convert surrogate pair to high codepoint only for valid second part (UTF-16) + // otherwise treat them independently (UCS-2 behavior) + if (0xDC00 <= second && second <= 0xDFFF) { + code = (code - 0xD800) * 0x400 + second - 0xDC00 + 0x10000; + } else { + result += wcwidth(second); } - code = ((code - 0xD800) * 0x400) + (low - 0xDC00) + 0x10000; - } - if (0xDC00 <= code && code <= 0xDFFF) { - continue; } result += wcwidth(code); } diff --git a/src/InputHandler.ts b/src/InputHandler.ts index a34590ef..893046bf 100644 --- a/src/InputHandler.ts +++ b/src/InputHandler.ts @@ -113,7 +113,7 @@ class DECRQSS implements IDcsHandler { * each function's header comment. */ export class InputHandler extends Disposable implements IInputHandler { - private _surrogateHigh: string; + private _surrogateFirst: string; constructor( protected _terminal: IInputHandlingTerminal, @@ -123,7 +123,7 @@ export class InputHandler extends Disposable implements IInputHandler { this.register(this._parser); - this._surrogateHigh = ''; + this._surrogateFirst = ''; /** * custom fallback handlers @@ -311,9 +311,9 @@ export class InputHandler extends Disposable implements IInputHandler { } // apply leftover surrogate high from last write - if (this._surrogateHigh) { - data = this._surrogateHigh + data; - this._surrogateHigh = ''; + if (this._surrogateFirst) { + data = this._surrogateFirst + data; + this._surrogateFirst = ''; } this._parser.parse(data); @@ -327,7 +327,7 @@ export class InputHandler extends Disposable implements IInputHandler { public print(data: string, start: number, end: number): void { let char: string; let code: number; - let low: number; + let second: number; let chWidth: number; const buffer: IBuffer = this._terminal.buffer; const charset: ICharset = this._terminal.charset; @@ -345,20 +345,25 @@ export class InputHandler extends Disposable implements IInputHandler { // surrogate pair handling if (0xD800 <= code && code <= 0xDBFF) { - // we got a surrogate high - // get surrogate low (next 2 bytes) - low = data.charCodeAt(stringPosition + 1); - if (isNaN(low)) { - // end of data stream, save surrogate high - this._surrogateHigh = char; + if (++stringPosition >= end) { + // end of input: + // handle pairs as true UTF-16 and wait for the second part + // since we expect the input comming from a stream there is + // a small chance that the surrogate pair got split + // therefore we dont process the first char here, instead + // it gets added as first char to the next processed chunk + this._surrogateFirst = char; continue; } - code = ((code - 0xD800) * 0x400) + (low - 0xDC00) + 0x10000; - char += data.charAt(stringPosition + 1); - } - // surrogate low - already handled above - if (0xDC00 <= code && code <= 0xDFFF) { - continue; + second = data.charCodeAt(stringPosition); + // if the second part is in surrogate pair range create the high codepoint + // otherwise fall back to UCS-2 behavior (handle codepoints independently) + if (0xDC00 <= second && second <= 0xDFFF) { + code = (code - 0xD800) * 0x400 + second - 0xDC00 + 0x10000; + char += data.charAt(stringPosition); + } else { + stringPosition--; + } } // calculate print space From 21f848d0ab21205dfe26f94ca0753ab461e3f591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= Date: Sun, 18 Nov 2018 23:48:59 +0100 Subject: [PATCH 2/7] regression test --- src/CharWidth.test.ts | 178 +++++++++++++++++++++++++++++++++++++++++- src/CharWidth.ts | 50 +++--------- 2 files changed, 190 insertions(+), 38 deletions(-) diff --git a/src/CharWidth.test.ts b/src/CharWidth.test.ts index 3242dd77..c2d10c5c 100644 --- a/src/CharWidth.test.ts +++ b/src/CharWidth.test.ts @@ -5,7 +5,7 @@ import { TestTerminal } from './utils/TestUtils.test'; import { assert } from 'chai'; -import { getStringCellWidth } from './CharWidth'; +import { getStringCellWidth, wcwidth } from './CharWidth'; import { IBuffer } from './Types'; import { CHAR_DATA_WIDTH_INDEX, CHAR_DATA_CHAR_INDEX } from './Buffer'; @@ -77,3 +77,179 @@ describe('getStringCellWidth', function(): void { }); // TODO: multiline tests once #1685 is resolved }); + +describe('wcwidth regression', function(): void { + // TODO: remove with one of the next releases + const wcwidthOld = (function(opts: {nul: number, control: number}): (ucs: number) => number { + // extracted from https://www.cl.cam.ac.uk/%7Emgk25/ucs/wcwidth.c + // combining characters + const COMBINING_BMP = [ + [0x0300, 0x036F], [0x0483, 0x0486], [0x0488, 0x0489], + [0x0591, 0x05BD], [0x05BF, 0x05BF], [0x05C1, 0x05C2], + [0x05C4, 0x05C5], [0x05C7, 0x05C7], [0x0600, 0x0603], + [0x0610, 0x0615], [0x064B, 0x065E], [0x0670, 0x0670], + [0x06D6, 0x06E4], [0x06E7, 0x06E8], [0x06EA, 0x06ED], + [0x070F, 0x070F], [0x0711, 0x0711], [0x0730, 0x074A], + [0x07A6, 0x07B0], [0x07EB, 0x07F3], [0x0901, 0x0902], + [0x093C, 0x093C], [0x0941, 0x0948], [0x094D, 0x094D], + [0x0951, 0x0954], [0x0962, 0x0963], [0x0981, 0x0981], + [0x09BC, 0x09BC], [0x09C1, 0x09C4], [0x09CD, 0x09CD], + [0x09E2, 0x09E3], [0x0A01, 0x0A02], [0x0A3C, 0x0A3C], + [0x0A41, 0x0A42], [0x0A47, 0x0A48], [0x0A4B, 0x0A4D], + [0x0A70, 0x0A71], [0x0A81, 0x0A82], [0x0ABC, 0x0ABC], + [0x0AC1, 0x0AC5], [0x0AC7, 0x0AC8], [0x0ACD, 0x0ACD], + [0x0AE2, 0x0AE3], [0x0B01, 0x0B01], [0x0B3C, 0x0B3C], + [0x0B3F, 0x0B3F], [0x0B41, 0x0B43], [0x0B4D, 0x0B4D], + [0x0B56, 0x0B56], [0x0B82, 0x0B82], [0x0BC0, 0x0BC0], + [0x0BCD, 0x0BCD], [0x0C3E, 0x0C40], [0x0C46, 0x0C48], + [0x0C4A, 0x0C4D], [0x0C55, 0x0C56], [0x0CBC, 0x0CBC], + [0x0CBF, 0x0CBF], [0x0CC6, 0x0CC6], [0x0CCC, 0x0CCD], + [0x0CE2, 0x0CE3], [0x0D41, 0x0D43], [0x0D4D, 0x0D4D], + [0x0DCA, 0x0DCA], [0x0DD2, 0x0DD4], [0x0DD6, 0x0DD6], + [0x0E31, 0x0E31], [0x0E34, 0x0E3A], [0x0E47, 0x0E4E], + [0x0EB1, 0x0EB1], [0x0EB4, 0x0EB9], [0x0EBB, 0x0EBC], + [0x0EC8, 0x0ECD], [0x0F18, 0x0F19], [0x0F35, 0x0F35], + [0x0F37, 0x0F37], [0x0F39, 0x0F39], [0x0F71, 0x0F7E], + [0x0F80, 0x0F84], [0x0F86, 0x0F87], [0x0F90, 0x0F97], + [0x0F99, 0x0FBC], [0x0FC6, 0x0FC6], [0x102D, 0x1030], + [0x1032, 0x1032], [0x1036, 0x1037], [0x1039, 0x1039], + [0x1058, 0x1059], [0x1160, 0x11FF], [0x135F, 0x135F], + [0x1712, 0x1714], [0x1732, 0x1734], [0x1752, 0x1753], + [0x1772, 0x1773], [0x17B4, 0x17B5], [0x17B7, 0x17BD], + [0x17C6, 0x17C6], [0x17C9, 0x17D3], [0x17DD, 0x17DD], + [0x180B, 0x180D], [0x18A9, 0x18A9], [0x1920, 0x1922], + [0x1927, 0x1928], [0x1932, 0x1932], [0x1939, 0x193B], + [0x1A17, 0x1A18], [0x1B00, 0x1B03], [0x1B34, 0x1B34], + [0x1B36, 0x1B3A], [0x1B3C, 0x1B3C], [0x1B42, 0x1B42], + [0x1B6B, 0x1B73], [0x1DC0, 0x1DCA], [0x1DFE, 0x1DFF], + [0x200B, 0x200F], [0x202A, 0x202E], [0x2060, 0x2063], + [0x206A, 0x206F], [0x20D0, 0x20EF], [0x302A, 0x302F], + [0x3099, 0x309A], [0xA806, 0xA806], [0xA80B, 0xA80B], + [0xA825, 0xA826], [0xFB1E, 0xFB1E], [0xFE00, 0xFE0F], + [0xFE20, 0xFE23], [0xFEFF, 0xFEFF], [0xFFF9, 0xFFFB] + ]; + const COMBINING_HIGH = [ + [0x10A01, 0x10A03], [0x10A05, 0x10A06], [0x10A0C, 0x10A0F], + [0x10A38, 0x10A3A], [0x10A3F, 0x10A3F], [0x1D167, 0x1D169], + [0x1D173, 0x1D182], [0x1D185, 0x1D18B], [0x1D1AA, 0x1D1AD], + [0x1D242, 0x1D244], [0xE0001, 0xE0001], [0xE0020, 0xE007F], + [0xE0100, 0xE01EF] + ]; + // binary search + function bisearch(ucs: number, data: number[][]): boolean { + let min = 0; + let max = data.length - 1; + let mid; + if (ucs < data[0][0] || ucs > data[max][1]) { + return false; + } + while (max >= min) { + mid = (min + max) >> 1; + if (ucs > data[mid][1]) { + min = mid + 1; + } else if (ucs < data[mid][0]) { + max = mid - 1; + } else { + return true; + } + } + return false; + } + function wcwidthBMP(ucs: number): number { + // test for 8-bit control characters + if (ucs === 0) { + return opts.nul; + } + if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0)) { + return opts.control; + } + // binary search in table of non-spacing characters + if (bisearch(ucs, COMBINING_BMP)) { + return 0; + } + // if we arrive here, ucs is not a combining or C0/C1 control character + if (isWideBMP(ucs)) { + return 2; + } + return 1; + } + function isWideBMP(ucs: number): boolean { + return ( + ucs >= 0x1100 && ( + ucs <= 0x115f || // Hangul Jamo init. consonants + ucs === 0x2329 || + ucs === 0x232a || + (ucs >= 0x2e80 && ucs <= 0xa4cf && ucs !== 0x303f) || // CJK..Yi + (ucs >= 0xac00 && ucs <= 0xd7a3) || // Hangul Syllables + (ucs >= 0xf900 && ucs <= 0xfaff) || // CJK Compat Ideographs + (ucs >= 0xfe10 && ucs <= 0xfe19) || // Vertical forms + (ucs >= 0xfe30 && ucs <= 0xfe6f) || // CJK Compat Forms + (ucs >= 0xff00 && ucs <= 0xff60) || // Fullwidth Forms + (ucs >= 0xffe0 && ucs <= 0xffe6))); + } + function wcwidthHigh(ucs: number): 0 | 1 | 2 { + if (bisearch(ucs, COMBINING_HIGH)) { + return 0; + } + if ((ucs >= 0x20000 && ucs <= 0x2fffd) || (ucs >= 0x30000 && ucs <= 0x3fffd)) { + return 2; + } + return 1; + } + const control = opts.control | 0; + let table: number[] | Uint32Array = null; + function initTable(): number[] | Uint32Array { + // lookup table for BMP + const CODEPOINTS = 65536; // BMP holds 65536 codepoints + const BITWIDTH = 2; // a codepoint can have a width of 0, 1 or 2 + const ITEMSIZE = 32; // using uint32_t + const CONTAINERSIZE = CODEPOINTS * BITWIDTH / ITEMSIZE; + const CODEPOINTS_PER_ITEM = ITEMSIZE / BITWIDTH; + table = (typeof Uint32Array === 'undefined') + ? new Array(CONTAINERSIZE) + : new Uint32Array(CONTAINERSIZE); + for (let i = 0; i < CONTAINERSIZE; ++i) { + let num = 0; + let pos = CODEPOINTS_PER_ITEM; + while (pos--) { + num = (num << 2) | wcwidthBMP(CODEPOINTS_PER_ITEM * i + pos); + } + table[i] = num; + } + return table; + } + // get width from lookup table + // position in container : num / CODEPOINTS_PER_ITEM + // ==> n = table[Math.floor(num / 16)] + // ==> n = table[num >> 4] + // 16 codepoints per number: FFEEDDCCBBAA99887766554433221100 + // position in number : (num % CODEPOINTS_PER_ITEM) * BITWIDTH + // ==> m = (n % 16) * 2 + // ==> m = (num & 15) << 1 + // right shift to position m + // ==> n = n >> m e.g. m=12 000000000000FFEEDDCCBBAA99887766 + // we are only interested in 2 LSBs, cut off higher bits + // ==> n = n & 3 e.g. 000000000000000000000000000000XX + return function (num: number): number { + num = num | 0; // get asm.js like optimization under V8 + if (num < 32) { + return control | 0; + } + if (num < 127) { + return 1; + } + const t = table || initTable(); + if (num < 65536) { + return t[num >> 4] >> ((num & 15) << 1) & 3; + } + // do a full search for high codepoints + return wcwidthHigh(num); + }; + })({nul: 0, control: 0}); // configurable options + + it('equality of old and new impl', function(): void { + for (let i = 0; i < 65536; ++i) { + assert.equal(wcwidth(i), wcwidthOld(i), `mismatch for i: ${i}`); + } + }); +}); diff --git a/src/CharWidth.ts b/src/CharWidth.ts index 045a796c..05fb5d37 100644 --- a/src/CharWidth.ts +++ b/src/CharWidth.ts @@ -78,38 +78,6 @@ export const wcwidth = (function(opts: {nul: number, control: number}): (ucs: nu } return false; } - function wcwidthBMP(ucs: number): number { - // test for 8-bit control characters - if (ucs === 0) { - return opts.nul; - } - if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0)) { - return opts.control; - } - // binary search in table of non-spacing characters - if (bisearch(ucs, COMBINING_BMP)) { - return 0; - } - // if we arrive here, ucs is not a combining or C0/C1 control character - if (isWideBMP(ucs)) { - return 2; - } - return 1; - } - function isWideBMP(ucs: number): boolean { - return ( - ucs >= 0x1100 && ( - ucs <= 0x115f || // Hangul Jamo init. consonants - ucs === 0x2329 || - ucs === 0x232a || - (ucs >= 0x2e80 && ucs <= 0xa4cf && ucs !== 0x303f) || // CJK..Yi - (ucs >= 0xac00 && ucs <= 0xd7a3) || // Hangul Syllables - (ucs >= 0xf900 && ucs <= 0xfaff) || // CJK Compat Ideographs - (ucs >= 0xfe10 && ucs <= 0xfe19) || // Vertical forms - (ucs >= 0xfe30 && ucs <= 0xfe6f) || // CJK Compat Forms - (ucs >= 0xff00 && ucs <= 0xff60) || // Fullwidth Forms - (ucs >= 0xffe0 && ucs <= 0xffe6))); - } function wcwidthHigh(ucs: number): 0 | 1 | 2 { if (bisearch(ucs, COMBINING_HIGH)) { return 0; @@ -129,16 +97,15 @@ export const wcwidth = (function(opts: {nul: number, control: number}): (ucs: nu // control chars table.subarray(1, 32).fill(opts.control); table.subarray(0x7f, 0xa0).fill(opts.control); - // combining 0 - for (let r = 0; r < COMBINING_BMP.length; ++r) { - table.subarray(COMBINING_BMP[r][0], COMBINING_BMP[r][1]).fill(0); - } + + // apply wide char rules first // wide chars table.subarray(0x1100, 0x1160).fill(2); table[0x2329] = 2; table[0x232a] = 2; table.subarray(0x2e80, 0xa4d0).fill(2); - table[0x303f] = 1; // wrongly added before + table[0x303f] = 1; // wrongly in last line + table.subarray(0xac00, 0xd7a4).fill(2); table.subarray(0xf900, 0xfb00).fill(2); table.subarray(0xfe10, 0xfe1a).fill(2); @@ -146,6 +113,15 @@ export const wcwidth = (function(opts: {nul: number, control: number}): (ucs: nu table.subarray(0xff00, 0xff61).fill(2); table.subarray(0xffe0, 0xffe7).fill(2); + // apply combining last to ensure we overwrite + // wrongly wide set chars: + // the original algo evals combining first and falls + // through to wide check so we simply do here the opposite + // combining 0 + for (let r = 0; r < COMBINING_BMP.length; ++r) { + table.subarray(COMBINING_BMP[r][0], COMBINING_BMP[r][1] + 1).fill(0); + } + return function (num: number): number { if (num < 32) { return control | 0; From 36db03e7133676e4044c94084bd4b7f9ebe8cab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= Date: Tue, 20 Nov 2018 22:35:51 +0100 Subject: [PATCH 3/7] keep old impl in test as reference; minor changes --- src/CharWidth.test.ts | 13 ++++++------- src/CharWidth.ts | 1 - src/InputHandler.ts | 3 +-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/CharWidth.test.ts b/src/CharWidth.test.ts index c2d10c5c..d4ddd24c 100644 --- a/src/CharWidth.test.ts +++ b/src/CharWidth.test.ts @@ -78,8 +78,8 @@ describe('getStringCellWidth', function(): void { // TODO: multiline tests once #1685 is resolved }); -describe('wcwidth regression', function(): void { - // TODO: remove with one of the next releases +it('wcwidth should match all values from the old implementation', function(): void { + // old implementation const wcwidthOld = (function(opts: {nul: number, control: number}): (ucs: number) => number { // extracted from https://www.cl.cam.ac.uk/%7Emgk25/ucs/wcwidth.c // combining characters @@ -247,9 +247,8 @@ describe('wcwidth regression', function(): void { }; })({nul: 0, control: 0}); // configurable options - it('equality of old and new impl', function(): void { - for (let i = 0; i < 65536; ++i) { - assert.equal(wcwidth(i), wcwidthOld(i), `mismatch for i: ${i}`); - } - }); + // test full BMP range old vs new implmenetation + for (let i = 0; i < 65536; ++i) { + assert.equal(wcwidth(i), wcwidthOld(i), `mismatch for i: ${i}`); + } }); diff --git a/src/CharWidth.ts b/src/CharWidth.ts index 05fb5d37..d099b109 100644 --- a/src/CharWidth.ts +++ b/src/CharWidth.ts @@ -90,7 +90,6 @@ export const wcwidth = (function(opts: {nul: number, control: number}): (ucs: nu const control = opts.control | 0; // create lookup table for BMP plane - // TODO: make callable/configurable from UnicodeManager const table = new Uint8Array(65536); table.fill(1); table[0] = opts.nul; diff --git a/src/InputHandler.ts b/src/InputHandler.ts index 893046bf..b4ce054f 100644 --- a/src/InputHandler.ts +++ b/src/InputHandler.ts @@ -327,7 +327,6 @@ export class InputHandler extends Disposable implements IInputHandler { public print(data: string, start: number, end: number): void { let char: string; let code: number; - let second: number; let chWidth: number; const buffer: IBuffer = this._terminal.buffer; const charset: ICharset = this._terminal.charset; @@ -355,7 +354,7 @@ export class InputHandler extends Disposable implements IInputHandler { this._surrogateFirst = char; continue; } - second = data.charCodeAt(stringPosition); + const second = data.charCodeAt(stringPosition); // if the second part is in surrogate pair range create the high codepoint // otherwise fall back to UCS-2 behavior (handle codepoints independently) if (0xDC00 <= second && second <= 0xDFFF) { From ef6fb93a928061bb2ef26563e76f0ed7298793fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= Date: Wed, 21 Nov 2018 00:07:28 +0100 Subject: [PATCH 4/7] polyfill for TypedArray.fill --- src/CharWidth.ts | 26 ++--- src/core/TypedArrayUtils.test.ts | 157 +++++++++++++++++++++++++++++++ src/core/TypedArrayUtils.ts | 40 ++++++++ 3 files changed, 211 insertions(+), 12 deletions(-) create mode 100644 src/core/TypedArrayUtils.test.ts create mode 100644 src/core/TypedArrayUtils.ts diff --git a/src/CharWidth.ts b/src/CharWidth.ts index d099b109..341b347a 100644 --- a/src/CharWidth.ts +++ b/src/CharWidth.ts @@ -1,3 +1,5 @@ +import { fill } from './core/TypedArrayUtils'; + /** * Copyright (c) 2016 The xterm.js authors. All rights reserved. * @license MIT @@ -91,26 +93,26 @@ export const wcwidth = (function(opts: {nul: number, control: number}): (ucs: nu // create lookup table for BMP plane const table = new Uint8Array(65536); - table.fill(1); + fill(table, 1); table[0] = opts.nul; // control chars - table.subarray(1, 32).fill(opts.control); - table.subarray(0x7f, 0xa0).fill(opts.control); + fill(table, opts.control, 1, 32); + fill(table, opts.control, 0x7f, 0xa0); // apply wide char rules first // wide chars - table.subarray(0x1100, 0x1160).fill(2); + fill(table, 2, 0x1100, 0x1160); table[0x2329] = 2; table[0x232a] = 2; - table.subarray(0x2e80, 0xa4d0).fill(2); + fill(table, 2, 0x2e80, 0xa4d0); table[0x303f] = 1; // wrongly in last line - table.subarray(0xac00, 0xd7a4).fill(2); - table.subarray(0xf900, 0xfb00).fill(2); - table.subarray(0xfe10, 0xfe1a).fill(2); - table.subarray(0xfe30, 0xfe70).fill(2); - table.subarray(0xff00, 0xff61).fill(2); - table.subarray(0xffe0, 0xffe7).fill(2); + fill(table, 2, 0xac00, 0xd7a4); + fill(table, 2, 0xf900, 0xfb00); + fill(table, 2, 0xfe10, 0xfe1a); + fill(table, 2, 0xfe30, 0xfe70); + fill(table, 2, 0xff00, 0xff61); + fill(table, 2, 0xffe0, 0xffe7); // apply combining last to ensure we overwrite // wrongly wide set chars: @@ -118,7 +120,7 @@ export const wcwidth = (function(opts: {nul: number, control: number}): (ucs: nu // through to wide check so we simply do here the opposite // combining 0 for (let r = 0; r < COMBINING_BMP.length; ++r) { - table.subarray(COMBINING_BMP[r][0], COMBINING_BMP[r][1] + 1).fill(0); + fill(table, 0, COMBINING_BMP[r][0], COMBINING_BMP[r][1] + 1); } return function (num: number): number { diff --git a/src/core/TypedArrayUtils.test.ts b/src/core/TypedArrayUtils.test.ts new file mode 100644 index 00000000..2e45d371 --- /dev/null +++ b/src/core/TypedArrayUtils.test.ts @@ -0,0 +1,157 @@ +/** + * Copyright (c) 2018 The xterm.js authors. All rights reserved. + * @license MIT + */ +import { assert } from 'chai'; +import { fill } from './TypedArrayUtils'; + +type TypedArray = Uint8Array | Uint16Array | Uint32Array | Uint8ClampedArray + | Int8Array | Int16Array | Int32Array + | Float32Array | Float64Array; + +function loopFill(array: TypedArray, value: number, start: number = 0, end?: number | undefined): TypedArray { + if (start >= array.length) { + return array; + } + start = (array.length + start) % array.length; + if (end === undefined) { + end = array.length; + } + if (end >= array.length) { + end = array.length; + } else { + end = (array.length + end) % array.length; + } + for (let i = start; i < end; ++i) { + array[i] = value; + } + return array; +} + +function deepEquals(a: TypedArray, b: TypedArray) { + assert.equal(a.length, b.length); + for (let i = 0; i < a.length; ++i) { + assert.equal(a[i], b[i]); + } +} + + +describe('polyfill conformance tests', function(): void { + describe('TypedArray.fill', function(): void { + it('should work with all typed array types', function(): void { + const u8_1 = new Uint8Array(5); + const u8_2 = new Uint8Array(5); + deepEquals(fill(u8_1, 2), u8_2.fill(2)); + deepEquals(fill(u8_1, -1), u8_2.fill(-1)); + const u16_1 = new Uint16Array(5); + const u16_2 = new Uint16Array(5); + deepEquals(fill(u16_1, 2), u16_2.fill(2)); + deepEquals(fill(u16_1, 65535), u16_2.fill(65535)); + deepEquals(fill(u16_1, -1), u16_2.fill(-1)); + const u32_1 = new Uint32Array(5); + const u32_2 = new Uint32Array(5); + deepEquals(fill(u32_1, 2), u32_2.fill(2)); + deepEquals(fill(u32_1, 65537), u32_2.fill(65537)); + deepEquals(fill(u32_1, -1), u32_2.fill(-1)); + const i8_1 = new Int8Array(5); + const i8_2 = new Int8Array(5); + deepEquals(fill(i8_1, 2), i8_2.fill(2)); + deepEquals(fill(i8_1, -1), i8_2.fill(-1)); + const i16_1 = new Int16Array(5); + const i16_2 = new Int16Array(5); + deepEquals(fill(i16_1, 2), i16_2.fill(2)); + deepEquals(fill(i16_1, 65535), i16_2.fill(65535)); + deepEquals(fill(i16_1, -1), i16_2.fill(-1)); + const i32_1 = new Int32Array(5); + const i32_2 = new Int32Array(5); + deepEquals(fill(i32_1, 2), i32_2.fill(2)); + deepEquals(fill(i32_1, 65537), i32_2.fill(65537)); + deepEquals(fill(i32_1, -1), i32_2.fill(-1)); + const f32_1 = new Float32Array(5); + const f32_2 = new Float32Array(5); + deepEquals(fill(f32_1, 1.2345), f32_2.fill(1.2345)); + const f64_1 = new Float64Array(5); + const f64_2 = new Float64Array(5); + deepEquals(fill(f64_1, 1.2345), f64_2.fill(1.2345)); + const u8Clamped_1 = new Uint8ClampedArray(5); + const u8Clamped_2 = new Uint8ClampedArray(5); + deepEquals(fill(u8Clamped_1, 2), u8Clamped_2.fill(2)); + deepEquals(fill(u8Clamped_1, 257), u8Clamped_2.fill(257)); + }); + it('should work with all typed array types - explicit looping', function(): void { + const u8_1 = new Uint8Array(5); + const u8_2 = new Uint8Array(5); + deepEquals(loopFill(u8_1, 2), u8_2.fill(2)); + deepEquals(loopFill(u8_1, -1), u8_2.fill(-1)); + const u16_1 = new Uint16Array(5); + const u16_2 = new Uint16Array(5); + deepEquals(loopFill(u16_1, 2), u16_2.fill(2)); + deepEquals(loopFill(u16_1, 65535), u16_2.fill(65535)); + deepEquals(loopFill(u16_1, -1), u16_2.fill(-1)); + const u32_1 = new Uint32Array(5); + const u32_2 = new Uint32Array(5); + deepEquals(loopFill(u32_1, 2), u32_2.fill(2)); + deepEquals(loopFill(u32_1, 65537), u32_2.fill(65537)); + deepEquals(loopFill(u32_1, -1), u32_2.fill(-1)); + const i8_1 = new Int8Array(5); + const i8_2 = new Int8Array(5); + deepEquals(loopFill(i8_1, 2), i8_2.fill(2)); + deepEquals(loopFill(i8_1, -1), i8_2.fill(-1)); + const i16_1 = new Int16Array(5); + const i16_2 = new Int16Array(5); + deepEquals(loopFill(i16_1, 2), i16_2.fill(2)); + deepEquals(loopFill(i16_1, 65535), i16_2.fill(65535)); + deepEquals(loopFill(i16_1, -1), i16_2.fill(-1)); + const i32_1 = new Int32Array(5); + const i32_2 = new Int32Array(5); + deepEquals(loopFill(i32_1, 2), i32_2.fill(2)); + deepEquals(loopFill(i32_1, 65537), i32_2.fill(65537)); + deepEquals(loopFill(i32_1, -1), i32_2.fill(-1)); + const f32_1 = new Float32Array(5); + const f32_2 = new Float32Array(5); + deepEquals(loopFill(f32_1, 1.2345), f32_2.fill(1.2345)); + const f64_1 = new Float64Array(5); + const f64_2 = new Float64Array(5); + deepEquals(loopFill(f64_1, 1.2345), f64_2.fill(1.2345)); + const u8Clamped_1 = new Uint8ClampedArray(5); + const u8Clamped_2 = new Uint8ClampedArray(5); + deepEquals(loopFill(u8Clamped_1, 2), u8Clamped_2.fill(2)); + deepEquals(loopFill(u8Clamped_1, 257), u8Clamped_2.fill(257)); + }); + it('start offset', function(): void { + for (let i = -2; i < 10; ++i) { + const u8_1 = new Uint8Array(5); + const u8_2 = new Uint8Array(5); + const u8_3 = new Uint8Array(5); + deepEquals(fill(u8_1, 2, i), u8_3.fill(2, i)); + deepEquals(fill(u8_1, -1, i), u8_3.fill(-1, i)); + deepEquals(loopFill(u8_2, 2, i), u8_3.fill(2, i)); + deepEquals(loopFill(u8_2, -1, i), u8_3.fill(-1, i)); + } + }); + it('end offset', function(): void { + for (let i = -2; i < 10; ++i) { + const u8_1 = new Uint8Array(5); + const u8_2 = new Uint8Array(5); + const u8_3 = new Uint8Array(5); + deepEquals(fill(u8_1, 2, 0, i), u8_3.fill(2, 0, i)); + deepEquals(fill(u8_1, -1, 0, i), u8_3.fill(-1, 0, i)); + deepEquals(loopFill(u8_2, 2, 0, i), u8_3.fill(2, 0, i)); + deepEquals(loopFill(u8_2, -1, 0, i), u8_3.fill(-1, 0, i)); + } + }); + it('start/end offset', function(): void { + for (let i = -2; i < 10; ++i) { + for (let j = -2; j < 10; ++j) { + const u8_1 = new Uint8Array(5); + const u8_2 = new Uint8Array(5); + const u8_3 = new Uint8Array(5); + deepEquals(fill(u8_1, 2, i, j), u8_3.fill(2, i, j)); + deepEquals(fill(u8_1, -1, i, j), u8_3.fill(-1, i, j)); + deepEquals(loopFill(u8_2, 2, i, j), u8_3.fill(2, i, j)); + deepEquals(loopFill(u8_2, -1, i, j), u8_3.fill(-1, i, j)); + } + } + }); + }); +}); diff --git a/src/core/TypedArrayUtils.ts b/src/core/TypedArrayUtils.ts new file mode 100644 index 00000000..56e9d7b0 --- /dev/null +++ b/src/core/TypedArrayUtils.ts @@ -0,0 +1,40 @@ +/** + * Copyright (c) 2018 The xterm.js authors. All rights reserved. + * @license MIT + */ + +/** + * polyfill for TypedArray.fill + * This is needed to support .fill in all safari versions and IE 11. + */ + +type TypedArray = Uint8Array | Uint16Array | Uint32Array | Uint8ClampedArray + | Int8Array | Int16Array | Int32Array + | Float32Array | Float64Array; + +export function fill(array: TypedArray, value: number, start: number = 0, end?: number | undefined): TypedArray { + // all modern engines that support .fill + if (array.fill) { + return array.fill(value, start, end); + } + // safari and IE 11 + // since IE 11 does not support Array.prototype.fill either + // we cannot use the suggested polyfill from MDN + // instead we simply fall back to looping + if (start >= array.length) { + return array; + } + start = (array.length + start) % array.length; + if (end === undefined) { + end = array.length; + } + if (end >= array.length) { + end = array.length; + } else { + end = (array.length + end) % array.length; + } + for (let i = start; i < end; ++i) { + array[i] = value; + } + return array; +} From 8c33de395d48fa6ee2ff4c4c3e2de830a82e70a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= Date: Wed, 21 Nov 2018 00:15:04 +0100 Subject: [PATCH 5/7] fix linter errors --- src/core/TypedArrayUtils.test.ts | 211 ++++++++++++++++--------------- 1 file changed, 106 insertions(+), 105 deletions(-) diff --git a/src/core/TypedArrayUtils.test.ts b/src/core/TypedArrayUtils.test.ts index 2e45d371..ef86d314 100644 --- a/src/core/TypedArrayUtils.test.ts +++ b/src/core/TypedArrayUtils.test.ts @@ -9,6 +9,7 @@ type TypedArray = Uint8Array | Uint16Array | Uint32Array | Uint8ClampedArray | Int8Array | Int16Array | Int32Array | Float32Array | Float64Array; +// we explicitly test against the looping version in the test cases function loopFill(array: TypedArray, value: number, start: number = 0, end?: number | undefined): TypedArray { if (start >= array.length) { return array; @@ -28,128 +29,128 @@ function loopFill(array: TypedArray, value: number, start: number = 0, end?: num return array; } -function deepEquals(a: TypedArray, b: TypedArray) { - assert.equal(a.length, b.length); - for (let i = 0; i < a.length; ++i) { - assert.equal(a[i], b[i]); - } -} - - describe('polyfill conformance tests', function(): void { + + function deepEquals(a: TypedArray, b: TypedArray): void { + assert.equal(a.length, b.length); + for (let i = 0; i < a.length; ++i) { + assert.equal(a[i], b[i]); + } + } + describe('TypedArray.fill', function(): void { it('should work with all typed array types', function(): void { - const u8_1 = new Uint8Array(5); - const u8_2 = new Uint8Array(5); - deepEquals(fill(u8_1, 2), u8_2.fill(2)); - deepEquals(fill(u8_1, -1), u8_2.fill(-1)); - const u16_1 = new Uint16Array(5); - const u16_2 = new Uint16Array(5); - deepEquals(fill(u16_1, 2), u16_2.fill(2)); - deepEquals(fill(u16_1, 65535), u16_2.fill(65535)); - deepEquals(fill(u16_1, -1), u16_2.fill(-1)); - const u32_1 = new Uint32Array(5); - const u32_2 = new Uint32Array(5); - deepEquals(fill(u32_1, 2), u32_2.fill(2)); - deepEquals(fill(u32_1, 65537), u32_2.fill(65537)); - deepEquals(fill(u32_1, -1), u32_2.fill(-1)); - const i8_1 = new Int8Array(5); - const i8_2 = new Int8Array(5); - deepEquals(fill(i8_1, 2), i8_2.fill(2)); - deepEquals(fill(i8_1, -1), i8_2.fill(-1)); - const i16_1 = new Int16Array(5); - const i16_2 = new Int16Array(5); - deepEquals(fill(i16_1, 2), i16_2.fill(2)); - deepEquals(fill(i16_1, 65535), i16_2.fill(65535)); - deepEquals(fill(i16_1, -1), i16_2.fill(-1)); - const i32_1 = new Int32Array(5); - const i32_2 = new Int32Array(5); - deepEquals(fill(i32_1, 2), i32_2.fill(2)); - deepEquals(fill(i32_1, 65537), i32_2.fill(65537)); - deepEquals(fill(i32_1, -1), i32_2.fill(-1)); - const f32_1 = new Float32Array(5); - const f32_2 = new Float32Array(5); - deepEquals(fill(f32_1, 1.2345), f32_2.fill(1.2345)); - const f64_1 = new Float64Array(5); - const f64_2 = new Float64Array(5); - deepEquals(fill(f64_1, 1.2345), f64_2.fill(1.2345)); - const u8Clamped_1 = new Uint8ClampedArray(5); - const u8Clamped_2 = new Uint8ClampedArray(5); - deepEquals(fill(u8Clamped_1, 2), u8Clamped_2.fill(2)); - deepEquals(fill(u8Clamped_1, 257), u8Clamped_2.fill(257)); + const u81 = new Uint8Array(5); + const u82 = new Uint8Array(5); + deepEquals(fill(u81, 2), u82.fill(2)); + deepEquals(fill(u81, -1), u82.fill(-1)); + const u161 = new Uint16Array(5); + const u162 = new Uint16Array(5); + deepEquals(fill(u161, 2), u162.fill(2)); + deepEquals(fill(u161, 65535), u162.fill(65535)); + deepEquals(fill(u161, -1), u162.fill(-1)); + const u321 = new Uint32Array(5); + const u322 = new Uint32Array(5); + deepEquals(fill(u321, 2), u322.fill(2)); + deepEquals(fill(u321, 65537), u322.fill(65537)); + deepEquals(fill(u321, -1), u322.fill(-1)); + const i81 = new Int8Array(5); + const i82 = new Int8Array(5); + deepEquals(fill(i81, 2), i82.fill(2)); + deepEquals(fill(i81, -1), i82.fill(-1)); + const i161 = new Int16Array(5); + const i162 = new Int16Array(5); + deepEquals(fill(i161, 2), i162.fill(2)); + deepEquals(fill(i161, 65535), i162.fill(65535)); + deepEquals(fill(i161, -1), i162.fill(-1)); + const i321 = new Int32Array(5); + const i322 = new Int32Array(5); + deepEquals(fill(i321, 2), i322.fill(2)); + deepEquals(fill(i321, 65537), i322.fill(65537)); + deepEquals(fill(i321, -1), i322.fill(-1)); + const f321 = new Float32Array(5); + const f322 = new Float32Array(5); + deepEquals(fill(f321, 1.2345), f322.fill(1.2345)); + const f641 = new Float64Array(5); + const f642 = new Float64Array(5); + deepEquals(fill(f641, 1.2345), f642.fill(1.2345)); + const u8Clamped1 = new Uint8ClampedArray(5); + const u8Clamped2 = new Uint8ClampedArray(5); + deepEquals(fill(u8Clamped1, 2), u8Clamped2.fill(2)); + deepEquals(fill(u8Clamped1, 257), u8Clamped2.fill(257)); }); it('should work with all typed array types - explicit looping', function(): void { - const u8_1 = new Uint8Array(5); - const u8_2 = new Uint8Array(5); - deepEquals(loopFill(u8_1, 2), u8_2.fill(2)); - deepEquals(loopFill(u8_1, -1), u8_2.fill(-1)); - const u16_1 = new Uint16Array(5); - const u16_2 = new Uint16Array(5); - deepEquals(loopFill(u16_1, 2), u16_2.fill(2)); - deepEquals(loopFill(u16_1, 65535), u16_2.fill(65535)); - deepEquals(loopFill(u16_1, -1), u16_2.fill(-1)); - const u32_1 = new Uint32Array(5); - const u32_2 = new Uint32Array(5); - deepEquals(loopFill(u32_1, 2), u32_2.fill(2)); - deepEquals(loopFill(u32_1, 65537), u32_2.fill(65537)); - deepEquals(loopFill(u32_1, -1), u32_2.fill(-1)); - const i8_1 = new Int8Array(5); - const i8_2 = new Int8Array(5); - deepEquals(loopFill(i8_1, 2), i8_2.fill(2)); - deepEquals(loopFill(i8_1, -1), i8_2.fill(-1)); - const i16_1 = new Int16Array(5); - const i16_2 = new Int16Array(5); - deepEquals(loopFill(i16_1, 2), i16_2.fill(2)); - deepEquals(loopFill(i16_1, 65535), i16_2.fill(65535)); - deepEquals(loopFill(i16_1, -1), i16_2.fill(-1)); - const i32_1 = new Int32Array(5); - const i32_2 = new Int32Array(5); - deepEquals(loopFill(i32_1, 2), i32_2.fill(2)); - deepEquals(loopFill(i32_1, 65537), i32_2.fill(65537)); - deepEquals(loopFill(i32_1, -1), i32_2.fill(-1)); - const f32_1 = new Float32Array(5); - const f32_2 = new Float32Array(5); - deepEquals(loopFill(f32_1, 1.2345), f32_2.fill(1.2345)); - const f64_1 = new Float64Array(5); - const f64_2 = new Float64Array(5); - deepEquals(loopFill(f64_1, 1.2345), f64_2.fill(1.2345)); - const u8Clamped_1 = new Uint8ClampedArray(5); - const u8Clamped_2 = new Uint8ClampedArray(5); - deepEquals(loopFill(u8Clamped_1, 2), u8Clamped_2.fill(2)); - deepEquals(loopFill(u8Clamped_1, 257), u8Clamped_2.fill(257)); + const u81 = new Uint8Array(5); + const u82 = new Uint8Array(5); + deepEquals(loopFill(u81, 2), u82.fill(2)); + deepEquals(loopFill(u81, -1), u82.fill(-1)); + const u161 = new Uint16Array(5); + const u162 = new Uint16Array(5); + deepEquals(loopFill(u161, 2), u162.fill(2)); + deepEquals(loopFill(u161, 65535), u162.fill(65535)); + deepEquals(loopFill(u161, -1), u162.fill(-1)); + const u321 = new Uint32Array(5); + const u322 = new Uint32Array(5); + deepEquals(loopFill(u321, 2), u322.fill(2)); + deepEquals(loopFill(u321, 65537), u322.fill(65537)); + deepEquals(loopFill(u321, -1), u322.fill(-1)); + const i81 = new Int8Array(5); + const i82 = new Int8Array(5); + deepEquals(loopFill(i81, 2), i82.fill(2)); + deepEquals(loopFill(i81, -1), i82.fill(-1)); + const i161 = new Int16Array(5); + const i162 = new Int16Array(5); + deepEquals(loopFill(i161, 2), i162.fill(2)); + deepEquals(loopFill(i161, 65535), i162.fill(65535)); + deepEquals(loopFill(i161, -1), i162.fill(-1)); + const i321 = new Int32Array(5); + const i322 = new Int32Array(5); + deepEquals(loopFill(i321, 2), i322.fill(2)); + deepEquals(loopFill(i321, 65537), i322.fill(65537)); + deepEquals(loopFill(i321, -1), i322.fill(-1)); + const f321 = new Float32Array(5); + const f322 = new Float32Array(5); + deepEquals(loopFill(f321, 1.2345), f322.fill(1.2345)); + const f641 = new Float64Array(5); + const f642 = new Float64Array(5); + deepEquals(loopFill(f641, 1.2345), f642.fill(1.2345)); + const u8Clamped1 = new Uint8ClampedArray(5); + const u8Clamped2 = new Uint8ClampedArray(5); + deepEquals(loopFill(u8Clamped1, 2), u8Clamped2.fill(2)); + deepEquals(loopFill(u8Clamped1, 257), u8Clamped2.fill(257)); }); it('start offset', function(): void { for (let i = -2; i < 10; ++i) { - const u8_1 = new Uint8Array(5); - const u8_2 = new Uint8Array(5); - const u8_3 = new Uint8Array(5); - deepEquals(fill(u8_1, 2, i), u8_3.fill(2, i)); - deepEquals(fill(u8_1, -1, i), u8_3.fill(-1, i)); - deepEquals(loopFill(u8_2, 2, i), u8_3.fill(2, i)); - deepEquals(loopFill(u8_2, -1, i), u8_3.fill(-1, i)); + const u81 = new Uint8Array(5); + const u82 = new Uint8Array(5); + const u83 = new Uint8Array(5); + deepEquals(fill(u81, 2, i), u83.fill(2, i)); + deepEquals(fill(u81, -1, i), u83.fill(-1, i)); + deepEquals(loopFill(u82, 2, i), u83.fill(2, i)); + deepEquals(loopFill(u82, -1, i), u83.fill(-1, i)); } }); it('end offset', function(): void { for (let i = -2; i < 10; ++i) { - const u8_1 = new Uint8Array(5); - const u8_2 = new Uint8Array(5); - const u8_3 = new Uint8Array(5); - deepEquals(fill(u8_1, 2, 0, i), u8_3.fill(2, 0, i)); - deepEquals(fill(u8_1, -1, 0, i), u8_3.fill(-1, 0, i)); - deepEquals(loopFill(u8_2, 2, 0, i), u8_3.fill(2, 0, i)); - deepEquals(loopFill(u8_2, -1, 0, i), u8_3.fill(-1, 0, i)); + const u81 = new Uint8Array(5); + const u82 = new Uint8Array(5); + const u83 = new Uint8Array(5); + deepEquals(fill(u81, 2, 0, i), u83.fill(2, 0, i)); + deepEquals(fill(u81, -1, 0, i), u83.fill(-1, 0, i)); + deepEquals(loopFill(u82, 2, 0, i), u83.fill(2, 0, i)); + deepEquals(loopFill(u82, -1, 0, i), u83.fill(-1, 0, i)); } }); it('start/end offset', function(): void { for (let i = -2; i < 10; ++i) { for (let j = -2; j < 10; ++j) { - const u8_1 = new Uint8Array(5); - const u8_2 = new Uint8Array(5); - const u8_3 = new Uint8Array(5); - deepEquals(fill(u8_1, 2, i, j), u8_3.fill(2, i, j)); - deepEquals(fill(u8_1, -1, i, j), u8_3.fill(-1, i, j)); - deepEquals(loopFill(u8_2, 2, i, j), u8_3.fill(2, i, j)); - deepEquals(loopFill(u8_2, -1, i, j), u8_3.fill(-1, i, j)); + const u81 = new Uint8Array(5); + const u82 = new Uint8Array(5); + const u83 = new Uint8Array(5); + deepEquals(fill(u81, 2, i, j), u83.fill(2, i, j)); + deepEquals(fill(u81, -1, i, j), u83.fill(-1, i, j)); + deepEquals(loopFill(u82, 2, i, j), u83.fill(2, i, j)); + deepEquals(loopFill(u82, -1, i, j), u83.fill(-1, i, j)); } } }); From dc0ae4a8156bd0cfff6e1884065757c4c9b9098f Mon Sep 17 00:00:00 2001 From: Daniel Imms Date: Fri, 23 Nov 2018 06:24:27 -0800 Subject: [PATCH 6/7] Fix indentation --- src/CharWidth.ts | 248 +++++++++++++++++++++++------------------------ 1 file changed, 124 insertions(+), 124 deletions(-) diff --git a/src/CharWidth.ts b/src/CharWidth.ts index 341b347a..5a484da8 100644 --- a/src/CharWidth.ts +++ b/src/CharWidth.ts @@ -6,136 +6,136 @@ import { fill } from './core/TypedArrayUtils'; */ export const wcwidth = (function(opts: {nul: number, control: number}): (ucs: number) => number { - // extracted from https://www.cl.cam.ac.uk/%7Emgk25/ucs/wcwidth.c - // combining characters - const COMBINING_BMP = [ - [0x0300, 0x036F], [0x0483, 0x0486], [0x0488, 0x0489], - [0x0591, 0x05BD], [0x05BF, 0x05BF], [0x05C1, 0x05C2], - [0x05C4, 0x05C5], [0x05C7, 0x05C7], [0x0600, 0x0603], - [0x0610, 0x0615], [0x064B, 0x065E], [0x0670, 0x0670], - [0x06D6, 0x06E4], [0x06E7, 0x06E8], [0x06EA, 0x06ED], - [0x070F, 0x070F], [0x0711, 0x0711], [0x0730, 0x074A], - [0x07A6, 0x07B0], [0x07EB, 0x07F3], [0x0901, 0x0902], - [0x093C, 0x093C], [0x0941, 0x0948], [0x094D, 0x094D], - [0x0951, 0x0954], [0x0962, 0x0963], [0x0981, 0x0981], - [0x09BC, 0x09BC], [0x09C1, 0x09C4], [0x09CD, 0x09CD], - [0x09E2, 0x09E3], [0x0A01, 0x0A02], [0x0A3C, 0x0A3C], - [0x0A41, 0x0A42], [0x0A47, 0x0A48], [0x0A4B, 0x0A4D], - [0x0A70, 0x0A71], [0x0A81, 0x0A82], [0x0ABC, 0x0ABC], - [0x0AC1, 0x0AC5], [0x0AC7, 0x0AC8], [0x0ACD, 0x0ACD], - [0x0AE2, 0x0AE3], [0x0B01, 0x0B01], [0x0B3C, 0x0B3C], - [0x0B3F, 0x0B3F], [0x0B41, 0x0B43], [0x0B4D, 0x0B4D], - [0x0B56, 0x0B56], [0x0B82, 0x0B82], [0x0BC0, 0x0BC0], - [0x0BCD, 0x0BCD], [0x0C3E, 0x0C40], [0x0C46, 0x0C48], - [0x0C4A, 0x0C4D], [0x0C55, 0x0C56], [0x0CBC, 0x0CBC], - [0x0CBF, 0x0CBF], [0x0CC6, 0x0CC6], [0x0CCC, 0x0CCD], - [0x0CE2, 0x0CE3], [0x0D41, 0x0D43], [0x0D4D, 0x0D4D], - [0x0DCA, 0x0DCA], [0x0DD2, 0x0DD4], [0x0DD6, 0x0DD6], - [0x0E31, 0x0E31], [0x0E34, 0x0E3A], [0x0E47, 0x0E4E], - [0x0EB1, 0x0EB1], [0x0EB4, 0x0EB9], [0x0EBB, 0x0EBC], - [0x0EC8, 0x0ECD], [0x0F18, 0x0F19], [0x0F35, 0x0F35], - [0x0F37, 0x0F37], [0x0F39, 0x0F39], [0x0F71, 0x0F7E], - [0x0F80, 0x0F84], [0x0F86, 0x0F87], [0x0F90, 0x0F97], - [0x0F99, 0x0FBC], [0x0FC6, 0x0FC6], [0x102D, 0x1030], - [0x1032, 0x1032], [0x1036, 0x1037], [0x1039, 0x1039], - [0x1058, 0x1059], [0x1160, 0x11FF], [0x135F, 0x135F], - [0x1712, 0x1714], [0x1732, 0x1734], [0x1752, 0x1753], - [0x1772, 0x1773], [0x17B4, 0x17B5], [0x17B7, 0x17BD], - [0x17C6, 0x17C6], [0x17C9, 0x17D3], [0x17DD, 0x17DD], - [0x180B, 0x180D], [0x18A9, 0x18A9], [0x1920, 0x1922], - [0x1927, 0x1928], [0x1932, 0x1932], [0x1939, 0x193B], - [0x1A17, 0x1A18], [0x1B00, 0x1B03], [0x1B34, 0x1B34], - [0x1B36, 0x1B3A], [0x1B3C, 0x1B3C], [0x1B42, 0x1B42], - [0x1B6B, 0x1B73], [0x1DC0, 0x1DCA], [0x1DFE, 0x1DFF], - [0x200B, 0x200F], [0x202A, 0x202E], [0x2060, 0x2063], - [0x206A, 0x206F], [0x20D0, 0x20EF], [0x302A, 0x302F], - [0x3099, 0x309A], [0xA806, 0xA806], [0xA80B, 0xA80B], - [0xA825, 0xA826], [0xFB1E, 0xFB1E], [0xFE00, 0xFE0F], - [0xFE20, 0xFE23], [0xFEFF, 0xFEFF], [0xFFF9, 0xFFFB] - ]; - const COMBINING_HIGH = [ - [0x10A01, 0x10A03], [0x10A05, 0x10A06], [0x10A0C, 0x10A0F], - [0x10A38, 0x10A3A], [0x10A3F, 0x10A3F], [0x1D167, 0x1D169], - [0x1D173, 0x1D182], [0x1D185, 0x1D18B], [0x1D1AA, 0x1D1AD], - [0x1D242, 0x1D244], [0xE0001, 0xE0001], [0xE0020, 0xE007F], - [0xE0100, 0xE01EF] - ]; - // binary search - function bisearch(ucs: number, data: number[][]): boolean { - let min = 0; - let max = data.length - 1; - let mid; - if (ucs < data[0][0] || ucs > data[max][1]) { - return false; - } - while (max >= min) { - mid = (min + max) >> 1; - if (ucs > data[mid][1]) { - min = mid + 1; - } else if (ucs < data[mid][0]) { - max = mid - 1; - } else { - return true; - } - } + // extracted from https://www.cl.cam.ac.uk/%7Emgk25/ucs/wcwidth.c + // combining characters + const COMBINING_BMP = [ + [0x0300, 0x036F], [0x0483, 0x0486], [0x0488, 0x0489], + [0x0591, 0x05BD], [0x05BF, 0x05BF], [0x05C1, 0x05C2], + [0x05C4, 0x05C5], [0x05C7, 0x05C7], [0x0600, 0x0603], + [0x0610, 0x0615], [0x064B, 0x065E], [0x0670, 0x0670], + [0x06D6, 0x06E4], [0x06E7, 0x06E8], [0x06EA, 0x06ED], + [0x070F, 0x070F], [0x0711, 0x0711], [0x0730, 0x074A], + [0x07A6, 0x07B0], [0x07EB, 0x07F3], [0x0901, 0x0902], + [0x093C, 0x093C], [0x0941, 0x0948], [0x094D, 0x094D], + [0x0951, 0x0954], [0x0962, 0x0963], [0x0981, 0x0981], + [0x09BC, 0x09BC], [0x09C1, 0x09C4], [0x09CD, 0x09CD], + [0x09E2, 0x09E3], [0x0A01, 0x0A02], [0x0A3C, 0x0A3C], + [0x0A41, 0x0A42], [0x0A47, 0x0A48], [0x0A4B, 0x0A4D], + [0x0A70, 0x0A71], [0x0A81, 0x0A82], [0x0ABC, 0x0ABC], + [0x0AC1, 0x0AC5], [0x0AC7, 0x0AC8], [0x0ACD, 0x0ACD], + [0x0AE2, 0x0AE3], [0x0B01, 0x0B01], [0x0B3C, 0x0B3C], + [0x0B3F, 0x0B3F], [0x0B41, 0x0B43], [0x0B4D, 0x0B4D], + [0x0B56, 0x0B56], [0x0B82, 0x0B82], [0x0BC0, 0x0BC0], + [0x0BCD, 0x0BCD], [0x0C3E, 0x0C40], [0x0C46, 0x0C48], + [0x0C4A, 0x0C4D], [0x0C55, 0x0C56], [0x0CBC, 0x0CBC], + [0x0CBF, 0x0CBF], [0x0CC6, 0x0CC6], [0x0CCC, 0x0CCD], + [0x0CE2, 0x0CE3], [0x0D41, 0x0D43], [0x0D4D, 0x0D4D], + [0x0DCA, 0x0DCA], [0x0DD2, 0x0DD4], [0x0DD6, 0x0DD6], + [0x0E31, 0x0E31], [0x0E34, 0x0E3A], [0x0E47, 0x0E4E], + [0x0EB1, 0x0EB1], [0x0EB4, 0x0EB9], [0x0EBB, 0x0EBC], + [0x0EC8, 0x0ECD], [0x0F18, 0x0F19], [0x0F35, 0x0F35], + [0x0F37, 0x0F37], [0x0F39, 0x0F39], [0x0F71, 0x0F7E], + [0x0F80, 0x0F84], [0x0F86, 0x0F87], [0x0F90, 0x0F97], + [0x0F99, 0x0FBC], [0x0FC6, 0x0FC6], [0x102D, 0x1030], + [0x1032, 0x1032], [0x1036, 0x1037], [0x1039, 0x1039], + [0x1058, 0x1059], [0x1160, 0x11FF], [0x135F, 0x135F], + [0x1712, 0x1714], [0x1732, 0x1734], [0x1752, 0x1753], + [0x1772, 0x1773], [0x17B4, 0x17B5], [0x17B7, 0x17BD], + [0x17C6, 0x17C6], [0x17C9, 0x17D3], [0x17DD, 0x17DD], + [0x180B, 0x180D], [0x18A9, 0x18A9], [0x1920, 0x1922], + [0x1927, 0x1928], [0x1932, 0x1932], [0x1939, 0x193B], + [0x1A17, 0x1A18], [0x1B00, 0x1B03], [0x1B34, 0x1B34], + [0x1B36, 0x1B3A], [0x1B3C, 0x1B3C], [0x1B42, 0x1B42], + [0x1B6B, 0x1B73], [0x1DC0, 0x1DCA], [0x1DFE, 0x1DFF], + [0x200B, 0x200F], [0x202A, 0x202E], [0x2060, 0x2063], + [0x206A, 0x206F], [0x20D0, 0x20EF], [0x302A, 0x302F], + [0x3099, 0x309A], [0xA806, 0xA806], [0xA80B, 0xA80B], + [0xA825, 0xA826], [0xFB1E, 0xFB1E], [0xFE00, 0xFE0F], + [0xFE20, 0xFE23], [0xFEFF, 0xFEFF], [0xFFF9, 0xFFFB] + ]; + const COMBINING_HIGH = [ + [0x10A01, 0x10A03], [0x10A05, 0x10A06], [0x10A0C, 0x10A0F], + [0x10A38, 0x10A3A], [0x10A3F, 0x10A3F], [0x1D167, 0x1D169], + [0x1D173, 0x1D182], [0x1D185, 0x1D18B], [0x1D1AA, 0x1D1AD], + [0x1D242, 0x1D244], [0xE0001, 0xE0001], [0xE0020, 0xE007F], + [0xE0100, 0xE01EF] + ]; + // binary search + function bisearch(ucs: number, data: number[][]): boolean { + let min = 0; + let max = data.length - 1; + let mid; + if (ucs < data[0][0] || ucs > data[max][1]) { return false; } - function wcwidthHigh(ucs: number): 0 | 1 | 2 { - if (bisearch(ucs, COMBINING_HIGH)) { - return 0; - } - if ((ucs >= 0x20000 && ucs <= 0x2fffd) || (ucs >= 0x30000 && ucs <= 0x3fffd)) { - return 2; + while (max >= min) { + mid = (min + max) >> 1; + if (ucs > data[mid][1]) { + min = mid + 1; + } else if (ucs < data[mid][0]) { + max = mid - 1; + } else { + return true; } + } + return false; + } + function wcwidthHigh(ucs: number): 0 | 1 | 2 { + if (bisearch(ucs, COMBINING_HIGH)) { + return 0; + } + if ((ucs >= 0x20000 && ucs <= 0x2fffd) || (ucs >= 0x30000 && ucs <= 0x3fffd)) { + return 2; + } + return 1; + } + const control = opts.control | 0; + + // create lookup table for BMP plane + const table = new Uint8Array(65536); + fill(table, 1); + table[0] = opts.nul; + // control chars + fill(table, opts.control, 1, 32); + fill(table, opts.control, 0x7f, 0xa0); + + // apply wide char rules first + // wide chars + fill(table, 2, 0x1100, 0x1160); + table[0x2329] = 2; + table[0x232a] = 2; + fill(table, 2, 0x2e80, 0xa4d0); + table[0x303f] = 1; // wrongly in last line + + fill(table, 2, 0xac00, 0xd7a4); + fill(table, 2, 0xf900, 0xfb00); + fill(table, 2, 0xfe10, 0xfe1a); + fill(table, 2, 0xfe30, 0xfe70); + fill(table, 2, 0xff00, 0xff61); + fill(table, 2, 0xffe0, 0xffe7); + + // apply combining last to ensure we overwrite + // wrongly wide set chars: + // the original algo evals combining first and falls + // through to wide check so we simply do here the opposite + // combining 0 + for (let r = 0; r < COMBINING_BMP.length; ++r) { + fill(table, 0, COMBINING_BMP[r][0], COMBINING_BMP[r][1] + 1); + } + + return function (num: number): number { + if (num < 32) { + return control | 0; + } + if (num < 127) { return 1; } - const control = opts.control | 0; - - // create lookup table for BMP plane - const table = new Uint8Array(65536); - fill(table, 1); - table[0] = opts.nul; - // control chars - fill(table, opts.control, 1, 32); - fill(table, opts.control, 0x7f, 0xa0); - - // apply wide char rules first - // wide chars - fill(table, 2, 0x1100, 0x1160); - table[0x2329] = 2; - table[0x232a] = 2; - fill(table, 2, 0x2e80, 0xa4d0); - table[0x303f] = 1; // wrongly in last line - - fill(table, 2, 0xac00, 0xd7a4); - fill(table, 2, 0xf900, 0xfb00); - fill(table, 2, 0xfe10, 0xfe1a); - fill(table, 2, 0xfe30, 0xfe70); - fill(table, 2, 0xff00, 0xff61); - fill(table, 2, 0xffe0, 0xffe7); - - // apply combining last to ensure we overwrite - // wrongly wide set chars: - // the original algo evals combining first and falls - // through to wide check so we simply do here the opposite - // combining 0 - for (let r = 0; r < COMBINING_BMP.length; ++r) { - fill(table, 0, COMBINING_BMP[r][0], COMBINING_BMP[r][1] + 1); + if (num < 65536) { + return table[num]; } - - return function (num: number): number { - if (num < 32) { - return control | 0; - } - if (num < 127) { - return 1; - } - if (num < 65536) { - return table[num]; - } - // do a full search for high codepoints - return wcwidthHigh(num); - }; + // do a full search for high codepoints + return wcwidthHigh(num); + }; })({nul: 0, control: 0}); // configurable options /** From 65c5df92d913936a9902acc665229293a41acd27 Mon Sep 17 00:00:00 2001 From: Daniel Imms Date: Fri, 23 Nov 2018 06:26:06 -0800 Subject: [PATCH 7/7] Move import below copyright --- src/CharWidth.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CharWidth.ts b/src/CharWidth.ts index 5a484da8..fd6ac55f 100644 --- a/src/CharWidth.ts +++ b/src/CharWidth.ts @@ -1,10 +1,10 @@ -import { fill } from './core/TypedArrayUtils'; - /** * Copyright (c) 2016 The xterm.js authors. All rights reserved. * @license MIT */ +import { fill } from './core/TypedArrayUtils'; + export const wcwidth = (function(opts: {nul: number, control: number}): (ucs: number) => number { // extracted from https://www.cl.cam.ac.uk/%7Emgk25/ucs/wcwidth.c // combining characters