waveterm/xterm.js
0
0
Fork 0
mirror of https://github.com/wavetermdev/xterm.js.git synced 2026-08-05 13:43:48 -07:00
Code Issues Packages Projects Releases Wiki Activity

add TextDecoder for string to UTF32

Browse Source
This commit is contained in:
Jörg Breitbart
2019-01-02 22:50:15 +01:00
parent 5fd764a89d
commit dfc853aef4
3 changed files with 169 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/EscapeSequenceParser.test.ts
+15 -15
View File
@@ -1192,7 +1192,7 @@ describe('EscapeSequenceParser', function (): void {
const csiCustom: [string, number[], string][] = [];
parser2.setCsiHandler('m', (params, collect) => csi.push(['m', params, collect]));
parser2.addCsiHandler('m', (params, collect) => { csiCustom.push(['m', params, collect]); return true; });
parser2.parse(INPUT);
parse(parser2, INPUT);
chai.expect(csi).eql([], 'Should not fallback to original handler');
chai.expect(csiCustom).eql([['m', [1, 31], ''], ['m', [0], '']]);
});
@@ -1200,7 +1200,7 @@ describe('EscapeSequenceParser', function (): void {
const csiCustom: [string, number[], string][] = [];
parser2.setCsiHandler('m', (params, collect) => csi.push(['m', params, collect]));
parser2.addCsiHandler('m', (params, collect) => { csiCustom.push(['m', params, collect]); return false; });
parser2.parse(INPUT);
parse(parser2, INPUT);
chai.expect(csi).eql([['m', [1, 31], ''], ['m', [0], '']], 'Should fallback to original handler');
chai.expect(csiCustom).eql([['m', [1, 31], ''], ['m', [0], '']]);
});
@@ -1210,7 +1210,7 @@ describe('EscapeSequenceParser', function (): void {
parser2.setCsiHandler('m', (params, collect) => csi.push(['m', params, collect]));
parser2.addCsiHandler('m', (params, collect) => { csiCustom.push(['m', params, collect]); return true; });
parser2.addCsiHandler('m', (params, collect) => { csiCustom2.push(['m', params, collect]); return false; });
parser2.parse(INPUT);
parse(parser2, INPUT);
chai.expect(csi).eql([], 'Should not fallback to original handler');
chai.expect(csiCustom).eql([['m', [1, 31], ''], ['m', [0], '']]);
chai.expect(csiCustom2).eql([['m', [1, 31], ''], ['m', [0], '']]);
@@ -1221,7 +1221,7 @@ describe('EscapeSequenceParser', function (): void {
parser2.setCsiHandler('m', (params, collect) => csi.push(['m', params, collect]));
parser2.addCsiHandler('m', (params, collect) => { csiCustom.push(['m', params, collect]); return true; });
parser2.addCsiHandler('m', (params, collect) => { csiCustom2.push(['m', params, collect]); return true; });
parser2.parse(INPUT);
parse(parser2, INPUT);
chai.expect(csi).eql([], 'Should not fallback to original handler');
chai.expect(csiCustom).eql([], 'Should not fallback once');
chai.expect(csiCustom2).eql([['m', [1, 31], ''], ['m', [0], '']]);
@@ -1231,15 +1231,15 @@ describe('EscapeSequenceParser', function (): void {
parser2.setCsiHandler('m', () => order.push(1));
parser2.addCsiHandler('m', () => { order.push(2); return false; });
parser2.addCsiHandler('m', () => { order.push(3); return false; });
parser2.parse('\x1b[0m');
chai.expect(order).eql([3, 2, 1]);
parse(parser2, INPUT);
chai.expect(order).eql([3, 2, 1, 3, 2, 1]);
});
it('Dispose should work', () => {
const csiCustom: [string, number[], string][] = [];
parser2.setCsiHandler('m', (params, collect) => csi.push(['m', params, collect]));
const customHandler = parser2.addCsiHandler('m', (params, collect) => { csiCustom.push(['m', params, collect]); return true; });
customHandler.dispose();
parser2.parse(INPUT);
parse(parser2, INPUT);
chai.expect(csi).eql([['m', [1, 31], ''], ['m', [0], '']]);
chai.expect(csiCustom).eql([], 'Should not use custom handler as it was disposed');
});
@@ -1249,7 +1249,7 @@ describe('EscapeSequenceParser', function (): void {
const customHandler = parser2.addCsiHandler('m', (params, collect) => { csiCustom.push(['m', params, collect]); return true; });
customHandler.dispose();
customHandler.dispose();
parser2.parse(INPUT);
parse(parser2, INPUT);
chai.expect(csi).eql([['m', [1, 31], ''], ['m', [0], '']]);
chai.expect(csiCustom).eql([], 'Should not use custom handler as it was disposed');
});
@@ -1286,7 +1286,7 @@ describe('EscapeSequenceParser', function (): void {
const oscCustom: [number, string][] = [];
parser2.setOscHandler(1, data => osc.push([1, data]));
parser2.addOscHandler(1, data => { oscCustom.push([1, data]); return true; });
parser2.parse(INPUT);
parse(parser2, INPUT);
chai.expect(osc).eql([], 'Should not fallback to original handler');
chai.expect(oscCustom).eql([[1, 'foo=bar']]);
});
@@ -1294,7 +1294,7 @@ describe('EscapeSequenceParser', function (): void {
const oscCustom: [number, string][] = [];
parser2.setOscHandler(1, data => osc.push([1, data]));
parser2.addOscHandler(1, data => { oscCustom.push([1, data]); return false; });
parser2.parse(INPUT);
parse(parser2, INPUT);
chai.expect(osc).eql([[1, 'foo=bar']], 'Should fallback to original handler');
chai.expect(oscCustom).eql([[1, 'foo=bar']]);
});
@@ -1304,7 +1304,7 @@ describe('EscapeSequenceParser', function (): void {
parser2.setOscHandler(1, data => osc.push([1, data]));
parser2.addOscHandler(1, data => { oscCustom.push([1, data]); return true; });
parser2.addOscHandler(1, data => { oscCustom2.push([1, data]); return false; });
parser2.parse(INPUT);
parse(parser2, INPUT);
chai.expect(osc).eql([], 'Should not fallback to original handler');
chai.expect(oscCustom).eql([[1, 'foo=bar']]);
chai.expect(oscCustom2).eql([[1, 'foo=bar']]);
@@ -1315,7 +1315,7 @@ describe('EscapeSequenceParser', function (): void {
parser2.setOscHandler(1, data => osc.push([1, data]));
parser2.addOscHandler(1, data => { oscCustom.push([1, data]); return true; });
parser2.addOscHandler(1, data => { oscCustom2.push([1, data]); return true; });
parser2.parse(INPUT);
parse(parser2, INPUT);
chai.expect(osc).eql([], 'Should not fallback to original handler');
chai.expect(oscCustom).eql([], 'Should not fallback once');
chai.expect(oscCustom2).eql([[1, 'foo=bar']]);
@@ -1325,7 +1325,7 @@ describe('EscapeSequenceParser', function (): void {
parser2.setOscHandler(1, () => order.push(1));
parser2.addOscHandler(1, () => { order.push(2); return false; });
parser2.addOscHandler(1, () => { order.push(3); return false; });
parser2.parse('\x1b]1;foo=bar\x1b\\');
parse(parser2, '\x1b]1;foo=bar\x1b\\');
chai.expect(order).eql([3, 2, 1]);
});
it('Dispose should work', () => {
@@ -1333,7 +1333,7 @@ describe('EscapeSequenceParser', function (): void {
parser2.setOscHandler(1, data => osc.push([1, data]));
const customHandler = parser2.addOscHandler(1, data => { oscCustom.push([1, data]); return true; });
customHandler.dispose();
parser2.parse(INPUT);
parse(parser2, INPUT);
chai.expect(osc).eql([[1, 'foo=bar']]);
chai.expect(oscCustom).eql([], 'Should not use custom handler as it was disposed');
});
@@ -1343,7 +1343,7 @@ describe('EscapeSequenceParser', function (): void {
const customHandler = parser2.addOscHandler(1, data => { oscCustom.push([1, data]); return true; });
customHandler.dispose();
customHandler.dispose();
parser2.parse(INPUT);
parse(parser2, INPUT);
chai.expect(osc).eql([[1, 'foo=bar']]);
chai.expect(oscCustom).eql([], 'Should not use custom handler as it was disposed');
});
src/core/input/TextDecoder.test.ts
+67
View File
@@ -0,0 +1,67 @@
/**
* Copyright (c) 2019 The xterm.js authors. All rights reserved.
* @license MIT
*/
import { assert } from 'chai';
import { StringToUtf32, stringFromCodePoint } from './TextDecoder';
// convert UTF32 codepoints to string
function toString(data: Uint32Array, length: number): string {
if ((String as any).fromCodePoint) {
return (String as any).fromCodePoint.apply(null, data.subarray(0, length));
}
let result = '';
for (let i = 0; i < length; ++i) {
result += stringFromCodePoint(data[i]);
}
return result;
}
describe('StringToUtf32 Decoder', () => {
describe('full codepoint test', () => {
it('0..65535', () => {
const decoder = new StringToUtf32();
const target = new Uint32Array(5);
for (let i = 0; i < 65536; ++i) {
// skip surrogate pairs
if (i >= 0xD800 && i <= 0xDFFF) {
continue;
}
const length = decoder.decode(String.fromCharCode(i), target);
assert.equal(length, 1);
assert.equal(target[0], i);
assert.equal(toString(target, length), String.fromCharCode(i));
decoder.clear();
}
});
it('65536..0x10FFFF (surrogates)', function(): void {
this.timeout(20000);
const decoder = new StringToUtf32();
const target = new Uint32Array(5);
for (let i = 65536; i < 0x10FFFF; ++i) {
const codePoint = i - 0x10000;
const s = String.fromCharCode((codePoint >> 10) + 0xD800) + String.fromCharCode((codePoint % 0x400) + 0xDC00);
const length = decoder.decode(s, target);
assert.equal(length, 1);
assert.equal(target[0], i);
assert.equal(toString(target, length), s);
decoder.clear();
}
});
});
describe('stream handling', () => {
it('surrogates mixed advance by 1', () => {
const decoder = new StringToUtf32();
const target = new Uint32Array(5);
const input = 'Ä€𝄞Ö𝄞€Ü𝄞€';
let decoded = '';
for (let i = 0; i < input.length; ++i) {
const written = decoder.decode(input[i], target);
decoded += toString(target, written);
}
assert(decoded, 'Ä€𝄞Ö𝄞€Ü𝄞€');
});
});
});
src/core/input/TextDecoder.ts
+87
View File
@@ -0,0 +1,87 @@
/**
* Copyright (c) 2019 The xterm.js authors. All rights reserved.
* @license MIT
*/
/**
* StringToUtf32 - decodes UTF16 sequences into UTF32 codepoints.
* To keep the decoder in line with JS strings it handles single surrogates as UCS2.
*/
export class StringToUtf32 {
private _interim: number = 0;
/**
* Clears interim and resets decoder to clean state.
*/
public clear(): void {
this._interim = 0;
}
/**
* Decode JS string to UTF32 codepoints.
* The methods assumes stream input and will store partly transmitted
* surrogate pairs and decode them with the next data chunk.
* Note: The method does no bound checks for target, therefore make sure
* the provided input data does not exceed the size of `target`.
* Returns the number of written codepoints in `target`.
*/
decode(input: string, target: Uint32Array): number {
const length = input.length;
if (!length) {
return 0;
}
let size = 0;
let startPos = 0;
// handle leftover surrogate high
if (this._interim) {
const second = input.charCodeAt(startPos++);
if (0xDC00 <= second && second <= 0xDFFF) {
target[size++] = (this._interim - 0xD800) * 0x400 + second - 0xDC00 + 0x10000;
} else {
// illegal codepoint (USC2 handling)
target[size++] = this._interim;
target[size++] = second;
}
this._interim = 0;
}
for (let i = startPos; i < length; ++i) {
const code = input.charCodeAt(i);
// surrogate pair first
if (0xD800 <= code && code <= 0xDBFF) {
if (++i >= length) {
this._interim = code;
return size;
}
const second = input.charCodeAt(i);
if (0xDC00 <= second && second <= 0xDFFF) {
target[size++] = (code - 0xD800) * 0x400 + second - 0xDC00 + 0x10000;
} else {
// illegal codepoint (USC2 handling)
target[size++] = code;
target[size++] = second;
}
continue;
}
target[size++] = code;
}
return size;
}
}
/**
* Polyfill - Convert UTF32 codepoint into JS string.
*/
export function stringFromCodePoint(codePoint: number): string {
if ((String as any).fromCodePoint) {
return (String as any).fromCodePoint(codePoint);
}
if (codePoint > 0xFFFF) {
codePoint -= 0x10000;
return String.fromCharCode((codePoint >> 10) + 0xD800) + String.fromCharCode((codePoint % 0x400) + 0xDC00);
}
return String.fromCharCode(codePoint);
}