mirror of
https://github.com/wavetermdev/xterm.js.git
synced 2026-08-05 13:43:48 -07:00
Merge pull request #4519 from PerBothner/clusters
New unicode-graphemes addon.
This commit is contained in:
Daniel Imms
@@ -30,6 +30,9 @@
|
||||
"addons/xterm-addon-serialize/benchmark/tsconfig.json",
|
||||
"addons/xterm-addon-unicode11/src/tsconfig.json",
|
||||
"addons/xterm-addon-unicode11/test/tsconfig.json",
|
||||
"addons/xterm-addon-unicode-graphemes/src/tsconfig.json",
|
||||
"addons/xterm-addon-unicode-graphemes/test/tsconfig.json",
|
||||
"addons/xterm-addon-unicode-graphemes/benchmark/tsconfig.json",
|
||||
"addons/xterm-addon-web-links/src/tsconfig.json",
|
||||
"addons/xterm-addon-web-links/test/tsconfig.json",
|
||||
"addons/xterm-addon-webgl/src/tsconfig.json",
|
||||
@@ -38,6 +41,7 @@
|
||||
"sourceType": "module"
|
||||
},
|
||||
"ignorePatterns": [
|
||||
"addons/*/src/third-party/*.ts",
|
||||
"**/inwasm-sdks/*",
|
||||
"**/typings/*.d.ts",
|
||||
"**/node_modules",
|
||||
|
||||
@@ -42,6 +42,8 @@ jobs:
|
||||
./addons/xterm-addon-serialize/out-test/* \
|
||||
./addons/xterm-addon-unicode11/out/* \
|
||||
./addons/xterm-addon-unicode11/out-test/* \
|
||||
./addons/xterm-addon-unicode-graphemes/out/* \
|
||||
./addons/xterm-addon-unicode-graphemes/out-test/* \
|
||||
./addons/xterm-addon-web-links/out/* \
|
||||
./addons/xterm-addon-web-links/out-test/* \
|
||||
./addons/xterm-addon-webgl/out/* \
|
||||
@@ -68,6 +70,8 @@ jobs:
|
||||
yarn --frozen-lockfile
|
||||
yarn install-addons
|
||||
- name: Lint code
|
||||
env:
|
||||
NODE_OPTIONS: --max_old_space_size=4096
|
||||
run: yarn lint
|
||||
- name: Lint API
|
||||
run: yarn lint-api
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
lib
|
||||
node_modules
|
||||
out-benchmark
|
||||
@@ -0,0 +1,29 @@
|
||||
# Blacklist - exclude everything except npm defaults such as LICENSE, etc
|
||||
*
|
||||
!*/
|
||||
|
||||
# Whitelist - lib/
|
||||
!lib/**/*.d.ts
|
||||
|
||||
!lib/**/*.js
|
||||
!lib/**/*.js.map
|
||||
|
||||
!lib/**/*.css
|
||||
|
||||
# Whitelist - src/
|
||||
!src/**/*.ts
|
||||
!src/**/*.d.ts
|
||||
|
||||
!src/**/*.js
|
||||
!src/**/*.js.map
|
||||
|
||||
!src/**/*.css
|
||||
|
||||
# Blacklist - src/ test files
|
||||
src/**/*.test.ts
|
||||
src/**/*.test.d.ts
|
||||
src/**/*.test.js
|
||||
src/**/*.test.js.map
|
||||
|
||||
# Whitelist - typings/
|
||||
!typings/*.d.ts
|
||||
@@ -0,0 +1,19 @@
|
||||
Copyright (c) 2023, The xterm.js authors (https://github.com/xtermjs/xterm.js)
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
@@ -0,0 +1,24 @@
|
||||
## xterm-addon-unicode-graphemes
|
||||
|
||||
⚠️ **This addon is currently experimental and may introduce unexpected and non-standard behavior**
|
||||
|
||||
An addon providing enhanced Unicode support (include grapheme clustering) for xterm.js.
|
||||
|
||||
The file `src/UnicodeProperties.ts` is generated and depends on the Unicode version. See [the unicode-properties project](https://github.com/PerBothner/unicode-properties) for credits and re-generation instructions.
|
||||
|
||||
### Install
|
||||
|
||||
```bash
|
||||
npm install --save xterm-addon-unicode-graphemes
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
```ts
|
||||
import { Terminal } from 'xterm';
|
||||
import { UnicodeGraphemeAddon } from 'xterm-addon-unicode-graphemes';
|
||||
|
||||
const terminal = new Terminal();
|
||||
const unicodeGraphemeAddon = new UnicodeGraphemeAddon();
|
||||
terminal.loadAddon(unicodeGraphemeAddon);
|
||||
```
|
||||
@@ -0,0 +1,78 @@
|
||||
/**
|
||||
* Copyright (c) 2019 The xterm.js authors. All rights reserved.
|
||||
* @license MIT
|
||||
*/
|
||||
|
||||
import { perfContext, before, ThroughputRuntimeCase } from 'xterm-benchmark';
|
||||
|
||||
import { spawn } from 'node-pty';
|
||||
import { Utf8ToUtf32, stringFromCodePoint } from 'common/input/TextDecoder';
|
||||
import { Terminal } from 'browser/Terminal';
|
||||
import { UnicodeGraphemeProvider } from 'UnicodeGraphemeProvider';
|
||||
|
||||
|
||||
function fakedAddonLoad(terminal: any): void {
|
||||
// resembles what UnicodeGraphemesAddon.activate does under the hood
|
||||
terminal.unicodeService.register(new UnicodeGraphemeProvider());
|
||||
terminal.unicodeService.activeVersion = '15-graphemes';
|
||||
}
|
||||
|
||||
|
||||
perfContext('Terminal: ls -lR /usr/lib', () => {
|
||||
let content = '';
|
||||
let contentUtf8: Uint8Array;
|
||||
|
||||
before(async () => {
|
||||
// grab output from "ls -lR /usr"
|
||||
const p = spawn('ls', ['--color=auto', '-lR', '/usr/lib'], {
|
||||
name: 'xterm-256color',
|
||||
cols: 80,
|
||||
rows: 25,
|
||||
cwd: process.env.HOME,
|
||||
env: process.env,
|
||||
encoding: (null as unknown as string) // needs to be fixed in node-pty
|
||||
});
|
||||
const chunks: Buffer[] = [];
|
||||
let length = 0;
|
||||
p.on('data', data => {
|
||||
chunks.push(data as unknown as Buffer);
|
||||
length += data.length;
|
||||
});
|
||||
await new Promise<void>(resolve => p.on('exit', () => resolve()));
|
||||
contentUtf8 = Buffer.concat(chunks, length);
|
||||
// translate to content string
|
||||
const buffer = new Uint32Array(contentUtf8.length);
|
||||
const decoder = new Utf8ToUtf32();
|
||||
const codepoints = decoder.decode(contentUtf8, buffer);
|
||||
for (let i = 0; i < codepoints; ++i) {
|
||||
content += stringFromCodePoint(buffer[i]);
|
||||
// peek into content to force flat repr in v8
|
||||
if (!(i % 10000000)) {
|
||||
content[i];
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
perfContext('write/string/async', () => {
|
||||
let terminal: Terminal;
|
||||
before(() => {
|
||||
terminal = new Terminal({ cols: 80, rows: 25, scrollback: 1000 });
|
||||
fakedAddonLoad(terminal);
|
||||
});
|
||||
new ThroughputRuntimeCase('', async () => {
|
||||
await new Promise<void>(res => terminal.write(content, res));
|
||||
return { payloadSize: contentUtf8.length };
|
||||
}, { fork: false }).showAverageThroughput();
|
||||
});
|
||||
|
||||
perfContext('write/Utf8/async', () => {
|
||||
let terminal: Terminal;
|
||||
before(() => {
|
||||
terminal = new Terminal({ cols: 80, rows: 25, scrollback: 1000 });
|
||||
});
|
||||
new ThroughputRuntimeCase('', async () => {
|
||||
await new Promise<void>(res => terminal.write(content, res));
|
||||
return { payloadSize: contentUtf8.length };
|
||||
}, { fork: false }).showAverageThroughput();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"APP_PATH": ".benchmark",
|
||||
"evalConfig": {
|
||||
"tolerance": {
|
||||
"*": [0.75, 1.5],
|
||||
"*.dev": [0.01, 1.5],
|
||||
"*.cv": [0.01, 1.5],
|
||||
"EscapeSequenceParser.benchmark.js.*.averageThroughput.mean": [0.9, 5]
|
||||
},
|
||||
"skip": [
|
||||
"*.median",
|
||||
"*.runs",
|
||||
"*.dev",
|
||||
"*.cv",
|
||||
"EscapeSequenceParser.benchmark.js.*.averageRuntime",
|
||||
"Terminal.benchmark.js.*.averageRuntime"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"lib": ["dom", "es6"],
|
||||
"outDir": "../out-benchmark",
|
||||
"types": ["../../../node_modules/@types/node"],
|
||||
"moduleResolution": "node",
|
||||
"strict": false,
|
||||
"target": "es2015",
|
||||
"module": "commonjs",
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"common/*": ["../../../src/common/*"],
|
||||
"browser/*": ["../../../src/browser/*"],
|
||||
"UnicodeGraphemeProvider": ["../src/UnicodeGraphemeProvider"]
|
||||
}
|
||||
},
|
||||
"include": ["../**/*", "../../../typings/xterm.d.ts"],
|
||||
"exclude": ["../../../**/*test.ts", "../../**/*api.ts"],
|
||||
"references": [
|
||||
{ "path": "../../../src/common" },
|
||||
{ "path": "../../../src/browser" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"name": "xterm-addon-unicode-graphemes",
|
||||
"version": "0.1.0",
|
||||
"author": {
|
||||
"name": "The xterm.js authors",
|
||||
"url": "https://xtermjs.org/"
|
||||
},
|
||||
"main": "lib/xterm-addon-unicode-graphemes.js",
|
||||
"types": "typings/xterm-addon-unicode-graphemes.d.ts",
|
||||
"repository": "https://github.com/xtermjs/xterm.js/tree/master/addons/xterm-addon-unicode-graphemes",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"terminal",
|
||||
"xterm",
|
||||
"xterm.js"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "../../node_modules/.bin/tsc -p .",
|
||||
"prepackage": "npm run build",
|
||||
"package": "../../node_modules/.bin/webpack",
|
||||
"prepublishOnly": "npm run package",
|
||||
"benchmark": "NODE_PATH=../../out:./out:./out-benchmark/ ../../node_modules/.bin/xterm-benchmark -r 5 -c benchmark/benchmark.json out-benchmark/benchmark/*benchmark.js",
|
||||
"benchmark-baseline": "NODE_PATH=../../out:./out:./out-benchmark/ ../../node_modules/.bin/xterm-benchmark -r 5 -c benchmark/benchmark.json --baseline out-benchmark/benchmark/*benchmark.js",
|
||||
"benchmark-eval": "NODE_PATH=../../out:./out:./out-benchmark/ ../../node_modules/.bin/xterm-benchmark -r 5 -c benchmark/benchmark.json --eval out-benchmark/benchmark/*benchmark.js"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"xterm": "^5.0.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Copyright (c) 2023 The xterm.js authors. All rights reserved.
|
||||
* @license MIT
|
||||
*/
|
||||
|
||||
import { IUnicodeVersionProvider } from 'xterm';
|
||||
import { UnicodeCharProperties, UnicodeCharWidth } from 'common/services/Services';
|
||||
import { UnicodeService } from 'common/services/UnicodeService';
|
||||
import * as UC from './third-party/UnicodeProperties';
|
||||
|
||||
export class UnicodeGraphemeProvider implements IUnicodeVersionProvider {
|
||||
public readonly version;
|
||||
public ambiguousCharsAreWide: boolean = false;
|
||||
public readonly handleGraphemes: boolean;
|
||||
|
||||
constructor(handleGraphemes: boolean = true) {
|
||||
this.version = handleGraphemes ? '15-graphemes' : '15';
|
||||
this.handleGraphemes = handleGraphemes;
|
||||
}
|
||||
|
||||
private static readonly _plainNarrowProperties: UnicodeCharProperties
|
||||
= UnicodeService.createPropertyValue(UC.GRAPHEME_BREAK_Other, 1, false);
|
||||
|
||||
public charProperties(codepoint: number, preceding: UnicodeCharProperties): UnicodeCharProperties {
|
||||
// Optimize the simple ASCII case, under the condition that
|
||||
// UnicodeService.extractCharKind(preceding) === GRAPHEME_BREAK_Other
|
||||
// (which also covers the case that preceding === 0).
|
||||
if ((codepoint >= 32 && codepoint < 127) && (preceding >> 3) === 0) {
|
||||
return UnicodeGraphemeProvider._plainNarrowProperties;
|
||||
}
|
||||
|
||||
let charInfo = UC.getInfo(codepoint);
|
||||
let w = UC.infoToWidthInfo(charInfo);
|
||||
let shouldJoin = false;
|
||||
if (w >= 2) {
|
||||
// Treat emoji_presentation_selector as WIDE.
|
||||
w = w === 3 || this.ambiguousCharsAreWide || codepoint === 0xfe0f ? 2 : 1;
|
||||
} else {
|
||||
w = 1;
|
||||
}
|
||||
if (preceding !== 0) {
|
||||
const oldWidth = UnicodeService.extractWidth(preceding);
|
||||
if (this.handleGraphemes) {
|
||||
charInfo = UC.shouldJoin(UnicodeService.extractCharKind(preceding), charInfo);
|
||||
} else {
|
||||
charInfo = w === 0 ? 1 : 0;
|
||||
}
|
||||
shouldJoin = charInfo > 0;
|
||||
if (shouldJoin) {
|
||||
if (oldWidth > w) {
|
||||
w = oldWidth;
|
||||
} else if (charInfo === 32) { // UC.GRAPHEME_BREAK_SAW_Regional_Pair)
|
||||
w = 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
return UnicodeService.createPropertyValue(charInfo, w, shouldJoin);
|
||||
}
|
||||
|
||||
public wcwidth(codepoint: number): UnicodeCharWidth {
|
||||
const charInfo = UC.getInfo(codepoint);
|
||||
const w = UC.infoToWidthInfo(charInfo);
|
||||
const kind = (charInfo & UC.GRAPHEME_BREAK_MASK) >> UC.GRAPHEME_BREAK_SHIFT;
|
||||
if (kind === UC.GRAPHEME_BREAK_Extend || kind === UC.GRAPHEME_BREAK_Prepend) {
|
||||
return 0;
|
||||
}
|
||||
if (w >= 2 && (w === 3 || this.ambiguousCharsAreWide)) {
|
||||
return 2;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* Copyright (c) 2023 The xterm.js authors. All rights reserved.
|
||||
* @license MIT
|
||||
*
|
||||
* UnicodeVersionProvider for V15 with grapeme cluster handleing.
|
||||
*/
|
||||
|
||||
import { Terminal, ITerminalAddon, IUnicodeHandling } from 'xterm';
|
||||
import { UnicodeGraphemeProvider } from './UnicodeGraphemeProvider';
|
||||
|
||||
|
||||
export class UnicodeGraphemesAddon implements ITerminalAddon {
|
||||
private _provider15Graphemes?: UnicodeGraphemeProvider;
|
||||
private _provider15?: UnicodeGraphemeProvider;
|
||||
private _unicode?: IUnicodeHandling;
|
||||
private _oldVersion: string = '';
|
||||
|
||||
public activate(terminal: Terminal): void {
|
||||
if (! this._provider15) {
|
||||
this._provider15 = new UnicodeGraphemeProvider(false);
|
||||
}
|
||||
if (! this._provider15Graphemes) {
|
||||
this._provider15Graphemes = new UnicodeGraphemeProvider(true);
|
||||
}
|
||||
const unicode = terminal.unicode;
|
||||
this._unicode = unicode;
|
||||
unicode.register(this._provider15);
|
||||
unicode.register(this._provider15Graphemes);
|
||||
this._oldVersion = unicode.activeVersion;
|
||||
unicode.activeVersion = '15-graphemes';
|
||||
}
|
||||
|
||||
public dispose(): void {
|
||||
if (this._unicode) {
|
||||
this._unicode.activeVersion = this._oldVersion;
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,380 @@
|
||||
var TINF_OK = 0;
|
||||
var TINF_DATA_ERROR = -3;
|
||||
|
||||
class Tree {
|
||||
table = new Uint16Array(16); /* table of code length counts */
|
||||
trans = new Uint16Array(288); /* code -> symbol translation table */
|
||||
};
|
||||
|
||||
class Data {
|
||||
tag: number = 0;
|
||||
bitcount: number = 0;
|
||||
destLen: number = 0;
|
||||
ltree: Tree;
|
||||
dtree: Tree;
|
||||
source: Uint8Array;
|
||||
dest: Uint8Array;
|
||||
sourceIndex: number = 0;
|
||||
|
||||
constructor(source: Uint8Array, dest: Uint8Array) {
|
||||
this.source = source;
|
||||
this.dest = dest;
|
||||
this.ltree = new Tree(); /* dynamic length/symbol tree */
|
||||
this.dtree = new Tree(); /* dynamic distance tree */
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------- *
|
||||
* -- uninitialized global data (static structures) -- *
|
||||
* --------------------------------------------------- */
|
||||
|
||||
var sltree = new Tree();
|
||||
var sdtree = new Tree();
|
||||
|
||||
/* extra bits and base tables for length codes */
|
||||
var length_bits = new Uint8Array(30);
|
||||
var length_base = new Uint16Array(30);
|
||||
|
||||
/* extra bits and base tables for distance codes */
|
||||
var dist_bits = new Uint8Array(30);
|
||||
var dist_base = new Uint16Array(30);
|
||||
|
||||
/* special ordering of code length codes */
|
||||
var clcidx = new Uint8Array([
|
||||
16, 17, 18, 0, 8, 7, 9, 6,
|
||||
10, 5, 11, 4, 12, 3, 13, 2,
|
||||
14, 1, 15
|
||||
]);
|
||||
|
||||
/* used by tinf_decode_trees, avoids allocations every call */
|
||||
const code_tree = new Tree();
|
||||
const lengths = new Uint8Array(288 + 32);
|
||||
|
||||
/* ----------------------- *
|
||||
* -- utility functions -- *
|
||||
* ----------------------- */
|
||||
|
||||
/* build extra bits and base tables */
|
||||
function tinf_build_bits_base(bits: Uint8Array, base: Uint16Array, delta: number, first: number): void {
|
||||
var i, sum;
|
||||
|
||||
/* build bits table */
|
||||
for (i = 0; i < delta; ++i) bits[i] = 0;
|
||||
for (i = 0; i < 30 - delta; ++i) bits[i + delta] = i / delta | 0;
|
||||
|
||||
/* build base table */
|
||||
for (sum = first, i = 0; i < 30; ++i) {
|
||||
base[i] = sum;
|
||||
sum += 1 << bits[i];
|
||||
}
|
||||
}
|
||||
|
||||
/* build the fixed huffman trees */
|
||||
function tinf_build_fixed_trees(lt: Tree, dt: Tree): void {
|
||||
var i;
|
||||
|
||||
/* build fixed length tree */
|
||||
for (i = 0; i < 7; ++i) lt.table[i] = 0;
|
||||
|
||||
lt.table[7] = 24;
|
||||
lt.table[8] = 152;
|
||||
lt.table[9] = 112;
|
||||
|
||||
for (i = 0; i < 24; ++i) lt.trans[i] = 256 + i;
|
||||
for (i = 0; i < 144; ++i) lt.trans[24 + i] = i;
|
||||
for (i = 0; i < 8; ++i) lt.trans[24 + 144 + i] = 280 + i;
|
||||
for (i = 0; i < 112; ++i) lt.trans[24 + 144 + 8 + i] = 144 + i;
|
||||
|
||||
/* build fixed distance tree */
|
||||
for (i = 0; i < 5; ++i) dt.table[i] = 0;
|
||||
|
||||
dt.table[5] = 32;
|
||||
|
||||
for (i = 0; i < 32; ++i) dt.trans[i] = i;
|
||||
}
|
||||
|
||||
/* given an array of code lengths, build a tree */
|
||||
var offs = new Uint16Array(16);
|
||||
|
||||
function tinf_build_tree(t: Tree, lengths: Uint8Array, off: number, num: number): void {
|
||||
var i, sum;
|
||||
|
||||
/* clear code length count table */
|
||||
for (i = 0; i < 16; ++i) t.table[i] = 0;
|
||||
|
||||
/* scan symbol lengths, and sum code length counts */
|
||||
for (i = 0; i < num; ++i) t.table[lengths[off + i]]++;
|
||||
|
||||
t.table[0] = 0;
|
||||
|
||||
/* compute offset table for distribution sort */
|
||||
for (sum = 0, i = 0; i < 16; ++i) {
|
||||
offs[i] = sum;
|
||||
sum += t.table[i];
|
||||
}
|
||||
|
||||
/* create code->symbol translation table (symbols sorted by code) */
|
||||
for (i = 0; i < num; ++i) {
|
||||
if (lengths[off + i]) t.trans[offs[lengths[off + i]]++] = i;
|
||||
}
|
||||
}
|
||||
|
||||
/* ---------------------- *
|
||||
* -- decode functions -- *
|
||||
* ---------------------- */
|
||||
|
||||
/* get one bit from source stream */
|
||||
function tinf_getbit(d: Data): number {
|
||||
/* check if tag is empty */
|
||||
if (!d.bitcount--) {
|
||||
/* load next tag */
|
||||
d.tag = d.source[d.sourceIndex++];
|
||||
d.bitcount = 7;
|
||||
}
|
||||
|
||||
/* shift bit out of tag */
|
||||
var bit = d.tag & 1;
|
||||
d.tag >>>= 1;
|
||||
|
||||
return bit;
|
||||
}
|
||||
|
||||
/* read a num bit value from a stream and add base */
|
||||
function tinf_read_bits(d: Data, num: number, base: number): number {
|
||||
if (!num)
|
||||
return base;
|
||||
|
||||
while (d.bitcount < 24) {
|
||||
d.tag |= d.source[d.sourceIndex++] << d.bitcount;
|
||||
d.bitcount += 8;
|
||||
}
|
||||
|
||||
var val = d.tag & (0xffff >>> (16 - num));
|
||||
d.tag >>>= num;
|
||||
d.bitcount -= num;
|
||||
return val + base;
|
||||
}
|
||||
|
||||
/* given a data stream and a tree, decode a symbol */
|
||||
function tinf_decode_symbol(d: Data, t: Tree): number {
|
||||
while (d.bitcount < 24) {
|
||||
d.tag |= d.source[d.sourceIndex++] << d.bitcount;
|
||||
d.bitcount += 8;
|
||||
}
|
||||
|
||||
var sum = 0, cur = 0, len = 0;
|
||||
var tag = d.tag;
|
||||
|
||||
/* get more bits while code value is above sum */
|
||||
do {
|
||||
cur = 2 * cur + (tag & 1);
|
||||
tag >>>= 1;
|
||||
++len;
|
||||
|
||||
sum += t.table[len];
|
||||
cur -= t.table[len];
|
||||
} while (cur >= 0);
|
||||
|
||||
d.tag = tag;
|
||||
d.bitcount -= len;
|
||||
|
||||
return t.trans[sum + cur];
|
||||
}
|
||||
|
||||
/* given a data stream, decode dynamic trees from it */
|
||||
function tinf_decode_trees(d: Data, lt: Tree, dt: Tree): void {
|
||||
var hlit, hdist, hclen;
|
||||
var i, num, length;
|
||||
|
||||
/* get 5 bits HLIT (257-286) */
|
||||
hlit = tinf_read_bits(d, 5, 257);
|
||||
|
||||
/* get 5 bits HDIST (1-32) */
|
||||
hdist = tinf_read_bits(d, 5, 1);
|
||||
|
||||
/* get 4 bits HCLEN (4-19) */
|
||||
hclen = tinf_read_bits(d, 4, 4);
|
||||
|
||||
for (i = 0; i < 19; ++i) lengths[i] = 0;
|
||||
|
||||
/* read code lengths for code length alphabet */
|
||||
for (i = 0; i < hclen; ++i) {
|
||||
/* get 3 bits code length (0-7) */
|
||||
var clen = tinf_read_bits(d, 3, 0);
|
||||
lengths[clcidx[i]] = clen;
|
||||
}
|
||||
|
||||
/* build code length tree */
|
||||
tinf_build_tree(code_tree, lengths, 0, 19);
|
||||
|
||||
/* decode code lengths for the dynamic trees */
|
||||
for (num = 0; num < hlit + hdist;) {
|
||||
var sym = tinf_decode_symbol(d, code_tree);
|
||||
|
||||
switch (sym) {
|
||||
case 16:
|
||||
/* copy previous code length 3-6 times (read 2 bits) */
|
||||
var prev = lengths[num - 1];
|
||||
for (length = tinf_read_bits(d, 2, 3); length; --length) {
|
||||
lengths[num++] = prev;
|
||||
}
|
||||
break;
|
||||
case 17:
|
||||
/* repeat code length 0 for 3-10 times (read 3 bits) */
|
||||
for (length = tinf_read_bits(d, 3, 3); length; --length) {
|
||||
lengths[num++] = 0;
|
||||
}
|
||||
break;
|
||||
case 18:
|
||||
/* repeat code length 0 for 11-138 times (read 7 bits) */
|
||||
for (length = tinf_read_bits(d, 7, 11); length; --length) {
|
||||
lengths[num++] = 0;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
/* values 0-15 represent the actual code lengths */
|
||||
lengths[num++] = sym;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* build dynamic trees */
|
||||
tinf_build_tree(lt, lengths, 0, hlit);
|
||||
tinf_build_tree(dt, lengths, hlit, hdist);
|
||||
}
|
||||
|
||||
/* ----------------------------- *
|
||||
* -- block inflate functions -- *
|
||||
* ----------------------------- */
|
||||
|
||||
/* given a stream and two trees, inflate a block of data */
|
||||
function tinf_inflate_block_data(d: Data, lt: Tree, dt: Tree): number {
|
||||
for (;;) {
|
||||
var sym = tinf_decode_symbol(d, lt);
|
||||
|
||||
/* check for end of block */
|
||||
if (sym === 256) {
|
||||
return TINF_OK;
|
||||
}
|
||||
|
||||
if (sym < 256) {
|
||||
d.dest[d.destLen++] = sym;
|
||||
} else {
|
||||
var length, dist, offs;
|
||||
var i;
|
||||
|
||||
sym -= 257;
|
||||
|
||||
/* possibly get more bits from length code */
|
||||
length = tinf_read_bits(d, length_bits[sym], length_base[sym]);
|
||||
|
||||
dist = tinf_decode_symbol(d, dt);
|
||||
|
||||
/* possibly get more bits from distance code */
|
||||
offs = d.destLen - tinf_read_bits(d, dist_bits[dist], dist_base[dist]);
|
||||
|
||||
/* copy match */
|
||||
for (i = offs; i < offs + length; ++i) {
|
||||
d.dest[d.destLen++] = d.dest[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* inflate an uncompressed block of data */
|
||||
function tinf_inflate_uncompressed_block(d: Data) {
|
||||
var length, invlength;
|
||||
var i;
|
||||
|
||||
/* unread from bitbuffer */
|
||||
while (d.bitcount > 8) {
|
||||
d.sourceIndex--;
|
||||
d.bitcount -= 8;
|
||||
}
|
||||
|
||||
/* get length */
|
||||
length = d.source[d.sourceIndex + 1];
|
||||
length = 256 * length + d.source[d.sourceIndex];
|
||||
|
||||
/* get one's complement of length */
|
||||
invlength = d.source[d.sourceIndex + 3];
|
||||
invlength = 256 * invlength + d.source[d.sourceIndex + 2];
|
||||
|
||||
/* check length */
|
||||
if (length !== (~invlength & 0x0000ffff))
|
||||
return TINF_DATA_ERROR;
|
||||
|
||||
d.sourceIndex += 4;
|
||||
|
||||
/* copy block */
|
||||
for (i = length; i; --i)
|
||||
d.dest[d.destLen++] = d.source[d.sourceIndex++];
|
||||
|
||||
/* make sure we start next block on a byte boundary */
|
||||
d.bitcount = 0;
|
||||
|
||||
return TINF_OK;
|
||||
}
|
||||
|
||||
/* inflate stream from source to dest */
|
||||
function tinf_uncompress(source: Uint8Array, dest: Uint8Array) {
|
||||
var d = new Data(source, dest);
|
||||
var bfinal, btype, res;
|
||||
|
||||
do {
|
||||
/* read final block flag */
|
||||
bfinal = tinf_getbit(d);
|
||||
|
||||
/* read block type (2 bits) */
|
||||
btype = tinf_read_bits(d, 2, 0);
|
||||
|
||||
/* decompress block */
|
||||
switch (btype) {
|
||||
case 0:
|
||||
/* decompress uncompressed block */
|
||||
res = tinf_inflate_uncompressed_block(d);
|
||||
break;
|
||||
case 1:
|
||||
/* decompress block with fixed huffman trees */
|
||||
res = tinf_inflate_block_data(d, sltree, sdtree);
|
||||
break;
|
||||
case 2:
|
||||
/* decompress block with dynamic huffman trees */
|
||||
tinf_decode_trees(d, d.ltree, d.dtree);
|
||||
res = tinf_inflate_block_data(d, d.ltree, d.dtree);
|
||||
break;
|
||||
default:
|
||||
res = TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
if (res !== TINF_OK)
|
||||
throw new Error('Data error');
|
||||
|
||||
} while (!bfinal);
|
||||
|
||||
if (d.destLen < d.dest.length) {
|
||||
if (typeof d.dest.slice === 'function')
|
||||
return d.dest.slice(0, d.destLen);
|
||||
else
|
||||
return d.dest.subarray(0, d.destLen);
|
||||
}
|
||||
|
||||
return d.dest;
|
||||
}
|
||||
|
||||
/* -------------------- *
|
||||
* -- initialization -- *
|
||||
* -------------------- */
|
||||
|
||||
/* build fixed huffman trees */
|
||||
tinf_build_fixed_trees(sltree, sdtree);
|
||||
|
||||
/* build extra bits and base tables */
|
||||
tinf_build_bits_base(length_bits, length_base, 4, 3);
|
||||
tinf_build_bits_base(dist_bits, dist_base, 2, 1);
|
||||
|
||||
/* fix a special case */
|
||||
length_bits[28] = 0;
|
||||
length_base[28] = 258;
|
||||
|
||||
export default tinf_uncompress
|
||||
@@ -0,0 +1,134 @@
|
||||
import inflate from './tiny-inflate'
|
||||
|
||||
// Shift size for getting the index-1 table offset.
|
||||
const SHIFT_1 = 6 + 5;
|
||||
|
||||
// Shift size for getting the index-2 table offset.
|
||||
const SHIFT_2 = 5;
|
||||
|
||||
// Difference between the two shift sizes,
|
||||
// for getting an index-1 offset from an index-2 offset. 6=11-5
|
||||
const SHIFT_1_2 = SHIFT_1 - SHIFT_2;
|
||||
|
||||
// Number of index-1 entries for the BMP. 32=0x20
|
||||
// This part of the index-1 table is omitted from the serialized form.
|
||||
const OMITTED_BMP_INDEX_1_LENGTH = 0x10000 >> SHIFT_1;
|
||||
|
||||
// Number of entries in an index-2 block. 64=0x40
|
||||
const INDEX_2_BLOCK_LENGTH = 1 << SHIFT_1_2;
|
||||
|
||||
// Mask for getting the lower bits for the in-index-2-block offset. */
|
||||
const INDEX_2_MASK = INDEX_2_BLOCK_LENGTH - 1;
|
||||
|
||||
// Shift size for shifting left the index array values.
|
||||
// Increases possible data size with 16-bit index values at the cost
|
||||
// of compactability.
|
||||
// This requires data blocks to be aligned by DATA_GRANULARITY.
|
||||
const INDEX_SHIFT = 2;
|
||||
|
||||
// Number of entries in a data block. 32=0x20
|
||||
const DATA_BLOCK_LENGTH = 1 << SHIFT_2;
|
||||
|
||||
// Mask for getting the lower bits for the in-data-block offset.
|
||||
const DATA_MASK = DATA_BLOCK_LENGTH - 1;
|
||||
|
||||
// The part of the index-2 table for U+D800..U+DBFF stores values for
|
||||
// lead surrogate code _units_ not code _points_.
|
||||
// Values for lead surrogate code _points_ are indexed with this portion of the table.
|
||||
// Length=32=0x20=0x400>>SHIFT_2. (There are 1024=0x400 lead surrogates.)
|
||||
const LSCP_INDEX_2_OFFSET = 0x10000 >> SHIFT_2;
|
||||
const LSCP_INDEX_2_LENGTH = 0x400 >> SHIFT_2;
|
||||
|
||||
// Count the lengths of both BMP pieces. 2080=0x820
|
||||
const INDEX_2_BMP_LENGTH = LSCP_INDEX_2_OFFSET + LSCP_INDEX_2_LENGTH;
|
||||
|
||||
// The 2-byte UTF-8 version of the index-2 table follows at offset 2080=0x820.
|
||||
// Length 32=0x20 for lead bytes C0..DF, regardless of SHIFT_2.
|
||||
const UTF8_2B_INDEX_2_OFFSET = INDEX_2_BMP_LENGTH;
|
||||
const UTF8_2B_INDEX_2_LENGTH = 0x800 >> 6; // U+0800 is the first code point after 2-byte UTF-8
|
||||
|
||||
// The index-1 table, only used for supplementary code points, at offset 2112=0x840.
|
||||
// Variable length, for code points up to highStart, where the last single-value range starts.
|
||||
// Maximum length 512=0x200=0x100000>>SHIFT_1.
|
||||
// (For 0x100000 supplementary code points U+10000..U+10ffff.)
|
||||
//
|
||||
// The part of the index-2 table for supplementary code points starts
|
||||
// after this index-1 table.
|
||||
//
|
||||
// Both the index-1 table and the following part of the index-2 table
|
||||
// are omitted completely if there is only BMP data.
|
||||
const INDEX_1_OFFSET = UTF8_2B_INDEX_2_OFFSET + UTF8_2B_INDEX_2_LENGTH;
|
||||
|
||||
// The alignment size of a data block. Also the granularity for compaction.
|
||||
const DATA_GRANULARITY = 1 << INDEX_SHIFT;
|
||||
|
||||
const isBigEndian = (new Uint8Array(new Uint32Array([0x12345678]).buffer)[0] === 0x12);
|
||||
|
||||
class UnicodeTrie {
|
||||
private data: Uint32Array;
|
||||
private highStart: number;
|
||||
private errorValue: number;
|
||||
constructor(data: Uint8Array) {
|
||||
// read binary format
|
||||
|
||||
const view = new DataView(data.buffer);
|
||||
this.highStart = view.getUint32(0, true);
|
||||
this.errorValue = view.getUint32(4, true);
|
||||
let uncompressedLength = view.getUint32(8, true);
|
||||
data = data.subarray(12);
|
||||
|
||||
// double inflate the actual trie data
|
||||
data = inflate(data, new Uint8Array(uncompressedLength));
|
||||
data = inflate(data, new Uint8Array(uncompressedLength));
|
||||
|
||||
if (isBigEndian) {
|
||||
// swap bytes from little-endian
|
||||
const len = data.length;
|
||||
for (let i = 0; i < len; i += 4) {
|
||||
// Exchange data[i] and data[i + 3]:
|
||||
let x = data[i]; data[i] = data[i+3]; data[i+3] = x;
|
||||
// Exchange data[i + 1] and data[i + 2]:
|
||||
let y = data[i+1]; data[i+1] = data[i+2]; data[i+2] = y;
|
||||
}
|
||||
}
|
||||
|
||||
this.data = new Uint32Array(data.buffer);
|
||||
|
||||
}
|
||||
|
||||
get(codePoint: number): number {
|
||||
let index;
|
||||
if ((codePoint < 0) || (codePoint > 0x10ffff)) {
|
||||
return this.errorValue;
|
||||
}
|
||||
|
||||
if ((codePoint < 0xd800) || ((codePoint > 0xdbff) && (codePoint <= 0xffff))) {
|
||||
// Ordinary BMP code point, excluding leading surrogates.
|
||||
// BMP uses a single level lookup. BMP index starts at offset 0 in the index.
|
||||
// data is stored in the index array itself.
|
||||
index = (this.data[codePoint >> SHIFT_2] << INDEX_SHIFT) + (codePoint & DATA_MASK);
|
||||
return this.data[index];
|
||||
}
|
||||
|
||||
if (codePoint <= 0xffff) {
|
||||
// Lead Surrogate Code Point. A Separate index section is stored for
|
||||
// lead surrogate code units and code points.
|
||||
// The main index has the code unit data.
|
||||
// For this function, we need the code point data.
|
||||
index = (this.data[LSCP_INDEX_2_OFFSET + ((codePoint - 0xd800) >> SHIFT_2)] << INDEX_SHIFT) + (codePoint & DATA_MASK);
|
||||
return this.data[index];
|
||||
}
|
||||
|
||||
if (codePoint < this.highStart) {
|
||||
// Supplemental code point, use two-level lookup.
|
||||
index = this.data[(INDEX_1_OFFSET - OMITTED_BMP_INDEX_1_LENGTH) + (codePoint >> SHIFT_1)];
|
||||
index = this.data[index + ((codePoint >> SHIFT_2) & INDEX_2_MASK)];
|
||||
index = (index << INDEX_SHIFT) + (codePoint & DATA_MASK);
|
||||
return this.data[index];
|
||||
}
|
||||
|
||||
return this.data[this.data.length - DATA_GRANULARITY];
|
||||
}
|
||||
}
|
||||
|
||||
export default UnicodeTrie
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "commonjs",
|
||||
"target": "es2021",
|
||||
"lib": [
|
||||
"dom",
|
||||
"es2021"
|
||||
],
|
||||
"rootDir": ".",
|
||||
"outDir": "../out",
|
||||
"sourceMap": true,
|
||||
"removeComments": true,
|
||||
"strict": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"common/*": [
|
||||
"../../../src/common/*"
|
||||
]
|
||||
},
|
||||
"types": [
|
||||
"../../../node_modules/@types/mocha"
|
||||
]
|
||||
},
|
||||
"include": [
|
||||
"./**/*",
|
||||
"../../../typings/xterm.d.ts"
|
||||
],
|
||||
"references": [
|
||||
{
|
||||
"path": "../../../src/common"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* Copyright (c) 2019 The xterm.js authors. All rights reserved.
|
||||
* @license MIT
|
||||
*/
|
||||
|
||||
import { assert } from 'chai';
|
||||
import { openTerminal, launchBrowser } from '../../../out-test/api/TestUtils';
|
||||
import { Browser, Page } from '@playwright/test';
|
||||
|
||||
const APP = 'http://127.0.0.1:3001/test';
|
||||
|
||||
let browser: Browser;
|
||||
let page: Page;
|
||||
const width = 800;
|
||||
const height = 600;
|
||||
|
||||
describe('UnicodeGraphemesAddon', () => {
|
||||
before(async function(): Promise<any> {
|
||||
browser = await launchBrowser();
|
||||
page = await (await browser.newContext()).newPage();
|
||||
await page.setViewportSize({ width, height });
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await browser.close();
|
||||
});
|
||||
|
||||
beforeEach(async function(): Promise<any> {
|
||||
await page.goto(APP);
|
||||
await openTerminal(page);
|
||||
});
|
||||
async function evalWidth(str: string): Promise<number> {
|
||||
return page.evaluate(`window.term._core.unicodeService.getStringCellWidth('${str}')`);
|
||||
}
|
||||
const ourVersion = '15-graphemes';
|
||||
it('wcwidth V15 emoji test', async () => {
|
||||
await page.evaluate(`
|
||||
window.unicode = new UnicodeGraphemesAddon();
|
||||
window.term.loadAddon(window.unicode);
|
||||
`);
|
||||
// should have loaded '15-graphemes'
|
||||
assert.deepEqual(await page.evaluate(`window.term.unicode.versions`), ['6', '15', '15-graphemes']);
|
||||
// switch should not throw
|
||||
await page.evaluate(`window.term.unicode.activeVersion = '${ourVersion}';`);
|
||||
assert.equal(await page.evaluate(`window.term.unicode.activeVersion`), ourVersion);
|
||||
assert.equal(await evalWidth('🤣🤣🤣🤣🤣🤣🤣🤣🤣🤣'), 20,
|
||||
'10 emoji - width 10 in V6; 20 in V11 or later');
|
||||
assert.equal(await evalWidth('\u{1F476}\u{1F3FF}\u{1F476}'), 4,
|
||||
'baby with emoji modifier fitzpatrick type-6; baby');
|
||||
assert.equal(await evalWidth('\u{1F469}\u200d\u{1f469}\u200d\u{1f466}'), 2,
|
||||
'woman+zwj+woman+zwj+boy');
|
||||
assert.equal(await evalWidth('=\u{1F3CB}\u{FE0F}=\u{F3CB}\u{1F3FE}\u200D\u2640='), 7,
|
||||
'person lifting weights (plain, emoji); woman lighting weights, medium dark');
|
||||
assert.equal(await evalWidth('\u{1F469}\u{1F469}\u{200D}\u{1F393}\u{1F468}\u{1F3FF}\u{200D}\u{1F393}'), 6,
|
||||
'woman; woman student; man student dark');
|
||||
assert.equal(await evalWidth('\u{1f1f3}\u{1f1f4}/'), 3,
|
||||
'regional indicator symbol letters N and O, cluster');
|
||||
assert.equal(await evalWidth('\u{1f1f3}/\u{1f1f4}'), 3,
|
||||
'regional indicator symbol letters N and O, separated');
|
||||
assert.equal(await evalWidth('\u0061\u0301'), 1,
|
||||
'letter a with acute accent');
|
||||
assert.equal(await evalWidth('{\u1100\u1161\u11a8\u1100\u1161}'), 6,
|
||||
'Korean Jamo');
|
||||
assert.equal(await evalWidth('\uAC00=\uD685='), 6,
|
||||
'Hangul syllables (pre-composed)');
|
||||
assert.equal(await evalWidth('(\u26b0\ufe0e)'), 3,
|
||||
'coffin with text presentation');
|
||||
assert.equal(await evalWidth('(\u26b0\ufe0f)'), 4,
|
||||
'coffin with emoji presentation');
|
||||
assert.equal(await evalWidth('<E\u0301\ufe0fg\ufe0fa\ufe0fl\ufe0fi\ufe0f\ufe0ft\ufe0fe\u0301\ufe0f>'), 16,
|
||||
'Égalité (using separate acute) emoij_presentation');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "commonjs",
|
||||
"target": "es2015",
|
||||
"lib": [
|
||||
"dom",
|
||||
"es2015"
|
||||
],
|
||||
"rootDir": ".",
|
||||
"outDir": "../out-test",
|
||||
"sourceMap": true,
|
||||
"removeComments": true,
|
||||
"strict": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"common/*": [
|
||||
"../../../src/common/*"
|
||||
]
|
||||
},
|
||||
"types": [
|
||||
"../../../node_modules/@types/mocha",
|
||||
"../../../node_modules/@types/node",
|
||||
"../../../out-test/api/TestUtils"
|
||||
]
|
||||
},
|
||||
"include": [
|
||||
"./**/*",
|
||||
"../../../typings/xterm.d.ts"
|
||||
],
|
||||
"references": [
|
||||
{
|
||||
"path": "../../../src/common"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"files": [],
|
||||
"include": [],
|
||||
"references": [
|
||||
{ "path": "./src" },
|
||||
{ "path": "./test" },
|
||||
{ "path": "./benchmark" }
|
||||
]
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
/**
|
||||
* Copyright (c) 2023 The xterm.js authors. All rights reserved.
|
||||
* @license MIT
|
||||
*/
|
||||
|
||||
import { Terminal, ITerminalAddon } from 'xterm';
|
||||
|
||||
declare module 'xterm-addon-unicode-graphemes' {
|
||||
export class Unicode11Addon implements ITerminalAddon {
|
||||
constructor();
|
||||
public activate(terminal: Terminal): void;
|
||||
public dispose(): void;
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user