mirror of
https://github.com/wavetermdev/xterm.js.git
synced 2026-08-05 13:43:48 -07:00
remove named colors
This commit is contained in:
Jörg Breitbart
@@ -1,14 +0,0 @@
|
||||
### Fixture for 11 color names
|
||||
|
||||
`rgb.txt` contains X11's defined color names, copied over from `/etc/X11/rgb.txt` on Ubuntu 18.
|
||||
|
||||
Run `create_module.js` to create a TS module containing the color definitions. The script performs these steps:
|
||||
- extract color definitions from `rgb.txt`
|
||||
- remove gray definitions (re-added programmatically later)
|
||||
- create a perfect hash function
|
||||
- calculate crc10 for basic collision prevention
|
||||
- run several collision tests
|
||||
- compress table and color data
|
||||
- write data with loading shim to `ColorNames.ts`
|
||||
|
||||
The final file is meant to be copied over to `../../src/common/data/`.
|
||||
@@ -1,447 +0,0 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
/**
|
||||
* For decoder side:
|
||||
* - reconstruct grays with f = lambda x: (x * 256 - x + 50) / 100 (needs value rounding check)
|
||||
* - apply nameFilter
|
||||
* - build lookup & extract functions (runtime decoding?)
|
||||
* - transfer hashtable and color table
|
||||
*/
|
||||
|
||||
// match color entry in rgb.txt
|
||||
const rexFile = /^\s*(\d+)\s*(\d+)\s*(\d+)\s*?[\t]+(.*)$/;
|
||||
// match greyXX|grayXX names
|
||||
const rexGrey = /^gr[ae]y\d+/;
|
||||
// name match
|
||||
const rexName = /^\w?[A-Za-z0-9 ]+\w/;
|
||||
|
||||
function parseX11Colors(filename) {
|
||||
const fileData = fs.readFileSync(filename, {encoding: 'utf8'});
|
||||
const colors = [];
|
||||
for (const line of fileData.split('\n')) {
|
||||
const m = rexFile.exec(line);
|
||||
if (m) {
|
||||
colors.push({
|
||||
color: [parseInt(m[1]), parseInt(m[2]), parseInt(m[3])],
|
||||
name: m[4]
|
||||
});
|
||||
}
|
||||
}
|
||||
return colors;
|
||||
}
|
||||
|
||||
// Use the FNV algorithm from http://isthe.com/chongo/tech/comp/fnv/
|
||||
function hash(d, name) {
|
||||
if (!d) d = 0x01000193;
|
||||
for (const c of name) {
|
||||
d = ( (d * 0x01000193) ^ c.charCodeAt(0) ) & 0xffffffff;
|
||||
}
|
||||
return d >>> 0;
|
||||
}
|
||||
|
||||
// inspired from http://stevehanov.ca/blog/?id=119
|
||||
function createMinimalPerfectHash(nameList) {
|
||||
const nameMap = Object.create(null);
|
||||
for (const [idx, name] of nameList.entries()) {
|
||||
nameMap[name] = idx;
|
||||
}
|
||||
const size = nameList.length;
|
||||
|
||||
const buckets = [];
|
||||
for (let i = 0; i < size; ++i) buckets.push([]);
|
||||
const g = new Array(size).fill(0);
|
||||
const v = new Array(size).fill(null);
|
||||
|
||||
for (const key of nameList) {
|
||||
buckets[hash(0, key) % size].push(key);
|
||||
}
|
||||
|
||||
buckets.sort((a, b) => b.length - a.length);
|
||||
let breakIdx = 0;
|
||||
for (let i = 0; i < size; ++i) {
|
||||
const bucket = buckets[i];
|
||||
if (bucket.length <= 1) {
|
||||
breakIdx = i;
|
||||
break;
|
||||
}
|
||||
let d = 1;
|
||||
let item = 0;
|
||||
let slots = [];
|
||||
|
||||
while (item < bucket.length) {
|
||||
const slot = hash(d, bucket[item]) % size;
|
||||
if (v[slot] !== null || slots.includes(slot)) {
|
||||
d++;
|
||||
item = 0;
|
||||
slots = [];
|
||||
} else {
|
||||
slots.push(slot);
|
||||
item++;
|
||||
}
|
||||
}
|
||||
|
||||
g[hash(0, bucket[0]) % size] = d;
|
||||
for (let k = 0; k < bucket.length; ++k) {
|
||||
v[slots[k]] = nameMap[bucket[k]];
|
||||
}
|
||||
}
|
||||
|
||||
const freeList = [];
|
||||
for (let i = 0; i < size; ++i) {
|
||||
if (v[i] === null) freeList.push(i);
|
||||
}
|
||||
|
||||
for (let i = breakIdx; i < size; ++i) {
|
||||
const bucket = buckets[i];
|
||||
if (!bucket.length) break;
|
||||
const slot = freeList.pop();
|
||||
g[hash(0, bucket[0]) % size] = -slot - 1;
|
||||
v[slot] = nameMap[bucket[0]];
|
||||
}
|
||||
|
||||
return [g, v];
|
||||
}
|
||||
|
||||
function lookupPerfectHash(g, v, key) {
|
||||
const d = g[hash(0, key) % g.length];
|
||||
if (d < 0) return v[-d - 1];
|
||||
return v[hash(d, key) % v.length];
|
||||
}
|
||||
|
||||
function checkPerfectHashTables(g, v, nameList) {
|
||||
let allPassed = true;
|
||||
for (const [idx, name] of nameList.entries()) {
|
||||
const lookup = lookupPerfectHash(g, v, name);
|
||||
if (lookup !== idx) {
|
||||
console.log(`\x1b[33mmismatch: '${name}' returns ${lookup} (orig: ${idx})\x1b[m`);
|
||||
allPassed = false;
|
||||
}
|
||||
}
|
||||
return allPassed;
|
||||
}
|
||||
|
||||
function crc10atm(name, crc) {
|
||||
if (!crc) crc = 0;
|
||||
for (const c of name) {
|
||||
const v = c.charCodeAt(0);
|
||||
crc ^= v << 2;
|
||||
for (let k = 0; k < 8; k++) {
|
||||
crc = crc & 0x200 ? (crc << 1) ^ 0x233 : crc << 1;
|
||||
}
|
||||
}
|
||||
crc &= 0x3ff;
|
||||
return crc >>> 0;
|
||||
}
|
||||
|
||||
function checkColorList(g, v, crc, colorNames) {
|
||||
let match = [];
|
||||
for (const word of colorNames) {
|
||||
if (!nameFilter(word)) continue;
|
||||
const idx = lookupPerfectHash(g, v, word);
|
||||
const crc10 = crc10atm(word);
|
||||
if (crc[idx] === crc10) {
|
||||
match.push({idx, word, colorName: colorNames[idx]});
|
||||
}
|
||||
}
|
||||
return {tested: colorNames.length, match};
|
||||
}
|
||||
|
||||
function checkWordlists(g, v, crc, colorNames, filename) {
|
||||
const fileData = fs.readFileSync(filename, {encoding: 'utf8'});
|
||||
const words = fileData.split('\n').filter(el => lookupPerfectHash.length !== 0);
|
||||
let collisions = [];
|
||||
for (const word of words) {
|
||||
if (!nameFilter(word)) continue;
|
||||
const idx = lookupPerfectHash(g, v, word);
|
||||
const crc10 = crc10atm(word);
|
||||
if (crc[idx] === crc10 && word !== colorNames[idx]) {
|
||||
collisions.push({idx, word, colorName: colorNames[idx]});
|
||||
}
|
||||
}
|
||||
return {tested: words.length, collisions};
|
||||
}
|
||||
|
||||
function nameFilter(name) {
|
||||
// length 3 - 22
|
||||
if (name.length < 3 || name.length > 22) return;
|
||||
// chars only in [A-Za-z0-9 ]
|
||||
if (!rexName.exec(name)) return;
|
||||
return name;
|
||||
}
|
||||
|
||||
function compressTables(g, v, crc, al) {
|
||||
// g | v | crc: all in 10 bit (<1024)
|
||||
// --> 30 bit
|
||||
// --> fits into 5 bytes of a 64-bit char alphabet
|
||||
|
||||
// g needs an offset for proper zero alignment
|
||||
const gOffset = -Math.min(...g);
|
||||
g = g.map(el => el + gOffset);
|
||||
|
||||
// assert we are in 0..2^10
|
||||
if (Math.min(...g) < 0 || Math.min(...v) < 0 || Math.min(...crc) < 0
|
||||
|| Math.max(...g) > 1023 || Math.max(...v) > 1023 || Math.max(...crc) > 1023
|
||||
) {
|
||||
console.log('\x1b[31mTables out of compressible range, manual fix needed.\x1b[m');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// assert we have same length
|
||||
if (g.length !== v.length || g.length !== crc.length) {
|
||||
console.log('\x1b[31mTables length mismatch, manual fix needed.\x1b[m');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// construct compressed data string
|
||||
let result = '';
|
||||
const length = g.length;
|
||||
for (let i = 0; i < length; ++i) {
|
||||
let value = (g[i] << 20) | (v[i] << 10) | crc[i];
|
||||
let bucket = '';
|
||||
for (let k = 0; k < 5; ++k) {
|
||||
bucket += al[value % al.length];
|
||||
value = Math.floor(value / al.length);
|
||||
}
|
||||
result += bucket.split('').reverse().join('');
|
||||
}
|
||||
|
||||
return [result, length, gOffset];
|
||||
}
|
||||
|
||||
function compressColors(colors, al) {
|
||||
let result = '';
|
||||
for (const color of colors) {
|
||||
let value = (color[0] << 16) | (color[1] << 8) | color[2];
|
||||
let bucket = '';
|
||||
for (let k = 0; k < 4; ++k) {
|
||||
bucket += al[value % al.length];
|
||||
value = Math.floor(value / al.length);
|
||||
}
|
||||
result += bucket.split('').reverse().join('');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function loadData(bucket, al) {
|
||||
let value = 0;
|
||||
for (const c of bucket) {
|
||||
value *= al.length;
|
||||
value += al.indexOf(c);
|
||||
}
|
||||
return [value >>> 20, (value >> 10) & 0x3FF, value & 0x3FF];
|
||||
}
|
||||
|
||||
function loadColor(al, data, idx) {
|
||||
// color buckets are hardcoded to 4 chars
|
||||
let value = 0;
|
||||
for (let i = idx * 4; i < idx * 4 + 4; ++i) {
|
||||
value *= al.length;
|
||||
value += al.indexOf(data[i]);
|
||||
}
|
||||
return [value >>> 16, (value >> 8) & 0xFF, value & 0xFF];
|
||||
}
|
||||
|
||||
function lookupIdx(al, data, length, gOffset, name) {
|
||||
const bl = data.length / length;
|
||||
let offset = (hash(0, name) % length) * bl;
|
||||
let [g, v, crc] = loadData(data.slice(offset, offset + bl), al);
|
||||
offset = g < gOffset
|
||||
? (-(g - gOffset) - 1) * bl
|
||||
: (hash(g - gOffset, name) % length) * bl;
|
||||
[_, v, _] = loadData(data.slice(offset, offset + bl), al);
|
||||
offset = v * bl;
|
||||
[_, _, crc] = loadData(data.slice(offset, offset + bl), al);
|
||||
return crc10atm(name) === crc ? v : -1;
|
||||
}
|
||||
|
||||
function lookup(al, data, colorData, length, gOffset, name) {
|
||||
const idx = lookupIdx(al, data, length, gOffset, name);
|
||||
if (idx === -1) return;
|
||||
return loadColor(al, colorData, idx);
|
||||
}
|
||||
|
||||
function createModule(tableData, colorData, alphabet, gOffset, length) {
|
||||
const TMPL = `/**
|
||||
* Copyright (c) 2021 The xterm.js authors. All rights reserved.
|
||||
* @license MIT
|
||||
*/
|
||||
|
||||
/**
|
||||
* This module enables X11 color name lookups with the help of a perfect hash function
|
||||
* to not penalize the package size too much.
|
||||
* saving: ~70% (from 17kB down to 5kB)
|
||||
*/
|
||||
|
||||
// Note: module and table data created with fixtures/x11-colornames/create_module.js
|
||||
const TABLE = '${tableData}';
|
||||
const COLORS = '${colorData}';
|
||||
const ALPHABET = '${alphabet}';
|
||||
const OFFSET = ${gOffset};
|
||||
const LENGTH = ${length};
|
||||
const BUCKET_LENGTH = TABLE.length / LENGTH;
|
||||
|
||||
// match greyXX|grayXX names
|
||||
const rexGrey = /^gr[ae]y(\\d+)/;
|
||||
// name match
|
||||
const rexName = /^\\w?[A-Za-z0-9 ]+\\w/;
|
||||
|
||||
function gray(n: number): [number, number, number] | undefined {
|
||||
if (0 <= n && n <= 100) {
|
||||
const v = Math.floor((n * 256 - n + 50) / 100);
|
||||
return [v, v, v];
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
function hash(d: number, name: string): number {
|
||||
if (!d) d = 0x01000193;
|
||||
for (const c of name) {
|
||||
d = ((d * 0x01000193) ^ c.charCodeAt(0)) & 0xffffffff;
|
||||
}
|
||||
return d >>> 0;
|
||||
}
|
||||
|
||||
function crc10(name: string, crc?: number): number {
|
||||
if (!crc) crc = 0;
|
||||
for (const c of name) {
|
||||
crc ^= c.charCodeAt(0) << 2;
|
||||
for (let k = 0; k < 8; k++) {
|
||||
crc = crc & 0x200 ? (crc << 1) ^ 0x233 : crc << 1;
|
||||
}
|
||||
}
|
||||
crc &= 0x3ff;
|
||||
return crc >>> 0;
|
||||
}
|
||||
|
||||
function loadData(idx: number): [number, number, number] {
|
||||
let value = 0;
|
||||
for (let i = idx * BUCKET_LENGTH; i < idx * BUCKET_LENGTH + BUCKET_LENGTH; ++i) {
|
||||
value *= ALPHABET.length;
|
||||
value += ALPHABET.indexOf(TABLE[i]);
|
||||
}
|
||||
return [value >>> 20, (value >> 10) & 0x3FF, value & 0x3FF];
|
||||
}
|
||||
|
||||
function loadColor(idx: number): [number, number, number] {
|
||||
// color buckets are hardcoded to 4 chars
|
||||
let v = 0;
|
||||
for (let i = idx * 4; i < idx * 4 + 4; ++i) {
|
||||
v *= ALPHABET.length;
|
||||
v += ALPHABET.indexOf(COLORS[i]);
|
||||
}
|
||||
return [v >>> 16, (v >> 8) & 0xFF, v & 0xFF];
|
||||
}
|
||||
|
||||
function lookupIdx(name: string): number {
|
||||
let b = loadData(hash(0, name) % LENGTH);
|
||||
b = loadData(b[0] < OFFSET ? (-(b[0] - OFFSET) - 1) : hash(b[0] - OFFSET, name) % LENGTH);
|
||||
const [ , , crc] = loadData(b[1]);
|
||||
return crc10(name) === crc ? b[1] : -1;
|
||||
}
|
||||
|
||||
export function getColorFromName(name: string): [number, number, number] | undefined {
|
||||
// basic name filtering
|
||||
if (name.length < 3 || name.length > 22 || !rexName.exec(name)) return;
|
||||
|
||||
// handle grays special
|
||||
const m = rexGrey.exec(name);
|
||||
if (m) return gray(parseInt(m[1]));
|
||||
|
||||
// grab crc checked idx from PHF
|
||||
const idx = lookupIdx(name);
|
||||
if (idx === -1) return;
|
||||
return loadColor(idx);
|
||||
}
|
||||
`;
|
||||
return TMPL;
|
||||
}
|
||||
|
||||
|
||||
function main() {
|
||||
// parse definitions from X11 file
|
||||
const colorList = parseX11Colors(path.join(__dirname, '/rgb.txt'))
|
||||
.filter(el => !rexGrey.exec(el.name)); // remove greys, as we can reconstruct them later
|
||||
const nameList = colorList.map(el => el.name);
|
||||
|
||||
// PHF creation and test
|
||||
const [g, v] = createMinimalPerfectHash(nameList);
|
||||
if (!checkPerfectHashTables(g, v, nameList)) {
|
||||
console.error('\x1b[31mPHF creation failed.\x1b[m');
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log('\x1b[32mPHF creation successful.\x1b[m');
|
||||
}
|
||||
|
||||
// calc 10-bit CRCs to rule to avoid collisions at ~1:1024
|
||||
const crc = nameList.map(el => crc10atm(el));
|
||||
|
||||
// collision check against colorNames - must all collide
|
||||
console.log('Self collision test:')
|
||||
const collSelf = checkColorList(g, v, crc, nameList);
|
||||
console.log('self:', {
|
||||
entries: collSelf.tested,
|
||||
matches: collSelf.match.length,
|
||||
rate: collSelf.tested / collSelf.match.length
|
||||
});
|
||||
if (collSelf.match.length !== collSelf.tested) {
|
||||
console.error('\x1b[31mSelf collision tested failed.\x1b[m');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// basic collision checks against dictionaries (rates should be greater than 900:1)
|
||||
console.log('Dictionary collision test:')
|
||||
const collEnglish = checkWordlists(g, v, crc, nameList, '/usr/share/dict/words');
|
||||
console.log('english:', {
|
||||
entries: collEnglish.tested,
|
||||
collisions: collEnglish.collisions.length,
|
||||
rate: collEnglish.tested / collEnglish.collisions.length
|
||||
});
|
||||
//console.log(collEnglish.collisions);
|
||||
const collnGerman = checkWordlists(g, v, crc, nameList, '/usr/share/dict/ngerman');
|
||||
console.log('ngerman:', {
|
||||
entries: collnGerman.tested,
|
||||
collisions: collnGerman.collisions.length,
|
||||
rate: collnGerman.tested / collnGerman.collisions.length
|
||||
});
|
||||
const colloGerman = checkWordlists(g, v, crc, nameList, '/usr/share/dict/ogerman');
|
||||
console.log('ogerman:', {
|
||||
entries: colloGerman.tested,
|
||||
collisions: colloGerman.collisions.length,
|
||||
rate: colloGerman.tested / colloGerman.collisions.length
|
||||
});
|
||||
|
||||
// compression with custom alphabet
|
||||
const ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!?';
|
||||
const [tableData, length, gOffset] = compressTables(g, v, crc, ALPHABET);
|
||||
const colorData = compressColors(colorList.map(el => el.color), ALPHABET);
|
||||
|
||||
// test loading of compressed table data
|
||||
let failed = false;
|
||||
for (const name of nameList) {
|
||||
const orig = lookupPerfectHash(g, v, name);
|
||||
const idx = lookupIdx(ALPHABET, tableData, length, gOffset, name);
|
||||
if (idx !== orig) {
|
||||
console.log('\x1b[31mcompressed loading failed for:\x1b[m', {name, idx, orig});
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
if (failed) process.exit(1);
|
||||
|
||||
// test full color decoding
|
||||
for (const color of colorList) {
|
||||
const loaded = lookup(ALPHABET, tableData, colorData, length, gOffset, color.name);
|
||||
if (loaded[0] !== color.color[0] || loaded[1] !== color.color[1] || loaded[2] !== color.color[2]) {
|
||||
console.log('\x1b[31mcolor loading failed for:\x1b[m', {color, loaded});
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
if (failed) process.exit(1);
|
||||
|
||||
// if we made it up to here, create decoding boilerplate
|
||||
const moduleData = createModule(tableData, colorData, ALPHABET, gOffset, length);
|
||||
fs.writeFileSync(path.join(__dirname, '/ColorNames.ts'), moduleData);
|
||||
console.log(`${moduleData.length} bytes written to ${path.join(__dirname, '/ColorNames.ts')}`);
|
||||
}
|
||||
|
||||
main();
|
||||
File diff suppressed because it is too large
Load Diff
@@ -55,7 +55,7 @@ class TestInputHandler extends InputHandler {
|
||||
}
|
||||
}
|
||||
|
||||
public parseColorSpec(data: string): void | [number, number, number] {
|
||||
public parseColorSpec(data: string): undefined | [number, number, number] {
|
||||
return this._parseColorSpec(data);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,6 @@ import { ICoreService, IBufferService, IOptionsService, ILogService, IDirtyRowSe
|
||||
import { OscHandler } from 'common/parser/OscParser';
|
||||
import { DcsHandler } from 'common/parser/DcsParser';
|
||||
import { IBuffer } from 'common/buffer/Types';
|
||||
import { getColorFromName } from 'common/data/ColorNames';
|
||||
|
||||
/**
|
||||
* Map collect to glevel. Used in `selectCharset`.
|
||||
@@ -2877,7 +2876,7 @@ export class InputHandler extends Disposable implements IInputHandler {
|
||||
* All other formats like rgbi: or device-independent string specifications
|
||||
* with float numbering are not supported.
|
||||
*/
|
||||
protected _parseColorSpec(data: string): [number, number, number] | void {
|
||||
protected _parseColorSpec(data: string): [number, number, number] | undefined {
|
||||
if (!data) return;
|
||||
// also handle uppercases
|
||||
let low = data.toLowerCase();
|
||||
@@ -2908,7 +2907,6 @@ export class InputHandler extends Disposable implements IInputHandler {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
return getColorFromName(data);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user