Allow selection in the a11y tree and sync the selection to terminal

This commit is contained in:
Jason Lin
2023-10-11 16:23:03 +11:00
parent 34017f32fb
commit 1b940de0bf
5 changed files with 239 additions and 53 deletions
+10 -1
View File
@@ -140,7 +140,7 @@
cursor: crosshair;
}
.xterm .xterm-accessibility,
.xterm .xterm-accessibility:not(.debug),
.xterm .xterm-message {
position: absolute;
left: 0;
@@ -152,6 +152,15 @@
pointer-events: none;
}
.xterm .xterm-accessibility-tree:not(.debug) *::selection {
color: transparent;
}
.xterm .xterm-accessibility-tree {
user-select: text;
white-space: pre;
}
.xterm .live-region {
position: absolute;
left: -9999px;
+131 -3
View File
@@ -19,11 +19,17 @@ const enum BoundaryPosition {
BOTTOM
}
// Turn this on to unhide the accessibility tree and display it under
// (instead of overlapping with) the terminal.
const DEBUG = false;
export class AccessibilityManager extends Disposable {
private _debugRootContainer: HTMLElement | undefined;
private _accessibilityContainer: HTMLElement;
private _rowContainer: HTMLElement;
private _rowElements: HTMLElement[];
private _rowColumns: WeakMap<HTMLElement, number[]> = new WeakMap();
private _liveRegion: HTMLElement;
private _liveRegionLineCount: number = 0;
@@ -81,7 +87,23 @@ export class AccessibilityManager extends Disposable {
if (!this._terminal.element) {
throw new Error('Cannot enable accessibility before Terminal.open');
}
this._terminal.element.insertAdjacentElement('afterbegin', this._accessibilityContainer);
if (DEBUG) {
this._accessibilityContainer.classList.add('debug');
this._rowContainer.classList.add('debug');
// Use a `<div class="xterm">` container so that the css will still apply.
this._debugRootContainer = document.createElement('div');
this._debugRootContainer.classList.add('xterm');
this._debugRootContainer.appendChild(document.createTextNode('------start a11y------'));
this._debugRootContainer.appendChild(this._accessibilityContainer);
this._debugRootContainer.appendChild(document.createTextNode('------end a11y------'));
this._terminal.element.insertAdjacentElement('afterend', this._debugRootContainer);
} else {
this._terminal.element.insertAdjacentElement('afterbegin', this._accessibilityContainer);
}
this.register(this._terminal.onResize(e => this._handleResize(e.rows)));
this.register(this._terminal.onRender(e => this._refreshRows(e.start, e.end)));
@@ -93,6 +115,7 @@ export class AccessibilityManager extends Disposable {
this.register(this._terminal.onKey(e => this._handleKey(e.key)));
this.register(this._terminal.onBlur(() => this._clearLiveRegion()));
this.register(this._renderService.onDimensionsChange(() => this._refreshRowsDimensions()));
this.register(addDisposableDomListener(document, 'selectionchange', () => this._handleSelectionChange()));
this._screenDprMonitor = new ScreenDprMonitor(window);
this.register(this._screenDprMonitor);
@@ -103,7 +126,11 @@ export class AccessibilityManager extends Disposable {
this._refreshRows();
this.register(toDisposable(() => {
this._accessibilityContainer.remove();
if (DEBUG) {
this._debugRootContainer!.remove();
} else {
this._accessibilityContainer.remove();
}
this._rowElements.length = 0;
}));
}
@@ -156,14 +183,18 @@ export class AccessibilityManager extends Disposable {
const buffer: IBuffer = this._terminal.buffer;
const setSize = buffer.lines.length.toString();
for (let i = start; i <= end; i++) {
const lineData = buffer.translateBufferLineToString(buffer.ydisp + i, true);
const line = buffer.lines.get(buffer.ydisp + i);
const columns: number[] = [];
const lineData = line?.translateToString(true, undefined, undefined, columns) || '';
const posInSet = (buffer.ydisp + i + 1).toString();
const element = this._rowElements[i];
if (element) {
if (lineData.length === 0) {
element.innerText = '\u00a0';
this._rowColumns.set(element, [0]);
} else {
element.textContent = lineData;
this._rowColumns.set(element, columns);
}
element.setAttribute('aria-posinset', posInSet);
element.setAttribute('aria-setsize', setSize);
@@ -240,6 +271,103 @@ export class AccessibilityManager extends Disposable {
e.stopImmediatePropagation();
}
private _handleSelectionChange(): void {
if (this._rowElements.length === 0) {
return;
}
const selection = document.getSelection();
if (!selection) {
return;
}
if (selection.isCollapsed) {
// Only do something when the anchorNode is inside the row container. This
// behavior mirrors what we do with mouse --- if the mouse clicks
// somewhere outside of the terminal, we don't clear the selection.
if (this._rowContainer.contains(selection.anchorNode)) {
this._terminal.clearSelection();
}
return;
}
if (!selection.anchorNode || !selection.focusNode) {
console.error('anchorNode and/or focusNode are null');
return;
}
// Sort the two selection points in document order.
let begin = { node: selection.anchorNode, offset: selection.anchorOffset };
let end = { node: selection.focusNode, offset: selection.focusOffset };
if ((begin.node.compareDocumentPosition(end.node) & Node.DOCUMENT_POSITION_PRECEDING) || (begin.node === end.node && begin.offset > end.offset) ) {
[begin, end] = [end, begin];
}
// Clamp begin/end to the inside of the row container.
if (begin.node.compareDocumentPosition(this._rowElements[0]) & (Node.DOCUMENT_POSITION_CONTAINED_BY | Node.DOCUMENT_POSITION_FOLLOWING)) {
begin = { node: this._rowElements[0].childNodes[0], offset: 0 };
}
if (!this._rowContainer.contains(begin.node)) {
// This happens when `begin` is below the last row.
return;
}
const lastRowElement = this._rowElements.slice(-1)[0];
if (end.node.compareDocumentPosition(lastRowElement) & (Node.DOCUMENT_POSITION_CONTAINED_BY | Node.DOCUMENT_POSITION_PRECEDING)) {
end = {
node: lastRowElement,
offset: lastRowElement.textContent?.length ?? 0
};
}
if (!this._rowContainer.contains(end.node)) {
// This happens when `end` is above the first row.
return;
}
const toRowColumn = ({ node, offset }: typeof begin): {row: number, column: number} | null => {
// `node` is either the row element or the Text node inside it.
const rowElement: any = node instanceof Text ? node.parentNode : node;
let row = parseInt(rowElement?.getAttribute('aria-posinset'), 10) - 1;
if (isNaN(row)) {
console.warn('row is invalid. Race condition?');
return null;
}
const columns = this._rowColumns.get(rowElement);
if (!columns) {
console.warn('columns is null. Race condition?');
return null;
}
let column = offset < columns.length ? columns[offset] : columns.slice(-1)[0] + 1;
if (column >= this._terminal.cols) {
++row;
column = 0;
}
return {
row,
column
};
};
const beginRowColumn = toRowColumn(begin);
const endRowColumn = toRowColumn(end);
if (!beginRowColumn || !endRowColumn) {
return;
}
if (beginRowColumn.row > endRowColumn.row || (beginRowColumn.row === endRowColumn.row && beginRowColumn.column >= endRowColumn.column)) {
// This should not happen unless we have some bugs.
throw new Error('invalid range');
}
this._terminal.select(
beginRowColumn.column,
beginRowColumn.row,
(endRowColumn.row - beginRowColumn.row) * this._terminal.cols - beginRowColumn.column + endRowColumn.column
);
}
private _handleResize(rows: number): void {
// Remove bottom boundary listener
this._rowElements[this._rowElements.length - 1].removeEventListener('focus', this._bottomBoundaryFocusListener);
+1 -1
View File
@@ -245,7 +245,7 @@ export interface IBufferLine {
clone(): IBufferLine;
getTrimmedLength(): number;
getNoBgTrimmedLength(): number;
translateToString(trimRight?: boolean, startCol?: number, endCol?: number): string;
translateToString(trimRight?: boolean, startCol?: number, endCol?: number, outColumns?: number[]): string;
/* direct access to cell attrs */
getWidth(index: number): number;
+76 -45
View File
@@ -331,56 +331,75 @@ describe('BufferLine', function(): void {
describe('translateToString with and w\'o trimming', function(): void {
it('empty line', function(): void {
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
assert.equal(line.translateToString(false), ' ');
assert.equal(line.translateToString(true), '');
const columns: number[] = [];
assert.equal(line.translateToString(false, undefined, undefined, columns), ' ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
assert.equal(line.translateToString(true, undefined, undefined, columns), '');
assert.deepEqual(columns, []);
});
it('ASCII', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(4, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(5, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
assert.equal(line.translateToString(false), 'a a aa ');
assert.equal(line.translateToString(true), 'a a aa');
assert.equal(line.translateToString(false, 0, 5), 'a a a');
assert.equal(line.translateToString(false, 0, 4), 'a a ');
assert.equal(line.translateToString(false, 0, 3), 'a a');
assert.equal(line.translateToString(true, 0, 5), 'a a a');
assert.equal(line.translateToString(true, 0, 4), 'a a ');
assert.equal(line.translateToString(true, 0, 3), 'a a');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a a aa ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a a aa');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5]);
for (const trimRight of [true, false]) {
assert.equal(line.translateToString(trimRight, 0, 5, columns), 'a a a');
assert.deepEqual(columns, [0, 1, 2, 3, 4]);
assert.equal(line.translateToString(trimRight, 0, 4, columns), 'a a ');
assert.deepEqual(columns, [0, 1, 2, 3]);
assert.equal(line.translateToString(trimRight, 0, 3, columns), 'a a');
assert.deepEqual(columns, [0, 1, 2]);
}
});
it('surrogate', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, '𝄞', 1, '𝄞'.charCodeAt(0)]));
line.setCell(4, CellData.fromCharData([1, '𝄞', 1, '𝄞'.charCodeAt(0)]));
line.setCell(5, CellData.fromCharData([1, '𝄞', 1, '𝄞'.charCodeAt(0)]));
assert.equal(line.translateToString(false), 'a 𝄞 𝄞𝄞 ');
assert.equal(line.translateToString(true), 'a 𝄞 𝄞𝄞');
assert.equal(line.translateToString(false, 0, 5), 'a 𝄞 𝄞');
assert.equal(line.translateToString(false, 0, 4), 'a 𝄞 ');
assert.equal(line.translateToString(false, 0, 3), 'a 𝄞');
assert.equal(line.translateToString(true, 0, 5), 'a 𝄞 𝄞');
assert.equal(line.translateToString(true, 0, 4), 'a 𝄞 ');
assert.equal(line.translateToString(true, 0, 3), 'a 𝄞');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a 𝄞 𝄞𝄞 ');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 8, 9]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a 𝄞 𝄞𝄞');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4, 5, 5]);
for (const trimRight of [true, false]) {
assert.equal(line.translateToString(trimRight, 0, 5, columns), 'a 𝄞 𝄞');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4]);
assert.equal(line.translateToString(trimRight, 0, 4, columns), 'a 𝄞 ');
assert.deepEqual(columns, [0, 1, 2, 2, 3]);
assert.equal(line.translateToString(trimRight, 0, 3, columns), 'a 𝄞');
assert.deepEqual(columns, [0, 1, 2, 2]);
}
});
it('combining', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, 'e\u0301', 1, '\u0301'.charCodeAt(0)]));
line.setCell(4, CellData.fromCharData([1, 'e\u0301', 1, '\u0301'.charCodeAt(0)]));
line.setCell(5, CellData.fromCharData([1, 'e\u0301', 1, '\u0301'.charCodeAt(0)]));
assert.equal(line.translateToString(false), 'a e\u0301 e\u0301e\u0301 ');
assert.equal(line.translateToString(true), 'a e\u0301 e\u0301e\u0301');
assert.equal(line.translateToString(false, 0, 5), 'a e\u0301 e\u0301');
assert.equal(line.translateToString(false, 0, 4), 'a e\u0301 ');
assert.equal(line.translateToString(false, 0, 3), 'a e\u0301');
assert.equal(line.translateToString(true, 0, 5), 'a e\u0301 e\u0301');
assert.equal(line.translateToString(true, 0, 4), 'a e\u0301 ');
assert.equal(line.translateToString(true, 0, 3), 'a e\u0301');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a e\u0301 e\u0301e\u0301 ');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 8, 9]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a e\u0301 e\u0301e\u0301');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4, 5, 5]);
for (const trimRight of [true, false]) {
assert.equal(line.translateToString(trimRight, 0, 5, columns), 'a e\u0301 e\u0301');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4]);
assert.equal(line.translateToString(trimRight, 0, 4, columns), 'a e\u0301 ');
assert.deepEqual(columns, [0, 1, 2, 2, 3]);
assert.equal(line.translateToString(trimRight, 0, 3, columns), 'a e\u0301');
assert.deepEqual(columns, [0, 1, 2, 2]);
}
});
it('fullwidth', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, '', 2, ''.charCodeAt(0)]));
@@ -389,43 +408,55 @@ describe('BufferLine', function(): void {
line.setCell(6, CellData.fromCharData([0, '', 0, 0]));
line.setCell(7, CellData.fromCharData([1, '', 2, ''.charCodeAt(0)]));
line.setCell(8, CellData.fromCharData([0, '', 0, 0]));
assert.equal(line.translateToString(false), 'a 11 ');
assert.equal(line.translateToString(true), 'a 11');
assert.equal(line.translateToString(false, 0, 7), 'a ');
assert.equal(line.translateToString(false, 0, 6), 'a ');
assert.equal(line.translateToString(false, 0, 5), 'a ');
assert.equal(line.translateToString(false, 0, 4), 'a ');
assert.equal(line.translateToString(false, 0, 3), 'a ');
assert.equal(line.translateToString(false, 0, 2), 'a ');
assert.equal(line.translateToString(true, 0, 7), 'a ');
assert.equal(line.translateToString(true, 0, 6), 'a ');
assert.equal(line.translateToString(true, 0, 5), 'a ');
assert.equal(line.translateToString(true, 0, 4), 'a ');
assert.equal(line.translateToString(true, 0, 3), 'a ');
assert.equal(line.translateToString(true, 0, 2), 'a ');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a 11 ');
assert.deepEqual(columns, [0, 1, 2, 4, 5, 7, 9]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a ');
assert.deepEqual(columns, [0, 1, 2, 4, 5, 7]);
for (const trimRight of [true, false]) {
assert.equal(line.translateToString(trimRight, 0, 7, columns), 'a ');
assert.deepEqual(columns, [0, 1, 2, 4, 5]);
assert.equal(line.translateToString(trimRight, 0, 6, columns), 'a ');
assert.deepEqual(columns, [0, 1, 2, 4, 5]);
assert.equal(line.translateToString(trimRight, 0, 5, columns), 'a ');
assert.deepEqual(columns, [0, 1, 2, 4]);
assert.equal(line.translateToString(trimRight, 0, 4, columns), 'a ');
assert.deepEqual(columns, [0, 1, 2]);
assert.equal(line.translateToString(trimRight, 0, 3, columns), 'a ');
assert.deepEqual(columns, [0, 1, 2]);
assert.equal(line.translateToString(trimRight, 0, 2, columns), 'a ');
assert.deepEqual(columns, [0, 1]);
}
});
it('space at end', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(4, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(5, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(6, CellData.fromCharData([1, ' ', 1, ' '.charCodeAt(0)]));
assert.equal(line.translateToString(false), 'a a aa ');
assert.equal(line.translateToString(true), 'a a aa ');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a a aa ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a a aa ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6]);
});
it('should always return some sane value', function(): void {
const columns: number[] = [];
// sanity check - broken line with invalid out of bound null width cells
// this can atm happen with deleting/inserting chars in inputhandler by "breaking"
// fullwidth pairs --> needs to be fixed after settling BufferLine impl
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, 0, NULL_CELL_CODE]), false);
assert.equal(line.translateToString(false), ' ');
assert.equal(line.translateToString(true), '');
assert.equal(line.translateToString(false, undefined, undefined, columns), ' ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
assert.equal(line.translateToString(true, undefined, undefined, columns), '');
assert.deepEqual(columns, []);
});
it('should work with endCol=0', () => {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, 0, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
assert.equal(line.translateToString(true, 0, 0), '');
assert.equal(line.translateToString(true, 0, 0, columns), '');
assert.deepEqual(columns, []);
});
});
describe('addCharToCell', () => {
+21 -3
View File
@@ -508,16 +508,34 @@ export class BufferLine implements IBufferLine {
}
}
public translateToString(trimRight: boolean = false, startCol: number = 0, endCol: number = this.length): string {
/**
* If outColumns is specified, it will be filled with column numbers such that
* returnedString[i] is display at outColumns[i] column. When a single cell is
* translated to multiple UTF-16 code units (e.g. surrogate pair) in the
* returned string, the corresponding entries in outColumns will have the same
* column number.
*/
public translateToString(trimRight?: boolean, startCol?: number, endCol?: number, outColumns?: number[]): string {
startCol = startCol ?? 0;
endCol = endCol ?? this.length;
if (trimRight) {
endCol = Math.min(endCol, this.getTrimmedLength());
}
if (outColumns) {
outColumns.length = 0;
}
let result = '';
while (startCol < endCol) {
const content = this._data[startCol * CELL_SIZE + Cell.CONTENT];
const cp = content & Content.CODEPOINT_MASK;
result += (content & Content.IS_COMBINED_MASK) ? this._combined[startCol] : (cp) ? stringFromCodePoint(cp) : WHITESPACE_CELL_CHAR;
startCol += (content >> Content.WIDTH_SHIFT) || 1; // always advance by 1
const chars = (content & Content.IS_COMBINED_MASK) ? this._combined[startCol] : (cp) ? stringFromCodePoint(cp) : WHITESPACE_CELL_CHAR;
result += chars;
if (outColumns) {
for (let i = 0; i < chars.length; ++i) {
outColumns.push(startCol);
}
}
startCol += (content >> Content.WIDTH_SHIFT) || 1; // always advance by at least 1
}
return result;
}