Merge pull request #4742 from JasonXJ/sync-select

Allow selection in the a11y tree and sync the selection to terminal
This commit is contained in:
Daniel Imms
2023-11-03 09:08:53 -07:00
committed by GitHub
5 changed files with 249 additions and 53 deletions
+10 -1
View File
@@ -140,7 +140,7 @@
cursor: crosshair;
}
.xterm .xterm-accessibility,
.xterm .xterm-accessibility:not(.debug),
.xterm .xterm-message {
position: absolute;
left: 0;
@@ -152,6 +152,15 @@
pointer-events: none;
}
.xterm .xterm-accessibility-tree:not(.debug) *::selection {
color: transparent;
}
.xterm .xterm-accessibility-tree {
user-select: text;
white-space: pre;
}
.xterm .live-region {
position: absolute;
left: -9999px;
+132 -3
View File
@@ -10,6 +10,7 @@ import { Disposable, toDisposable } from 'common/Lifecycle';
import { ICoreBrowserService, IRenderService } from 'browser/services/Services';
import { IBuffer } from 'common/buffer/Types';
import { IInstantiationService } from 'common/services/Services';
import { addDisposableDomListener } from 'browser/Lifecycle';
const MAX_ROWS_TO_READ = 20;
@@ -18,11 +19,17 @@ const enum BoundaryPosition {
BOTTOM
}
// Turn this on to unhide the accessibility tree and display it under
// (instead of overlapping with) the terminal.
const DEBUG = false;
export class AccessibilityManager extends Disposable {
private _debugRootContainer: HTMLElement | undefined;
private _accessibilityContainer: HTMLElement;
private _rowContainer: HTMLElement;
private _rowElements: HTMLElement[];
private _rowColumns: WeakMap<HTMLElement, number[]> = new WeakMap();
private _liveRegion: HTMLElement;
private _liveRegionLineCount: number = 0;
@@ -80,7 +87,23 @@ export class AccessibilityManager extends Disposable {
if (!this._terminal.element) {
throw new Error('Cannot enable accessibility before Terminal.open');
}
this._terminal.element.insertAdjacentElement('afterbegin', this._accessibilityContainer);
if (DEBUG) {
this._accessibilityContainer.classList.add('debug');
this._rowContainer.classList.add('debug');
// Use a `<div class="xterm">` container so that the css will still apply.
this._debugRootContainer = document.createElement('div');
this._debugRootContainer.classList.add('xterm');
this._debugRootContainer.appendChild(document.createTextNode('------start a11y------'));
this._debugRootContainer.appendChild(this._accessibilityContainer);
this._debugRootContainer.appendChild(document.createTextNode('------end a11y------'));
this._terminal.element.insertAdjacentElement('afterend', this._debugRootContainer);
} else {
this._terminal.element.insertAdjacentElement('afterbegin', this._accessibilityContainer);
}
this.register(this._terminal.onResize(e => this._handleResize(e.rows)));
this.register(this._terminal.onRender(e => this._refreshRows(e.start, e.end)));
@@ -92,11 +115,16 @@ export class AccessibilityManager extends Disposable {
this.register(this._terminal.onKey(e => this._handleKey(e.key)));
this.register(this._terminal.onBlur(() => this._clearLiveRegion()));
this.register(this._renderService.onDimensionsChange(() => this._refreshRowsDimensions()));
this.register(addDisposableDomListener(document, 'selectionchange', () => this._handleSelectionChange()));
this.register(this._coreBrowserService.onDprChange(() => this._refreshRowsDimensions()));
this._refreshRows();
this.register(toDisposable(() => {
this._accessibilityContainer.remove();
if (DEBUG) {
this._debugRootContainer!.remove();
} else {
this._accessibilityContainer.remove();
}
this._rowElements.length = 0;
}));
}
@@ -149,14 +177,18 @@ export class AccessibilityManager extends Disposable {
const buffer: IBuffer = this._terminal.buffer;
const setSize = buffer.lines.length.toString();
for (let i = start; i <= end; i++) {
const lineData = buffer.translateBufferLineToString(buffer.ydisp + i, true);
const line = buffer.lines.get(buffer.ydisp + i);
const columns: number[] = [];
const lineData = line?.translateToString(true, undefined, undefined, columns) || '';
const posInSet = (buffer.ydisp + i + 1).toString();
const element = this._rowElements[i];
if (element) {
if (lineData.length === 0) {
element.innerText = '\u00a0';
this._rowColumns.set(element, [0, 1]);
} else {
element.textContent = lineData;
this._rowColumns.set(element, columns);
}
element.setAttribute('aria-posinset', posInSet);
element.setAttribute('aria-setsize', setSize);
@@ -233,6 +265,103 @@ export class AccessibilityManager extends Disposable {
e.stopImmediatePropagation();
}
private _handleSelectionChange(): void {
if (this._rowElements.length === 0) {
return;
}
const selection = document.getSelection();
if (!selection) {
return;
}
if (selection.isCollapsed) {
// Only do something when the anchorNode is inside the row container. This
// behavior mirrors what we do with mouse --- if the mouse clicks
// somewhere outside of the terminal, we don't clear the selection.
if (this._rowContainer.contains(selection.anchorNode)) {
this._terminal.clearSelection();
}
return;
}
if (!selection.anchorNode || !selection.focusNode) {
console.error('anchorNode and/or focusNode are null');
return;
}
// Sort the two selection points in document order.
let begin = { node: selection.anchorNode, offset: selection.anchorOffset };
let end = { node: selection.focusNode, offset: selection.focusOffset };
if ((begin.node.compareDocumentPosition(end.node) & Node.DOCUMENT_POSITION_PRECEDING) || (begin.node === end.node && begin.offset > end.offset) ) {
[begin, end] = [end, begin];
}
// Clamp begin/end to the inside of the row container.
if (begin.node.compareDocumentPosition(this._rowElements[0]) & (Node.DOCUMENT_POSITION_CONTAINED_BY | Node.DOCUMENT_POSITION_FOLLOWING)) {
begin = { node: this._rowElements[0].childNodes[0], offset: 0 };
}
if (!this._rowContainer.contains(begin.node)) {
// This happens when `begin` is below the last row.
return;
}
const lastRowElement = this._rowElements.slice(-1)[0];
if (end.node.compareDocumentPosition(lastRowElement) & (Node.DOCUMENT_POSITION_CONTAINED_BY | Node.DOCUMENT_POSITION_PRECEDING)) {
end = {
node: lastRowElement,
offset: lastRowElement.textContent?.length ?? 0
};
}
if (!this._rowContainer.contains(end.node)) {
// This happens when `end` is above the first row.
return;
}
const toRowColumn = ({ node, offset }: typeof begin): {row: number, column: number} | null => {
// `node` is either the row element or the Text node inside it.
const rowElement: any = node instanceof Text ? node.parentNode : node;
let row = parseInt(rowElement?.getAttribute('aria-posinset'), 10) - 1;
if (isNaN(row)) {
console.warn('row is invalid. Race condition?');
return null;
}
const columns = this._rowColumns.get(rowElement);
if (!columns) {
console.warn('columns is null. Race condition?');
return null;
}
let column = offset < columns.length ? columns[offset] : columns.slice(-1)[0] + 1;
if (column >= this._terminal.cols) {
++row;
column = 0;
}
return {
row,
column
};
};
const beginRowColumn = toRowColumn(begin);
const endRowColumn = toRowColumn(end);
if (!beginRowColumn || !endRowColumn) {
return;
}
if (beginRowColumn.row > endRowColumn.row || (beginRowColumn.row === endRowColumn.row && beginRowColumn.column >= endRowColumn.column)) {
// This should not happen unless we have some bugs.
throw new Error('invalid range');
}
this._terminal.select(
beginRowColumn.column,
beginRowColumn.row,
(endRowColumn.row - beginRowColumn.row) * this._terminal.cols - beginRowColumn.column + endRowColumn.column
);
}
private _handleResize(rows: number): void {
// Remove bottom boundary listener
this._rowElements[this._rowElements.length - 1].removeEventListener('focus', this._bottomBoundaryFocusListener);
+1 -1
View File
@@ -247,7 +247,7 @@ export interface IBufferLine {
clone(): IBufferLine;
getTrimmedLength(): number;
getNoBgTrimmedLength(): number;
translateToString(trimRight?: boolean, startCol?: number, endCol?: number): string;
translateToString(trimRight?: boolean, startCol?: number, endCol?: number, outColumns?: number[]): string;
/* direct access to cell attrs */
getWidth(index: number): number;
+76 -45
View File
@@ -343,56 +343,75 @@ describe('BufferLine', function(): void {
describe('translateToString with and w\'o trimming', function(): void {
it('empty line', function(): void {
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
assert.equal(line.translateToString(false), ' ');
assert.equal(line.translateToString(true), '');
const columns: number[] = [];
assert.equal(line.translateToString(false, undefined, undefined, columns), ' ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
assert.equal(line.translateToString(true, undefined, undefined, columns), '');
assert.deepEqual(columns, [0]);
});
it('ASCII', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(4, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(5, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
assert.equal(line.translateToString(false), 'a a aa ');
assert.equal(line.translateToString(true), 'a a aa');
assert.equal(line.translateToString(false, 0, 5), 'a a a');
assert.equal(line.translateToString(false, 0, 4), 'a a ');
assert.equal(line.translateToString(false, 0, 3), 'a a');
assert.equal(line.translateToString(true, 0, 5), 'a a a');
assert.equal(line.translateToString(true, 0, 4), 'a a ');
assert.equal(line.translateToString(true, 0, 3), 'a a');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a a aa ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a a aa');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6]);
for (const trimRight of [true, false]) {
assert.equal(line.translateToString(trimRight, 0, 5, columns), 'a a a');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5]);
assert.equal(line.translateToString(trimRight, 0, 4, columns), 'a a ');
assert.deepEqual(columns, [0, 1, 2, 3, 4]);
assert.equal(line.translateToString(trimRight, 0, 3, columns), 'a a');
assert.deepEqual(columns, [0, 1, 2, 3]);
}
});
it('surrogate', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, '𝄞', 1, '𝄞'.charCodeAt(0)]));
line.setCell(4, CellData.fromCharData([1, '𝄞', 1, '𝄞'.charCodeAt(0)]));
line.setCell(5, CellData.fromCharData([1, '𝄞', 1, '𝄞'.charCodeAt(0)]));
assert.equal(line.translateToString(false), 'a 𝄞 𝄞𝄞 ');
assert.equal(line.translateToString(true), 'a 𝄞 𝄞𝄞');
assert.equal(line.translateToString(false, 0, 5), 'a 𝄞 𝄞');
assert.equal(line.translateToString(false, 0, 4), 'a 𝄞 ');
assert.equal(line.translateToString(false, 0, 3), 'a 𝄞');
assert.equal(line.translateToString(true, 0, 5), 'a 𝄞 𝄞');
assert.equal(line.translateToString(true, 0, 4), 'a 𝄞 ');
assert.equal(line.translateToString(true, 0, 3), 'a 𝄞');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a 𝄞 𝄞𝄞 ');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a 𝄞 𝄞𝄞');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4, 5, 5, 6]);
for (const trimRight of [true, false]) {
assert.equal(line.translateToString(trimRight, 0, 5, columns), 'a 𝄞 𝄞');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4, 5]);
assert.equal(line.translateToString(trimRight, 0, 4, columns), 'a 𝄞 ');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4]);
assert.equal(line.translateToString(trimRight, 0, 3, columns), 'a 𝄞');
assert.deepEqual(columns, [0, 1, 2, 2, 3]);
}
});
it('combining', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, 'e\u0301', 1, '\u0301'.charCodeAt(0)]));
line.setCell(4, CellData.fromCharData([1, 'e\u0301', 1, '\u0301'.charCodeAt(0)]));
line.setCell(5, CellData.fromCharData([1, 'e\u0301', 1, '\u0301'.charCodeAt(0)]));
assert.equal(line.translateToString(false), 'a e\u0301 e\u0301e\u0301 ');
assert.equal(line.translateToString(true), 'a e\u0301 e\u0301e\u0301');
assert.equal(line.translateToString(false, 0, 5), 'a e\u0301 e\u0301');
assert.equal(line.translateToString(false, 0, 4), 'a e\u0301 ');
assert.equal(line.translateToString(false, 0, 3), 'a e\u0301');
assert.equal(line.translateToString(true, 0, 5), 'a e\u0301 e\u0301');
assert.equal(line.translateToString(true, 0, 4), 'a e\u0301 ');
assert.equal(line.translateToString(true, 0, 3), 'a e\u0301');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a e\u0301 e\u0301e\u0301 ');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a e\u0301 e\u0301e\u0301');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4, 5, 5, 6]);
for (const trimRight of [true, false]) {
assert.equal(line.translateToString(trimRight, 0, 5, columns), 'a e\u0301 e\u0301');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4, 4, 5]);
assert.equal(line.translateToString(trimRight, 0, 4, columns), 'a e\u0301 ');
assert.deepEqual(columns, [0, 1, 2, 2, 3, 4]);
assert.equal(line.translateToString(trimRight, 0, 3, columns), 'a e\u0301');
assert.deepEqual(columns, [0, 1, 2, 2, 3]);
}
});
it('fullwidth', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, '', 2, ''.charCodeAt(0)]));
@@ -401,43 +420,55 @@ describe('BufferLine', function(): void {
line.setCell(6, CellData.fromCharData([0, '', 0, 0]));
line.setCell(7, CellData.fromCharData([1, '', 2, ''.charCodeAt(0)]));
line.setCell(8, CellData.fromCharData([0, '', 0, 0]));
assert.equal(line.translateToString(false), 'a 11 ');
assert.equal(line.translateToString(true), 'a 11');
assert.equal(line.translateToString(false, 0, 7), 'a ');
assert.equal(line.translateToString(false, 0, 6), 'a ');
assert.equal(line.translateToString(false, 0, 5), 'a ');
assert.equal(line.translateToString(false, 0, 4), 'a ');
assert.equal(line.translateToString(false, 0, 3), 'a ');
assert.equal(line.translateToString(false, 0, 2), 'a ');
assert.equal(line.translateToString(true, 0, 7), 'a ');
assert.equal(line.translateToString(true, 0, 6), 'a ');
assert.equal(line.translateToString(true, 0, 5), 'a ');
assert.equal(line.translateToString(true, 0, 4), 'a ');
assert.equal(line.translateToString(true, 0, 3), 'a ');
assert.equal(line.translateToString(true, 0, 2), 'a ');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a 11 ');
assert.deepEqual(columns, [0, 1, 2, 4, 5, 7, 9, 10]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a ');
assert.deepEqual(columns, [0, 1, 2, 4, 5, 7, 9]);
for (const trimRight of [true, false]) {
assert.equal(line.translateToString(trimRight, 0, 7, columns), 'a ');
assert.deepEqual(columns, [0, 1, 2, 4, 5, 7]);
assert.equal(line.translateToString(trimRight, 0, 6, columns), 'a ');
assert.deepEqual(columns, [0, 1, 2, 4, 5, 7]);
assert.equal(line.translateToString(trimRight, 0, 5, columns), 'a ');
assert.deepEqual(columns, [0, 1, 2, 4, 5]);
assert.equal(line.translateToString(trimRight, 0, 4, columns), 'a ');
assert.deepEqual(columns, [0, 1, 2, 4]);
assert.equal(line.translateToString(trimRight, 0, 3, columns), 'a ');
assert.deepEqual(columns, [0, 1, 2, 4]);
assert.equal(line.translateToString(trimRight, 0, 2, columns), 'a ');
assert.deepEqual(columns, [0, 1, 2]);
}
});
it('space at end', function(): void {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, NULL_CELL_WIDTH, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(2, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(4, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(5, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
line.setCell(6, CellData.fromCharData([1, ' ', 1, ' '.charCodeAt(0)]));
assert.equal(line.translateToString(false), 'a a aa ');
assert.equal(line.translateToString(true), 'a a aa ');
assert.equal(line.translateToString(false, undefined, undefined, columns), 'a a aa ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
assert.equal(line.translateToString(true, undefined, undefined, columns), 'a a aa ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6, 7]);
});
it('should always return some sane value', function(): void {
const columns: number[] = [];
// sanity check - broken line with invalid out of bound null width cells
// this can atm happen with deleting/inserting chars in inputhandler by "breaking"
// fullwidth pairs --> needs to be fixed after settling BufferLine impl
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, 0, NULL_CELL_CODE]), false);
assert.equal(line.translateToString(false), ' ');
assert.equal(line.translateToString(true), '');
assert.equal(line.translateToString(false, undefined, undefined, columns), ' ');
assert.deepEqual(columns, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
assert.equal(line.translateToString(true, undefined, undefined, columns), '');
assert.deepEqual(columns, [0]);
});
it('should work with endCol=0', () => {
const columns: number[] = [];
const line = new TestBufferLine(10, CellData.fromCharData([DEFAULT_ATTR, NULL_CELL_CHAR, 0, NULL_CELL_CODE]), false);
line.setCell(0, CellData.fromCharData([1, 'a', 1, 'a'.charCodeAt(0)]));
assert.equal(line.translateToString(true, 0, 0), '');
assert.equal(line.translateToString(true, 0, 0, columns), '');
assert.deepEqual(columns, [0]);
});
});
describe('addCharToCell', () => {
+30 -3
View File
@@ -508,16 +508,43 @@ export class BufferLine implements IBufferLine {
}
}
public translateToString(trimRight: boolean = false, startCol: number = 0, endCol: number = this.length): string {
/**
* Translates the buffer line to a string.
*
* @param trimRight Whether to trim any empty cells on the right.
* @param startCol The column to start the string (0-based inclusive).
* @param endCol The column to end the string (0-based exclusive).
* @param outColumns if specified, this array will be filled with column numbers such that
* `returnedString[i]` is displayed at `outColumns[i]` column. `outColumns[returnedString.length]`
* is where the character following `returnedString` will be displayed.
*
* When a single cell is translated to multiple UTF-16 code units (e.g. surrogate pair) in the
* returned string, the corresponding entries in `outColumns` will have the same column number.
*/
public translateToString(trimRight?: boolean, startCol?: number, endCol?: number, outColumns?: number[]): string {
startCol = startCol ?? 0;
endCol = endCol ?? this.length;
if (trimRight) {
endCol = Math.min(endCol, this.getTrimmedLength());
}
if (outColumns) {
outColumns.length = 0;
}
let result = '';
while (startCol < endCol) {
const content = this._data[startCol * CELL_SIZE + Cell.CONTENT];
const cp = content & Content.CODEPOINT_MASK;
result += (content & Content.IS_COMBINED_MASK) ? this._combined[startCol] : (cp) ? stringFromCodePoint(cp) : WHITESPACE_CELL_CHAR;
startCol += (content >> Content.WIDTH_SHIFT) || 1; // always advance by 1
const chars = (content & Content.IS_COMBINED_MASK) ? this._combined[startCol] : (cp) ? stringFromCodePoint(cp) : WHITESPACE_CELL_CHAR;
result += chars;
if (outColumns) {
for (let i = 0; i < chars.length; ++i) {
outColumns.push(startCol);
}
}
startCol += (content >> Content.WIDTH_SHIFT) || 1; // always advance by at least 1
}
if (outColumns) {
outColumns.push(startCol);
}
return result;
}