From 1d0cf32b280295bbd30304902838c7b23ec24862 Mon Sep 17 00:00:00 2001 From: Anthony Kim Date: Wed, 11 Feb 2026 14:19:27 -0800 Subject: [PATCH] support z-index + dual canvas for DOM. Doesnt work for webgl --- addons/addon-image/src/ImageRenderer.ts | 115 ++++++++++++------ addons/addon-image/src/ImageStorage.ts | 50 ++++++-- addons/addon-image/src/Types.ts | 3 + .../src/kitty/KittyGraphicsHandler.ts | 12 +- .../src/kitty/KittyGraphicsTypes.test.ts | 20 +++ .../src/kitty/KittyGraphicsTypes.ts | 6 +- addons/addon-image/test/KittyGraphics.test.ts | 79 ++++++++++++ 7 files changed, 235 insertions(+), 50 deletions(-) diff --git a/addons/addon-image/src/ImageRenderer.ts b/addons/addon-image/src/ImageRenderer.ts index 1e87aa97..24a15d43 100644 --- a/addons/addon-image/src/ImageRenderer.ts +++ b/addons/addon-image/src/ImageRenderer.ts @@ -5,7 +5,7 @@ import { toRGBA8888 } from 'sixel/lib/Colors'; import { IDisposable } from '@xterm/xterm'; -import { ICellSize, ITerminalExt, IImageSpec, IRenderDimensions, IRenderService } from './Types'; +import { ICellSize, ImageLayer, ITerminalExt, IImageSpec, IRenderDimensions, IRenderService } from './Types'; import { Disposable, MutableDisposable, toDisposable } from 'vs/base/common/lifecycle'; const PLACEHOLDER_LENGTH = 4096; @@ -18,8 +18,12 @@ const PLACEHOLDER_HEIGHT = 24; * - draw image tiles onRender */ export class ImageRenderer extends Disposable implements IDisposable { - public canvas: HTMLCanvasElement | undefined; - private _ctx: CanvasRenderingContext2D | null | undefined; + /** @deprecated Use canvasTop instead. Kept for backward compat — points to canvasTop. */ + public get canvas(): HTMLCanvasElement | undefined { return this._canvasTop; } + private _canvasTop: HTMLCanvasElement | undefined; + private _canvasBottom: HTMLCanvasElement | undefined; + private _ctxTop: CanvasRenderingContext2D | null | undefined; + private _ctxBottom: CanvasRenderingContext2D | null | undefined; private _placeholder: HTMLCanvasElement | undefined; private _placeholderBitmap: ImageBitmap | undefined; private _optionsRefresh = this._register(new MutableDisposable()); @@ -86,6 +90,7 @@ export class ImageRenderer extends Disposable implements IDisposable { }); this._register(toDisposable(() => { this.removeLayerFromDom(); + this.removeLayerFromDom('bottom'); if (this._terminal._core && this._oldOpen) { this._terminal._core.open = this._oldOpen; this._oldOpen = undefined; @@ -95,8 +100,10 @@ export class ImageRenderer extends Disposable implements IDisposable { this._oldSetRenderer = undefined; } this._renderService = undefined; - this.canvas = undefined; - this._ctx = undefined; + this._canvasTop = undefined; + this._canvasBottom = undefined; + this._ctxTop = undefined; + this._ctxBottom = undefined; this._placeholderBitmap?.close(); this._placeholderBitmap = undefined; this._placeholder = undefined; @@ -140,27 +147,36 @@ export class ImageRenderer extends Disposable implements IDisposable { /** * Clear a region of the image layer canvas. */ - public clearLines(start: number, end: number): void { - this._ctx?.clearRect( - 0, - start * (this.dimensions?.css.cell.height || 0), - this.dimensions?.css.canvas.width || 0, - (++end - start) * (this.dimensions?.css.cell.height || 0) - ); + public clearLines(start: number, end: number, layer?: ImageLayer): void { + const y = start * (this.dimensions?.css.cell.height || 0); + const w = this.dimensions?.css.canvas.width || 0; + const h = (++end - start) * (this.dimensions?.css.cell.height || 0); + if (!layer || layer === 'top') { + this._ctxTop?.clearRect(0, y, w, h); + } + if (!layer || layer === 'bottom') { + this._ctxBottom?.clearRect(0, y, w, h); + } } /** * Clear whole image canvas. */ - public clearAll(): void { - this._ctx?.clearRect(0, 0, this.canvas?.width || 0, this.canvas?.height || 0); + public clearAll(layer?: ImageLayer): void { + if (!layer || layer === 'top') { + this._ctxTop?.clearRect(0, 0, this._canvasTop?.width || 0, this._canvasTop?.height || 0); + } + if (!layer || layer === 'bottom') { + this._ctxBottom?.clearRect(0, 0, this._canvasBottom?.width || 0, this._canvasBottom?.height || 0); + } } /** * Draw neighboring tiles on the image layer canvas. */ public draw(imgSpec: IImageSpec, tileId: number, col: number, row: number, count: number = 1): void { - if (!this._ctx) { + const ctx = imgSpec.layer === 'bottom' ? this._ctxBottom : this._ctxTop; + if (!ctx) { return; } const { width, height } = this.cellSize; @@ -187,7 +203,7 @@ export class ImageRenderer extends Disposable implements IDisposable { // Note: For not pixel perfect aligned cells like in the DOM renderer // this will move a tile slightly to the top/left (subpixel range, thus ignore it). // FIX #34: avoid striping on displays with pixelDeviceRatio != 1 by ceiling height and width - this._ctx.drawImage( + ctx.drawImage( img, Math.floor(sx), Math.floor(sy), Math.ceil(finalWidth), Math.ceil(finalHeight), Math.floor(dx), Math.floor(dy), Math.ceil(finalWidth), Math.ceil(finalHeight) @@ -227,7 +243,7 @@ export class ImageRenderer extends Disposable implements IDisposable { * Draw a line with placeholder on the image layer canvas. */ public drawPlaceholder(col: number, row: number, count: number = 1): void { - if (this._ctx) { + if (this._ctxTop) { const { width, height } = this.cellSize; // Don't try to draw anything, if we cannot get valid renderer metrics. @@ -241,7 +257,7 @@ export class ImageRenderer extends Disposable implements IDisposable { this._createPlaceHolder(height + 1); } if (!this._placeholder) return; - this._ctx.drawImage( + this._ctxTop.drawImage( this._placeholderBitmap ?? this._placeholder!, col * width, (row * height) % 2 ? 0 : 1, // needs %2 offset correction @@ -260,12 +276,15 @@ export class ImageRenderer extends Disposable implements IDisposable { * Checked once from `ImageStorage.render`. */ public rescaleCanvas(): void { - if (!this.canvas) { - return; + const w = this.dimensions?.css.canvas.width || 0; + const h = this.dimensions?.css.canvas.height || 0; + if (this._canvasTop && (this._canvasTop.width !== w || this._canvasTop.height !== h)) { + this._canvasTop.width = w; + this._canvasTop.height = h; } - if (this.canvas.width !== this.dimensions!.css.canvas.width || this.canvas.height !== this.dimensions!.css.canvas.height) { - this.canvas.width = this.dimensions!.css.canvas.width || 0; - this.canvas.height = this.dimensions!.css.canvas.height || 0; + if (this._canvasBottom && (this._canvasBottom.width !== w || this._canvasBottom.height !== h)) { + this._canvasBottom.width = w; + this._canvasBottom.height = h; } } @@ -305,34 +324,60 @@ export class ImageRenderer extends Disposable implements IDisposable { this._oldSetRenderer = this._renderService.setRenderer.bind(this._renderService); this._renderService.setRenderer = (renderer: any) => { this.removeLayerFromDom(); + this.removeLayerFromDom('bottom'); this._oldSetRenderer?.call(this._renderService, renderer); }; } - public insertLayerToDom(): void { + public insertLayerToDom(layer: ImageLayer = 'top'): void { // make sure that the terminal is attached to a document and to DOM if (this.document && this._terminal._core.screenElement) { - if (!this.canvas) { - this.canvas = ImageRenderer.createCanvas( + if (layer === 'top' && !this._canvasTop) { + this._canvasTop = ImageRenderer.createCanvas( this.document, this.dimensions?.css.canvas.width || 0, this.dimensions?.css.canvas.height || 0 ); - this.canvas.classList.add('xterm-image-layer'); - this._terminal._core.screenElement.appendChild(this.canvas); - this._ctx = this.canvas.getContext('2d', { alpha: true, desynchronized: true }); - this.clearAll(); + this._canvasTop.classList.add('xterm-image-layer-top'); + this._terminal._core.screenElement.appendChild(this._canvasTop); + this._ctxTop = this._canvasTop.getContext('2d', { alpha: true, desynchronized: true }); + this.clearAll('top'); + } + if (layer === 'bottom' && !this._canvasBottom) { + this._canvasBottom = ImageRenderer.createCanvas( + this.document, this.dimensions?.css.canvas.width || 0, + this.dimensions?.css.canvas.height || 0 + ); + this._canvasBottom.classList.add('xterm-image-layer-bottom'); + // Use z-index:-1 so it paints behind non-positioned text elements. + // The screen element needs to be a stacking context to contain the + // negative z-index, otherwise it would go behind the entire terminal. + this._canvasBottom.style.zIndex = '-1'; + const screenElement = this._terminal._core.screenElement; + screenElement.style.zIndex = '0'; + screenElement.insertBefore(this._canvasBottom, screenElement.firstChild); + this._ctxBottom = this._canvasBottom.getContext('2d', { alpha: true, desynchronized: true }); + this.clearAll('bottom'); } } else { console.warn('image addon: cannot insert output canvas to DOM, missing document or screenElement'); } } - public removeLayerFromDom(): void { - if (this.canvas) { - this._ctx = undefined; - this.canvas.remove(); - this.canvas = undefined; + public removeLayerFromDom(layer: ImageLayer = 'top'): void { + if (layer === 'top' && this._canvasTop) { + this._ctxTop = undefined; + this._canvasTop.remove(); + this._canvasTop = undefined; } + if (layer === 'bottom' && this._canvasBottom) { + this._ctxBottom = undefined; + this._canvasBottom.remove(); + this._canvasBottom = undefined; + } + } + + public hasLayer(layer: ImageLayer): boolean { + return layer === 'top' ? !!this._canvasTop : !!this._canvasBottom; } private _createPlaceHolder(height: number = PLACEHOLDER_HEIGHT): void { diff --git a/addons/addon-image/src/ImageStorage.ts b/addons/addon-image/src/ImageStorage.ts index 03925f0b..3609b279 100644 --- a/addons/addon-image/src/ImageStorage.ts +++ b/addons/addon-image/src/ImageStorage.ts @@ -5,7 +5,7 @@ import { IDisposable } from '@xterm/xterm'; import { ImageRenderer } from './ImageRenderer'; -import { ITerminalExt, IExtendedAttrsImage, IImageAddonOptions, IImageSpec, IBufferLineExt, BgFlags, Cell, Content, ICellSize, ExtFlags, Attributes, UnderlineStyle } from './Types'; +import { ITerminalExt, IExtendedAttrsImage, IImageAddonOptions, IImageSpec, IBufferLineExt, BgFlags, Cell, Content, ICellSize, ExtFlags, Attributes, UnderlineStyle, ImageLayer } from './Types'; // fallback default cell size @@ -250,7 +250,7 @@ export class ImageStorage implements IDisposable { * Method to add an image to the storage. * Returns the internal image ID assigned to the stored image. */ - public addImage(img: HTMLCanvasElement | ImageBitmap): number { + public addImage(img: HTMLCanvasElement | ImageBitmap, layer: ImageLayer = 'top'): number { // never allow storage to exceed memory limit this._evictOldest(img.width * img.height); @@ -339,7 +339,8 @@ export class ImageStorage implements IDisposable { actualCellSize: { ...cellSize }, // clone needed, since later modified marker: endMarker || undefined, tileCount, - bufferType: this._terminal.buffer.active.type + bufferType: this._terminal.buffer.active.type, + layer }; // finally add the image @@ -354,16 +355,30 @@ export class ImageStorage implements IDisposable { */ // TODO: Should we move this to the ImageRenderer? public render(range: { start: number, end: number }): void { - // setup image canvas in case we have none yet, but have images in store - if (!this._renderer.canvas && this._images.size) { - this._renderer.insertLayerToDom(); - // safety measure - in case we cannot spawn a canvas at all, just exit - if (!this._renderer.canvas) { - return; + // Determine which layers have images + let hasTopImages = false; + let hasBottomImages = false; + for (const spec of this._images.values()) { + if (spec.layer === 'bottom') { + hasBottomImages = true; + } else { + hasTopImages = true; } + if (hasTopImages && hasBottomImages) break; } + + // Lazily insert layers that are needed + if (hasTopImages && !this._renderer.hasLayer('top')) { + this._renderer.insertLayerToDom('top'); + if (!this._renderer.hasLayer('top')) return; + } + if (hasBottomImages && !this._renderer.hasLayer('bottom')) { + this._renderer.insertLayerToDom('bottom'); + } + // rescale if needed this._renderer.rescaleCanvas(); + // exit early if we dont have any images to test for if (!this._images.size) { if (!this._fullyCleared) { @@ -371,12 +386,25 @@ export class ImageStorage implements IDisposable { this._fullyCleared = true; this._needsFullClear = false; } - if (this._renderer.canvas) { - this._renderer.removeLayerFromDom(); + if (this._renderer.hasLayer('top')) { + this._renderer.removeLayerFromDom('top'); + } + if (this._renderer.hasLayer('bottom')) { + this._renderer.removeLayerFromDom('bottom'); } return; } + // Remove layers no longer needed + if (!hasTopImages && this._renderer.hasLayer('top')) { + this._renderer.clearAll('top'); + this._renderer.removeLayerFromDom('top'); + } + if (!hasBottomImages && this._renderer.hasLayer('bottom')) { + this._renderer.clearAll('bottom'); + this._renderer.removeLayerFromDom('bottom'); + } + // buffer switches force a full clear if (this._needsFullClear) { this._renderer.clearAll(); diff --git a/addons/addon-image/src/Types.ts b/addons/addon-image/src/Types.ts index 480a4853..a9f3310b 100644 --- a/addons/addon-image/src/Types.ts +++ b/addons/addon-image/src/Types.ts @@ -99,6 +99,8 @@ export interface ICellSize { height: number; } +export type ImageLayer = 'top' | 'bottom'; + export interface IImageSpec { orig: HTMLCanvasElement | ImageBitmap | undefined; origCellSize: ICellSize; @@ -107,4 +109,5 @@ export interface IImageSpec { marker: IMarker | undefined; tileCount: number; bufferType: 'alternate' | 'normal'; + layer: ImageLayer; } diff --git a/addons/addon-image/src/kitty/KittyGraphicsHandler.ts b/addons/addon-image/src/kitty/KittyGraphicsHandler.ts index 84bd4d8c..8e3821f1 100644 --- a/addons/addon-image/src/kitty/KittyGraphicsHandler.ts +++ b/addons/addon-image/src/kitty/KittyGraphicsHandler.ts @@ -3,7 +3,7 @@ * @license MIT */ -import { IApcHandler, IImageAddonOptions, IResetHandler, ITerminalExt } from '../Types'; +import { IApcHandler, IImageAddonOptions, IResetHandler, ITerminalExt, ImageLayer } from '../Types'; import { ImageRenderer } from '../ImageRenderer'; import { ImageStorage, CELL_SIZE_DEFAULT } from '../ImageStorage'; import Base64Decoder, { type DecodeStatus } from 'xterm-wasm-parts/lib/base64/Base64Decoder.wasm'; @@ -504,12 +504,18 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler { const savedY = buffer.y; const savedYbase = buffer.ybase; + // Determine layer based on z-index: negative = behind text, 0+ = on top. + // Bottom layer only works when allowTransparency is enabled (otherwise text + // canvas background is opaque and hides the bottom canvas). Fall back to top. + const wantsBottom = cmd.zIndex !== undefined && cmd.zIndex < 0; + const layer: ImageLayer = (wantsBottom && this._coreTerminal.options.allowTransparency) ? 'bottom' : 'top'; + let storageId: number; if (w !== bitmap.width || h !== bitmap.height) { const resized = await createImageBitmap(bitmap, { resizeWidth: w, resizeHeight: h }); - storageId = this._storage.addImage(resized); + storageId = this._storage.addImage(resized, layer); } else { - storageId = this._storage.addImage(bitmap); + storageId = this._storage.addImage(bitmap, layer); } this._kittyIdToStorageId.set(image.id, storageId); diff --git a/addons/addon-image/src/kitty/KittyGraphicsTypes.test.ts b/addons/addon-image/src/kitty/KittyGraphicsTypes.test.ts index ac79c09a..04d95dd3 100644 --- a/addons/addon-image/src/kitty/KittyGraphicsTypes.test.ts +++ b/addons/addon-image/src/kitty/KittyGraphicsTypes.test.ts @@ -86,5 +86,25 @@ describe('KittyGraphicsTypes', () => { assert.ok(isNaN(cmd.format!)); assert.strictEqual(cmd.id, 5); }); + + it('should parse z-index key with positive value', () => { + const cmd = parseKittyCommand('a=T,f=100,z=10'); + assert.strictEqual(cmd.zIndex, 10); + }); + + it('should parse z-index key with zero', () => { + const cmd = parseKittyCommand('a=T,f=100,z=0'); + assert.strictEqual(cmd.zIndex, 0); + }); + + it('should parse z-index key with negative value', () => { + const cmd = parseKittyCommand('a=T,f=100,z=-1'); + assert.strictEqual(cmd.zIndex, -1); + }); + + it('should leave zIndex undefined when not specified', () => { + const cmd = parseKittyCommand('a=T,f=100'); + assert.strictEqual(cmd.zIndex, undefined); + }); }); }); diff --git a/addons/addon-image/src/kitty/KittyGraphicsTypes.ts b/addons/addon-image/src/kitty/KittyGraphicsTypes.ts index f49df028..b3f871fd 100644 --- a/addons/addon-image/src/kitty/KittyGraphicsTypes.ts +++ b/addons/addon-image/src/kitty/KittyGraphicsTypes.ts @@ -70,7 +70,9 @@ export const enum KittyKey { // Quiet mode (1=suppress OK responses, 2=suppress error responses) QUIET = 'q', // Cursor movement policy (0=move cursor after image, 1=don't move cursor) - CURSOR_MOVEMENT = 'C' + CURSOR_MOVEMENT = 'C', + // Z-index for image layering (negative = behind text, 0+ = on top) + Z_INDEX = 'z' } // Pixel format constants @@ -95,6 +97,7 @@ export interface IKittyCommand { more?: number; quiet?: number; cursorMovement?: number; + zIndex?: number; compression?: string; payload?: string; } @@ -164,6 +167,7 @@ export function parseKittyCommand(data: string): IKittyCommand { case KittyKey.MORE: cmd.more = numValue; break; case KittyKey.QUIET: cmd.quiet = numValue; break; case KittyKey.CURSOR_MOVEMENT: cmd.cursorMovement = numValue; break; + case KittyKey.Z_INDEX: cmd.zIndex = numValue; break; } } diff --git a/addons/addon-image/test/KittyGraphics.test.ts b/addons/addon-image/test/KittyGraphics.test.ts index f317ebde..38a70c17 100644 --- a/addons/addon-image/test/KittyGraphics.test.ts +++ b/addons/addon-image/test/KittyGraphics.test.ts @@ -546,6 +546,85 @@ test.describe('Kitty Graphics Protocol', () => { }); }); + test.describe('Z-index layer placement', () => { + test('default placement (no z key) stores image on top layer', async () => { + await ctx.proxy.write(`\x1b_Ga=T,f=100;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + strictEqual(await getImageStorageLength(), 1); + strictEqual(await ctx.page.evaluate(`window.imageAddon._storage._images.get(1).layer`), 'top'); + }); + + test('z=0 stores image on top layer', async () => { + await ctx.proxy.write(`\x1b_Ga=T,f=100,z=0;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + strictEqual(await getImageStorageLength(), 1); + strictEqual(await ctx.page.evaluate(`window.imageAddon._storage._images.get(1).layer`), 'top'); + }); + + test('z=1 (positive) stores image on top layer', async () => { + await ctx.proxy.write(`\x1b_Ga=T,f=100,z=1;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + strictEqual(await getImageStorageLength(), 1); + strictEqual(await ctx.page.evaluate(`window.imageAddon._storage._images.get(1).layer`), 'top'); + }); + + test('z=-1 falls back to top layer when allowTransparency is disabled', async () => { + await ctx.page.evaluate(`window.term.options.allowTransparency = false`); + await ctx.proxy.write(`\x1b_Ga=T,f=100,z=-1;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + strictEqual(await getImageStorageLength(), 1); + strictEqual(await ctx.page.evaluate(`window.imageAddon._storage._images.get(1).layer`), 'top'); + }); + + test('z=-1 (negative) stores image on bottom layer when allowTransparency is enabled', async () => { + await ctx.page.evaluate(`window.term.options.allowTransparency = true`); + await ctx.proxy.write(`\x1b_Ga=T,f=100,z=-1;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + strictEqual(await getImageStorageLength(), 1); + strictEqual(await ctx.page.evaluate(`window.imageAddon._storage._images.get(1).layer`), 'bottom'); + }); + + test('z=-100 (large negative) stores image on bottom layer when allowTransparency is enabled', async () => { + await ctx.page.evaluate(`window.term.options.allowTransparency = true`); + await ctx.proxy.write(`\x1b_Ga=T,f=100,z=-100;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + strictEqual(await getImageStorageLength(), 1); + strictEqual(await ctx.page.evaluate(`window.imageAddon._storage._images.get(1).layer`), 'bottom'); + }); + + test('top layer canvas has correct CSS class', async () => { + await ctx.proxy.write(`\x1b_Ga=T,f=100;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + const hasClass = await ctx.page.evaluate(() => { + const el = document.querySelector('.xterm-image-layer-top'); + return el !== null; + }); + strictEqual(hasClass, true); + }); + + test('bottom layer canvas has correct CSS class', async () => { + await ctx.page.evaluate(`window.term.options.allowTransparency = true`); + await ctx.proxy.write(`\x1b_Ga=T,f=100,z=-1;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + const hasClass = await ctx.page.evaluate(() => { + const el = document.querySelector('.xterm-image-layer-bottom'); + return el !== null; + }); + strictEqual(hasClass, true); + }); + + test('bottom layer canvas is before text canvas in DOM order', async () => { + await ctx.page.evaluate(`window.term.options.allowTransparency = true`); + await ctx.proxy.write(`\x1b_Ga=T,f=100,z=-1;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + const isFirst = await ctx.page.evaluate(() => { + const screen = document.querySelector('.xterm-screen'); + return screen?.firstElementChild?.classList.contains('xterm-image-layer-bottom') ?? false; + }); + strictEqual(isFirst, true); + }); + }); + test.describe('Pixel verification', () => { test('renders 1x1 black PNG at cursor position', async () => { const seq = `\x1b_Ga=T,f=100;${KITTY_BLACK_1X1_BASE64}\x1b\\`;