diff --git a/addons/addon-kitty-graphics/src/KittyGraphicsAddon.ts b/addons/addon-kitty-graphics/src/KittyGraphicsAddon.ts index 0dde0671..24d280f2 100644 --- a/addons/addon-kitty-graphics/src/KittyGraphicsAddon.ts +++ b/addons/addon-kitty-graphics/src/KittyGraphicsAddon.ts @@ -5,6 +5,7 @@ import type { Terminal, ITerminalAddon, IDisposable } from '@xterm/xterm'; import type { KittyGraphicsAddon as IKittyGraphicsApi, IKittyGraphicsOptions, IKittyImage } from '@xterm/addon-kitty-graphics'; +import { KittyImageRenderer } from './KittyImageRenderer'; /** * Kitty graphics protocol action types. @@ -81,7 +82,9 @@ export function parseKittyCommand(data: string): IKittyCommand { export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi { private _terminal: Terminal | undefined; private _apcHandler: IDisposable | undefined; + private _renderer: KittyImageRenderer | undefined; private _images: Map = new Map(); + private _decodedImages: Map = new Map(); private _pendingData: Map = new Map(); private _nextImageId = 1; private _debug: boolean; @@ -96,8 +99,11 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi { public activate(terminal: Terminal): void { this._terminal = terminal; - // TODO: Remove console log - console.log('[KittyGraphicsAddon] Activated'); + this._renderer = new KittyImageRenderer(terminal); + + if (this._debug) { + console.log('[KittyGraphicsAddon] Activating, registering APC handler for G (0x47)'); + } // Register APC handler for 'G' (0x47) - Kitty graphics protocol // APC sequence format: ESC _ G ESC \ @@ -108,8 +114,16 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi { public dispose(): void { this._apcHandler?.dispose(); + this._renderer?.dispose(); this._images.clear(); this._pendingData.clear(); + + // Close all decoded bitmaps + for (const bitmap of this._decodedImages.values()) { + bitmap.close(); + } + this._decodedImages.clear(); + this._terminal = undefined; } @@ -175,11 +189,130 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi { } private _handleTransmitDisplay(cmd: IKittyCommand): boolean { + // First store the image this._handleTransmit(cmd); - // TODO: Display image at cursor position (canvas layer) + + // Get the image ID (same logic as _handleTransmit) + const id = cmd.id || (this._nextImageId - 1); + const image = this._images.get(id); + + if (!image) { + return true; + } + + // Decode and display with sizing + this._decodeAndDisplay(image, cmd.columns, cmd.rows).catch(err => { + if (this._debug) { + console.error(`[KittyGraphicsAddon] Failed to decode/display image ${id}:`, err); + } + }); + return true; } + /** + * Decode base64 image data into an ImageBitmap. + */ + private async _decodeImage(image: IKittyImage): Promise { + const format = image.format; + const base64Data = image.data as string; + + // Decode base64 to binary + const binaryString = atob(base64Data); + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + + if (format === KittyFormat.PNG) { + // PNG: create blob and decode + const blob = new Blob([bytes], { type: 'image/png' }); + return createImageBitmap(blob); + } + + // RGB (24) or RGBA (32): create ImageData + const width = image.width; + const height = image.height; + + if (!width || !height) { + throw new Error('Width and height required for raw pixel data'); + } + + const bytesPerPixel = format === KittyFormat.RGBA ? 4 : 3; + const expectedBytes = width * height * bytesPerPixel; + + if (bytes.length < expectedBytes) { + throw new Error(`Insufficient pixel data: got ${bytes.length}, expected ${expectedBytes}`); + } + + // Convert to RGBA ImageData + const imageData = new ImageData(width, height); + const data = imageData.data; + + for (let i = 0; i < width * height; i++) { + const srcOffset = i * bytesPerPixel; + const dstOffset = i * 4; + + data[dstOffset] = bytes[srcOffset]; // R + data[dstOffset + 1] = bytes[srcOffset + 1]; // G + data[dstOffset + 2] = bytes[srcOffset + 2]; // B + data[dstOffset + 3] = format === KittyFormat.RGBA + ? bytes[srcOffset + 3] // A from source + : 255; // Fully opaque for RGB + } + + return createImageBitmap(imageData); + } + + /** + * Decode an image and display it at the cursor position. + * @param columns - Optional: number of terminal columns to span + * @param rows - Optional: number of terminal rows to span + */ + private async _decodeAndDisplay(image: IKittyImage, columns?: number, rows?: number): Promise { + if (!this._renderer) { + return; + } + + // Check if already decoded + let bitmap = this._decodedImages.get(image.id); + + if (!bitmap) { + bitmap = await this._decodeImage(image); + this._decodedImages.set(image.id, bitmap); + + if (this._debug) { + console.log(`[KittyGraphicsAddon] Decoded image ${image.id}: ${bitmap.width}x${bitmap.height}`); + } + } + + // Calculate pixel dimensions from columns/rows if specified + let width = 0; + let height = 0; + if (columns || rows) { + const cellSize = this._renderer.getCellSize(); + if (columns) { + width = columns * cellSize.width; + } + if (rows) { + height = rows * cellSize.height; + } + // If only one dimension specified, maintain aspect ratio + if (width && !height) { + height = Math.round(width * (bitmap.height / bitmap.width)); + } else if (height && !width) { + width = Math.round(height * (bitmap.width / bitmap.height)); + } + } + + // Place at cursor position + this._renderer.placeImage(bitmap, image.id, undefined, undefined, width, height); + + if (this._debug) { + console.log(`[KittyGraphicsAddon] Placed image ${image.id} at cursor, size: ${width || bitmap.width}x${height || bitmap.height}`); + } + } + private _handleQuery(cmd: IKittyCommand): boolean { // TODO: Respond with APC sequence indicating graphics support // Protocol: terminal should reply with ESC _ G i=;OK ESC \ @@ -192,8 +325,23 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi { private _handleDelete(cmd: IKittyCommand): boolean { if (cmd.id !== undefined) { this._images.delete(cmd.id); + // Close and remove decoded bitmap + const bitmap = this._decodedImages.get(cmd.id); + if (bitmap) { + bitmap.close(); + this._decodedImages.delete(cmd.id); + } + // Remove from renderer + this._renderer?.removeByImageId(cmd.id); } else { this._images.clear(); + // Close all decoded bitmaps + for (const bitmap of this._decodedImages.values()) { + bitmap.close(); + } + this._decodedImages.clear(); + // Clear all from renderer + this._renderer?.clearAll(); } return true; } diff --git a/addons/addon-kitty-graphics/src/KittyImageRenderer.ts b/addons/addon-kitty-graphics/src/KittyImageRenderer.ts new file mode 100644 index 00000000..d23698ff --- /dev/null +++ b/addons/addon-kitty-graphics/src/KittyImageRenderer.ts @@ -0,0 +1,240 @@ +/** + * Copyright (c) 2025 The xterm.js authors. All rights reserved. + * @license MIT + */ + +import type { Terminal, IDisposable } from '@xterm/xterm'; + +/** + * A placed image in the terminal. + */ +export interface IPlacedImage { + /** The decoded image bitmap */ + bitmap: ImageBitmap; + /** Column position in terminal */ + col: number; + /** Row position in terminal (relative to buffer, not viewport) */ + row: number; + /** Width to render (in pixels, 0 = original) */ + width: number; + /** Height to render (in pixels, 0 = original) */ + height: number; + /** The image ID for reference */ + id: number; +} + +/** + * Handles canvas layer management and image rendering for Kitty graphics. + * + * Similar to ImageRenderer in addon-image but simplified for Kitty protocol. + */ +export class KittyImageRenderer implements IDisposable { + private _canvas: HTMLCanvasElement | undefined; + private _ctx: CanvasRenderingContext2D | null | undefined; + private _terminal: Terminal; + private _placements: Map = new Map(); + private _placementIdCounter = 0; + private _renderDisposable: IDisposable | undefined; + private _resizeDisposable: IDisposable | undefined; + + constructor(terminal: Terminal) { + this._terminal = terminal; + } + + /** + * Initialize the canvas layer. Called when first image is placed. + */ + public ensureCanvasLayer(): void { + if (this._canvas) { + return; + } + + // Access internal screenElement + const core = (this._terminal as any)._core; + const screenElement = core?.screenElement; + if (!screenElement) { + console.warn('[KittyGraphicsAddon] Cannot create canvas: no screenElement'); + return; + } + + // Get dimensions from terminal + const dimensions = (this._terminal as any).dimensions; + const width = dimensions?.css?.canvas?.width || 800; + const height = dimensions?.css?.canvas?.height || 600; + + // Create canvas + this._canvas = document.createElement('canvas'); + this._canvas.width = width; + this._canvas.height = height; + this._canvas.classList.add('xterm-kitty-graphics-layer'); + + // Position absolutely over the terminal + this._canvas.style.position = 'absolute'; + this._canvas.style.top = '0'; + this._canvas.style.left = '0'; + this._canvas.style.pointerEvents = 'none'; + this._canvas.style.zIndex = '10'; + + screenElement.appendChild(this._canvas); + this._ctx = this._canvas.getContext('2d', { alpha: true }); + + // Hook into render events to redraw when terminal scrolls + this._renderDisposable = this._terminal.onRender(() => this._onRender()); + + // Handle resize + this._resizeDisposable = this._terminal.onResize(() => this._onResize()); + } + + /** + * Get cell dimensions from terminal. + */ + public getCellSize(): { width: number; height: number } { + const dimensions = (this._terminal as any).dimensions; + return { + width: dimensions?.css?.cell?.width || 9, + height: dimensions?.css?.cell?.height || 17 + }; + } + + /** + * Place a decoded image at cursor position. + * @param width - Optional width in pixels (0 = original) + * @param height - Optional height in pixels (0 = original) + */ + public placeImage(bitmap: ImageBitmap, id: number, col?: number, row?: number, width?: number, height?: number): number { + this.ensureCanvasLayer(); + + const buffer = this._terminal.buffer.active; + const placementId = ++this._placementIdCounter; + + const placement: IPlacedImage = { + bitmap, + col: col ?? buffer.cursorX, + row: row ?? (buffer.cursorY + buffer.baseY), + width: width || 0, + height: height || 0, + id + }; + + this._placements.set(placementId, placement); + this._draw(); + + return placementId; + } + + /** + * Remove a placed image. + */ + public removePlacement(placementId: number): void { + const placement = this._placements.get(placementId); + if (placement) { + this._placements.delete(placementId); + this._draw(); + } + } + + /** + * Remove all placements for an image ID. + */ + public removeByImageId(imageId: number): void { + const toDelete: number[] = []; + for (const [placementId, placement] of this._placements) { + if (placement.id === imageId) { + toDelete.push(placementId); + } + } + for (const id of toDelete) { + this._placements.delete(id); + } + if (toDelete.length > 0) { + this._draw(); + } + } + + /** + * Clear all images. + */ + public clearAll(): void { + this._placements.clear(); + if (this._ctx && this._canvas) { + this._ctx.clearRect(0, 0, this._canvas.width, this._canvas.height); + } + } + + /** + * Redraw all placed images. + */ + private _draw(): void { + if (!this._ctx || !this._canvas) { + return; + } + + // Clear canvas + this._ctx.clearRect(0, 0, this._canvas.width, this._canvas.height); + + const buffer = this._terminal.buffer.active; + const viewportStartRow = buffer.baseY; + const viewportRows = this._terminal.rows; + const cellSize = this.getCellSize(); + + // Draw each placement that's visible + for (const placement of this._placements.values()) { + // Check if placement is in viewport + const relativeRow = placement.row - viewportStartRow; + if (relativeRow < 0 || relativeRow >= viewportRows) { + continue; // Not in viewport + } + + const x = placement.col * cellSize.width; + const y = relativeRow * cellSize.height; + + const width = placement.width || placement.bitmap.width; + const height = placement.height || placement.bitmap.height; + + this._ctx.drawImage(placement.bitmap, x, y, width, height); + } + } + + /** + * Called when terminal renders (scrolling, content change). + */ + private _onRender(): void { + this._draw(); + } + + /** + * Called when terminal resizes. + */ + private _onResize(): void { + if (!this._canvas) { + return; + } + + const dimensions = (this._terminal as any).dimensions; + const width = dimensions?.css?.canvas?.width || 800; + const height = dimensions?.css?.canvas?.height || 600; + + if (this._canvas.width !== width || this._canvas.height !== height) { + this._canvas.width = width; + this._canvas.height = height; + this._draw(); + } + } + + public dispose(): void { + this._renderDisposable?.dispose(); + this._resizeDisposable?.dispose(); + + // Close all bitmaps + for (const placement of this._placements.values()) { + placement.bitmap.close(); + } + this._placements.clear(); + + if (this._canvas) { + this._canvas.remove(); + this._canvas = undefined; + this._ctx = undefined; + } + } +} diff --git a/addons/addon-kitty-graphics/test/KittyGraphicsAddon.test.ts b/addons/addon-kitty-graphics/test/KittyGraphicsAddon.test.ts index c937620f..307604f2 100644 --- a/addons/addon-kitty-graphics/test/KittyGraphicsAddon.test.ts +++ b/addons/addon-kitty-graphics/test/KittyGraphicsAddon.test.ts @@ -127,8 +127,8 @@ test.describe('KittyGraphicsAddon', () => { }); }); - // TODO: These tests will fail until decode + rendering is implemented - test.describe.skip('pixel verification', () => { + test.describe('pixel verification', () => { + // TODO: Add more intense ones. test('renders 1x1 black PNG at cursor position', async () => { // Send image with a=T to transmit and display const seq = `\x1b_Ga=T,f=100;${BLACK_1X1_BASE64}\x1b\\`; diff --git a/send-png b/send-png index 9afe64c5..8bb820d9 100755 --- a/send-png +++ b/send-png @@ -5,8 +5,9 @@ Minimal Kitty Graphics Protocol image sender. Based on: https://sw.kovidgoyal.net/kitty/graphics-protocol/#a-minimal-example Usage: - ./send-png black-1x1.png - ./send-png rgb-3x1.png + ./send-png image.png # Original size + ./send-png image.png 10 # Scale to 10 columns wide + ./send-png image.png 10 5 # Scale to 10 columns x 5 rows """ import sys from base64 import standard_b64encode @@ -16,18 +17,25 @@ def serialize_image(path): data = f.read() return standard_b64encode(data).decode('ascii') -def write_chunked(data): +def write_chunked(data, columns=None, rows=None): # a=T means transmit and display # f=100 means PNG format - # For small images, single chunk (no chunking needed) - sys.stdout.write(f'\x1b_Ga=T,f=100;{data}\x1b\\') + # c=columns, r=rows for sizing + params = 'a=T,f=100' + if columns: + params += f',c={columns}' + if rows: + params += f',r={rows}' + sys.stdout.write(f'\x1b_G{params};{data}\x1b\\') sys.stdout.flush() if __name__ == '__main__': if len(sys.argv) < 2: - print("Usage: ./send-png ", file=sys.stderr) + print("Usage: ./send-png [columns] [rows]", file=sys.stderr) sys.exit(1) img_data = serialize_image(sys.argv[1]) - write_chunked(img_data) + columns = int(sys.argv[2]) if len(sys.argv) > 2 else None + rows = int(sys.argv[3]) if len(sys.argv) > 3 else None + write_chunked(img_data, columns, rows) print() # newline after the sequence