Add rendering, tweak send-png

addons/addon-kitty-graphics/src/KittyGraphicsAddon.ts

@@ -5,6 +5,7 @@
 
 import type { Terminal, ITerminalAddon, IDisposable } from '@xterm/xterm';
 import type { KittyGraphicsAddon as IKittyGraphicsApi, IKittyGraphicsOptions, IKittyImage } from '@xterm/addon-kitty-graphics';
+import { KittyImageRenderer } from './KittyImageRenderer';
 
 /**
  * Kitty graphics protocol action types.
@@ -81,7 +82,9 @@ export function parseKittyCommand(data: string): IKittyCommand {
 export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi {
   private _terminal: Terminal | undefined;
   private _apcHandler: IDisposable | undefined;
+  private _renderer: KittyImageRenderer | undefined;
   private _images: Map<number, IKittyImage> = new Map();
+  private _decodedImages: Map<number, ImageBitmap> = new Map();
   private _pendingData: Map<number, string> = new Map();
   private _nextImageId = 1;
   private _debug: boolean;
@@ -96,8 +99,11 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi {
 
   public activate(terminal: Terminal): void {
     this._terminal = terminal;
-    // TODO: Remove console log
-    console.log('[KittyGraphicsAddon] Activated');
+    this._renderer = new KittyImageRenderer(terminal);
+
+    if (this._debug) {
+      console.log('[KittyGraphicsAddon] Activating, registering APC handler for G (0x47)');
+    }
 
     // Register APC handler for 'G' (0x47) - Kitty graphics protocol
     // APC sequence format: ESC _ G <data> ESC \
@@ -108,8 +114,16 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi {
 
   public dispose(): void {
     this._apcHandler?.dispose();
+    this._renderer?.dispose();
     this._images.clear();
     this._pendingData.clear();
+
+    // Close all decoded bitmaps
+    for (const bitmap of this._decodedImages.values()) {
+      bitmap.close();
+    }
+    this._decodedImages.clear();
+
     this._terminal = undefined;
   }
 
@@ -175,11 +189,130 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi {
   }
 
   private _handleTransmitDisplay(cmd: IKittyCommand): boolean {
+    // First store the image
     this._handleTransmit(cmd);
-    // TODO: Display image at cursor position (canvas layer)
+
+    // Get the image ID (same logic as _handleTransmit)
+    const id = cmd.id || (this._nextImageId - 1);
+    const image = this._images.get(id);
+
+    if (!image) {
+      return true;
+    }
+
+    // Decode and display with sizing
+    this._decodeAndDisplay(image, cmd.columns, cmd.rows).catch(err => {
+      if (this._debug) {
+        console.error(`[KittyGraphicsAddon] Failed to decode/display image ${id}:`, err);
+      }
+    });
+
     return true;
   }
 
+  /**
+   * Decode base64 image data into an ImageBitmap.
+   */
+  private async _decodeImage(image: IKittyImage): Promise<ImageBitmap> {
+    const format = image.format;
+    const base64Data = image.data as string;
+
+    // Decode base64 to binary
+    const binaryString = atob(base64Data);
+    const bytes = new Uint8Array(binaryString.length);
+    for (let i = 0; i < binaryString.length; i++) {
+      bytes[i] = binaryString.charCodeAt(i);
+    }
+
+    if (format === KittyFormat.PNG) {
+      // PNG: create blob and decode
+      const blob = new Blob([bytes], { type: 'image/png' });
+      return createImageBitmap(blob);
+    }
+
+    // RGB (24) or RGBA (32): create ImageData
+    const width = image.width;
+    const height = image.height;
+
+    if (!width || !height) {
+      throw new Error('Width and height required for raw pixel data');
+    }
+
+    const bytesPerPixel = format === KittyFormat.RGBA ? 4 : 3;
+    const expectedBytes = width * height * bytesPerPixel;
+
+    if (bytes.length < expectedBytes) {
+      throw new Error(`Insufficient pixel data: got ${bytes.length}, expected ${expectedBytes}`);
+    }
+
+    // Convert to RGBA ImageData
+    const imageData = new ImageData(width, height);
+    const data = imageData.data;
+
+    for (let i = 0; i < width * height; i++) {
+      const srcOffset = i * bytesPerPixel;
+      const dstOffset = i * 4;
+
+      data[dstOffset] = bytes[srcOffset];         // R
+      data[dstOffset + 1] = bytes[srcOffset + 1]; // G
+      data[dstOffset + 2] = bytes[srcOffset + 2]; // B
+      data[dstOffset + 3] = format === KittyFormat.RGBA
+        ? bytes[srcOffset + 3]  // A from source
+        : 255;                   // Fully opaque for RGB
+    }
+
+    return createImageBitmap(imageData);
+  }
+
+  /**
+   * Decode an image and display it at the cursor position.
+   * @param columns - Optional: number of terminal columns to span
+   * @param rows - Optional: number of terminal rows to span
+   */
+  private async _decodeAndDisplay(image: IKittyImage, columns?: number, rows?: number): Promise<void> {
+    if (!this._renderer) {
+      return;
+    }
+
+    // Check if already decoded
+    let bitmap = this._decodedImages.get(image.id);
+
+    if (!bitmap) {
+      bitmap = await this._decodeImage(image);
+      this._decodedImages.set(image.id, bitmap);
+
+      if (this._debug) {
+        console.log(`[KittyGraphicsAddon] Decoded image ${image.id}: ${bitmap.width}x${bitmap.height}`);
+      }
+    }
+
+    // Calculate pixel dimensions from columns/rows if specified
+    let width = 0;
+    let height = 0;
+    if (columns || rows) {
+      const cellSize = this._renderer.getCellSize();
+      if (columns) {
+        width = columns * cellSize.width;
+      }
+      if (rows) {
+        height = rows * cellSize.height;
+      }
+      // If only one dimension specified, maintain aspect ratio
+      if (width && !height) {
+        height = Math.round(width * (bitmap.height / bitmap.width));
+      } else if (height && !width) {
+        width = Math.round(height * (bitmap.width / bitmap.height));
+      }
+    }
+
+    // Place at cursor position
+    this._renderer.placeImage(bitmap, image.id, undefined, undefined, width, height);
+
+    if (this._debug) {
+      console.log(`[KittyGraphicsAddon] Placed image ${image.id} at cursor, size: ${width || bitmap.width}x${height || bitmap.height}`);
+    }
+  }
+
   private _handleQuery(cmd: IKittyCommand): boolean {
     // TODO: Respond with APC sequence indicating graphics support
     // Protocol: terminal should reply with ESC _ G i=<id>;OK ESC \
@@ -192,8 +325,23 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi {
   private _handleDelete(cmd: IKittyCommand): boolean {
     if (cmd.id !== undefined) {
       this._images.delete(cmd.id);
+      // Close and remove decoded bitmap
+      const bitmap = this._decodedImages.get(cmd.id);
+      if (bitmap) {
+        bitmap.close();
+        this._decodedImages.delete(cmd.id);
+      }
+      // Remove from renderer
+      this._renderer?.removeByImageId(cmd.id);
     } else {
       this._images.clear();
+      // Close all decoded bitmaps
+      for (const bitmap of this._decodedImages.values()) {
+        bitmap.close();
+      }
+      this._decodedImages.clear();
+      // Clear all from renderer
+      this._renderer?.clearAll();
     }
     return true;
   }

addons/addon-kitty-graphics/src/KittyImageRenderer.ts

@@ -0,0 +1,240 @@
+/**
+ * Copyright (c) 2025 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+
+import type { Terminal, IDisposable } from '@xterm/xterm';
+
+/**
+ * A placed image in the terminal.
+ */
+export interface IPlacedImage {
+  /** The decoded image bitmap */
+  bitmap: ImageBitmap;
+  /** Column position in terminal */
+  col: number;
+  /** Row position in terminal (relative to buffer, not viewport) */
+  row: number;
+  /** Width to render (in pixels, 0 = original) */
+  width: number;
+  /** Height to render (in pixels, 0 = original) */
+  height: number;
+  /** The image ID for reference */
+  id: number;
+}
+
+/**
+ * Handles canvas layer management and image rendering for Kitty graphics.
+ *
+ * Similar to ImageRenderer in addon-image but simplified for Kitty protocol.
+ */
+export class KittyImageRenderer implements IDisposable {
+  private _canvas: HTMLCanvasElement | undefined;
+  private _ctx: CanvasRenderingContext2D | null | undefined;
+  private _terminal: Terminal;
+  private _placements: Map<number, IPlacedImage> = new Map();
+  private _placementIdCounter = 0;
+  private _renderDisposable: IDisposable | undefined;
+  private _resizeDisposable: IDisposable | undefined;
+
+  constructor(terminal: Terminal) {
+    this._terminal = terminal;
+  }
+
+  /**
+   * Initialize the canvas layer. Called when first image is placed.
+   */
+  public ensureCanvasLayer(): void {
+    if (this._canvas) {
+      return;
+    }
+
+    // Access internal screenElement
+    const core = (this._terminal as any)._core;
+    const screenElement = core?.screenElement;
+    if (!screenElement) {
+      console.warn('[KittyGraphicsAddon] Cannot create canvas: no screenElement');
+      return;
+    }
+
+    // Get dimensions from terminal
+    const dimensions = (this._terminal as any).dimensions;
+    const width = dimensions?.css?.canvas?.width || 800;
+    const height = dimensions?.css?.canvas?.height || 600;
+
+    // Create canvas
+    this._canvas = document.createElement('canvas');
+    this._canvas.width = width;
+    this._canvas.height = height;
+    this._canvas.classList.add('xterm-kitty-graphics-layer');
+
+    // Position absolutely over the terminal
+    this._canvas.style.position = 'absolute';
+    this._canvas.style.top = '0';
+    this._canvas.style.left = '0';
+    this._canvas.style.pointerEvents = 'none';
+    this._canvas.style.zIndex = '10';
+
+    screenElement.appendChild(this._canvas);
+    this._ctx = this._canvas.getContext('2d', { alpha: true });
+
+    // Hook into render events to redraw when terminal scrolls
+    this._renderDisposable = this._terminal.onRender(() => this._onRender());
+
+    // Handle resize
+    this._resizeDisposable = this._terminal.onResize(() => this._onResize());
+  }
+
+  /**
+   * Get cell dimensions from terminal.
+   */
+  public getCellSize(): { width: number; height: number } {
+    const dimensions = (this._terminal as any).dimensions;
+    return {
+      width: dimensions?.css?.cell?.width || 9,
+      height: dimensions?.css?.cell?.height || 17
+    };
+  }
+
+  /**
+   * Place a decoded image at cursor position.
+   * @param width - Optional width in pixels (0 = original)
+   * @param height - Optional height in pixels (0 = original)
+   */
+  public placeImage(bitmap: ImageBitmap, id: number, col?: number, row?: number, width?: number, height?: number): number {
+    this.ensureCanvasLayer();
+
+    const buffer = this._terminal.buffer.active;
+    const placementId = ++this._placementIdCounter;
+
+    const placement: IPlacedImage = {
+      bitmap,
+      col: col ?? buffer.cursorX,
+      row: row ?? (buffer.cursorY + buffer.baseY),
+      width: width || 0,
+      height: height || 0,
+      id
+    };
+
+    this._placements.set(placementId, placement);
+    this._draw();
+
+    return placementId;
+  }
+
+  /**
+   * Remove a placed image.
+   */
+  public removePlacement(placementId: number): void {
+    const placement = this._placements.get(placementId);
+    if (placement) {
+      this._placements.delete(placementId);
+      this._draw();
+    }
+  }
+
+  /**
+   * Remove all placements for an image ID.
+   */
+  public removeByImageId(imageId: number): void {
+    const toDelete: number[] = [];
+    for (const [placementId, placement] of this._placements) {
+      if (placement.id === imageId) {
+        toDelete.push(placementId);
+      }
+    }
+    for (const id of toDelete) {
+      this._placements.delete(id);
+    }
+    if (toDelete.length > 0) {
+      this._draw();
+    }
+  }
+
+  /**
+   * Clear all images.
+   */
+  public clearAll(): void {
+    this._placements.clear();
+    if (this._ctx && this._canvas) {
+      this._ctx.clearRect(0, 0, this._canvas.width, this._canvas.height);
+    }
+  }
+
+  /**
+   * Redraw all placed images.
+   */
+  private _draw(): void {
+    if (!this._ctx || !this._canvas) {
+      return;
+    }
+
+    // Clear canvas
+    this._ctx.clearRect(0, 0, this._canvas.width, this._canvas.height);
+
+    const buffer = this._terminal.buffer.active;
+    const viewportStartRow = buffer.baseY;
+    const viewportRows = this._terminal.rows;
+    const cellSize = this.getCellSize();
+
+    // Draw each placement that's visible
+    for (const placement of this._placements.values()) {
+      // Check if placement is in viewport
+      const relativeRow = placement.row - viewportStartRow;
+      if (relativeRow < 0 || relativeRow >= viewportRows) {
+        continue; // Not in viewport
+      }
+
+      const x = placement.col * cellSize.width;
+      const y = relativeRow * cellSize.height;
+
+      const width = placement.width || placement.bitmap.width;
+      const height = placement.height || placement.bitmap.height;
+
+      this._ctx.drawImage(placement.bitmap, x, y, width, height);
+    }
+  }
+
+  /**
+   * Called when terminal renders (scrolling, content change).
+   */
+  private _onRender(): void {
+    this._draw();
+  }
+
+  /**
+   * Called when terminal resizes.
+   */
+  private _onResize(): void {
+    if (!this._canvas) {
+      return;
+    }
+
+    const dimensions = (this._terminal as any).dimensions;
+    const width = dimensions?.css?.canvas?.width || 800;
+    const height = dimensions?.css?.canvas?.height || 600;
+
+    if (this._canvas.width !== width || this._canvas.height !== height) {
+      this._canvas.width = width;
+      this._canvas.height = height;
+      this._draw();
+    }
+  }
+
+  public dispose(): void {
+    this._renderDisposable?.dispose();
+    this._resizeDisposable?.dispose();
+
+    // Close all bitmaps
+    for (const placement of this._placements.values()) {
+      placement.bitmap.close();
+    }
+    this._placements.clear();
+
+    if (this._canvas) {
+      this._canvas.remove();
+      this._canvas = undefined;
+      this._ctx = undefined;
+    }
+  }
+}

addons/addon-kitty-graphics/test/KittyGraphicsAddon.test.ts

@@ -127,8 +127,8 @@ test.describe('KittyGraphicsAddon', () => {
     });
   });
 
-  // TODO: These tests will fail until decode + rendering is implemented
-  test.describe.skip('pixel verification', () => {
+  test.describe('pixel verification', () => {
+    // TODO: Add more intense ones.
     test('renders 1x1 black PNG at cursor position', async () => {
       // Send image with a=T to transmit and display
       const seq = `\x1b_Ga=T,f=100;${BLACK_1X1_BASE64}\x1b\\`;

send-png

@@ -5,8 +5,9 @@ Minimal Kitty Graphics Protocol image sender.
 Based on: https://sw.kovidgoyal.net/kitty/graphics-protocol/#a-minimal-example
 
 Usage:
-    ./send-png black-1x1.png
-    ./send-png rgb-3x1.png
+    ./send-png image.png           # Original size
+    ./send-png image.png 10        # Scale to 10 columns wide
+    ./send-png image.png 10 5      # Scale to 10 columns x 5 rows
 """
 import sys
 from base64 import standard_b64encode
@@ -16,18 +17,25 @@ def serialize_image(path):
         data = f.read()
     return standard_b64encode(data).decode('ascii')
 
-def write_chunked(data):
+def write_chunked(data, columns=None, rows=None):
     # a=T means transmit and display
     # f=100 means PNG format
-    # For small images, single chunk (no chunking needed)
-    sys.stdout.write(f'\x1b_Ga=T,f=100;{data}\x1b\\')
+    # c=columns, r=rows for sizing
+    params = 'a=T,f=100'
+    if columns:
+        params += f',c={columns}'
+    if rows:
+        params += f',r={rows}'
+    sys.stdout.write(f'\x1b_G{params};{data}\x1b\\')
     sys.stdout.flush()
 
 if __name__ == '__main__':
     if len(sys.argv) < 2:
-        print("Usage: ./send-png <image.png>", file=sys.stderr)
+        print("Usage: ./send-png <image.png> [columns] [rows]", file=sys.stderr)
         sys.exit(1)
 
     img_data = serialize_image(sys.argv[1])
-    write_chunked(img_data)
+    columns = int(sys.argv[2]) if len(sys.argv) > 2 else None
+    rows = int(sys.argv[3]) if len(sys.argv) > 3 else None
+    write_chunked(img_data, columns, rows)
     print()  # newline after the sequence