From 1442f84e4ef93c3e930f446988e5426928dc061b Mon Sep 17 00:00:00 2001 From: Anthony Kim Date: Tue, 27 Jan 2026 17:21:38 -0800 Subject: [PATCH] Fix chunking for large data, allow pixels --- .../src/KittyGraphicsAddon.ts | 200 ++++++++++++++++-- .../typings/addon-kitty-graphics.d.ts | 7 + demo/client/client.ts | 8 +- 3 files changed, 192 insertions(+), 23 deletions(-) diff --git a/addons/addon-kitty-graphics/src/KittyGraphicsAddon.ts b/addons/addon-kitty-graphics/src/KittyGraphicsAddon.ts index 51e67681..10be97b7 100644 --- a/addons/addon-kitty-graphics/src/KittyGraphicsAddon.ts +++ b/addons/addon-kitty-graphics/src/KittyGraphicsAddon.ts @@ -31,6 +31,15 @@ const enum KittyFormat { PNG = 100 } +/** + * Kitty graphics protocol compression types. + * See: https://sw.kovidgoyal.net/kitty/graphics-protocol/#control-data-reference under key 'o'. + */ +const enum KittyCompression { + NONE = '', + ZLIB = 'z' +} + /** * Parsed Kitty graphics command. */ @@ -46,6 +55,7 @@ export interface IKittyCommand { rows?: number; more?: number; quiet?: number; + compression?: string; payload?: string; } @@ -66,8 +76,6 @@ export function parseKittyCommand(data: string): IKittyCommand { switch (key) { // These are key(s) from: https://sw.kovidgoyal.net/kitty/graphics-protocol/#control-data-reference - // Question: How do we know which radix to use? - // TODO: Support more keys like type of compression, animations, etc case 'a': cmd.action = value; break; case 'f': cmd.format = parseInt(value); break; case 'i': cmd.id = parseInt(value); break; @@ -79,12 +87,24 @@ export function parseKittyCommand(data: string): IKittyCommand { case 'r': cmd.rows = parseInt(value); break; case 'm': cmd.more = parseInt(value); break; case 'q': cmd.quiet = parseInt(value); break; + case 'o': cmd.compression = value; break; } } return cmd; } +/** + * Pending chunked transmission state. + * Stores metadata from the first chunk while accumulating payload data. + */ +interface IPendingTransmission { + /** The parsed command from the first chunk (contains action, format, dimensions, etc.) */ + cmd: IKittyCommand; + /** Accumulated base64 payload data */ + data: string; +} + export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi { private _terminal: ITerminalExt | undefined; private _apcHandler: IDisposable | undefined; @@ -93,7 +113,11 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi { // Maybe add more, rename to IKittyImageStorage instead of IKittyImage? private _images: Map = new Map(); private _decodedImages: Map = new Map(); - private _pendingData: Map = new Map(); + /** + * Pending chunked transmissions keyed by image ID. + * ID 0 is used for transmissions without an explicit ID (the "anonymous" transmission). + */ + private _pendingTransmissions: Map = new Map(); private _nextImageId = 1; private _debug: boolean; @@ -124,7 +148,7 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi { this._apcHandler?.dispose(); this._renderer?.dispose(); this._images.clear(); - this._pendingData.clear(); + this._pendingTransmissions.clear(); // Close all decoded bitmaps for (const bitmap of this._decodedImages.values()) { @@ -165,27 +189,88 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi { } private _handleTransmit(cmd: IKittyCommand): boolean { - const id = cmd.id || this._nextImageId++; - const payload = cmd.payload || ''; + const payload = cmd.payload ?? ''; + // Use 0 as the key for anonymous transmissions (no explicit i=) + const pendingKey = cmd.id ?? 0; - if (cmd.more === 1) { - const existing = this._pendingData.get(id) || ''; - this._pendingData.set(id, existing + payload); + // Check if this is a continuation of a chunked transmission + const isMoreComing = cmd.more === 1; + const pending = this._pendingTransmissions.get(pendingKey); + + if (pending) { + // This is a continuation chunk - append data + pending.data += payload; + + if (this._debug) { + console.log(`[KittyGraphicsAddon] Chunk continuation for id=${pendingKey}, total size=${pending.data.length}, more=${isMoreComing}`); + } + + if (isMoreComing) { + // More chunks coming, keep waiting + return true; + } + + // Final chunk - use the stored command's metadata + const originalCmd = pending.cmd; + const fullPayload = pending.data; + this._pendingTransmissions.delete(pendingKey); + + // Assign a real ID if this was anonymous (pendingKey === 0) + const id = originalCmd.id ?? this._nextImageId++; + + const image: IKittyImage = { + id, + data: fullPayload, + width: originalCmd.width ?? 0, + height: originalCmd.height ?? 0, + format: (originalCmd.format ?? KittyFormat.PNG) as 24 | 32 | 100, + compression: originalCmd.compression ?? '' + }; + + this._images.set(id, image); + + if (this._debug) { + console.log(`[KittyGraphicsAddon] Stored chunked image ${id}, payload size=${fullPayload.length}`); + } + + // If the original action was 'T' (transmit+display), display the image now + if (originalCmd.action === KittyAction.TRANSMIT_DISPLAY) { + this._decodeAndDisplay(image, originalCmd.columns, originalCmd.rows).catch(err => { + if (this._debug) { + console.error(`[KittyGraphicsAddon] Failed to decode/display chunked image ${id}:`, err); + } + }); + } + + cmd.id = id; return true; } - let fullPayload = payload; - if (this._pendingData.has(id)) { - fullPayload = this._pendingData.get(id)! + payload; - this._pendingData.delete(id); + // This is the first chunk (or a non-chunked transmission) + if (isMoreComing) { + // First chunk of a multi-chunk transmission - store the command and start accumulating + this._pendingTransmissions.set(pendingKey, { + cmd: { ...cmd }, // Clone the command to preserve metadata + data: payload + }); + + if (this._debug) { + console.log(`[KittyGraphicsAddon] Started chunked transmission, id=${pendingKey}, format=${cmd.format}, compression=${cmd.compression}, size=${cmd.width}x${cmd.height}, initial payload=${payload.length}`); + } + + return true; } + // Non-chunked transmission - store immediately + const id = cmd.id ?? this._nextImageId++; + const image: IKittyImage = { id, - data: fullPayload, - width: cmd.width || 0, - height: cmd.height || 0, - format: (cmd.format || KittyFormat.PNG) as 24 | 32 | 100 + data: payload, + width: cmd.width ?? 0, + height: cmd.height ?? 0, + format: (cmd.format ?? KittyFormat.PNG) as 24 | 32 | 100, + compression: cmd.compression ?? '' }; this._images.set(id, image); @@ -194,18 +279,41 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi { console.log(`[KittyGraphicsAddon] Stored image ${id}`); } + // Update cmd.id for _handleTransmitDisplay + cmd.id = id; return true; } private _handleTransmitDisplay(cmd: IKittyCommand): boolean { - // First store the image + const pendingKey = cmd.id ?? 0; + const wasPendingBefore = this._pendingTransmissions.has(pendingKey); + + // Delegate to _handleTransmit which will: + // 1. Accumulate chunks if m=1 + // 2. Store the image when complete + // 3. Trigger display when chunked transmission completes (checks original action) this._handleTransmit(cmd); - // Get the image ID (same logic as _handleTransmit) - const id = cmd.id || (this._nextImageId - 1); + // If this was a chunked transmission (first chunk with m=1), _handleTransmit + // stored it as pending and will display when complete. Don't display now. + if (cmd.more === 1) { + return true; + } + + // If there was a pending transmission that just completed, _handleTransmit + // already triggered the display. Don't display again. + if (wasPendingBefore) { + return true; + } + + // For non-chunked transmission, display now + const id = cmd.id!; const image = this._images.get(id); if (!image) { + if (this._debug) { + console.log(`[KittyGraphicsAddon] No image found for id=${id} after transmit`); + } return true; } @@ -228,11 +336,19 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi { // Decode base64 to binary const binaryString = atob(base64Data); - const bytes = new Uint8Array(binaryString.length); + let bytes = new Uint8Array(binaryString.length); for (let i = 0; i < binaryString.length; i++) { bytes[i] = binaryString.charCodeAt(i); } + // Decompress if needed (o=z means zlib/deflate compression) + if (image.compression === KittyCompression.ZLIB) { + bytes = await this._decompressZlib(bytes); + if (this._debug) { + console.log(`[KittyGraphicsAddon] Decompressed ${binaryString.length} -> ${bytes.length} bytes`); + } + } + if (format === KittyFormat.PNG) { // PNG: create blob and decode const blob = new Blob([bytes], { type: 'image/png' }); @@ -273,6 +389,48 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi { return createImageBitmap(imageData); } + /** + * Decompress zlib/deflate compressed data using the browser's DecompressionStream API. + */ + private async _decompressZlib(compressed: Uint8Array): Promise { + // Use DecompressionStream API (available in modern browsers) + // 'deflate-raw' is the format used by zlib without headers + // Try 'deflate' first (zlib with header), fall back to 'deflate-raw' if that fails + try { + return await this._decompress(compressed, 'deflate'); + } catch { + // If 'deflate' fails, try 'deflate-raw' + return await this._decompress(compressed, 'deflate-raw'); + } + } + + private async _decompress(compressed: Uint8Array, format: 'deflate' | 'deflate-raw'): Promise { + const ds = new DecompressionStream(format); + const writer = ds.writable.getWriter(); + writer.write(compressed); + writer.close(); + + const chunks: Uint8Array[] = []; + const reader = ds.readable.getReader(); + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + } + + // Combine chunks into single Uint8Array + const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + const result = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.length; + } + + return result; + } + /** * Decode an image and display it at the cursor position. * @param image - The Kitty image to decode and display diff --git a/addons/addon-kitty-graphics/typings/addon-kitty-graphics.d.ts b/addons/addon-kitty-graphics/typings/addon-kitty-graphics.d.ts index 792f516d..32a137cb 100644 --- a/addons/addon-kitty-graphics/typings/addon-kitty-graphics.d.ts +++ b/addons/addon-kitty-graphics/typings/addon-kitty-graphics.d.ts @@ -78,5 +78,12 @@ declare module '@xterm/addon-kitty-graphics' { * - 100: PNG */ format: 24 | 32 | 100; + + /** + * Compression type. + * - '': no compression + * - 'z': zlib compression + */ + compression?: string; } } diff --git a/demo/client/client.ts b/demo/client/client.ts index 70ed39b5..a2ccd40a 100644 --- a/demo/client/client.ts +++ b/demo/client/client.ts @@ -327,7 +327,9 @@ function createTerminal(): Terminal { } const cols = size.cols; const rows = size.rows; - const url = '/terminals/' + pid + '/size?cols=' + cols + '&rows=' + rows; + const pixelWidth = Math.round(term.dimensions?.css?.canvas?.width ?? 0); + const pixelHeight = Math.round(term.dimensions?.css?.canvas?.height ?? 0); + const url = '/terminals/' + pid + '/size?cols=' + cols + '&rows=' + rows + '&pixelWidth=' + pixelWidth + '&pixelHeight=' + pixelHeight; fetch(url, { method: 'POST' }); }); @@ -378,7 +380,9 @@ function createTerminal(): Terminal { if (useRealTerminal instanceof HTMLInputElement && !useRealTerminal.checked) { runFakeTerminal(); } else { - const res = await fetch('/terminals?cols=' + term.cols + '&rows=' + term.rows, { method: 'POST' }); + const pixelWidth = Math.round(term.dimensions?.css?.canvas?.width ?? 0); + const pixelHeight = Math.round(term.dimensions?.css?.canvas?.height ?? 0); + const res = await fetch('/terminals?cols=' + term.cols + '&rows=' + term.rows + '&pixelWidth=' + pixelWidth + '&pixelHeight=' + pixelHeight, { method: 'POST' }); const processId = await res.text(); pid = processId; socketURL += processId;