diff --git a/addons/addon-image/src/kitty/KittyGraphicsHandler.ts b/addons/addon-image/src/kitty/KittyGraphicsHandler.ts index bc5d6c02..6886df4e 100644 --- a/addons/addon-image/src/kitty/KittyGraphicsHandler.ts +++ b/addons/addon-image/src/kitty/KittyGraphicsHandler.ts @@ -1,9 +1,6 @@ /** * Copyright (c) 2025 The xterm.js authors. All rights reserved. * @license MIT - * - * Kitty graphics protocol APC handler. - * Implements IApcHandler to integrate with xterm.js parser. */ import { IApcHandler, IImageAddonOptions, IResetHandler, ITerminalExt } from '../Types'; @@ -26,41 +23,53 @@ import { // Memory limit for base64 decoder (4MB, same as IIPHandler) const DECODER_KEEP_DATA = 4194304; -// Base64 shard size for chunked decoding (1MB encoded → ~768KB decoded) -// This prevents memory pressure with large images +// Base64 shard size (~1MB, must be divisible by 4) const BASE64_SHARD_SIZE = 1048576; +// Decoded shard size: 1MB base64 → 768KB decoded +const DECODED_SHARD_SIZE = 786432; + +// Maximum control data size +const MAX_CONTROL_DATA_SIZE = 4096; + +// Semicolon codepoint +const SEMICOLON = 0x3B; + +// Padding character '=' +const EQUALS = 0x3D; + /** - * Kitty graphics protocol handler. - * - * Handles APC sequences for the Kitty graphics protocol, supporting: - * - Image transmission (a=t) - * - Image transmission + display (a=T) - * - Query (a=q) - * - Delete (a=d) - * - * TODO: File transmission (t=f) is not supported since browsers cannot access the filesystem. - * Explore using the File System Access API for opt-in filesystem access when available. + * Kitty graphics protocol handler with streaming base64 decoding. */ export class KittyGraphicsHandler implements IApcHandler, IResetHandler { private _aborted = false; - private _data: string = ''; + private _decodeError = false; + + /** Reusable base64 decoder - avoids WASM cold-start on each chunk */ + private _decoder = new Base64Decoder(DECODER_KEEP_DATA); + + // Streaming related states + + /** True while receiving control data (before semicolon). */ + private _inControlData = true; + + /** Buffer for control data. */ + private _controlData = new Uint32Array(MAX_CONTROL_DATA_SIZE); + private _controlLength = 0; + + /** Shard buffer for base64 bytes (filled until 4-byte aligned boundary). */ + private _shardBuffer = new Uint32Array(BASE64_SHARD_SIZE); + private _shardBufferPos = 0; + + /** Accumulated decoded chunks. */ + private _decodedChunks: Uint8Array[] = []; + private _totalDecodedSize = 0; + + // Storage related states - /** - * Pending chunked transmissions keyed by image ID. - * ID 0 is used for transmissions without an explicit ID (the "anonymous" transmission). - */ private _pendingTransmissions: Map = new Map(); private _nextImageId = 1; - - /** - * Stored images - kept for protocol compliance (transmit without display, then display later). - */ private _images: Map = new Map(); - - /** - * Cached decoded bitmaps. - */ private _decodedImages: Map = new Map(); constructor( @@ -71,217 +80,473 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler { ) {} public reset(): void { - // Clear pending transmissions this._pendingTransmissions.clear(); - // Clear stored images this._images.clear(); - // Close decoded bitmaps for (const bitmap of this._decodedImages.values()) { bitmap.close(); } this._decodedImages.clear(); } - /** - * Called at the start of a new APC sequence. - */ public start(): void { this._aborted = false; - this._data = ''; + this._decodeError = false; + this._inControlData = true; + this._controlLength = 0; + this._shardBufferPos = 0; + this._decodedChunks = []; + this._totalDecodedSize = 0; } - /** - * Called for each chunk of data in the APC sequence. - * Accumulates codepoints as a string for control data parsing. - * The base64 payload is decoded later using the wasm decoder. - */ public put(data: Uint32Array, start: number, end: number): void { if (this._aborted) return; + if (this._inControlData) { + // Scan for semicolon + let controlEnd = end; + for (let i = start; i < end; i++) { + if (data[i] === SEMICOLON) { + this._inControlData = false; + controlEnd = i; + break; + } + } + + // Copy control data + const copyLength = controlEnd - start; + if (this._controlLength + copyLength > MAX_CONTROL_DATA_SIZE) { + this._aborted = true; + return; + } + this._controlData.set(data.subarray(start, controlEnd), this._controlLength); + this._controlLength += copyLength; + + if (!this._inControlData) { + // Found semicolon - stream remaining as payload + const payloadStart = controlEnd + 1; + if (payloadStart < end) { + this._streamPayload(data, payloadStart, end); + } + } + } else { + this._streamPayload(data, start, end); + } + } + + /** + * Stream payload bytes, decode at 4-byte aligned boundaries. + */ + private _streamPayload(data: Uint32Array, start: number, end: number): void { + if (this._aborted) return; + // Check size limit - if (this._data.length + (end - start) > this._opts.kittySizeLimit) { - console.warn('[KittyGraphicsHandler] Data exceeds size limit, aborting'); + const estimatedTotal = this._totalDecodedSize + Math.ceil(this._shardBufferPos * 3 / 4) + Math.ceil((end - start) * 3 / 4); + if (estimatedTotal > this._opts.kittySizeLimit) { this._aborted = true; return; } - // Convert Uint32Array codepoints to string for control data parsing - // Note: We accumulate as string because we need to parse control data (before semicolon). - // The base64 payload is decoded efficiently using the wasm-based decoder in _decodeImage(). - for (let i = start; i < end; i++) { - this._data += String.fromCodePoint(data[i]); + const inputLength = end - start; + const freeSpace = BASE64_SHARD_SIZE - this._shardBufferPos; + + if (inputLength < freeSpace) { + // Fits in current shard + this._shardBuffer.set(data.subarray(start, end), this._shardBufferPos); + this._shardBufferPos += inputLength; + return; + } + + // Fill shard to capacity + this._shardBuffer.set(data.subarray(start, start + freeSpace), this._shardBufferPos); + this._shardBufferPos = BASE64_SHARD_SIZE; + + // Decode full shard (exactly 1MB → 768KB) + if (!this._decodeFullShard()) { + this._aborted = true; + return; + } + + // Reset and process remaining + this._shardBufferPos = 0; + const remaining = start + freeSpace; + if (remaining < end) { + this._streamPayload(data, remaining, end); } } /** - * Called at the end of the APC sequence. + * Decode a full 1MB shard (768KB output). */ + private _decodeFullShard(): boolean { + this._decoder.init(DECODED_SHARD_SIZE); + + if (this._decoder.put(this._shardBuffer, 0, BASE64_SHARD_SIZE)) { + this._decoder.release(); + this._decodeError = true; + return false; + } + + if (this._decoder.end()) { + this._decoder.release(); + this._decodeError = true; + return false; + } + + const chunk = new Uint8Array(this._decoder.data8); + this._decodedChunks.push(chunk); + this._totalDecodedSize += chunk.length; + this._decoder.release(); + return true; + } + public end(success: boolean): boolean | Promise { if (this._aborted || !success) { return true; } - return this._handleCommand(this._data); + // No semicolon = no payload (delete, capability query) + if (this._inControlData) { + return this._handleNoPayloadCommand(); + } + + // Parse command to check m=1 and get pending key + const cmd = parseKittyCommand(this._parseControlDataString()); + const pendingKey = cmd.id ?? 0; + const isMoreComing = cmd.more === 1; + const pending = this._pendingTransmissions.get(pendingKey); + + // If continuing a pending transmission, prepend leftover bytes + if (pending && pending.leftoverLength > 0) { + // Shift current buffer to make room for leftover + const newPos = pending.leftoverLength + this._shardBufferPos; + if (newPos <= BASE64_SHARD_SIZE) { + // Shift existing data right + for (let i = this._shardBufferPos - 1; i >= 0; i--) { + this._shardBuffer[i + pending.leftoverLength] = this._shardBuffer[i]; + } + // Prepend leftover + this._shardBuffer.set(pending.leftover.subarray(0, pending.leftoverLength), 0); + this._shardBufferPos = newPos; + } + pending.leftoverLength = 0; + } + + // Decode with 4-byte alignment, preserving leftover if m=1 + if (this._shardBufferPos > 0) { + if (isMoreComing) { + this._decodePartialShardWithLeftover(pendingKey, cmd); + } else { + this._decodePartialShard(); + } + } + + // Combine all decoded chunks + const imageBytes = this._combineChunks(); + + return this._handleCommandWithBytesAndCmd(cmd, imageBytes, this._decodeError); } /** - * Handle a complete Kitty graphics command. + * Decode partial shard, storing leftover bytes for m=1 chunks. */ - private _handleCommand(data: string): boolean | Promise { - const semiIdx = data.indexOf(';'); - const controlData = semiIdx === -1 ? data : data.substring(0, semiIdx); - const payload = semiIdx === -1 ? '' : data.substring(semiIdx + 1); + private _decodePartialShardWithLeftover(pendingKey: number, cmd: IKittyCommand): boolean { + let length = this._shardBufferPos; + if (length === 0) return true; - const cmd = parseKittyCommand(controlData); - cmd.payload = payload; + // Align to 4 bytes + const aligned = Math.floor(length / 4) * 4; + const leftoverCount = length - aligned; + // Store leftover bytes for next chunk + if (leftoverCount > 0) { + let pending = this._pendingTransmissions.get(pendingKey); + if (!pending) { + pending = { + cmd: { ...cmd }, + chunks: [], + totalSize: 0, + leftover: new Uint32Array(4), + leftoverLength: 0 + }; + this._pendingTransmissions.set(pendingKey, pending); + } + pending.leftover.set(this._shardBuffer.subarray(aligned, length), 0); + pending.leftoverLength = leftoverCount; + } + + if (aligned === 0) return true; + + // Count padding (shouldn't have padding in m=1 chunks, but be safe) + let padding = 0; + if (aligned >= 1 && this._shardBuffer[aligned - 1] === EQUALS) padding++; + if (aligned >= 2 && this._shardBuffer[aligned - 2] === EQUALS) padding++; + + const decodedSize = Math.floor(aligned * 3 / 4) - padding; + if (decodedSize <= 0) return true; + + this._decoder.init(decodedSize); + + if (this._decoder.put(this._shardBuffer, 0, aligned)) { + this._decoder.release(); + this._decodeError = true; + return false; + } + + if (this._decoder.end()) { + this._decoder.release(); + this._decodeError = true; + return false; + } + + const chunk = new Uint8Array(this._decoder.data8); + this._decodedChunks.push(chunk); + this._totalDecodedSize += chunk.length; + this._decoder.release(); + return true; + } + + /** + * Decode partial shard with proper 4-byte alignment. + * For final chunks, pad with = if not aligned instead of truncating. + */ + private _decodePartialShard(): boolean { + let length = this._shardBufferPos; + if (length === 0) return true; + + + // Count ORIGINAL padding BEFORE we add any + let padding = 0; + if (length >= 1 && this._shardBuffer[length - 1] === EQUALS) padding++; + if (length >= 2 && this._shardBuffer[length - 2] === EQUALS) padding++; + + // Pad to 4-byte boundary with = if needed (instead of truncating) + const remainder = length % 4; + if (remainder > 0) { + const paddingNeeded = 4 - remainder; + for (let i = 0; i < paddingNeeded; i++) { + this._shardBuffer[length + i] = EQUALS; + } + length += paddingNeeded; + // Count the artificial padding too! + padding += paddingNeeded; + } + + // Calculate decoded size + const decodedSize = Math.floor(length * 3 / 4) - padding; + if (decodedSize <= 0) return true; + + this._decoder.init(decodedSize); + + if (this._decoder.put(this._shardBuffer, 0, length)) { + this._decoder.release(); + this._decodeError = true; + return false; + } + + if (this._decoder.end()) { + this._decoder.release(); + this._decodeError = true; + return false; + } + + const chunk = new Uint8Array(this._decoder.data8); + this._decodedChunks.push(chunk); + this._totalDecodedSize += chunk.length; + this._decoder.release(); + return true; + } + + private _combineChunks(): Uint8Array { + if (this._decodedChunks.length === 0) return new Uint8Array(0); + if (this._decodedChunks.length === 1) return this._decodedChunks[0]; + + const result = new Uint8Array(this._totalDecodedSize); + let offset = 0; + for (const chunk of this._decodedChunks) { + result.set(chunk, offset); + offset += chunk.length; + } + return result; + } + + // Command handling + + private _parseControlDataString(): string { + let str = ''; + for (let i = 0; i < this._controlLength; i++) { + str += String.fromCodePoint(this._controlData[i]); + } + return str; + } + + private _handleNoPayloadCommand(): boolean | Promise { + const cmd = parseKittyCommand(this._parseControlDataString()); const action = cmd.action ?? 't'; switch (action) { - case KittyAction.TRANSMIT: - return this._handleTransmit(cmd); - case KittyAction.TRANSMIT_DISPLAY: - return this._handleTransmitDisplay(cmd); - case KittyAction.QUERY: - return this._handleQuery(cmd); case KittyAction.DELETE: return this._handleDelete(cmd); + case KittyAction.QUERY: + this._sendResponse(cmd.id ?? 0, 'OK', cmd.quiet ?? 0); + return true; default: return true; } } - private _handleTransmit(cmd: IKittyCommand): boolean { - const payload = cmd.payload ?? ''; + private _handleCommandWithBytesAndCmd(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean | Promise { + const action = cmd.action ?? 't'; + + switch (action) { + case KittyAction.TRANSMIT: + return this._handleTransmit(cmd, bytes, decodeError); + case KittyAction.TRANSMIT_DISPLAY: + return this._handleTransmitDisplay(cmd, bytes, decodeError); + case KittyAction.QUERY: + return this._handleQuery(cmd, bytes, decodeError); + default: + return true; + } + } + + private _handleTransmit(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean { const pendingKey = cmd.id ?? 0; const isMoreComing = cmd.more === 1; const pending = this._pendingTransmissions.get(pendingKey); - if (pending) { - pending.data += payload; - if (isMoreComing) { - return true; + // For m=1 intermediate chunks, reject if decode error + if (decodeError && !pending && isMoreComing) return true; + + if (pending) { + // Append to existing pending transmission (even if decode error, we keep accumulated data) + if (bytes.length > 0) { + pending.chunks.push(bytes); + pending.totalSize += bytes.length; } - const originalCmd = pending.cmd; - const fullPayload = pending.data; + if (isMoreComing) return true; + + // Final chunk - merge all (even if the final chunk had a decode error) + const fullData = this._mergeChunks(pending.chunks, pending.totalSize); this._pendingTransmissions.delete(pendingKey); - const id = originalCmd.id ?? this._nextImageId++; + if (fullData.length === 0) return true; // Nothing to store + + const id = pending.cmd.id ?? this._nextImageId++; const image: IKittyImageData = { id, - data: fullPayload, - width: originalCmd.width ?? 0, - height: originalCmd.height ?? 0, - format: (originalCmd.format ?? KittyFormat.PNG) as 24 | 32 | 100, - compression: originalCmd.compression ?? '' + data: fullData, + width: pending.cmd.width ?? 0, + height: pending.cmd.height ?? 0, + format: (pending.cmd.format ?? KittyFormat.PNG) as 24 | 32 | 100, + compression: pending.cmd.compression ?? '' }; + this._images.set(id, image); - this._images.set(image.id, image); - - if (originalCmd.action === KittyAction.TRANSMIT_DISPLAY) { - // Fire-and-forget the display, transmit itself succeeded - void this._displayImage(image, originalCmd.columns, originalCmd.rows); - return true; - } - - cmd.id = id; + // Note: Don't display here - _handleTransmitDisplay will do it return true; } if (isMoreComing) { + // Start new pending transmission (reject if decode error for first chunk) + if (decodeError) return true; this._pendingTransmissions.set(pendingKey, { cmd: { ...cmd }, - data: payload + chunks: bytes.length > 0 ? [bytes] : [], + totalSize: bytes.length, + leftover: new Uint32Array(4), + leftoverLength: 0 }); return true; } - const id = cmd.id ?? this._nextImageId++; + // Single-chunk transmission - reject if decode error or no data + if (decodeError || bytes.length === 0) return true; + const id = cmd.id ?? this._nextImageId++; const image: IKittyImageData = { id, - data: payload, + data: bytes, width: cmd.width ?? 0, height: cmd.height ?? 0, format: (cmd.format ?? KittyFormat.PNG) as 24 | 32 | 100, compression: cmd.compression ?? '' }; - - this._images.set(image.id, image); - cmd.id = id; + this._images.set(id, image); return true; } - private _handleTransmitDisplay(cmd: IKittyCommand): boolean | Promise { + private _mergeChunks(chunks: Uint8Array[], totalSize: number): Uint8Array { + if (chunks.length === 1) return chunks[0]; + const result = new Uint8Array(totalSize); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.length; + } + return result; + } + + private _handleTransmitDisplay(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean | Promise { + if (decodeError) return true; + const pendingKey = cmd.id ?? 0; - const wasPendingBefore = this._pendingTransmissions.has(pendingKey); - this._handleTransmit(cmd); + this._handleTransmit(cmd, bytes, decodeError); - if (cmd.more === 1) { - return true; - } + // If still accumulating chunks, don't display yet + if (this._pendingTransmissions.has(pendingKey)) return true; - if (wasPendingBefore) { - return true; - } - - const id = cmd.id!; + // Display the completed image + const id = cmd.id ?? this._nextImageId - 1; const image = this._images.get(id); if (image) { return this._displayImage(image, cmd.columns, cmd.rows); } - return true; } - /** - * Handle query action (a=q). - */ - private _handleQuery(cmd: IKittyCommand): boolean { + private _handleQuery(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean { const id = cmd.id ?? 0; const quiet = cmd.quiet ?? 0; - const payload = cmd.payload || ''; - if (!payload) { + // Check decode error first (invalid base64) + if (decodeError) { + this._sendResponse(id, 'EINVAL:invalid base64 data', quiet); + return true; + } + + // Capability query (no payload) - just respond OK + if (bytes.length === 0) { this._sendResponse(id, 'OK', quiet); return true; } - try { - const bytes = this._decodeBase64(payload); + const format = cmd.format ?? KittyFormat.RGBA; - const format = cmd.format || KittyFormat.RGBA; + if (format === KittyFormat.PNG) { + this._sendResponse(id, 'OK', quiet); + } else { + const width = cmd.width ?? 0; + const height = cmd.height ?? 0; - if (format === KittyFormat.PNG) { - this._sendResponse(id, 'OK', quiet); - } else { - const width = cmd.width || 0; - const height = cmd.height || 0; - - if (!width || !height) { - this._sendResponse(id, 'EINVAL:width and height required for raw pixel data', quiet); - return true; - } - - const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB; - const expectedBytes = width * height * bytesPerPixel; - - if (bytes.length < expectedBytes) { - this._sendResponse(id, `EINVAL:insufficient pixel data, got ${bytes.length}, expected ${expectedBytes}`, quiet); - return true; - } - - this._sendResponse(id, 'OK', quiet); + if (!width || !height) { + this._sendResponse(id, 'EINVAL:width and height required for raw pixel data', quiet); + return true; } - } catch (e) { - const errorMsg = e instanceof Error ? e.message : 'unknown error'; - this._sendResponse(id, `EINVAL:${errorMsg}`, quiet); - } + const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB; + const expectedBytes = width * height * bytesPerPixel; + + if (bytes.length < expectedBytes) { + this._sendResponse(id, `EINVAL:insufficient pixel data`, quiet); + return true; + } + + this._sendResponse(id, 'OK', quiet); + } return true; } @@ -302,13 +567,9 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler { } this._decodedImages.clear(); } - return true; } - /** - * Send a response back to the client. - */ private _sendResponse(id: number, message: string, quiet: number): void { const isOk = message === 'OK'; if (isOk && quiet === 1) return; @@ -318,27 +579,22 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler { this._coreTerminal._core.coreService.triggerDataEvent(response); } - /** - * Decode and display an image using the shared ImageStorage. - */ + // Image display + private _displayImage(image: IKittyImageData, columns?: number, rows?: number): boolean | Promise { return this._decodeAndDisplay(image, columns, rows) .then(() => true) - .catch(err => { - console.warn('[KittyHandler] Failed to decode/display image:', err); - return true; - }); + .catch(() => true); } private async _decodeAndDisplay(image: IKittyImageData, columns?: number, rows?: number): Promise { let bitmap = this._decodedImages.get(image.id); if (!bitmap) { - bitmap = await this._decodeImage(image); + bitmap = await this._createBitmap(image); this._decodedImages.set(image.id, bitmap); } - // Calculate display size let w = bitmap.width; let h = bitmap.height; @@ -346,30 +602,16 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler { const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width; const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height; - if (columns) { - w = columns * cw; - } - if (rows) { - h = rows * ch; - } - // Maintain aspect ratio if only one dimension specified - if (columns && !rows) { - h = Math.round(w * (bitmap.height / bitmap.width)); - } else if (rows && !columns) { - w = Math.round(h * (bitmap.width / bitmap.height)); - } + if (columns) w = columns * cw; + if (rows) h = rows * ch; + + if (columns && !rows) h = Math.round(w * (bitmap.height / bitmap.width)); + else if (rows && !columns) w = Math.round(h * (bitmap.width / bitmap.height)); } - // Check pixel limit - if (w * h > this._opts.pixelLimit) { - console.warn('[KittyHandler] Image exceeds pixel limit'); - return; - } + if (w * h > this._opts.pixelLimit) return; - // Use shared ImageStorage to add the image - // This handles cursor movement and integration with the terminal if (w !== bitmap.width || h !== bitmap.height) { - // Resize if needed const resized = await createImageBitmap(bitmap, { resizeWidth: w, resizeHeight: h }); this._storage.addImage(resized); } else { @@ -378,23 +620,18 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler { } /** - * Decode base64 image data into an ImageBitmap. - * Uses the wasm-based decoder from xterm-wasm-parts for better performance. + * Create ImageBitmap from already-decoded image data. */ - private async _decodeImage(image: IKittyImageData): Promise { - const format = image.format; - const base64Data = image.data; + private async _createBitmap(image: IKittyImageData): Promise { + let bytes = image.data; - // TODO: Get this confirmed! - let bytes = this._decodeBase64(base64Data); if (image.compression === KittyCompression.ZLIB) { - bytes = await this._decompressZlib(bytes) as Uint8Array; + bytes = await this._decompressZlib(bytes); } - if (format === KittyFormat.PNG) { - const blob = new Blob([new Uint8Array(bytes)], { type: 'image/png' }); - // Safari fallback pattern (from IIPHandler) + if (image.format === KittyFormat.PNG) { + const blob = new Blob([bytes as BlobPart], { type: 'image/png' }); if (!window.createImageBitmap) { const url = URL.createObjectURL(blob); const img = new Image(); @@ -412,7 +649,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler { return createImageBitmap(blob); } - // Raw pixel data (RGB or RGBA) + // Raw pixel data const width = image.width; const height = image.height; @@ -420,26 +657,23 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler { throw new Error('Width and height required for raw pixel data'); } - const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB; + const bytesPerPixel = image.format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB; const expectedBytes = width * height * bytesPerPixel; if (bytes.length < expectedBytes) { - throw new Error(`Insufficient pixel data: got ${bytes.length}, expected ${expectedBytes}`); + throw new Error('Insufficient pixel data'); } - // Convert to RGBA ImageData - // TODO: For RGBA, use bytes directly. For RGB, use Uint32Array bit manipulation - // for 5-6x speedup. const pixelCount = width * height; const data = new Uint8ClampedArray(pixelCount * BYTES_PER_PIXEL_RGBA); - const isRgba = format === KittyFormat.RGBA; + const isRgba = image.format === KittyFormat.RGBA; let srcOffset = 0; let dstOffset = 0; for (let i = 0; i < pixelCount; i++) { - data[dstOffset ] = bytes[srcOffset ]; // R - data[dstOffset + 1] = bytes[srcOffset + 1]; // G - data[dstOffset + 2] = bytes[srcOffset + 2]; // B + data[dstOffset] = bytes[srcOffset]; + data[dstOffset + 1] = bytes[srcOffset + 1]; + data[dstOffset + 2] = bytes[srcOffset + 2]; data[dstOffset + 3] = isRgba ? bytes[srcOffset + 3] : ALPHA_OPAQUE; srcOffset += bytesPerPixel; dstOffset += BYTES_PER_PIXEL_RGBA; @@ -448,13 +682,10 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler { return createImageBitmap(new ImageData(data, width, height)); } - /** - * Decompress zlib/deflate compressed data using the browser's DecompressionStream API. - */ private async _decompressZlib(compressed: Uint8Array): Promise { try { return await this._decompress(compressed, 'deflate'); - } catch { + } catch (e) { return await this._decompress(compressed, 'deflate-raw'); } } @@ -462,7 +693,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler { private async _decompress(compressed: Uint8Array, format: 'deflate' | 'deflate-raw'): Promise { const ds = new DecompressionStream(format); const writer = ds.writable.getWriter(); - writer.write(new Uint8Array(compressed) as Uint8Array); + writer.write(compressed as BufferSource); writer.close(); const chunks: Uint8Array[] = []; @@ -481,82 +712,9 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler { result.set(chunk, offset); offset += chunk.length; } - return result; } - /** - * Decode base64 string using the wasm-based decoder for better performance. - * Uses shard-based decoding (1MB chunks) to prevent memory pressure with large images. - */ - private _decodeBase64(base64Data: string): Uint8Array { - const length = base64Data.length; - - // For small data, decode directly without sharding - if (length <= BASE64_SHARD_SIZE) { - return this._decodeBase64Shard(base64Data); - } - - // For large data, decode in 1MB shards and accumulate - // Base64 must be decoded in multiples of 4 chars, so align shard boundaries - const shardSize = BASE64_SHARD_SIZE - (BASE64_SHARD_SIZE % 4); - const chunks: Uint8Array[] = []; - let totalLength = 0; - - for (let offset = 0; offset < length; offset += shardSize) { - const end = Math.min(offset + shardSize, length); - const shard = base64Data.substring(offset, end); - const decoded = this._decodeBase64Shard(shard); - chunks.push(decoded); - totalLength += decoded.length; - } - - // Combine all chunks into final result - const result = new Uint8Array(totalLength); - let writeOffset = 0; - for (const chunk of chunks) { - result.set(chunk, writeOffset); - writeOffset += chunk.length; - } - - return result; - } - - /** - * Decode a single base64 shard using the wasm decoder. - */ - private _decodeBase64Shard(base64Data: string): Uint8Array { - // Calculate exact decoded size from base64 string - // Formula: (encoded_length * 3) / 4 - padding_count - let padding = 0; - if (base64Data.endsWith('==')) padding = 2; - else if (base64Data.endsWith('=')) padding = 1; - const decodedSize = Math.floor(base64Data.length * 3 / 4) - padding; - - // Convert string to Uint32Array of codepoints for the wasm decoder - const codepoints = new Uint32Array(base64Data.length); - for (let i = 0; i < base64Data.length; i++) { - codepoints[i] = base64Data.charCodeAt(i); - } - - const decoder = new Base64Decoder(DECODER_KEEP_DATA); - decoder.init(decodedSize); - decoder.put(codepoints, 0, codepoints.length); - - if (decoder.end()) { - decoder.release(); - throw new Error('Base64 decode failed'); - } - - // Copy the decoded data before releasing - const result = new Uint8Array(decoder.data8); - decoder.release(); - return result; - } - - /** - * Get stored images (for testing/debugging). - */ public get images(): ReadonlyMap { return this._images; } diff --git a/addons/addon-image/src/kitty/KittyGraphicsTypes.ts b/addons/addon-image/src/kitty/KittyGraphicsTypes.ts index 7afb5b3b..17ebefc1 100644 --- a/addons/addon-image/src/kitty/KittyGraphicsTypes.ts +++ b/addons/addon-image/src/kitty/KittyGraphicsTypes.ts @@ -93,13 +93,19 @@ export interface IKittyCommand { /** * Pending chunked transmission state. - * Stores metadata from the first chunk while accumulating payload data. + * Stores metadata from the first chunk while accumulating decoded payload data. */ export interface IPendingTransmission { /** The parsed command from the first chunk (contains action, format, dimensions, etc.) */ cmd: IKittyCommand; - /** Accumulated base64 payload data */ - data: string; + /** Accumulated decoded payload chunks */ + chunks: Uint8Array[]; + /** Total size of accumulated chunks */ + totalSize: number; + /** Leftover base64 bytes (0-3) that weren't aligned in previous chunk */ + leftover: Uint32Array; + /** Number of valid bytes in leftover */ + leftoverLength: number; } /** @@ -107,7 +113,8 @@ export interface IPendingTransmission { */ export interface IKittyImageData { id: number; - data: string; + /** Decoded image bytes (already decoded from base64) */ + data: Uint8Array; width: number; height: number; format: 24 | 32 | 100; diff --git a/addons/addon-image/test/ImageAddon.test.ts b/addons/addon-image/test/ImageAddon.test.ts index a7bb2543..923da858 100644 --- a/addons/addon-image/test/ImageAddon.test.ts +++ b/addons/addon-image/test/ImageAddon.test.ts @@ -77,6 +77,7 @@ const TESTDATA_IIP: [string, [number, number]][] = [ // Kitty graphics test images const KITTY_BLACK_1X1_BASE64 = readFileSync('./addons/addon-image/fixture/kitty/black-1x1.png').toString('base64'); +const KITTY_BLACK_1X1_BYTES = Array.from(readFileSync('./addons/addon-image/fixture/kitty/black-1x1.png')); const KITTY_RGB_3X1_BASE64 = readFileSync('./addons/addon-image/fixture/kitty/rgb-3x1.png').toString('base64'); let ctx: ITestContext; @@ -386,8 +387,8 @@ test.describe('ImageAddon', () => { await ctx.proxy.write(`\x1b_Ga=t,f=100,i=99;${part2}\x1b\\`); await timeout(100); - const storedData = await ctx.page.evaluate(`window.imageAddon._handlers.get('kitty').images.get(99).data`); - strictEqual(storedData, KITTY_BLACK_1X1_BASE64); + const storedData = await ctx.page.evaluate(`Array.from(window.imageAddon._handlers.get('kitty').images.get(99).data)`); + deepStrictEqual(storedData, KITTY_BLACK_1X1_BYTES); }); test('delete command (a=d) removes specific image by id', async () => {