From 304943658db9a5df9b5da82a38da5b628f3a34d4 Mon Sep 17 00:00:00 2001 From: Anthony Kim Date: Thu, 22 Jan 2026 10:44:38 -0800 Subject: [PATCH] Support Kitty graphics protocol --- addons/addon-kitty-graphics/LICENSE | 19 ++ addons/addon-kitty-graphics/README.md | 47 ++++ .../fixture/black-1x1.png | Bin 0 -> 123 bytes .../addon-kitty-graphics/fixture/rgb-3x1.png | Bin 0 -> 131 bytes addons/addon-kitty-graphics/package.json | 27 +++ .../src/KittyGraphicsAddon.test.ts | 156 ++++++++++++++ .../src/KittyGraphicsAddon.ts | 200 ++++++++++++++++++ addons/addon-kitty-graphics/src/tsconfig.json | 27 +++ .../test/KittyGraphicsAddon.test.ts | 41 ++++ .../test/playwright.config.ts | 35 +++ .../addon-kitty-graphics/test/tsconfig.json | 30 +++ addons/addon-kitty-graphics/tsconfig.json | 5 + .../typings/addon-kitty-graphics.d.ts | 82 +++++++ addons/addon-kitty-graphics/webpack.config.js | 44 ++++ bin/esbuild.mjs | 1 + bin/test_integration.js | 1 + demo/client/client.ts | 8 +- demo/client/tsconfig.json | 1 + demo/client/types.ts | 45 ++-- send-png | 33 +++ tsconfig.all.json | 1 + 21 files changed, 781 insertions(+), 22 deletions(-) create mode 100644 addons/addon-kitty-graphics/LICENSE create mode 100644 addons/addon-kitty-graphics/README.md create mode 100644 addons/addon-kitty-graphics/fixture/black-1x1.png create mode 100644 addons/addon-kitty-graphics/fixture/rgb-3x1.png create mode 100644 addons/addon-kitty-graphics/package.json create mode 100644 addons/addon-kitty-graphics/src/KittyGraphicsAddon.test.ts create mode 100644 addons/addon-kitty-graphics/src/KittyGraphicsAddon.ts create mode 100644 addons/addon-kitty-graphics/src/tsconfig.json create mode 100644 addons/addon-kitty-graphics/test/KittyGraphicsAddon.test.ts create mode 100644 addons/addon-kitty-graphics/test/playwright.config.ts create mode 100644 addons/addon-kitty-graphics/test/tsconfig.json create mode 100644 addons/addon-kitty-graphics/tsconfig.json create mode 100644 addons/addon-kitty-graphics/typings/addon-kitty-graphics.d.ts create mode 100644 addons/addon-kitty-graphics/webpack.config.js create mode 100755 send-png diff --git a/addons/addon-kitty-graphics/LICENSE b/addons/addon-kitty-graphics/LICENSE new file mode 100644 index 00000000..a6db9edc --- /dev/null +++ b/addons/addon-kitty-graphics/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2025, The xterm.js authors (https://github.com/xtermjs/xterm.js) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/addons/addon-kitty-graphics/README.md b/addons/addon-kitty-graphics/README.md new file mode 100644 index 00000000..650f84e0 --- /dev/null +++ b/addons/addon-kitty-graphics/README.md @@ -0,0 +1,47 @@ +# @xterm/addon-kitty-graphics + +An addon for [xterm.js](https://github.com/xtermjs/xterm.js) that adds support for the [Kitty graphics protocol](https://sw.kovidgoyal.net/kitty/graphics-protocol/). + +## Install + +```bash +npm install --save @xterm/addon-kitty-graphics @xterm/xterm +``` + +## Usage + +```typescript +import { Terminal } from '@xterm/xterm'; +import { KittyGraphicsAddon } from '@xterm/addon-kitty-graphics'; + +const terminal = new Terminal(); +const kittyGraphicsAddon = new KittyGraphicsAddon(); +terminal.loadAddon(kittyGraphicsAddon); +``` + +## Features + +This addon implements the Kitty graphics protocol, allowing applications to display images directly in the terminal using APC (Application Program Command) escape sequences. + +### Supported Features + +- PNG image transmission (f=100) +- Direct RGB/RGBA pixel data (f=24, f=32) +- Image placement at cursor position +- Basic query support (a=q) + +### Protocol Format + +The Kitty graphics protocol uses APC escape sequences: + +``` +_G=,=,...;\ +``` + +Key parameters: +- `a`: Action (t=transmit, T=transmit+display, q=query) +- `f`: Format (100=PNG, 24=RGB, 32=RGBA) +- `i`: Image ID +- `m`: More data follows (1=yes, 0=no) + +See the [Kitty graphics protocol documentation](https://sw.kovidgoyal.net/kitty/graphics-protocol/) for full details. diff --git a/addons/addon-kitty-graphics/fixture/black-1x1.png b/addons/addon-kitty-graphics/fixture/black-1x1.png new file mode 100644 index 0000000000000000000000000000000000000000..9456317423574592cd13da42412c3b4a28fd663e GIT binary patch literal 123 zcmeAS@N?(olHy`uVBq!ia0y~yU|mpLFo3bH*V!%x z1_mBa7srr_ImrnLDNGCu91JXG4>l}gU|^6fag8Vm&QB{TPb^AhaL6gmODsst%q!6^ d$V=Bv&QD2A{^~3#$H2hA;OXk;vd$@?2>^69A_f2e literal 0 HcmV?d00001 diff --git a/addons/addon-kitty-graphics/package.json b/addons/addon-kitty-graphics/package.json new file mode 100644 index 00000000..5bb560e7 --- /dev/null +++ b/addons/addon-kitty-graphics/package.json @@ -0,0 +1,27 @@ +{ + "name": "@xterm/addon-kitty-graphics", + "version": "0.1.0", + "author": { + "name": "The xterm.js authors", + "url": "https://xtermjs.org/" + }, + "main": "lib/addon-kitty-graphics.js", + "module": "lib/addon-kitty-graphics.mjs", + "types": "typings/addon-kitty-graphics.d.ts", + "repository": "https://github.com/xtermjs/xterm.js/tree/master/addons/addon-kitty-graphics", + "license": "MIT", + "keywords": [ + "terminal", + "xterm", + "xterm.js", + "kitty", + "graphics" + ], + "scripts": { + "build": "../../node_modules/.bin/tsc -p .", + "prepackage": "npm run build", + "package": "../../node_modules/.bin/webpack", + "prepublishOnly": "npm run package", + "start": "node ../../demo/start" + } +} diff --git a/addons/addon-kitty-graphics/src/KittyGraphicsAddon.test.ts b/addons/addon-kitty-graphics/src/KittyGraphicsAddon.test.ts new file mode 100644 index 00000000..9378e954 --- /dev/null +++ b/addons/addon-kitty-graphics/src/KittyGraphicsAddon.test.ts @@ -0,0 +1,156 @@ +/** + * Copyright (c) 2025 The xterm.js authors. All rights reserved. + * @license MIT + */ +import { assert } from 'chai'; +import { Terminal } from 'browser/public/Terminal'; +import { KittyGraphicsAddon, parseKittyCommand } from './KittyGraphicsAddon'; + +/** + * Write data to terminal and wait for completion. + */ +function writeP(terminal: Terminal, data: string | Uint8Array): Promise { + return new Promise(r => terminal.write(data, r)); +} + +// Test image: 1x1 black PNG (captured from `send-png fixture/black-1x1.png`) +// Get the below base64-encoded PNG file by: `python3 send-png addons/addon-kitty-graphics/fixture/black-1x1.png` +const BLACK_1X1_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAMAAAAoyzS7AAAAA1BMVEUAAACnej3aAAAACklEQVR4nGNgAAAAAgABSK+kcQAAAAt0RVh0Q29tbWVudAAA1LTqjgAAAApJREFUeJxjYAAAAGQA2AAAAAt0RVh0Q29tbWVudAAA1LTqjg5JREFUAAAAASUVORK5CYII='; + +// Test image: 3x1 RGB PNG (red, green, blue pixels) +const RGB_3X1_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAMAAAABCAMAAAAsPuSGAAAACVBMVEX/AAAA/wAAAP8tSs2KAAAADElEQVR4nGNgYGQCAAAIAAQ24LCmAAAAHXRFWHRTb2Z0d2FyZQBAbHVuYXBhaW50L3BuZy1jb2RlY/VDGR4AAAAASUVORK5CYII='; + +// Currently tests the flow: write escape sequence -> addon stores image. +// Pixel-level verification of rendered images is done in Playwright tests. +describe('KittyGraphicsAddon', () => { + let terminal: Terminal; + let addon: KittyGraphicsAddon; + + beforeEach(() => { + terminal = new Terminal({ cols: 80, rows: 24, allowProposedApi: true }); + addon = new KittyGraphicsAddon({ debug: false }); + terminal.loadAddon(addon); + }); + + describe('parseKittyCommand', () => { + it('should parse control data with action and format', () => { + const cmd = parseKittyCommand('a=T,f=100'); + assert.equal(cmd.action, 'T'); + assert.equal(cmd.format, 100); + }); + + it('should parse control data with all options', () => { + const cmd = parseKittyCommand('a=t,f=32,i=5,s=10,v=20,c=3,r=2,m=1,q=2'); + assert.equal(cmd.action, 't'); + assert.equal(cmd.format, 32); + assert.equal(cmd.id, 5); + assert.equal(cmd.width, 10); + assert.equal(cmd.height, 20); + assert.equal(cmd.columns, 3); + assert.equal(cmd.rows, 2); + assert.equal(cmd.more, 1); + assert.equal(cmd.quiet, 2); + }); + + it('should handle empty control data', () => { + const cmd = parseKittyCommand(''); + assert.equal(cmd.action, undefined); + assert.equal(cmd.format, undefined); + }); + + it('should parse transmit action', () => { + const cmd = parseKittyCommand('a=t,f=100'); + assert.equal(cmd.action, 't'); + assert.equal(cmd.format, 100); + }); + + it('should parse delete action', () => { + const cmd = parseKittyCommand('a=d,i=5'); + assert.equal(cmd.action, 'd'); + assert.equal(cmd.id, 5); + }); + }); + + describe('APC handler', () => { + it('should store image when transmit+display sequence is written', async () => { + // Write Kitty graphics sequence: ESC _ G ; ESC \ + const sequence = `\x1b_Ga=T,f=100;${BLACK_1X1_BASE64}\x1b\\`; + await writeP(terminal, sequence); + + // Addon should have stored the image + assert.equal(addon.images.size, 1); + + const image = addon.images.get(1)!; + assert.exists(image); + assert.equal(image.format, 100); // PNG format + assert.equal(image.data, BLACK_1X1_BASE64); + }); + + it('should store RGB image with correct payload', async () => { + const sequence = `\x1b_Ga=T,f=100;${RGB_3X1_BASE64}\x1b\\`; + await writeP(terminal, sequence); + + assert.equal(addon.images.size, 1); + const image = addon.images.get(1)!; + assert.equal(image.data, RGB_3X1_BASE64); + }); + + it('should use explicit image id when provided', async () => { + const sequence = `\x1b_Ga=T,f=100,i=42;${BLACK_1X1_BASE64}\x1b\\`; + await writeP(terminal, sequence); + + assert.equal(addon.images.size, 1); + assert.isTrue(addon.images.has(42)); + assert.isFalse(addon.images.has(1)); + }); + + it('should handle transmit-only (a=t) without display', async () => { + const sequence = `\x1b_Ga=t,f=100;${BLACK_1X1_BASE64}\x1b\\`; + await writeP(terminal, sequence); + + // Image should still be stored?? + assert.equal(addon.images.size, 1); + }); + + it('should delete image by id', async () => { + // First store an image with id=5 + await writeP(terminal, `\x1b_Ga=T,f=100,i=5;${BLACK_1X1_BASE64}\x1b\\`); + assert.equal(addon.images.size, 1); + + // Delete it + await writeP(terminal, `\x1b_Ga=d,i=5\x1b\\`); + assert.equal(addon.images.size, 0); + }); + + it('should delete all images when no id specified', async () => { + // Store multiple images + await writeP(terminal, `\x1b_Ga=T,f=100,i=1;${BLACK_1X1_BASE64}\x1b\\`); + await writeP(terminal, `\x1b_Ga=T,f=100,i=2;${RGB_3X1_BASE64}\x1b\\`); + assert.equal(addon.images.size, 2); + + // Delete all + await writeP(terminal, `\x1b_Ga=d\x1b\\`); + assert.equal(addon.images.size, 0); + }); + + it('should handle chunked transmission (m=1 flag)', async () => { + // Split payload into chunks using m=1 (more data coming) + const half = Math.floor(BLACK_1X1_BASE64.length / 2); + const chunk1 = BLACK_1X1_BASE64.substring(0, half); + const chunk2 = BLACK_1X1_BASE64.substring(half); + + // First chunk with m=1 + await writeP(terminal, `\x1b_Ga=t,f=100,i=10,m=1;${chunk1}\x1b\\`); + // Image not complete yet (pending) + assert.equal(addon.images.size, 0); + + // Final chunk without m=1 + await writeP(terminal, `\x1b_Ga=t,f=100,i=10;${chunk2}\x1b\\`); + // Now image should be stored + assert.equal(addon.images.size, 1); + + const image = addon.images.get(10)!; + assert.equal(image.data, BLACK_1X1_BASE64); + }); + }); +}); diff --git a/addons/addon-kitty-graphics/src/KittyGraphicsAddon.ts b/addons/addon-kitty-graphics/src/KittyGraphicsAddon.ts new file mode 100644 index 00000000..0dde0671 --- /dev/null +++ b/addons/addon-kitty-graphics/src/KittyGraphicsAddon.ts @@ -0,0 +1,200 @@ +/** + * Copyright (c) 2025 The xterm.js authors. All rights reserved. + * @license MIT + */ + +import type { Terminal, ITerminalAddon, IDisposable } from '@xterm/xterm'; +import type { KittyGraphicsAddon as IKittyGraphicsApi, IKittyGraphicsOptions, IKittyImage } from '@xterm/addon-kitty-graphics'; + +/** + * Kitty graphics protocol action types. + */ +const enum KittyAction { + TRANSMIT = 't', + TRANSMIT_DISPLAY = 'T', + QUERY = 'q', + PLACEMENT = 'p', + DELETE = 'd' +} + +/** + * Kitty graphics protocol format types. + */ +const enum KittyFormat { + RGB = 24, + RGBA = 32, + PNG = 100 +} + +/** + * Parsed Kitty graphics command. + */ +export interface IKittyCommand { + action?: string; + format?: number; + id?: number; + width?: number; + height?: number; + x?: number; + y?: number; + columns?: number; + rows?: number; + more?: number; + quiet?: number; + payload?: string; +} + +/** + * Parses Kitty graphics control data into a command object. + * Exported for testing. + */ +export function parseKittyCommand(data: string): IKittyCommand { + const cmd: IKittyCommand = {}; + const parts = data.split(','); + + for (const part of parts) { + const eqIdx = part.indexOf('='); + if (eqIdx === -1) continue; + + const key = part.substring(0, eqIdx); + const value = part.substring(eqIdx + 1); + + switch (key) { + // Question: How do we know which radix to use? + case 'a': cmd.action = value; break; + case 'f': cmd.format = parseInt(value); break; + case 'i': cmd.id = parseInt(value); break; + case 's': cmd.width = parseInt(value); break; + case 'v': cmd.height = parseInt(value); break; + case 'x': cmd.x = parseInt(value); break; + case 'y': cmd.y = parseInt(value); break; + case 'c': cmd.columns = parseInt(value); break; + case 'r': cmd.rows = parseInt(value); break; + case 'm': cmd.more = parseInt(value); break; + case 'q': cmd.quiet = parseInt(value); break; + } + } + + return cmd; +} + +export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi { + private _terminal: Terminal | undefined; + private _apcHandler: IDisposable | undefined; + private _images: Map = new Map(); + private _pendingData: Map = new Map(); + private _nextImageId = 1; + private _debug: boolean; + + constructor(options?: IKittyGraphicsOptions) { + this._debug = options?.debug ?? false; + } + + public get images(): ReadonlyMap { + return this._images; + } + + public activate(terminal: Terminal): void { + this._terminal = terminal; + // TODO: Remove console log + console.log('[KittyGraphicsAddon] Activated'); + + // Register APC handler for 'G' (0x47) - Kitty graphics protocol + // APC sequence format: ESC _ G ESC \ + this._apcHandler = terminal.parser.registerApcHandler(0x47, (data: string) => { + return this._handleKittyGraphics(data); + }); + } + + public dispose(): void { + this._apcHandler?.dispose(); + this._images.clear(); + this._pendingData.clear(); + this._terminal = undefined; + } + + private _handleKittyGraphics(data: string): boolean { + const semiIdx = data.indexOf(';'); + const controlData = semiIdx === -1 ? data : data.substring(0, semiIdx); + const payload = semiIdx === -1 ? '' : data.substring(semiIdx + 1); + + const cmd = parseKittyCommand(controlData); + cmd.payload = payload; + + if (this._debug) { + console.log('[KittyGraphicsAddon] Received command:', cmd); + } + + const action = cmd.action || 't'; + + switch (action) { + case KittyAction.TRANSMIT: + return this._handleTransmit(cmd); + case KittyAction.TRANSMIT_DISPLAY: + return this._handleTransmitDisplay(cmd); + case KittyAction.QUERY: + return this._handleQuery(cmd); + case KittyAction.DELETE: + return this._handleDelete(cmd); + default: + return true; + } + } + + private _handleTransmit(cmd: IKittyCommand): boolean { + const id = cmd.id || this._nextImageId++; + const payload = cmd.payload || ''; + + if (cmd.more === 1) { + const existing = this._pendingData.get(id) || ''; + this._pendingData.set(id, existing + payload); + return true; + } + + let fullPayload = payload; + if (this._pendingData.has(id)) { + fullPayload = this._pendingData.get(id)! + payload; + this._pendingData.delete(id); + } + + const image: IKittyImage = { + id, + data: fullPayload, + width: cmd.width || 0, + height: cmd.height || 0, + format: (cmd.format || KittyFormat.PNG) as 24 | 32 | 100 + }; + + this._images.set(id, image); + + if (this._debug) { + console.log(`[KittyGraphicsAddon] Stored image ${id}`); + } + + return true; + } + + private _handleTransmitDisplay(cmd: IKittyCommand): boolean { + this._handleTransmit(cmd); + // TODO: Display image at cursor position (canvas layer) + return true; + } + + private _handleQuery(cmd: IKittyCommand): boolean { + // TODO: Respond with APC sequence indicating graphics support + // Protocol: terminal should reply with ESC _ G i=;OK ESC \ + if (this._debug) { + console.log('[KittyGraphicsAddon] Query received'); + } + return true; + } + + private _handleDelete(cmd: IKittyCommand): boolean { + if (cmd.id !== undefined) { + this._images.delete(cmd.id); + } else { + this._images.clear(); + } + return true; + } +} diff --git a/addons/addon-kitty-graphics/src/tsconfig.json b/addons/addon-kitty-graphics/src/tsconfig.json new file mode 100644 index 00000000..570d4af5 --- /dev/null +++ b/addons/addon-kitty-graphics/src/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es2021", + "lib": ["dom", "es2015"], + "rootDir": ".", + "outDir": "../out", + "sourceMap": true, + "removeComments": true, + "strict": true, + "types": ["../../../node_modules/@types/mocha"], + "paths": { + "browser/*": ["../../../src/browser/*"], + "vs/*": ["../../../src/vs/*"], + "@xterm/addon-kitty-graphics": ["../typings/addon-kitty-graphics.d.ts"] + } + }, + "include": ["./**/*", "../../../typings/xterm.d.ts"], + "references": [ + { + "path": "../../../src/browser" + }, + { + "path": "../../../src/vs" + } + ] +} diff --git a/addons/addon-kitty-graphics/test/KittyGraphicsAddon.test.ts b/addons/addon-kitty-graphics/test/KittyGraphicsAddon.test.ts new file mode 100644 index 00000000..114df461 --- /dev/null +++ b/addons/addon-kitty-graphics/test/KittyGraphicsAddon.test.ts @@ -0,0 +1,41 @@ +/** + * Copyright (c) 2025 The xterm.js authors. All rights reserved. + * @license MIT + * + * Playwright integration tests for Kitty Graphics Addon. + * These test the addon in a real browser with visual verification. + * + * Unit tests for parsing are in src/KittyGraphicsAddon.test.ts + */ + +import test from '@playwright/test'; +import { deepStrictEqual, strictEqual } from 'assert'; +import { ITestContext, createTestContext, openTerminal } from '../../../test/playwright/TestUtils'; + +let ctx: ITestContext; + +test.beforeAll(async ({ browser }) => { + ctx = await createTestContext(browser); +}); + +test.afterAll(async () => { + await ctx.page.close(); +}); + +test.describe('KittyGraphicsAddon', () => { + test.beforeEach(async () => { + await openTerminal(ctx, { cols: 80, rows: 24 }); + await ctx.page.evaluate(` + window.term.reset(); + `); + }); + + test('addon should be loaded and activated', async () => { + // Verify the addon is available on the window + const hasAddon = await ctx.page.evaluate(`typeof window.KittyGraphicsAddon !== 'undefined'`); + strictEqual(hasAddon, true, 'KittyGraphicsAddon should be available'); + }); + + // TODO: WAYYYYYY More tests that would initially fail, but work when addon work is complete. Like implementation of the handler. + +}); diff --git a/addons/addon-kitty-graphics/test/playwright.config.ts b/addons/addon-kitty-graphics/test/playwright.config.ts new file mode 100644 index 00000000..22834be1 --- /dev/null +++ b/addons/addon-kitty-graphics/test/playwright.config.ts @@ -0,0 +1,35 @@ +import { PlaywrightTestConfig } from '@playwright/test'; + +const config: PlaywrightTestConfig = { + testDir: '.', + timeout: 10000, + projects: [ + { + name: 'ChromeStable', + use: { + browserName: 'chromium', + channel: 'chrome' + } + }, + { + name: 'FirefoxStable', + use: { + browserName: 'firefox' + } + }, + { + name: 'WebKit', + use: { + browserName: 'webkit' + } + } + ], + reporter: 'list', + webServer: { + command: 'npm run start', + port: 3000, + timeout: 120000, + reuseExistingServer: !process.env.CI + } +}; +export default config; diff --git a/addons/addon-kitty-graphics/test/tsconfig.json b/addons/addon-kitty-graphics/test/tsconfig.json new file mode 100644 index 00000000..8a742670 --- /dev/null +++ b/addons/addon-kitty-graphics/test/tsconfig.json @@ -0,0 +1,30 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "ESNext", + "lib": ["es2021"], + "rootDir": ".", + "outDir": "../out-test", + "sourceMap": true, + "removeComments": true, + "baseUrl": ".", + "paths": { + "common/*": ["../../../src/common/*"], + "browser/*": ["../../../src/browser/*"] + }, + "strict": true, + "types": ["../../../node_modules/@types/node"] + }, + "include": ["./**/*", "../../../typings/xterm.d.ts"], + "references": [ + { + "path": "../../../src/common" + }, + { + "path": "../../../src/browser" + }, + { + "path": "../../../test/playwright" + } + ] +} diff --git a/addons/addon-kitty-graphics/tsconfig.json b/addons/addon-kitty-graphics/tsconfig.json new file mode 100644 index 00000000..6531dac8 --- /dev/null +++ b/addons/addon-kitty-graphics/tsconfig.json @@ -0,0 +1,5 @@ +{ + "files": [], + "include": [], + "references": [{ "path": "./src" }, { "path": "./test" }] +} diff --git a/addons/addon-kitty-graphics/typings/addon-kitty-graphics.d.ts b/addons/addon-kitty-graphics/typings/addon-kitty-graphics.d.ts new file mode 100644 index 00000000..792f516d --- /dev/null +++ b/addons/addon-kitty-graphics/typings/addon-kitty-graphics.d.ts @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2025 The xterm.js authors. All rights reserved. + * @license MIT + */ + +import { Terminal, ITerminalAddon, IDisposable } from '@xterm/xterm'; + +declare module '@xterm/addon-kitty-graphics' { + /** + * An xterm.js addon that provides support for the Kitty graphics protocol. + * This allows applications to display images in the terminal using APC + * escape sequences. + */ + export class KittyGraphicsAddon implements ITerminalAddon, IDisposable { + /** + * Creates a new Kitty graphics addon. + * @param options Optional configuration for the addon. + */ + constructor(options?: IKittyGraphicsOptions); + + /** + * Activates the addon. + * @param terminal The terminal the addon is being loaded in. + */ + public activate(terminal: Terminal): void; + + /** + * Disposes the addon. + */ + public dispose(): void; + + /** + * Gets the current images stored in the addon. + * Returns a map of image IDs to their image data. + */ + public readonly images: ReadonlyMap; + } + + /** + * Options for the Kitty graphics addon. + */ + export interface IKittyGraphicsOptions { + /** + * Enable debug logging of received graphics commands. + * Default: false + */ + debug?: boolean; + } + + /** + * Represents a stored Kitty graphics image. + */ + export interface IKittyImage { + /** + * The image ID assigned by the terminal or the application. + */ + id: number; + + /** + * The image data as a base64 string or ImageData object. + */ + data: string | ImageData; + + /** + * Width of the image in pixels. + */ + width: number; + + /** + * Height of the image in pixels. + */ + height: number; + + /** + * Format of the image data. + * - 24: RGB (3 bytes per pixel) + * - 32: RGBA (4 bytes per pixel) + * - 100: PNG + */ + format: 24 | 32 | 100; + } +} diff --git a/addons/addon-kitty-graphics/webpack.config.js b/addons/addon-kitty-graphics/webpack.config.js new file mode 100644 index 00000000..89ed51de --- /dev/null +++ b/addons/addon-kitty-graphics/webpack.config.js @@ -0,0 +1,44 @@ +/** + * Copyright (c) 2025 The xterm.js authors. All rights reserved. + * @license MIT + */ + +const path = require('path'); + +const addonName = 'KittyGraphicsAddon'; +const mainFile = 'addon-kitty-graphics.js'; + +const addon = { + entry: `./out/${addonName}.js`, + devtool: 'source-map', + module: { + rules: [ + { + test: /\.js$/, + use: ["source-map-loader"], + enforce: "pre", + exclude: /node_modules/ + } + ] + }, + resolve: { + modules: ['./node_modules'], + extensions: [ '.js' ], + alias: { + common: path.resolve('../../out/common'), + browser: path.resolve('../../out/browser'), + vs: path.resolve('../../out/vs') + } + }, + output: { + filename: mainFile, + path: path.resolve('./lib'), + library: addonName, + libraryTarget: 'umd', + // Force usage of globalThis instead of global / self. (This is cross-env compatible) + globalObject: 'globalThis', + }, + mode: 'production' +}; + +module.exports = [addon]; diff --git a/bin/esbuild.mjs b/bin/esbuild.mjs index 5e0a370c..5c7e5afe 100644 --- a/bin/esbuild.mjs +++ b/bin/esbuild.mjs @@ -137,6 +137,7 @@ if (config.addon) { "@xterm/addon-clipboard": "./addons/addon-clipboard/lib/addon-clipboard.mjs", "@xterm/addon-fit": "./addons/addon-fit/lib/addon-fit.mjs", "@xterm/addon-image": "./addons/addon-image/lib/addon-image.mjs", + "@xterm/addon-kitty-graphics": "./addons/addon-kitty-graphics/lib/addon-kitty-graphics.mjs", "@xterm/addon-progress": "./addons/addon-progress/lib/addon-progress.mjs", "@xterm/addon-search": "./addons/addon-search/lib/addon-search.mjs", "@xterm/addon-serialize": "./addons/addon-serialize/lib/addon-serialize.mjs", diff --git a/bin/test_integration.js b/bin/test_integration.js index 37e6aca7..caceaf9b 100644 --- a/bin/test_integration.js +++ b/bin/test_integration.js @@ -25,6 +25,7 @@ const addons = [ 'clipboard', 'fit', 'image', + 'kitty-graphics', 'progress', 'search', 'serialize', diff --git a/demo/client/client.ts b/demo/client/client.ts index 1ed83669..70ed39b5 100644 --- a/demo/client/client.ts +++ b/demo/client/client.ts @@ -29,6 +29,7 @@ import { TestWindow } from './components/window/testWindow'; import { VtWindow } from './components/window/vtWindow'; import { ClipboardAddon } from '@xterm/addon-clipboard'; import { FitAddon } from '@xterm/addon-fit'; +import { KittyGraphicsAddon } from '@xterm/addon-kitty-graphics'; import { LigaturesAddon } from '@xterm/addon-ligatures'; import { ProgressAddon } from '@xterm/addon-progress'; import { SearchAddon, ISearchOptions } from '@xterm/addon-search'; @@ -47,6 +48,7 @@ export interface IWindowWithTerminal extends Window { ClipboardAddon?: typeof ClipboardAddon; FitAddon?: typeof FitAddon; ImageAddon?: typeof ImageAddon; + KittyGraphicsAddon?: typeof KittyGraphicsAddon; ProgressAddon?: typeof ProgressAddon; SearchAddon?: typeof SearchAddon; SerializeAddon?: typeof SerializeAddon; @@ -74,6 +76,7 @@ const addons: AddonCollection = { clipboard: { name: 'clipboard', ctor: ClipboardAddon, canChange: true }, fit: { name: 'fit', ctor: FitAddon, canChange: false }, image: { name: 'image', ctor: ImageAddon, canChange: true }, + kittyGraphics: { name: 'kittyGraphics', ctor: KittyGraphicsAddon, canChange: true }, progress: { name: 'progress', ctor: ProgressAddon, canChange: true }, search: { name: 'search', ctor: SearchAddon, canChange: true }, serialize: { name: 'serialize', ctor: SerializeAddon, canChange: true }, @@ -196,6 +199,7 @@ if (document.location.pathname === '/test') { window.ClipboardAddon = ClipboardAddon; window.FitAddon = FitAddon; window.ImageAddon = ImageAddon; + window.KittyGraphicsAddon = KittyGraphicsAddon; window.ProgressAddon = ProgressAddon; window.SearchAddon = SearchAddon; window.SerializeAddon = SerializeAddon; @@ -294,6 +298,7 @@ function createTerminal(): Terminal { addons.serialize.instance = new SerializeAddon(); addons.fit.instance = new FitAddon(); addons.image.instance = new ImageAddon(); + addons.kittyGraphics.instance = new KittyGraphicsAddon(); addons.progress.instance = new ProgressAddon(); addons.unicodeGraphemes.instance = new UnicodeGraphemesAddon(); addons.clipboard.instance = new ClipboardAddon(); @@ -306,6 +311,7 @@ function createTerminal(): Terminal { addons.webFonts.instance = new WebFontsAddon(); typedTerm.loadAddon(addons.fit.instance); typedTerm.loadAddon(addons.image.instance); + typedTerm.loadAddon(addons.kittyGraphics.instance); typedTerm.loadAddon(addons.progress.instance); typedTerm.loadAddon(addons.search.instance); typedTerm.loadAddon(addons.serialize.instance); @@ -607,7 +613,7 @@ function updateTerminalSize(): void { function getBox(width: number, height: number): any { return { string: '+', - style: 'font-size: 1px; padding: ' + Math.floor(height/2) + 'px ' + Math.floor(width/2) + 'px; line-height: ' + height + 'px;' + style: 'font-size: 1px; padding: ' + Math.floor(height / 2) + 'px ' + Math.floor(width / 2) + 'px; line-height: ' + height + 'px;' }; } if (source instanceof HTMLCanvasElement) { diff --git a/demo/client/tsconfig.json b/demo/client/tsconfig.json index 17aae876..554272de 100644 --- a/demo/client/tsconfig.json +++ b/demo/client/tsconfig.json @@ -11,6 +11,7 @@ "@xterm/addon-clipboard": ["../../addons/addon-clipboard"], "@xterm/addon-fit": ["../../addons/addon-fit"], "@xterm/addon-image": ["../../addons/addon-image"], + "@xterm/addon-kitty-graphics": ["../../addons/addon-kitty-graphics"], "@xterm/addon-progress": ["../../addons/addon-progress"], "@xterm/addon-search": ["../../addons/addon-search"], "@xterm/addon-serialize": ["../../addons/addon-serialize"], diff --git a/demo/client/types.ts b/demo/client/types.ts index 4790fd4f..8cafdd50 100644 --- a/demo/client/types.ts +++ b/demo/client/types.ts @@ -9,6 +9,7 @@ import type { ImageAddon } from '@xterm/addon-image'; import type { AttachAddon } from '@xterm/addon-attach'; import type { ClipboardAddon } from '@xterm/addon-clipboard'; import type { FitAddon } from '@xterm/addon-fit'; +import type { KittyGraphicsAddon } from '@xterm/addon-kitty-graphics'; import type { LigaturesAddon } from '@xterm/addon-ligatures'; import type { ProgressAddon } from '@xterm/addon-progress'; import type { SearchAddon } from '@xterm/addon-search'; @@ -19,7 +20,7 @@ import type { WebFontsAddon } from '@xterm/addon-web-fonts'; import type { WebLinksAddon } from '@xterm/addon-web-links'; import type { WebglAddon } from '@xterm/addon-webgl'; -export type AddonType = 'attach' | 'clipboard' | 'fit' | 'image' | 'progress' | 'search' | 'serialize' | 'unicode11' | 'unicodeGraphemes' | 'webFonts' | 'webLinks' | 'webgl' | 'ligatures'; +export type AddonType = 'attach' | 'clipboard' | 'fit' | 'image' | 'kittyGraphics' | 'progress' | 'search' | 'serialize' | 'unicode11' | 'unicodeGraphemes' | 'webFonts' | 'webLinks' | 'webgl' | 'ligatures'; export interface IDemoAddon { name: T; @@ -29,32 +30,34 @@ export interface IDemoAddon { T extends 'clipboard' ? typeof ClipboardAddon : T extends 'fit' ? typeof FitAddon : T extends 'image' ? typeof ImageAddon : - T extends 'ligatures' ? typeof LigaturesAddon : - T extends 'progress' ? typeof ProgressAddon : - T extends 'search' ? typeof SearchAddon : - T extends 'serialize' ? typeof SerializeAddon : - T extends 'webFonts' ? typeof WebFontsAddon : - T extends 'webLinks' ? typeof WebLinksAddon : - T extends 'unicode11' ? typeof Unicode11Addon : - T extends 'unicodeGraphemes' ? typeof UnicodeGraphemesAddon : - T extends 'webgl' ? typeof WebglAddon : - never + T extends 'kittyGraphics' ? typeof KittyGraphicsAddon : + T extends 'ligatures' ? typeof LigaturesAddon : + T extends 'progress' ? typeof ProgressAddon : + T extends 'search' ? typeof SearchAddon : + T extends 'serialize' ? typeof SerializeAddon : + T extends 'webFonts' ? typeof WebFontsAddon : + T extends 'webLinks' ? typeof WebLinksAddon : + T extends 'unicode11' ? typeof Unicode11Addon : + T extends 'unicodeGraphemes' ? typeof UnicodeGraphemesAddon : + T extends 'webgl' ? typeof WebglAddon : + never ); instance?: ( T extends 'attach' ? AttachAddon : T extends 'clipboard' ? ClipboardAddon : T extends 'fit' ? FitAddon : T extends 'image' ? ImageAddon : - T extends 'ligatures' ? LigaturesAddon : - T extends 'progress' ? ProgressAddon : - T extends 'search' ? SearchAddon : - T extends 'serialize' ? SerializeAddon : - T extends 'webFonts' ? WebFontsAddon : - T extends 'webLinks' ? WebLinksAddon : - T extends 'unicode11' ? Unicode11Addon : - T extends 'unicodeGraphemes' ? UnicodeGraphemesAddon : - T extends 'webgl' ? WebglAddon : - never + T extends 'kittyGraphics' ? KittyGraphicsAddon : + T extends 'ligatures' ? LigaturesAddon : + T extends 'progress' ? ProgressAddon : + T extends 'search' ? SearchAddon : + T extends 'serialize' ? SerializeAddon : + T extends 'webFonts' ? WebFontsAddon : + T extends 'webLinks' ? WebLinksAddon : + T extends 'unicode11' ? Unicode11Addon : + T extends 'unicodeGraphemes' ? UnicodeGraphemesAddon : + T extends 'webgl' ? WebglAddon : + never ); } diff --git a/send-png b/send-png new file mode 100755 index 00000000..9afe64c5 --- /dev/null +++ b/send-png @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# TODO: Consider removing when Kitty Graphics Addon is shipped. +""" +Minimal Kitty Graphics Protocol image sender. +Based on: https://sw.kovidgoyal.net/kitty/graphics-protocol/#a-minimal-example + +Usage: + ./send-png black-1x1.png + ./send-png rgb-3x1.png +""" +import sys +from base64 import standard_b64encode + +def serialize_image(path): + with open(path, 'rb') as f: + data = f.read() + return standard_b64encode(data).decode('ascii') + +def write_chunked(data): + # a=T means transmit and display + # f=100 means PNG format + # For small images, single chunk (no chunking needed) + sys.stdout.write(f'\x1b_Ga=T,f=100;{data}\x1b\\') + sys.stdout.flush() + +if __name__ == '__main__': + if len(sys.argv) < 2: + print("Usage: ./send-png ", file=sys.stderr) + sys.exit(1) + + img_data = serialize_image(sys.argv[1]) + write_chunked(img_data) + print() # newline after the sequence diff --git a/tsconfig.all.json b/tsconfig.all.json index b0ed0279..2f682739 100644 --- a/tsconfig.all.json +++ b/tsconfig.all.json @@ -11,6 +11,7 @@ { "path": "./addons/addon-clipboard" }, { "path": "./addons/addon-fit" }, { "path": "./addons/addon-image" }, + { "path": "./addons/addon-kitty-graphics" }, { "path": "./addons/addon-ligatures" }, { "path": "./addons/addon-progress" }, { "path": "./addons/addon-search" },