Merge pull request #4631 from jerch/missing_image_changes

image addon fixes

.eslintrc.json

@@ -36,6 +36,7 @@
     "sourceType": "module"
   },
   "ignorePatterns": [
+    "**/inwasm-sdks/*",
     "**/typings/*.d.ts",
     "**/node_modules",
     "**/*.js"

addons/xterm-addon-image/README.md

@@ -1,45 +1,17 @@
 ## xterm-addon-image
 
-Image output in xterm.js.
+Inline image output in xterm.js. Supports SIXEL and iTerm's inline image protocol (IIP).
 
 
 ![](fixture/example.png)
 
 
-### Important note
-
-Version 0.4.x will be the last version from this single repo.
-Future versions will reside as addon in the xterm.js main repo.
-
-
 ### Install from npm
 
 ```bash
 npm install --save xterm-addon-image
 ```
 
-### Release Compatibility
-
-- 0.4.2 - compatible to xterm.js 5.2.0
-- 0.4.1 - compatible to xterm.js 5.2.0
-- 0.4.0 - compatible to xterm.js 5.1.0
-- 0.3.1 - compatible to xterm.js 5.1.0
-- 0.3.0 - compatible to xterm.js 5.0.0
-- 0.2.0 - compatible to xterm.js 5.0.0
-- 0.1.x - compatible to xterm.js 4.16.0 - 4.19.0
-
-
-### Clone & Build
-
-The addon integrates tightly with the xterm.js base repo, esp. for tests and the demo.
-To properly set up all needed resources see `bootstrap.sh` or run it directly with
-
-```bash
-curl -s https://raw.githubusercontent.com/jerch/xterm-addon-image/master/bootstrap.sh | XTERMJS=5.2.0 IMAGEADDON=master bash
-```
-
-The addon sources and npm package definition reside under `addons/xterm-addon-image`.
-
 
 ### Usage
 
@@ -243,6 +215,8 @@ _How can I adjust the memory usage?_
 
 ### Changelog
 
+- 0.5.0 integrate with xtermjs base repo (at v0.4.3)
+- 0.4.3 defer canvas creation
 - 0.4.2 fix image canvas resize
 - 0.4.1 compat release for xterm.js 5.2.0
 - 0.4.0 IIP support

addons/xterm-addon-image/src/IIPHandler.ts

@@ -108,14 +108,13 @@ export class IIPHandler implements IOscHandler, IResetHandler {
     const blob = new Blob([this._dec.data8], { type: this._metrics.mime });
     this._dec.release();
 
-    const win = this._coreTerminal._core._coreBrowserService.window;
-    if (!win.createImageBitmap) {
+    if (!window.createImageBitmap) {
       const url = URL.createObjectURL(blob);
       const img = new Image();
       return new Promise<boolean>(r => {
         img.addEventListener('load', () => {
           URL.revokeObjectURL(url);
-          const canvas = ImageRenderer.createCanvas(win, w, h);
+          const canvas = ImageRenderer.createCanvas(window.document, w, h);
           canvas.getContext('2d')?.drawImage(img, 0, 0, w, h);
           this._storage.addImage(canvas);
           r(true);
@@ -126,9 +125,9 @@ export class IIPHandler implements IOscHandler, IResetHandler {
         setTimeout(() => r(true), 1000);
       });
     }
-    return win.createImageBitmap(blob, { resizeWidth: w, resizeHeight: h })
+    return createImageBitmap(blob, { resizeWidth: w, resizeHeight: h })
       .then(bm => {
-        this._storage.addImage(bm as unknown as HTMLCanvasElement);
+        this._storage.addImage(bm);
         return true;
       });
   }

addons/xterm-addon-image/src/ImageAddon.ts

@@ -80,7 +80,7 @@ export class ImageAddon implements ITerminalAddon {
     this._terminal = terminal;
 
     // internal data structures
-    this._renderer = new ImageRenderer(terminal, this._opts.showPlaceholder);
+    this._renderer = new ImageRenderer(terminal);
     this._storage = new ImageStorage(terminal, this._renderer, this._opts);
 
     // enable size reports

addons/xterm-addon-image/src/ImageRenderer.ts

@@ -28,8 +28,17 @@ export class ImageRenderer implements IDisposable {
   private _oldSetRenderer: ((renderer: any) => void) | undefined;
 
   // drawing primitive - canvas
-  public static createCanvas(base: Window, width: number, height: number): HTMLCanvasElement {
-    const canvas = base.document.createElement('canvas');
+  public static createCanvas(localDocument: Document | undefined, width: number, height: number): HTMLCanvasElement {
+    /**
+     * NOTE: We normally dont care, from which document the canvas
+     * gets created, so we can fall back to global document,
+     * if the terminal has no document associated yet.
+     * This way early image loads before calling .open keep working
+     * (still discouraged though, as the metrics will be screwed up).
+     * Only the DOM output canvas should be on the terminal's document,
+     * which gets explicitly checked in `insertLayerToDom`.
+     */
+    const canvas = (localDocument || document).createElement('canvas');
     canvas.width = width | 0;
     canvas.height = height | 0;
     return canvas;
@@ -58,7 +67,7 @@ export class ImageRenderer implements IDisposable {
   }
 
 
-  constructor(private _terminal: ITerminalExt, private _showPlaceholder: boolean) {
+  constructor(private _terminal: ITerminalExt) {
     this._oldOpen = this._terminal._core.open;
     this._terminal._core.open = (parent: HTMLElement): void => {
       this._oldOpen?.call(this._terminal._core, parent);
@@ -79,7 +88,7 @@ export class ImageRenderer implements IDisposable {
 
   public dispose(): void {
     this._optionsRefresh?.dispose();
-    this._removeLayerFromDom();
+    this.removeLayerFromDom();
     if (this._terminal._core && this._oldOpen) {
       this._terminal._core.open = this._oldOpen;
       this._oldOpen = undefined;
@@ -204,7 +213,7 @@ export class ImageRenderer implements IDisposable {
     const finalWidth = width + sx > img.width ? img.width - sx : width;
     const finalHeight = sy + height > img.height ? img.height - sy : height;
 
-    const canvas = ImageRenderer.createCanvas(this._terminal._core._coreBrowserService.window, finalWidth, finalHeight);
+    const canvas = ImageRenderer.createCanvas(this.document, finalWidth, finalHeight);
     const ctx = canvas.getContext('2d');
     if (ctx) {
       ctx.drawImage(
@@ -220,7 +229,7 @@ export class ImageRenderer implements IDisposable {
    * Draw a line with placeholder on the image layer canvas.
    */
   public drawPlaceholder(col: number, row: number, count: number = 1): void {
-    if ((this._placeholderBitmap || this._placeholder) && this._ctx) {
+    if (this._ctx) {
       const { width, height } = this.cellSize;
 
       // Don't try to draw anything, if we cannot get valid renderer metrics.
@@ -228,9 +237,12 @@ export class ImageRenderer implements IDisposable {
         return;
       }
 
-      if (height >= this._placeholder!.height) {
+      if (!this._placeholder) {
+        this._createPlaceHolder(Math.max(height + 1, PLACEHOLDER_HEIGHT));
+      } else if (height >= this._placeholder!.height) {
         this._createPlaceHolder(height + 1);
       }
+      if (!this._placeholder) return;
       this._ctx.drawImage(
         this._placeholderBitmap || this._placeholder!,
         col * width,
@@ -274,7 +286,7 @@ export class ImageRenderer implements IDisposable {
       return;
     }
     const canvas = ImageRenderer.createCanvas(
-      this._terminal._core._coreBrowserService.window,
+      this.document,
       Math.ceil(spec.orig!.width * currentWidth / originalWidth),
       Math.ceil(spec.orig!.height * currentHeight / originalHeight)
     );
@@ -294,25 +306,35 @@ export class ImageRenderer implements IDisposable {
     this._renderService = this._terminal._core._renderService;
     this._oldSetRenderer = this._renderService.setRenderer.bind(this._renderService);
     this._renderService.setRenderer = (renderer: any) => {
-      this._removeLayerFromDom();
+      this.removeLayerFromDom();
       this._oldSetRenderer?.call(this._renderService, renderer);
-      this._insertLayerToDom();
     };
-    this._insertLayerToDom();
-    if (this._showPlaceholder) {
-      this._createPlaceHolder();
+  }
+
+  public insertLayerToDom(): void {
+    // make sure that the terminal is attached to a document and to DOM
+    if (this.document && this._terminal._core.screenElement) {
+      if (!this.canvas) {
+        this.canvas = ImageRenderer.createCanvas(
+          this.document, this.dimensions?.css.canvas.width || 0,
+          this.dimensions?.css.canvas.height || 0
+        );
+        this.canvas.classList.add('xterm-image-layer');
+        this._terminal._core.screenElement.appendChild(this.canvas);
+        this._ctx = this.canvas.getContext('2d', { alpha: true, desynchronized: true });
+        this.clearAll();
+      }
+    } else {
+      console.warn('image addon: cannot insert output canvas to DOM, missing document or screenElement');
     }
   }
 
-  private _insertLayerToDom(): void {
-    this.canvas = ImageRenderer.createCanvas(this._terminal._core._coreBrowserService.window, this.dimensions?.css.canvas.width || 0, this.dimensions?.css.canvas.height || 0);
-    this.canvas.classList.add('xterm-image-layer');
-    this._terminal._core.screenElement?.appendChild(this.canvas);
-    this._ctx = this.canvas.getContext('2d', { alpha: true, desynchronized: true });
-  }
-
-  private _removeLayerFromDom(): void {
-    this.canvas?.parentNode?.removeChild(this.canvas);
+  public removeLayerFromDom(): void {
+    if (this.canvas) {
+      this._ctx = undefined;
+      this.canvas.remove();
+      this.canvas = undefined;
+    }
   }
 
   private _createPlaceHolder(height: number = PLACEHOLDER_HEIGHT): void {
@@ -321,7 +343,7 @@ export class ImageRenderer implements IDisposable {
 
     // create blueprint to fill placeholder with
     const bWidth = 32;  // must be 2^n
-    const blueprint = ImageRenderer.createCanvas(this._terminal._core._coreBrowserService.window, bWidth, height);
+    const blueprint = ImageRenderer.createCanvas(this.document, bWidth, height);
     const ctx = blueprint.getContext('2d', { alpha: false });
     if (!ctx) return;
     const imgData = ImageRenderer.createImageData(ctx, bWidth, height);
@@ -340,7 +362,7 @@ export class ImageRenderer implements IDisposable {
 
     // create placeholder line, width aligned to blueprint width
     const width = (screen.width + bWidth - 1) & ~(bWidth - 1) || PLACEHOLDER_LENGTH;
-    this._placeholder = ImageRenderer.createCanvas(this._terminal._core._coreBrowserService.window, width, height);
+    this._placeholder = ImageRenderer.createCanvas(this.document, width, height);
     const ctx2 = this._placeholder.getContext('2d', { alpha: false });
     if (!ctx2) {
       this._placeholder = undefined;
@@ -351,4 +373,8 @@ export class ImageRenderer implements IDisposable {
     }
     ImageRenderer.createImageBitmap(this._placeholder).then(bitmap => this._placeholderBitmap = bitmap);
   }
+
+  public get document(): Document | undefined {
+    return this._terminal._core._coreBrowserService?.window.document;
+  }
 }

addons/xterm-addon-image/src/ImageStorage.ts

@@ -149,8 +149,8 @@ export class ImageStorage implements IDisposable {
   }
 
   public setLimit(value: number): void {
-    if (value < 1 || value > 1000) {
-      throw RangeError('invalid storageLimit, should be at least 1 MB and not exceed 1G');
+    if (value < 0.5 || value > 1000) {
+      throw RangeError('invalid storageLimit, should be at least 0.5 MB and not exceed 1G');
     }
     this._pixelLimit = (value / 4 * 1000000) >>> 0;
     this._evictOldest(0);
@@ -177,8 +177,7 @@ export class ImageStorage implements IDisposable {
     const spec = this._images.get(id);
     this._images.delete(id);
     // FIXME: really ugly workaround to get bitmaps deallocated :(
-    const win = this._terminal._core._coreBrowserService.window;
-    if (spec && win.ImageBitmap && spec.orig instanceof ImageBitmap) {
+    if (spec && window.ImageBitmap && spec.orig instanceof ImageBitmap) {
       spec.orig.close();
     }
   }
@@ -224,7 +223,7 @@ export class ImageStorage implements IDisposable {
   /**
    * Method to add an image to the storage.
    */
-  public addImage(img: HTMLCanvasElement): void {
+  public addImage(img: HTMLCanvasElement | ImageBitmap): void {
     // never allow storage to exceed memory limit
     this._evictOldest(img.width * img.height);
 
@@ -327,9 +326,13 @@ export class ImageStorage implements IDisposable {
    */
   // TODO: Should we move this to the ImageRenderer?
   public render(range: { start: number, end: number }): void {
-    // exit early if we dont have a canvas
-    if (!this._renderer.canvas) {
-      return;
+    // setup image canvas in case we have none yet, but have images in store
+    if (!this._renderer.canvas && this._images.size) {
+      this._renderer.insertLayerToDom();
+      // safety measure - in case we cannot spawn a canvas at all, just exit
+      if (!this._renderer.canvas) {
+        return;
+      }
     }
     // rescale if needed
     this._renderer.rescaleCanvas();
@@ -340,6 +343,9 @@ export class ImageStorage implements IDisposable {
         this._fullyCleared = true;
         this._needsFullClear = false;
       }
+      if (this._renderer.canvas) {
+        this._renderer.removeLayerFromDom();
+      }
       return;
     }
 
@@ -472,9 +478,8 @@ export class ImageStorage implements IDisposable {
       const e: IExtendedAttrsImage = line._extendedAttrs[x] || EMPTY_ATTRS;
       if (e.imageId && e.imageId !== -1) {
         const orig = this._images.get(e.imageId)?.orig;
-        const win = this._terminal._core._coreBrowserService.window;
-        if (win.ImageBitmap && orig instanceof ImageBitmap) {
-          const canvas = ImageRenderer.createCanvas(win, orig.width, orig.height);
+        if (window.ImageBitmap && orig instanceof ImageBitmap) {
+          const canvas = ImageRenderer.createCanvas(window.document, orig.width, orig.height);
           canvas.getContext('2d')?.drawImage(orig, 0, 0, orig.width, orig.height);
           return canvas;
         }

addons/xterm-addon-image/src/SixelHandler.ts

@@ -57,7 +57,7 @@ export class SixelHandler implements IDcsHandler, IResetHandler {
     if (this._dec) {
       const fillColor = params.params[1] === 1 ? 0 : extractActiveBg(
         this._coreTerminal._core._inputHandler._curAttrData,
-        this._coreTerminal._core._themeService.colors);
+        this._coreTerminal._core._themeService?.colors);
       this._dec.init(fillColor, null, this._opts.sixelPaletteLimit);
     }
   }
@@ -98,7 +98,7 @@ export class SixelHandler implements IDcsHandler, IResetHandler {
       return true;
     }
 
-    const canvas = ImageRenderer.createCanvas(this._coreTerminal._core._coreBrowserService.window, width, height);
+    const canvas = ImageRenderer.createCanvas(undefined, width, height);
     canvas.getContext('2d')?.putImageData(new ImageData(this._dec.data8, width, height), 0, 0);
     if (this._dec.memoryUsage > MEM_PERMA_LIMIT) {
       this._dec.release();
@@ -115,8 +115,13 @@ export class SixelHandler implements IDcsHandler, IResetHandler {
 
 // get currently active background color from terminal
 // also respect INVERSE setting
-function extractActiveBg(attr: AttributeData, colors: ReadonlyColorSet): RGBA8888 {
+function extractActiveBg(attr: AttributeData, colors: ReadonlyColorSet | undefined): RGBA8888 {
   let bg = 0;
+  if (!colors) {
+    // FIXME: theme service is prolly not available yet,
+    // happens if .open() was not called yet (bug in core?)
+    return bg;
+  }
   if (attr.isInverse()) {
     if (attr.isFgDefault()) {
       bg = convertLe(colors.foreground.rgba);

addons/xterm-addon-image/src/Types.d.ts

@@ -77,10 +77,10 @@ interface IInputHandlerExt extends IInputHandler {
 }
 
 export interface ICoreTerminalExt extends ITerminal {
-  _themeService: IThemeService;
+  _themeService: IThemeService | undefined;
   _inputHandler: IInputHandlerExt;
   _renderService: IRenderService;
-  _coreBrowserService: ICoreBrowserService;
+  _coreBrowserService: ICoreBrowserService | undefined;
 }
 
 export interface ITerminalExt extends Terminal {

addons/xterm-addon-image/test/ImageAddon.api.ts

@@ -222,11 +222,11 @@ describe('ImageAddon', () => {
       assert.equal(usage as number < 1, true);
     });
     it('set storageLimit removes images synchronously', async () => {
-      await writeToTerminal(SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0);
+      await writeToTerminal(SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0);
       const usage: number = await page.evaluate('imageAddon.storageUsage');
-      const newUsage: number = await page.evaluate('imageAddon.storageLimit = 1; imageAddon.storageUsage');
+      const newUsage: number = await page.evaluate('imageAddon.storageLimit = 0.5; imageAddon.storageUsage');
       assert.equal(newUsage < usage, true);
-      assert.equal(newUsage < 1, true);
+      assert.equal(newUsage < 0.5, true);
     });
     it('clear alternate images on buffer change', async () => {
       assert.equal(await page.evaluate('imageAddon.storageUsage'), 0);
@@ -236,6 +236,7 @@ describe('ImageAddon', () => {
       assert.equal(await page.evaluate('imageAddon.storageUsage'), 0);
     });
     it('evict tiles by in-place overwrites (only full overwrite tested)', async () => {
+      await new Promise(r => setTimeout(r, 50));
       await writeToTerminal('\x1b[H' + SIXEL_SEQ_0 + '\x1b[100;100H');
       let usage = await page.evaluate('imageAddon.storageUsage');
       while (usage === 0) {
@@ -243,8 +244,6 @@ describe('ImageAddon', () => {
         usage = await page.evaluate('imageAddon.storageUsage');
       }
       await writeToTerminal('\x1b[H' + SIXEL_SEQ_0 + '\x1b[100;100H');
-      await writeToTerminal('\x1b[H' + SIXEL_SEQ_0 + '\x1b[100;100H');
-      await writeToTerminal('\x1b[H' + SIXEL_SEQ_0 + '\x1b[100;100H');
       await new Promise(r => setTimeout(r, 200));  // wait some time and re-check
       assert.equal(await page.evaluate('imageAddon.storageUsage'), usage);
     });

bin/publish.js

@@ -30,6 +30,7 @@ const addonPackageDirs = [
   path.resolve(__dirname, '../addons/xterm-addon-attach'),
   path.resolve(__dirname, '../addons/xterm-addon-canvas'),
   path.resolve(__dirname, '../addons/xterm-addon-fit'),
+  path.resolve(__dirname, '../addons/xterm-addon-image'),
   path.resolve(__dirname, '../addons/xterm-addon-ligatures'),
   path.resolve(__dirname, '../addons/xterm-addon-search'),
   path.resolve(__dirname, '../addons/xterm-addon-serialize'),