diff --git a/src/renderer/BaseRenderLayer.ts b/src/renderer/BaseRenderLayer.ts
index 8f655de5..4cc07472 100644
--- a/src/renderer/BaseRenderLayer.ts
+++ b/src/renderer/BaseRenderLayer.ts
@@ -57,6 +57,9 @@ export abstract class BaseRenderLayer implements IRenderLayer {
    * @param colorSet The color set to use for the char atlas.
    */
   private _refreshCharAtlas(terminal: ITerminal, colorSet: IColorSet): void {
+    if (this._scaledCharWidth > 0 && this._scaledCharHeight > 0) {
+      return;
+    }
     this._charAtlas = null;
     const result = acquireCharAtlas(terminal, this._colors, this._scaledCharWidth, this._scaledCharHeight);
     if (result instanceof HTMLCanvasElement) {