diff --git a/addons/xterm-addon-webgl/src/atlas/WebglCharAtlas.ts b/addons/xterm-addon-webgl/src/atlas/WebglCharAtlas.ts
index dda4c2fc..c17c54a5 100644
--- a/addons/xterm-addon-webgl/src/atlas/WebglCharAtlas.ts
+++ b/addons/xterm-addon-webgl/src/atlas/WebglCharAtlas.ts
@@ -15,6 +15,7 @@ import { color, rgba } from 'common/Color';
 import { tryDrawCustomChar } from 'browser/renderer/CustomGlyphs';
 import { excludeFromContrastRatioDemands, isPowerlineGlyph, isRestrictedPowerlineGlyph } from 'browser/renderer/RendererUtils';
 import { IUnicodeService } from 'common/services/Services';
+import { FourKeyMap } from 'common/MultiKeyMap';
 
 // For debugging purposes, it can be useful to set this to a really tiny value,
 // to verify that LRU eviction works.
@@ -53,23 +54,15 @@ interface ICharAtlasActiveRow {
 }
 
 /** Work variables to avoid garbage collection. */
-const w: {
-  glyphSet: IRasterizedGlyphSet | undefined;
-  glyphSetBg: { [fg: number]: { [ext: number]: IRasterizedGlyph } } | undefined;
-  glyphSetFg: { [ext: number]: IRasterizedGlyph } | undefined;
-  glyph: IRasterizedGlyph | undefined;
-} = {
-  glyphSet: undefined,
-  glyphSetBg: undefined,
-  glyphSetFg: undefined,
+const w: { glyph: IRasterizedGlyph | undefined } = {
   glyph: undefined
 };
 
 export class WebglCharAtlas implements IDisposable {
   private _didWarmUp: boolean = false;
 
-  private _cacheMap: { [code: number]: IRasterizedGlyphSet } = {};
-  private _cacheMapCombined: { [chars: string]: IRasterizedGlyphSet } = {};
+  private _cacheMap: FourKeyMap<number, number, number, number, IRasterizedGlyph> = new FourKeyMap();
+  private _cacheMapCombined: FourKeyMap<string, number, number, number, IRasterizedGlyph> = new FourKeyMap();
 
   // The texture that the atlas is drawn to
   public cacheCanvas: HTMLCanvasElement;
@@ -137,13 +130,7 @@ export class WebglCharAtlas implements IDisposable {
     // Pre-fill with ASCII 33-126
     for (let i = 33; i < 126; i++) {
       const rasterizedGlyph = this._drawToCache(i, DEFAULT_COLOR, DEFAULT_COLOR, DEFAULT_EXT);
-      this._cacheMap[i] = {
-        [DEFAULT_COLOR]: {
-          [DEFAULT_COLOR]: {
-            [DEFAULT_EXT]: rasterizedGlyph
-          }
-        }
-      };
+      this._cacheMap.set(i, DEFAULT_COLOR, DEFAULT_COLOR, DEFAULT_EXT, rasterizedGlyph);
     }
   }
 
@@ -161,8 +148,8 @@ export class WebglCharAtlas implements IDisposable {
       return;
     }
     this._cacheCtx.clearRect(0, 0, TEXTURE_WIDTH, TEXTURE_HEIGHT);
-    this._cacheMap = {};
-    this._cacheMapCombined = {};
+    this._cacheMap.clear();
+    this._cacheMapCombined.clear();
     this._currentRow.x = 0;
     this._currentRow.y = 0;
     this._currentRow.height = 0;
@@ -182,38 +169,17 @@ export class WebglCharAtlas implements IDisposable {
    * Gets the glyphs texture coords, drawing the texture if it's not already
    */
   private _getFromCacheMap(
-    cacheMap: { [key: string | number]: IRasterizedGlyphSet },
+    cacheMap: FourKeyMap<string | number, number, number, number, IRasterizedGlyph>,
     key: string | number,
     bg: number,
     fg: number,
     ext: number
   ): IRasterizedGlyph {
-    w.glyphSet = cacheMap[key];
-    if (!w.glyphSet) {
-      w.glyphSet = {};
-      cacheMap[key] = w.glyphSet;
-    }
-
-    w.glyphSetBg = w.glyphSet[bg];
-    if (!w.glyphSetBg) {
-      w.glyphSetBg = {};
-      w.glyphSet[bg] = w.glyphSetBg;
-    }
-
-    w.glyph = undefined;
-    w.glyphSetFg = w.glyphSetBg[fg];
-    if (!w.glyphSetFg) {
-      w.glyphSetFg = {};
-      w.glyphSetBg[fg] = w.glyphSetFg;
-    } else {
-      w.glyph = w.glyphSetFg[ext];
-    }
-
+    w.glyph = cacheMap.get(key, bg, fg, ext);
     if (!w.glyph) {
       w.glyph = this._drawToCache(key, bg, fg, ext);
-      w.glyphSetFg[ext] = w.glyph;
+      cacheMap.set(key, bg, fg, ext, w.glyph);
     }
-
     return w.glyph;
   }
 
diff --git a/src/browser/ColorContrastCache.ts b/src/browser/ColorContrastCache.ts
index 73b7a0b7..0c60e8db 100644
--- a/src/browser/ColorContrastCache.ts
+++ b/src/browser/ColorContrastCache.ts
@@ -5,35 +5,30 @@
 
 import { IColorContrastCache } from 'browser/Types';
 import { IColor } from 'common/Types';
+import { TwoKeyMap } from 'common/MultiKeyMap';
 
 export class ColorContrastCache implements IColorContrastCache {
-  private _color: { [bg: number]: { [fg: number]: IColor | null | undefined } | undefined } = {};
-  private _rgba: { [bg: number]: { [fg: number]: string | null | undefined } | undefined } = {};
-
-  public clear(): void {
-    this._color = {};
-    this._rgba = {};
-  }
+  private _color: TwoKeyMap</* bg */number, /* fg */number, IColor | null> = new TwoKeyMap();
+  private _css: TwoKeyMap</* bg */number, /* fg */number, string | null> = new TwoKeyMap();
 
   public setCss(bg: number, fg: number, value: string | null): void {
-    if (!this._rgba[bg]) {
-      this._rgba[bg] = {};
-    }
-    this._rgba[bg]![fg] = value;
+    this._css.set(bg, fg, value);
   }
 
   public getCss(bg: number, fg: number): string | null | undefined {
-    return this._rgba[bg] ? this._rgba[bg]![fg] : undefined;
+    return this._css.get(bg, fg);
   }
 
   public setColor(bg: number, fg: number, value: IColor | null): void {
-    if (!this._color[bg]) {
-      this._color[bg] = {};
-    }
-    this._color[bg]![fg] = value;
+    this._color.set(bg, fg, value);
   }
 
   public getColor(bg: number, fg: number): IColor | null | undefined {
-    return this._color[bg] ? this._color[bg]![fg] : undefined;
+    return this._color.get(bg, fg);
+  }
+
+  public clear(): void {
+    this._color.clear();
+    this._css.clear();
   }
 }
diff --git a/src/common/MultiKeyMap.ts b/src/common/MultiKeyMap.ts
new file mode 100644
index 00000000..6287a8f2
--- /dev/null
+++ b/src/common/MultiKeyMap.ts
@@ -0,0 +1,42 @@
+/**
+ * Copyright (c) 2022 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+
+export class TwoKeyMap<TFirst extends string | number, TSecond extends string | number, TValue> {
+  private _data: { [bg: string | number]: { [fg: string | number]: TValue | undefined } | undefined } = {};
+
+  public set(first: TFirst, second: TSecond, value: TValue): void {
+    if (!this._data[first]) {
+      this._data[first] = {};
+    }
+    this._data[first as string | number]![second] = value;
+  }
+
+  public get(first: TFirst, second: TSecond): TValue | undefined {
+    return this._data[first as string | number] ? this._data[first as string | number]![second] : undefined;
+  }
+
+  public clear(): void {
+    this._data = {};
+  }
+}
+
+export class FourKeyMap<TFirst extends string | number, TSecond extends string | number, TThird extends string | number, TFourth extends string | number, TValue> {
+  private _data: TwoKeyMap<TFirst, TSecond, TwoKeyMap<TThird, TFourth, TValue>> = new TwoKeyMap();
+
+  public set(first: TFirst, second: TSecond, third: TThird, fourth: TFourth, value: TValue): void {
+    if (!this._data.get(first, second)) {
+      this._data.set(first, second, new TwoKeyMap());
+    }
+    this._data.get(first, second)!.set(third, fourth, value);
+  }
+
+  public get(first: TFirst, second: TSecond, third: TThird, fourth: TFourth): TValue | undefined {
+    return this._data.get(first, second)?.get(third, fourth);
+  }
+
+  public clear(): void {
+    this._data.clear();
+  }
+}