Fix infinite recursion OOM crash in addon-ligatures

1. processLookahead/BacktrackPosition created separate entry objects
   for each glyph at each position (O(N^M) with N glyphs, M positions).
   Now all glyphs at a position share one entry object.
2. cloneEntry, cloneTree, flattenEntry, and mergeTreeEntry could
   infinitely recurse or duplicate work on shared/cyclic references.
   Added visited/merged tracking maps to cache results.

Note that parsing CommitMono locks up the renderer for some time still
so there is still a performance issue here, this fixes the OOM though.
This was mostly defensive generated code since the codebase is
unfamiliar to me.

Fixes #5570
This commit is contained in:
Daniel Imms
2026-01-03 14:06:57 -08:00
parent d06d278f0c
commit 8712f8067d
3 changed files with 103 additions and 53 deletions
@@ -1,13 +1,13 @@
import { ILookupTree, IFlattenedLookupTree, ILookupTreeEntry, IFlattenedLookupTreeEntry } from './types';
export default function flatten(tree: ILookupTree): IFlattenedLookupTree {
export default function flatten(tree: ILookupTree, visited: Map<ILookupTreeEntry, IFlattenedLookupTreeEntry> = new Map()): IFlattenedLookupTree {
const result: IFlattenedLookupTree = {};
for (const [glyphId, entry] of Object.entries(tree.individual)) {
result[glyphId] = flattenEntry(entry);
result[glyphId] = flattenEntry(entry, visited);
}
for (const { range, entry } of tree.range) {
const flattened = flattenEntry(entry);
const flattened = flattenEntry(entry, visited);
for (let glyphId = range[0]; glyphId < range[1]; glyphId++) {
result[glyphId] = flattened;
}
@@ -16,15 +16,20 @@ export default function flatten(tree: ILookupTree): IFlattenedLookupTree {
return result;
}
function flattenEntry(entry: ILookupTreeEntry): IFlattenedLookupTreeEntry {
function flattenEntry(entry: ILookupTreeEntry, visited: Map<ILookupTreeEntry, IFlattenedLookupTreeEntry>): IFlattenedLookupTreeEntry {
if (visited.has(entry)) {
return visited.get(entry)!;
}
const result: IFlattenedLookupTreeEntry = {};
visited.set(entry, result);
if (entry.forward) {
result.forward = flatten(entry.forward);
result.forward = flatten(entry.forward, visited);
}
if (entry.reverse) {
result.reverse = flatten(entry.reverse);
result.reverse = flatten(entry.reverse, visited);
}
if (entry.lookup) {
@@ -14,8 +14,9 @@ export default function mergeTrees(trees: ILookupTree[]): ILookupTree {
range: []
};
const mergedEntries = new WeakMap<ILookupTreeEntry, Set<ILookupTreeEntry>>();
for (const tree of trees) {
mergeSubtree(result, tree);
mergeSubtree(result, tree, mergedEntries);
}
return result;
@@ -26,15 +27,16 @@ export default function mergeTrees(trees: ILookupTree[]): ILookupTree {
*
* @param mainTree The tree where the values should be merged
* @param mergeTree The tree to be merged into the mainTree
* @param mergedEntries WeakMap to track already merged entry pairs
*/
function mergeSubtree(mainTree: ILookupTree, mergeTree: ILookupTree): void {
function mergeSubtree(mainTree: ILookupTree, mergeTree: ILookupTree, mergedEntries: WeakMap<ILookupTreeEntry, Set<ILookupTreeEntry>>): void {
// Need to fix this recursively (and handle lookups)
for (const [glyphId, value] of Object.entries(mergeTree.individual)) {
// The main tree is guaranteed to have no overlaps between the
// individual and range values, so if we match an invididual, there
// must not be a range
if (mainTree.individual[glyphId]) {
mergeTreeEntry(mainTree.individual[glyphId], value);
mergeTreeEntry(mainTree.individual[glyphId], value, mergedEntries);
} else {
let matched = false;
for (const [index, { range, entry }] of mainTree.range.entries()) {
@@ -50,7 +52,7 @@ function mergeSubtree(mainTree: ILookupTree, mergeTree: ILookupTree): void {
// If they overlap, we have to split the range and then
// merge the overlap
mainTree.individual[glyphId] = value;
mergeTreeEntry(mainTree.individual[glyphId], cloneEntry(entry));
mergeTreeEntry(mainTree.individual[glyphId], cloneEntry(entry), mergedEntries);
// When there's an overlap, we also have to fix up the range
// that we had already processed
@@ -101,7 +103,7 @@ function mergeSubtree(mainTree: ILookupTree, mergeTree: ILookupTree): void {
mainTree.individual[overlap.both] = entryToMerge;
}
mergeTreeEntry(entryToMerge, cloneEntry(entry));
mergeTreeEntry(entryToMerge, cloneEntry(entry), mergedEntries);
for (const second of overlap.second) {
if (Array.isArray(second)) {
@@ -124,7 +126,7 @@ function mergeSubtree(mainTree: ILookupTree, mergeTree: ILookupTree): void {
// If they overlap, we have to split the range and then
// merge the overlap
mainTree.individual[remainingRange] = cloneEntry(entry);
mergeTreeEntry(mainTree.individual[remainingRange], cloneEntry(resultEntry));
mergeTreeEntry(mainTree.individual[remainingRange], cloneEntry(resultEntry), mergedEntries);
// When there's an overlap, we also have to fix up the range
// that we had already processed
@@ -158,14 +160,14 @@ function mergeSubtree(mainTree: ILookupTree, mergeTree: ILookupTree): void {
}
// If they overlap, we have to merge the overlap
mergeTreeEntry(mainTree.individual[glyphId], cloneEntry(entry));
mergeTreeEntry(mainTree.individual[glyphId], cloneEntry(entry), mergedEntries);
// Update the remaining ranges
remainingRanges.splice(remainingIndex, 1, ...overlap.second);
break;
} else {
if (Number(glyphId) === remainingRange) {
mergeTreeEntry(mainTree.individual[glyphId], cloneEntry(entry));
mergeTreeEntry(mainTree.individual[glyphId], cloneEntry(entry), mergedEntries);
break;
}
}
@@ -191,8 +193,20 @@ function mergeSubtree(mainTree: ILookupTree, mergeTree: ILookupTree): void {
*
* @param mainTree The entry where the values should be merged
* @param mergeTree The entry to merge into the mainTree
* @param mergedEntries WeakMap to track already merged entry pairs
*/
function mergeTreeEntry(mainTree: ILookupTreeEntry, mergeTree: ILookupTreeEntry): void {
function mergeTreeEntry(mainTree: ILookupTreeEntry, mergeTree: ILookupTreeEntry, mergedEntries: WeakMap<ILookupTreeEntry, Set<ILookupTreeEntry>>): void {
// Check if we've already merged this pair
let mergedSet = mergedEntries.get(mainTree);
if (mergedSet?.has(mergeTree)) {
return;
}
if (!mergedSet) {
mergedSet = new Set();
mergedEntries.set(mainTree, mergedSet);
}
mergedSet.add(mergeTree);
if (
mergeTree.lookup && (
!mainTree.lookup ||
@@ -207,7 +221,7 @@ function mergeTreeEntry(mainTree: ILookupTreeEntry, mergeTree: ILookupTreeEntry)
if (!mainTree.forward) {
mainTree.forward = mergeTree.forward;
} else {
mergeSubtree(mainTree.forward, mergeTree.forward);
mergeSubtree(mainTree.forward, mergeTree.forward, mergedEntries);
}
}
@@ -215,7 +229,7 @@ function mergeTreeEntry(mainTree: ILookupTreeEntry, mergeTree: ILookupTreeEntry)
if (!mainTree.reverse) {
mainTree.reverse = mergeTree.reverse;
} else {
mergeSubtree(mainTree.reverse, mergeTree.reverse);
mergeSubtree(mainTree.reverse, mergeTree.reverse, mergedEntries);
}
}
}
@@ -326,16 +340,22 @@ function rangeOrIndividual(start: number, end: number): number | [number, number
* Clones an individual lookup tree entry.
*
* @param entry Lookup tree entry to clone
* @param visited Map to track already cloned entries (prevents infinite loops)
*/
function cloneEntry(entry: ILookupTreeEntry): ILookupTreeEntry {
function cloneEntry(entry: ILookupTreeEntry, visited: Map<ILookupTreeEntry, ILookupTreeEntry> = new Map()): ILookupTreeEntry {
if (visited.has(entry)) {
return visited.get(entry)!;
}
const result: ILookupTreeEntry = {};
visited.set(entry, result);
if (entry.forward) {
result.forward = cloneTree(entry.forward);
result.forward = cloneTree(entry.forward, visited);
}
if (entry.reverse) {
result.reverse = cloneTree(entry.reverse);
result.reverse = cloneTree(entry.reverse, visited);
}
if (entry.lookup) {
@@ -355,18 +375,19 @@ function cloneEntry(entry: ILookupTreeEntry): ILookupTreeEntry {
* Clones a lookup tree.
*
* @param tree Lookup tree to clone
* @param visited Map to track already cloned entries (prevents infinite loops)
*/
function cloneTree(tree: ILookupTree): ILookupTree {
function cloneTree(tree: ILookupTree, visited: Map<ILookupTreeEntry, ILookupTreeEntry> = new Map()): ILookupTree {
const individual: { [glyphId: string]: ILookupTreeEntry } = {};
for (const [glyphId, entry] of Object.entries(tree.individual)) {
individual[glyphId] = cloneEntry(entry);
individual[glyphId] = cloneEntry(entry, visited);
}
return {
individual,
range: tree.range.map(({ range, entry }) => ({
range: range.slice() as [number, number],
entry: cloneEntry(entry)
entry: cloneEntry(entry, visited)
}))
};
}
@@ -43,29 +43,41 @@ export function processLookaheadPosition(
currentEntries: IEntryMeta[]
): IEntryMeta[] {
const nextEntries: IEntryMeta[] = [];
for (const currentEntry of currentEntries) {
for (const glyph of glyphs) {
const entry: ILookupTreeEntry = {};
if (!currentEntry.entry.forward) {
currentEntry.entry.forward = {
individual: {},
range: []
};
}
nextEntries.push({
entry,
substitutions: currentEntry.substitutions
});
const processedEntries = new Set<ILookupTreeEntry>();
for (const currentEntry of currentEntries) {
// Skip if we've already processed this entry object
if (processedEntries.has(currentEntry.entry)) {
continue;
}
processedEntries.add(currentEntry.entry);
if (!currentEntry.entry.forward) {
currentEntry.entry.forward = {
individual: {},
range: []
};
}
// All glyphs at this position share ONE entry - lookahead just needs to match,
// all paths lead to the same result
const sharedEntry: ILookupTreeEntry = {};
for (const glyph of glyphs) {
if (Array.isArray(glyph)) {
currentEntry.entry.forward.range.push({
entry,
entry: sharedEntry,
range: glyph
});
} else {
currentEntry.entry.forward.individual[glyph] = entry;
currentEntry.entry.forward.individual[glyph] = sharedEntry;
}
}
nextEntries.push({
entry: sharedEntry,
substitutions: currentEntry.substitutions
});
}
return nextEntries;
@@ -76,29 +88,41 @@ export function processBacktrackPosition(
currentEntries: IEntryMeta[]
): IEntryMeta[] {
const nextEntries: IEntryMeta[] = [];
for (const currentEntry of currentEntries) {
for (const glyph of glyphs) {
const entry: ILookupTreeEntry = {};
if (!currentEntry.entry.reverse) {
currentEntry.entry.reverse = {
individual: {},
range: []
};
}
nextEntries.push({
entry,
substitutions: currentEntry.substitutions
});
const processedEntries = new Set<ILookupTreeEntry>();
for (const currentEntry of currentEntries) {
// Skip if we've already processed this entry object
if (processedEntries.has(currentEntry.entry)) {
continue;
}
processedEntries.add(currentEntry.entry);
if (!currentEntry.entry.reverse) {
currentEntry.entry.reverse = {
individual: {},
range: []
};
}
// All glyphs at this position share ONE entry - backtrack just needs to match,
// all paths lead to the same result
const sharedEntry: ILookupTreeEntry = {};
for (const glyph of glyphs) {
if (Array.isArray(glyph)) {
currentEntry.entry.reverse.range.push({
entry,
entry: sharedEntry,
range: glyph
});
} else {
currentEntry.entry.reverse.individual[glyph] = entry;
currentEntry.entry.reverse.individual[glyph] = sharedEntry;
}
}
nextEntries.push({
entry: sharedEntry,
substitutions: currentEntry.substitutions
});
}
return nextEntries;