mirror of
https://github.com/storytold/indd-utils.git
synced 2026-11-01 21:09:58 -08:00
aiongg
10b008e77a
tools/root_values.py collects the object styles other than [None] and the preferences from the pairs compare.py compares, like the root styles, instead of the part of the corpus it had first: 1,585 pairs, 1,448 distinct IDML files. tools/element_values.py reads 1,495 distinct IDML files. The DOM 8 to 11 merge of the DOM 12 object style frame values is gone: the file has those versions itself. Values that vary in the larger corpus are dropped, also where only one document differs and the converter does not read the value from the INDD: 209 keys, most of them preferences (EPubExportPreference, TextDefault, PageItemDefault, IndexOptions, TextPreference), object style values (line ends, gap colour, stroke tint, anchored object and text wrap settings, baseline frame grid before DOM 18), Spread PageTransitionType and PageTransitionDirection, and character style PreviewColor. Evidence in idml-values.md. compare.py --exclude own/, trustworthy pairs: coverage 97.79 % -> 97.09 % (-212,444 values on dropped keys, +11,501 on keys whose first version is now earlier); documents >= 99 % 514 -> 106 of 1,251; all pairs 96.55 % -> 95.85 %. Extras 94,610 -> 94,328. Every lost value is on a regenerated key. No new failures; story text unchanged.
422 lines
17 KiB
Python
422 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Find the values that every corpus IDML has on its root styles, on its
|
|
other object styles and in its preferences.
|
|
|
|
Reads Resources/Styles.xml and Resources/Preferences.xml of the
|
|
reference IDML of every pair that compare.py compares (distinct files
|
|
only) and collects each attribute,
|
|
Properties child and child element attribute of the root styles and of
|
|
the top-level preference elements. A value is kept when it is the same in
|
|
every IDML from some DOM version on and present in all of them; the
|
|
earliest such version is its minimum version. Attributes the converter
|
|
reads from the INDD (the TEXT_ATTRS table in src/idml/attrs.rs) are left
|
|
out for the paragraph and character root styles, and the document setup
|
|
it reads is left out of the preferences. Preference values that refer to
|
|
a style other than a root style are left out, since the package need not
|
|
have that style.
|
|
|
|
Object styles other than the root style `[None]` are collected the same
|
|
way, all styles of a file together: a value is kept when every such style
|
|
in every IDML from some DOM version on has it, with the same value.
|
|
|
|
Usage: python3 -I tools/root_values.py [--write]
|
|
Prints the evidence counts. With --write, regenerates
|
|
src/idml/root_values.xml, src/idml/object_style_values.xml and
|
|
src/idml/preference_values.xml. See docs/format/idml-values.md.
|
|
"""
|
|
|
|
import hashlib
|
|
import re
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
import zipfile
|
|
from collections import OrderedDict, defaultdict
|
|
from pathlib import Path
|
|
from xml.sax.saxutils import quoteattr
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
OUT = ROOT / "src" / "idml" / "root_values.xml"
|
|
PREF_OUT = ROOT / "src" / "idml" / "preference_values.xml"
|
|
OBJECT_OUT = ROOT / "src" / "idml" / "object_style_values.xml"
|
|
|
|
# The values are collected from the pairs that tools/compare.py compares,
|
|
# without the files whose path under corpus/ starts with one of these
|
|
# (privately held samples).
|
|
EXCLUDE = ("own/",)
|
|
|
|
# Preference attributes the converter reads from the INDD.
|
|
PREF_SKIP = {
|
|
("DocumentPreference", "PageHeight"), ("DocumentPreference", "PageWidth"),
|
|
("DocumentPreference", "FacingPages"), ("DocumentPreference", "Intent"),
|
|
("DocumentPreference", "DocumentBleedTopOffset"),
|
|
("DocumentPreference", "DocumentBleedBottomOffset"),
|
|
("DocumentPreference", "DocumentBleedInsideOrLeftOffset"),
|
|
("DocumentPreference", "DocumentBleedOutsideOrRightOffset"),
|
|
("ViewPreference", "RulerOrigin"),
|
|
}
|
|
# Styles a preference value may refer to: those every package has.
|
|
ROOT_STYLES = {
|
|
"ParagraphStyle/$ID/[No paragraph style]",
|
|
"CharacterStyle/$ID/[No character style]",
|
|
"ObjectStyle/$ID/[None]",
|
|
}
|
|
# A preference value first seen in a later version is kept only when at
|
|
# least this many files show it.
|
|
MIN_FILES = 10
|
|
STYLE_REF = re.compile(r"(?:>|^)(Paragraph|Character|Object|Cell|Table)Style/[^<]*")
|
|
|
|
PREF_HEADER = """<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Values that every IDML exported by InDesign in the corpus has in
|
|
Resources/Preferences.xml. The converter writes them; values it reads
|
|
from the INDD take precedence. Values in a block with MinimumVersion are
|
|
written only for documents of that InDesign version or later. Generated
|
|
by tools/root_values.py; evidence in docs/format/idml-values.md.
|
|
-->
|
|
"""
|
|
|
|
TARGETS = OrderedDict([
|
|
("ParagraphStyle", "ParagraphStyle/$ID/[No paragraph style]"),
|
|
("CharacterStyle", "CharacterStyle/$ID/[No character style]"),
|
|
("ObjectStyle", "ObjectStyle/$ID/[None]"),
|
|
("CellStyle", "CellStyle/$ID/[None]"),
|
|
("TableStyle", "TableStyle/$ID/[No table style]"),
|
|
])
|
|
# Written by the converter itself.
|
|
SKIP = {"Self", "Name"}
|
|
|
|
HEADER = """<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Values that every IDML exported by InDesign in the corpus has on the root
|
|
styles. The converter writes them on its root styles; values it reads from
|
|
the INDD take precedence. Values in a block with MinimumVersion are written
|
|
only for documents of that InDesign version or later. Generated by
|
|
tools/root_values.py; evidence in docs/format/idml-values.md.
|
|
-->
|
|
"""
|
|
|
|
|
|
def decoded_names():
|
|
src = (ROOT / "src" / "idml" / "attrs.rs").read_text()
|
|
table = src[src.index("const TEXT_ATTRS"):src.index("/// An attribute written as a")]
|
|
return set(re.findall(r'0x[0-9A-F]+,\s*"(\w+)"', table))
|
|
|
|
|
|
def corpus_idmls():
|
|
"""The reference IDML of every pair that tools/compare.py compares: a
|
|
little-endian INDD whose IDML has the same major version, not in
|
|
EXCLUDE. Each distinct IDML counts once."""
|
|
lines = (ROOT / "corpus" / "inventory.tsv").read_text().splitlines()
|
|
cols = lines[0].split("\t")
|
|
seen, out, pairs = set(), [], 0
|
|
for line in lines[1:]:
|
|
row = dict(zip(cols, line.split("\t")))
|
|
path, order, ver, dom = row["path"], row["order"], row["version"], row["idml_dom"]
|
|
if order != "LE" or not dom or ver.split(".")[0] != dom.split(".")[0]:
|
|
continue
|
|
elif path.startswith(EXCLUDE):
|
|
continue
|
|
pairs += 1
|
|
idml = ROOT / "corpus" / (row.get("idml") or str(Path(path).with_suffix(".idml")))
|
|
digest = hashlib.md5(idml.read_bytes()).hexdigest()
|
|
if digest not in seen:
|
|
seen.add(digest)
|
|
out.append((int(dom.split(".")[0]), idml))
|
|
return pairs, out
|
|
|
|
|
|
def leaves(el, path, out, order):
|
|
"""Flatten an element: (path, '@attr') and (path, 'P/tag') keys."""
|
|
for k, v in el.attrib.items():
|
|
out[(path, "@" + k)] = v
|
|
order.setdefault((path, "@" + k), len(order))
|
|
for c in el:
|
|
if c.tag == "Properties":
|
|
for p in c:
|
|
key = (path, "P/" + p.tag)
|
|
out[key] = re.sub(r">\s+<", "><", ET.tostring(p, encoding="unicode").strip())
|
|
order.setdefault(key, len(order))
|
|
else:
|
|
order.setdefault((path + (c.tag,), ""), len(order))
|
|
leaves(c, path + (c.tag,), out, order)
|
|
|
|
|
|
def analyse():
|
|
decoded = decoded_names()
|
|
pairs, idmls = corpus_idmls()
|
|
doms = [d for d, _ in idmls]
|
|
values = defaultdict(dict)
|
|
order = defaultdict(dict)
|
|
for i, (_dom, idml) in enumerate(idmls):
|
|
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml"))
|
|
for el in root.iter():
|
|
if el.tag in TARGETS and TARGETS[el.tag] == el.get("Self"):
|
|
out = {}
|
|
leaves(el, (), out, order[el.tag])
|
|
for k, v in out.items():
|
|
values[(el.tag, k)][i] = v
|
|
kept = defaultdict(list)
|
|
left_out = defaultdict(lambda: defaultdict(list))
|
|
for (tag, key), by_file in values.items():
|
|
path, name = key
|
|
if not path and name[1:] in SKIP:
|
|
continue
|
|
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
|
|
if not path and tag in ("ParagraphStyle", "CharacterStyle") and plain in decoded:
|
|
left_out[tag]["read from the INDD"].append(plain)
|
|
continue
|
|
since = None
|
|
for m in sorted({doms[i] for i in by_file}):
|
|
files = [i for i, d in enumerate(doms) if d >= m]
|
|
if all(i in by_file for i in files):
|
|
if len({by_file[i] for i in files}) == 1:
|
|
since = (m, len(files))
|
|
break
|
|
if since is None:
|
|
left_out[tag]["varies"].append("/".join(path + (plain,)))
|
|
continue
|
|
kept[tag].append((key, next(iter(by_file.values())), since))
|
|
return pairs, idmls, doms, kept, left_out, order
|
|
|
|
|
|
def since_of(by_file, doms):
|
|
"""The first DOM version from which every file has the same value, and
|
|
the number of those files; None if there is none."""
|
|
for m in sorted({doms[i] for i in by_file}):
|
|
files = [i for i, d in enumerate(doms) if d >= m]
|
|
if all(i in by_file for i in files):
|
|
if len({by_file[i] for i in files}) == 1:
|
|
return (m, len(files))
|
|
return None
|
|
return None
|
|
|
|
|
|
def style_ref(v):
|
|
m = STYLE_REF.search(v)
|
|
return m and m.group(0).lstrip(">") not in ROOT_STYLES
|
|
|
|
|
|
def analyse_preferences(idmls, doms):
|
|
values = defaultdict(dict)
|
|
order = defaultdict(dict)
|
|
tags = OrderedDict()
|
|
for i, (_dom, idml) in enumerate(idmls):
|
|
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Preferences.xml"))
|
|
for el in root:
|
|
tags.setdefault(el.tag, len(tags))
|
|
out = {}
|
|
leaves(el, (), out, order[el.tag])
|
|
for k, v in out.items():
|
|
values[(el.tag, k)][i] = v
|
|
kept = defaultdict(list)
|
|
left_out = defaultdict(lambda: defaultdict(list))
|
|
for (tag, key), by_file in values.items():
|
|
path, name = key
|
|
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
|
|
if not path and (tag, plain) in PREF_SKIP:
|
|
left_out[tag]["read from the INDD"].append(plain)
|
|
continue
|
|
since = since_of(by_file, doms)
|
|
if since is None:
|
|
left_out[tag]["varies"].append("/".join(path + (plain,)))
|
|
continue
|
|
if since[1] < MIN_FILES:
|
|
left_out[tag][f"in fewer than {MIN_FILES} files"].append("/".join(path + (plain,)))
|
|
continue
|
|
value = next(iter(by_file.values()))
|
|
if style_ref(value):
|
|
left_out[tag]["refers to a style"].append("/".join(path + (plain,)))
|
|
continue
|
|
kept[tag].append((key, value, since))
|
|
return tags, kept, left_out, order
|
|
|
|
|
|
OBJECT_HEADER = """<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Values that every object style other than [None] has in every IDML
|
|
exported by InDesign in the corpus. The converter writes them on those
|
|
styles; values it reads from the INDD take precedence. Values in a block
|
|
with MinimumVersion are written only for documents of that InDesign
|
|
version or later. Generated by tools/root_values.py; evidence in
|
|
docs/format/idml-values.md.
|
|
-->
|
|
"""
|
|
|
|
|
|
def analyse_object_styles(idmls, doms):
|
|
"""Values of the object styles other than the root style. A file
|
|
contributes a value for a key when all its such styles have the key
|
|
with one value; the key is kept when every file with such styles from
|
|
some DOM version on has the same value."""
|
|
values = defaultdict(dict)
|
|
order = {}
|
|
files = []
|
|
styles = 0
|
|
for i, (_dom, idml) in enumerate(idmls):
|
|
root = ET.fromstring(zipfile.ZipFile(idml).read("Resources/Styles.xml"))
|
|
found = [el for el in root.iter("ObjectStyle")
|
|
if el.get("Self") != TARGETS["ObjectStyle"]]
|
|
if not found:
|
|
continue
|
|
files.append(i)
|
|
styles += len(found)
|
|
per_key = defaultdict(list)
|
|
for el in found:
|
|
out = {}
|
|
leaves(el, (), out, order)
|
|
for k, v in out.items():
|
|
per_key[k].append(v)
|
|
for k, vs in per_key.items():
|
|
if len(vs) == len(found) and len(set(vs)) == 1:
|
|
values[k][i] = vs[0]
|
|
sub_doms = [doms[i] if i in files else -1 for i in range(len(doms))]
|
|
kept, left_out = [], []
|
|
for key, by_file in values.items():
|
|
path, name = key
|
|
plain = name.split("/", 1)[1] if name.startswith("P/") else name[1:]
|
|
if not path and (plain in SKIP or plain == "BasedOn"):
|
|
continue
|
|
since = None
|
|
for m in sorted({sub_doms[i] for i in by_file}):
|
|
within = [i for i in files if sub_doms[i] >= m]
|
|
if all(i in by_file for i in within):
|
|
if len({by_file[i] for i in within}) == 1:
|
|
since = (m, len(within))
|
|
break
|
|
if since is None or since[1] < MIN_FILES:
|
|
left_out.append("/".join(path + (plain,)))
|
|
continue
|
|
kept.append((key, by_file[next(iter(by_file))], since))
|
|
return files, styles, kept, left_out, order
|
|
|
|
|
|
def write_object_styles(kept, order, first):
|
|
blocks = []
|
|
for since in sorted({s for _, _, (s, _) in kept}):
|
|
gate = "" if since == first else f' MinimumVersion="{since}"'
|
|
items = sorted((it for it in kept if it[2][0] == since), key=lambda it: order[it[0]])
|
|
tree = {"attrs": [], "props": [], "children": OrderedDict()}
|
|
for (path, name), v, _ in items:
|
|
node = tree
|
|
for p in path:
|
|
node = node["children"].setdefault(
|
|
p, {"attrs": [], "props": [], "children": OrderedDict()})
|
|
if name.startswith("@"):
|
|
node["attrs"].append((name[1:], v))
|
|
else:
|
|
node["props"].append(v)
|
|
blocks.append(f"\t<Values{gate}>\n" + element("ObjectStyle", tree, "\t\t") + "\t</Values>\n")
|
|
OBJECT_OUT.write_text(OBJECT_HEADER + "<ObjectStyleValues>\n" + "".join(blocks)
|
|
+ "</ObjectStyleValues>\n")
|
|
|
|
|
|
def write_preferences(tags, kept, order, first):
|
|
blocks = []
|
|
for since in sorted({s for items in kept.values() for _, _, (s, _) in items}):
|
|
gate = "" if since == first else f' MinimumVersion="{since}"'
|
|
s = f"\t<Values{gate}>\n"
|
|
for tag in tags:
|
|
items = sorted((it for it in kept[tag] if it[2][0] == since),
|
|
key=lambda it: order[tag][it[0]])
|
|
if not items:
|
|
continue
|
|
tree = {"attrs": [], "props": [], "children": OrderedDict()}
|
|
for (path, name), v, _ in items:
|
|
node = tree
|
|
for p in path:
|
|
node = node["children"].setdefault(
|
|
p, {"attrs": [], "props": [], "children": OrderedDict()})
|
|
if name.startswith("@"):
|
|
node["attrs"].append((name[1:], v))
|
|
else:
|
|
node["props"].append(v)
|
|
s += element(tag, tree, "\t\t")
|
|
blocks.append(s + "\t</Values>\n")
|
|
PREF_OUT.write_text(PREF_HEADER + "<PreferenceValues>\n" + "".join(blocks) + "</PreferenceValues>\n")
|
|
|
|
|
|
def element(tag, node, indent):
|
|
attrs = "".join(f" {k}={quoteattr(v)}" for k, v in node["attrs"])
|
|
if not node["props"] and not node["children"]:
|
|
return f"{indent}<{tag}{attrs} />\n"
|
|
s = f"{indent}<{tag}{attrs}>\n"
|
|
if node["props"]:
|
|
s += f"{indent}\t<Properties>\n"
|
|
s += "".join(f"{indent}\t\t{p}\n" for p in node["props"])
|
|
s += f"{indent}\t</Properties>\n"
|
|
for t, c in node["children"].items():
|
|
s += element(t, c, indent + "\t")
|
|
return s + f"{indent}</{tag}>\n"
|
|
|
|
|
|
def write(kept, order, first):
|
|
blocks = []
|
|
for since in sorted({s for items in kept.values() for _, _, (s, _) in items}):
|
|
gate = "" if since == first else f' MinimumVersion="{since}"'
|
|
s = f"\t<Values{gate}>\n"
|
|
for tag in TARGETS:
|
|
items = sorted((it for it in kept[tag] if it[2][0] == since),
|
|
key=lambda it: order[tag][it[0]])
|
|
if not items:
|
|
continue
|
|
tree = {"attrs": [], "props": [], "children": OrderedDict()}
|
|
for (path, name), v, _ in items:
|
|
node = tree
|
|
for p in path:
|
|
node = node["children"].setdefault(
|
|
p, {"attrs": [], "props": [], "children": OrderedDict()})
|
|
if name.startswith("@"):
|
|
node["attrs"].append((name[1:], v))
|
|
else:
|
|
node["props"].append(v)
|
|
s += element(tag, tree, "\t\t")
|
|
blocks.append(s + "\t</Values>\n")
|
|
OUT.write_text(HEADER + "<RootStyleValues>\n" + "".join(blocks) + "</RootStyleValues>\n")
|
|
|
|
|
|
def main():
|
|
pairs, idmls, doms, kept, left_out, order = analyse()
|
|
print(f"{pairs} pairs, {len(idmls)} distinct IDML files, DOM versions {sorted(set(doms))}")
|
|
for tag in TARGETS:
|
|
counts = defaultdict(int)
|
|
for _, _, since in kept[tag]:
|
|
counts[since] += 1
|
|
print(f"\n{tag}:")
|
|
for (since, n), k in sorted(counts.items()):
|
|
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
|
|
for why, names in left_out[tag].items():
|
|
print(f" {len(names):4} left out ({why}): {', '.join(sorted(names))}")
|
|
tags, pkept, pleft, porder = analyse_preferences(idmls, doms)
|
|
print("\nPreferences:")
|
|
total = defaultdict(int)
|
|
for tag in tags:
|
|
for _, _, since in pkept[tag]:
|
|
total[since] += 1
|
|
for (since, n), k in sorted(total.items()):
|
|
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
|
|
for tag in tags:
|
|
counts = {why: len(names) for why, names in pleft[tag].items()}
|
|
print(f" {tag}: {len(pkept[tag])} kept, left out {counts}")
|
|
if pleft[tag].get("refers to a style") or pleft[tag].get("read from the INDD"):
|
|
print(f" {pleft[tag].get('refers to a style', [])} {pleft[tag].get('read from the INDD', [])}")
|
|
files, styles, okept, oleft, oorder = analyse_object_styles(idmls, doms)
|
|
print(f"\nObject styles other than [None]: {styles} in {len(files)} IDML files")
|
|
counts = defaultdict(int)
|
|
for _, _, since in okept:
|
|
counts[since] += 1
|
|
for (since, n), k in sorted(counts.items()):
|
|
print(f" {k:4} values, the same in {n} of {n} IDML files with DOM >= {since}")
|
|
print(f" {len(oleft)} left out (vary, not in every style, or in fewer than {MIN_FILES} files):")
|
|
print(f" {', '.join(sorted(oleft))}")
|
|
if "--write" in sys.argv:
|
|
write_object_styles(okept, oorder, min(doms))
|
|
print(f"wrote {OBJECT_OUT.relative_to(ROOT)}")
|
|
write(kept, order, min(doms))
|
|
print(f"\nwrote {OUT.relative_to(ROOT)}")
|
|
write_preferences(tags, pkept, porder, min(doms))
|
|
print(f"wrote {PREF_OUT.relative_to(ROOT)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|