diff options
Diffstat (limited to 'Lib/fontTools/varLib/merger.py')
| -rw-r--r-- | Lib/fontTools/varLib/merger.py | 472 |
1 files changed, 458 insertions, 14 deletions
diff --git a/Lib/fontTools/varLib/merger.py b/Lib/fontTools/varLib/merger.py index 5a3a4f34..c9a1d3e3 100644 --- a/Lib/fontTools/varLib/merger.py +++ b/Lib/fontTools/varLib/merger.py @@ -3,22 +3,26 @@ Merge OpenType Layout tables (GDEF / GPOS / GSUB). """ import os import copy +import enum from operator import ior import logging +from fontTools.colorLib.builder import MAX_PAINT_COLR_LAYER_COUNT, LayerReuseCache from fontTools.misc import classifyTools from fontTools.misc.roundTools import otRound +from fontTools.misc.treeTools import build_n_ary_tree from fontTools.ttLib.tables import otTables as ot from fontTools.ttLib.tables import otBase as otBase +from fontTools.ttLib.tables.otConverters import BaseFixedValue +from fontTools.ttLib.tables.otTraverse import dfs_base_table from fontTools.ttLib.tables.DefaultTable import DefaultTable from fontTools.varLib import builder, models, varStore -from fontTools.varLib.models import nonNone, allNone, allEqual, allEqualTo +from fontTools.varLib.models import nonNone, allNone, allEqual, allEqualTo, subList from fontTools.varLib.varStore import VarStoreInstancer from functools import reduce from fontTools.otlLib.builder import buildSinglePos from fontTools.otlLib.optimize.gpos import ( - compact_pair_pos, - GPOS_COMPACT_MODE_DEFAULT, - GPOS_COMPACT_MODE_ENV_KEY, + _compression_level_from_env, + compact_pair_pos, ) log = logging.getLogger("fontTools.varLib.merger") @@ -27,11 +31,12 @@ from .errors import ( ShouldBeConstant, FoundANone, MismatchedTypes, + NotANone, LengthsDiffer, KeysDiffer, InconsistentGlyphOrder, InconsistentExtensions, - UnsupportedFormat, + InconsistentFormats, UnsupportedFormat, VarLibMergeError, ) @@ -40,13 +45,15 @@ class Merger(object): def __init__(self, font=None): self.font = font + # mergeTables populates this from the parent's master ttfs + self.ttfs = None @classmethod def merger(celf, clazzes, attrs=(None,)): assert celf != Merger, 'Subclass Merger instead.' if 'mergers' not in celf.__dict__: celf.mergers = {} - if type(clazzes) == type: + if type(clazzes) in (type, enum.EnumMeta): clazzes = (clazzes,) if type(attrs) == str: attrs = (attrs,) @@ -82,10 +89,10 @@ class Merger(object): def mergeObjects(self, out, lst, exclude=()): if hasattr(out, "ensureDecompiled"): - out.ensureDecompiled() + out.ensureDecompiled(recurse=False) for item in lst: if hasattr(item, "ensureDecompiled"): - item.ensureDecompiled() + item.ensureDecompiled(recurse=False) keys = sorted(vars(out).keys()) if not all(keys == sorted(vars(v).keys()) for v in lst): raise KeysDiffer(self, expected=keys, @@ -123,6 +130,11 @@ class Merger(object): mergerFunc = self.mergersFor(out).get(None, None) if mergerFunc is not None: mergerFunc(self, out, lst) + elif isinstance(out, enum.Enum): + # need to special-case Enums as have __dict__ but are not regular 'objects', + # otherwise mergeObjects/mergeThings get trapped in a RecursionError + if not allEqualTo(out, lst): + raise ShouldBeConstant(self, expected=out, got=lst) elif hasattr(out, '__dict__'): self.mergeObjects(out, lst) elif isinstance(out, list): @@ -135,9 +147,8 @@ class Merger(object): for tag in tableTags: if tag not in font: continue try: - self.ttfs = [m for m in master_ttfs if tag in m] - self.mergeThings(font[tag], [m[tag] if tag in m else None - for m in master_ttfs]) + self.ttfs = master_ttfs + self.mergeThings(font[tag], [m.get(tag) for m in master_ttfs]) except VarLibMergeError as e: e.stack.append(tag) raise @@ -217,6 +228,20 @@ def _merge_GlyphOrders(font, lst, values_lst=None, default=None): for dict_set in dict_sets] return order, padded +@AligningMerger.merger(otBase.ValueRecord) +def merge(merger, self, lst): + # Code below sometimes calls us with self being + # a new object. Copy it from lst and recurse. + self.__dict__ = lst[0].__dict__.copy() + merger.mergeObjects(self, lst) + +@AligningMerger.merger(ot.Anchor) +def merge(merger, self, lst): + # Code below sometimes calls us with self being + # a new object. Copy it from lst and recurse. + self.__dict__ = lst[0].__dict__.copy() + merger.mergeObjects(self, lst) + def _Lookup_SinglePos_get_effective_value(merger, subtables, glyph): for self in subtables: if self is None or \ @@ -850,10 +875,14 @@ def merge(merger, self, lst): # Compact the merged subtables # This is a good moment to do it because the compaction should create # smaller subtables, which may prevent overflows from happening. - mode = os.environ.get(GPOS_COMPACT_MODE_ENV_KEY, GPOS_COMPACT_MODE_DEFAULT) - if mode and mode != "0": + # Keep reading the value from the ENV until ufo2ft switches to the config system + level = merger.font.cfg.get( + "fontTools.otlLib.optimize.gpos:COMPRESSION_LEVEL", + default=_compression_level_from_env(), + ) + if level != 0: log.info("Compacting GPOS...") - self.SubTable = compact_pair_pos(merger.font, mode, self.SubTable) + self.SubTable = compact_pair_pos(merger.font, level, self.SubTable) self.SubTableCount = len(self.SubTable) elif isSinglePos and flattened: @@ -1033,11 +1062,19 @@ class VariationMerger(AligningMerger): def mergeThings(self, out, lst): masterModel = None + origTTFs = None if None in lst: if allNone(lst): if out is not None: raise FoundANone(self, got=lst) return + + # temporarily subset the list of master ttfs to the ones for which + # master values are not None + origTTFs = self.ttfs + if self.ttfs: + self.ttfs = subList([v is not None for v in lst], self.ttfs) + masterModel = self.model model, lst = masterModel.getSubModel(lst) self.setModel(model) @@ -1046,6 +1083,8 @@ class VariationMerger(AligningMerger): if masterModel: self.setModel(masterModel) + if origTTFs: + self.ttfs = origTTFs def buildVarDevTable(store_builder, master_values): @@ -1096,3 +1135,408 @@ def merge(merger, self, lst): setattr(self, name, value) if deviceTable: setattr(self, tableName, deviceTable) + + +class COLRVariationMerger(VariationMerger): + """A specialized VariationMerger that takes multiple master fonts containing + COLRv1 tables, and builds a variable COLR font. + + COLR tables are special in that variable subtables can be associated with + multiple delta-set indices (via VarIndexBase). + They also contain tables that must change their type (not simply the Format) + as they become variable (e.g. Affine2x3 -> VarAffine2x3) so this merger takes + care of that too. + """ + + def __init__(self, model, axisTags, font, allowLayerReuse=True): + VariationMerger.__init__(self, model, axisTags, font) + # maps {tuple(varIdxes): VarIndexBase} to facilitate reuse of VarIndexBase + # between variable tables with same varIdxes. + self.varIndexCache = {} + # flat list of all the varIdxes generated while merging + self.varIdxes = [] + # set of id()s of the subtables that contain variations after merging + # and need to be upgraded to the associated VarType. + self.varTableIds = set() + # we keep these around for rebuilding a LayerList while merging PaintColrLayers + self.layers = [] + self.layerReuseCache = None + if allowLayerReuse: + self.layerReuseCache = LayerReuseCache() + # flag to ensure BaseGlyphList is fully merged before LayerList gets processed + self._doneBaseGlyphs = False + + def mergeTables(self, font, master_ttfs, tableTags=("COLR",)): + if "COLR" in tableTags and "COLR" in font: + # The merger modifies the destination COLR table in-place. If this contains + # multiple PaintColrLayers referencing the same layers from LayerList, it's + # a problem because we may risk modifying the same paint more than once, or + # worse, fail while attempting to do that. + # We don't know whether the master COLR table was built with layer reuse + # disabled, thus to be safe we rebuild its LayerList so that it contains only + # unique layers referenced from non-overlapping PaintColrLayers throughout + # the base paint graphs. + self.expandPaintColrLayers(font["COLR"].table) + VariationMerger.mergeTables(self, font, master_ttfs, tableTags) + + def checkFormatEnum(self, out, lst, validate=lambda _: True): + fmt = out.Format + formatEnum = out.formatEnum + ok = False + try: + fmt = formatEnum(fmt) + except ValueError: + pass + else: + ok = validate(fmt) + if not ok: + raise UnsupportedFormat( + self, subtable=type(out).__name__, value=fmt + ) + expected = fmt + got = [] + for v in lst: + fmt = getattr(v, "Format", None) + try: + fmt = formatEnum(fmt) + except ValueError: + pass + got.append(fmt) + if not allEqualTo(expected, got): + raise InconsistentFormats( + self, + subtable=type(out).__name__, + expected=expected, + got=got, + ) + return expected + + def mergeSparseDict(self, out, lst): + for k in out.keys(): + try: + self.mergeThings(out[k], [v.get(k) for v in lst]) + except VarLibMergeError as e: + e.stack.append(f"[{k!r}]") + raise + + def mergeAttrs(self, out, lst, attrs): + for attr in attrs: + value = getattr(out, attr) + values = [getattr(item, attr) for item in lst] + try: + self.mergeThings(value, values) + except VarLibMergeError as e: + e.stack.append(f".{attr}") + raise + + def storeMastersForAttr(self, out, lst, attr): + master_values = [getattr(item, attr) for item in lst] + + # VarStore treats deltas for fixed-size floats as integers, so we + # must convert master values to int before storing them in the builder + # then back to float. + is_fixed_size_float = False + conv = out.getConverterByName(attr) + if isinstance(conv, BaseFixedValue): + is_fixed_size_float = True + master_values = [conv.toInt(v) for v in master_values] + + baseValue = master_values[0] + varIdx = ot.NO_VARIATION_INDEX + if not allEqual(master_values): + baseValue, varIdx = self.store_builder.storeMasters(master_values) + + if is_fixed_size_float: + baseValue = conv.fromInt(baseValue) + + return baseValue, varIdx + + def storeVariationIndices(self, varIdxes) -> int: + # try to reuse an existing VarIndexBase for the same varIdxes, or else + # create a new one + key = tuple(varIdxes) + varIndexBase = self.varIndexCache.get(key) + + if varIndexBase is None: + # scan for a full match anywhere in the self.varIdxes + for i in range(len(self.varIdxes) - len(varIdxes) + 1): + if self.varIdxes[i:i+len(varIdxes)] == varIdxes: + self.varIndexCache[key] = varIndexBase = i + break + + if varIndexBase is None: + # try find a partial match at the end of the self.varIdxes + for n in range(len(varIdxes)-1, 0, -1): + if self.varIdxes[-n:] == varIdxes[:n]: + varIndexBase = len(self.varIdxes) - n + self.varIndexCache[key] = varIndexBase + self.varIdxes.extend(varIdxes[n:]) + break + + if varIndexBase is None: + # no match found, append at the end + self.varIndexCache[key] = varIndexBase = len(self.varIdxes) + self.varIdxes.extend(varIdxes) + + return varIndexBase + + def mergeVariableAttrs(self, out, lst, attrs) -> int: + varIndexBase = ot.NO_VARIATION_INDEX + varIdxes = [] + for attr in attrs: + baseValue, varIdx = self.storeMastersForAttr(out, lst, attr) + setattr(out, attr, baseValue) + varIdxes.append(varIdx) + + if any(v != ot.NO_VARIATION_INDEX for v in varIdxes): + varIndexBase = self.storeVariationIndices(varIdxes) + + return varIndexBase + + @classmethod + def convertSubTablesToVarType(cls, table): + for path in dfs_base_table( + table, + skip_root=True, + predicate=lambda path: ( + getattr(type(path[-1].value), "VarType", None) is not None + ) + ): + st = path[-1] + subTable = st.value + varType = type(subTable).VarType + newSubTable = varType() + newSubTable.__dict__.update(subTable.__dict__) + newSubTable.populateDefaults() + parent = path[-2].value + if st.index is not None: + getattr(parent, st.name)[st.index] = newSubTable + else: + setattr(parent, st.name, newSubTable) + + @staticmethod + def expandPaintColrLayers(colr): + """Rebuild LayerList without PaintColrLayers reuse. + + Each base paint graph is fully DFS-traversed (with exception of PaintColrGlyph + which are irrelevant for this); any layers referenced via PaintColrLayers are + collected into a new LayerList and duplicated when reuse is detected, to ensure + that all paints are distinct objects at the end of the process. + PaintColrLayers's FirstLayerIndex/NumLayers are updated so that no overlap + is left. Also, any consecutively nested PaintColrLayers are flattened. + The COLR table's LayerList is replaced with the new unique layers. + A side effect is also that any layer from the old LayerList which is not + referenced by any PaintColrLayers is dropped. + """ + if not colr.LayerList: + # if no LayerList, there's nothing to expand + return + uniqueLayerIDs = set() + newLayerList = [] + for rec in colr.BaseGlyphList.BaseGlyphPaintRecord: + frontier = [rec.Paint] + while frontier: + paint = frontier.pop() + if paint.Format == ot.PaintFormat.PaintColrGlyph: + # don't traverse these, we treat them as constant for merging + continue + elif paint.Format == ot.PaintFormat.PaintColrLayers: + # de-treeify any nested PaintColrLayers, append unique copies to + # the new layer list and update PaintColrLayers index/count + children = list(_flatten_layers(paint, colr)) + first_layer_index = len(newLayerList) + for layer in children: + if id(layer) in uniqueLayerIDs: + layer = copy.deepcopy(layer) + assert id(layer) not in uniqueLayerIDs + newLayerList.append(layer) + uniqueLayerIDs.add(id(layer)) + paint.FirstLayerIndex = first_layer_index + paint.NumLayers = len(children) + else: + children = paint.getChildren(colr) + frontier.extend(reversed(children)) + # sanity check all the new layers are distinct objects + assert len(newLayerList) == len(uniqueLayerIDs) + colr.LayerList.Paint = newLayerList + colr.LayerList.LayerCount = len(newLayerList) + + +@COLRVariationMerger.merger(ot.BaseGlyphList) +def merge(merger, self, lst): + # ignore BaseGlyphCount, allow sparse glyph sets across masters + out = {rec.BaseGlyph: rec for rec in self.BaseGlyphPaintRecord} + masters = [{rec.BaseGlyph: rec for rec in m.BaseGlyphPaintRecord} for m in lst] + + for i, g in enumerate(out.keys()): + try: + # missing base glyphs don't participate in the merge + merger.mergeThings(out[g], [v.get(g) for v in masters]) + except VarLibMergeError as e: + e.stack.append(f".BaseGlyphPaintRecord[{i}]") + e.cause["location"] = f"base glyph {g!r}" + raise + + merger._doneBaseGlyphs = True + + +@COLRVariationMerger.merger(ot.LayerList) +def merge(merger, self, lst): + # nothing to merge for LayerList, assuming we have already merged all PaintColrLayers + # found while traversing the paint graphs rooted at BaseGlyphPaintRecords. + assert merger._doneBaseGlyphs, "BaseGlyphList must be merged before LayerList" + # Simply flush the final list of layers and go home. + self.LayerCount = len(merger.layers) + self.Paint = merger.layers + + +def _flatten_layers(root, colr): + assert root.Format == ot.PaintFormat.PaintColrLayers + for paint in root.getChildren(colr): + if paint.Format == ot.PaintFormat.PaintColrLayers: + yield from _flatten_layers(paint, colr) + else: + yield paint + + +def _merge_PaintColrLayers(self, out, lst): + # we only enforce that the (flat) number of layers is the same across all masters + # but we allow FirstLayerIndex to differ to acommodate for sparse glyph sets. + + out_layers = list(_flatten_layers(out, self.font["COLR"].table)) + + # sanity check ttfs are subset to current values (see VariationMerger.mergeThings) + # before matching each master PaintColrLayers to its respective COLR by position + assert len(self.ttfs) == len(lst) + master_layerses = [ + list(_flatten_layers(lst[i], self.ttfs[i]["COLR"].table)) + for i in range(len(lst)) + ] + + try: + self.mergeLists(out_layers, master_layerses) + except VarLibMergeError as e: + # NOTE: This attribute doesn't actually exist in PaintColrLayers but it's + # handy to have it in the stack trace for debugging. + e.stack.append(".Layers") + raise + + # following block is very similar to LayerListBuilder._beforeBuildPaintColrLayers + # but I couldn't find a nice way to share the code between the two... + + if self.layerReuseCache is not None: + # successful reuse can make the list smaller + out_layers = self.layerReuseCache.try_reuse(out_layers) + + # if the list is still too big we need to tree-fy it + is_tree = len(out_layers) > MAX_PAINT_COLR_LAYER_COUNT + out_layers = build_n_ary_tree(out_layers, n=MAX_PAINT_COLR_LAYER_COUNT) + + # We now have a tree of sequences with Paint leaves. + # Convert the sequences into PaintColrLayers. + def listToColrLayers(paint): + if isinstance(paint, list): + layers = [listToColrLayers(l) for l in paint] + paint = ot.Paint() + paint.Format = int(ot.PaintFormat.PaintColrLayers) + paint.NumLayers = len(layers) + paint.FirstLayerIndex = len(self.layers) + self.layers.extend(layers) + if self.layerReuseCache is not None: + self.layerReuseCache.add(layers, paint.FirstLayerIndex) + return paint + + out_layers = [listToColrLayers(l) for l in out_layers] + + if len(out_layers) == 1 and out_layers[0].Format == ot.PaintFormat.PaintColrLayers: + # special case when the reuse cache finds a single perfect PaintColrLayers match + # (it can only come from a successful reuse, _flatten_layers has gotten rid of + # all nested PaintColrLayers already); we assign it directly and avoid creating + # an extra table + out.NumLayers = out_layers[0].NumLayers + out.FirstLayerIndex = out_layers[0].FirstLayerIndex + else: + out.NumLayers = len(out_layers) + out.FirstLayerIndex = len(self.layers) + + self.layers.extend(out_layers) + + # Register our parts for reuse provided we aren't a tree + # If we are a tree the leaves registered for reuse and that will suffice + if self.layerReuseCache is not None and not is_tree: + self.layerReuseCache.add(out_layers, out.FirstLayerIndex) + + +@COLRVariationMerger.merger((ot.Paint, ot.ClipBox)) +def merge(merger, self, lst): + fmt = merger.checkFormatEnum(self, lst, lambda fmt: not fmt.is_variable()) + + if fmt is ot.PaintFormat.PaintColrLayers: + _merge_PaintColrLayers(merger, self, lst) + return + + varFormat = fmt.as_variable() + + varAttrs = () + if varFormat is not None: + varAttrs = otBase.getVariableAttrs(type(self), varFormat) + staticAttrs = (c.name for c in self.getConverters() if c.name not in varAttrs) + + merger.mergeAttrs(self, lst, staticAttrs) + + varIndexBase = merger.mergeVariableAttrs(self, lst, varAttrs) + + subTables = [st.value for st in self.iterSubTables()] + + # Convert table to variable if itself has variations or any subtables have + isVariable = ( + varIndexBase != ot.NO_VARIATION_INDEX + or any(id(table) in merger.varTableIds for table in subTables) + ) + + if isVariable: + if varAttrs: + # Some PaintVar* don't have any scalar attributes that can vary, + # only indirect offsets to other variable subtables, thus have + # no VarIndexBase of their own (e.g. PaintVarTransform) + self.VarIndexBase = varIndexBase + + if subTables: + # Convert Affine2x3 -> VarAffine2x3, ColorLine -> VarColorLine, etc. + merger.convertSubTablesToVarType(self) + + assert varFormat is not None + self.Format = int(varFormat) + + +@COLRVariationMerger.merger((ot.Affine2x3, ot.ColorStop)) +def merge(merger, self, lst): + varType = type(self).VarType + + varAttrs = otBase.getVariableAttrs(varType) + staticAttrs = (c.name for c in self.getConverters() if c.name not in varAttrs) + + merger.mergeAttrs(self, lst, staticAttrs) + + varIndexBase = merger.mergeVariableAttrs(self, lst, varAttrs) + + if varIndexBase != ot.NO_VARIATION_INDEX: + self.VarIndexBase = varIndexBase + # mark as having variations so the parent table will convert to Var{Type} + merger.varTableIds.add(id(self)) + + +@COLRVariationMerger.merger(ot.ColorLine) +def merge(merger, self, lst): + merger.mergeAttrs(self, lst, (c.name for c in self.getConverters())) + + if any(id(stop) in merger.varTableIds for stop in self.ColorStop): + merger.convertSubTablesToVarType(self) + merger.varTableIds.add(id(self)) + + +@COLRVariationMerger.merger(ot.ClipList, "clips") +def merge(merger, self, lst): + # 'sparse' in that we allow non-default masters to omit ClipBox entries + # for some/all glyphs (i.e. they don't participate) + merger.mergeSparseDict(self, lst) |