diff options
Diffstat (limited to 'yapf/yapflib/split_penalty.py')
| -rw-r--r-- | yapf/yapflib/split_penalty.py | 249 |
1 files changed, 134 insertions, 115 deletions
diff --git a/yapf/yapflib/split_penalty.py b/yapf/yapflib/split_penalty.py index 416eda3..643ae24 100644 --- a/yapf/yapflib/split_penalty.py +++ b/yapf/yapflib/split_penalty.py @@ -16,21 +16,24 @@ import re from lib2to3 import pytree +from lib2to3.pgen2 import token as grammar_token from yapf.yapflib import format_token from yapf.yapflib import py3compat from yapf.yapflib import pytree_utils from yapf.yapflib import pytree_visitor from yapf.yapflib import style +from yapf.yapflib import subtypes # TODO(morbo): Document the annotations in a centralized place. E.g., the # README file. UNBREAKABLE = 1000 * 1000 -NAMED_ASSIGN = 11000 +NAMED_ASSIGN = 15000 DOTTED_NAME = 4000 VERY_STRONGLY_CONNECTED = 3500 STRONGLY_CONNECTED = 3000 CONNECTED = 500 +TOGETHER = 100 OR_TEST = 1000 AND_TEST = 1100 @@ -46,7 +49,8 @@ TERM = 2000 FACTOR = 2100 POWER = 2200 ATOM = 2300 -ONE_ELEMENT_ARGUMENT = 2500 +ONE_ELEMENT_ARGUMENT = 500 +SUBSCRIPT = 6000 def ComputeSplitPenalties(tree): @@ -64,6 +68,10 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): Split penalties are attached as annotations to tokens. """ + def Visit(self, node): + if not hasattr(node, 'is_pseudo'): # Ignore pseudo tokens. + super(_SplitPenaltyAssigner, self).Visit(node) + def Visit_import_as_names(self, node): # pyline: disable=invalid-name # import_as_names ::= import_as_name (',' import_as_name)* [','] self.DefaultNodeVisit(node) @@ -117,15 +125,15 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): allow_multiline_lambdas = style.Get('ALLOW_MULTILINE_LAMBDAS') if not allow_multiline_lambdas: for child in node.children: - if pytree_utils.NodeName(child) == 'COMMENT': + if child.type == grammar_token.COMMENT: if re.search(r'pylint:.*disable=.*\bg-long-lambda', child.value): allow_multiline_lambdas = True break if allow_multiline_lambdas: - _SetStronglyConnected(node) + _SetExpressionPenalty(node, STRONGLY_CONNECTED) else: - self._SetUnbreakableOnChildren(node) + _SetExpressionPenalty(node, VERY_STRONGLY_CONNECTED) def Visit_parameters(self, node): # pylint: disable=invalid-name # parameters ::= '(' [typedargslist] ')' @@ -133,18 +141,25 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): # Can't break before the opening paren of a parameter list. _SetUnbreakable(node.children[0]) - if not style.Get('DEDENT_CLOSING_BRACKETS'): + if not (style.Get('INDENT_CLOSING_BRACKETS') or + style.Get('DEDENT_CLOSING_BRACKETS')): _SetStronglyConnected(node.children[-1]) def Visit_arglist(self, node): # pylint: disable=invalid-name # arglist ::= argument (',' argument)* [','] + if node.children[0].type == grammar_token.STAR: + # Python 3 treats a star expression as a specific expression type. + # Process it in that method. + self.Visit_star_expr(node) + return + self.DefaultNodeVisit(node) - index = 1 - while index < len(node.children): + + for index in py3compat.range(1, len(node.children)): child = node.children[index] if isinstance(child, pytree.Leaf) and child.value == ',': _SetUnbreakable(child) - index += 1 + for child in node.children: if pytree_utils.NodeName(child) == 'atom': _IncreasePenalty(child, CONNECTED) @@ -152,32 +167,34 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): def Visit_argument(self, node): # pylint: disable=invalid-name # argument ::= test [comp_for] | test '=' test # Really [keyword '='] test self.DefaultNodeVisit(node) - index = 1 - while index < len(node.children) - 1: + + for index in py3compat.range(1, len(node.children) - 1): child = node.children[index] if isinstance(child, pytree.Leaf) and child.value == '=': _SetSplitPenalty( pytree_utils.FirstLeafNode(node.children[index]), NAMED_ASSIGN) _SetSplitPenalty( pytree_utils.FirstLeafNode(node.children[index + 1]), NAMED_ASSIGN) - index += 1 def Visit_tname(self, node): # pylint: disable=invalid-name # tname ::= NAME [':' test] self.DefaultNodeVisit(node) - index = 1 - while index < len(node.children) - 1: + + for index in py3compat.range(1, len(node.children) - 1): child = node.children[index] if isinstance(child, pytree.Leaf) and child.value == ':': _SetSplitPenalty( pytree_utils.FirstLeafNode(node.children[index]), NAMED_ASSIGN) _SetSplitPenalty( pytree_utils.FirstLeafNode(node.children[index + 1]), NAMED_ASSIGN) - index += 1 def Visit_dotted_name(self, node): # pylint: disable=invalid-name # dotted_name ::= NAME ('.' NAME)* - self._SetUnbreakableOnChildren(node) + for child in node.children: + self.Visit(child) + start = 2 if hasattr(node.children[0], 'is_pseudo') else 1 + for i in py3compat.range(start, len(node.children)): + _SetUnbreakable(node.children[i]) def Visit_dictsetmaker(self, node): # pylint: disable=invalid-name # dictsetmaker ::= ( (test ':' test @@ -185,7 +202,7 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): # (test (comp_for | (',' test)* [','])) ) for child in node.children: self.Visit(child) - if pytree_utils.NodeName(child) == 'COLON': + if child.type == grammar_token.COLON: # This is a key to a dictionary. We don't want to split the key if at # all possible. _SetStronglyConnected(child) @@ -193,8 +210,11 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): def Visit_trailer(self, node): # pylint: disable=invalid-name # trailer ::= '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME if node.children[0].value == '.': - self._SetUnbreakableOnChildren(node) - _SetSplitPenalty(node.children[1], DOTTED_NAME) + before = style.Get('SPLIT_BEFORE_DOT') + _SetSplitPenalty(node.children[0], + VERY_STRONGLY_CONNECTED if before else DOTTED_NAME) + _SetSplitPenalty(node.children[1], + DOTTED_NAME if before else VERY_STRONGLY_CONNECTED) elif len(node.children) == 2: # Don't split an empty argument list if at all possible. _SetSplitPenalty(node.children[1], VERY_STRONGLY_CONNECTED) @@ -211,7 +231,7 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): _SetSplitPenalty( pytree_utils.FirstLeafNode(node.children[1]), ONE_ELEMENT_ARGUMENT) - elif (pytree_utils.NodeName(node.children[0]) == 'LSQB' and + elif (node.children[0].type == grammar_token.LSQB and len(node.children[1].children) > 2 and (name.endswith('_test') or name.endswith('_expr'))): _SetStronglyConnected(node.children[1].children[0]) @@ -230,13 +250,23 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): pytree_utils.FirstLeafNode(node.children[1].children[2]), 0) # Don't split the ending bracket of a subscript list. - _SetVeryStronglyConnected(node.children[-1]) + _RecAnnotate(node.children[-1], pytree_utils.Annotation.SPLIT_PENALTY, + VERY_STRONGLY_CONNECTED) elif name not in { 'arglist', 'argument', 'term', 'or_test', 'and_test', 'comparison', 'atom', 'power' }: # Don't split an argument list with one element if at all possible. - _SetStronglyConnected(node.children[1], node.children[2]) + stypes = pytree_utils.GetNodeAnnotation( + pytree_utils.FirstLeafNode(node), pytree_utils.Annotation.SUBTYPE) + if stypes and subtypes.SUBSCRIPT_BRACKET in stypes: + _IncreasePenalty(node, SUBSCRIPT) + + # Bump up the split penalty for the first part of a subscript. We + # would rather not split there. + _IncreasePenalty(node.children[1], CONNECTED) + else: + _SetStronglyConnected(node.children[1], node.children[2]) if name == 'arglist': _SetStronglyConnected(node.children[-1]) @@ -253,7 +283,9 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): pytree_utils.NodeName(node.children[1]) == 'trailer'): # children[1] itself is a whole trailer: we don't want to # mark all of it as unbreakable, only its first token: (, [ or . - _SetUnbreakable(node.children[1].children[0]) + first = pytree_utils.FirstLeafNode(node.children[1]) + if first.value != '.': + _SetUnbreakable(node.children[1].children[0]) # A special case when there are more trailers in the sequence. Given: # atom tr1 tr2 @@ -265,20 +297,20 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): while prev_trailer_idx < len(node.children) - 1: cur_trailer_idx = prev_trailer_idx + 1 cur_trailer = node.children[cur_trailer_idx] - if pytree_utils.NodeName(cur_trailer) == 'trailer': - # Now we know we have two trailers one after the other - prev_trailer = node.children[prev_trailer_idx] - if prev_trailer.children[-1].value != ')': - # Set the previous node unbreakable if it's not a function call: - # atom tr1() tr2 - # It may be necessary (though undesirable) to split up a previous - # function call's parentheses to the next line. - _SetStronglyConnected(prev_trailer.children[-1]) - _SetStronglyConnected(cur_trailer.children[0]) - prev_trailer_idx = cur_trailer_idx - else: + if pytree_utils.NodeName(cur_trailer) != 'trailer': break + # Now we know we have two trailers one after the other + prev_trailer = node.children[prev_trailer_idx] + if prev_trailer.children[-1].value != ')': + # Set the previous node unbreakable if it's not a function call: + # atom tr1() tr2 + # It may be necessary (though undesirable) to split up a previous + # function call's parentheses to the next line. + _SetStronglyConnected(prev_trailer.children[-1]) + _SetStronglyConnected(cur_trailer.children[0]) + prev_trailer_idx = cur_trailer_idx + # We don't want to split before the last ')' of a function call. This also # takes care of the special case of: # atom tr1 tr2 ... trn @@ -288,16 +320,17 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): break if trailer.children[0].value in '([': if len(trailer.children) > 2: - subtypes = pytree_utils.GetNodeAnnotation( + stypes = pytree_utils.GetNodeAnnotation( trailer.children[0], pytree_utils.Annotation.SUBTYPE) - if subtypes and format_token.Subtype.SUBSCRIPT_BRACKET in subtypes: + if stypes and subtypes.SUBSCRIPT_BRACKET in stypes: _SetStronglyConnected( pytree_utils.FirstLeafNode(trailer.children[1])) last_child_node = pytree_utils.LastLeafNode(trailer) if last_child_node.value.strip().startswith('#'): last_child_node = last_child_node.prev_sibling - if not style.Get('DEDENT_CLOSING_BRACKETS'): + if not (style.Get('INDENT_CLOSING_BRACKETS') or + style.Get('DEDENT_CLOSING_BRACKETS')): last = pytree_utils.LastLeafNode(last_child_node.prev_sibling) if last.value != ',': if last_child_node.value == ']': @@ -310,9 +343,15 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): # split the two. _SetStronglyConnected(trailer.children[-1]) - # If the original source has a "builder" style calls, then we should allow - # the reformatter to retain that. - _AllowBuilderStyleCalls(node) + def Visit_subscriptlist(self, node): # pylint: disable=invalid-name + # subscriptlist ::= subscript (',' subscript)* [','] + self.DefaultNodeVisit(node) + _SetSplitPenalty(pytree_utils.FirstLeafNode(node), 0) + prev_child = None + for child in node.children: + if prev_child and prev_child.type == grammar_token.COMMA: + _SetSplitPenalty(pytree_utils.FirstLeafNode(child), 0) + prev_child = child def Visit_subscript(self, node): # pylint: disable=invalid-name # subscript ::= test | [test] ':' [test] [sliceop] @@ -325,6 +364,10 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): _SetStronglyConnected(*node.children[1:]) self.DefaultNodeVisit(node) + def Visit_old_comp_for(self, node): # pylint: disable=invalid-name + # Python 3.7 + self.Visit_comp_for(node) + def Visit_comp_if(self, node): # pylint: disable=invalid-name # comp_if ::= 'if' old_test [comp_iter] _SetSplitPenalty(node.children[0], @@ -332,6 +375,15 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): _SetStronglyConnected(*node.children[1:]) self.DefaultNodeVisit(node) + def Visit_old_comp_if(self, node): # pylint: disable=invalid-name + # Python 3.7 + self.Visit_comp_if(node) + + def Visit_test(self, node): # pylint: disable=invalid-name + # test ::= or_test ['if' or_test 'else' test] | lambdef + _IncreasePenalty(node, OR_TEST) + self.DefaultNodeVisit(node) + def Visit_or_test(self, node): # pylint: disable=invalid-name # or_test ::= and_test ('or' and_test)* self.DefaultNodeVisit(node) @@ -369,8 +421,7 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): # comparison ::= expr (comp_op expr)* self.DefaultNodeVisit(node) if len(node.children) == 3 and _StronglyConnectedCompOp(node): - _SetSplitPenalty( - pytree_utils.FirstLeafNode(node.children[1]), STRONGLY_CONNECTED) + _IncreasePenalty(node.children[1], VERY_STRONGLY_CONNECTED) _SetSplitPenalty( pytree_utils.FirstLeafNode(node.children[2]), STRONGLY_CONNECTED) else: @@ -385,27 +436,19 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): # expr ::= xor_expr ('|' xor_expr)* self.DefaultNodeVisit(node) _IncreasePenalty(node, EXPR) - index = 1 - while index < len(node.children) - 1: - child = node.children[index] - if isinstance(child, pytree.Leaf) and child.value == '|': - if style.Get('SPLIT_BEFORE_BITWISE_OPERATOR'): - _SetSplitPenalty(child, style.Get('SPLIT_PENALTY_BITWISE_OPERATOR')) - else: - _SetSplitPenalty( - pytree_utils.FirstLeafNode(node.children[index + 1]), - style.Get('SPLIT_PENALTY_BITWISE_OPERATOR')) - index += 1 + _SetBitwiseOperandPenalty(node, '|') def Visit_xor_expr(self, node): # pylint: disable=invalid-name # xor_expr ::= and_expr ('^' and_expr)* self.DefaultNodeVisit(node) _IncreasePenalty(node, XOR_EXPR) + _SetBitwiseOperandPenalty(node, '^') def Visit_and_expr(self, node): # pylint: disable=invalid-name # and_expr ::= shift_expr ('&' shift_expr)* self.DefaultNodeVisit(node) _IncreasePenalty(node, AND_EXPR) + _SetBitwiseOperandPenalty(node, '&') def Visit_shift_expr(self, node): # pylint: disable=invalid-name # shift_expr ::= arith_expr (('<<'|'>>') arith_expr)* @@ -418,14 +461,7 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): # arith_expr ::= term (('+'|'-') term)* self.DefaultNodeVisit(node) _IncreasePenalty(node, ARITH_EXPR) - - index = 1 - while index < len(node.children) - 1: - child = node.children[index] - if pytree_utils.NodeName(child) in self._ARITH_OPS: - next_node = pytree_utils.FirstLeafNode(node.children[index + 1]) - _SetSplitPenalty(next_node, ARITH_EXPR) - index += 1 + _SetExpressionOperandPenalty(node, self._ARITH_OPS) _TERM_OPS = frozenset({'STAR', 'AT', 'SLASH', 'PERCENT', 'DOUBLESLASH'}) @@ -433,14 +469,7 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): # term ::= factor (('*'|'@'|'/'|'%'|'//') factor)* self.DefaultNodeVisit(node) _IncreasePenalty(node, TERM) - - index = 1 - while index < len(node.children) - 1: - child = node.children[index] - if pytree_utils.NodeName(child) in self._TERM_OPS: - next_node = pytree_utils.FirstLeafNode(node.children[index + 1]) - _SetSplitPenalty(next_node, TERM) - index += 1 + _SetExpressionOperandPenalty(node, self._TERM_OPS) def Visit_factor(self, node): # pyline: disable=invalid-name # factor ::= ('+'|'-'|'~') factor | power @@ -452,7 +481,8 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): # '[' [listmaker] ']' | # '{' [dictsetmaker] '}') self.DefaultNodeVisit(node) - if node.children[0].value == '(': + if (node.children[0].value == '(' and + not hasattr(node.children[0], 'is_pseudo')): if node.children[-1].value == ')': if pytree_utils.NodeName(node.parent) == 'if_stmt': _SetSplitPenalty(node.children[-1], STRONGLY_CONNECTED) @@ -473,20 +503,9 @@ class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): prev_was_comma = True else: if prev_was_comma: - _SetSplitPenalty(pytree_utils.FirstLeafNode(child), 0) + _SetSplitPenalty(pytree_utils.FirstLeafNode(child), TOGETHER) prev_was_comma = False - ############################################################################ - # Helper methods that set the annotations. - - def _SetUnbreakableOnChildren(self, node): - """Set an UNBREAKABLE penalty annotation on children of node.""" - for child in node.children: - self.Visit(child) - start = 2 if hasattr(node.children[0], 'is_pseudo') else 1 - for i in py3compat.range(start, len(node.children)): - _SetUnbreakable(node.children[i]) - def _SetUnbreakable(node): """Set an UNBREAKABLE penalty annotation for the given node.""" @@ -500,13 +519,6 @@ def _SetStronglyConnected(*nodes): STRONGLY_CONNECTED) -def _SetVeryStronglyConnected(*nodes): - """Set a VERY_STRONGLY_CONNECTED penalty annotation for the given nodes.""" - for node in nodes: - _RecAnnotate(node, pytree_utils.Annotation.SPLIT_PENALTY, - VERY_STRONGLY_CONNECTED) - - def _SetExpressionPenalty(node, penalty): """Set a penalty annotation on children nodes.""" @@ -528,6 +540,30 @@ def _SetExpressionPenalty(node, penalty): RecExpression(node, pytree_utils.FirstLeafNode(node)) +def _SetBitwiseOperandPenalty(node, op): + for index in py3compat.range(1, len(node.children) - 1): + child = node.children[index] + if isinstance(child, pytree.Leaf) and child.value == op: + if style.Get('SPLIT_BEFORE_BITWISE_OPERATOR'): + _SetSplitPenalty(child, style.Get('SPLIT_PENALTY_BITWISE_OPERATOR')) + else: + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index + 1]), + style.Get('SPLIT_PENALTY_BITWISE_OPERATOR')) + + +def _SetExpressionOperandPenalty(node, ops): + for index in py3compat.range(1, len(node.children) - 1): + child = node.children[index] + if pytree_utils.NodeName(child) in ops: + if style.Get('SPLIT_BEFORE_ARITHMETIC_OPERATOR'): + _SetSplitPenalty(child, style.Get('SPLIT_PENALTY_ARITHMETIC_OPERATOR')) + else: + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index + 1]), + style.Get('SPLIT_PENALTY_ARITHMETIC_OPERATOR')) + + def _IncreasePenalty(node, amt): """Increase a penalty annotation on children nodes.""" @@ -536,7 +572,7 @@ def _IncreasePenalty(node, amt): return if isinstance(node, pytree.Leaf): - if node.value in {'(', 'for', 'if'}: + if node.value in {'(', 'for'}: return penalty = pytree_utils.GetNodeAnnotation( node, pytree_utils.Annotation.SPLIT_PENALTY, default=0) @@ -570,10 +606,13 @@ def _RecAnnotate(tree, annotate_name, annotate_value): def _StronglyConnectedCompOp(op): if (len(op.children[1].children) == 2 and - pytree_utils.NodeName(op.children[1]) == 'comp_op' and - pytree_utils.FirstLeafNode(op.children[1]).value == 'not' and - pytree_utils.LastLeafNode(op.children[1]).value == 'in'): - return True + pytree_utils.NodeName(op.children[1]) == 'comp_op'): + if (pytree_utils.FirstLeafNode(op.children[1]).value == 'not' and + pytree_utils.LastLeafNode(op.children[1]).value == 'in'): + return True + if (pytree_utils.FirstLeafNode(op.children[1]).value == 'is' and + pytree_utils.LastLeafNode(op.children[1]).value == 'not'): + return True if (isinstance(op.children[1], pytree.Leaf) and op.children[1].value in {'==', 'in'}): return True @@ -590,23 +629,3 @@ def _DecrementSplitPenalty(node, amt): def _SetSplitPenalty(node, penalty): pytree_utils.SetNodeAnnotation(node, pytree_utils.Annotation.SPLIT_PENALTY, penalty) - - -def _AllowBuilderStyleCalls(node): - """Allow splitting before '.' if it's a builder style function call.""" - - def RecGetLeaves(node): - if isinstance(node, pytree.Leaf): - return [node] - children = [] - for child in node.children: - children += RecGetLeaves(child) - return children - - list_of_children = RecGetLeaves(node) - prev_child = None - for child in list_of_children: - if child.value == '.': - if prev_child.lineno != child.lineno: - _SetSplitPenalty(child, 0) - prev_child = child |