Changeset 391 for python/trunk/Lib/lib2to3/refactor.py

Timestamp:

Mar 19, 2014, 11:31:01 PM (11 years ago)

Author:

dmik

Message:

python: Merge vendor 2.7.6 to trunk.

Location:

python/trunk

Files:

• . (modified) (1 prop)
• Lib/lib2to3/refactor.py (modified) (28 diffs)

python/trunk/Lib/lib2to3/refactor.py

@@ -8 +8 @@
 provides infrastructure to write your own refactoring tool.
 """
+
+from __future__ import with_statement
 
 __author__ = "Guido van Rossum <guido@python.org>"
@@ -23 +25 @@
 # Local imports
 from .pgen2 import driver, tokenize, token
+from .fixer_util import find_root
 from . import pytree, pygram
+from . import btm_utils as bu
+from . import btm_matcher as bm
 
 
@@ -123 +128 @@
 
 
-def _detect_future_print(source):
+def _detect_future_features(source):
     have_docstring = False
     gen = tokenize.generate_tokens(StringIO.StringIO(source).readline)
     def advance():
-        tok = next(gen)
+        tok = gen.next()
         return tok[0], tok[1]
     ignore = frozenset((token.NEWLINE, tokenize.NL, token.COMMENT))
+    features = set()
     try:
         while True:
@@ -141 +147 @@
             elif tp == token.NAME and value == u"from":
                 tp, value = advance()
-                if tp != token.NAME and value != u"__future__":
+                if tp != token.NAME or value != u"__future__":
                     break
                 tp, value = advance()
-                if tp != token.NAME and value != u"import":
+                if tp != token.NAME or value != u"import":
                     break
                 tp, value = advance()
@@ -150 +156 @@
                     tp, value = advance()
                 while tp == token.NAME:
-                    if value == u"print_function":
-                        return True
+                    features.add(value)
                     tp, value = advance()
-                    if tp != token.OP and value != u",":
+                    if tp != token.OP or value != u",":
                         break
                     tp, value = advance()
@@ -160 +165 @@
     except StopIteration:
         pass
-    return False
+    return frozenset(features)
 
 
@@ -169 +174 @@
 class RefactoringTool(object):
 
-    _default_options = {"print_function" : False}
+    _default_options = {"print_function" : False,
+                        "write_unchanged_files" : False}
 
     CLASS_PREFIX = "Fix" # The prefix for fixer classes
@@ -191 +197 @@
         else:
             self.grammar = pygram.python_grammar
+        # When this is True, the refactor*() methods will call write_file() for
+        # files processed even if they were not changed during refactoring. If
+        # and only if the refactor method's write parameter was True.
+        self.write_unchanged_files = self.options.get("write_unchanged_files")
         self.errors = []
         self.logger = logging.getLogger("RefactoringTool")
@@ -200 +210 @@
         self.pre_order, self.post_order = self.get_fixers()
 
-        self.pre_order_heads = _get_headnode_dict(self.pre_order)
-        self.post_order_heads = _get_headnode_dict(self.post_order)
 
         self.files = []  # List of files that were or should be modified
+
+        self.BM = bm.BottomMatcher()
+        self.bmi_pre_order = [] # Bottom Matcher incompatible fixers
+        self.bmi_post_order = []
+
+        for fixer in chain(self.post_order, self.pre_order):
+            if fixer.BM_compatible:
+                self.BM.add_fixer(fixer)
+                # remove fixers that will be handled by the bottom-up
+                # matcher
+            elif fixer in self.pre_order:
+                self.bmi_pre_order.append(fixer)
+            elif fixer in self.post_order:
+                self.bmi_post_order.append(fixer)
+
+        self.bmi_pre_order_heads = _get_headnode_dict(self.bmi_pre_order)
+        self.bmi_post_order_heads = _get_headnode_dict(self.bmi_post_order)
+
+
 
     def get_fixers(self):
@@ -267 +294 @@
     def refactor(self, items, write=False, doctests_only=False):
         """Refactor a list of files and directories."""
+
         for dir_or_file in items:
             if os.path.isdir(dir_or_file):
@@ -280 +308 @@
         Files and subdirectories starting with '.' are skipped.
         """
+        py_ext = os.extsep + "py"
         for dirpath, dirnames, filenames in os.walk(dir_name):
             self.log_debug("Descending into %s", dirpath)
@@ -285 +314 @@
             filenames.sort()
             for name in filenames:
-                if not name.startswith(".") and \
-                        os.path.splitext(name)[1].endswith("py"):
+                if (not name.startswith(".") and
+                    os.path.splitext(name)[1] == py_ext):
                     fullname = os.path.join(dirpath, name)
                     self.refactor_file(fullname, write, doctests_only)
@@ -298 +327 @@
         try:
             f = open(filename, "rb")
-        except IOError, err:
+        except IOError as err:
             self.log_error("Can't open %s: %s", filename, err)
             return None, None
@@ -318 +347 @@
             self.log_debug("Refactoring doctests in %s", filename)
             output = self.refactor_docstring(input, filename)
-            if output != input:
+            if self.write_unchanged_files or output != input:
                 self.processed_file(output, filename, input, write, encoding)
             else:
@@ -324 +353 @@
         else:
             tree = self.refactor_string(input, filename)
-            if tree and tree.was_changed:
+            if self.write_unchanged_files or (tree and tree.was_changed):
                 # The [:-1] is to take off the \n we added earlier
                 self.processed_file(unicode(tree)[:-1], filename,
@@ -342 +371 @@
             there were errors during the parse.
         """
-        if _detect_future_print(data):
+        features = _detect_future_features(data)
+        if "print_function" in features:
             self.driver.grammar = pygram.python_grammar_no_print_statement
         try:
             tree = self.driver.parse_string(data)
-        except Exception, err:
+        except Exception as err:
             self.log_error("Can't parse %s: %s: %s",
                            name, err.__class__.__name__, err)
@@ -352 +382 @@
         finally:
             self.driver.grammar = self.grammar
+        tree.future_features = features
         self.log_debug("Refactoring %s", name)
         self.refactor_tree(tree, name)
@@ -361 +392 @@
             self.log_debug("Refactoring doctests in stdin")
             output = self.refactor_docstring(input, "<stdin>")
-            if output != input:
+            if self.write_unchanged_files or output != input:
                 self.processed_file(output, "<stdin>", input)
             else:
@@ -367 +398 @@
         else:
             tree = self.refactor_string(input, "<stdin>")
-            if tree and tree.was_changed:
+            if self.write_unchanged_files or (tree and tree.was_changed):
                 self.processed_file(unicode(tree), "<stdin>", input)
             else:
@@ -374 +405 @@
     def refactor_tree(self, tree, name):
         """Refactors a parse tree (modifying the tree in place).
+
+        For compatible patterns the bottom matcher module is
+        used. Otherwise the tree is traversed node-to-node for
+        matches.
 
         Args:
@@ -383 +418 @@
             True if the tree was modified, False otherwise.
         """
+
         for fixer in chain(self.pre_order, self.post_order):
             fixer.start_tree(tree, name)
 
-        self.traverse_by(self.pre_order_heads, tree.pre_order())
-        self.traverse_by(self.post_order_heads, tree.post_order())
+        #use traditional matching for the incompatible fixers
+        self.traverse_by(self.bmi_pre_order_heads, tree.pre_order())
+        self.traverse_by(self.bmi_post_order_heads, tree.post_order())
+
+        # obtain a set of candidate nodes
+        match_set = self.BM.run(tree.leaves())
+
+        while any(match_set.values()):
+            for fixer in self.BM.fixers:
+                if fixer in match_set and match_set[fixer]:
+                    #sort by depth; apply fixers from bottom(of the AST) to top
+                    match_set[fixer].sort(key=pytree.Base.depth, reverse=True)
+
+                    if fixer.keep_line_order:
+                        #some fixers(eg fix_imports) must be applied
+                        #with the original file's line order
+                        match_set[fixer].sort(key=pytree.Base.get_lineno)
+
+                    for node in list(match_set[fixer]):
+                        if node in match_set[fixer]:
+                            match_set[fixer].remove(node)
+
+                        try:
+                            find_root(node)
+                        except ValueError:
+                            # this node has been cut off from a
+                            # previous transformation ; skip
+                            continue
+
+                        if node.fixers_applied and fixer in node.fixers_applied:
+                            # do not apply the same fixer again
+                            continue
+
+                        results = fixer.match(node)
+
+                        if results:
+                            new = fixer.transform(node, results)
+                            if new is not None:
+                                node.replace(new)
+                                #new.fixers_applied.append(fixer)
+                                for node in new.post_order():
+                                    # do not apply the fixer again to
+                                    # this or any subnode
+                                    if not node.fixers_applied:
+                                        node.fixers_applied = []
+                                    node.fixers_applied.append(fixer)
+
+                                # update the original match set for
+                                # the added code
+                                new_matches = self.BM.run(new.leaves())
+                                for fxr in new_matches:
+                                    if not fxr in match_set:
+                                        match_set[fxr]=[]
+
+                                    match_set[fxr].extend(new_matches[fxr])
 
         for fixer in chain(self.pre_order, self.post_order):
@@ -419 +508 @@
                        encoding=None):
         """
-        Called when a file has been refactored, and there are changes.
+        Called when a file has been refactored and there may be changes.
         """
         self.files.append(filename)
@@ -430 +519 @@
         if equal:
             self.log_debug("No changes to %s", filename)
-            return
+            if not self.write_unchanged_files:
+                return
         if write:
             self.write_file(new_text, filename, old_text, encoding)
@@ -445 +535 @@
         try:
             f = _open_with_encoding(filename, "w", encoding=encoding)
-        except os.error, err:
+        except os.error as err:
             self.log_error("Can't create %s: %s", filename, err)
             return
         try:
             f.write(_to_system_newlines(new_text))
-        except os.error, err:
+        except os.error as err:
             self.log_error("Can't write %s: %s", filename, err)
         finally:
@@ -513 +603 @@
         try:
             tree = self.parse_block(block, lineno, indent)
-        except Exception, err:
-            if self.log.isEnabledFor(logging.DEBUG):
+        except Exception as err:
+            if self.logger.isEnabledFor(logging.DEBUG):
                 for line in block:
                     self.log_debug("Source: %s", line.rstrip(u"\n"))
@@ -561 +651 @@
         in the parser diagnostics and embedded into the parse tree.
         """
-        return self.driver.parse_tokens(self.wrap_toks(block, lineno, indent))
+        tree = self.driver.parse_tokens(self.wrap_toks(block, lineno, indent))
+        tree.future_features = frozenset()
+        return tree
 
     def wrap_toks(self, block, lineno, indent):
@@ -606 +698 @@
         super(MultiprocessRefactoringTool, self).__init__(*args, **kwargs)
         self.queue = None
+        self.output_lock = None
 
     def refactor(self, items, write=False, doctests_only=False,
@@ -619 +712 @@
             raise RuntimeError("already doing multiple processes")
         self.queue = multiprocessing.JoinableQueue()
+        self.output_lock = multiprocessing.Lock()
         processes = [multiprocessing.Process(target=self._child)
                      for i in xrange(num_processes)]