Changeset 391 for python/trunk/Lib/lib2to3

python/trunk

Property svn:mergeinfo set to
/python/vendor/Python-2.7.6 merged eligible
/python/vendor/current merged eligible

python/trunk/Lib/lib2to3/Grammar.txt

-              r2
+              r391
 # Grammar for Python
+# Grammar for 2to3. This grammar supports Python 2.x and 3.x.
 # Note:  Changing the grammar specified in this file will most likely
 …
        NAME | NUMBER | STRING+ | '.' '.' '.')
 listmaker: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
 testlist_gexp: test ( comp_for | (',' (test|star_expr))* [','] )
+testlist_gexp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
 lambdef: 'lambda' [varargslist] ':' test
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME

python/trunk/Lib/lib2to3/fixer_base.py

-              r2
+              r391
     PATTERN = None  # Most subclasses should override with a string literal
     pattern = None  # Compiled pattern, set by compile_pattern()
+    pattern_tree = None # Tree representation of the pattern
     options = None  # Options object passed to initializer
     filename = None # The filename (set by set_filename)
 …
     _accept_type = None # [Advanced and not public] This tells RefactoringTool
                         # which node type to accept when there's not a pattern.
+    keep_line_order = False # For the bottom matcher: match with the
+                            # original line order
+    BM_compatible = False # Compatibility with the bottom matching
+                          # module; every fixer should set this
+                          # manually
     # Shortcut for access to Python grammar symbols
 …
         """
         if self.PATTERN is not None:
+            self.pattern = PatternCompiler().compile_pattern(self.PATTERN)
+            PC = PatternCompiler()
+            self.pattern, self.pattern_tree = PC.compile_pattern(self.PATTERN,
+                                                                 with_tree=True)
     def set_filename(self, filename):

python/trunk/Lib/lib2to3/fixer_util.py

-              r2
+              r391
 """Utility functions, node construction macros, etc."""
 # Author: Collin Winter
+from itertools import islice
 # Local imports
 …
 def KeywordArg(keyword, value):
     return Node(syms.argument,
                 [keyword, Leaf(token.EQUAL, u'='), value])
+                [keyword, Leaf(token.EQUAL, u"="), value])
 def LParen():
 …
 def Subscript(index_node):
     """A numeric or string subscript"""
     return Node(syms.trailer, [Leaf(token.LBRACE, u'['),
+    return Node(syms.trailer, [Leaf(token.LBRACE, u"["),
                                index_node,
                                Leaf(token.RBRACE, u']')])
+                               Leaf(token.RBRACE, u"]")])
 def String(string, prefix=None):
 …
         leaf.remove()
     children = [Leaf(token.NAME, u'from'),
+    children = [Leaf(token.NAME, u"from"),
                 Leaf(token.NAME, package_name, prefix=u" "),
                 Leaf(token.NAME, u'import', prefix=u" "),
+                Leaf(token.NAME, u"import", prefix=u" "),
                 Node(syms.import_as_names, name_leafs)]
     imp = Node(syms.import_from, children)
 …
 consuming_calls = set(["sorted", "list", "set", "any", "all", "tuple", "sum",
                        "min", "max"])
+                       "min", "max", "enumerate"])
 def attr_chain(obj, attr):
 …
 power<
     ( 'iter' | 'list' | 'tuple' | 'sorted' | 'set' | 'sum' |
       'any' | 'all' | (any* trailer< '.' 'join' >) )
+      'any' | 'all' | 'enumerate' | (any* trailer< '.' 'join' >) )
     trailer< '(' node=any ')' >
     any*
 …
 p2 = """
 power<
     'sorted'
+    ( 'sorted' | 'enumerate' )
     trailer< '(' arglist<node=any any*> ')' >
     any*
 …
 def in_special_context(node):
     """ Returns true if node is in an environment where all that is required
         of it is being itterable (ie, it doesn't matter if it returns a list
         or an itterator).
+        of it is being iterable (ie, it doesn't matter if it returns a list
+        or an iterator).
         See test_map_nochange in test_fixers.py for some examples and tests.
         """
     global p0, p1, p2, pats_built
     if not pats_built:
+        p0 = patcomp.compile_pattern(p0)
         p1 = patcomp.compile_pattern(p1)
-        p0 = patcomp.compile_pattern(p0)
         p2 = patcomp.compile_pattern(p2)
         pats_built = True
 …
     return True
+def find_indentation(node):
+    """Find the indentation of *node*."""
+    while node is not None:
+        if node.type == syms.suite and len(node.children) > 2:
+            indent = node.children[1]
+            if indent.type == token.INDENT:
+                return indent.value
+        node = node.parent
+    return u""
 ###########################################################
 ### The following functions are to find bindings in a suite
 …
     # Scamper up to the top level namespace
     while node.type != syms.file_input:
-        assert node.parent, "Tree is insane! root found before "\
-                           "file_input node was found."
         node = node.parent
+        if not node:
+            raise ValueError("root found before file_input node was found.")
     return node
 …
         if it was not imported. """
     def is_import_stmt(node):
         return node.type == syms.simple_stmt and node.children and \
                is_import(node.children[0])
+        return (node.type == syms.simple_stmt and node.children and
+                is_import(node.children[0]))
     root = find_root(node)
 …
     if insert_pos == 0:
         for idx, node in enumerate(root.children):
             if node.type == syms.simple_stmt and node.children and \
                node.children[0].type == token.STRING:
+            if (node.type == syms.simple_stmt and node.children and
+               node.children[0].type == token.STRING):
                 insert_pos = idx + 1
                 break
 …
     if package is None:
         import_ = Node(syms.import_name, [
             Leaf(token.NAME, u'import'),
             Leaf(token.NAME, name, prefix=u' ')
+            Leaf(token.NAME, u"import"),
+            Leaf(token.NAME, name, prefix=u" ")
         ])
     else:
         import_ = FromImport(package, [Leaf(token.NAME, name, prefix=u' ')])
+        import_ = FromImport(package, [Leaf(token.NAME, name, prefix=u" ")])
     children = [import_, Newline()]
 …
             return None
         n = node.children[3]
         if package and _find(u'as', n):
+        if package and _find(u"as", n):
             # See test_from_import_as for explanation
             return None

python/trunk/Lib/lib2to3/fixes/fix_apply.py

r2	r391
13	13
14	14	class FixApply(fixer_base.BaseFix):
	15	BM_compatible = True
15	16
16	17	PATTERN = """

python/trunk/Lib/lib2to3/fixes/fix_basestring.py

r2	r391
7	7
8	8	class FixBasestring(fixer_base.BaseFix):
	9	BM_compatible = True
9	10
10	11	PATTERN = "'basestring'"

python/trunk/Lib/lib2to3/fixes/fix_buffer.py

r2	r391
10	10
11	11	class FixBuffer(fixer_base.BaseFix):
	12	BM_compatible = True
12	13
13	14	explicit = True # The user must ask for this fixer

python/trunk/Lib/lib2to3/fixes/fix_callable.py

-              r2
+              r391
 class FixCallable(fixer_base.BaseFix):
+    BM_compatible = True
+    order = "pre"
     # Ignore callable(*args) or use of keywords.

python/trunk/Lib/lib2to3/fixes/fix_dict.py

r2	r391
41	41
42	42	class FixDict(fixer_base.BaseFix):
	43	BM_compatible = True
	44
43	45	PATTERN = """
44	46	power< head=any+

python/trunk/Lib/lib2to3/fixes/fix_except.py

r2	r391
35	35
36	36	class FixExcept(fixer_base.BaseFix):
	37	BM_compatible = True
37	38
38	39	PATTERN = """

python/trunk/Lib/lib2to3/fixes/fix_exec.py

r2	r391
17	17
18	18	class FixExec(fixer_base.BaseFix):
	19	BM_compatible = True
19	20
20	21	PATTERN = """

python/trunk/Lib/lib2to3/fixes/fix_execfile.py

r2	r391
14	14
15	15	class FixExecfile(fixer_base.BaseFix):
	16	BM_compatible = True
16	17
17	18	PATTERN = """

python/trunk/Lib/lib2to3/fixes/fix_filter.py

r2	r391
20	20
21	21	class FixFilter(fixer_base.ConditionalFix):
	22	BM_compatible = True
22	23
23	24	PATTERN = """

python/trunk/Lib/lib2to3/fixes/fix_funcattrs.py

r2	r391
8	8
9	9	class FixFuncattrs(fixer_base.BaseFix):
	10	BM_compatible = True
	11
10	12	PATTERN = """
11	13	power< any+ trailer< '.' attr=('func_closure' \| 'func_doc' \| 'func_globals'

python/trunk/Lib/lib2to3/fixes/fix_future.py

r2	r391
10	10
11	11	class FixFuture(fixer_base.BaseFix):
	12	BM_compatible = True
	13
12	14	PATTERN = """import_from< 'from' module_name="__future__" 'import' any >"""
13	15

python/trunk/Lib/lib2to3/fixes/fix_getcwdu.py

r2	r391
9	9
10	10	class FixGetcwdu(fixer_base.BaseFix):
	11	BM_compatible = True
11	12
12	13	PATTERN = """

python/trunk/Lib/lib2to3/fixes/fix_has_key.py

r2	r391
38	38
39	39	class FixHasKey(fixer_base.BaseFix):
	40	BM_compatible = True
40	41
41	42	PATTERN = """

python/trunk/Lib/lib2to3/fixes/fix_idioms.py

r2	r391
36	36
37	37	class FixIdioms(fixer_base.BaseFix):
38
39	38	explicit = True # The user must ask for this fixer
40	39

python/trunk/Lib/lib2to3/fixes/fix_import.py

-              r2
+              r391
 class FixImport(fixer_base.BaseFix):
+    BM_compatible = True
     PATTERN = """
 …
     """
+    def start_tree(self, tree, name):
+        super(FixImport, self).start_tree(tree, name)
+        self.skip = "absolute_import" in tree.future_features
     def transform(self, node, results):
+        if self.skip:
+            return
         imp = results['imp']
 …
                 return
             new = FromImport('.', [imp])
+            new = FromImport(u".", [imp])
             new.prefix = node.prefix
             return new
     def probably_a_local_import(self, imp_name):
+        imp_name = imp_name.split('.', 1)[0]
+        if imp_name.startswith(u"."):
+            # Relative imports are certainly not local imports.
+            return False
+        imp_name = imp_name.split(u".", 1)[0]
         base_path = dirname(self.filename)
         base_path = join(base_path, imp_name)
         # If there is no __init__.py next to the file its not in a package
         # so can't be a relative import.
         if not exists(join(dirname(base_path), '__init__.py')):
+        if not exists(join(dirname(base_path), "__init__.py")):
             return False
         for ext in ['.py', sep, '.pyc', '.so', '.sl', '.pyd']:
+        for ext in [".py", sep, ".pyc", ".so", ".sl", ".pyd"]:
             if exists(base_path + ext):
                 return True

python/trunk/Lib/lib2to3/fixes/fix_imports.py

r2	r391
85	85	class FixImports(fixer_base.BaseFix):
86	86
	87	BM_compatible = True
	88	keep_line_order = True
87	89	# This is overridden in fix_imports2.
88	90	mapping = MAPPING

python/trunk/Lib/lib2to3/fixes/fix_input.py

r2	r391
12	12
13	13	class FixInput(fixer_base.BaseFix):
14
	14	BM_compatible = True
15	15	PATTERN = """
16	16	power< 'input' args=trailer< '(' [any] ')' > >

python/trunk/Lib/lib2to3/fixes/fix_intern.py

r2	r391
13	13
14	14	class FixIntern(fixer_base.BaseFix):
	15	BM_compatible = True
	16	order = "pre"
15	17
16	18	PATTERN = """

python/trunk/Lib/lib2to3/fixes/fix_isinstance.py

r2	r391
15	15
16	16	class FixIsinstance(fixer_base.BaseFix):
17
	17	BM_compatible = True
18	18	PATTERN = """
19	19	power<

python/trunk/Lib/lib2to3/fixes/fix_itertools.py

-              r2
+              r391
 class FixItertools(fixer_base.BaseFix):
+    it_funcs = "('imap'|'ifilter'|'izip'|'ifilterfalse')"
+    BM_compatible = True
+    it_funcs = "('imap'|'ifilter'|'izip'|'izip_longest'|'ifilterfalse')"
     PATTERN = """
               power< it='itertools'
 …
         prefix = None
         func = results['func'][0]
+        if 'it' in results and func.value != u'ifilterfalse':
+        if ('it' in results and
+            func.value not in (u'ifilterfalse', u'izip_longest')):
             dot, it = (results['dot'], results['it'])
             # Remove the 'itertools'
             prefix = it.prefix
             it.remove()
             # Replace the node wich contains ('.', 'function') with the
             # function (to be consistant with the second part of the pattern)
+            # Replace the node which contains ('.', 'function') with the
+            # function (to be consistent with the second part of the pattern)
             dot.remove()
             func.parent.replace(func)

python/trunk/Lib/lib2to3/fixes/fix_itertools_imports.py

-              r2
+              r391
 class FixItertoolsImports(fixer_base.BaseFix):
+    BM_compatible = True
     PATTERN = """
               import_from< 'from' 'itertools' 'import' imports=any >
 …
                 member = child.value
                 name_node = child
+            elif child.type == token.STAR:
+                # Just leave the import as is.
+                return
             else:
                 assert child.type == syms.import_as_name
 …
                 child.value = None
                 child.remove()
             elif member_name == u'ifilterfalse':
+            elif member_name in (u'ifilterfalse', u'izip_longest'):
                 node.changed()
+                name_node.value = u'filterfalse'
+                name_node.value = (u'filterfalse' if member_name[1] == u'f'
+                                   else u'zip_longest')
         # Make sure the import statement is still sane
 …
                 remove_comma ^= True
         if children[-1].type == token.COMMA:
             children[-1].remove()
+        while children and children[-1].type == token.COMMA:
+            children.pop().remove()
         # If there are no imports left, just get rid of the entire statement
         if not (imports.children or getattr(imports, 'value', None)) or \
                 imports.parent is None:
+        if (not (imports.children or getattr(imports, 'value', None)) or
+            imports.parent is None):
             p = node.prefix
             node = BlankLine()

python/trunk/Lib/lib2to3/fixes/fix_long.py

r2	r391
11	11
12	12	class FixLong(fixer_base.BaseFix):
13
	13	BM_compatible = True
14	14	PATTERN = "'long'"
15	15

python/trunk/Lib/lib2to3/fixes/fix_map.py

r2	r391
27	27
28	28	class FixMap(fixer_base.ConditionalFix):
	29	BM_compatible = True
29	30
30	31	PATTERN = """

python/trunk/Lib/lib2to3/fixes/fix_metaclass.py

-              r2
+              r391
     for node in cls_node.children:
         if node.type == syms.suite:
             # already in the prefered format, do nothing
+            # already in the preferred format, do nothing
             return
 …
     """ if there is a semi-colon all the parts count as part of the same
         simple_stmt.  We just want the __metaclass__ part so we move
         everything efter the semi-colon into its own simple_stmt node
+        everything after the semi-colon into its own simple_stmt node
     """
     for semi_ind, node in enumerate(stmt_node.children):
 …
 class FixMetaclass(fixer_base.BaseFix):
+    BM_compatible = True
     PATTERN = """

python/trunk/Lib/lib2to3/fixes/fix_methodattrs.py

r2	r391
14	14
15	15	class FixMethodattrs(fixer_base.BaseFix):
	16	BM_compatible = True
16	17	PATTERN = """
17	18	power< any+ trailer< '.' attr=('im_func' \| 'im_self' \| 'im_class') > any* >

python/trunk/Lib/lib2to3/fixes/fix_next.py

r2	r391
16	16
17	17	class FixNext(fixer_base.BaseFix):
	18	BM_compatible = True
18	19	PATTERN = """
19	20	power< base=any+ trailer< '.' attr='next' > trailer< '(' ')' > >

python/trunk/Lib/lib2to3/fixes/fix_nonzero.py

r2	r391
7	7
8	8	class FixNonzero(fixer_base.BaseFix):
	9	BM_compatible = True
9	10	PATTERN = """
10	11	classdef< 'class' any+ ':'

python/trunk/Lib/lib2to3/fixes/fix_operator.py

-              r2
+              r391
 """Fixer for operator.{isCallable,sequenceIncludes}
+"""Fixer for operator functions.
 operator.isCallable(obj) -> hasattr(obj, '__call__')
+operator.isCallable(obj)       -> hasattr(obj, '__call__')
 operator.sequenceIncludes(obj) -> operator.contains(obj)
+operator.isSequenceType(obj)   -> isinstance(obj, collections.Sequence)
+operator.isMappingType(obj)    -> isinstance(obj, collections.Mapping)
+operator.isNumberType(obj)     -> isinstance(obj, numbers.Number)
+operator.repeat(obj, n)        -> operator.mul(obj, n)
+operator.irepeat(obj, n)       -> operator.imul(obj, n)
 """
 # Local imports
+from .. import fixer_base
+from ..fixer_util import Call, Name, String
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import Call, Name, String, touch_import
+def invocation(s):
+    def dec(f):
+        f.invocation = s
+        return f
+    return dec
 class FixOperator(fixer_base.BaseFix):
+    BM_compatible = True
+    order = "pre"
+    methods = "method=('isCallable'|'sequenceIncludes')"
+    func = "'(' func=any ')'"
+    methods = """
+              method=('isCallable'|'sequenceIncludes'
+                     |'isSequenceType'|'isMappingType'|'isNumberType'
+                     |'repeat'|'irepeat')
+              """
+    obj = "'(' obj=any ')'"
     PATTERN = """
               power< module='operator'
                 trailer< '.' {methods} > trailer< {func} > >
+                trailer< '.' %(methods)s > trailer< %(obj)s > >
+              |
               power< {methods} trailer< {func} > >
               """.format(methods=methods, func=func)
+              power< %(methods)s trailer< %(obj)s > >
+              """ % dict(methods=methods, obj=obj)
     def transform(self, node, results):
+        method = self._check_method(node, results)
+        if method is not None:
+            return method(node, results)
+    @invocation("operator.contains(%s)")
+    def _sequenceIncludes(self, node, results):
+        return self._handle_rename(node, results, u"contains")
+    @invocation("hasattr(%s, '__call__')")
+    def _isCallable(self, node, results):
+        obj = results["obj"]
+        args = [obj.clone(), String(u", "), String(u"'__call__'")]
+        return Call(Name(u"hasattr"), args, prefix=node.prefix)
+    @invocation("operator.mul(%s)")
+    def _repeat(self, node, results):
+        return self._handle_rename(node, results, u"mul")
+    @invocation("operator.imul(%s)")
+    def _irepeat(self, node, results):
+        return self._handle_rename(node, results, u"imul")
+    @invocation("isinstance(%s, collections.Sequence)")
+    def _isSequenceType(self, node, results):
+        return self._handle_type2abc(node, results, u"collections", u"Sequence")
+    @invocation("isinstance(%s, collections.Mapping)")
+    def _isMappingType(self, node, results):
+        return self._handle_type2abc(node, results, u"collections", u"Mapping")
+    @invocation("isinstance(%s, numbers.Number)")
+    def _isNumberType(self, node, results):
+        return self._handle_type2abc(node, results, u"numbers", u"Number")
+    def _handle_rename(self, node, results, name):
         method = results["method"][0]
+        method.value = name
+        method.changed()
+        if method.value == u"sequenceIncludes":
+            if "module" not in results:
+                # operator may not be in scope, so we can't make a change.
+                self.warning(node, "You should use operator.contains here.")
+    def _handle_type2abc(self, node, results, module, abc):
+        touch_import(None, module, node)
+        obj = results["obj"]
+        args = [obj.clone(), String(u", " + u".".join([module, abc]))]
+        return Call(Name(u"isinstance"), args, prefix=node.prefix)
+    def _check_method(self, node, results):
+        method = getattr(self, "_" + results["method"][0].value.encode("ascii"))
+        if callable(method):
+            if "module" in results:
+                return method
             else:
+                method.value = u"contains"
+                method.changed()
+        elif method.value == u"isCallable":
+            if "module" not in results:
+                self.warning(node,
+                             "You should use hasattr(%s, '__call__') here." %
+                             results["func"].value)
+            else:
+                func = results["func"]
+                args = [func.clone(), String(u", "), String(u"'__call__'")]
+                return Call(Name(u"hasattr"), args, prefix=node.prefix)
+                sub = (unicode(results["obj"]),)
+                invocation_str = unicode(method.invocation) % sub
+                self.warning(node, u"You should use '%s' here." % invocation_str)
+        return None

python/trunk/Lib/lib2to3/fixes/fix_paren.py

r2	r391
11	11	# XXX This doesn't support nested for loops like [x for x in 1, 2 for x in 1, 2]
12	12	class FixParen(fixer_base.BaseFix):
	13	BM_compatible = True
	14
13	15	PATTERN = """
14	16	atom< ('[' \| '(')

python/trunk/Lib/lib2to3/fixes/fix_print.py

r2	r391
28	28
29	29	class FixPrint(fixer_base.BaseFix):
	30
	31	BM_compatible = True
30	32
31	33	PATTERN = """

python/trunk/Lib/lib2to3/fixes/fix_raise.py

-              r2
+              r391
 raise E, V    -> raise E(V)
 raise E, V, T -> raise E(V).with_traceback(T)
+raise E, None, T -> raise E.with_traceback(T)
 raise (((E, E'), E''), E'''), V -> raise E(V)
 …
 class FixRaise(fixer_base.BaseFix):
+    BM_compatible = True
     PATTERN = """
     raise_stmt< 'raise' exc=any [',' val=any [',' tb=any]] >
 …
         exc = results["exc"].clone()
+        if exc.type is token.STRING:
+            self.cannot_convert(node, "Python 3 does not support string exceptions")
+        if exc.type == token.STRING:
+            msg = "Python 3 does not support string exceptions"
+            self.cannot_convert(node, msg)
             return
 …
                 # exc.children[1].children[0] is the first element of the tuple
                 exc = exc.children[1].children[0].clone()
             exc.prefix = " "
+            exc.prefix = u" "
         if "val" not in results:
 …
             tb.prefix = u""
+            e = Call(exc, args)
+            e = exc
+            # If there's a traceback and None is passed as the value, then don't
+            # add a call, since the user probably just wants to add a
+            # traceback. See issue #9661.
+            if val.type != token.NAME or val.value != u"None":
+                e = Call(exc, args)
             with_tb = Attr(e, Name(u'with_traceback')) + [ArgList([tb])]
             new = pytree.Node(syms.simple_stmt, [Name(u"raise")] + with_tb)

python/trunk/Lib/lib2to3/fixes/fix_raw_input.py

r2	r391
8	8	class FixRawInput(fixer_base.BaseFix):
9	9
	10	BM_compatible = True
10	11	PATTERN = """
11	12	power< name='raw_input' trailer< '(' [any] ')' > any* >

python/trunk/Lib/lib2to3/fixes/fix_reduce.py

-              r2
+              r391
 """
+from .. import pytree
+from .. import fixer_base
+from ..fixer_util import Name, Attr, touch_import
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import touch_import
 class FixReduce(fixer_base.BaseFix):
+    BM_compatible = True
+    order = "pre"
     PATTERN = """

python/trunk/Lib/lib2to3/fixes/fix_renames.py

r2	r391
41	41
42	42	class FixRenames(fixer_base.BaseFix):
	43	BM_compatible = True
43	44	PATTERN = "\|".join(build_pattern())
44	45

python/trunk/Lib/lib2to3/fixes/fix_repr.py

r2	r391
11	11	class FixRepr(fixer_base.BaseFix):
12	12
	13	BM_compatible = True
13	14	PATTERN = """
14	15	atom < '`' expr=any '`' >

python/trunk/Lib/lib2to3/fixes/fix_set_literal.py

r2	r391
12	12	class FixSetLiteral(fixer_base.BaseFix):
13	13
	14	BM_compatible = True
14	15	explicit = True
15	16

python/trunk/Lib/lib2to3/fixes/fix_standarderror.py

r2	r391
10	10
11	11	class FixStandarderror(fixer_base.BaseFix):
12
	12	BM_compatible = True
13	13	PATTERN = """
14	14	'StandardError'

python/trunk/Lib/lib2to3/fixes/fix_sys_exc.py

-              r2
+              r391
 class FixSysExc(fixer_base.BaseFix):
     # This order matches the ordering of sys.exc_info().
+    exc_info = ["exc_type", "exc_value", "exc_traceback"]
+    exc_info = [u"exc_type", u"exc_value", u"exc_traceback"]
+    BM_compatible = True
     PATTERN = """
               power< 'sys' trailer< dot='.' attribute=(%s) > >

python/trunk/Lib/lib2to3/fixes/fix_throw.py

r2	r391
15	15
16	16	class FixThrow(fixer_base.BaseFix):
17
	17	BM_compatible = True
18	18	PATTERN = """
19	19	power< any trailer< '.' 'throw' >

python/trunk/Lib/lib2to3/fixes/fix_tuple_params.py

-              r2
+              r391
 class FixTupleParams(fixer_base.BaseFix):
+    run_order = 4 #use a lower order since lambda is part of other
+                  #patterns
+    BM_compatible = True
     PATTERN = """
               funcdef< 'def' any parameters< '(' args=any ')' >
 …
         else:
             start = 0
             indent = "; "
+            indent = u"; "
             end = pytree.Leaf(token.INDENT, u"")
 …
         d = {}
     for i, obj in enumerate(param_list):
         trailer = [Subscript(Number(i))]
+        trailer = [Subscript(Number(unicode(i)))]
         if isinstance(obj, list):
             map_to_index(obj, trailer, d=d)

python/trunk/Lib/lib2to3/fixes/fix_types.py

r2	r391
53	53
54	54	class FixTypes(fixer_base.BaseFix):
55
	55	BM_compatible = True
56	56	PATTERN = '\|'.join(_pats)
57	57

python/trunk/Lib/lib2to3/fixes/fix_unicode.py

-              r2
+              r391
+"""Fixer that changes unicode to str, unichr to chr, and u"..." into "...".
+r"""Fixer for unicode.
+* Changes unicode to str and unichr to chr.
+* If "...\u..." is not unicode literal change it into "...\\u...".
+* Change u"..." into "...".
 """
-import re
 from ..pgen2 import token
 from .. import fixer_base
 _mapping = {u"unichr" : u"chr", u"unicode" : u"str"}
-_literal_re = re.compile(ur"[uU][rR]?[\'\"]")
 class FixUnicode(fixer_base.BaseFix):
+    BM_compatible = True
+    PATTERN = "STRING | 'unicode' | 'unichr'"
+    PATTERN = "STRING | 'unicode' | 'unichr'"
+    def start_tree(self, tree, filename):
+        super(FixUnicode, self).start_tree(tree, filename)
+        self.unicode_literals = 'unicode_literals' in tree.future_features
     def transform(self, node, results):
 …
             return new
         elif node.type == token.STRING:
+            if _literal_re.match(node.value):
+                new = node.clone()
+                new.value = new.value[1:]
+                return new
+            val = node.value
+            if not self.unicode_literals and val[0] in u'\'"' and u'\\' in val:
+                val = ur'\\'.join([
+                    v.replace(u'\\u', ur'\\u').replace(u'\\U', ur'\\U')
+                    for v in val.split(ur'\\')
+                ])
+            if val[0] in u'uU':
+                val = val[1:]
+            if val == node.value:
+                return node
+            new = node.clone()
+            new.value = val
+            return new

python/trunk/Lib/lib2to3/fixes/fix_urllib.py

-              r2
+              r391
 # Local imports
+from .fix_imports import alternates, FixImports
+from .. import fixer_base
+from ..fixer_util import Name, Comma, FromImport, Newline, attr_chain
+from lib2to3.fixes.fix_imports import alternates, FixImports
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import (Name, Comma, FromImport, Newline,
+                                find_indentation, Node, syms)
 MAPPING = {'urllib':  [
                 ('urllib.request',
                     ['URLOpener', 'FancyURLOpener', 'urlretrieve',
                      '_urlopener', 'urlopen', 'urlcleanup',
                      'pathname2url', 'url2pathname']),
                 ('urllib.parse',
                     ['quote', 'quote_plus', 'unquote', 'unquote_plus',
                      'urlencode', 'splitattr', 'splithost', 'splitnport',
                      'splitpasswd', 'splitport', 'splitquery', 'splittag',
                      'splittype', 'splituser', 'splitvalue', ]),
                 ('urllib.error',
                     ['ContentTooShortError'])],
            'urllib2' : [
                 ('urllib.request',
                     ['urlopen', 'install_opener', 'build_opener',
                      'Request', 'OpenerDirector', 'BaseHandler',
                      'HTTPDefaultErrorHandler', 'HTTPRedirectHandler',
                      'HTTPCookieProcessor', 'ProxyHandler',
                      'HTTPPasswordMgr',
                      'HTTPPasswordMgrWithDefaultRealm',
                      'AbstractBasicAuthHandler',
                      'HTTPBasicAuthHandler', 'ProxyBasicAuthHandler',
                      'AbstractDigestAuthHandler',
                      'HTTPDigestAuthHandler', 'ProxyDigestAuthHandler',
                      'HTTPHandler', 'HTTPSHandler', 'FileHandler',
                      'FTPHandler', 'CacheFTPHandler',
                      'UnknownHandler']),
                 ('urllib.error',
                     ['URLError', 'HTTPError']),
+MAPPING = {"urllib":  [
+                ("urllib.request",
+                    ["URLopener", "FancyURLopener", "urlretrieve",
+                     "_urlopener", "urlopen", "urlcleanup",
+                     "pathname2url", "url2pathname"]),
+                ("urllib.parse",
+                    ["quote", "quote_plus", "unquote", "unquote_plus",
+                     "urlencode", "splitattr", "splithost", "splitnport",
+                     "splitpasswd", "splitport", "splitquery", "splittag",
+                     "splittype", "splituser", "splitvalue", ]),
+                ("urllib.error",
+                    ["ContentTooShortError"])],
+           "urllib2" : [
+                ("urllib.request",
+                    ["urlopen", "install_opener", "build_opener",
+                     "Request", "OpenerDirector", "BaseHandler",
+                     "HTTPDefaultErrorHandler", "HTTPRedirectHandler",
+                     "HTTPCookieProcessor", "ProxyHandler",
+                     "HTTPPasswordMgr",
+                     "HTTPPasswordMgrWithDefaultRealm",
+                     "AbstractBasicAuthHandler",
+                     "HTTPBasicAuthHandler", "ProxyBasicAuthHandler",
+                     "AbstractDigestAuthHandler",
+                     "HTTPDigestAuthHandler", "ProxyDigestAuthHandler",
+                     "HTTPHandler", "HTTPSHandler", "FileHandler",
+                     "FTPHandler", "CacheFTPHandler",
+                     "UnknownHandler"]),
+                ("urllib.error",
+                    ["URLError", "HTTPError"]),
+           ]
+}
 …
            replacements.
         """
         import_mod = results.get('module')
+        import_mod = results.get("module")
         pref = import_mod.prefix
 …
            module.
         """
         mod_member = results.get('mod_member')
+        mod_member = results.get("mod_member")
         pref = mod_member.prefix
         member = results.get('member')
+        member = results.get("member")
         # Simple case with only a single member being imported
 …
                 mod_member.replace(Name(new_name, prefix=pref))
             else:
+                self.cannot_convert(node,
+                                    'This is an invalid module element')
+                self.cannot_convert(node, "This is an invalid module element")
         # Multiple members being imported
 …
             modules = []
             mod_dict = {}
             members = results.get('members')
+            members = results["members"]
             for member in members:
-                member = member.value
                 # we only care about the actual members
+                if member != ',':
+                if member.type == syms.import_as_name:
+                    as_name = member.children[2].value
+                    member_name = member.children[0].value
+                else:
+                    member_name = member.value
+                    as_name = None
+                if member_name != u",":
                     for change in MAPPING[mod_member.value]:
+                        if member in change[1]:
+                            if change[0] in mod_dict:
+                                mod_dict[change[0]].append(member)
+                            else:
+                                mod_dict[change[0]] = [member]
+                        if member_name in change[1]:
+                            if change[0] not in mod_dict:
                                 modules.append(change[0])
+                            mod_dict.setdefault(change[0], []).append(member)
             new_nodes = []
+            indentation = find_indentation(node)
+            first = True
+            def handle_name(name, prefix):
+                if name.type == syms.import_as_name:
+                    kids = [Name(name.children[0].value, prefix=prefix),
+                            name.children[1].clone(),
+                            name.children[2].clone()]
+                    return [Node(syms.import_as_name, kids)]
+                return [Name(name.value, prefix=prefix)]
             for module in modules:
                 elts = mod_dict[module]
                 names = []
                 for elt in elts[:-1]:
+                    names.extend([Name(elt, prefix=pref), Comma()])
+                names.append(Name(elts[-1], prefix=pref))
+                new_nodes.append(FromImport(module, names))
+                    names.extend(handle_name(elt, pref))
+                    names.append(Comma())
+                names.extend(handle_name(elts[-1], pref))
+                new = FromImport(module, names)
+                if not first or node.parent.prefix.endswith(indentation):
+                    new.prefix = indentation
+                new_nodes.append(new)
+                first = False
             if new_nodes:
                 nodes = []
 …
                 node.replace(nodes)
             else:
                 self.cannot_convert(node, 'All module elements are invalid')
+                self.cannot_convert(node, "All module elements are invalid")
     def transform_dot(self, node, results):
         """Transform for calls to module members in code."""
         module_dot = results.get('bare_with_attr')
         member = results.get('member')
+        module_dot = results.get("bare_with_attr")
+        member = results.get("member")
         new_name = None
         if isinstance(member, list):
 …
                                     prefix=module_dot.prefix))
         else:
             self.cannot_convert(node, 'This is an invalid module element')
+            self.cannot_convert(node, "This is an invalid module element")
     def transform(self, node, results):
         if results.get('module'):
+        if results.get("module"):
             self.transform_import(node, results)
         elif results.get('mod_member'):
+        elif results.get("mod_member"):
             self.transform_member(node, results)
         elif results.get('bare_with_attr'):
+        elif results.get("bare_with_attr"):
             self.transform_dot(node, results)
         # Renaming and star imports are not supported for these modules.
         elif results.get('module_star'):
             self.cannot_convert(node, 'Cannot handle star imports.')
         elif results.get('module_as'):
             self.cannot_convert(node, 'This module is now multiple modules')
+        elif results.get("module_star"):
+            self.cannot_convert(node, "Cannot handle star imports.")
+        elif results.get("module_as"):
+            self.cannot_convert(node, "This module is now multiple modules")

python/trunk/Lib/lib2to3/fixes/fix_xrange.py

-              r2
+              r391
 class FixXrange(fixer_base.BaseFix):
+    BM_compatible = True
     PATTERN = """
               power<
 …
               rest=any* >
               """
+    def start_tree(self, tree, filename):
+        super(FixXrange, self).start_tree(tree, filename)
+        self.transformed_xranges = set()
+    def finish_tree(self, tree, filename):
+        self.transformed_xranges = None
     def transform(self, node, results):
 …
         name = results["name"]
         name.replace(Name(u"range", prefix=name.prefix))
+        # This prevents the new range call from being wrapped in a list later.
+        self.transformed_xranges.add(id(node))
     def transform_range(self, node, results):
+        if not self.in_special_context(node):
+        if (id(node) not in self.transformed_xranges and
+            not self.in_special_context(node)):
             range_call = Call(Name(u"range"), [results["args"].clone()])
             # Encase the range call in list().

python/trunk/Lib/lib2to3/fixes/fix_xreadlines.py

r2	r391
10	10
11	11	class FixXreadlines(fixer_base.BaseFix):
	12	BM_compatible = True
12	13	PATTERN = """
13	14	power< call=any+ trailer< '.' 'xreadlines' > trailer< '(' ')' > >

python/trunk/Lib/lib2to3/fixes/fix_zip.py

r2	r391
14	14	class FixZip(fixer_base.ConditionalFix):
15	15
	16	BM_compatible = True
16	17	PATTERN = """
17	18	power< 'zip' args=trailer< '(' [any] ')' >

python/trunk/Lib/lib2to3/main.py

-              r2
+              r391
 Main program for 2to3.
 """
+from __future__ import with_statement
 import sys
 …
 class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
     """
+    A refactoring tool that can avoid overwriting its input files.
     Prints output to stdout.
+    Output files can optionally be written to a different directory and or
+    have an extra file suffix appended to their name for use in situations
+    where you do not want to replace the input files.
     """
+    def __init__(self, fixers, options, explicit, nobackups, show_diffs):
+    def __init__(self, fixers, options, explicit, nobackups, show_diffs,
+                 input_base_dir='', output_dir='', append_suffix=''):
+        """
+        Args:
+            fixers: A list of fixers to import.
+            options: A dict with RefactoringTool configuration.
+            explicit: A list of fixers to run even if they are explicit.
+            nobackups: If true no backup '.bak' files will be created for those
+                files that are being refactored.
+            show_diffs: Should diffs of the refactoring be printed to stdout?
+            input_base_dir: The base directory for all input files.  This class
+                will strip this path prefix off of filenames before substituting
+                it with output_dir.  Only meaningful if output_dir is supplied.
+                All files processed by refactor() must start with this path.
+            output_dir: If supplied, all converted files will be written into
+                this directory tree instead of input_base_dir.
+            append_suffix: If supplied, all files output by this tool will have
+                this appended to their filename.  Useful for changing .py to
+                .py3 for example by passing append_suffix='3'.
+        """
         self.nobackups = nobackups
         self.show_diffs = show_diffs
+        if input_base_dir and not input_base_dir.endswith(os.sep):
+            input_base_dir += os.sep
+        self._input_base_dir = input_base_dir
+        self._output_dir = output_dir
+        self._append_suffix = append_suffix
         super(StdoutRefactoringTool, self).__init__(fixers, options, explicit)
 …
     def write_file(self, new_text, filename, old_text, encoding):
+        orig_filename = filename
+        if self._output_dir:
+            if filename.startswith(self._input_base_dir):
+                filename = os.path.join(self._output_dir,
+                                        filename[len(self._input_base_dir):])
+            else:
+                raise ValueError('filename %s does not start with the '
+                                 'input_base_dir %s' % (
+                                         filename, self._input_base_dir))
+        if self._append_suffix:
+            filename += self._append_suffix
+        if orig_filename != filename:
+            output_dir = os.path.dirname(filename)
+            if not os.path.isdir(output_dir):
+                os.makedirs(output_dir)
+            self.log_message('Writing converted %s to %s.', orig_filename,
+                             filename)
         if not self.nobackups:
             # Make backup
 …
         if not self.nobackups:
             shutil.copymode(backup, filename)
+        if orig_filename != filename:
+            # Preserve the file mode in the new output directory.
+            shutil.copymode(orig_filename, filename)
     def print_output(self, old, new, filename, equal):
 …
                 diff_lines = diff_texts(old, new, filename)
                 try:
+                    for line in diff_lines:
+                        print line
+                    if self.output_lock is not None:
+                        with self.output_lock:
+                            for line in diff_lines:
+                                print line
+                            sys.stdout.flush()
+                    else:
+                        for line in diff_lines:
+                            print line
                 except UnicodeEncodeError:
                     warn("couldn't encode %s's diff for your terminal" %
 …
                       type="int", help="Run 2to3 concurrently")
     parser.add_option("-x", "--nofix", action="append", default=[],
                       help="Prevent a fixer from being run.")
+                      help="Prevent a transformation from being run")
     parser.add_option("-l", "--list-fixes", action="store_true",
                       help="List available transformations (fixes/fix_*.py)")
+                      help="List available transformations")
     parser.add_option("-p", "--print-function", action="store_true",
                       help="Modify the grammar so that print() is a function")
 …
                       help="Write back modified files")
     parser.add_option("-n", "--nobackups", action="store_true", default=False,
+                      help="Don't write backups for modified files.")
+                      help="Don't write backups for modified files")
+    parser.add_option("-o", "--output-dir", action="store", type="str",
+                      default="", help="Put output files in this directory "
+                      "instead of overwriting the input files.  Requires -n.")
+    parser.add_option("-W", "--write-unchanged-files", action="store_true",
+                      help="Also write files even if no changes were required"
+                      " (useful with --output-dir); implies -w.")
+    parser.add_option("--add-suffix", action="store", type="str", default="",
+                      help="Append this string to all output filenames."
+                      " Requires -n if non-empty.  "
+                      "ex: --add-suffix='3' will generate .py3 files.")
     # Parse command line arguments
 …
     flags = {}
     options, args = parser.parse_args(args)
+    if options.write_unchanged_files:
+        flags["write_unchanged_files"] = True
+        if not options.write:
+            warn("--write-unchanged-files/-W implies -w.")
+        options.write = True
+    # If we allowed these, the original files would be renamed to backup names
+    # but not replaced.
+    if options.output_dir and not options.nobackups:
+        parser.error("Can't use --output-dir/-o without -n.")
+    if options.add_suffix and not options.nobackups:
+        parser.error("Can't use --add-suffix without -n.")
     if not options.write and options.no_diffs:
         warn("not writing files and not printing diffs; that's not very useful")
 …
     level = logging.DEBUG if options.verbose else logging.INFO
     logging.basicConfig(format='%(name)s: %(message)s', level=level)
+    logger = logging.getLogger('lib2to3.main')
     # Initialize the refactoring tool
 …
         requested = avail_fixes.union(explicit)
     fixer_names = requested.difference(unwanted_fixes)
+    rt = StdoutRefactoringTool(sorted(fixer_names), flags, sorted(explicit),
+                               options.nobackups, not options.no_diffs)
+    input_base_dir = os.path.commonprefix(args)
+    if (input_base_dir and not input_base_dir.endswith(os.sep)
+        and not os.path.isdir(input_base_dir)):
+        # One or more similar names were passed, their directory is the base.
+        # os.path.commonprefix() is ignorant of path elements, this corrects
+        # for that weird API.
+        input_base_dir = os.path.dirname(input_base_dir)
+    if options.output_dir:
+        input_base_dir = input_base_dir.rstrip(os.sep)
+        logger.info('Output in %r will mirror the input directory %r layout.',
+                    options.output_dir, input_base_dir)
+    rt = StdoutRefactoringTool(
+            sorted(fixer_names), flags, sorted(explicit),
+            options.nobackups, not options.no_diffs,
+            input_base_dir=input_base_dir,
+            output_dir=options.output_dir,
+            append_suffix=options.add_suffix)
     # Refactor all files and directories passed as arguments

python/trunk/Lib/lib2to3/patcomp.py

-              r2
+              r391
 # Python imports
 import os
+import StringIO
 # Fairly local imports
 …
     """Tokenizes a string suppressing significant whitespace."""
     skip = set((token.NEWLINE, token.INDENT, token.DEDENT))
     tokens = tokenize.generate_tokens(driver.generate_lines(input).next)
+    tokens = tokenize.generate_tokens(StringIO.StringIO(input).readline)
     for quintuple in tokens:
         type, value, start, end, line_text = quintuple
 …
         self.driver = driver.Driver(self.grammar, convert=pattern_convert)
     def compile_pattern(self, input, debug=False):
+    def compile_pattern(self, input, debug=False, with_tree=False):
         """Compiles a pattern string to a nested pytree.*Pattern object."""
         tokens = tokenize_wrapper(input)
 …
         except parse.ParseError as e:
             raise PatternSyntaxError(str(e))
+        return self.compile_node(root)
+        if with_tree:
+            return self.compile_node(root), root
+        else:
+            return self.compile_node(root)
     def compile_node(self, node):

python/trunk/Lib/lib2to3/pgen2/conv.py

-              r2
+              r391
     def parse_graminit_h(self, filename):
         """Parse the .h file writen by pgen.  (Internal)
+        """Parse the .h file written by pgen.  (Internal)
         This file is a sequence of #define statements defining the
 …
     def parse_graminit_c(self, filename):
         """Parse the .c file writen by pgen.  (Internal)
+        """Parse the .c file written by pgen.  (Internal)
         The file looks as follows.  The first two lines are always this:

python/trunk/Lib/lib2to3/pgen2/driver.py

-              r2
+              r391
 import os
 import logging
+import StringIO
 import sys
 …
     def parse_string(self, text, debug=False):
         """Parse a string and return the syntax tree."""
         tokens = tokenize.generate_tokens(generate_lines(text).next)
+        tokens = tokenize.generate_tokens(StringIO.StringIO(text).readline)
         return self.parse_tokens(tokens, debug)
-def generate_lines(text):
-    """Generator that behaves like readline without using StringIO."""
-    for line in text.splitlines(True):
-        yield line
-    while True:
-        yield ""
 …
         return True
     return os.path.getmtime(a) >= os.path.getmtime(b)
+def main(*args):
+    """Main program, when run as a script: produce grammar pickle files.
+    Calls load_grammar for each argument, a path to a grammar text file.
+    """
+    if not args:
+        args = sys.argv[1:]
+    logging.basicConfig(level=logging.INFO, stream=sys.stdout,
+                        format='%(message)s')
+    for gt in args:
+        load_grammar(gt, save=True, force=True)
+    return True
+if __name__ == "__main__":
+    sys.exit(int(not main()))

python/trunk/Lib/lib2to3/pgen2/grammar.py

-              r2
+              r391
 class Grammar(object):
     """Pgen parsing tables tables conversion class.
+    """Pgen parsing tables conversion class.
     Once initialized, this class supplies the grammar tables for the
 …
     states        -- a list of DFAs, where each DFA is a list of
                      states, each state is is a list of arcs, and each
+                     states, each state is a list of arcs, and each
                      arc is a (i, j) pair where i is a label and j is
                      a state number.  The DFA number is the index into

python/trunk/Lib/lib2to3/pgen2/tokenize.py

-              r2
+              r391
            "generate_tokens", "untokenize"]
 del token
+try:
+    bytes
+except NameError:
+    # Support bytes type in Python <= 2.5, so 2to3 turns itself into
+    # valid Python 3 code.
+    bytes = str
 def group(*choices): return '(' + '|'.join(choices) + ')'
 …
             toks_append(tokval)
 cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
+cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)')
 def _get_normal_name(orig_enc):
 …
     It detects the encoding from the presence of a utf-8 bom or an encoding
+    cookie as specified in pep-0263. If both a bom and a cookie are present,
+    but disagree, a SyntaxError will be raised. If the encoding cookie is an
+    invalid charset, raise a SyntaxError.
+    cookie as specified in pep-0263. If both a bom and a cookie are present, but
+    disagree, a SyntaxError will be raised. If the encoding cookie is an invalid
+    charset, raise a SyntaxError.  Note that if a utf-8 bom is found,
+    'utf-8-sig' is returned.
     If no encoding is specified, then the default of 'utf-8' will be returned.
 …
     bom_found = False
     encoding = None
+    default = 'utf-8'
     def read_or_stop():
         try:
             return readline()
         except StopIteration:
             return b''
+            return bytes()
     def find_cookie(line):
 …
         except UnicodeDecodeError:
             return None
+        matches = cookie_re.findall(line_string)
+        if not matches:
+        match = cookie_re.match(line_string)
+        if not match:
             return None
         encoding = _get_normal_name(matches[0])
+        encoding = _get_normal_name(match.group(1))
         try:
             codec = lookup(encoding)
 …
                 # This behaviour mimics the Python interpreter
                 raise SyntaxError('encoding problem: utf-8')
+            else:
+                # Allow it to be properly encoded and decoded.
+                encoding = 'utf-8-sig'
+            encoding += '-sig'
         return encoding
 …
         bom_found = True
         first = first[3:]
+        default = 'utf-8-sig'
     if not first:
         return 'utf-8', []
+        return default, []
     encoding = find_cookie(first)
 …
     second = read_or_stop()
     if not second:
         return 'utf-8', [first]
+        return default, [first]
     encoding = find_cookie(second)
 …
         return encoding, [first, second]
     return 'utf-8', [first, second]
+    return default, [first, second]
 def untokenize(iterable):

python/trunk/Lib/lib2to3/pygram.py

-              r2
+              r391
 # The grammar file
 _GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "Grammar.txt")
+_PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__),
+                                     "PatternGrammar.txt")
 …
 python_grammar_no_print_statement = python_grammar.copy()
 del python_grammar_no_print_statement.keywords["print"]
+pattern_grammar = driver.load_grammar(_PATTERN_GRAMMAR_FILE)
+pattern_symbols = Symbols(pattern_grammar)

python/trunk/Lib/lib2to3/pytree.py

-              r2
+              r391
 import warnings
 from StringIO import StringIO
 HUGE = 0x7FFFFFFF  # maximum repeat count, default max
 …
     return _type_reprs.setdefault(type_num, type_num)
 class Base(object):
 …
     children = ()  # Tuple of subnodes
     was_changed = False
+    was_checked = False
     def __new__(cls, *args, **kwds):
 …
                 return self.parent.children[i-1]
+    def leaves(self):
+        for child in self.children:
+            for x in child.leaves():
+                yield x
+    def depth(self):
+        if self.parent is None:
+            return 0
+        return 1 + self.parent.depth()
     def get_suffix(self):
         """
 …
             return unicode(self).encode("ascii")
 class Node(Base):
     """Concrete implementation for interior nodes."""
+    def __init__(self, type, children, context=None, prefix=None):
+    def __init__(self,type, children,
+                 context=None,
+                 prefix=None,
+                 fixers_applied=None):
         """
         Initializer.
 …
         if prefix is not None:
             self.prefix = prefix
+        if fixers_applied:
+            self.fixers_applied = fixers_applied[:]
+        else:
+            self.fixers_applied = None
     def __repr__(self):
 …
     def clone(self):
         """Return a cloned (deep) copy of self."""
+        return Node(self.type, [ch.clone() for ch in self.children])
+        return Node(self.type, [ch.clone() for ch in self.children],
+                    fixers_applied=self.fixers_applied)
     def post_order(self):
 …
         yield self
         for child in self.children:
             for node in child.post_order():
+            for node in child.pre_order():
                 yield node
+    @property
+    def prefix(self):
+    def _prefix_getter(self):
         """
         The whitespace and comments preceding this node in the input.
 …
         return self.children[0].prefix
+    @prefix.setter
+    def prefix(self, prefix):
+    def _prefix_setter(self, prefix):
         if self.children:
             self.children[0].prefix = prefix
+    prefix = property(_prefix_getter, _prefix_setter)
     def set_child(self, i, child):
 …
     column = 0    # Column where this token tarts in the input
+    def __init__(self, type, value, context=None, prefix=None):
+    def __init__(self, type, value,
+                 context=None,
+                 prefix=None,
+                 fixers_applied=[]):
         """
         Initializer.
 …
         if prefix is not None:
             self._prefix = prefix
+        self.fixers_applied = fixers_applied[:]
     def __repr__(self):
 …
         """Return a cloned (deep) copy of self."""
         return Leaf(self.type, self.value,
+                    (self.prefix, (self.lineno, self.column)))
+                    (self.prefix, (self.lineno, self.column)),
+                    fixers_applied=self.fixers_applied)
+    def leaves(self):
+        yield self
     def post_order(self):
 …
         yield self
+    @property
+    def prefix(self):
+    def _prefix_getter(self):
         """
         The whitespace and comments preceding this token in the input.
 …
         return self._prefix
+    @prefix.setter
+    def prefix(self, prefix):
+    def _prefix_setter(self, prefix):
         self.changed()
         self._prefix = prefix
+    prefix = property(_prefix_getter, _prefix_setter)
 def convert(gr, raw_node):
 …
                      if absent, matches one node;
                      if present, each subsequence is an alternative [*]
             min: optinal minumum number of times to match, default 0
             max: optional maximum number of times tro match, default HUGE
+            min: optional minimum number of times to match, default 0
+            max: optional maximum number of times to match, default HUGE
             name: optional name assigned to this match
 …
             # The reason for this is that hitting the recursion limit usually
             # results in some ugly messages about how RuntimeErrors are being
+            # ignored.
+            save_stderr = sys.stderr
+            sys.stderr = StringIO()
+            # ignored. We don't do this on non-CPython implementation because
+            # they don't have this problem.
+            if hasattr(sys, "getrefcount"):
+                save_stderr = sys.stderr
+                sys.stderr = StringIO()
             try:
                 for count, r in self._recursive_matches(nodes, 0):
 …
                     yield count, r
             finally:
+                sys.stderr = save_stderr
+                if hasattr(sys, "getrefcount"):
+                    sys.stderr = save_stderr
     def _iterative_matches(self, nodes):

python/trunk/Lib/lib2to3/refactor.py

-              r2
+              r391
 provides infrastructure to write your own refactoring tool.
 """
+from __future__ import with_statement
 __author__ = "Guido van Rossum <guido@python.org>"
 …
 # Local imports
 from .pgen2 import driver, tokenize, token
+from .fixer_util import find_root
 from . import pytree, pygram
+from . import btm_utils as bu
+from . import btm_matcher as bm
 …
 def _detect_future_print(source):
+def _detect_future_features(source):
     have_docstring = False
     gen = tokenize.generate_tokens(StringIO.StringIO(source).readline)
     def advance():
         tok = next(gen)
+        tok = gen.next()
         return tok[0], tok[1]
     ignore = frozenset((token.NEWLINE, tokenize.NL, token.COMMENT))
+    features = set()
     try:
         while True:
 …
             elif tp == token.NAME and value == u"from":
                 tp, value = advance()
                 if tp != token.NAME and value != u"__future__":
+                if tp != token.NAME or value != u"__future__":
                     break
                 tp, value = advance()
                 if tp != token.NAME and value != u"import":
+                if tp != token.NAME or value != u"import":
                     break
                 tp, value = advance()
 …
                     tp, value = advance()
                 while tp == token.NAME:
+                    if value == u"print_function":
+                        return True
+                    features.add(value)
                     tp, value = advance()
                     if tp != token.OP and value != u",":
+                    if tp != token.OP or value != u",":
                         break
                     tp, value = advance()
 …
     except StopIteration:
         pass
     return False
+    return frozenset(features)
 …
 class RefactoringTool(object):
+    _default_options = {"print_function" : False}
+    _default_options = {"print_function" : False,
+                        "write_unchanged_files" : False}
     CLASS_PREFIX = "Fix" # The prefix for fixer classes
 …
         else:
             self.grammar = pygram.python_grammar
+        # When this is True, the refactor*() methods will call write_file() for
+        # files processed even if they were not changed during refactoring. If
+        # and only if the refactor method's write parameter was True.
+        self.write_unchanged_files = self.options.get("write_unchanged_files")
         self.errors = []
         self.logger = logging.getLogger("RefactoringTool")
 …
         self.pre_order, self.post_order = self.get_fixers()
-        self.pre_order_heads = _get_headnode_dict(self.pre_order)
-        self.post_order_heads = _get_headnode_dict(self.post_order)
         self.files = []  # List of files that were or should be modified
+        self.BM = bm.BottomMatcher()
+        self.bmi_pre_order = [] # Bottom Matcher incompatible fixers
+        self.bmi_post_order = []
+        for fixer in chain(self.post_order, self.pre_order):
+            if fixer.BM_compatible:
+                self.BM.add_fixer(fixer)
+                # remove fixers that will be handled by the bottom-up
+                # matcher
+            elif fixer in self.pre_order:
+                self.bmi_pre_order.append(fixer)
+            elif fixer in self.post_order:
+                self.bmi_post_order.append(fixer)
+        self.bmi_pre_order_heads = _get_headnode_dict(self.bmi_pre_order)
+        self.bmi_post_order_heads = _get_headnode_dict(self.bmi_post_order)
     def get_fixers(self):
 …
     def refactor(self, items, write=False, doctests_only=False):
         """Refactor a list of files and directories."""
         for dir_or_file in items:
             if os.path.isdir(dir_or_file):
 …
         Files and subdirectories starting with '.' are skipped.
         """
+        py_ext = os.extsep + "py"
         for dirpath, dirnames, filenames in os.walk(dir_name):
             self.log_debug("Descending into %s", dirpath)
 …
             filenames.sort()
             for name in filenames:
                 if not name.startswith(".") and \
                         os.path.splitext(name)[1].endswith("py"):
+                if (not name.startswith(".") and
+                    os.path.splitext(name)[1] == py_ext):
                     fullname = os.path.join(dirpath, name)
                     self.refactor_file(fullname, write, doctests_only)
 …
         try:
             f = open(filename, "rb")
         except IOError, err:
+        except IOError as err:
             self.log_error("Can't open %s: %s", filename, err)
             return None, None
 …
             self.log_debug("Refactoring doctests in %s", filename)
             output = self.refactor_docstring(input, filename)
             if output != input:
+            if self.write_unchanged_files or output != input:
                 self.processed_file(output, filename, input, write, encoding)
             else:
 …
         else:
             tree = self.refactor_string(input, filename)
             if tree and tree.was_changed:
+            if self.write_unchanged_files or (tree and tree.was_changed):
                 # The [:-1] is to take off the \n we added earlier
                 self.processed_file(unicode(tree)[:-1], filename,
 …
             there were errors during the parse.
         """
+        if _detect_future_print(data):
+        features = _detect_future_features(data)
+        if "print_function" in features:
             self.driver.grammar = pygram.python_grammar_no_print_statement
         try:
             tree = self.driver.parse_string(data)
         except Exception, err:
+        except Exception as err:
             self.log_error("Can't parse %s: %s: %s",
                            name, err.__class__.__name__, err)
 …
         finally:
             self.driver.grammar = self.grammar
+        tree.future_features = features
         self.log_debug("Refactoring %s", name)
         self.refactor_tree(tree, name)
 …
             self.log_debug("Refactoring doctests in stdin")
             output = self.refactor_docstring(input, "<stdin>")
             if output != input:
+            if self.write_unchanged_files or output != input:
                 self.processed_file(output, "<stdin>", input)
             else:
 …
         else:
             tree = self.refactor_string(input, "<stdin>")
             if tree and tree.was_changed:
+            if self.write_unchanged_files or (tree and tree.was_changed):
                 self.processed_file(unicode(tree), "<stdin>", input)
             else:
 …
     def refactor_tree(self, tree, name):
         """Refactors a parse tree (modifying the tree in place).
+        For compatible patterns the bottom matcher module is
+        used. Otherwise the tree is traversed node-to-node for
+        matches.
         Args:
 …
             True if the tree was modified, False otherwise.
         """
         for fixer in chain(self.pre_order, self.post_order):
             fixer.start_tree(tree, name)
+        self.traverse_by(self.pre_order_heads, tree.pre_order())
+        self.traverse_by(self.post_order_heads, tree.post_order())
+        #use traditional matching for the incompatible fixers
+        self.traverse_by(self.bmi_pre_order_heads, tree.pre_order())
+        self.traverse_by(self.bmi_post_order_heads, tree.post_order())
+        # obtain a set of candidate nodes
+        match_set = self.BM.run(tree.leaves())
+        while any(match_set.values()):
+            for fixer in self.BM.fixers:
+                if fixer in match_set and match_set[fixer]:
+                    #sort by depth; apply fixers from bottom(of the AST) to top
+                    match_set[fixer].sort(key=pytree.Base.depth, reverse=True)
+                    if fixer.keep_line_order:
+                        #some fixers(eg fix_imports) must be applied
+                        #with the original file's line order
+                        match_set[fixer].sort(key=pytree.Base.get_lineno)
+                    for node in list(match_set[fixer]):
+                        if node in match_set[fixer]:
+                            match_set[fixer].remove(node)
+                        try:
+                            find_root(node)
+                        except ValueError:
+                            # this node has been cut off from a
+                            # previous transformation ; skip
+                            continue
+                        if node.fixers_applied and fixer in node.fixers_applied:
+                            # do not apply the same fixer again
+                            continue
+                        results = fixer.match(node)
+                        if results:
+                            new = fixer.transform(node, results)
+                            if new is not None:
+                                node.replace(new)
+                                #new.fixers_applied.append(fixer)
+                                for node in new.post_order():
+                                    # do not apply the fixer again to
+                                    # this or any subnode
+                                    if not node.fixers_applied:
+                                        node.fixers_applied = []
+                                    node.fixers_applied.append(fixer)
+                                # update the original match set for
+                                # the added code
+                                new_matches = self.BM.run(new.leaves())
+                                for fxr in new_matches:
+                                    if not fxr in match_set:
+                                        match_set[fxr]=[]
+                                    match_set[fxr].extend(new_matches[fxr])
         for fixer in chain(self.pre_order, self.post_order):
 …
                        encoding=None):
         """
         Called when a file has been refactored, and there are changes.
+        Called when a file has been refactored and there may be changes.
         """
         self.files.append(filename)
 …
         if equal:
             self.log_debug("No changes to %s", filename)
+            return
+            if not self.write_unchanged_files:
+                return
         if write:
             self.write_file(new_text, filename, old_text, encoding)
 …
         try:
             f = _open_with_encoding(filename, "w", encoding=encoding)
         except os.error, err:
+        except os.error as err:
             self.log_error("Can't create %s: %s", filename, err)
             return
         try:
             f.write(_to_system_newlines(new_text))
         except os.error, err:
+        except os.error as err:
             self.log_error("Can't write %s: %s", filename, err)
         finally:
 …
         try:
             tree = self.parse_block(block, lineno, indent)
         except Exception, err:
             if self.log.isEnabledFor(logging.DEBUG):
+        except Exception as err:
+            if self.logger.isEnabledFor(logging.DEBUG):
                 for line in block:
                     self.log_debug("Source: %s", line.rstrip(u"\n"))
 …
         in the parser diagnostics and embedded into the parse tree.
         """
+        return self.driver.parse_tokens(self.wrap_toks(block, lineno, indent))
+        tree = self.driver.parse_tokens(self.wrap_toks(block, lineno, indent))
+        tree.future_features = frozenset()
+        return tree
     def wrap_toks(self, block, lineno, indent):
 …
         super(MultiprocessRefactoringTool, self).__init__(*args, **kwargs)
         self.queue = None
+        self.output_lock = None
     def refactor(self, items, write=False, doctests_only=False,
 …
             raise RuntimeError("already doing multiple processes")
         self.queue = multiprocessing.JoinableQueue()
+        self.output_lock = multiprocessing.Lock()
         processes = [multiprocessing.Process(target=self._child)
                      for i in xrange(num_processes)]

python/trunk/Lib/lib2to3/tests/data/bom.py

r2	r391
1	1	# coding: utf-8
2	2	print "BOM BOOM!"
3

python/trunk/Lib/lib2to3/tests/data/py2_test_grammar.py

r2	r391
317	317	x = 1; pass; del x
318	318	def foo():
319		# verify statments that end with semi-colons
	319	# verify statements that end with semi-colons
320	320	x = 1; pass; del x;
321	321	foo()

python/trunk/Lib/lib2to3/tests/data/py3_test_grammar.py

r2	r391
357	357	x = 1; pass; del x
358	358	def foo():
359		# verify statments that end with semi-colons
	359	# verify statements that end with semi-colons
360	360	x = 1; pass; del x;
361	361	foo()

python/trunk/Lib/lib2to3/tests/test_fixers.py

-              r2
+              r391
         self.check(b, a)
+    def test_None_value(self):
+        b = """raise Exception(5), None, tb"""
+        a = """raise Exception(5).with_traceback(tb)"""
+        self.check(b, a)
     def test_tuple_value(self):
         b = """raise Exception, (5, 6, 7)"""
 …
         self.check(b, a)
     def test_14(self):
+    def test_28(self):
         b = "[i for i in d.viewkeys()]"
         a = "[i for i in d.keys()]"
         self.check(b, a)
     def test_15(self):
+    def test_29(self):
         b = "(i for i in d.viewkeys())"
         a = "(i for i in d.keys())"
         self.check(b, a)
     def test_17(self):
+    def test_30(self):
         b = "iter(d.viewkeys())"
         a = "iter(d.keys())"
         self.check(b, a)
     def test_18(self):
+    def test_31(self):
         b = "list(d.viewkeys())"
         a = "list(d.keys())"
         self.check(b, a)
     def test_19(self):
+    def test_32(self):
         b = "sorted(d.viewkeys())"
         a = "sorted(d.keys())"
 …
         for call in fixer_util.consuming_calls:
             self.unchanged("a = %s(range(10))" % call)
+class Test_xrange_with_reduce(FixerTestCase):
+    def setUp(self):
+        super(Test_xrange_with_reduce, self).setUp(["xrange", "reduce"])
+    def test_double_transform(self):
+        b = """reduce(x, xrange(5))"""
+        a = """from functools import reduce
+reduce(x, range(5))"""
+        self.check(b, a)
 class Test_raw_input(FixerTestCase):
 …
                     a = "from %s import %s as foo_bar" % (new, member)
                     self.check(b, a)
+                    b = "from %s import %s as blah, %s" % (old, member, member)
+                    a = "from %s import %s as blah, %s" % (new, member, member)
+                    self.check(b, a)
     def test_star(self):
 …
             s = "from %s import *" % old
             self.warns_unchanged(s, "Cannot handle star imports")
+    def test_indented(self):
+        b = """
+def foo():
+    from urllib import urlencode, urlopen
+"""
+        a = """
+def foo():
+    from urllib.parse import urlencode
+    from urllib.request import urlopen
+"""
+        self.check(b, a)
+        b = """
+def foo():
+    other()
+    from urllib import urlencode, urlopen
+"""
+        a = """
+def foo():
+    other()
+    from urllib.parse import urlencode
+    from urllib.request import urlopen
+"""
+        self.check(b, a)
     def test_import_module_usage(self):
 …
         self.check(b, a)
+    def test_native_literal_escape_u(self):
+        b = """'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        a = """'\\\\\\\\u20ac\\\\U0001d121\\\\u20ac'"""
+        self.check(b, a)
+        b = """r'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        a = """r'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        self.check(b, a)
+    def test_bytes_literal_escape_u(self):
+        b = """b'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        a = """b'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        self.check(b, a)
+        b = """br'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        a = """br'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        self.check(b, a)
+    def test_unicode_literal_escape_u(self):
+        b = """u'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        a = """'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        self.check(b, a)
+        b = """ur'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        a = """r'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        self.check(b, a)
+    def test_native_unicode_literal_escape_u(self):
+        f = 'from __future__ import unicode_literals\n'
+        b = f + """'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        a = f + """'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        self.check(b, a)
+        b = f + """r'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        a = f + """r'\\\\\\u20ac\\U0001d121\\\\u20ac'"""
+        self.check(b, a)
 class Test_callable(FixerTestCase):
     fixer = "callable"
 …
         a = """sorted(filter(f, 'abc'), key=blah)[0]"""
         self.unchanged(a)
+        a = """enumerate(filter(f, 'abc'))"""
+        self.unchanged(a)
+        a = """enumerate(filter(f, 'abc'), start=1)"""
+        self.unchanged(a)
         a = """for i in filter(f, 'abc'): pass"""
         self.unchanged(a)
 …
         a = """sorted(map(f, 'abc'), key=blah)[0]"""
         self.unchanged(a)
+        a = """enumerate(map(f, 'abc'))"""
+        self.unchanged(a)
+        a = """enumerate(map(f, 'abc'), start=1)"""
+        self.unchanged(a)
         a = """for i in map(f, 'abc'): pass"""
         self.unchanged(a)
 …
         self.unchanged(a)
         a = """sorted(zip(a, b), key=blah)[0]"""
+        self.unchanged(a)
+        a = """enumerate(zip(a, b))"""
+        self.unchanged(a)
+        a = """enumerate(zip(a, b), start=1)"""
         self.unchanged(a)
         a = """for i in zip(a, b): pass"""
 …
         self.checkall(b, a)
     def test_2(self):
+    def test_qualified(self):
         b = """itertools.ifilterfalse(a, b)"""
         a = """itertools.filterfalse(a, b)"""
         self.check(b, a)
+    def test_4(self):
+        b = """itertools.izip_longest(a, b)"""
+        a = """itertools.zip_longest(a, b)"""
+        self.check(b, a)
+    def test_2(self):
         b = """ifilterfalse(a, b)"""
         a = """filterfalse(a, b)"""
+        self.check(b, a)
+        b = """izip_longest(a, b)"""
+        a = """zip_longest(a, b)"""
         self.check(b, a)
 …
         self.check(b, a)
+        b = """    itertools.izip_longest(a, b)"""
+        a = """    itertools.zip_longest(a, b)"""
+        self.check(b, a)
     def test_run_order(self):
         self.assert_runs_after('map', 'zip', 'filter')
 class Test_itertools_imports(FixerTestCase):
 …
         self.check(b, a)
+        b = "from itertools import chain, imap, izip"
+        a = "from itertools import chain"
+        self.check(b, a)
     def test_comments(self):
         b = "#foo\nfrom itertools import imap, izip"
 …
         self.unchanged(s)
+    def test_ifilter(self):
+        b = "from itertools import ifilterfalse"
+        a = "from itertools import filterfalse"
+        self.check(b, a)
+        b = "from itertools import imap, ifilterfalse, foo"
+        a = "from itertools import filterfalse, foo"
+        self.check(b, a)
+        b = "from itertools import bar, ifilterfalse, foo"
+        a = "from itertools import bar, filterfalse, foo"
+        self.check(b, a)
+    def test_ifilter_and_zip_longest(self):
+        for name in "filterfalse", "zip_longest":
+            b = "from itertools import i%s" % (name,)
+            a = "from itertools import %s" % (name,)
+            self.check(b, a)
+            b = "from itertools import imap, i%s, foo" % (name,)
+            a = "from itertools import %s, foo" % (name,)
+            self.check(b, a)
+            b = "from itertools import bar, i%s, foo" % (name,)
+            a = "from itertools import bar, %s, foo" % (name,)
+            self.check(b, a)
+    def test_import_star(self):
+        s = "from itertools import *"
+        self.unchanged(s)
 …
             return self.always_exists or (name in self.present_files)
         from ..fixes import fix_import
+        from lib2to3.fixes import fix_import
         fix_import.exists = fake_exists
 …
         self.unchanged(s)
+    def test_with_absolute_import_enabled(self):
+        s = "from __future__ import absolute_import\nimport bar"
+        self.always_exists = False
+        self.present_files = set(["__init__.py", "bar.py"])
+        self.unchanged(s)
     def test_in_package(self):
         b = "import bar"
 …
         self.present_files = set(["__init__.py", "bar" + os.path.sep])
         self.check(b, a)
+    def test_already_relative_import(self):
+        s = "from . import bar"
+        self.unchanged(s)
     def test_comments_and_indent(self):
 …
         self.check(b, a)
+        b = "operator .sequenceIncludes(x, y)"
+        a = "operator .contains(x, y)"
+        self.check(b, a)
+        b = "operator.  sequenceIncludes(x, y)"
+        a = "operator.  contains(x, y)"
+        self.check(b, a)
+    def test_operator_isSequenceType(self):
+        b = "operator.isSequenceType(x)"
+        a = "import collections\nisinstance(x, collections.Sequence)"
+        self.check(b, a)
+    def test_operator_isMappingType(self):
+        b = "operator.isMappingType(x)"
+        a = "import collections\nisinstance(x, collections.Mapping)"
+        self.check(b, a)
+    def test_operator_isNumberType(self):
+        b = "operator.isNumberType(x)"
+        a = "import numbers\nisinstance(x, numbers.Number)"
+        self.check(b, a)
+    def test_operator_repeat(self):
+        b = "operator.repeat(x, n)"
+        a = "operator.mul(x, n)"
+        self.check(b, a)
+        b = "operator .repeat(x, n)"
+        a = "operator .mul(x, n)"
+        self.check(b, a)
+        b = "operator.  repeat(x, n)"
+        a = "operator.  mul(x, n)"
+        self.check(b, a)
+    def test_operator_irepeat(self):
+        b = "operator.irepeat(x, n)"
+        a = "operator.imul(x, n)"
+        self.check(b, a)
+        b = "operator .irepeat(x, n)"
+        a = "operator .imul(x, n)"
+        self.check(b, a)
+        b = "operator.  irepeat(x, n)"
+        a = "operator.  imul(x, n)"
+        self.check(b, a)
     def test_bare_isCallable(self):
         s = "isCallable(x)"
+        self.warns_unchanged(s, "You should use hasattr(x, '__call__') here.")
+        t = "You should use 'hasattr(x, '__call__')' here."
+        self.warns_unchanged(s, t)
     def test_bare_sequenceIncludes(self):
         s = "sequenceIncludes(x, y)"
+        self.warns_unchanged(s, "You should use operator.contains here.")
+        t = "You should use 'operator.contains(x, y)' here."
+        self.warns_unchanged(s, t)
+    def test_bare_operator_isSequenceType(self):
+        s = "isSequenceType(z)"
+        t = "You should use 'isinstance(z, collections.Sequence)' here."
+        self.warns_unchanged(s, t)
+    def test_bare_operator_isMappingType(self):
+        s = "isMappingType(x)"
+        t = "You should use 'isinstance(x, collections.Mapping)' here."
+        self.warns_unchanged(s, t)
+    def test_bare_operator_isNumberType(self):
+        s = "isNumberType(y)"
+        t = "You should use 'isinstance(y, numbers.Number)' here."
+        self.warns_unchanged(s, t)
+    def test_bare_operator_repeat(self):
+        s = "repeat(x, n)"
+        t = "You should use 'operator.mul(x, n)' here."
+        self.warns_unchanged(s, t)
+    def test_bare_operator_irepeat(self):
+        s = "irepeat(y, 187)"
+        t = "You should use 'operator.imul(y, 187)' here."
+        self.warns_unchanged(s, t)
+class Test_exitfunc(FixerTestCase):
+    fixer = "exitfunc"
+    def test_simple(self):
+        b = """
+            import sys
+            sys.exitfunc = my_atexit
+            """
+        a = """
+            import sys
+            import atexit
+            atexit.register(my_atexit)
+            """
+        self.check(b, a)
+    def test_names_import(self):
+        b = """
+            import sys, crumbs
+            sys.exitfunc = my_func
+            """
+        a = """
+            import sys, crumbs, atexit
+            atexit.register(my_func)
+            """
+        self.check(b, a)
+    def test_complex_expression(self):
+        b = """
+            import sys
+            sys.exitfunc = do(d)/a()+complex(f=23, g=23)*expression
+            """
+        a = """
+            import sys
+            import atexit
+            atexit.register(do(d)/a()+complex(f=23, g=23)*expression)
+            """
+        self.check(b, a)
+    def test_comments(self):
+        b = """
+            import sys # Foo
+            sys.exitfunc = f # Blah
+            """
+        a = """
+            import sys
+            import atexit # Foo
+            atexit.register(f) # Blah
+            """
+        self.check(b, a)
+        b = """
+            import apples, sys, crumbs, larry # Pleasant comments
+            sys.exitfunc = func
+            """
+        a = """
+            import apples, sys, crumbs, larry, atexit # Pleasant comments
+            atexit.register(func)
+            """
+        self.check(b, a)
+    def test_in_a_function(self):
+        b = """
+            import sys
+            def f():
+                sys.exitfunc = func
+            """
+        a = """
+            import sys
+            import atexit
+            def f():
+                atexit.register(func)
+             """
+        self.check(b, a)
+    def test_no_sys_import(self):
+        b = """sys.exitfunc = f"""
+        a = """atexit.register(f)"""
+        msg = ("Can't find sys import; Please add an atexit import at the "
+            "top of your file.")
+        self.warns(b, a, msg)
+    def test_unchanged(self):
+        s = """f(sys.exitfunc)"""
+        self.unchanged(s)

python/trunk/Lib/lib2to3/tests/test_main.py

-              r2
+              r391
 import codecs
 import logging
+import os
+import re
+import shutil
 import StringIO
+import sys
+import tempfile
 import unittest
 …
+TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
+PY2_TEST_MODULE = os.path.join(TEST_DATA_DIR, "py2_test_grammar.py")
 class TestMain(unittest.TestCase):
+    if not hasattr(unittest.TestCase, 'assertNotRegex'):
+        # This method was only introduced in 3.2.
+        def assertNotRegex(self, text, regexp, msg=None):
+            import re
+            if not hasattr(regexp, 'search'):
+                regexp = re.compile(regexp)
+            if regexp.search(text):
+                self.fail("regexp %s MATCHED text %r" % (regexp.pattern, text))
+    def setUp(self):
+        self.temp_dir = None  # tearDown() will rmtree this directory if set.
     def tearDown(self):
         # Clean up logging configuration down by main.
         del logging.root.handlers[:]
+        if self.temp_dir:
+            shutil.rmtree(self.temp_dir)
     def run_2to3_capture(self, args, in_capture, out_capture, err_capture):
 …
         self.assertTrue("WARNING: couldn't encode <stdin>'s diff for "
                         "your terminal" in err.getvalue())
+    def setup_test_source_trees(self):
+        """Setup a test source tree and output destination tree."""
+        self.temp_dir = tempfile.mkdtemp()  # tearDown() cleans this up.
+        self.py2_src_dir = os.path.join(self.temp_dir, "python2_project")
+        self.py3_dest_dir = os.path.join(self.temp_dir, "python3_project")
+        os.mkdir(self.py2_src_dir)
+        os.mkdir(self.py3_dest_dir)
+        # Turn it into a package with a few files.
+        self.setup_files = []
+        open(os.path.join(self.py2_src_dir, "__init__.py"), "w").close()
+        self.setup_files.append("__init__.py")
+        shutil.copy(PY2_TEST_MODULE, self.py2_src_dir)
+        self.setup_files.append(os.path.basename(PY2_TEST_MODULE))
+        self.trivial_py2_file = os.path.join(self.py2_src_dir, "trivial.py")
+        self.init_py2_file = os.path.join(self.py2_src_dir, "__init__.py")
+        with open(self.trivial_py2_file, "w") as trivial:
+            trivial.write("print 'I need a simple conversion.'")
+        self.setup_files.append("trivial.py")
+    def test_filename_changing_on_output_single_dir(self):
+        """2to3 a single directory with a new output dir and suffix."""
+        self.setup_test_source_trees()
+        out = StringIO.StringIO()
+        err = StringIO.StringIO()
+        suffix = "TEST"
+        ret = self.run_2to3_capture(
+                ["-n", "--add-suffix", suffix, "--write-unchanged-files",
+                 "--no-diffs", "--output-dir",
+                 self.py3_dest_dir, self.py2_src_dir],
+                StringIO.StringIO(""), out, err)
+        self.assertEqual(ret, 0)
+        stderr = err.getvalue()
+        self.assertIn(" implies -w.", stderr)
+        self.assertIn(
+                "Output in %r will mirror the input directory %r layout" % (
+                        self.py3_dest_dir, self.py2_src_dir), stderr)
+        self.assertEqual(set(name+suffix for name in self.setup_files),
+                         set(os.listdir(self.py3_dest_dir)))
+        for name in self.setup_files:
+            self.assertIn("Writing converted %s to %s" % (
+                    os.path.join(self.py2_src_dir, name),
+                    os.path.join(self.py3_dest_dir, name+suffix)), stderr)
+        sep = re.escape(os.sep)
+        self.assertRegexpMatches(
+                stderr, r"No changes to .*/__init__\.py".replace("/", sep))
+        self.assertNotRegex(
+                stderr, r"No changes to .*/trivial\.py".replace("/", sep))
+    def test_filename_changing_on_output_two_files(self):
+        """2to3 two files in one directory with a new output dir."""
+        self.setup_test_source_trees()
+        err = StringIO.StringIO()
+        py2_files = [self.trivial_py2_file, self.init_py2_file]
+        expected_files = set(os.path.basename(name) for name in py2_files)
+        ret = self.run_2to3_capture(
+                ["-n", "-w", "--write-unchanged-files",
+                 "--no-diffs", "--output-dir", self.py3_dest_dir] + py2_files,
+                StringIO.StringIO(""), StringIO.StringIO(), err)
+        self.assertEqual(ret, 0)
+        stderr = err.getvalue()
+        self.assertIn(
+                "Output in %r will mirror the input directory %r layout" % (
+                        self.py3_dest_dir, self.py2_src_dir), stderr)
+        self.assertEqual(expected_files, set(os.listdir(self.py3_dest_dir)))
+    def test_filename_changing_on_output_single_file(self):
+        """2to3 a single file with a new output dir."""
+        self.setup_test_source_trees()
+        err = StringIO.StringIO()
+        ret = self.run_2to3_capture(
+                ["-n", "-w", "--no-diffs", "--output-dir", self.py3_dest_dir,
+                 self.trivial_py2_file],
+                StringIO.StringIO(""), StringIO.StringIO(), err)
+        self.assertEqual(ret, 0)
+        stderr = err.getvalue()
+        self.assertIn(
+                "Output in %r will mirror the input directory %r layout" % (
+                        self.py3_dest_dir, self.py2_src_dir), stderr)
+        self.assertEqual(set([os.path.basename(self.trivial_py2_file)]),
+                         set(os.listdir(self.py3_dest_dir)))
+if __name__ == '__main__':
+    unittest.main()

python/trunk/Lib/lib2to3/tests/test_parser.py

-              r2
+              r391
 """
+from __future__ import with_statement
 # Testing imports
 from . import support
 …
 # Python imports
 import os
-import io
 import sys
 …
 from lib2to3.pgen2 import tokenize
 from ..pgen2.parse import ParseError
+from lib2to3.pygram import python_symbols as syms
+class TestDriver(support.TestCase):
+    def test_formfeed(self):
+        s = """print 1\n\x0Cprint 2\n"""
+        t = driver.parse_string(s)
+        self.assertEqual(t.children[0].children[0].type, syms.print_stmt)
+        self.assertEqual(t.children[1].children[0].type, syms.print_stmt)
 …
 # Adapated from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef
+# Adaptated from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef
 class TestFunctionAnnotations(GrammarTest):
     def test_1(self):
 …
             self.assertTrue(encoding is not None,
                             "can't detect encoding for %s" % filepath)
             with io.open(filepath, "r", encoding=encoding) as fp:
+            with open(filepath, "r") as fp:
                 source = fp.read()
+                source = source.decode(encoding)
             tree = driver.parse_string(source)
             new = unicode(tree)
 …
 def diff(fn, result, encoding):
     f = io.open("@", "w", encoding=encoding)
+    f = open("@", "w")
     try:
         f.write(result)
+        f.write(result.encode(encoding))
     finally:
         f.close()
     try:
+        return os.system("diff -u %r @" % fn)
+        fn = fn.replace('"', '\\"')
+        return os.system('diff -u "%s" @' % fn)
     finally:
         os.remove("@")

python/trunk/Lib/lib2to3/tests/test_pytree.py

-              r2
+              r391
 """
+from __future__ import with_statement
+import sys
 import warnings
 …
     """Unit tests for nodes (Base, Leaf, Node)."""
+    def test_deprecated_prefix_methods(self):
+        l = pytree.Leaf(100, "foo")
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always", DeprecationWarning)
+            self.assertEqual(l.get_prefix(), "")
+            l.set_prefix("hi")
+        self.assertEqual(l.prefix, "hi")
+        self.assertEqual(len(w), 2)
+        for warning in w:
+            self.assertTrue(warning.category is DeprecationWarning)
+        self.assertEqual(str(w[0].message), "get_prefix() is deprecated; " \
+                             "use the prefix property")
+        self.assertEqual(str(w[1].message), "set_prefix() is deprecated; " \
+                             "use the prefix property")
+    if sys.version_info >= (2,6):
+        # warnings.catch_warnings is new in 2.6.
+        def test_deprecated_prefix_methods(self):
+            l = pytree.Leaf(100, "foo")
+            with warnings.catch_warnings(record=True) as w:
+                warnings.simplefilter("always", DeprecationWarning)
+                self.assertEqual(l.get_prefix(), "")
+                l.set_prefix("hi")
+            self.assertEqual(l.prefix, "hi")
+            self.assertEqual(len(w), 2)
+            for warning in w:
+                self.assertTrue(warning.category is DeprecationWarning)
+            self.assertEqual(str(w[0].message), "get_prefix() is deprecated; " \
+                                 "use the prefix property")
+            self.assertEqual(str(w[1].message), "set_prefix() is deprecated; " \
+                                 "use the prefix property")
     def test_instantiate_base(self):
 …
         self.assertTrue(isinstance(n1.children, list))
+    def test_leaves(self):
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "bar")
+        l3 = pytree.Leaf(100, "fooey")
+        n2 = pytree.Node(1000, [l1, l2])
+        n3 = pytree.Node(1000, [l3])
+        n1 = pytree.Node(1000, [n2, n3])
+        self.assertEqual(list(n1.leaves()), [l1, l2, l3])
+    def test_depth(self):
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "bar")
+        n2 = pytree.Node(1000, [l1, l2])
+        n3 = pytree.Node(1000, [])
+        n1 = pytree.Node(1000, [n2, n3])
+        self.assertEqual(l1.depth(), 2)
+        self.assertEqual(n3.depth(), 1)
+        self.assertEqual(n1.depth(), 0)
     def test_post_order(self):
         l1 = pytree.Leaf(100, "foo")
         l2 = pytree.Leaf(100, "bar")
+        n1 = pytree.Node(1000, [l1, l2])
+        self.assertEqual(list(n1.post_order()), [l1, l2, n1])
+        l3 = pytree.Leaf(100, "fooey")
+        c1 = pytree.Node(1000, [l1, l2])
+        n1 = pytree.Node(1000, [c1, l3])
+        self.assertEqual(list(n1.post_order()), [l1, l2, c1, l3, n1])
     def test_pre_order(self):
         l1 = pytree.Leaf(100, "foo")
         l2 = pytree.Leaf(100, "bar")
+        n1 = pytree.Node(1000, [l1, l2])
+        self.assertEqual(list(n1.pre_order()), [n1, l1, l2])
+        l3 = pytree.Leaf(100, "fooey")
+        c1 = pytree.Node(1000, [l1, l2])
+        n1 = pytree.Node(1000, [c1, l3])
+        self.assertEqual(list(n1.pre_order()), [n1, c1, l1, l2, l3])
     def test_changed(self):

python/trunk/Lib/lib2to3/tests/test_refactor.py

-              r2
+              r391
 Unit tests for refactor.py.
 """
+from __future__ import with_statement
 import sys
 …
                         pygram.python_grammar_no_print_statement)
+    def test_write_unchanged_files_option(self):
+        rt = self.rt()
+        self.assertFalse(rt.write_unchanged_files)
+        rt = self.rt({"write_unchanged_files" : True})
+        self.assertTrue(rt.write_unchanged_files)
     def test_fixer_loading_helpers(self):
         contents = ["explicit", "first", "last", "parrot", "preorder"]
 …
                          ["myfixes.fix_" + name for name in contents])
+    def test_detect_future_print(self):
+        run = refactor._detect_future_print
+        self.assertFalse(run(""))
+        self.assertTrue(run("from __future__ import print_function"))
+        self.assertFalse(run("from __future__ import generators"))
+        self.assertFalse(run("from __future__ import generators, feature"))
+        input = "from __future__ import generators, print_function"
+        self.assertTrue(run(input))
+        input ="from __future__ import print_function, generators"
+        self.assertTrue(run(input))
+        input = "from __future__ import (print_function,)"
+        self.assertTrue(run(input))
+        input = "from __future__ import (generators, print_function)"
+        self.assertTrue(run(input))
+        input = "from __future__ import (generators, nested_scopes)"
+        self.assertFalse(run(input))
+        input = """from __future__ import generators
+    def test_detect_future_features(self):
+        run = refactor._detect_future_features
+        fs = frozenset
+        empty = fs()
+        self.assertEqual(run(""), empty)
+        self.assertEqual(run("from __future__ import print_function"),
+                         fs(("print_function",)))
+        self.assertEqual(run("from __future__ import generators"),
+                         fs(("generators",)))
+        self.assertEqual(run("from __future__ import generators, feature"),
+                         fs(("generators", "feature")))
+        inp = "from __future__ import generators, print_function"
+        self.assertEqual(run(inp), fs(("generators", "print_function")))
+        inp ="from __future__ import print_function, generators"
+        self.assertEqual(run(inp), fs(("print_function", "generators")))
+        inp = "from __future__ import (print_function,)"
+        self.assertEqual(run(inp), fs(("print_function",)))
+        inp = "from __future__ import (generators, print_function)"
+        self.assertEqual(run(inp), fs(("generators", "print_function")))
+        inp = "from __future__ import (generators, nested_scopes)"
+        self.assertEqual(run(inp), fs(("generators", "nested_scopes")))
+        inp = """from __future__ import generators
 from __future__ import print_function"""
+        self.assertTrue(run(input))
+        self.assertFalse(run("from"))
+        self.assertFalse(run("from 4"))
+        self.assertFalse(run("from x"))
+        self.assertFalse(run("from x 5"))
+        self.assertFalse(run("from x im"))
+        self.assertFalse(run("from x import"))
+        self.assertFalse(run("from x import 4"))
+        input = "'docstring'\nfrom __future__ import print_function"
+        self.assertTrue(run(input))
+        input = "'docstring'\n'somng'\nfrom __future__ import print_function"
+        self.assertFalse(run(input))
+        input = "# comment\nfrom __future__ import print_function"
+        self.assertTrue(run(input))
+        input = "# comment\n'doc'\nfrom __future__ import print_function"
+        self.assertTrue(run(input))
+        input = "class x: pass\nfrom __future__ import print_function"
+        self.assertFalse(run(input))
+        self.assertEqual(run(inp), fs(("generators", "print_function")))
+        invalid = ("from",
+                   "from 4",
+                   "from x",
+                   "from x 5",
+                   "from x im",
+                   "from x import",
+                   "from x import 4",
+                   )
+        for inp in invalid:
+            self.assertEqual(run(inp), empty)
+        inp = "'docstring'\nfrom __future__ import print_function"
+        self.assertEqual(run(inp), fs(("print_function",)))
+        inp = "'docstring'\n'somng'\nfrom __future__ import print_function"
+        self.assertEqual(run(inp), empty)
+        inp = "# comment\nfrom __future__ import print_function"
+        self.assertEqual(run(inp), fs(("print_function",)))
+        inp = "# comment\n'doc'\nfrom __future__ import print_function"
+        self.assertEqual(run(inp), fs(("print_function",)))
+        inp = "class x: pass\nfrom __future__ import print_function"
+        self.assertEqual(run(inp), empty)
     def test_get_headnode_dict(self):
 …
         self.assertEqual(results, expected)
+    def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS):
+    def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS,
+                               options=None, mock_log_debug=None,
+                               actually_write=True):
+        tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor")
+        self.addCleanup(shutil.rmtree, tmpdir)
+        # make a copy of the tested file that we can write to
+        shutil.copy(test_file, tmpdir)
+        test_file = os.path.join(tmpdir, os.path.basename(test_file))
+        os.chmod(test_file, 0o644)
         def read_file():
             with open(test_file, "rb") as fp:
                 return fp.read()
         old_contents = read_file()
+        rt = self.rt(fixers=fixers)
+        rt = self.rt(fixers=fixers, options=options)
+        if mock_log_debug:
+            rt.log_debug = mock_log_debug
         rt.refactor_file(test_file)
         self.assertEqual(old_contents, read_file())
+        try:
+            rt.refactor_file(test_file, True)
+            new_contents = read_file()
+            self.assertNotEqual(old_contents, new_contents)
+        finally:
+            with open(test_file, "wb") as fp:
+                fp.write(old_contents)
+        if not actually_write:
+            return
+        rt.refactor_file(test_file, True)
+        new_contents = read_file()
+        self.assertNotEqual(old_contents, new_contents)
         return new_contents
 …
         test_file = os.path.join(FIXER_DIR, "parrot_example.py")
         self.check_file_refactoring(test_file, _DEFAULT_FIXERS)
+    def test_refactor_file_write_unchanged_file(self):
+        test_file = os.path.join(FIXER_DIR, "parrot_example.py")
+        debug_messages = []
+        def recording_log_debug(msg, *args):
+            debug_messages.append(msg % args)
+        self.check_file_refactoring(test_file, fixers=(),
+                                    options={"write_unchanged_files": True},
+                                    mock_log_debug=recording_log_debug,
+                                    actually_write=False)
+        # Testing that it logged this message when write=False was passed is
+        # sufficient to see that it did not bail early after "No changes".
+        message_regex = r"Not writing changes to .*%s%s" % (
+                os.sep, os.path.basename(test_file))
+        for message in debug_messages:
+            if "Not writing changes" in message:
+                self.assertRegexpMatches(message, message_regex)
+                break
+        else:
+            self.fail("%r not matched in %r" % (message_regex, debug_messages))
     def test_refactor_dir(self):
 …
                 ".dumb",
                 ".after.py",
+                "notpy.npy",
                 "sappy"]
         expected = ["hi.py"]
 …
         fn = os.path.join(TEST_DATA_DIR, "different_encoding.py")
         self.check_file_refactoring(fn)
+    def test_false_file_encoding(self):
+        fn = os.path.join(TEST_DATA_DIR, "false_encoding.py")
+        data = self.check_file_refactoring(fn)
     def test_bom(self):

python/trunk/Lib/lib2to3/tests/test_util.py

-              r2
+              r391
     def test_from_import(self):
         node = parse('bar()')
         fixer_util.touch_import("cgi", "escape", node)
         self.assertEqual(str(node), 'from cgi import escape\nbar()\n\n')
+        fixer_util.touch_import("html", "escape", node)
+        self.assertEqual(str(node), 'from html import escape\nbar()\n\n')
     def test_name_import(self):
 …
         fixer_util.touch_import(None, "cgi", node)
         self.assertEqual(str(node), 'import cgi\nbar()\n\n')
+class Test_find_indentation(support.TestCase):
+    def test_nothing(self):
+        fi = fixer_util.find_indentation
+        node = parse("node()")
+        self.assertEqual(fi(node), u"")
+        node = parse("")
+        self.assertEqual(fi(node), u"")
+    def test_simple(self):
+        fi = fixer_util.find_indentation
+        node = parse("def f():\n    x()")
+        self.assertEqual(fi(node), u"")
+        self.assertEqual(fi(node.children[0].children[4].children[2]), u"    ")
+        node = parse("def f():\n    x()\n    y()")
+        self.assertEqual(fi(node.children[0].children[4].children[4]), u"    ")

/python/vendor/Python-2.7.6	merged	eligible
/python/vendor/current	merged	eligible

Context Navigation

Legend:

Download in other formats: