Changeset 391 for python/trunk/Lib/test/test_tokenize.py

Timestamp:

Mar 19, 2014, 11:31:01 PM (11 years ago)

Author:

dmik

Message:

python: Merge vendor 2.7.6 to trunk.

Location:

python/trunk

Files:

• . (modified) (1 prop)
• Lib/test/test_tokenize.py (modified) (5 diffs)

python/trunk/Lib/test/test_tokenize.py

@@ -57 +57 @@
     ...
 
-There are some standard formattig practises that are easy to get right.
+There are some standard formatting practices that are easy to get right.
 
     >>> roundtrip("if x == 1:\\n"
@@ -279 +279 @@
     STRING     'UR"ABC"'     (1, 34) (1, 41)
 
+    >>> dump_tokens("b'abc' + B'abc'")
+    STRING     "b'abc'"      (1, 0) (1, 6)
+    OP         '+'           (1, 7) (1, 8)
+    STRING     "B'abc'"      (1, 9) (1, 15)
+    >>> dump_tokens('b"abc" + B"abc"')
+    STRING     'b"abc"'      (1, 0) (1, 6)
+    OP         '+'           (1, 7) (1, 8)
+    STRING     'B"abc"'      (1, 9) (1, 15)
+    >>> dump_tokens("br'abc' + bR'abc' + Br'abc' + BR'abc'")
+    STRING     "br'abc'"     (1, 0) (1, 7)
+    OP         '+'           (1, 8) (1, 9)
+    STRING     "bR'abc'"     (1, 10) (1, 17)
+    OP         '+'           (1, 18) (1, 19)
+    STRING     "Br'abc'"     (1, 20) (1, 27)
+    OP         '+'           (1, 28) (1, 29)
+    STRING     "BR'abc'"     (1, 30) (1, 37)
+    >>> dump_tokens('br"abc" + bR"abc" + Br"abc" + BR"abc"')
+    STRING     'br"abc"'     (1, 0) (1, 7)
+    OP         '+'           (1, 8) (1, 9)
+    STRING     'bR"abc"'     (1, 10) (1, 17)
+    OP         '+'           (1, 18) (1, 19)
+    STRING     'Br"abc"'     (1, 20) (1, 27)
+    OP         '+'           (1, 28) (1, 29)
+    STRING     'BR"abc"'     (1, 30) (1, 37)
+
 Operators
 
@@ -494 +519 @@
 
 Test roundtrip on random python modules.
-pass the '-ucompiler' option to process the full directory.
+pass the '-ucpu' option to process the full directory.
 
     >>>
@@ -500 +525 @@
     >>> testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
 
-    >>> if not test_support.is_resource_enabled("compiler"):
+    >>> if not test_support.is_resource_enabled("cpu"):
     ...     testfiles = random.sample(testfiles, 10)
     ...
@@ -509 +534 @@
     ... else: True
     True
+
+Evil tabs
+    >>> dump_tokens("def f():\\n\\tif x\\n        \\tpass")
+    NAME       'def'         (1, 0) (1, 3)
+    NAME       'f'           (1, 4) (1, 5)
+    OP         '('           (1, 5) (1, 6)
+    OP         ')'           (1, 6) (1, 7)
+    OP         ':'           (1, 7) (1, 8)
+    NEWLINE    '\\n'          (1, 8) (1, 9)
+    INDENT     '\\t'          (2, 0) (2, 1)
+    NAME       'if'          (2, 1) (2, 3)
+    NAME       'x'           (2, 4) (2, 5)
+    NEWLINE    '\\n'          (2, 5) (2, 6)
+    INDENT     '        \\t'  (3, 0) (3, 9)
+    NAME       'pass'        (3, 9) (3, 13)
+    DEDENT     ''            (4, 0) (4, 0)
+    DEDENT     ''            (4, 0) (4, 0)
+
+Pathological whitespace (http://bugs.python.org/issue16152)
+    >>> dump_tokens("@          ")
+    OP         '@'           (1, 0) (1, 1)
 """
 
 
 from test import test_support
-from tokenize import (tokenize, untokenize, generate_tokens, NUMBER, NAME, OP,
+from tokenize import (untokenize, generate_tokens, NUMBER, NAME, OP,
                      STRING, ENDMARKER, tok_name)
 from StringIO import StringIO