Changeset 391 for python/trunk/Lib/lib2to3/pgen2/tokenize.py

Timestamp:

Mar 19, 2014, 11:31:01 PM (11 years ago)

Author:

dmik

Message:

python: Merge vendor 2.7.6 to trunk.

Location:

python/trunk

Files:

• . (modified) (1 prop)
• Lib/lib2to3/pgen2/tokenize.py (modified) (9 diffs)

python/trunk/Lib/lib2to3/pgen2/tokenize.py

@@ -38 +38 @@
            "generate_tokens", "untokenize"]
 del token
+
+try:
+    bytes
+except NameError:
+    # Support bytes type in Python <= 2.5, so 2to3 turns itself into
+    # valid Python 3 code.
+    bytes = str
 
 def group(*choices): return '(' + '|'.join(choices) + ')'
@@ -230 +237 @@
             toks_append(tokval)
 
-cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
+cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)')
 
 def _get_normal_name(orig_enc):
@@ -254 +261 @@
 
     It detects the encoding from the presence of a utf-8 bom or an encoding
-    cookie as specified in pep-0263. If both a bom and a cookie are present,
-    but disagree, a SyntaxError will be raised. If the encoding cookie is an
-    invalid charset, raise a SyntaxError.
+    cookie as specified in pep-0263. If both a bom and a cookie are present, but
+    disagree, a SyntaxError will be raised. If the encoding cookie is an invalid
+    charset, raise a SyntaxError.  Note that if a utf-8 bom is found,
+    'utf-8-sig' is returned.
 
     If no encoding is specified, then the default of 'utf-8' will be returned.
@@ -262 +270 @@
     bom_found = False
     encoding = None
+    default = 'utf-8'
     def read_or_stop():
         try:
             return readline()
         except StopIteration:
-            return b''
+            return bytes()
 
     def find_cookie(line):
@@ -273 +282 @@
         except UnicodeDecodeError:
             return None
-
-        matches = cookie_re.findall(line_string)
-        if not matches:
+        match = cookie_re.match(line_string)
+        if not match:
             return None
-        encoding = _get_normal_name(matches[0])
+        encoding = _get_normal_name(match.group(1))
         try:
             codec = lookup(encoding)
@@ -288 +296 @@
                 # This behaviour mimics the Python interpreter
                 raise SyntaxError('encoding problem: utf-8')
-            else:
-                # Allow it to be properly encoded and decoded.
-                encoding = 'utf-8-sig'
+            encoding += '-sig'
         return encoding
 
@@ -297 +303 @@
         bom_found = True
         first = first[3:]
+        default = 'utf-8-sig'
     if not first:
-        return 'utf-8', []
+        return default, []
 
     encoding = find_cookie(first)
@@ -306 +313 @@
     second = read_or_stop()
     if not second:
-        return 'utf-8', [first]
+        return default, [first]
 
     encoding = find_cookie(second)
@@ -312 +319 @@
         return encoding, [first, second]
 
-    return 'utf-8', [first, second]
+    return default, [first, second]
 
 def untokenize(iterable):