source: python/trunk/Lib/json/decoder.py@ 389

Last change on this file since 389 was 2, checked in by Yuri Dario, 15 years ago

Initial import for vendor code.
  • Property svn:eol-style set to native
File size: 10.8 KB
Line 
1"""Implementation of JSONDecoder
2"""
3
4import re
5import sys
6
7from json.scanner import Scanner, pattern
8try:
9 from _json import scanstring as c_scanstring
10except ImportError:
11 c_scanstring = None
12
13__all__ = ['JSONDecoder']
14
15FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
16
17NaN, PosInf, NegInf = float('nan'), float('inf'), float('-inf')
18
19
20def linecol(doc, pos):
21 lineno = doc.count('\n', 0, pos) + 1
22 if lineno == 1:
23 colno = pos
24 else:
25 colno = pos - doc.rindex('\n', 0, pos)
26 return lineno, colno
27
28
29def errmsg(msg, doc, pos, end=None):
30 lineno, colno = linecol(doc, pos)
31 if end is None:
32 fmt = '{0}: line {1} column {2} (char {3})'
33 return fmt.format(msg, lineno, colno, pos)
34 endlineno, endcolno = linecol(doc, end)
35 fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
36 return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
37
38
39_CONSTANTS = {
40 '-Infinity': NegInf,
41 'Infinity': PosInf,
42 'NaN': NaN,
43 'true': True,
44 'false': False,
45 'null': None,
46}
47
48
49def JSONConstant(match, context, c=_CONSTANTS):
50 s = match.group(0)
51 fn = getattr(context, 'parse_constant', None)
52 if fn is None:
53 rval = c[s]
54 else:
55 rval = fn(s)
56 return rval, None
57pattern('(-?Infinity|NaN|true|false|null)')(JSONConstant)
58
59
60def JSONNumber(match, context):
61 match = JSONNumber.regex.match(match.string, *match.span())
62 integer, frac, exp = match.groups()
63 if frac or exp:
64 fn = getattr(context, 'parse_float', None) or float
65 res = fn(integer + (frac or '') + (exp or ''))
66 else:
67 fn = getattr(context, 'parse_int', None) or int
68 res = fn(integer)
69 return res, None
70pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber)
71
72
73STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
74BACKSLASH = {
75 '"': u'"', '\\': u'\\', '/': u'/',
76 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
77}
78
79DEFAULT_ENCODING = "utf-8"
80
81
82def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match):
83 if encoding is None:
84 encoding = DEFAULT_ENCODING
85 chunks = []
86 _append = chunks.append
87 begin = end - 1
88 while 1:
89 chunk = _m(s, end)
90 if chunk is None:
91 raise ValueError(
92 errmsg("Unterminated string starting at", s, begin))
93 end = chunk.end()
94 content, terminator = chunk.groups()
95 if content:
96 if not isinstance(content, unicode):
97 content = unicode(content, encoding)
98 _append(content)
99 if terminator == '"':
100 break
101 elif terminator != '\\':
102 if strict:
103 msg = "Invalid control character {0!r} at".format(terminator)
104 raise ValueError(errmsg(msg, s, end))
105 else:
106 _append(terminator)
107 continue
108 try:
109 esc = s[end]
110 except IndexError:
111 raise ValueError(
112 errmsg("Unterminated string starting at", s, begin))
113 if esc != 'u':
114 try:
115 m = _b[esc]
116 except KeyError:
117 msg = "Invalid \\escape: {0!r}".format(esc)
118 raise ValueError(errmsg(msg, s, end))
119 end += 1
120 else:
121 esc = s[end + 1:end + 5]
122 next_end = end + 5
123 msg = "Invalid \\uXXXX escape"
124 try:
125 if len(esc) != 4:
126 raise ValueError
127 uni = int(esc, 16)
128 if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
129 msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
130 if not s[end + 5:end + 7] == '\\u':
131 raise ValueError
132 esc2 = s[end + 7:end + 11]
133 if len(esc2) != 4:
134 raise ValueError
135 uni2 = int(esc2, 16)
136 uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
137 next_end += 6
138 m = unichr(uni)
139 except ValueError:
140 raise ValueError(errmsg(msg, s, end))
141 end = next_end
142 _append(m)
143 return u''.join(chunks), end
144
145
146# Use speedup
147if c_scanstring is not None:
148 scanstring = c_scanstring
149else:
150 scanstring = py_scanstring
151
152def JSONString(match, context):
153 encoding = getattr(context, 'encoding', None)
154 strict = getattr(context, 'strict', True)
155 return scanstring(match.string, match.end(), encoding, strict)
156pattern(r'"')(JSONString)
157
158
159WHITESPACE = re.compile(r'\s*', FLAGS)
160
161
162def JSONObject(match, context, _w=WHITESPACE.match):
163 pairs = {}
164 s = match.string
165 end = _w(s, match.end()).end()
166 nextchar = s[end:end + 1]
167 # Trivial empty object
168 if nextchar == '}':
169 return pairs, end + 1
170 if nextchar != '"':
171 raise ValueError(errmsg("Expecting property name", s, end))
172 end += 1
173 encoding = getattr(context, 'encoding', None)
174 strict = getattr(context, 'strict', True)
175 iterscan = JSONScanner.iterscan
176 while True:
177 key, end = scanstring(s, end, encoding, strict)
178 end = _w(s, end).end()
179 if s[end:end + 1] != ':':
180 raise ValueError(errmsg("Expecting : delimiter", s, end))
181 end = _w(s, end + 1).end()
182 try:
183 value, end = iterscan(s, idx=end, context=context).next()
184 except StopIteration:
185 raise ValueError(errmsg("Expecting object", s, end))
186 pairs[key] = value
187 end = _w(s, end).end()
188 nextchar = s[end:end + 1]
189 end += 1
190 if nextchar == '}':
191 break
192 if nextchar != ',':
193 raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
194 end = _w(s, end).end()
195 nextchar = s[end:end + 1]
196 end += 1
197 if nextchar != '"':
198 raise ValueError(errmsg("Expecting property name", s, end - 1))
199 object_hook = getattr(context, 'object_hook', None)
200 if object_hook is not None:
201 pairs = object_hook(pairs)
202 return pairs, end
203pattern(r'{')(JSONObject)
204
205
206def JSONArray(match, context, _w=WHITESPACE.match):
207 values = []
208 s = match.string
209 end = _w(s, match.end()).end()
210 # Look-ahead for trivial empty array
211 nextchar = s[end:end + 1]
212 if nextchar == ']':
213 return values, end + 1
214 iterscan = JSONScanner.iterscan
215 while True:
216 try:
217 value, end = iterscan(s, idx=end, context=context).next()
218 except StopIteration:
219 raise ValueError(errmsg("Expecting object", s, end))
220 values.append(value)
221 end = _w(s, end).end()
222 nextchar = s[end:end + 1]
223 end += 1
224 if nextchar == ']':
225 break
226 if nextchar != ',':
227 raise ValueError(errmsg("Expecting , delimiter", s, end))
228 end = _w(s, end).end()
229 return values, end
230pattern(r'\[')(JSONArray)
231
232
233ANYTHING = [
234 JSONObject,
235 JSONArray,
236 JSONString,
237 JSONConstant,
238 JSONNumber,
239]
240
241JSONScanner = Scanner(ANYTHING)
242
243
244class JSONDecoder(object):
245 """Simple JSON <http://json.org> decoder
246
247 Performs the following translations in decoding by default:
248
249 +---------------+-------------------+
250 | JSON | Python |
251 +===============+===================+
252 | object | dict |
253 +---------------+-------------------+
254 | array | list |
255 +---------------+-------------------+
256 | string | unicode |
257 +---------------+-------------------+
258 | number (int) | int, long |
259 +---------------+-------------------+
260 | number (real) | float |
261 +---------------+-------------------+
262 | true | True |
263 +---------------+-------------------+
264 | false | False |
265 +---------------+-------------------+
266 | null | None |
267 +---------------+-------------------+
268
269 It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
270 their corresponding ``float`` values, which is outside the JSON spec.
271 """
272
273 _scanner = Scanner(ANYTHING)
274 __all__ = ['__init__', 'decode', 'raw_decode']
275
276 def __init__(self, encoding=None, object_hook=None, parse_float=None,
277 parse_int=None, parse_constant=None, strict=True):
278 """``encoding`` determines the encoding used to interpret any ``str``
279 objects decoded by this instance (utf-8 by default). It has no
280 effect when decoding ``unicode`` objects.
281
282 Note that currently only encodings that are a superset of ASCII work,
283 strings of other encodings should be passed in as ``unicode``.
284
285 ``object_hook``, if specified, will be called with the result of
286 every JSON object decoded and its return value will be used in
287 place of the given ``dict``. This can be used to provide custom
288 deserializations (e.g. to support JSON-RPC class hinting).
289
290 ``parse_float``, if specified, will be called with the string
291 of every JSON float to be decoded. By default this is equivalent to
292 float(num_str). This can be used to use another datatype or parser
293 for JSON floats (e.g. decimal.Decimal).
294
295 ``parse_int``, if specified, will be called with the string
296 of every JSON int to be decoded. By default this is equivalent to
297 int(num_str). This can be used to use another datatype or parser
298 for JSON integers (e.g. float).
299
300 ``parse_constant``, if specified, will be called with one of the
301 following strings: -Infinity, Infinity, NaN, null, true, false.
302 This can be used to raise an exception if invalid JSON numbers
303 are encountered.
304
305 """
306 self.encoding = encoding
307 self.object_hook = object_hook
308 self.parse_float = parse_float
309 self.parse_int = parse_int
310 self.parse_constant = parse_constant
311 self.strict = strict
312
313 def decode(self, s, _w=WHITESPACE.match):
314 """
315 Return the Python representation of ``s`` (a ``str`` or ``unicode``
316 instance containing a JSON document)
317
318 """
319 obj, end = self.raw_decode(s, idx=_w(s, 0).end())
320 end = _w(s, end).end()
321 if end != len(s):
322 raise ValueError(errmsg("Extra data", s, end, len(s)))
323 return obj
324
325 def raw_decode(self, s, **kw):
326 """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
327 with a JSON document) and return a 2-tuple of the Python
328 representation and the index in ``s`` where the document ended.
329
330 This can be used to decode a JSON document from a string that may
331 have extraneous data at the end.
332
333 """
334 kw.setdefault('context', self)
335 try:
336 obj, end = self._scanner.iterscan(s, **kw).next()
337 except StopIteration:
338 raise ValueError("No JSON object could be decoded")
339 return obj, end
Note: See TracBrowser for help on using the repository browser.