source: python/vendor/Python-2.6.5/Parser/asdl_c.py

Last change on this file was 2, checked in by Yuri Dario, 15 years ago

Initial import for vendor code.
  • Property svn:eol-style set to native
File size: 39.9 KB
Line 
1#! /usr/bin/env python
2"""Generate C code from an ASDL description."""
3
4# TO DO
5# handle fields that have a type but no name
6
7import os, sys
8
9import asdl
10
11TABSIZE = 8
12MAX_COL = 80
13
14def get_c_type(name):
15 """Return a string for the C name of the type.
16
17 This function special cases the default types provided by asdl:
18 identifier, string, int, bool.
19 """
20 # XXX ack! need to figure out where Id is useful and where string
21 if isinstance(name, asdl.Id):
22 name = name.value
23 if name in asdl.builtin_types:
24 return name
25 else:
26 return "%s_ty" % name
27
28def reflow_lines(s, depth):
29 """Reflow the line s indented depth tabs.
30
31 Return a sequence of lines where no line extends beyond MAX_COL
32 when properly indented. The first line is properly indented based
33 exclusively on depth * TABSIZE. All following lines -- these are
34 the reflowed lines generated by this function -- start at the same
35 column as the first character beyond the opening { in the first
36 line.
37 """
38 size = MAX_COL - depth * TABSIZE
39 if len(s) < size:
40 return [s]
41
42 lines = []
43 cur = s
44 padding = ""
45 while len(cur) > size:
46 i = cur.rfind(' ', 0, size)
47 # XXX this should be fixed for real
48 if i == -1 and 'GeneratorExp' in cur:
49 i = size + 3
50 assert i != -1, "Impossible line %d to reflow: %r" % (size, s)
51 lines.append(padding + cur[:i])
52 if len(lines) == 1:
53 # find new size based on brace
54 j = cur.find('{', 0, i)
55 if j >= 0:
56 j += 2 # account for the brace and the space after it
57 size -= j
58 padding = " " * j
59 else:
60 j = cur.find('(', 0, i)
61 if j >= 0:
62 j += 1 # account for the paren (no space after it)
63 size -= j
64 padding = " " * j
65 cur = cur[i+1:]
66 else:
67 lines.append(padding + cur)
68 return lines
69
70def is_simple(sum):
71 """Return True if a sum is a simple.
72
73 A sum is simple if its types have no fields, e.g.
74 unaryop = Invert | Not | UAdd | USub
75 """
76 for t in sum.types:
77 if t.fields:
78 return False
79 return True
80
81
82class EmitVisitor(asdl.VisitorBase):
83 """Visit that emits lines"""
84
85 def __init__(self, file):
86 self.file = file
87 super(EmitVisitor, self).__init__()
88
89 def emit(self, s, depth, reflow=1):
90 # XXX reflow long lines?
91 if reflow:
92 lines = reflow_lines(s, depth)
93 else:
94 lines = [s]
95 for line in lines:
96 line = (" " * TABSIZE * depth) + line + "\n"
97 self.file.write(line)
98
99
100class TypeDefVisitor(EmitVisitor):
101 def visitModule(self, mod):
102 for dfn in mod.dfns:
103 self.visit(dfn)
104
105 def visitType(self, type, depth=0):
106 self.visit(type.value, type.name, depth)
107
108 def visitSum(self, sum, name, depth):
109 if is_simple(sum):
110 self.simple_sum(sum, name, depth)
111 else:
112 self.sum_with_constructors(sum, name, depth)
113
114 def simple_sum(self, sum, name, depth):
115 enum = []
116 for i in range(len(sum.types)):
117 type = sum.types[i]
118 enum.append("%s=%d" % (type.name, i + 1))
119 enums = ", ".join(enum)
120 ctype = get_c_type(name)
121 s = "typedef enum _%s { %s } %s;" % (name, enums, ctype)
122 self.emit(s, depth)
123 self.emit("", depth)
124
125 def sum_with_constructors(self, sum, name, depth):
126 ctype = get_c_type(name)
127 s = "typedef struct _%(name)s *%(ctype)s;" % locals()
128 self.emit(s, depth)
129 self.emit("", depth)
130
131 def visitProduct(self, product, name, depth):
132 ctype = get_c_type(name)
133 s = "typedef struct _%(name)s *%(ctype)s;" % locals()
134 self.emit(s, depth)
135 self.emit("", depth)
136
137
138class StructVisitor(EmitVisitor):
139 """Visitor to generate typdefs for AST."""
140
141 def visitModule(self, mod):
142 for dfn in mod.dfns:
143 self.visit(dfn)
144
145 def visitType(self, type, depth=0):
146 self.visit(type.value, type.name, depth)
147
148 def visitSum(self, sum, name, depth):
149 if not is_simple(sum):
150 self.sum_with_constructors(sum, name, depth)
151
152 def sum_with_constructors(self, sum, name, depth):
153 def emit(s, depth=depth):
154 self.emit(s % sys._getframe(1).f_locals, depth)
155 enum = []
156 for i in range(len(sum.types)):
157 type = sum.types[i]
158 enum.append("%s_kind=%d" % (type.name, i + 1))
159
160 emit("enum _%(name)s_kind {" + ", ".join(enum) + "};")
161
162 emit("struct _%(name)s {")
163 emit("enum _%(name)s_kind kind;", depth + 1)
164 emit("union {", depth + 1)
165 for t in sum.types:
166 self.visit(t, depth + 2)
167 emit("} v;", depth + 1)
168 for field in sum.attributes:
169 # rudimentary attribute handling
170 type = str(field.type)
171 assert type in asdl.builtin_types, type
172 emit("%s %s;" % (type, field.name), depth + 1);
173 emit("};")
174 emit("")
175
176 def visitConstructor(self, cons, depth):
177 if cons.fields:
178 self.emit("struct {", depth)
179 for f in cons.fields:
180 self.visit(f, depth + 1)
181 self.emit("} %s;" % cons.name, depth)
182 self.emit("", depth)
183 else:
184 # XXX not sure what I want here, nothing is probably fine
185 pass
186
187 def visitField(self, field, depth):
188 # XXX need to lookup field.type, because it might be something
189 # like a builtin...
190 ctype = get_c_type(field.type)
191 name = field.name
192 if field.seq:
193 if field.type.value in ('cmpop',):
194 self.emit("asdl_int_seq *%(name)s;" % locals(), depth)
195 else:
196 self.emit("asdl_seq *%(name)s;" % locals(), depth)
197 else:
198 self.emit("%(ctype)s %(name)s;" % locals(), depth)
199
200 def visitProduct(self, product, name, depth):
201 self.emit("struct _%(name)s {" % locals(), depth)
202 for f in product.fields:
203 self.visit(f, depth + 1)
204 self.emit("};", depth)
205 self.emit("", depth)
206
207
208class PrototypeVisitor(EmitVisitor):
209 """Generate function prototypes for the .h file"""
210
211 def visitModule(self, mod):
212 for dfn in mod.dfns:
213 self.visit(dfn)
214
215 def visitType(self, type):
216 self.visit(type.value, type.name)
217
218 def visitSum(self, sum, name):
219 if is_simple(sum):
220 pass # XXX
221 else:
222 for t in sum.types:
223 self.visit(t, name, sum.attributes)
224
225 def get_args(self, fields):
226 """Return list of C argument into, one for each field.
227
228 Argument info is 3-tuple of a C type, variable name, and flag
229 that is true if type can be NULL.
230 """
231 args = []
232 unnamed = {}
233 for f in fields:
234 if f.name is None:
235 name = f.type
236 c = unnamed[name] = unnamed.get(name, 0) + 1
237 if c > 1:
238 name = "name%d" % (c - 1)
239 else:
240 name = f.name
241 # XXX should extend get_c_type() to handle this
242 if f.seq:
243 if f.type.value in ('cmpop',):
244 ctype = "asdl_int_seq *"
245 else:
246 ctype = "asdl_seq *"
247 else:
248 ctype = get_c_type(f.type)
249 args.append((ctype, name, f.opt or f.seq))
250 return args
251
252 def visitConstructor(self, cons, type, attrs):
253 args = self.get_args(cons.fields)
254 attrs = self.get_args(attrs)
255 ctype = get_c_type(type)
256 self.emit_function(cons.name, ctype, args, attrs)
257
258 def emit_function(self, name, ctype, args, attrs, union=1):
259 args = args + attrs
260 if args:
261 argstr = ", ".join(["%s %s" % (atype, aname)
262 for atype, aname, opt in args])
263 argstr += ", PyArena *arena"
264 else:
265 argstr = "PyArena *arena"
266 margs = "a0"
267 for i in range(1, len(args)+1):
268 margs += ", a%d" % i
269 self.emit("#define %s(%s) _Py_%s(%s)" % (name, margs, name, margs), 0,
270 reflow = 0)
271 self.emit("%s _Py_%s(%s);" % (ctype, name, argstr), 0)
272
273 def visitProduct(self, prod, name):
274 self.emit_function(name, get_c_type(name),
275 self.get_args(prod.fields), [], union=0)
276
277
278class FunctionVisitor(PrototypeVisitor):
279 """Visitor to generate constructor functions for AST."""
280
281 def emit_function(self, name, ctype, args, attrs, union=1):
282 def emit(s, depth=0, reflow=1):
283 self.emit(s, depth, reflow)
284 argstr = ", ".join(["%s %s" % (atype, aname)
285 for atype, aname, opt in args + attrs])
286 if argstr:
287 argstr += ", PyArena *arena"
288 else:
289 argstr = "PyArena *arena"
290 self.emit("%s" % ctype, 0)
291 emit("%s(%s)" % (name, argstr))
292 emit("{")
293 emit("%s p;" % ctype, 1)
294 for argtype, argname, opt in args:
295 # XXX hack alert: false is allowed for a bool
296 if not opt and not (argtype == "bool" or argtype == "int"):
297 emit("if (!%s) {" % argname, 1)
298 emit("PyErr_SetString(PyExc_ValueError,", 2)
299 msg = "field %s is required for %s" % (argname, name)
300 emit(' "%s");' % msg,
301 2, reflow=0)
302 emit('return NULL;', 2)
303 emit('}', 1)
304
305 emit("p = (%s)PyArena_Malloc(arena, sizeof(*p));" % ctype, 1);
306 emit("if (!p)", 1)
307 emit("return NULL;", 2)
308 if union:
309 self.emit_body_union(name, args, attrs)
310 else:
311 self.emit_body_struct(name, args, attrs)
312 emit("return p;", 1)
313 emit("}")
314 emit("")
315
316 def emit_body_union(self, name, args, attrs):
317 def emit(s, depth=0, reflow=1):
318 self.emit(s, depth, reflow)
319 emit("p->kind = %s_kind;" % name, 1)
320 for argtype, argname, opt in args:
321 emit("p->v.%s.%s = %s;" % (name, argname, argname), 1)
322 for argtype, argname, opt in attrs:
323 emit("p->%s = %s;" % (argname, argname), 1)
324
325 def emit_body_struct(self, name, args, attrs):
326 def emit(s, depth=0, reflow=1):
327 self.emit(s, depth, reflow)
328 for argtype, argname, opt in args:
329 emit("p->%s = %s;" % (argname, argname), 1)
330 assert not attrs
331
332
333class PickleVisitor(EmitVisitor):
334
335 def visitModule(self, mod):
336 for dfn in mod.dfns:
337 self.visit(dfn)
338
339 def visitType(self, type):
340 self.visit(type.value, type.name)
341
342 def visitSum(self, sum, name):
343 pass
344
345 def visitProduct(self, sum, name):
346 pass
347
348 def visitConstructor(self, cons, name):
349 pass
350
351 def visitField(self, sum):
352 pass
353
354
355class Obj2ModPrototypeVisitor(PickleVisitor):
356 def visitProduct(self, prod, name):
357 code = "static int obj2ast_%s(PyObject* obj, %s* out, PyArena* arena);"
358 self.emit(code % (name, get_c_type(name)), 0)
359
360 visitSum = visitProduct
361
362
363class Obj2ModVisitor(PickleVisitor):
364 def funcHeader(self, name):
365 ctype = get_c_type(name)
366 self.emit("int", 0)
367 self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)
368 self.emit("{", 0)
369 self.emit("PyObject* tmp = NULL;", 1)
370 self.emit("int isinstance;", 1)
371 self.emit("", 0)
372
373 def sumTrailer(self, name):
374 self.emit("", 0)
375 self.emit("tmp = PyObject_Repr(obj);", 1)
376 # there's really nothing more we can do if this fails ...
377 self.emit("if (tmp == NULL) goto failed;", 1)
378 error = "expected some sort of %s, but got %%.400s" % name
379 format = "PyErr_Format(PyExc_TypeError, \"%s\", PyString_AS_STRING(tmp));"
380 self.emit(format % error, 1, reflow=False)
381 self.emit("failed:", 0)
382 self.emit("Py_XDECREF(tmp);", 1)
383 self.emit("return 1;", 1)
384 self.emit("}", 0)
385 self.emit("", 0)
386
387 def simpleSum(self, sum, name):
388 self.funcHeader(name)
389 for t in sum.types:
390 line = ("isinstance = PyObject_IsInstance(obj, "
391 "(PyObject *)%s_type);")
392 self.emit(line % (t.name,), 1)
393 self.emit("if (isinstance == -1) {", 1)
394 self.emit("return 1;", 2)
395 self.emit("}", 1)
396 self.emit("if (isinstance) {", 1)
397 self.emit("*out = %s;" % t.name, 2)
398 self.emit("return 0;", 2)
399 self.emit("}", 1)
400 self.sumTrailer(name)
401
402 def buildArgs(self, fields):
403 return ", ".join(fields + ["arena"])
404
405 def complexSum(self, sum, name):
406 self.funcHeader(name)
407 for a in sum.attributes:
408 self.visitAttributeDeclaration(a, name, sum=sum)
409 self.emit("", 0)
410 # XXX: should we only do this for 'expr'?
411 self.emit("if (obj == Py_None) {", 1)
412 self.emit("*out = NULL;", 2)
413 self.emit("return 0;", 2)
414 self.emit("}", 1)
415 for a in sum.attributes:
416 self.visitField(a, name, sum=sum, depth=1)
417 for t in sum.types:
418 line = "isinstance = PyObject_IsInstance(obj, (PyObject*)%s_type);"
419 self.emit(line % (t.name,), 1)
420 self.emit("if (isinstance == -1) {", 1)
421 self.emit("return 1;", 2)
422 self.emit("}", 1)
423 self.emit("if (isinstance) {", 1)
424 for f in t.fields:
425 self.visitFieldDeclaration(f, t.name, sum=sum, depth=2)
426 self.emit("", 0)
427 for f in t.fields:
428 self.visitField(f, t.name, sum=sum, depth=2)
429 args = [f.name.value for f in t.fields] + [a.name.value for a in sum.attributes]
430 self.emit("*out = %s(%s);" % (t.name, self.buildArgs(args)), 2)
431 self.emit("if (*out == NULL) goto failed;", 2)
432 self.emit("return 0;", 2)
433 self.emit("}", 1)
434 self.sumTrailer(name)
435
436 def visitAttributeDeclaration(self, a, name, sum=sum):
437 ctype = get_c_type(a.type)
438 self.emit("%s %s;" % (ctype, a.name), 1)
439
440 def visitSum(self, sum, name):
441 if is_simple(sum):
442 self.simpleSum(sum, name)
443 else:
444 self.complexSum(sum, name)
445
446 def visitProduct(self, prod, name):
447 ctype = get_c_type(name)
448 self.emit("int", 0)
449 self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)
450 self.emit("{", 0)
451 self.emit("PyObject* tmp = NULL;", 1)
452 for f in prod.fields:
453 self.visitFieldDeclaration(f, name, prod=prod, depth=1)
454 self.emit("", 0)
455 for f in prod.fields:
456 self.visitField(f, name, prod=prod, depth=1)
457 args = [f.name.value for f in prod.fields]
458 self.emit("*out = %s(%s);" % (name, self.buildArgs(args)), 1)
459 self.emit("return 0;", 1)
460 self.emit("failed:", 0)
461 self.emit("Py_XDECREF(tmp);", 1)
462 self.emit("return 1;", 1)
463 self.emit("}", 0)
464 self.emit("", 0)
465
466 def visitFieldDeclaration(self, field, name, sum=None, prod=None, depth=0):
467 ctype = get_c_type(field.type)
468 if field.seq:
469 if self.isSimpleType(field):
470 self.emit("asdl_int_seq* %s;" % field.name, depth)
471 else:
472 self.emit("asdl_seq* %s;" % field.name, depth)
473 else:
474 ctype = get_c_type(field.type)
475 self.emit("%s %s;" % (ctype, field.name), depth)
476
477 def isSimpleSum(self, field):
478 # XXX can the members of this list be determined automatically?
479 return field.type.value in ('expr_context', 'boolop', 'operator',
480 'unaryop', 'cmpop')
481
482 def isNumeric(self, field):
483 return get_c_type(field.type) in ("int", "bool")
484
485 def isSimpleType(self, field):
486 return self.isSimpleSum(field) or self.isNumeric(field)
487
488 def visitField(self, field, name, sum=None, prod=None, depth=0):
489 ctype = get_c_type(field.type)
490 self.emit("if (PyObject_HasAttrString(obj, \"%s\")) {" % field.name, depth)
491 self.emit("int res;", depth+1)
492 if field.seq:
493 self.emit("Py_ssize_t len;", depth+1)
494 self.emit("Py_ssize_t i;", depth+1)
495 self.emit("tmp = PyObject_GetAttrString(obj, \"%s\");" % field.name, depth+1)
496 self.emit("if (tmp == NULL) goto failed;", depth+1)
497 if field.seq:
498 self.emit("if (!PyList_Check(tmp)) {", depth+1)
499 self.emit("PyErr_Format(PyExc_TypeError, \"%s field \\\"%s\\\" must "
500 "be a list, not a %%.200s\", tmp->ob_type->tp_name);" %
501 (name, field.name),
502 depth+2, reflow=False)
503 self.emit("goto failed;", depth+2)
504 self.emit("}", depth+1)
505 self.emit("len = PyList_GET_SIZE(tmp);", depth+1)
506 if self.isSimpleType(field):
507 self.emit("%s = asdl_int_seq_new(len, arena);" % field.name, depth+1)
508 else:
509 self.emit("%s = asdl_seq_new(len, arena);" % field.name, depth+1)
510 self.emit("if (%s == NULL) goto failed;" % field.name, depth+1)
511 self.emit("for (i = 0; i < len; i++) {", depth+1)
512 self.emit("%s value;" % ctype, depth+2)
513 self.emit("res = obj2ast_%s(PyList_GET_ITEM(tmp, i), &value, arena);" %
514 field.type, depth+2, reflow=False)
515 self.emit("if (res != 0) goto failed;", depth+2)
516 self.emit("asdl_seq_SET(%s, i, value);" % field.name, depth+2)
517 self.emit("}", depth+1)
518 else:
519 self.emit("res = obj2ast_%s(tmp, &%s, arena);" %
520 (field.type, field.name), depth+1)
521 self.emit("if (res != 0) goto failed;", depth+1)
522
523 self.emit("Py_XDECREF(tmp);", depth+1)
524 self.emit("tmp = NULL;", depth+1)
525 self.emit("} else {", depth)
526 if not field.opt:
527 message = "required field \\\"%s\\\" missing from %s" % (field.name, name)
528 format = "PyErr_SetString(PyExc_TypeError, \"%s\");"
529 self.emit(format % message, depth+1, reflow=False)
530 self.emit("return 1;", depth+1)
531 else:
532 if self.isNumeric(field):
533 self.emit("%s = 0;" % field.name, depth+1)
534 elif not self.isSimpleType(field):
535 self.emit("%s = NULL;" % field.name, depth+1)
536 else:
537 raise TypeError("could not determine the default value for %s" % field.name)
538 self.emit("}", depth)
539
540
541class MarshalPrototypeVisitor(PickleVisitor):
542
543 def prototype(self, sum, name):
544 ctype = get_c_type(name)
545 self.emit("static int marshal_write_%s(PyObject **, int *, %s);"
546 % (name, ctype), 0)
547
548 visitProduct = visitSum = prototype
549
550
551class PyTypesDeclareVisitor(PickleVisitor):
552
553 def visitProduct(self, prod, name):
554 self.emit("static PyTypeObject *%s_type;" % name, 0)
555 self.emit("static PyObject* ast2obj_%s(void*);" % name, 0)
556 if prod.fields:
557 self.emit("static char *%s_fields[]={" % name,0)
558 for f in prod.fields:
559 self.emit('"%s",' % f.name, 1)
560 self.emit("};", 0)
561
562 def visitSum(self, sum, name):
563 self.emit("static PyTypeObject *%s_type;" % name, 0)
564 if sum.attributes:
565 self.emit("static char *%s_attributes[] = {" % name, 0)
566 for a in sum.attributes:
567 self.emit('"%s",' % a.name, 1)
568 self.emit("};", 0)
569 ptype = "void*"
570 if is_simple(sum):
571 ptype = get_c_type(name)
572 tnames = []
573 for t in sum.types:
574 tnames.append(str(t.name)+"_singleton")
575 tnames = ", *".join(tnames)
576 self.emit("static PyObject *%s;" % tnames, 0)
577 self.emit("static PyObject* ast2obj_%s(%s);" % (name, ptype), 0)
578 for t in sum.types:
579 self.visitConstructor(t, name)
580
581 def visitConstructor(self, cons, name):
582 self.emit("static PyTypeObject *%s_type;" % cons.name, 0)
583 if cons.fields:
584 self.emit("static char *%s_fields[]={" % cons.name, 0)
585 for t in cons.fields:
586 self.emit('"%s",' % t.name, 1)
587 self.emit("};",0)
588
589class PyTypesVisitor(PickleVisitor):
590
591 def visitModule(self, mod):
592 self.emit("""
593static int
594ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
595{
596 Py_ssize_t i, numfields = 0;
597 int res = -1;
598 PyObject *key, *value, *fields;
599 fields = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "_fields");
600 if (!fields)
601 PyErr_Clear();
602 if (fields) {
603 numfields = PySequence_Size(fields);
604 if (numfields == -1)
605 goto cleanup;
606 }
607 res = 0; /* if no error occurs, this stays 0 to the end */
608 if (PyTuple_GET_SIZE(args) > 0) {
609 if (numfields != PyTuple_GET_SIZE(args)) {
610 PyErr_Format(PyExc_TypeError, "%.400s constructor takes %s"
611 "%zd positional argument%s",
612 Py_TYPE(self)->tp_name,
613 numfields == 0 ? "" : "either 0 or ",
614 numfields, numfields == 1 ? "" : "s");
615 res = -1;
616 goto cleanup;
617 }
618 for (i = 0; i < PyTuple_GET_SIZE(args); i++) {
619 /* cannot be reached when fields is NULL */
620 PyObject *name = PySequence_GetItem(fields, i);
621 if (!name) {
622 res = -1;
623 goto cleanup;
624 }
625 res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i));
626 Py_DECREF(name);
627 if (res < 0)
628 goto cleanup;
629 }
630 }
631 if (kw) {
632 i = 0; /* needed by PyDict_Next */
633 while (PyDict_Next(kw, &i, &key, &value)) {
634 res = PyObject_SetAttr(self, key, value);
635 if (res < 0)
636 goto cleanup;
637 }
638 }
639 cleanup:
640 Py_XDECREF(fields);
641 return res;
642}
643
644/* Pickling support */
645static PyObject *
646ast_type_reduce(PyObject *self, PyObject *unused)
647{
648 PyObject *res;
649 PyObject *dict = PyObject_GetAttrString(self, "__dict__");
650 if (dict == NULL) {
651 if (PyErr_ExceptionMatches(PyExc_AttributeError))
652 PyErr_Clear();
653 else
654 return NULL;
655 }
656 if (dict) {
657 res = Py_BuildValue("O()O", Py_TYPE(self), dict);
658 Py_DECREF(dict);
659 return res;
660 }
661 return Py_BuildValue("O()", Py_TYPE(self));
662}
663
664static PyMethodDef ast_type_methods[] = {
665 {"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
666 {NULL}
667};
668
669static PyTypeObject AST_type = {
670 PyVarObject_HEAD_INIT(&PyType_Type, 0)
671 "_ast.AST",
672 sizeof(PyObject),
673 0,
674 0, /* tp_dealloc */
675 0, /* tp_print */
676 0, /* tp_getattr */
677 0, /* tp_setattr */
678 0, /* tp_compare */
679 0, /* tp_repr */
680 0, /* tp_as_number */
681 0, /* tp_as_sequence */
682 0, /* tp_as_mapping */
683 0, /* tp_hash */
684 0, /* tp_call */
685 0, /* tp_str */
686 PyObject_GenericGetAttr, /* tp_getattro */
687 PyObject_GenericSetAttr, /* tp_setattro */
688 0, /* tp_as_buffer */
689 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
690 0, /* tp_doc */
691 0, /* tp_traverse */
692 0, /* tp_clear */
693 0, /* tp_richcompare */
694 0, /* tp_weaklistoffset */
695 0, /* tp_iter */
696 0, /* tp_iternext */
697 ast_type_methods, /* tp_methods */
698 0, /* tp_members */
699 0, /* tp_getset */
700 0, /* tp_base */
701 0, /* tp_dict */
702 0, /* tp_descr_get */
703 0, /* tp_descr_set */
704 0, /* tp_dictoffset */
705 (initproc)ast_type_init, /* tp_init */
706 PyType_GenericAlloc, /* tp_alloc */
707 PyType_GenericNew, /* tp_new */
708 PyObject_Del, /* tp_free */
709};
710
711
712static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int num_fields)
713{
714 PyObject *fnames, *result;
715 int i;
716 fnames = PyTuple_New(num_fields);
717 if (!fnames) return NULL;
718 for (i = 0; i < num_fields; i++) {
719 PyObject *field = PyString_FromString(fields[i]);
720 if (!field) {
721 Py_DECREF(fnames);
722 return NULL;
723 }
724 PyTuple_SET_ITEM(fnames, i, field);
725 }
726 result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){sOss}",
727 type, base, "_fields", fnames, "__module__", "_ast");
728 Py_DECREF(fnames);
729 return (PyTypeObject*)result;
730}
731
732static int add_attributes(PyTypeObject* type, char**attrs, int num_fields)
733{
734 int i, result;
735 PyObject *s, *l = PyTuple_New(num_fields);
736 if (!l) return 0;
737 for(i = 0; i < num_fields; i++) {
738 s = PyString_FromString(attrs[i]);
739 if (!s) {
740 Py_DECREF(l);
741 return 0;
742 }
743 PyTuple_SET_ITEM(l, i, s);
744 }
745 result = PyObject_SetAttrString((PyObject*)type, "_attributes", l) >= 0;
746 Py_DECREF(l);
747 return result;
748}
749
750/* Conversion AST -> Python */
751
752static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
753{
754 int i, n = asdl_seq_LEN(seq);
755 PyObject *result = PyList_New(n);
756 PyObject *value;
757 if (!result)
758 return NULL;
759 for (i = 0; i < n; i++) {
760 value = func(asdl_seq_GET(seq, i));
761 if (!value) {
762 Py_DECREF(result);
763 return NULL;
764 }
765 PyList_SET_ITEM(result, i, value);
766 }
767 return result;
768}
769
770static PyObject* ast2obj_object(void *o)
771{
772 if (!o)
773 o = Py_None;
774 Py_INCREF((PyObject*)o);
775 return (PyObject*)o;
776}
777#define ast2obj_identifier ast2obj_object
778#define ast2obj_string ast2obj_object
779static PyObject* ast2obj_bool(bool b)
780{
781 return PyBool_FromLong(b);
782}
783
784static PyObject* ast2obj_int(long b)
785{
786 return PyInt_FromLong(b);
787}
788
789/* Conversion Python -> AST */
790
791static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
792{
793 if (obj == Py_None)
794 obj = NULL;
795 if (obj)
796 PyArena_AddPyObject(arena, obj);
797 Py_XINCREF(obj);
798 *out = obj;
799 return 0;
800}
801
802#define obj2ast_identifier obj2ast_object
803#define obj2ast_string obj2ast_object
804
805static int obj2ast_int(PyObject* obj, int* out, PyArena* arena)
806{
807 int i;
808 if (!PyInt_Check(obj) && !PyLong_Check(obj)) {
809 PyObject *s = PyObject_Repr(obj);
810 if (s == NULL) return 1;
811 PyErr_Format(PyExc_ValueError, "invalid integer value: %.400s",
812 PyString_AS_STRING(s));
813 Py_DECREF(s);
814 return 1;
815 }
816
817 i = (int)PyLong_AsLong(obj);
818 if (i == -1 && PyErr_Occurred())
819 return 1;
820 *out = i;
821 return 0;
822}
823
824static int obj2ast_bool(PyObject* obj, bool* out, PyArena* arena)
825{
826 if (!PyBool_Check(obj)) {
827 PyObject *s = PyObject_Repr(obj);
828 if (s == NULL) return 1;
829 PyErr_Format(PyExc_ValueError, "invalid boolean value: %.400s",
830 PyString_AS_STRING(s));
831 Py_DECREF(s);
832 return 1;
833 }
834
835 *out = (obj == Py_True);
836 return 0;
837}
838
839static int add_ast_fields(void)
840{
841 PyObject *empty_tuple, *d;
842 if (PyType_Ready(&AST_type) < 0)
843 return -1;
844 d = AST_type.tp_dict;
845 empty_tuple = PyTuple_New(0);
846 if (!empty_tuple ||
847 PyDict_SetItemString(d, "_fields", empty_tuple) < 0 ||
848 PyDict_SetItemString(d, "_attributes", empty_tuple) < 0) {
849 Py_XDECREF(empty_tuple);
850 return -1;
851 }
852 Py_DECREF(empty_tuple);
853 return 0;
854}
855
856""", 0, reflow=False)
857
858 self.emit("static int init_types(void)",0)
859 self.emit("{", 0)
860 self.emit("static int initialized;", 1)
861 self.emit("if (initialized) return 1;", 1)
862 self.emit("if (add_ast_fields() < 0) return 0;", 1)
863 for dfn in mod.dfns:
864 self.visit(dfn)
865 self.emit("initialized = 1;", 1)
866 self.emit("return 1;", 1);
867 self.emit("}", 0)
868
869 def visitProduct(self, prod, name):
870 if prod.fields:
871 fields = name.value+"_fields"
872 else:
873 fields = "NULL"
874 self.emit('%s_type = make_type("%s", &AST_type, %s, %d);' %
875 (name, name, fields, len(prod.fields)), 1)
876 self.emit("if (!%s_type) return 0;" % name, 1)
877
878 def visitSum(self, sum, name):
879 self.emit('%s_type = make_type("%s", &AST_type, NULL, 0);' %
880 (name, name), 1)
881 self.emit("if (!%s_type) return 0;" % name, 1)
882 if sum.attributes:
883 self.emit("if (!add_attributes(%s_type, %s_attributes, %d)) return 0;" %
884 (name, name, len(sum.attributes)), 1)
885 else:
886 self.emit("if (!add_attributes(%s_type, NULL, 0)) return 0;" % name, 1)
887 simple = is_simple(sum)
888 for t in sum.types:
889 self.visitConstructor(t, name, simple)
890
891 def visitConstructor(self, cons, name, simple):
892 if cons.fields:
893 fields = cons.name.value+"_fields"
894 else:
895 fields = "NULL"
896 self.emit('%s_type = make_type("%s", %s_type, %s, %d);' %
897 (cons.name, cons.name, name, fields, len(cons.fields)), 1)
898 self.emit("if (!%s_type) return 0;" % cons.name, 1)
899 if simple:
900 self.emit("%s_singleton = PyType_GenericNew(%s_type, NULL, NULL);" %
901 (cons.name, cons.name), 1)
902 self.emit("if (!%s_singleton) return 0;" % cons.name, 1)
903
904
905def parse_version(mod):
906 return mod.version.value[12:-3]
907
908class ASTModuleVisitor(PickleVisitor):
909
910 def visitModule(self, mod):
911 self.emit("PyMODINIT_FUNC", 0)
912 self.emit("init_ast(void)", 0)
913 self.emit("{", 0)
914 self.emit("PyObject *m, *d;", 1)
915 self.emit("if (!init_types()) return;", 1)
916 self.emit('m = Py_InitModule3("_ast", NULL, NULL);', 1)
917 self.emit("if (!m) return;", 1)
918 self.emit("d = PyModule_GetDict(m);", 1)
919 self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return;', 1)
920 self.emit('if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0)', 1)
921 self.emit("return;", 2)
922 # Value of version: "$Revision: 76775 $"
923 self.emit('if (PyModule_AddStringConstant(m, "__version__", "%s") < 0)'
924 % parse_version(mod), 1)
925 self.emit("return;", 2)
926 for dfn in mod.dfns:
927 self.visit(dfn)
928 self.emit("}", 0)
929
930 def visitProduct(self, prod, name):
931 self.addObj(name)
932
933 def visitSum(self, sum, name):
934 self.addObj(name)
935 for t in sum.types:
936 self.visitConstructor(t, name)
937
938 def visitConstructor(self, cons, name):
939 self.addObj(cons.name)
940
941 def addObj(self, name):
942 self.emit('if (PyDict_SetItemString(d, "%s", (PyObject*)%s_type) < 0) return;' % (name, name), 1)
943
944
945_SPECIALIZED_SEQUENCES = ('stmt', 'expr')
946
947def find_sequence(fields, doing_specialization):
948 """Return True if any field uses a sequence."""
949 for f in fields:
950 if f.seq:
951 if not doing_specialization:
952 return True
953 if str(f.type) not in _SPECIALIZED_SEQUENCES:
954 return True
955 return False
956
957def has_sequence(types, doing_specialization):
958 for t in types:
959 if find_sequence(t.fields, doing_specialization):
960 return True
961 return False
962
963
964class StaticVisitor(PickleVisitor):
965 CODE = '''Very simple, always emit this static code. Overide CODE'''
966
967 def visit(self, object):
968 self.emit(self.CODE, 0, reflow=False)
969
970
971class ObjVisitor(PickleVisitor):
972
973 def func_begin(self, name):
974 ctype = get_c_type(name)
975 self.emit("PyObject*", 0)
976 self.emit("ast2obj_%s(void* _o)" % (name), 0)
977 self.emit("{", 0)
978 self.emit("%s o = (%s)_o;" % (ctype, ctype), 1)
979 self.emit("PyObject *result = NULL, *value = NULL;", 1)
980 self.emit('if (!o) {', 1)
981 self.emit("Py_INCREF(Py_None);", 2)
982 self.emit('return Py_None;', 2)
983 self.emit("}", 1)
984 self.emit('', 0)
985
986 def func_end(self):
987 self.emit("return result;", 1)
988 self.emit("failed:", 0)
989 self.emit("Py_XDECREF(value);", 1)
990 self.emit("Py_XDECREF(result);", 1)
991 self.emit("return NULL;", 1)
992 self.emit("}", 0)
993 self.emit("", 0)
994
995 def visitSum(self, sum, name):
996 if is_simple(sum):
997 self.simpleSum(sum, name)
998 return
999 self.func_begin(name)
1000 self.emit("switch (o->kind) {", 1)
1001 for i in range(len(sum.types)):
1002 t = sum.types[i]
1003 self.visitConstructor(t, i + 1, name)
1004 self.emit("}", 1)
1005 for a in sum.attributes:
1006 self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1)
1007 self.emit("if (!value) goto failed;", 1)
1008 self.emit('if (PyObject_SetAttrString(result, "%s", value) < 0)' % a.name, 1)
1009 self.emit('goto failed;', 2)
1010 self.emit('Py_DECREF(value);', 1)
1011 self.func_end()
1012
1013 def simpleSum(self, sum, name):
1014 self.emit("PyObject* ast2obj_%s(%s_ty o)" % (name, name), 0)
1015 self.emit("{", 0)
1016 self.emit("switch(o) {", 1)
1017 for t in sum.types:
1018 self.emit("case %s:" % t.name, 2)
1019 self.emit("Py_INCREF(%s_singleton);" % t.name, 3)
1020 self.emit("return %s_singleton;" % t.name, 3)
1021 self.emit("default:" % name, 2)
1022 self.emit('/* should never happen, but just in case ... */', 3)
1023 code = "PyErr_Format(PyExc_SystemError, \"unknown %s found\");" % name
1024 self.emit(code, 3, reflow=False)
1025 self.emit("return NULL;", 3)
1026 self.emit("}", 1)
1027 self.emit("}", 0)
1028
1029 def visitProduct(self, prod, name):
1030 self.func_begin(name)
1031 self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % name, 1);
1032 self.emit("if (!result) return NULL;", 1)
1033 for field in prod.fields:
1034 self.visitField(field, name, 1, True)
1035 self.func_end()
1036
1037 def visitConstructor(self, cons, enum, name):
1038 self.emit("case %s_kind:" % cons.name, 1)
1039 self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % cons.name, 2);
1040 self.emit("if (!result) goto failed;", 2)
1041 for f in cons.fields:
1042 self.visitField(f, cons.name, 2, False)
1043 self.emit("break;", 2)
1044
1045 def visitField(self, field, name, depth, product):
1046 def emit(s, d):
1047 self.emit(s, depth + d)
1048 if product:
1049 value = "o->%s" % field.name
1050 else:
1051 value = "o->v.%s.%s" % (name, field.name)
1052 self.set(field, value, depth)
1053 emit("if (!value) goto failed;", 0)
1054 emit('if (PyObject_SetAttrString(result, "%s", value) == -1)' % field.name, 0)
1055 emit("goto failed;", 1)
1056 emit("Py_DECREF(value);", 0)
1057
1058 def emitSeq(self, field, value, depth, emit):
1059 emit("seq = %s;" % value, 0)
1060 emit("n = asdl_seq_LEN(seq);", 0)
1061 emit("value = PyList_New(n);", 0)
1062 emit("if (!value) goto failed;", 0)
1063 emit("for (i = 0; i < n; i++) {", 0)
1064 self.set("value", field, "asdl_seq_GET(seq, i)", depth + 1)
1065 emit("if (!value1) goto failed;", 1)
1066 emit("PyList_SET_ITEM(value, i, value1);", 1)
1067 emit("value1 = NULL;", 1)
1068 emit("}", 0)
1069
1070 def set(self, field, value, depth):
1071 if field.seq:
1072 # XXX should really check for is_simple, but that requires a symbol table
1073 if field.type.value == "cmpop":
1074 # While the sequence elements are stored as void*,
1075 # ast2obj_cmpop expects an enum
1076 self.emit("{", depth)
1077 self.emit("int i, n = asdl_seq_LEN(%s);" % value, depth+1)
1078 self.emit("value = PyList_New(n);", depth+1)
1079 self.emit("if (!value) goto failed;", depth+1)
1080 self.emit("for(i = 0; i < n; i++)", depth+1)
1081 # This cannot fail, so no need for error handling
1082 self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i)));" % value,
1083 depth+2, reflow=False)
1084 self.emit("}", depth)
1085 else:
1086 self.emit("value = ast2obj_list(%s, ast2obj_%s);" % (value, field.type), depth)
1087 else:
1088 ctype = get_c_type(field.type)
1089 self.emit("value = ast2obj_%s(%s);" % (field.type, value), depth, reflow=False)
1090
1091
1092class PartingShots(StaticVisitor):
1093
1094 CODE = """
1095PyObject* PyAST_mod2obj(mod_ty t)
1096{
1097 init_types();
1098 return ast2obj_mod(t);
1099}
1100
1101/* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */
1102mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
1103{
1104 mod_ty res;
1105 PyObject *req_type[] = {(PyObject*)Module_type, (PyObject*)Expression_type,
1106 (PyObject*)Interactive_type};
1107 char *req_name[] = {"Module", "Expression", "Interactive"};
1108 int isinstance;
1109 assert(0 <= mode && mode <= 2);
1110
1111 init_types();
1112
1113 isinstance = PyObject_IsInstance(ast, req_type[mode]);
1114 if (isinstance == -1)
1115 return NULL;
1116 if (!isinstance) {
1117 PyErr_Format(PyExc_TypeError, "expected %s node, got %.400s",
1118 req_name[mode], Py_TYPE(ast)->tp_name);
1119 return NULL;
1120 }
1121 if (obj2ast_mod(ast, &res, arena) != 0)
1122 return NULL;
1123 else
1124 return res;
1125}
1126
1127int PyAST_Check(PyObject* obj)
1128{
1129 init_types();
1130 return PyObject_IsInstance(obj, (PyObject*)&AST_type);
1131}
1132"""
1133
1134class ChainOfVisitors:
1135 def __init__(self, *visitors):
1136 self.visitors = visitors
1137
1138 def visit(self, object):
1139 for v in self.visitors:
1140 v.visit(object)
1141 v.emit("", 0)
1142
1143common_msg = "/* File automatically generated by %s. */\n\n"
1144
1145c_file_msg = """
1146/*
1147 __version__ %s.
1148
1149 This module must be committed separately after each AST grammar change;
1150 The __version__ number is set to the revision number of the commit
1151 containing the grammar change.
1152*/
1153
1154"""
1155
1156def main(srcfile):
1157 argv0 = sys.argv[0]
1158 components = argv0.split(os.sep)
1159 argv0 = os.sep.join(components[-2:])
1160 auto_gen_msg = common_msg % argv0
1161 mod = asdl.parse(srcfile)
1162 if not asdl.check(mod):
1163 sys.exit(1)
1164 if INC_DIR:
1165 p = "%s/%s-ast.h" % (INC_DIR, mod.name)
1166 f = open(p, "wb")
1167 f.write(auto_gen_msg)
1168 f.write('#include "asdl.h"\n\n')
1169 c = ChainOfVisitors(TypeDefVisitor(f),
1170 StructVisitor(f),
1171 PrototypeVisitor(f),
1172 )
1173 c.visit(mod)
1174 f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
1175 f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
1176 f.write("int PyAST_Check(PyObject* obj);\n")
1177 f.close()
1178
1179 if SRC_DIR:
1180 p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c")
1181 f = open(p, "wb")
1182 f.write(auto_gen_msg)
1183 f.write(c_file_msg % parse_version(mod))
1184 f.write('#include "Python.h"\n')
1185 f.write('#include "%s-ast.h"\n' % mod.name)
1186 f.write('\n')
1187 f.write("static PyTypeObject AST_type;\n")
1188 v = ChainOfVisitors(
1189 PyTypesDeclareVisitor(f),
1190 PyTypesVisitor(f),
1191 Obj2ModPrototypeVisitor(f),
1192 FunctionVisitor(f),
1193 ObjVisitor(f),
1194 Obj2ModVisitor(f),
1195 ASTModuleVisitor(f),
1196 PartingShots(f),
1197 )
1198 v.visit(mod)
1199 f.close()
1200
1201if __name__ == "__main__":
1202 import sys
1203 import getopt
1204
1205 INC_DIR = ''
1206 SRC_DIR = ''
1207 opts, args = getopt.getopt(sys.argv[1:], "h:c:")
1208 if len(opts) != 1:
1209 print "Must specify exactly one output file"
1210 sys.exit(1)
1211 for o, v in opts:
1212 if o == '-h':
1213 INC_DIR = v
1214 if o == '-c':
1215 SRC_DIR = v
1216 if len(args) != 1:
1217 print "Must specify single input file"
1218 sys.exit(1)
1219 main(args[0])
Note: See TracBrowser for help on using the repository browser.