source: python/trunk/Python/marshal.c@ 389

Last change on this file since 389 was 2, checked in by Yuri Dario, 15 years ago

Initial import for vendor code.
  • Property svn:eol-style set to native
File size: 28.9 KB
Line 
1
2/* Write Python objects to files and read them back.
3 This is intended for writing and reading compiled Python code only;
4 a true persistent storage facility would be much harder, since
5 it would have to take circular links and sharing into account. */
6
7#define PY_SSIZE_T_CLEAN
8
9#include "Python.h"
10#include "longintrepr.h"
11#include "code.h"
12#include "marshal.h"
13
14/* High water mark to determine when the marshalled object is dangerously deep
15 * and risks coring the interpreter. When the object stack gets this deep,
16 * raise an exception instead of continuing.
17 */
18#define MAX_MARSHAL_STACK_DEPTH 2000
19
20#define TYPE_NULL '0'
21#define TYPE_NONE 'N'
22#define TYPE_FALSE 'F'
23#define TYPE_TRUE 'T'
24#define TYPE_STOPITER 'S'
25#define TYPE_ELLIPSIS '.'
26#define TYPE_INT 'i'
27#define TYPE_INT64 'I'
28#define TYPE_FLOAT 'f'
29#define TYPE_BINARY_FLOAT 'g'
30#define TYPE_COMPLEX 'x'
31#define TYPE_BINARY_COMPLEX 'y'
32#define TYPE_LONG 'l'
33#define TYPE_STRING 's'
34#define TYPE_INTERNED 't'
35#define TYPE_STRINGREF 'R'
36#define TYPE_TUPLE '('
37#define TYPE_LIST '['
38#define TYPE_DICT '{'
39#define TYPE_CODE 'c'
40#define TYPE_UNICODE 'u'
41#define TYPE_UNKNOWN '?'
42#define TYPE_SET '<'
43#define TYPE_FROZENSET '>'
44
45typedef struct {
46 FILE *fp;
47 int error;
48 int depth;
49 /* If fp == NULL, the following are valid: */
50 PyObject *str;
51 char *ptr;
52 char *end;
53 PyObject *strings; /* dict on marshal, list on unmarshal */
54 int version;
55} WFILE;
56
57#define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
58 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
59 else w_more(c, p)
60
61static void
62w_more(int c, WFILE *p)
63{
64 Py_ssize_t size, newsize;
65 if (p->str == NULL)
66 return; /* An error already occurred */
67 size = PyString_Size(p->str);
68 newsize = size + size + 1024;
69 if (newsize > 32*1024*1024) {
70 newsize = size + (size >> 3); /* 12.5% overallocation */
71 }
72 if (_PyString_Resize(&p->str, newsize) != 0) {
73 p->ptr = p->end = NULL;
74 }
75 else {
76 p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
77 p->end =
78 PyString_AS_STRING((PyStringObject *)p->str) + newsize;
79 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
80 }
81}
82
83static void
84w_string(char *s, int n, WFILE *p)
85{
86 if (p->fp != NULL) {
87 fwrite(s, 1, n, p->fp);
88 }
89 else {
90 while (--n >= 0) {
91 w_byte(*s, p);
92 s++;
93 }
94 }
95}
96
97static void
98w_short(int x, WFILE *p)
99{
100 w_byte((char)( x & 0xff), p);
101 w_byte((char)((x>> 8) & 0xff), p);
102}
103
104static void
105w_long(long x, WFILE *p)
106{
107 w_byte((char)( x & 0xff), p);
108 w_byte((char)((x>> 8) & 0xff), p);
109 w_byte((char)((x>>16) & 0xff), p);
110 w_byte((char)((x>>24) & 0xff), p);
111}
112
113#if SIZEOF_LONG > 4
114static void
115w_long64(long x, WFILE *p)
116{
117 w_long(x, p);
118 w_long(x>>32, p);
119}
120#endif
121
122static void
123w_object(PyObject *v, WFILE *p)
124{
125 Py_ssize_t i, n;
126
127 p->depth++;
128
129 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
130 p->error = 2;
131 }
132 else if (v == NULL) {
133 w_byte(TYPE_NULL, p);
134 }
135 else if (v == Py_None) {
136 w_byte(TYPE_NONE, p);
137 }
138 else if (v == PyExc_StopIteration) {
139 w_byte(TYPE_STOPITER, p);
140 }
141 else if (v == Py_Ellipsis) {
142 w_byte(TYPE_ELLIPSIS, p);
143 }
144 else if (v == Py_False) {
145 w_byte(TYPE_FALSE, p);
146 }
147 else if (v == Py_True) {
148 w_byte(TYPE_TRUE, p);
149 }
150 else if (PyInt_CheckExact(v)) {
151 long x = PyInt_AS_LONG((PyIntObject *)v);
152#if SIZEOF_LONG > 4
153 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
154 if (y && y != -1) {
155 w_byte(TYPE_INT64, p);
156 w_long64(x, p);
157 }
158 else
159#endif
160 {
161 w_byte(TYPE_INT, p);
162 w_long(x, p);
163 }
164 }
165 else if (PyLong_CheckExact(v)) {
166 PyLongObject *ob = (PyLongObject *)v;
167 w_byte(TYPE_LONG, p);
168 n = ob->ob_size;
169 w_long((long)n, p);
170 if (n < 0)
171 n = -n;
172 for (i = 0; i < n; i++)
173 w_short(ob->ob_digit[i], p);
174 }
175 else if (PyFloat_CheckExact(v)) {
176 if (p->version > 1) {
177 unsigned char buf[8];
178 if (_PyFloat_Pack8(PyFloat_AsDouble(v),
179 buf, 1) < 0) {
180 p->error = 1;
181 return;
182 }
183 w_byte(TYPE_BINARY_FLOAT, p);
184 w_string((char*)buf, 8, p);
185 }
186 else {
187 char buf[256]; /* Plenty to format any double */
188 PyFloat_AsReprString(buf, (PyFloatObject *)v);
189 n = strlen(buf);
190 w_byte(TYPE_FLOAT, p);
191 w_byte((int)n, p);
192 w_string(buf, (int)n, p);
193 }
194 }
195#ifndef WITHOUT_COMPLEX
196 else if (PyComplex_CheckExact(v)) {
197 if (p->version > 1) {
198 unsigned char buf[8];
199 if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
200 buf, 1) < 0) {
201 p->error = 1;
202 return;
203 }
204 w_byte(TYPE_BINARY_COMPLEX, p);
205 w_string((char*)buf, 8, p);
206 if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
207 buf, 1) < 0) {
208 p->error = 1;
209 return;
210 }
211 w_string((char*)buf, 8, p);
212 }
213 else {
214 char buf[256]; /* Plenty to format any double */
215 PyFloatObject *temp;
216 w_byte(TYPE_COMPLEX, p);
217 temp = (PyFloatObject*)PyFloat_FromDouble(
218 PyComplex_RealAsDouble(v));
219 if (!temp) {
220 p->error = 1;
221 return;
222 }
223 PyFloat_AsReprString(buf, temp);
224 Py_DECREF(temp);
225 n = strlen(buf);
226 w_byte((int)n, p);
227 w_string(buf, (int)n, p);
228 temp = (PyFloatObject*)PyFloat_FromDouble(
229 PyComplex_ImagAsDouble(v));
230 if (!temp) {
231 p->error = 1;
232 return;
233 }
234 PyFloat_AsReprString(buf, temp);
235 Py_DECREF(temp);
236 n = strlen(buf);
237 w_byte((int)n, p);
238 w_string(buf, (int)n, p);
239 }
240 }
241#endif
242 else if (PyString_CheckExact(v)) {
243 if (p->strings && PyString_CHECK_INTERNED(v)) {
244 PyObject *o = PyDict_GetItem(p->strings, v);
245 if (o) {
246 long w = PyInt_AsLong(o);
247 w_byte(TYPE_STRINGREF, p);
248 w_long(w, p);
249 goto exit;
250 }
251 else {
252 int ok;
253 o = PyInt_FromSsize_t(PyDict_Size(p->strings));
254 ok = o &&
255 PyDict_SetItem(p->strings, v, o) >= 0;
256 Py_XDECREF(o);
257 if (!ok) {
258 p->depth--;
259 p->error = 1;
260 return;
261 }
262 w_byte(TYPE_INTERNED, p);
263 }
264 }
265 else {
266 w_byte(TYPE_STRING, p);
267 }
268 n = PyString_GET_SIZE(v);
269 if (n > INT_MAX) {
270 /* huge strings are not supported */
271 p->depth--;
272 p->error = 1;
273 return;
274 }
275 w_long((long)n, p);
276 w_string(PyString_AS_STRING(v), (int)n, p);
277 }
278#ifdef Py_USING_UNICODE
279 else if (PyUnicode_CheckExact(v)) {
280 PyObject *utf8;
281 utf8 = PyUnicode_AsUTF8String(v);
282 if (utf8 == NULL) {
283 p->depth--;
284 p->error = 1;
285 return;
286 }
287 w_byte(TYPE_UNICODE, p);
288 n = PyString_GET_SIZE(utf8);
289 if (n > INT_MAX) {
290 p->depth--;
291 p->error = 1;
292 return;
293 }
294 w_long((long)n, p);
295 w_string(PyString_AS_STRING(utf8), (int)n, p);
296 Py_DECREF(utf8);
297 }
298#endif
299 else if (PyTuple_CheckExact(v)) {
300 w_byte(TYPE_TUPLE, p);
301 n = PyTuple_Size(v);
302 w_long((long)n, p);
303 for (i = 0; i < n; i++) {
304 w_object(PyTuple_GET_ITEM(v, i), p);
305 }
306 }
307 else if (PyList_CheckExact(v)) {
308 w_byte(TYPE_LIST, p);
309 n = PyList_GET_SIZE(v);
310 w_long((long)n, p);
311 for (i = 0; i < n; i++) {
312 w_object(PyList_GET_ITEM(v, i), p);
313 }
314 }
315 else if (PyDict_CheckExact(v)) {
316 Py_ssize_t pos;
317 PyObject *key, *value;
318 w_byte(TYPE_DICT, p);
319 /* This one is NULL object terminated! */
320 pos = 0;
321 while (PyDict_Next(v, &pos, &key, &value)) {
322 w_object(key, p);
323 w_object(value, p);
324 }
325 w_object((PyObject *)NULL, p);
326 }
327 else if (PyAnySet_CheckExact(v)) {
328 PyObject *value, *it;
329
330 if (PyObject_TypeCheck(v, &PySet_Type))
331 w_byte(TYPE_SET, p);
332 else
333 w_byte(TYPE_FROZENSET, p);
334 n = PyObject_Size(v);
335 if (n == -1) {
336 p->depth--;
337 p->error = 1;
338 return;
339 }
340 w_long((long)n, p);
341 it = PyObject_GetIter(v);
342 if (it == NULL) {
343 p->depth--;
344 p->error = 1;
345 return;
346 }
347 while ((value = PyIter_Next(it)) != NULL) {
348 w_object(value, p);
349 Py_DECREF(value);
350 }
351 Py_DECREF(it);
352 if (PyErr_Occurred()) {
353 p->depth--;
354 p->error = 1;
355 return;
356 }
357 }
358 else if (PyCode_Check(v)) {
359 PyCodeObject *co = (PyCodeObject *)v;
360 w_byte(TYPE_CODE, p);
361 w_long(co->co_argcount, p);
362 w_long(co->co_nlocals, p);
363 w_long(co->co_stacksize, p);
364 w_long(co->co_flags, p);
365 w_object(co->co_code, p);
366 w_object(co->co_consts, p);
367 w_object(co->co_names, p);
368 w_object(co->co_varnames, p);
369 w_object(co->co_freevars, p);
370 w_object(co->co_cellvars, p);
371 w_object(co->co_filename, p);
372 w_object(co->co_name, p);
373 w_long(co->co_firstlineno, p);
374 w_object(co->co_lnotab, p);
375 }
376 else if (PyObject_CheckReadBuffer(v)) {
377 /* Write unknown buffer-style objects as a string */
378 char *s;
379 PyBufferProcs *pb = v->ob_type->tp_as_buffer;
380 w_byte(TYPE_STRING, p);
381 n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
382 if (n > INT_MAX) {
383 p->depth--;
384 p->error = 1;
385 return;
386 }
387 w_long((long)n, p);
388 w_string(s, (int)n, p);
389 }
390 else {
391 w_byte(TYPE_UNKNOWN, p);
392 p->error = 1;
393 }
394 exit:
395 p->depth--;
396}
397
398/* version currently has no effect for writing longs. */
399void
400PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
401{
402 WFILE wf;
403 wf.fp = fp;
404 wf.error = 0;
405 wf.depth = 0;
406 wf.strings = NULL;
407 wf.version = version;
408 w_long(x, &wf);
409}
410
411void
412PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
413{
414 WFILE wf;
415 wf.fp = fp;
416 wf.error = 0;
417 wf.depth = 0;
418 wf.strings = (version > 0) ? PyDict_New() : NULL;
419 wf.version = version;
420 w_object(x, &wf);
421 Py_XDECREF(wf.strings);
422}
423
424typedef WFILE RFILE; /* Same struct with different invariants */
425
426#define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
427
428#define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
429
430static int
431r_string(char *s, int n, RFILE *p)
432{
433 if (p->fp != NULL)
434 /* The result fits into int because it must be <=n. */
435 return (int)fread(s, 1, n, p->fp);
436 if (p->end - p->ptr < n)
437 n = (int)(p->end - p->ptr);
438 memcpy(s, p->ptr, n);
439 p->ptr += n;
440 return n;
441}
442
443static int
444r_short(RFILE *p)
445{
446 register short x;
447 x = r_byte(p);
448 x |= r_byte(p) << 8;
449 /* Sign-extension, in case short greater than 16 bits */
450 x |= -(x & 0x8000);
451 return x;
452}
453
454static long
455r_long(RFILE *p)
456{
457 register long x;
458 register FILE *fp = p->fp;
459 if (fp) {
460 x = getc(fp);
461 x |= (long)getc(fp) << 8;
462 x |= (long)getc(fp) << 16;
463 x |= (long)getc(fp) << 24;
464 }
465 else {
466 x = rs_byte(p);
467 x |= (long)rs_byte(p) << 8;
468 x |= (long)rs_byte(p) << 16;
469 x |= (long)rs_byte(p) << 24;
470 }
471#if SIZEOF_LONG > 4
472 /* Sign extension for 64-bit machines */
473 x |= -(x & 0x80000000L);
474#endif
475 return x;
476}
477
478/* r_long64 deals with the TYPE_INT64 code. On a machine with
479 sizeof(long) > 4, it returns a Python int object, else a Python long
480 object. Note that w_long64 writes out TYPE_INT if 32 bits is enough,
481 so there's no inefficiency here in returning a PyLong on 32-bit boxes
482 for everything written via TYPE_INT64 (i.e., if an int is written via
483 TYPE_INT64, it *needs* more than 32 bits).
484*/
485static PyObject *
486r_long64(RFILE *p)
487{
488 long lo4 = r_long(p);
489 long hi4 = r_long(p);
490#if SIZEOF_LONG > 4
491 long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
492 return PyInt_FromLong(x);
493#else
494 unsigned char buf[8];
495 int one = 1;
496 int is_little_endian = (int)*(char*)&one;
497 if (is_little_endian) {
498 memcpy(buf, &lo4, 4);
499 memcpy(buf+4, &hi4, 4);
500 }
501 else {
502 memcpy(buf, &hi4, 4);
503 memcpy(buf+4, &lo4, 4);
504 }
505 return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
506#endif
507}
508
509static PyObject *
510r_object(RFILE *p)
511{
512 /* NULL is a valid return value, it does not necessarily means that
513 an exception is set. */
514 PyObject *v, *v2;
515 long i, n;
516 int type = r_byte(p);
517 PyObject *retval;
518
519 p->depth++;
520
521 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
522 p->depth--;
523 PyErr_SetString(PyExc_ValueError, "recursion limit exceeded");
524 return NULL;
525 }
526
527 switch (type) {
528
529 case EOF:
530 PyErr_SetString(PyExc_EOFError,
531 "EOF read where object expected");
532 retval = NULL;
533 break;
534
535 case TYPE_NULL:
536 retval = NULL;
537 break;
538
539 case TYPE_NONE:
540 Py_INCREF(Py_None);
541 retval = Py_None;
542 break;
543
544 case TYPE_STOPITER:
545 Py_INCREF(PyExc_StopIteration);
546 retval = PyExc_StopIteration;
547 break;
548
549 case TYPE_ELLIPSIS:
550 Py_INCREF(Py_Ellipsis);
551 retval = Py_Ellipsis;
552 break;
553
554 case TYPE_FALSE:
555 Py_INCREF(Py_False);
556 retval = Py_False;
557 break;
558
559 case TYPE_TRUE:
560 Py_INCREF(Py_True);
561 retval = Py_True;
562 break;
563
564 case TYPE_INT:
565 retval = PyInt_FromLong(r_long(p));
566 break;
567
568 case TYPE_INT64:
569 retval = r_long64(p);
570 break;
571
572 case TYPE_LONG:
573 {
574 int size;
575 PyLongObject *ob;
576 n = r_long(p);
577 if (n < -INT_MAX || n > INT_MAX) {
578 PyErr_SetString(PyExc_ValueError,
579 "bad marshal data");
580 retval = NULL;
581 break;
582 }
583 size = n<0 ? -n : n;
584 ob = _PyLong_New(size);
585 if (ob == NULL) {
586 retval = NULL;
587 break;
588 }
589 ob->ob_size = n;
590 for (i = 0; i < size; i++) {
591 int digit = r_short(p);
592 if (digit < 0 ||
593 (digit == 0 && i == size-1)) {
594 Py_DECREF(ob);
595 PyErr_SetString(PyExc_ValueError,
596 "bad marshal data");
597 ob = NULL;
598 break;
599 }
600 if (ob != NULL)
601 ob->ob_digit[i] = digit;
602 }
603 retval = (PyObject *)ob;
604 break;
605 }
606
607 case TYPE_FLOAT:
608 {
609 char buf[256];
610 double dx;
611 n = r_byte(p);
612 if (n == EOF || r_string(buf, (int)n, p) != n) {
613 PyErr_SetString(PyExc_EOFError,
614 "EOF read where object expected");
615 retval = NULL;
616 break;
617 }
618 buf[n] = '\0';
619 retval = NULL;
620 PyFPE_START_PROTECT("atof", break)
621 dx = PyOS_ascii_atof(buf);
622 PyFPE_END_PROTECT(dx)
623 retval = PyFloat_FromDouble(dx);
624 break;
625 }
626
627 case TYPE_BINARY_FLOAT:
628 {
629 unsigned char buf[8];
630 double x;
631 if (r_string((char*)buf, 8, p) != 8) {
632 PyErr_SetString(PyExc_EOFError,
633 "EOF read where object expected");
634 retval = NULL;
635 break;
636 }
637 x = _PyFloat_Unpack8(buf, 1);
638 if (x == -1.0 && PyErr_Occurred()) {
639 retval = NULL;
640 break;
641 }
642 retval = PyFloat_FromDouble(x);
643 break;
644 }
645
646#ifndef WITHOUT_COMPLEX
647 case TYPE_COMPLEX:
648 {
649 char buf[256];
650 Py_complex c;
651 n = r_byte(p);
652 if (n == EOF || r_string(buf, (int)n, p) != n) {
653 PyErr_SetString(PyExc_EOFError,
654 "EOF read where object expected");
655 retval = NULL;
656 break;
657 }
658 buf[n] = '\0';
659 retval = NULL;
660 PyFPE_START_PROTECT("atof", break;)
661 c.real = PyOS_ascii_atof(buf);
662 PyFPE_END_PROTECT(c)
663 n = r_byte(p);
664 if (n == EOF || r_string(buf, (int)n, p) != n) {
665 PyErr_SetString(PyExc_EOFError,
666 "EOF read where object expected");
667 retval = NULL;
668 break;
669 }
670 buf[n] = '\0';
671 PyFPE_START_PROTECT("atof", break)
672 c.imag = PyOS_ascii_atof(buf);
673 PyFPE_END_PROTECT(c)
674 retval = PyComplex_FromCComplex(c);
675 break;
676 }
677
678 case TYPE_BINARY_COMPLEX:
679 {
680 unsigned char buf[8];
681 Py_complex c;
682 if (r_string((char*)buf, 8, p) != 8) {
683 PyErr_SetString(PyExc_EOFError,
684 "EOF read where object expected");
685 retval = NULL;
686 break;
687 }
688 c.real = _PyFloat_Unpack8(buf, 1);
689 if (c.real == -1.0 && PyErr_Occurred()) {
690 retval = NULL;
691 break;
692 }
693 if (r_string((char*)buf, 8, p) != 8) {
694 PyErr_SetString(PyExc_EOFError,
695 "EOF read where object expected");
696 retval = NULL;
697 break;
698 }
699 c.imag = _PyFloat_Unpack8(buf, 1);
700 if (c.imag == -1.0 && PyErr_Occurred()) {
701 retval = NULL;
702 break;
703 }
704 retval = PyComplex_FromCComplex(c);
705 break;
706 }
707#endif
708
709 case TYPE_INTERNED:
710 case TYPE_STRING:
711 n = r_long(p);
712 if (n < 0 || n > INT_MAX) {
713 PyErr_SetString(PyExc_ValueError, "bad marshal data");
714 retval = NULL;
715 break;
716 }
717 v = PyString_FromStringAndSize((char *)NULL, n);
718 if (v == NULL) {
719 retval = NULL;
720 break;
721 }
722 if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {
723 Py_DECREF(v);
724 PyErr_SetString(PyExc_EOFError,
725 "EOF read where object expected");
726 retval = NULL;
727 break;
728 }
729 if (type == TYPE_INTERNED) {
730 PyString_InternInPlace(&v);
731 if (PyList_Append(p->strings, v) < 0) {
732 retval = NULL;
733 break;
734 }
735 }
736 retval = v;
737 break;
738
739 case TYPE_STRINGREF:
740 n = r_long(p);
741 if (n < 0 || n >= PyList_GET_SIZE(p->strings)) {
742 PyErr_SetString(PyExc_ValueError, "bad marshal data");
743 retval = NULL;
744 break;
745 }
746 v = PyList_GET_ITEM(p->strings, n);
747 Py_INCREF(v);
748 retval = v;
749 break;
750
751#ifdef Py_USING_UNICODE
752 case TYPE_UNICODE:
753 {
754 char *buffer;
755
756 n = r_long(p);
757 if (n < 0 || n > INT_MAX) {
758 PyErr_SetString(PyExc_ValueError, "bad marshal data");
759 retval = NULL;
760 break;
761 }
762 buffer = PyMem_NEW(char, n);
763 if (buffer == NULL) {
764 retval = PyErr_NoMemory();
765 break;
766 }
767 if (r_string(buffer, (int)n, p) != n) {
768 PyMem_DEL(buffer);
769 PyErr_SetString(PyExc_EOFError,
770 "EOF read where object expected");
771 retval = NULL;
772 break;
773 }
774 v = PyUnicode_DecodeUTF8(buffer, n, NULL);
775 PyMem_DEL(buffer);
776 retval = v;
777 break;
778 }
779#endif
780
781 case TYPE_TUPLE:
782 n = r_long(p);
783 if (n < 0 || n > INT_MAX) {
784 PyErr_SetString(PyExc_ValueError, "bad marshal data");
785 retval = NULL;
786 break;
787 }
788 v = PyTuple_New((int)n);
789 if (v == NULL) {
790 retval = NULL;
791 break;
792 }
793 for (i = 0; i < n; i++) {
794 v2 = r_object(p);
795 if ( v2 == NULL ) {
796 if (!PyErr_Occurred())
797 PyErr_SetString(PyExc_TypeError,
798 "NULL object in marshal data");
799 Py_DECREF(v);
800 v = NULL;
801 break;
802 }
803 PyTuple_SET_ITEM(v, (int)i, v2);
804 }
805 retval = v;
806 break;
807
808 case TYPE_LIST:
809 n = r_long(p);
810 if (n < 0 || n > INT_MAX) {
811 PyErr_SetString(PyExc_ValueError, "bad marshal data");
812 retval = NULL;
813 break;
814 }
815 v = PyList_New((int)n);
816 if (v == NULL) {
817 retval = NULL;
818 break;
819 }
820 for (i = 0; i < n; i++) {
821 v2 = r_object(p);
822 if ( v2 == NULL ) {
823 if (!PyErr_Occurred())
824 PyErr_SetString(PyExc_TypeError,
825 "NULL object in marshal data");
826 Py_DECREF(v);
827 v = NULL;
828 break;
829 }
830 PyList_SET_ITEM(v, (int)i, v2);
831 }
832 retval = v;
833 break;
834
835 case TYPE_DICT:
836 v = PyDict_New();
837 if (v == NULL) {
838 retval = NULL;
839 break;
840 }
841 for (;;) {
842 PyObject *key, *val;
843 key = r_object(p);
844 if (key == NULL)
845 break;
846 val = r_object(p);
847 if (val != NULL)
848 PyDict_SetItem(v, key, val);
849 Py_DECREF(key);
850 Py_XDECREF(val);
851 }
852 if (PyErr_Occurred()) {
853 Py_DECREF(v);
854 v = NULL;
855 }
856 retval = v;
857 break;
858
859 case TYPE_SET:
860 case TYPE_FROZENSET:
861 n = r_long(p);
862 if (n < 0 || n > INT_MAX) {
863 PyErr_SetString(PyExc_ValueError, "bad marshal data");
864 retval = NULL;
865 break;
866 }
867 v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
868 if (v == NULL) {
869 retval = NULL;
870 break;
871 }
872 for (i = 0; i < n; i++) {
873 v2 = r_object(p);
874 if ( v2 == NULL ) {
875 if (!PyErr_Occurred())
876 PyErr_SetString(PyExc_TypeError,
877 "NULL object in marshal data");
878 Py_DECREF(v);
879 v = NULL;
880 break;
881 }
882 if (PySet_Add(v, v2) == -1) {
883 Py_DECREF(v);
884 Py_DECREF(v2);
885 v = NULL;
886 break;
887 }
888 Py_DECREF(v2);
889 }
890 retval = v;
891 break;
892
893 case TYPE_CODE:
894 if (PyEval_GetRestricted()) {
895 PyErr_SetString(PyExc_RuntimeError,
896 "cannot unmarshal code objects in "
897 "restricted execution mode");
898 retval = NULL;
899 break;
900 }
901 else {
902 int argcount;
903 int nlocals;
904 int stacksize;
905 int flags;
906 PyObject *code = NULL;
907 PyObject *consts = NULL;
908 PyObject *names = NULL;
909 PyObject *varnames = NULL;
910 PyObject *freevars = NULL;
911 PyObject *cellvars = NULL;
912 PyObject *filename = NULL;
913 PyObject *name = NULL;
914 int firstlineno;
915 PyObject *lnotab = NULL;
916
917 v = NULL;
918
919 /* XXX ignore long->int overflows for now */
920 argcount = (int)r_long(p);
921 nlocals = (int)r_long(p);
922 stacksize = (int)r_long(p);
923 flags = (int)r_long(p);
924 code = r_object(p);
925 if (code == NULL)
926 goto code_error;
927 consts = r_object(p);
928 if (consts == NULL)
929 goto code_error;
930 names = r_object(p);
931 if (names == NULL)
932 goto code_error;
933 varnames = r_object(p);
934 if (varnames == NULL)
935 goto code_error;
936 freevars = r_object(p);
937 if (freevars == NULL)
938 goto code_error;
939 cellvars = r_object(p);
940 if (cellvars == NULL)
941 goto code_error;
942 filename = r_object(p);
943 if (filename == NULL)
944 goto code_error;
945 name = r_object(p);
946 if (name == NULL)
947 goto code_error;
948 firstlineno = (int)r_long(p);
949 lnotab = r_object(p);
950 if (lnotab == NULL)
951 goto code_error;
952
953 v = (PyObject *) PyCode_New(
954 argcount, nlocals, stacksize, flags,
955 code, consts, names, varnames,
956 freevars, cellvars, filename, name,
957 firstlineno, lnotab);
958
959 code_error:
960 Py_XDECREF(code);
961 Py_XDECREF(consts);
962 Py_XDECREF(names);
963 Py_XDECREF(varnames);
964 Py_XDECREF(freevars);
965 Py_XDECREF(cellvars);
966 Py_XDECREF(filename);
967 Py_XDECREF(name);
968 Py_XDECREF(lnotab);
969
970 }
971 retval = v;
972 break;
973
974 default:
975 /* Bogus data got written, which isn't ideal.
976 This will let you keep working and recover. */
977 PyErr_SetString(PyExc_ValueError, "bad marshal data");
978 retval = NULL;
979 break;
980
981 }
982 p->depth--;
983 return retval;
984}
985
986static PyObject *
987read_object(RFILE *p)
988{
989 PyObject *v;
990 if (PyErr_Occurred()) {
991 fprintf(stderr, "XXX readobject called with exception set\n");
992 return NULL;
993 }
994 v = r_object(p);
995 if (v == NULL && !PyErr_Occurred())
996 PyErr_SetString(PyExc_TypeError, "NULL object in marshal data");
997 return v;
998}
999
1000int
1001PyMarshal_ReadShortFromFile(FILE *fp)
1002{
1003 RFILE rf;
1004 assert(fp);
1005 rf.fp = fp;
1006 rf.strings = NULL;
1007 rf.end = rf.ptr = NULL;
1008 return r_short(&rf);
1009}
1010
1011long
1012PyMarshal_ReadLongFromFile(FILE *fp)
1013{
1014 RFILE rf;
1015 rf.fp = fp;
1016 rf.strings = NULL;
1017 rf.ptr = rf.end = NULL;
1018 return r_long(&rf);
1019}
1020
1021#ifdef HAVE_FSTAT
1022/* Return size of file in bytes; < 0 if unknown. */
1023static off_t
1024getfilesize(FILE *fp)
1025{
1026 struct stat st;
1027 if (fstat(fileno(fp), &st) != 0)
1028 return -1;
1029 else
1030 return st.st_size;
1031}
1032#endif
1033
1034/* If we can get the size of the file up-front, and it's reasonably small,
1035 * read it in one gulp and delegate to ...FromString() instead. Much quicker
1036 * than reading a byte at a time from file; speeds .pyc imports.
1037 * CAUTION: since this may read the entire remainder of the file, don't
1038 * call it unless you know you're done with the file.
1039 */
1040PyObject *
1041PyMarshal_ReadLastObjectFromFile(FILE *fp)
1042{
1043/* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
1044 * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
1045 */
1046#define SMALL_FILE_LIMIT (1L << 14)
1047#define REASONABLE_FILE_LIMIT (1L << 18)
1048#ifdef HAVE_FSTAT
1049 off_t filesize;
1050#endif
1051#ifdef HAVE_FSTAT
1052 filesize = getfilesize(fp);
1053 if (filesize > 0) {
1054 char buf[SMALL_FILE_LIMIT];
1055 char* pBuf = NULL;
1056 if (filesize <= SMALL_FILE_LIMIT)
1057 pBuf = buf;
1058 else if (filesize <= REASONABLE_FILE_LIMIT)
1059 pBuf = (char *)PyMem_MALLOC(filesize);
1060 if (pBuf != NULL) {
1061 PyObject* v;
1062 size_t n;
1063 /* filesize must fit into an int, because it
1064 is smaller than REASONABLE_FILE_LIMIT */
1065 n = fread(pBuf, 1, (int)filesize, fp);
1066 v = PyMarshal_ReadObjectFromString(pBuf, n);
1067 if (pBuf != buf)
1068 PyMem_FREE(pBuf);
1069 return v;
1070 }
1071
1072 }
1073#endif
1074 /* We don't have fstat, or we do but the file is larger than
1075 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1076 */
1077 return PyMarshal_ReadObjectFromFile(fp);
1078
1079#undef SMALL_FILE_LIMIT
1080#undef REASONABLE_FILE_LIMIT
1081}
1082
1083PyObject *
1084PyMarshal_ReadObjectFromFile(FILE *fp)
1085{
1086 RFILE rf;
1087 PyObject *result;
1088 rf.fp = fp;
1089 rf.strings = PyList_New(0);
1090 rf.depth = 0;
1091 rf.ptr = rf.end = NULL;
1092 result = r_object(&rf);
1093 Py_DECREF(rf.strings);
1094 return result;
1095}
1096
1097PyObject *
1098PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
1099{
1100 RFILE rf;
1101 PyObject *result;
1102 rf.fp = NULL;
1103 rf.ptr = str;
1104 rf.end = str + len;
1105 rf.strings = PyList_New(0);
1106 rf.depth = 0;
1107 result = r_object(&rf);
1108 Py_DECREF(rf.strings);
1109 return result;
1110}
1111
1112PyObject *
1113PyMarshal_WriteObjectToString(PyObject *x, int version)
1114{
1115 WFILE wf;
1116 wf.fp = NULL;
1117 wf.str = PyString_FromStringAndSize((char *)NULL, 50);
1118 if (wf.str == NULL)
1119 return NULL;
1120 wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
1121 wf.end = wf.ptr + PyString_Size(wf.str);
1122 wf.error = 0;
1123 wf.depth = 0;
1124 wf.version = version;
1125 wf.strings = (version > 0) ? PyDict_New() : NULL;
1126 w_object(x, &wf);
1127 Py_XDECREF(wf.strings);
1128 if (wf.str != NULL) {
1129 char *base = PyString_AS_STRING((PyStringObject *)wf.str);
1130 if (wf.ptr - base > PY_SSIZE_T_MAX) {
1131 Py_DECREF(wf.str);
1132 PyErr_SetString(PyExc_OverflowError,
1133 "too much marshall data for a string");
1134 return NULL;
1135 }
1136 _PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base));
1137 }
1138 if (wf.error) {
1139 Py_XDECREF(wf.str);
1140 PyErr_SetString(PyExc_ValueError,
1141 (wf.error==1)?"unmarshallable object"
1142 :"object too deeply nested to marshal");
1143 return NULL;
1144 }
1145 return wf.str;
1146}
1147
1148/* And an interface for Python programs... */
1149
1150static PyObject *
1151marshal_dump(PyObject *self, PyObject *args)
1152{
1153 WFILE wf;
1154 PyObject *x;
1155 PyObject *f;
1156 int version = Py_MARSHAL_VERSION;
1157 if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version))
1158 return NULL;
1159 if (!PyFile_Check(f)) {
1160 PyErr_SetString(PyExc_TypeError,
1161 "marshal.dump() 2nd arg must be file");
1162 return NULL;
1163 }
1164 wf.fp = PyFile_AsFile(f);
1165 wf.str = NULL;
1166 wf.ptr = wf.end = NULL;
1167 wf.error = 0;
1168 wf.depth = 0;
1169 wf.strings = (version > 0) ? PyDict_New() : 0;
1170 wf.version = version;
1171 w_object(x, &wf);
1172 Py_XDECREF(wf.strings);
1173 if (wf.error) {
1174 PyErr_SetString(PyExc_ValueError,
1175 (wf.error==1)?"unmarshallable object"
1176 :"object too deeply nested to marshal");
1177 return NULL;
1178 }
1179 Py_INCREF(Py_None);
1180 return Py_None;
1181}
1182
1183PyDoc_STRVAR(dump_doc,
1184"dump(value, file[, version])\n\
1185\n\
1186Write the value on the open file. The value must be a supported type.\n\
1187The file must be an open file object such as sys.stdout or returned by\n\
1188open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
1189\n\
1190If the value has (or contains an object that has) an unsupported type, a\n\
1191ValueError exception is raised — but garbage data will also be written\n\
1192to the file. The object will not be properly read back by load()\n\
1193\n\
1194New in version 2.4: The version argument indicates the data format that\n\
1195dump should use.");
1196
1197static PyObject *
1198marshal_load(PyObject *self, PyObject *f)
1199{
1200 RFILE rf;
1201 PyObject *result;
1202 if (!PyFile_Check(f)) {
1203 PyErr_SetString(PyExc_TypeError,
1204 "marshal.load() arg must be file");
1205 return NULL;
1206 }
1207 rf.fp = PyFile_AsFile(f);
1208 rf.strings = PyList_New(0);
1209 rf.depth = 0;
1210 result = read_object(&rf);
1211 Py_DECREF(rf.strings);
1212 return result;
1213}
1214
1215PyDoc_STRVAR(load_doc,
1216"load(file)\n\
1217\n\
1218Read one value from the open file and return it. If no valid value is\n\
1219read (e.g. because the data has a different Python version’s\n\
1220incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
1221The file must be an open file object opened in binary mode ('rb' or\n\
1222'r+b').\n\
1223\n\
1224Note: If an object containing an unsupported type was marshalled with\n\
1225dump(), load() will substitute None for the unmarshallable type.");
1226
1227
1228static PyObject *
1229marshal_dumps(PyObject *self, PyObject *args)
1230{
1231 PyObject *x;
1232 int version = Py_MARSHAL_VERSION;
1233 if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version))
1234 return NULL;
1235 return PyMarshal_WriteObjectToString(x, version);
1236}
1237
1238PyDoc_STRVAR(dumps_doc,
1239"dumps(value[, version])\n\
1240\n\
1241Return the string that would be written to a file by dump(value, file).\n\
1242The value must be a supported type. Raise a ValueError exception if\n\
1243value has (or contains an object that has) an unsupported type.\n\
1244\n\
1245New in version 2.4: The version argument indicates the data format that\n\
1246dumps should use.");
1247
1248
1249static PyObject *
1250marshal_loads(PyObject *self, PyObject *args)
1251{
1252 RFILE rf;
1253 char *s;
1254 Py_ssize_t n;
1255 PyObject* result;
1256 if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
1257 return NULL;
1258 rf.fp = NULL;
1259 rf.ptr = s;
1260 rf.end = s + n;
1261 rf.strings = PyList_New(0);
1262 rf.depth = 0;
1263 result = read_object(&rf);
1264 Py_DECREF(rf.strings);
1265 return result;
1266}
1267
1268PyDoc_STRVAR(loads_doc,
1269"loads(string)\n\
1270\n\
1271Convert the string to a value. If no valid value is found, raise\n\
1272EOFError, ValueError or TypeError. Extra characters in the string are\n\
1273ignored.");
1274
1275static PyMethodDef marshal_methods[] = {
1276 {"dump", marshal_dump, METH_VARARGS, dump_doc},
1277 {"load", marshal_load, METH_O, load_doc},
1278 {"dumps", marshal_dumps, METH_VARARGS, dumps_doc},
1279 {"loads", marshal_loads, METH_VARARGS, loads_doc},
1280 {NULL, NULL} /* sentinel */
1281};
1282
1283PyDoc_STRVAR(marshal_doc,
1284"This module contains functions that can read and write Python values in\n\
1285a binary format. The format is specific to Python, but independent of\n\
1286machine architecture issues.\n\
1287\n\
1288Not all Python object types are supported; in general, only objects\n\
1289whose value is independent from a particular invocation of Python can be\n\
1290written and read by this module. The following types are supported:\n\
1291None, integers, long integers, floating point numbers, strings, Unicode\n\
1292objects, tuples, lists, sets, dictionaries, and code objects, where it\n\
1293should be understood that tuples, lists and dictionaries are only\n\
1294supported as long as the values contained therein are themselves\n\
1295supported; and recursive lists and dictionaries should not be written\n\
1296(they will cause infinite loops).\n\
1297\n\
1298Variables:\n\
1299\n\
1300version -- indicates the format that the module uses. Version 0 is the\n\
1301 historical format, version 1 (added in Python 2.4) shares interned\n\
1302 strings and version 2 (added in Python 2.5) uses a binary format for\n\
1303 floating point numbers. (New in version 2.4)\n\
1304\n\
1305Functions:\n\
1306\n\
1307dump() -- write value to a file\n\
1308load() -- read value from a file\n\
1309dumps() -- write value to a string\n\
1310loads() -- read value from a string");
1311
1312
1313PyMODINIT_FUNC
1314PyMarshal_Init(void)
1315{
1316 PyObject *mod = Py_InitModule3("marshal", marshal_methods,
1317 marshal_doc);
1318 if (mod == NULL)
1319 return;
1320 PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
1321}
Note: See TracBrowser for help on using the repository browser.