source: python/trunk/Python/marshal.c@ 391

Last change on this file since 391 was 391, checked in by dmik, 11 years ago

python: Merge vendor 2.7.6 to trunk.
  • Property svn:eol-style set to native
File size: 38.3 KB
Line 
1
2/* Write Python objects to files and read them back.
3 This is intended for writing and reading compiled Python code only;
4 a true persistent storage facility would be much harder, since
5 it would have to take circular links and sharing into account. */
6
7#define PY_SSIZE_T_CLEAN
8
9#include "Python.h"
10#include "longintrepr.h"
11#include "code.h"
12#include "marshal.h"
13
14#define ABS(x) ((x) < 0 ? -(x) : (x))
15
16/* High water mark to determine when the marshalled object is dangerously deep
17 * and risks coring the interpreter. When the object stack gets this deep,
18 * raise an exception instead of continuing.
19 */
20#define MAX_MARSHAL_STACK_DEPTH 2000
21
22#define TYPE_NULL '0'
23#define TYPE_NONE 'N'
24#define TYPE_FALSE 'F'
25#define TYPE_TRUE 'T'
26#define TYPE_STOPITER 'S'
27#define TYPE_ELLIPSIS '.'
28#define TYPE_INT 'i'
29#define TYPE_INT64 'I'
30#define TYPE_FLOAT 'f'
31#define TYPE_BINARY_FLOAT 'g'
32#define TYPE_COMPLEX 'x'
33#define TYPE_BINARY_COMPLEX 'y'
34#define TYPE_LONG 'l'
35#define TYPE_STRING 's'
36#define TYPE_INTERNED 't'
37#define TYPE_STRINGREF 'R'
38#define TYPE_TUPLE '('
39#define TYPE_LIST '['
40#define TYPE_DICT '{'
41#define TYPE_CODE 'c'
42#define TYPE_UNICODE 'u'
43#define TYPE_UNKNOWN '?'
44#define TYPE_SET '<'
45#define TYPE_FROZENSET '>'
46
47#define WFERR_OK 0
48#define WFERR_UNMARSHALLABLE 1
49#define WFERR_NESTEDTOODEEP 2
50#define WFERR_NOMEMORY 3
51
52typedef struct {
53 FILE *fp;
54 int error; /* see WFERR_* values */
55 int depth;
56 /* If fp == NULL, the following are valid: */
57 PyObject *str;
58 char *ptr;
59 char *end;
60 PyObject *strings; /* dict on marshal, list on unmarshal */
61 int version;
62} WFILE;
63
64#define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
65 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
66 else w_more(c, p)
67
68static void
69w_more(int c, WFILE *p)
70{
71 Py_ssize_t size, newsize;
72 if (p->str == NULL)
73 return; /* An error already occurred */
74 size = PyString_Size(p->str);
75 newsize = size + size + 1024;
76 if (newsize > 32*1024*1024) {
77 newsize = size + (size >> 3); /* 12.5% overallocation */
78 }
79 if (_PyString_Resize(&p->str, newsize) != 0) {
80 p->ptr = p->end = NULL;
81 }
82 else {
83 p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
84 p->end =
85 PyString_AS_STRING((PyStringObject *)p->str) + newsize;
86 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
87 }
88}
89
90static void
91w_string(const char *s, Py_ssize_t n, WFILE *p)
92{
93 if (p->fp != NULL) {
94 fwrite(s, 1, n, p->fp);
95 }
96 else {
97 while (--n >= 0) {
98 w_byte(*s, p);
99 s++;
100 }
101 }
102}
103
104static void
105w_short(int x, WFILE *p)
106{
107 w_byte((char)( x & 0xff), p);
108 w_byte((char)((x>> 8) & 0xff), p);
109}
110
111static void
112w_long(long x, WFILE *p)
113{
114 w_byte((char)( x & 0xff), p);
115 w_byte((char)((x>> 8) & 0xff), p);
116 w_byte((char)((x>>16) & 0xff), p);
117 w_byte((char)((x>>24) & 0xff), p);
118}
119
120#if SIZEOF_LONG > 4
121static void
122w_long64(long x, WFILE *p)
123{
124 w_long(x, p);
125 w_long(x>>32, p);
126}
127#endif
128
129#define SIZE32_MAX 0x7FFFFFFF
130
131#if SIZEOF_SIZE_T > 4
132# define W_SIZE(n, p) do { \
133 if ((n) > SIZE32_MAX) { \
134 (p)->depth--; \
135 (p)->error = WFERR_UNMARSHALLABLE; \
136 return; \
137 } \
138 w_long((long)(n), p); \
139 } while(0)
140#else
141# define W_SIZE w_long
142#endif
143
144static void
145w_pstring(const char *s, Py_ssize_t n, WFILE *p)
146{
147 W_SIZE(n, p);
148 w_string(s, n, p);
149}
150
151/* We assume that Python longs are stored internally in base some power of
152 2**15; for the sake of portability we'll always read and write them in base
153 exactly 2**15. */
154
155#define PyLong_MARSHAL_SHIFT 15
156#define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)
157#define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)
158#if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0
159#error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"
160#endif
161#define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)
162
163static void
164w_PyLong(const PyLongObject *ob, WFILE *p)
165{
166 Py_ssize_t i, j, n, l;
167 digit d;
168
169 w_byte(TYPE_LONG, p);
170 if (Py_SIZE(ob) == 0) {
171 w_long((long)0, p);
172 return;
173 }
174
175 /* set l to number of base PyLong_MARSHAL_BASE digits */
176 n = ABS(Py_SIZE(ob));
177 l = (n-1) * PyLong_MARSHAL_RATIO;
178 d = ob->ob_digit[n-1];
179 assert(d != 0); /* a PyLong is always normalized */
180 do {
181 d >>= PyLong_MARSHAL_SHIFT;
182 l++;
183 } while (d != 0);
184 if (l > SIZE32_MAX) {
185 p->depth--;
186 p->error = WFERR_UNMARSHALLABLE;
187 return;
188 }
189 w_long((long)(Py_SIZE(ob) > 0 ? l : -l), p);
190
191 for (i=0; i < n-1; i++) {
192 d = ob->ob_digit[i];
193 for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
194 w_short(d & PyLong_MARSHAL_MASK, p);
195 d >>= PyLong_MARSHAL_SHIFT;
196 }
197 assert (d == 0);
198 }
199 d = ob->ob_digit[n-1];
200 do {
201 w_short(d & PyLong_MARSHAL_MASK, p);
202 d >>= PyLong_MARSHAL_SHIFT;
203 } while (d != 0);
204}
205
206static void
207w_object(PyObject *v, WFILE *p)
208{
209 Py_ssize_t i, n;
210
211 p->depth++;
212
213 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
214 p->error = WFERR_NESTEDTOODEEP;
215 }
216 else if (v == NULL) {
217 w_byte(TYPE_NULL, p);
218 }
219 else if (v == Py_None) {
220 w_byte(TYPE_NONE, p);
221 }
222 else if (v == PyExc_StopIteration) {
223 w_byte(TYPE_STOPITER, p);
224 }
225 else if (v == Py_Ellipsis) {
226 w_byte(TYPE_ELLIPSIS, p);
227 }
228 else if (v == Py_False) {
229 w_byte(TYPE_FALSE, p);
230 }
231 else if (v == Py_True) {
232 w_byte(TYPE_TRUE, p);
233 }
234 else if (PyInt_CheckExact(v)) {
235 long x = PyInt_AS_LONG((PyIntObject *)v);
236#if SIZEOF_LONG > 4
237 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
238 if (y && y != -1) {
239 w_byte(TYPE_INT64, p);
240 w_long64(x, p);
241 }
242 else
243#endif
244 {
245 w_byte(TYPE_INT, p);
246 w_long(x, p);
247 }
248 }
249 else if (PyLong_CheckExact(v)) {
250 PyLongObject *ob = (PyLongObject *)v;
251 w_PyLong(ob, p);
252 }
253 else if (PyFloat_CheckExact(v)) {
254 if (p->version > 1) {
255 unsigned char buf[8];
256 if (_PyFloat_Pack8(PyFloat_AsDouble(v),
257 buf, 1) < 0) {
258 p->error = WFERR_UNMARSHALLABLE;
259 return;
260 }
261 w_byte(TYPE_BINARY_FLOAT, p);
262 w_string((char*)buf, 8, p);
263 }
264 else {
265 char *buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v),
266 'g', 17, 0, NULL);
267 if (!buf) {
268 p->error = WFERR_NOMEMORY;
269 return;
270 }
271 n = strlen(buf);
272 w_byte(TYPE_FLOAT, p);
273 w_byte((int)n, p);
274 w_string(buf, n, p);
275 PyMem_Free(buf);
276 }
277 }
278#ifndef WITHOUT_COMPLEX
279 else if (PyComplex_CheckExact(v)) {
280 if (p->version > 1) {
281 unsigned char buf[8];
282 if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
283 buf, 1) < 0) {
284 p->error = WFERR_UNMARSHALLABLE;
285 return;
286 }
287 w_byte(TYPE_BINARY_COMPLEX, p);
288 w_string((char*)buf, 8, p);
289 if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
290 buf, 1) < 0) {
291 p->error = WFERR_UNMARSHALLABLE;
292 return;
293 }
294 w_string((char*)buf, 8, p);
295 }
296 else {
297 char *buf;
298 w_byte(TYPE_COMPLEX, p);
299 buf = PyOS_double_to_string(PyComplex_RealAsDouble(v),
300 'g', 17, 0, NULL);
301 if (!buf) {
302 p->error = WFERR_NOMEMORY;
303 return;
304 }
305 n = strlen(buf);
306 w_byte((int)n, p);
307 w_string(buf, n, p);
308 PyMem_Free(buf);
309 buf = PyOS_double_to_string(PyComplex_ImagAsDouble(v),
310 'g', 17, 0, NULL);
311 if (!buf) {
312 p->error = WFERR_NOMEMORY;
313 return;
314 }
315 n = strlen(buf);
316 w_byte((int)n, p);
317 w_string(buf, n, p);
318 PyMem_Free(buf);
319 }
320 }
321#endif
322 else if (PyString_CheckExact(v)) {
323 if (p->strings && PyString_CHECK_INTERNED(v)) {
324 PyObject *o = PyDict_GetItem(p->strings, v);
325 if (o) {
326 long w = PyInt_AsLong(o);
327 w_byte(TYPE_STRINGREF, p);
328 w_long(w, p);
329 goto exit;
330 }
331 else {
332 int ok;
333 o = PyInt_FromSsize_t(PyDict_Size(p->strings));
334 ok = o &&
335 PyDict_SetItem(p->strings, v, o) >= 0;
336 Py_XDECREF(o);
337 if (!ok) {
338 p->depth--;
339 p->error = WFERR_UNMARSHALLABLE;
340 return;
341 }
342 w_byte(TYPE_INTERNED, p);
343 }
344 }
345 else {
346 w_byte(TYPE_STRING, p);
347 }
348 w_pstring(PyBytes_AS_STRING(v), PyString_GET_SIZE(v), p);
349 }
350#ifdef Py_USING_UNICODE
351 else if (PyUnicode_CheckExact(v)) {
352 PyObject *utf8;
353 utf8 = PyUnicode_AsUTF8String(v);
354 if (utf8 == NULL) {
355 p->depth--;
356 p->error = WFERR_UNMARSHALLABLE;
357 return;
358 }
359 w_byte(TYPE_UNICODE, p);
360 w_pstring(PyString_AS_STRING(utf8), PyString_GET_SIZE(utf8), p);
361 Py_DECREF(utf8);
362 }
363#endif
364 else if (PyTuple_CheckExact(v)) {
365 w_byte(TYPE_TUPLE, p);
366 n = PyTuple_Size(v);
367 W_SIZE(n, p);
368 for (i = 0; i < n; i++) {
369 w_object(PyTuple_GET_ITEM(v, i), p);
370 }
371 }
372 else if (PyList_CheckExact(v)) {
373 w_byte(TYPE_LIST, p);
374 n = PyList_GET_SIZE(v);
375 W_SIZE(n, p);
376 for (i = 0; i < n; i++) {
377 w_object(PyList_GET_ITEM(v, i), p);
378 }
379 }
380 else if (PyDict_CheckExact(v)) {
381 Py_ssize_t pos;
382 PyObject *key, *value;
383 w_byte(TYPE_DICT, p);
384 /* This one is NULL object terminated! */
385 pos = 0;
386 while (PyDict_Next(v, &pos, &key, &value)) {
387 w_object(key, p);
388 w_object(value, p);
389 }
390 w_object((PyObject *)NULL, p);
391 }
392 else if (PyAnySet_CheckExact(v)) {
393 PyObject *value, *it;
394
395 if (PyObject_TypeCheck(v, &PySet_Type))
396 w_byte(TYPE_SET, p);
397 else
398 w_byte(TYPE_FROZENSET, p);
399 n = PyObject_Size(v);
400 if (n == -1) {
401 p->depth--;
402 p->error = WFERR_UNMARSHALLABLE;
403 return;
404 }
405 W_SIZE(n, p);
406 it = PyObject_GetIter(v);
407 if (it == NULL) {
408 p->depth--;
409 p->error = WFERR_UNMARSHALLABLE;
410 return;
411 }
412 while ((value = PyIter_Next(it)) != NULL) {
413 w_object(value, p);
414 Py_DECREF(value);
415 }
416 Py_DECREF(it);
417 if (PyErr_Occurred()) {
418 p->depth--;
419 p->error = WFERR_UNMARSHALLABLE;
420 return;
421 }
422 }
423 else if (PyCode_Check(v)) {
424 PyCodeObject *co = (PyCodeObject *)v;
425 w_byte(TYPE_CODE, p);
426 w_long(co->co_argcount, p);
427 w_long(co->co_nlocals, p);
428 w_long(co->co_stacksize, p);
429 w_long(co->co_flags, p);
430 w_object(co->co_code, p);
431 w_object(co->co_consts, p);
432 w_object(co->co_names, p);
433 w_object(co->co_varnames, p);
434 w_object(co->co_freevars, p);
435 w_object(co->co_cellvars, p);
436 w_object(co->co_filename, p);
437 w_object(co->co_name, p);
438 w_long(co->co_firstlineno, p);
439 w_object(co->co_lnotab, p);
440 }
441 else if (PyObject_CheckReadBuffer(v)) {
442 /* Write unknown buffer-style objects as a string */
443 char *s;
444 PyBufferProcs *pb = v->ob_type->tp_as_buffer;
445 w_byte(TYPE_STRING, p);
446 n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
447 w_pstring(s, n, p);
448 }
449 else {
450 w_byte(TYPE_UNKNOWN, p);
451 p->error = WFERR_UNMARSHALLABLE;
452 }
453 exit:
454 p->depth--;
455}
456
457/* version currently has no effect for writing longs. */
458void
459PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
460{
461 WFILE wf;
462 wf.fp = fp;
463 wf.error = WFERR_OK;
464 wf.depth = 0;
465 wf.strings = NULL;
466 wf.version = version;
467 w_long(x, &wf);
468}
469
470void
471PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
472{
473 WFILE wf;
474 wf.fp = fp;
475 wf.error = WFERR_OK;
476 wf.depth = 0;
477 wf.strings = (version > 0) ? PyDict_New() : NULL;
478 wf.version = version;
479 w_object(x, &wf);
480 Py_XDECREF(wf.strings);
481}
482
483typedef WFILE RFILE; /* Same struct with different invariants */
484
485#define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
486
487#define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
488
489static Py_ssize_t
490r_string(char *s, Py_ssize_t n, RFILE *p)
491{
492 if (p->fp != NULL)
493 /* The result fits into int because it must be <=n. */
494 return fread(s, 1, n, p->fp);
495 if (p->end - p->ptr < n)
496 n = p->end - p->ptr;
497 memcpy(s, p->ptr, n);
498 p->ptr += n;
499 return n;
500}
501
502static int
503r_short(RFILE *p)
504{
505 register short x;
506 x = r_byte(p);
507 x |= r_byte(p) << 8;
508 /* Sign-extension, in case short greater than 16 bits */
509 x |= -(x & 0x8000);
510 return x;
511}
512
513static long
514r_long(RFILE *p)
515{
516 register long x;
517 register FILE *fp = p->fp;
518 if (fp) {
519 x = getc(fp);
520 x |= (long)getc(fp) << 8;
521 x |= (long)getc(fp) << 16;
522 x |= (long)getc(fp) << 24;
523 }
524 else {
525 x = rs_byte(p);
526 x |= (long)rs_byte(p) << 8;
527 x |= (long)rs_byte(p) << 16;
528 x |= (long)rs_byte(p) << 24;
529 }
530#if SIZEOF_LONG > 4
531 /* Sign extension for 64-bit machines */
532 x |= -(x & 0x80000000L);
533#endif
534 return x;
535}
536
537/* r_long64 deals with the TYPE_INT64 code. On a machine with
538 sizeof(long) > 4, it returns a Python int object, else a Python long
539 object. Note that w_long64 writes out TYPE_INT if 32 bits is enough,
540 so there's no inefficiency here in returning a PyLong on 32-bit boxes
541 for everything written via TYPE_INT64 (i.e., if an int is written via
542 TYPE_INT64, it *needs* more than 32 bits).
543*/
544static PyObject *
545r_long64(RFILE *p)
546{
547 long lo4 = r_long(p);
548 long hi4 = r_long(p);
549#if SIZEOF_LONG > 4
550 long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
551 return PyInt_FromLong(x);
552#else
553 unsigned char buf[8];
554 int one = 1;
555 int is_little_endian = (int)*(char*)&one;
556 if (is_little_endian) {
557 memcpy(buf, &lo4, 4);
558 memcpy(buf+4, &hi4, 4);
559 }
560 else {
561 memcpy(buf, &hi4, 4);
562 memcpy(buf+4, &lo4, 4);
563 }
564 return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
565#endif
566}
567
568static PyObject *
569r_PyLong(RFILE *p)
570{
571 PyLongObject *ob;
572 long n, size, i;
573 int j, md, shorts_in_top_digit;
574 digit d;
575
576 n = r_long(p);
577 if (n == 0)
578 return (PyObject *)_PyLong_New(0);
579 if (n < -SIZE32_MAX || n > SIZE32_MAX) {
580 PyErr_SetString(PyExc_ValueError,
581 "bad marshal data (long size out of range)");
582 return NULL;
583 }
584
585 size = 1 + (ABS(n) - 1) / PyLong_MARSHAL_RATIO;
586 shorts_in_top_digit = 1 + (ABS(n) - 1) % PyLong_MARSHAL_RATIO;
587 ob = _PyLong_New(size);
588 if (ob == NULL)
589 return NULL;
590 Py_SIZE(ob) = n > 0 ? size : -size;
591
592 for (i = 0; i < size-1; i++) {
593 d = 0;
594 for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
595 md = r_short(p);
596 if (md < 0 || md > PyLong_MARSHAL_BASE)
597 goto bad_digit;
598 d += (digit)md << j*PyLong_MARSHAL_SHIFT;
599 }
600 ob->ob_digit[i] = d;
601 }
602 d = 0;
603 for (j=0; j < shorts_in_top_digit; j++) {
604 md = r_short(p);
605 if (md < 0 || md > PyLong_MARSHAL_BASE)
606 goto bad_digit;
607 /* topmost marshal digit should be nonzero */
608 if (md == 0 && j == shorts_in_top_digit - 1) {
609 Py_DECREF(ob);
610 PyErr_SetString(PyExc_ValueError,
611 "bad marshal data (unnormalized long data)");
612 return NULL;
613 }
614 d += (digit)md << j*PyLong_MARSHAL_SHIFT;
615 }
616 /* top digit should be nonzero, else the resulting PyLong won't be
617 normalized */
618 ob->ob_digit[size-1] = d;
619 return (PyObject *)ob;
620 bad_digit:
621 Py_DECREF(ob);
622 PyErr_SetString(PyExc_ValueError,
623 "bad marshal data (digit out of range in long)");
624 return NULL;
625}
626
627
628static PyObject *
629r_object(RFILE *p)
630{
631 /* NULL is a valid return value, it does not necessarily means that
632 an exception is set. */
633 PyObject *v, *v2;
634 long i, n;
635 int type = r_byte(p);
636 PyObject *retval;
637
638 p->depth++;
639
640 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
641 p->depth--;
642 PyErr_SetString(PyExc_ValueError, "recursion limit exceeded");
643 return NULL;
644 }
645
646 switch (type) {
647
648 case EOF:
649 PyErr_SetString(PyExc_EOFError,
650 "EOF read where object expected");
651 retval = NULL;
652 break;
653
654 case TYPE_NULL:
655 retval = NULL;
656 break;
657
658 case TYPE_NONE:
659 Py_INCREF(Py_None);
660 retval = Py_None;
661 break;
662
663 case TYPE_STOPITER:
664 Py_INCREF(PyExc_StopIteration);
665 retval = PyExc_StopIteration;
666 break;
667
668 case TYPE_ELLIPSIS:
669 Py_INCREF(Py_Ellipsis);
670 retval = Py_Ellipsis;
671 break;
672
673 case TYPE_FALSE:
674 Py_INCREF(Py_False);
675 retval = Py_False;
676 break;
677
678 case TYPE_TRUE:
679 Py_INCREF(Py_True);
680 retval = Py_True;
681 break;
682
683 case TYPE_INT:
684 retval = PyInt_FromLong(r_long(p));
685 break;
686
687 case TYPE_INT64:
688 retval = r_long64(p);
689 break;
690
691 case TYPE_LONG:
692 retval = r_PyLong(p);
693 break;
694
695 case TYPE_FLOAT:
696 {
697 char buf[256];
698 double dx;
699 n = r_byte(p);
700 if (n == EOF || r_string(buf, n, p) != n) {
701 PyErr_SetString(PyExc_EOFError,
702 "EOF read where object expected");
703 retval = NULL;
704 break;
705 }
706 buf[n] = '\0';
707 dx = PyOS_string_to_double(buf, NULL, NULL);
708 if (dx == -1.0 && PyErr_Occurred()) {
709 retval = NULL;
710 break;
711 }
712 retval = PyFloat_FromDouble(dx);
713 break;
714 }
715
716 case TYPE_BINARY_FLOAT:
717 {
718 unsigned char buf[8];
719 double x;
720 if (r_string((char*)buf, 8, p) != 8) {
721 PyErr_SetString(PyExc_EOFError,
722 "EOF read where object expected");
723 retval = NULL;
724 break;
725 }
726 x = _PyFloat_Unpack8(buf, 1);
727 if (x == -1.0 && PyErr_Occurred()) {
728 retval = NULL;
729 break;
730 }
731 retval = PyFloat_FromDouble(x);
732 break;
733 }
734
735#ifndef WITHOUT_COMPLEX
736 case TYPE_COMPLEX:
737 {
738 char buf[256];
739 Py_complex c;
740 n = r_byte(p);
741 if (n == EOF || r_string(buf, n, p) != n) {
742 PyErr_SetString(PyExc_EOFError,
743 "EOF read where object expected");
744 retval = NULL;
745 break;
746 }
747 buf[n] = '\0';
748 c.real = PyOS_string_to_double(buf, NULL, NULL);
749 if (c.real == -1.0 && PyErr_Occurred()) {
750 retval = NULL;
751 break;
752 }
753 n = r_byte(p);
754 if (n == EOF || r_string(buf, n, p) != n) {
755 PyErr_SetString(PyExc_EOFError,
756 "EOF read where object expected");
757 retval = NULL;
758 break;
759 }
760 buf[n] = '\0';
761 c.imag = PyOS_string_to_double(buf, NULL, NULL);
762 if (c.imag == -1.0 && PyErr_Occurred()) {
763 retval = NULL;
764 break;
765 }
766 retval = PyComplex_FromCComplex(c);
767 break;
768 }
769
770 case TYPE_BINARY_COMPLEX:
771 {
772 unsigned char buf[8];
773 Py_complex c;
774 if (r_string((char*)buf, 8, p) != 8) {
775 PyErr_SetString(PyExc_EOFError,
776 "EOF read where object expected");
777 retval = NULL;
778 break;
779 }
780 c.real = _PyFloat_Unpack8(buf, 1);
781 if (c.real == -1.0 && PyErr_Occurred()) {
782 retval = NULL;
783 break;
784 }
785 if (r_string((char*)buf, 8, p) != 8) {
786 PyErr_SetString(PyExc_EOFError,
787 "EOF read where object expected");
788 retval = NULL;
789 break;
790 }
791 c.imag = _PyFloat_Unpack8(buf, 1);
792 if (c.imag == -1.0 && PyErr_Occurred()) {
793 retval = NULL;
794 break;
795 }
796 retval = PyComplex_FromCComplex(c);
797 break;
798 }
799#endif
800
801 case TYPE_INTERNED:
802 case TYPE_STRING:
803 n = r_long(p);
804 if (n < 0 || n > SIZE32_MAX) {
805 PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)");
806 retval = NULL;
807 break;
808 }
809 v = PyString_FromStringAndSize((char *)NULL, n);
810 if (v == NULL) {
811 retval = NULL;
812 break;
813 }
814 if (r_string(PyString_AS_STRING(v), n, p) != n) {
815 Py_DECREF(v);
816 PyErr_SetString(PyExc_EOFError,
817 "EOF read where object expected");
818 retval = NULL;
819 break;
820 }
821 if (type == TYPE_INTERNED) {
822 PyString_InternInPlace(&v);
823 if (PyList_Append(p->strings, v) < 0) {
824 retval = NULL;
825 break;
826 }
827 }
828 retval = v;
829 break;
830
831 case TYPE_STRINGREF:
832 n = r_long(p);
833 if (n < 0 || n >= PyList_GET_SIZE(p->strings)) {
834 PyErr_SetString(PyExc_ValueError, "bad marshal data (string ref out of range)");
835 retval = NULL;
836 break;
837 }
838 v = PyList_GET_ITEM(p->strings, n);
839 Py_INCREF(v);
840 retval = v;
841 break;
842
843#ifdef Py_USING_UNICODE
844 case TYPE_UNICODE:
845 {
846 char *buffer;
847
848 n = r_long(p);
849 if (n < 0 || n > SIZE32_MAX) {
850 PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)");
851 retval = NULL;
852 break;
853 }
854 buffer = PyMem_NEW(char, n);
855 if (buffer == NULL) {
856 retval = PyErr_NoMemory();
857 break;
858 }
859 if (r_string(buffer, n, p) != n) {
860 PyMem_DEL(buffer);
861 PyErr_SetString(PyExc_EOFError,
862 "EOF read where object expected");
863 retval = NULL;
864 break;
865 }
866 v = PyUnicode_DecodeUTF8(buffer, n, NULL);
867 PyMem_DEL(buffer);
868 retval = v;
869 break;
870 }
871#endif
872
873 case TYPE_TUPLE:
874 n = r_long(p);
875 if (n < 0 || n > SIZE32_MAX) {
876 PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)");
877 retval = NULL;
878 break;
879 }
880 v = PyTuple_New(n);
881 if (v == NULL) {
882 retval = NULL;
883 break;
884 }
885 for (i = 0; i < n; i++) {
886 v2 = r_object(p);
887 if ( v2 == NULL ) {
888 if (!PyErr_Occurred())
889 PyErr_SetString(PyExc_TypeError,
890 "NULL object in marshal data for tuple");
891 Py_DECREF(v);
892 v = NULL;
893 break;
894 }
895 PyTuple_SET_ITEM(v, i, v2);
896 }
897 retval = v;
898 break;
899
900 case TYPE_LIST:
901 n = r_long(p);
902 if (n < 0 || n > SIZE32_MAX) {
903 PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)");
904 retval = NULL;
905 break;
906 }
907 v = PyList_New(n);
908 if (v == NULL) {
909 retval = NULL;
910 break;
911 }
912 for (i = 0; i < n; i++) {
913 v2 = r_object(p);
914 if ( v2 == NULL ) {
915 if (!PyErr_Occurred())
916 PyErr_SetString(PyExc_TypeError,
917 "NULL object in marshal data for list");
918 Py_DECREF(v);
919 v = NULL;
920 break;
921 }
922 PyList_SET_ITEM(v, i, v2);
923 }
924 retval = v;
925 break;
926
927 case TYPE_DICT:
928 v = PyDict_New();
929 if (v == NULL) {
930 retval = NULL;
931 break;
932 }
933 for (;;) {
934 PyObject *key, *val;
935 key = r_object(p);
936 if (key == NULL)
937 break;
938 val = r_object(p);
939 if (val != NULL)
940 PyDict_SetItem(v, key, val);
941 Py_DECREF(key);
942 Py_XDECREF(val);
943 }
944 if (PyErr_Occurred()) {
945 Py_DECREF(v);
946 v = NULL;
947 }
948 retval = v;
949 break;
950
951 case TYPE_SET:
952 case TYPE_FROZENSET:
953 n = r_long(p);
954 if (n < 0 || n > SIZE32_MAX) {
955 PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)");
956 retval = NULL;
957 break;
958 }
959 v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
960 if (v == NULL) {
961 retval = NULL;
962 break;
963 }
964 for (i = 0; i < n; i++) {
965 v2 = r_object(p);
966 if ( v2 == NULL ) {
967 if (!PyErr_Occurred())
968 PyErr_SetString(PyExc_TypeError,
969 "NULL object in marshal data for set");
970 Py_DECREF(v);
971 v = NULL;
972 break;
973 }
974 if (PySet_Add(v, v2) == -1) {
975 Py_DECREF(v);
976 Py_DECREF(v2);
977 v = NULL;
978 break;
979 }
980 Py_DECREF(v2);
981 }
982 retval = v;
983 break;
984
985 case TYPE_CODE:
986 if (PyEval_GetRestricted()) {
987 PyErr_SetString(PyExc_RuntimeError,
988 "cannot unmarshal code objects in "
989 "restricted execution mode");
990 retval = NULL;
991 break;
992 }
993 else {
994 int argcount;
995 int nlocals;
996 int stacksize;
997 int flags;
998 PyObject *code = NULL;
999 PyObject *consts = NULL;
1000 PyObject *names = NULL;
1001 PyObject *varnames = NULL;
1002 PyObject *freevars = NULL;
1003 PyObject *cellvars = NULL;
1004 PyObject *filename = NULL;
1005 PyObject *name = NULL;
1006 int firstlineno;
1007 PyObject *lnotab = NULL;
1008
1009 v = NULL;
1010
1011 /* XXX ignore long->int overflows for now */
1012 argcount = (int)r_long(p);
1013 nlocals = (int)r_long(p);
1014 stacksize = (int)r_long(p);
1015 flags = (int)r_long(p);
1016 code = r_object(p);
1017 if (code == NULL)
1018 goto code_error;
1019 consts = r_object(p);
1020 if (consts == NULL)
1021 goto code_error;
1022 names = r_object(p);
1023 if (names == NULL)
1024 goto code_error;
1025 varnames = r_object(p);
1026 if (varnames == NULL)
1027 goto code_error;
1028 freevars = r_object(p);
1029 if (freevars == NULL)
1030 goto code_error;
1031 cellvars = r_object(p);
1032 if (cellvars == NULL)
1033 goto code_error;
1034 filename = r_object(p);
1035 if (filename == NULL)
1036 goto code_error;
1037 name = r_object(p);
1038 if (name == NULL)
1039 goto code_error;
1040 firstlineno = (int)r_long(p);
1041 lnotab = r_object(p);
1042 if (lnotab == NULL)
1043 goto code_error;
1044
1045 v = (PyObject *) PyCode_New(
1046 argcount, nlocals, stacksize, flags,
1047 code, consts, names, varnames,
1048 freevars, cellvars, filename, name,
1049 firstlineno, lnotab);
1050
1051 code_error:
1052 Py_XDECREF(code);
1053 Py_XDECREF(consts);
1054 Py_XDECREF(names);
1055 Py_XDECREF(varnames);
1056 Py_XDECREF(freevars);
1057 Py_XDECREF(cellvars);
1058 Py_XDECREF(filename);
1059 Py_XDECREF(name);
1060 Py_XDECREF(lnotab);
1061
1062 }
1063 retval = v;
1064 break;
1065
1066 default:
1067 /* Bogus data got written, which isn't ideal.
1068 This will let you keep working and recover. */
1069 PyErr_SetString(PyExc_ValueError, "bad marshal data (unknown type code)");
1070 retval = NULL;
1071 break;
1072
1073 }
1074 p->depth--;
1075 return retval;
1076}
1077
1078static PyObject *
1079read_object(RFILE *p)
1080{
1081 PyObject *v;
1082 if (PyErr_Occurred()) {
1083 fprintf(stderr, "XXX readobject called with exception set\n");
1084 return NULL;
1085 }
1086 v = r_object(p);
1087 if (v == NULL && !PyErr_Occurred())
1088 PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object");
1089 return v;
1090}
1091
1092int
1093PyMarshal_ReadShortFromFile(FILE *fp)
1094{
1095 RFILE rf;
1096 assert(fp);
1097 rf.fp = fp;
1098 rf.strings = NULL;
1099 rf.end = rf.ptr = NULL;
1100 return r_short(&rf);
1101}
1102
1103long
1104PyMarshal_ReadLongFromFile(FILE *fp)
1105{
1106 RFILE rf;
1107 rf.fp = fp;
1108 rf.strings = NULL;
1109 rf.ptr = rf.end = NULL;
1110 return r_long(&rf);
1111}
1112
1113#ifdef HAVE_FSTAT
1114/* Return size of file in bytes; < 0 if unknown. */
1115static off_t
1116getfilesize(FILE *fp)
1117{
1118 struct stat st;
1119 if (fstat(fileno(fp), &st) != 0)
1120 return -1;
1121 else
1122 return st.st_size;
1123}
1124#endif
1125
1126/* If we can get the size of the file up-front, and it's reasonably small,
1127 * read it in one gulp and delegate to ...FromString() instead. Much quicker
1128 * than reading a byte at a time from file; speeds .pyc imports.
1129 * CAUTION: since this may read the entire remainder of the file, don't
1130 * call it unless you know you're done with the file.
1131 */
1132PyObject *
1133PyMarshal_ReadLastObjectFromFile(FILE *fp)
1134{
1135/* REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc. */
1136#define REASONABLE_FILE_LIMIT (1L << 18)
1137#ifdef HAVE_FSTAT
1138 off_t filesize;
1139 filesize = getfilesize(fp);
1140 if (filesize > 0 && filesize <= REASONABLE_FILE_LIMIT) {
1141 char* pBuf = (char *)PyMem_MALLOC(filesize);
1142 if (pBuf != NULL) {
1143 size_t n = fread(pBuf, 1, (size_t)filesize, fp);
1144 PyObject* v = PyMarshal_ReadObjectFromString(pBuf, n);
1145 PyMem_FREE(pBuf);
1146 return v;
1147 }
1148
1149 }
1150#endif
1151 /* We don't have fstat, or we do but the file is larger than
1152 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1153 */
1154 return PyMarshal_ReadObjectFromFile(fp);
1155
1156#undef REASONABLE_FILE_LIMIT
1157}
1158
1159PyObject *
1160PyMarshal_ReadObjectFromFile(FILE *fp)
1161{
1162 RFILE rf;
1163 PyObject *result;
1164 rf.fp = fp;
1165 rf.strings = PyList_New(0);
1166 rf.depth = 0;
1167 rf.ptr = rf.end = NULL;
1168 result = r_object(&rf);
1169 Py_DECREF(rf.strings);
1170 return result;
1171}
1172
1173PyObject *
1174PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
1175{
1176 RFILE rf;
1177 PyObject *result;
1178 rf.fp = NULL;
1179 rf.ptr = str;
1180 rf.end = str + len;
1181 rf.strings = PyList_New(0);
1182 rf.depth = 0;
1183 result = r_object(&rf);
1184 Py_DECREF(rf.strings);
1185 return result;
1186}
1187
1188static void
1189set_error(int error)
1190{
1191 switch (error) {
1192 case WFERR_NOMEMORY:
1193 PyErr_NoMemory();
1194 break;
1195 case WFERR_UNMARSHALLABLE:
1196 PyErr_SetString(PyExc_ValueError, "unmarshallable object");
1197 break;
1198 case WFERR_NESTEDTOODEEP:
1199 default:
1200 PyErr_SetString(PyExc_ValueError,
1201 "object too deeply nested to marshal");
1202 break;
1203 }
1204}
1205
1206PyObject *
1207PyMarshal_WriteObjectToString(PyObject *x, int version)
1208{
1209 WFILE wf;
1210 wf.fp = NULL;
1211 wf.str = PyString_FromStringAndSize((char *)NULL, 50);
1212 if (wf.str == NULL)
1213 return NULL;
1214 wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
1215 wf.end = wf.ptr + PyString_Size(wf.str);
1216 wf.error = WFERR_OK;
1217 wf.depth = 0;
1218 wf.version = version;
1219 wf.strings = (version > 0) ? PyDict_New() : NULL;
1220 w_object(x, &wf);
1221 Py_XDECREF(wf.strings);
1222 if (wf.str != NULL) {
1223 char *base = PyString_AS_STRING((PyStringObject *)wf.str);
1224 if (wf.ptr - base > PY_SSIZE_T_MAX) {
1225 Py_DECREF(wf.str);
1226 PyErr_SetString(PyExc_OverflowError,
1227 "too much marshall data for a string");
1228 return NULL;
1229 }
1230 if (_PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)))
1231 return NULL;
1232 }
1233 if (wf.error != WFERR_OK) {
1234 Py_XDECREF(wf.str);
1235 set_error(wf.error);
1236 return NULL;
1237 }
1238 return wf.str;
1239}
1240
1241/* And an interface for Python programs... */
1242
1243static PyObject *
1244marshal_dump(PyObject *self, PyObject *args)
1245{
1246 WFILE wf;
1247 PyObject *x;
1248 PyObject *f;
1249 int version = Py_MARSHAL_VERSION;
1250 if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version))
1251 return NULL;
1252 if (!PyFile_Check(f)) {
1253 PyErr_SetString(PyExc_TypeError,
1254 "marshal.dump() 2nd arg must be file");
1255 return NULL;
1256 }
1257 wf.fp = PyFile_AsFile(f);
1258 wf.str = NULL;
1259 wf.ptr = wf.end = NULL;
1260 wf.error = WFERR_OK;
1261 wf.depth = 0;
1262 wf.strings = (version > 0) ? PyDict_New() : 0;
1263 wf.version = version;
1264 w_object(x, &wf);
1265 Py_XDECREF(wf.strings);
1266 if (wf.error != WFERR_OK) {
1267 set_error(wf.error);
1268 return NULL;
1269 }
1270 Py_INCREF(Py_None);
1271 return Py_None;
1272}
1273
1274PyDoc_STRVAR(dump_doc,
1275"dump(value, file[, version])\n\
1276\n\
1277Write the value on the open file. The value must be a supported type.\n\
1278The file must be an open file object such as sys.stdout or returned by\n\
1279open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
1280\n\
1281If the value has (or contains an object that has) an unsupported type, a\n\
1282ValueError exception is raised — but garbage data will also be written\n\
1283to the file. The object will not be properly read back by load()\n\
1284\n\
1285New in version 2.4: The version argument indicates the data format that\n\
1286dump should use.");
1287
1288static PyObject *
1289marshal_load(PyObject *self, PyObject *f)
1290{
1291 RFILE rf;
1292 PyObject *result;
1293 if (!PyFile_Check(f)) {
1294 PyErr_SetString(PyExc_TypeError,
1295 "marshal.load() arg must be file");
1296 return NULL;
1297 }
1298 rf.fp = PyFile_AsFile(f);
1299 rf.strings = PyList_New(0);
1300 rf.depth = 0;
1301 result = read_object(&rf);
1302 Py_DECREF(rf.strings);
1303 return result;
1304}
1305
1306PyDoc_STRVAR(load_doc,
1307"load(file)\n\
1308\n\
1309Read one value from the open file and return it. If no valid value is\n\
1310read (e.g. because the data has a different Python version’s\n\
1311incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
1312The file must be an open file object opened in binary mode ('rb' or\n\
1313'r+b').\n\
1314\n\
1315Note: If an object containing an unsupported type was marshalled with\n\
1316dump(), load() will substitute None for the unmarshallable type.");
1317
1318
1319static PyObject *
1320marshal_dumps(PyObject *self, PyObject *args)
1321{
1322 PyObject *x;
1323 int version = Py_MARSHAL_VERSION;
1324 if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version))
1325 return NULL;
1326 return PyMarshal_WriteObjectToString(x, version);
1327}
1328
1329PyDoc_STRVAR(dumps_doc,
1330"dumps(value[, version])\n\
1331\n\
1332Return the string that would be written to a file by dump(value, file).\n\
1333The value must be a supported type. Raise a ValueError exception if\n\
1334value has (or contains an object that has) an unsupported type.\n\
1335\n\
1336New in version 2.4: The version argument indicates the data format that\n\
1337dumps should use.");
1338
1339
1340static PyObject *
1341marshal_loads(PyObject *self, PyObject *args)
1342{
1343 RFILE rf;
1344 char *s;
1345 Py_ssize_t n;
1346 PyObject* result;
1347 if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
1348 return NULL;
1349 rf.fp = NULL;
1350 rf.ptr = s;
1351 rf.end = s + n;
1352 rf.strings = PyList_New(0);
1353 rf.depth = 0;
1354 result = read_object(&rf);
1355 Py_DECREF(rf.strings);
1356 return result;
1357}
1358
1359PyDoc_STRVAR(loads_doc,
1360"loads(string)\n\
1361\n\
1362Convert the string to a value. If no valid value is found, raise\n\
1363EOFError, ValueError or TypeError. Extra characters in the string are\n\
1364ignored.");
1365
1366static PyMethodDef marshal_methods[] = {
1367 {"dump", marshal_dump, METH_VARARGS, dump_doc},
1368 {"load", marshal_load, METH_O, load_doc},
1369 {"dumps", marshal_dumps, METH_VARARGS, dumps_doc},
1370 {"loads", marshal_loads, METH_VARARGS, loads_doc},
1371 {NULL, NULL} /* sentinel */
1372};
1373
1374PyDoc_STRVAR(marshal_doc,
1375"This module contains functions that can read and write Python values in\n\
1376a binary format. The format is specific to Python, but independent of\n\
1377machine architecture issues.\n\
1378\n\
1379Not all Python object types are supported; in general, only objects\n\
1380whose value is independent from a particular invocation of Python can be\n\
1381written and read by this module. The following types are supported:\n\
1382None, integers, long integers, floating point numbers, strings, Unicode\n\
1383objects, tuples, lists, sets, dictionaries, and code objects, where it\n\
1384should be understood that tuples, lists and dictionaries are only\n\
1385supported as long as the values contained therein are themselves\n\
1386supported; and recursive lists and dictionaries should not be written\n\
1387(they will cause infinite loops).\n\
1388\n\
1389Variables:\n\
1390\n\
1391version -- indicates the format that the module uses. Version 0 is the\n\
1392 historical format, version 1 (added in Python 2.4) shares interned\n\
1393 strings and version 2 (added in Python 2.5) uses a binary format for\n\
1394 floating point numbers. (New in version 2.4)\n\
1395\n\
1396Functions:\n\
1397\n\
1398dump() -- write value to a file\n\
1399load() -- read value from a file\n\
1400dumps() -- write value to a string\n\
1401loads() -- read value from a string");
1402
1403
1404PyMODINIT_FUNC
1405PyMarshal_Init(void)
1406{
1407 PyObject *mod = Py_InitModule3("marshal", marshal_methods,
1408 marshal_doc);
1409 if (mod == NULL)
1410 return;
1411 PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
1412}
Note: See TracBrowser for help on using the repository browser.