Changeset 391 for python/trunk/Lib/mailbox.py
- Timestamp:
- Mar 19, 2014, 11:31:01 PM (11 years ago)
- Location:
- python/trunk
- Files:
-
- 2 edited
-
. (modified) (1 prop)
-
Lib/mailbox.py (modified) (32 diffs)
Legend:
- Unmodified
- Added
- Removed
-
python/trunk
-
Property svn:mergeinfo
set to
/python/vendor/Python-2.7.6 merged eligible /python/vendor/current merged eligible
-
Property svn:mergeinfo
set to
-
python/trunk/Lib/mailbox.py
r2 r391 19 19 import email.message 20 20 import email.generator 21 import rfc82222 21 import StringIO 23 22 try: … … 28 27 except ImportError: 29 28 fcntl = None 29 30 import warnings 31 with warnings.catch_warnings(): 32 if sys.py3kwarning: 33 warnings.filterwarnings("ignore", ".*rfc822 has been removed", 34 DeprecationWarning) 35 import rfc822 30 36 31 37 __all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF', … … 191 197 """Flush and close the mailbox.""" 192 198 raise NotImplementedError('Method must be implemented by subclass') 199 200 # Whether each message must end in a newline 201 _append_newline = False 193 202 194 203 def _dump_message(self, message, target, mangle_from_=False): … … 202 211 gen.flatten(message) 203 212 buffer.seek(0) 204 target.write(buffer.read().replace('\n', os.linesep)) 213 data = buffer.read().replace('\n', os.linesep) 214 target.write(data) 215 if self._append_newline and not data.endswith(os.linesep): 216 # Make sure the message ends with a newline 217 target.write(os.linesep) 205 218 elif isinstance(message, str): 206 219 if mangle_from_: … … 208 221 message = message.replace('\n', os.linesep) 209 222 target.write(message) 223 if self._append_newline and not message.endswith(os.linesep): 224 # Make sure the message ends with a newline 225 target.write(os.linesep) 210 226 elif hasattr(message, 'read'): 227 lastline = None 211 228 while True: 212 229 line = message.readline() … … 217 234 line = line.replace('\n', os.linesep) 218 235 target.write(line) 236 lastline = line 237 if self._append_newline and lastline and not lastline.endswith(os.linesep): 238 # Make sure the message ends with a newline 239 target.write(os.linesep) 219 240 else: 220 241 raise TypeError('Invalid message type: %s' % type(message)) … … 229 250 """Initialize a Maildir instance.""" 230 251 Mailbox.__init__(self, dirname, factory, create) 252 self._paths = { 253 'tmp': os.path.join(self._path, 'tmp'), 254 'new': os.path.join(self._path, 'new'), 255 'cur': os.path.join(self._path, 'cur'), 256 } 231 257 if not os.path.exists(self._path): 232 258 if create: 233 259 os.mkdir(self._path, 0700) 234 os.mkdir(os.path.join(self._path, 'tmp'), 0700) 235 os.mkdir(os.path.join(self._path, 'new'), 0700) 236 os.mkdir(os.path.join(self._path, 'cur'), 0700) 260 for path in self._paths.values(): 261 os.mkdir(path, 0o700) 237 262 else: 238 263 raise NoSuchMailboxError(self._path) 239 264 self._toc = {} 265 self._toc_mtimes = {'cur': 0, 'new': 0} 266 self._last_read = 0 # Records last time we read cur/new 267 self._skewfactor = 0.1 # Adjust if os/fs clocks are skewing 240 268 241 269 def add(self, message): … … 244 272 try: 245 273 self._dump_message(message, tmp_file) 246 finally: 247 _sync_close(tmp_file) 274 except BaseException: 275 tmp_file.close() 276 os.remove(tmp_file.name) 277 raise 278 _sync_close(tmp_file) 248 279 if isinstance(message, MaildirMessage): 249 280 subdir = message.get_subdir() … … 256 287 uniq = os.path.basename(tmp_file.name).split(self.colon)[0] 257 288 dest = os.path.join(self._path, subdir, uniq + suffix) 289 if isinstance(message, MaildirMessage): 290 os.utime(tmp_file.name, 291 (os.path.getatime(tmp_file.name), message.get_date())) 292 # No file modification should be done after the file is moved to its 293 # final position in order to prevent race conditions with changes 294 # from other programs 258 295 try: 259 296 if hasattr(os, 'link'): … … 269 306 else: 270 307 raise 271 if isinstance(message, MaildirMessage):272 os.utime(dest, (os.path.getatime(dest), message.get_date()))273 308 return uniq 274 309 … … 305 340 suffix = '' 306 341 self.discard(key) 342 tmp_path = os.path.join(self._path, temp_subpath) 307 343 new_path = os.path.join(self._path, subdir, key + suffix) 308 os.rename(os.path.join(self._path, temp_subpath), new_path)309 344 if isinstance(message, MaildirMessage): 310 os.utime(new_path, (os.path.getatime(new_path), 311 message.get_date())) 345 os.utime(tmp_path, 346 (os.path.getatime(tmp_path), message.get_date())) 347 # No file modification should be done after the file is moved to its 348 # final position in order to prevent race conditions with changes 349 # from other programs 350 os.rename(tmp_path, new_path) 312 351 313 352 def get_message(self, key): … … 364 403 def flush(self): 365 404 """Write any pending changes to disk.""" 366 return # Maildir changes are always written immediately. 405 # Maildir changes are always written immediately, so there's nothing 406 # to do. 407 pass 367 408 368 409 def lock(self): … … 462 503 def _refresh(self): 463 504 """Update table of contents mapping.""" 505 # If it has been less than two seconds since the last _refresh() call, 506 # we have to unconditionally re-read the mailbox just in case it has 507 # been modified, because os.path.mtime() has a 2 sec resolution in the 508 # most common worst case (FAT) and a 1 sec resolution typically. This 509 # results in a few unnecessary re-reads when _refresh() is called 510 # multiple times in that interval, but once the clock ticks over, we 511 # will only re-read as needed. Because the filesystem might be being 512 # served by an independent system with its own clock, we record and 513 # compare with the mtimes from the filesystem. Because the other 514 # system's clock might be skewing relative to our clock, we add an 515 # extra delta to our wait. The default is one tenth second, but is an 516 # instance variable and so can be adjusted if dealing with a 517 # particularly skewed or irregular system. 518 if time.time() - self._last_read > 2 + self._skewfactor: 519 refresh = False 520 for subdir in self._toc_mtimes: 521 mtime = os.path.getmtime(self._paths[subdir]) 522 if mtime > self._toc_mtimes[subdir]: 523 refresh = True 524 self._toc_mtimes[subdir] = mtime 525 if not refresh: 526 return 527 # Refresh toc 464 528 self._toc = {} 465 for subdir in ('new', 'cur'):466 subdir_path = os.path.join(self._path, subdir)467 for entry in os.listdir( subdir_path):468 p = os.path.join( subdir_path, entry)529 for subdir in self._toc_mtimes: 530 path = self._paths[subdir] 531 for entry in os.listdir(path): 532 p = os.path.join(path, entry) 469 533 if os.path.isdir(p): 470 534 continue 471 535 uniq = entry.split(self.colon)[0] 472 536 self._toc[uniq] = os.path.join(subdir, entry) 537 self._last_read = time.time() 473 538 474 539 def _lookup(self, key): … … 513 578 else: 514 579 raise NoSuchMailboxError(self._path) 515 elif e.errno == errno.EACCES:580 elif e.errno in (errno.EACCES, errno.EROFS): 516 581 f = open(self._path, 'rb') 517 582 else: … … 520 585 self._toc = None 521 586 self._next_key = 0 522 self._pending = False # No changes require rewriting the file. 587 self._pending = False # No changes require rewriting the file. 588 self._pending_sync = False # No need to sync the file 523 589 self._locked = False 524 self._file_length = None 590 self._file_length = None # Used to record mailbox size 525 591 526 592 def add(self, message): … … 529 595 self._toc[self._next_key] = self._append_message(message) 530 596 self._next_key += 1 531 self._pending = True 597 # _append_message appends the message to the mailbox file. We 598 # don't need a full rewrite + rename, sync is enough. 599 self._pending_sync = True 532 600 return self._next_key - 1 533 601 … … 575 643 """Write any pending changes to disk.""" 576 644 if not self._pending: 645 if self._pending_sync: 646 # Messages have only been added, so syncing the file 647 # is enough. 648 _sync_flush(self._file) 649 self._pending_sync = False 577 650 return 578 651 … … 608 681 new_toc[key] = (new_start, new_file.tell()) 609 682 self._post_message_hook(new_file) 683 self._file_length = new_file.tell() 610 684 except: 611 685 new_file.close() … … 615 689 # self._file is about to get replaced, so no need to sync. 616 690 self._file.close() 691 # Make sure the new file's mode is the same as the old file's 692 mode = os.stat(self._path).st_mode 693 os.chmod(new_file.name, mode) 617 694 try: 618 695 os.rename(new_file.name, self._path) … … 627 704 self._toc = new_toc 628 705 self._pending = False 706 self._pending_sync = False 629 707 if self._locked: 630 708 _lock_file(self._file, dotlock=False) … … 662 740 """Append message to mailbox and return (start, stop) offsets.""" 663 741 self._file.seek(0, 2) 664 self._pre_message_hook(self._file) 665 offsets = self._install_message(message) 666 self._post_message_hook(self._file) 742 before = self._file.tell() 743 if len(self._toc) == 0 and not self._pending: 744 # This is the first message, and the _pre_mailbox_hook 745 # hasn't yet been called. If self._pending is True, 746 # messages have been removed, so _pre_mailbox_hook must 747 # have been called already. 748 self._pre_mailbox_hook(self._file) 749 try: 750 self._pre_message_hook(self._file) 751 offsets = self._install_message(message) 752 self._post_message_hook(self._file) 753 except BaseException: 754 self._file.truncate(before) 755 raise 667 756 self._file.flush() 668 757 self._file_length = self._file.tell() # Record current length of mailbox … … 732 821 _mangle_from_ = True 733 822 823 # All messages must end in a newline character, and 824 # _post_message_hooks outputs an empty line between messages. 825 _append_newline = True 826 734 827 def __init__(self, path, factory=None, create=True): 735 828 """Initialize an mbox mailbox.""" … … 737 830 _mboxMMDF.__init__(self, path, factory, create) 738 831 739 def _pre_message_hook(self, f): 740 """Called before writing each message to file f.""" 741 if f.tell() != 0: 742 f.write(os.linesep) 832 def _post_message_hook(self, f): 833 """Called after writing each message to file f.""" 834 f.write(os.linesep) 743 835 744 836 def _generate_toc(self): 745 837 """Generate key-to-(start, stop) table of contents.""" 746 838 starts, stops = [], [] 839 last_was_empty = False 747 840 self._file.seek(0) 748 841 while True: … … 751 844 if line.startswith('From '): 752 845 if len(stops) < len(starts): 846 if last_was_empty: 847 stops.append(line_pos - len(os.linesep)) 848 else: 849 # The last line before the "From " line wasn't 850 # blank, but we consider it a start of a 851 # message anyway. 852 stops.append(line_pos) 853 starts.append(line_pos) 854 last_was_empty = False 855 elif not line: 856 if last_was_empty: 753 857 stops.append(line_pos - len(os.linesep)) 754 starts.append(line_pos) 755 elif line == '': 756 stops.append(line_pos) 858 else: 859 stops.append(line_pos) 757 860 break 861 elif line == os.linesep: 862 last_was_empty = True 863 else: 864 last_was_empty = False 758 865 self._toc = dict(enumerate(zip(starts, stops))) 759 866 self._next_key = len(self._toc) … … 830 937 new_path = os.path.join(self._path, str(new_key)) 831 938 f = _create_carefully(new_path) 939 closed = False 832 940 try: 833 941 if self._locked: 834 942 _lock_file(f) 835 943 try: 836 self._dump_message(message, f) 944 try: 945 self._dump_message(message, f) 946 except BaseException: 947 # Unlock and close so it can be deleted on Windows 948 if self._locked: 949 _unlock_file(f) 950 _sync_close(f) 951 closed = True 952 os.remove(new_path) 953 raise 837 954 if isinstance(message, MHMessage): 838 955 self._dump_sequences(message, new_key) … … 841 958 _unlock_file(f) 842 959 finally: 843 _sync_close(f) 960 if not closed: 961 _sync_close(f) 844 962 return new_key 845 963 … … 854 972 else: 855 973 raise 856 try: 857 if self._locked: 858 _lock_file(f) 859 try: 860 f.close() 861 os.remove(os.path.join(self._path, str(key))) 862 finally: 863 if self._locked: 864 _unlock_file(f) 865 finally: 974 else: 866 975 f.close() 976 os.remove(path) 867 977 868 978 def __setitem__(self, key, message): … … 1318 1428 self._file.write(line.replace('\n', os.linesep)) 1319 1429 if line == '\n' or line == '': 1320 self._file.write('*** EOOH ***' + os.linesep)1321 1430 if first_pass: 1322 1431 first_pass = False 1432 self._file.write('*** EOOH ***' + os.linesep) 1323 1433 message.seek(original_pos) 1324 1434 else: … … 1803 1913 def close(self): 1804 1914 """Close the file.""" 1805 del self._file 1915 if hasattr(self, '_file'): 1916 if hasattr(self._file, 'close'): 1917 self._file.close() 1918 del self._file 1806 1919 1807 1920 def _read(self, size, read_method): … … 1847 1960 return _ProxyFile._read(self, size, read_method) 1848 1961 1962 def close(self): 1963 # do *not* close the underlying file object for partial files, 1964 # since it's global to the mailbox object 1965 if hasattr(self, '_file'): 1966 del self._file 1967 1849 1968 1850 1969 def _lock_file(f, dotlock=True): … … 1856 1975 fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB) 1857 1976 except IOError, e: 1858 if e.errno in (errno.EAGAIN, errno.EACCES ):1977 if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS): 1859 1978 raise ExternalClashError('lockf: lock unavailable: %s' % 1860 1979 f.name) … … 1866 1985 pre_lock.close() 1867 1986 except IOError, e: 1868 if e.errno == errno.EACCES:1987 if e.errno in (errno.EACCES, errno.EROFS): 1869 1988 return # Without write access, just skip dotlocking. 1870 1989 else:
Note:
See TracChangeset
for help on using the changeset viewer.
