1 | #----------------------------------------------------------------------
|
---|
2 | # Copyright (c) 1999-2001, Digital Creations, Fredericksburg, VA, USA
|
---|
3 | # and Andrew Kuchling. All rights reserved.
|
---|
4 | #
|
---|
5 | # Redistribution and use in source and binary forms, with or without
|
---|
6 | # modification, are permitted provided that the following conditions are
|
---|
7 | # met:
|
---|
8 | #
|
---|
9 | # o Redistributions of source code must retain the above copyright
|
---|
10 | # notice, this list of conditions, and the disclaimer that follows.
|
---|
11 | #
|
---|
12 | # o Redistributions in binary form must reproduce the above copyright
|
---|
13 | # notice, this list of conditions, and the following disclaimer in
|
---|
14 | # the documentation and/or other materials provided with the
|
---|
15 | # distribution.
|
---|
16 | #
|
---|
17 | # o Neither the name of Digital Creations nor the names of its
|
---|
18 | # contributors may be used to endorse or promote products derived
|
---|
19 | # from this software without specific prior written permission.
|
---|
20 | #
|
---|
21 | # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS AND CONTRIBUTORS *AS
|
---|
22 | # IS* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
---|
23 | # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
|
---|
24 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL
|
---|
25 | # CREATIONS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
---|
26 | # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
---|
27 | # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
|
---|
28 | # OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
---|
29 | # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
|
---|
30 | # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
|
---|
31 | # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
|
---|
32 | # DAMAGE.
|
---|
33 | #----------------------------------------------------------------------
|
---|
34 |
|
---|
35 |
|
---|
36 | """Support for Berkeley DB 4.3 through 5.3 with a simple interface.
|
---|
37 |
|
---|
38 | For the full featured object oriented interface use the bsddb.db module
|
---|
39 | instead. It mirrors the Oracle Berkeley DB C API.
|
---|
40 | """
|
---|
41 |
|
---|
42 | import sys
|
---|
43 | absolute_import = (sys.version_info[0] >= 3)
|
---|
44 |
|
---|
45 | if (sys.version_info >= (2, 6)) and (sys.version_info < (3, 0)) :
|
---|
46 | import warnings
|
---|
47 | if sys.py3kwarning and (__name__ != 'bsddb3') :
|
---|
48 | warnings.warnpy3k("in 3.x, the bsddb module has been removed; "
|
---|
49 | "please use the pybsddb project instead",
|
---|
50 | DeprecationWarning, 2)
|
---|
51 | warnings.filterwarnings("ignore", ".*CObject.*", DeprecationWarning,
|
---|
52 | "bsddb.__init__")
|
---|
53 |
|
---|
54 | try:
|
---|
55 | if __name__ == 'bsddb3':
|
---|
56 | # import _pybsddb binary as it should be the more recent version from
|
---|
57 | # a standalone pybsddb addon package than the version included with
|
---|
58 | # python as bsddb._bsddb.
|
---|
59 | if absolute_import :
|
---|
60 | # Because this syntaxis is not valid before Python 2.5
|
---|
61 | exec("from . import _pybsddb")
|
---|
62 | else :
|
---|
63 | import _pybsddb
|
---|
64 | _bsddb = _pybsddb
|
---|
65 | from bsddb3.dbutils import DeadlockWrap as _DeadlockWrap
|
---|
66 | else:
|
---|
67 | import _bsddb
|
---|
68 | from bsddb.dbutils import DeadlockWrap as _DeadlockWrap
|
---|
69 | except ImportError:
|
---|
70 | # Remove ourselves from sys.modules
|
---|
71 | import sys
|
---|
72 | del sys.modules[__name__]
|
---|
73 | raise
|
---|
74 |
|
---|
75 | # bsddb3 calls it db, but provide _db for backwards compatibility
|
---|
76 | db = _db = _bsddb
|
---|
77 | __version__ = db.__version__
|
---|
78 |
|
---|
79 | error = db.DBError # So bsddb.error will mean something...
|
---|
80 |
|
---|
81 | #----------------------------------------------------------------------
|
---|
82 |
|
---|
83 | import sys, os
|
---|
84 |
|
---|
85 | from weakref import ref
|
---|
86 |
|
---|
87 | if sys.version_info < (2, 6) :
|
---|
88 | import UserDict
|
---|
89 | MutableMapping = UserDict.DictMixin
|
---|
90 | else :
|
---|
91 | import collections
|
---|
92 | MutableMapping = collections.MutableMapping
|
---|
93 |
|
---|
94 | class _iter_mixin(MutableMapping):
|
---|
95 | def _make_iter_cursor(self):
|
---|
96 | cur = _DeadlockWrap(self.db.cursor)
|
---|
97 | key = id(cur)
|
---|
98 | self._cursor_refs[key] = ref(cur, self._gen_cref_cleaner(key))
|
---|
99 | return cur
|
---|
100 |
|
---|
101 | def _gen_cref_cleaner(self, key):
|
---|
102 | # use generate the function for the weakref callback here
|
---|
103 | # to ensure that we do not hold a strict reference to cur
|
---|
104 | # in the callback.
|
---|
105 | return lambda ref: self._cursor_refs.pop(key, None)
|
---|
106 |
|
---|
107 | def __iter__(self):
|
---|
108 | self._kill_iteration = False
|
---|
109 | self._in_iter += 1
|
---|
110 | try:
|
---|
111 | try:
|
---|
112 | cur = self._make_iter_cursor()
|
---|
113 |
|
---|
114 | # FIXME-20031102-greg: race condition. cursor could
|
---|
115 | # be closed by another thread before this call.
|
---|
116 |
|
---|
117 | # since we're only returning keys, we call the cursor
|
---|
118 | # methods with flags=0, dlen=0, dofs=0
|
---|
119 | key = _DeadlockWrap(cur.first, 0,0,0)[0]
|
---|
120 | yield key
|
---|
121 |
|
---|
122 | next = getattr(cur, "next")
|
---|
123 | while 1:
|
---|
124 | try:
|
---|
125 | key = _DeadlockWrap(next, 0,0,0)[0]
|
---|
126 | yield key
|
---|
127 | except _bsddb.DBCursorClosedError:
|
---|
128 | if self._kill_iteration:
|
---|
129 | raise RuntimeError('Database changed size '
|
---|
130 | 'during iteration.')
|
---|
131 | cur = self._make_iter_cursor()
|
---|
132 | # FIXME-20031101-greg: race condition. cursor could
|
---|
133 | # be closed by another thread before this call.
|
---|
134 | _DeadlockWrap(cur.set, key,0,0,0)
|
---|
135 | next = getattr(cur, "next")
|
---|
136 | except _bsddb.DBNotFoundError:
|
---|
137 | pass
|
---|
138 | except _bsddb.DBCursorClosedError:
|
---|
139 | # the database was modified during iteration. abort.
|
---|
140 | pass
|
---|
141 | # When Python 2.4 not supported in bsddb3, we can change this to "finally"
|
---|
142 | except :
|
---|
143 | self._in_iter -= 1
|
---|
144 | raise
|
---|
145 |
|
---|
146 | self._in_iter -= 1
|
---|
147 |
|
---|
148 | def iteritems(self):
|
---|
149 | if not self.db:
|
---|
150 | return
|
---|
151 | self._kill_iteration = False
|
---|
152 | self._in_iter += 1
|
---|
153 | try:
|
---|
154 | try:
|
---|
155 | cur = self._make_iter_cursor()
|
---|
156 |
|
---|
157 | # FIXME-20031102-greg: race condition. cursor could
|
---|
158 | # be closed by another thread before this call.
|
---|
159 |
|
---|
160 | kv = _DeadlockWrap(cur.first)
|
---|
161 | key = kv[0]
|
---|
162 | yield kv
|
---|
163 |
|
---|
164 | next = getattr(cur, "next")
|
---|
165 | while 1:
|
---|
166 | try:
|
---|
167 | kv = _DeadlockWrap(next)
|
---|
168 | key = kv[0]
|
---|
169 | yield kv
|
---|
170 | except _bsddb.DBCursorClosedError:
|
---|
171 | if self._kill_iteration:
|
---|
172 | raise RuntimeError('Database changed size '
|
---|
173 | 'during iteration.')
|
---|
174 | cur = self._make_iter_cursor()
|
---|
175 | # FIXME-20031101-greg: race condition. cursor could
|
---|
176 | # be closed by another thread before this call.
|
---|
177 | _DeadlockWrap(cur.set, key,0,0,0)
|
---|
178 | next = getattr(cur, "next")
|
---|
179 | except _bsddb.DBNotFoundError:
|
---|
180 | pass
|
---|
181 | except _bsddb.DBCursorClosedError:
|
---|
182 | # the database was modified during iteration. abort.
|
---|
183 | pass
|
---|
184 | # When Python 2.4 not supported in bsddb3, we can change this to "finally"
|
---|
185 | except :
|
---|
186 | self._in_iter -= 1
|
---|
187 | raise
|
---|
188 |
|
---|
189 | self._in_iter -= 1
|
---|
190 |
|
---|
191 |
|
---|
192 | class _DBWithCursor(_iter_mixin):
|
---|
193 | """
|
---|
194 | A simple wrapper around DB that makes it look like the bsddbobject in
|
---|
195 | the old module. It uses a cursor as needed to provide DB traversal.
|
---|
196 | """
|
---|
197 | def __init__(self, db):
|
---|
198 | self.db = db
|
---|
199 | self.db.set_get_returns_none(0)
|
---|
200 |
|
---|
201 | # FIXME-20031101-greg: I believe there is still the potential
|
---|
202 | # for deadlocks in a multithreaded environment if someone
|
---|
203 | # attempts to use the any of the cursor interfaces in one
|
---|
204 | # thread while doing a put or delete in another thread. The
|
---|
205 | # reason is that _checkCursor and _closeCursors are not atomic
|
---|
206 | # operations. Doing our own locking around self.dbc,
|
---|
207 | # self.saved_dbc_key and self._cursor_refs could prevent this.
|
---|
208 | # TODO: A test case demonstrating the problem needs to be written.
|
---|
209 |
|
---|
210 | # self.dbc is a DBCursor object used to implement the
|
---|
211 | # first/next/previous/last/set_location methods.
|
---|
212 | self.dbc = None
|
---|
213 | self.saved_dbc_key = None
|
---|
214 |
|
---|
215 | # a collection of all DBCursor objects currently allocated
|
---|
216 | # by the _iter_mixin interface.
|
---|
217 | self._cursor_refs = {}
|
---|
218 | self._in_iter = 0
|
---|
219 | self._kill_iteration = False
|
---|
220 |
|
---|
221 | def __del__(self):
|
---|
222 | self.close()
|
---|
223 |
|
---|
224 | def _checkCursor(self):
|
---|
225 | if self.dbc is None:
|
---|
226 | self.dbc = _DeadlockWrap(self.db.cursor)
|
---|
227 | if self.saved_dbc_key is not None:
|
---|
228 | _DeadlockWrap(self.dbc.set, self.saved_dbc_key)
|
---|
229 | self.saved_dbc_key = None
|
---|
230 |
|
---|
231 | # This method is needed for all non-cursor DB calls to avoid
|
---|
232 | # Berkeley DB deadlocks (due to being opened with DB_INIT_LOCK
|
---|
233 | # and DB_THREAD to be thread safe) when intermixing database
|
---|
234 | # operations that use the cursor internally with those that don't.
|
---|
235 | def _closeCursors(self, save=1):
|
---|
236 | if self.dbc:
|
---|
237 | c = self.dbc
|
---|
238 | self.dbc = None
|
---|
239 | if save:
|
---|
240 | try:
|
---|
241 | self.saved_dbc_key = _DeadlockWrap(c.current, 0,0,0)[0]
|
---|
242 | except db.DBError:
|
---|
243 | pass
|
---|
244 | _DeadlockWrap(c.close)
|
---|
245 | del c
|
---|
246 | for cref in self._cursor_refs.values():
|
---|
247 | c = cref()
|
---|
248 | if c is not None:
|
---|
249 | _DeadlockWrap(c.close)
|
---|
250 |
|
---|
251 | def _checkOpen(self):
|
---|
252 | if self.db is None:
|
---|
253 | raise error, "BSDDB object has already been closed"
|
---|
254 |
|
---|
255 | def isOpen(self):
|
---|
256 | return self.db is not None
|
---|
257 |
|
---|
258 | def __len__(self):
|
---|
259 | self._checkOpen()
|
---|
260 | return _DeadlockWrap(lambda: len(self.db)) # len(self.db)
|
---|
261 |
|
---|
262 | if sys.version_info >= (2, 6) :
|
---|
263 | def __repr__(self) :
|
---|
264 | if self.isOpen() :
|
---|
265 | return repr(dict(_DeadlockWrap(self.db.items)))
|
---|
266 | return repr(dict())
|
---|
267 |
|
---|
268 | def __getitem__(self, key):
|
---|
269 | self._checkOpen()
|
---|
270 | return _DeadlockWrap(lambda: self.db[key]) # self.db[key]
|
---|
271 |
|
---|
272 | def __setitem__(self, key, value):
|
---|
273 | self._checkOpen()
|
---|
274 | self._closeCursors()
|
---|
275 | if self._in_iter and key not in self:
|
---|
276 | self._kill_iteration = True
|
---|
277 | def wrapF():
|
---|
278 | self.db[key] = value
|
---|
279 | _DeadlockWrap(wrapF) # self.db[key] = value
|
---|
280 |
|
---|
281 | def __delitem__(self, key):
|
---|
282 | self._checkOpen()
|
---|
283 | self._closeCursors()
|
---|
284 | if self._in_iter and key in self:
|
---|
285 | self._kill_iteration = True
|
---|
286 | def wrapF():
|
---|
287 | del self.db[key]
|
---|
288 | _DeadlockWrap(wrapF) # del self.db[key]
|
---|
289 |
|
---|
290 | def close(self):
|
---|
291 | self._closeCursors(save=0)
|
---|
292 | if self.dbc is not None:
|
---|
293 | _DeadlockWrap(self.dbc.close)
|
---|
294 | v = 0
|
---|
295 | if self.db is not None:
|
---|
296 | v = _DeadlockWrap(self.db.close)
|
---|
297 | self.dbc = None
|
---|
298 | self.db = None
|
---|
299 | return v
|
---|
300 |
|
---|
301 | def keys(self):
|
---|
302 | self._checkOpen()
|
---|
303 | return _DeadlockWrap(self.db.keys)
|
---|
304 |
|
---|
305 | def has_key(self, key):
|
---|
306 | self._checkOpen()
|
---|
307 | return _DeadlockWrap(self.db.has_key, key)
|
---|
308 |
|
---|
309 | def set_location(self, key):
|
---|
310 | self._checkOpen()
|
---|
311 | self._checkCursor()
|
---|
312 | return _DeadlockWrap(self.dbc.set_range, key)
|
---|
313 |
|
---|
314 | def next(self): # Renamed by "2to3"
|
---|
315 | self._checkOpen()
|
---|
316 | self._checkCursor()
|
---|
317 | rv = _DeadlockWrap(getattr(self.dbc, "next"))
|
---|
318 | return rv
|
---|
319 |
|
---|
320 | if sys.version_info[0] >= 3 : # For "2to3" conversion
|
---|
321 | next = __next__
|
---|
322 |
|
---|
323 | def previous(self):
|
---|
324 | self._checkOpen()
|
---|
325 | self._checkCursor()
|
---|
326 | rv = _DeadlockWrap(self.dbc.prev)
|
---|
327 | return rv
|
---|
328 |
|
---|
329 | def first(self):
|
---|
330 | self._checkOpen()
|
---|
331 | # fix 1725856: don't needlessly try to restore our cursor position
|
---|
332 | self.saved_dbc_key = None
|
---|
333 | self._checkCursor()
|
---|
334 | rv = _DeadlockWrap(self.dbc.first)
|
---|
335 | return rv
|
---|
336 |
|
---|
337 | def last(self):
|
---|
338 | self._checkOpen()
|
---|
339 | # fix 1725856: don't needlessly try to restore our cursor position
|
---|
340 | self.saved_dbc_key = None
|
---|
341 | self._checkCursor()
|
---|
342 | rv = _DeadlockWrap(self.dbc.last)
|
---|
343 | return rv
|
---|
344 |
|
---|
345 | def sync(self):
|
---|
346 | self._checkOpen()
|
---|
347 | return _DeadlockWrap(self.db.sync)
|
---|
348 |
|
---|
349 |
|
---|
350 | #----------------------------------------------------------------------
|
---|
351 | # Compatibility object factory functions
|
---|
352 |
|
---|
353 | def hashopen(file, flag='c', mode=0666, pgsize=None, ffactor=None, nelem=None,
|
---|
354 | cachesize=None, lorder=None, hflags=0):
|
---|
355 |
|
---|
356 | flags = _checkflag(flag, file)
|
---|
357 | e = _openDBEnv(cachesize)
|
---|
358 | d = db.DB(e)
|
---|
359 | d.set_flags(hflags)
|
---|
360 | if pgsize is not None: d.set_pagesize(pgsize)
|
---|
361 | if lorder is not None: d.set_lorder(lorder)
|
---|
362 | if ffactor is not None: d.set_h_ffactor(ffactor)
|
---|
363 | if nelem is not None: d.set_h_nelem(nelem)
|
---|
364 | d.open(file, db.DB_HASH, flags, mode)
|
---|
365 | return _DBWithCursor(d)
|
---|
366 |
|
---|
367 | #----------------------------------------------------------------------
|
---|
368 |
|
---|
369 | def btopen(file, flag='c', mode=0666,
|
---|
370 | btflags=0, cachesize=None, maxkeypage=None, minkeypage=None,
|
---|
371 | pgsize=None, lorder=None):
|
---|
372 |
|
---|
373 | flags = _checkflag(flag, file)
|
---|
374 | e = _openDBEnv(cachesize)
|
---|
375 | d = db.DB(e)
|
---|
376 | if pgsize is not None: d.set_pagesize(pgsize)
|
---|
377 | if lorder is not None: d.set_lorder(lorder)
|
---|
378 | d.set_flags(btflags)
|
---|
379 | if minkeypage is not None: d.set_bt_minkey(minkeypage)
|
---|
380 | if maxkeypage is not None: d.set_bt_maxkey(maxkeypage)
|
---|
381 | d.open(file, db.DB_BTREE, flags, mode)
|
---|
382 | return _DBWithCursor(d)
|
---|
383 |
|
---|
384 | #----------------------------------------------------------------------
|
---|
385 |
|
---|
386 |
|
---|
387 | def rnopen(file, flag='c', mode=0666,
|
---|
388 | rnflags=0, cachesize=None, pgsize=None, lorder=None,
|
---|
389 | rlen=None, delim=None, source=None, pad=None):
|
---|
390 |
|
---|
391 | flags = _checkflag(flag, file)
|
---|
392 | e = _openDBEnv(cachesize)
|
---|
393 | d = db.DB(e)
|
---|
394 | if pgsize is not None: d.set_pagesize(pgsize)
|
---|
395 | if lorder is not None: d.set_lorder(lorder)
|
---|
396 | d.set_flags(rnflags)
|
---|
397 | if delim is not None: d.set_re_delim(delim)
|
---|
398 | if rlen is not None: d.set_re_len(rlen)
|
---|
399 | if source is not None: d.set_re_source(source)
|
---|
400 | if pad is not None: d.set_re_pad(pad)
|
---|
401 | d.open(file, db.DB_RECNO, flags, mode)
|
---|
402 | return _DBWithCursor(d)
|
---|
403 |
|
---|
404 | #----------------------------------------------------------------------
|
---|
405 |
|
---|
406 | def _openDBEnv(cachesize):
|
---|
407 | e = db.DBEnv()
|
---|
408 | if cachesize is not None:
|
---|
409 | if cachesize >= 20480:
|
---|
410 | e.set_cachesize(0, cachesize)
|
---|
411 | else:
|
---|
412 | raise error, "cachesize must be >= 20480"
|
---|
413 | e.set_lk_detect(db.DB_LOCK_DEFAULT)
|
---|
414 | e.open('.', db.DB_PRIVATE | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_LOCK | db.DB_INIT_MPOOL)
|
---|
415 | return e
|
---|
416 |
|
---|
417 | def _checkflag(flag, file):
|
---|
418 | if flag == 'r':
|
---|
419 | flags = db.DB_RDONLY
|
---|
420 | elif flag == 'rw':
|
---|
421 | flags = 0
|
---|
422 | elif flag == 'w':
|
---|
423 | flags = db.DB_CREATE
|
---|
424 | elif flag == 'c':
|
---|
425 | flags = db.DB_CREATE
|
---|
426 | elif flag == 'n':
|
---|
427 | flags = db.DB_CREATE
|
---|
428 | #flags = db.DB_CREATE | db.DB_TRUNCATE
|
---|
429 | # we used db.DB_TRUNCATE flag for this before but Berkeley DB
|
---|
430 | # 4.2.52 changed to disallowed truncate with txn environments.
|
---|
431 | if file is not None and os.path.isfile(file):
|
---|
432 | os.unlink(file)
|
---|
433 | else:
|
---|
434 | raise error, "flags should be one of 'r', 'w', 'c' or 'n'"
|
---|
435 | return flags | db.DB_THREAD
|
---|
436 |
|
---|
437 | #----------------------------------------------------------------------
|
---|
438 |
|
---|
439 |
|
---|
440 | # This is a silly little hack that allows apps to continue to use the
|
---|
441 | # DB_THREAD flag even on systems without threads without freaking out
|
---|
442 | # Berkeley DB.
|
---|
443 | #
|
---|
444 | # This assumes that if Python was built with thread support then
|
---|
445 | # Berkeley DB was too.
|
---|
446 |
|
---|
447 | try:
|
---|
448 | # 2to3 automatically changes "import thread" to "import _thread"
|
---|
449 | import thread as T
|
---|
450 | del T
|
---|
451 |
|
---|
452 | except ImportError:
|
---|
453 | db.DB_THREAD = 0
|
---|
454 |
|
---|
455 | #----------------------------------------------------------------------
|
---|