source: python/trunk/Lib/imputil.py

Last change on this file was 391, checked in by dmik, 11 years ago

python: Merge vendor 2.7.6 to trunk.
  • Property svn:eol-style set to native
File size: 25.2 KB
RevLine 
[2]1"""
2Import utilities
3
4Exported classes:
5 ImportManager Manage the import process
6
7 Importer Base class for replacing standard import functions
8 BuiltinImporter Emulate the import mechanism for builtin and frozen modules
9
10 DynLoadSuffixImporter
11"""
12from warnings import warnpy3k
13warnpy3k("the imputil module has been removed in Python 3.0", stacklevel=2)
14del warnpy3k
15
16# note: avoid importing non-builtin modules
17import imp ### not available in Jython?
18import sys
19import __builtin__
20
21# for the DirectoryImporter
22import struct
23import marshal
24
25__all__ = ["ImportManager","Importer","BuiltinImporter"]
26
27_StringType = type('')
28_ModuleType = type(sys) ### doesn't work in Jython...
29
30class ImportManager:
31 "Manage the import process."
32
33 def install(self, namespace=vars(__builtin__)):
34 "Install this ImportManager into the specified namespace."
35
36 if isinstance(namespace, _ModuleType):
37 namespace = vars(namespace)
38
39 # Note: we have no notion of "chaining"
40
41 # Record the previous import hook, then install our own.
42 self.previous_importer = namespace['__import__']
43 self.namespace = namespace
44 namespace['__import__'] = self._import_hook
45
46 ### fix this
47 #namespace['reload'] = self._reload_hook
48
49 def uninstall(self):
50 "Restore the previous import mechanism."
51 self.namespace['__import__'] = self.previous_importer
52
53 def add_suffix(self, suffix, importFunc):
[391]54 assert hasattr(importFunc, '__call__')
[2]55 self.fs_imp.add_suffix(suffix, importFunc)
56
57 ######################################################################
58 #
59 # PRIVATE METHODS
60 #
61
62 clsFilesystemImporter = None
63
64 def __init__(self, fs_imp=None):
65 # we're definitely going to be importing something in the future,
66 # so let's just load the OS-related facilities.
67 if not _os_stat:
68 _os_bootstrap()
69
70 # This is the Importer that we use for grabbing stuff from the
71 # filesystem. It defines one more method (import_from_dir) for our use.
72 if fs_imp is None:
73 cls = self.clsFilesystemImporter or _FilesystemImporter
74 fs_imp = cls()
75 self.fs_imp = fs_imp
76
77 # Initialize the set of suffixes that we recognize and import.
78 # The default will import dynamic-load modules first, followed by
79 # .py files (or a .py file's cached bytecode)
80 for desc in imp.get_suffixes():
81 if desc[2] == imp.C_EXTENSION:
82 self.add_suffix(desc[0],
83 DynLoadSuffixImporter(desc).import_file)
84 self.add_suffix('.py', py_suffix_importer)
85
86 def _import_hook(self, fqname, globals=None, locals=None, fromlist=None):
87 """Python calls this hook to locate and import a module."""
88
89 parts = fqname.split('.')
90
91 # determine the context of this import
92 parent = self._determine_import_context(globals)
93
94 # if there is a parent, then its importer should manage this import
95 if parent:
96 module = parent.__importer__._do_import(parent, parts, fromlist)
97 if module:
98 return module
99
100 # has the top module already been imported?
101 try:
102 top_module = sys.modules[parts[0]]
103 except KeyError:
104
105 # look for the topmost module
106 top_module = self._import_top_module(parts[0])
107 if not top_module:
108 # the topmost module wasn't found at all.
109 raise ImportError, 'No module named ' + fqname
110
111 # fast-path simple imports
112 if len(parts) == 1:
113 if not fromlist:
114 return top_module
115
116 if not top_module.__dict__.get('__ispkg__'):
117 # __ispkg__ isn't defined (the module was not imported by us),
118 # or it is zero.
119 #
120 # In the former case, there is no way that we could import
121 # sub-modules that occur in the fromlist (but we can't raise an
122 # error because it may just be names) because we don't know how
123 # to deal with packages that were imported by other systems.
124 #
125 # In the latter case (__ispkg__ == 0), there can't be any sub-
126 # modules present, so we can just return.
127 #
128 # In both cases, since len(parts) == 1, the top_module is also
129 # the "bottom" which is the defined return when a fromlist
130 # exists.
131 return top_module
132
133 importer = top_module.__dict__.get('__importer__')
134 if importer:
135 return importer._finish_import(top_module, parts[1:], fromlist)
136
137 # Grrr, some people "import os.path" or do "from os.path import ..."
138 if len(parts) == 2 and hasattr(top_module, parts[1]):
139 if fromlist:
140 return getattr(top_module, parts[1])
141 else:
142 return top_module
143
144 # If the importer does not exist, then we have to bail. A missing
145 # importer means that something else imported the module, and we have
146 # no knowledge of how to get sub-modules out of the thing.
147 raise ImportError, 'No module named ' + fqname
148
149 def _determine_import_context(self, globals):
150 """Returns the context in which a module should be imported.
151
152 The context could be a loaded (package) module and the imported module
153 will be looked for within that package. The context could also be None,
154 meaning there is no context -- the module should be looked for as a
155 "top-level" module.
156 """
157
158 if not globals or not globals.get('__importer__'):
159 # globals does not refer to one of our modules or packages. That
160 # implies there is no relative import context (as far as we are
161 # concerned), and it should just pick it off the standard path.
162 return None
163
164 # The globals refer to a module or package of ours. It will define
165 # the context of the new import. Get the module/package fqname.
166 parent_fqname = globals['__name__']
167
168 # if a package is performing the import, then return itself (imports
169 # refer to pkg contents)
170 if globals['__ispkg__']:
171 parent = sys.modules[parent_fqname]
172 assert globals is parent.__dict__
173 return parent
174
175 i = parent_fqname.rfind('.')
176
177 # a module outside of a package has no particular import context
178 if i == -1:
179 return None
180
181 # if a module in a package is performing the import, then return the
182 # package (imports refer to siblings)
183 parent_fqname = parent_fqname[:i]
184 parent = sys.modules[parent_fqname]
185 assert parent.__name__ == parent_fqname
186 return parent
187
188 def _import_top_module(self, name):
189 # scan sys.path looking for a location in the filesystem that contains
190 # the module, or an Importer object that can import the module.
191 for item in sys.path:
192 if isinstance(item, _StringType):
193 module = self.fs_imp.import_from_dir(item, name)
194 else:
195 module = item.import_top(name)
196 if module:
197 return module
198 return None
199
200 def _reload_hook(self, module):
201 "Python calls this hook to reload a module."
202
203 # reloading of a module may or may not be possible (depending on the
204 # importer), but at least we can validate that it's ours to reload
205 importer = module.__dict__.get('__importer__')
206 if not importer:
207 ### oops. now what...
208 pass
209
210 # okay. it is using the imputil system, and we must delegate it, but
211 # we don't know what to do (yet)
212 ### we should blast the module dict and do another get_code(). need to
213 ### flesh this out and add proper docco...
214 raise SystemError, "reload not yet implemented"
215
216
217class Importer:
218 "Base class for replacing standard import functions."
219
220 def import_top(self, name):
221 "Import a top-level module."
222 return self._import_one(None, name, name)
223
224 ######################################################################
225 #
226 # PRIVATE METHODS
227 #
228 def _finish_import(self, top, parts, fromlist):
229 # if "a.b.c" was provided, then load the ".b.c" portion down from
230 # below the top-level module.
231 bottom = self._load_tail(top, parts)
232
233 # if the form is "import a.b.c", then return "a"
234 if not fromlist:
235 # no fromlist: return the top of the import tree
236 return top
237
238 # the top module was imported by self.
239 #
240 # this means that the bottom module was also imported by self (just
241 # now, or in the past and we fetched it from sys.modules).
242 #
243 # since we imported/handled the bottom module, this means that we can
244 # also handle its fromlist (and reliably use __ispkg__).
245
246 # if the bottom node is a package, then (potentially) import some
247 # modules.
248 #
249 # note: if it is not a package, then "fromlist" refers to names in
250 # the bottom module rather than modules.
251 # note: for a mix of names and modules in the fromlist, we will
252 # import all modules and insert those into the namespace of
253 # the package module. Python will pick up all fromlist names
254 # from the bottom (package) module; some will be modules that
255 # we imported and stored in the namespace, others are expected
256 # to be present already.
257 if bottom.__ispkg__:
258 self._import_fromlist(bottom, fromlist)
259
260 # if the form is "from a.b import c, d" then return "b"
261 return bottom
262
263 def _import_one(self, parent, modname, fqname):
264 "Import a single module."
265
266 # has the module already been imported?
267 try:
268 return sys.modules[fqname]
269 except KeyError:
270 pass
271
272 # load the module's code, or fetch the module itself
273 result = self.get_code(parent, modname, fqname)
274 if result is None:
275 return None
276
277 module = self._process_result(result, fqname)
278
279 # insert the module into its parent
280 if parent:
281 setattr(parent, modname, module)
282 return module
283
[391]284 def _process_result(self, result, fqname):
285 ispkg, code, values = result
[2]286 # did get_code() return an actual module? (rather than a code object)
287 is_module = isinstance(code, _ModuleType)
288
289 # use the returned module, or create a new one to exec code into
290 if is_module:
291 module = code
292 else:
293 module = imp.new_module(fqname)
294
295 ### record packages a bit differently??
296 module.__importer__ = self
297 module.__ispkg__ = ispkg
298
299 # insert additional values into the module (before executing the code)
300 module.__dict__.update(values)
301
302 # the module is almost ready... make it visible
303 sys.modules[fqname] = module
304
305 # execute the code within the module's namespace
306 if not is_module:
307 try:
308 exec code in module.__dict__
309 except:
310 if fqname in sys.modules:
311 del sys.modules[fqname]
312 raise
313
314 # fetch from sys.modules instead of returning module directly.
315 # also make module's __name__ agree with fqname, in case
316 # the "exec code in module.__dict__" played games on us.
317 module = sys.modules[fqname]
318 module.__name__ = fqname
319 return module
320
321 def _load_tail(self, m, parts):
322 """Import the rest of the modules, down from the top-level module.
323
324 Returns the last module in the dotted list of modules.
325 """
326 for part in parts:
327 fqname = "%s.%s" % (m.__name__, part)
328 m = self._import_one(m, part, fqname)
329 if not m:
330 raise ImportError, "No module named " + fqname
331 return m
332
333 def _import_fromlist(self, package, fromlist):
334 'Import any sub-modules in the "from" list.'
335
336 # if '*' is present in the fromlist, then look for the '__all__'
337 # variable to find additional items (modules) to import.
338 if '*' in fromlist:
339 fromlist = list(fromlist) + \
340 list(package.__dict__.get('__all__', []))
341
342 for sub in fromlist:
343 # if the name is already present, then don't try to import it (it
344 # might not be a module!).
345 if sub != '*' and not hasattr(package, sub):
346 subname = "%s.%s" % (package.__name__, sub)
347 submod = self._import_one(package, sub, subname)
348 if not submod:
349 raise ImportError, "cannot import name " + subname
350
351 def _do_import(self, parent, parts, fromlist):
352 """Attempt to import the module relative to parent.
353
354 This method is used when the import context specifies that <self>
355 imported the parent module.
356 """
357 top_name = parts[0]
358 top_fqname = parent.__name__ + '.' + top_name
359 top_module = self._import_one(parent, top_name, top_fqname)
360 if not top_module:
361 # this importer and parent could not find the module (relatively)
362 return None
363
364 return self._finish_import(top_module, parts[1:], fromlist)
365
366 ######################################################################
367 #
368 # METHODS TO OVERRIDE
369 #
370 def get_code(self, parent, modname, fqname):
371 """Find and retrieve the code for the given module.
372
373 parent specifies a parent module to define a context for importing. It
374 may be None, indicating no particular context for the search.
375
376 modname specifies a single module (not dotted) within the parent.
377
378 fqname specifies the fully-qualified module name. This is a
379 (potentially) dotted name from the "root" of the module namespace
380 down to the modname.
381 If there is no parent, then modname==fqname.
382
383 This method should return None, or a 3-tuple.
384
385 * If the module was not found, then None should be returned.
386
387 * The first item of the 2- or 3-tuple should be the integer 0 or 1,
388 specifying whether the module that was found is a package or not.
389
390 * The second item is the code object for the module (it will be
391 executed within the new module's namespace). This item can also
392 be a fully-loaded module object (e.g. loaded from a shared lib).
393
394 * The third item is a dictionary of name/value pairs that will be
395 inserted into new module before the code object is executed. This
396 is provided in case the module's code expects certain values (such
397 as where the module was found). When the second item is a module
398 object, then these names/values will be inserted *after* the module
399 has been loaded/initialized.
400 """
401 raise RuntimeError, "get_code not implemented"
402
403
404######################################################################
405#
406# Some handy stuff for the Importers
407#
408
409# byte-compiled file suffix character
410_suffix_char = __debug__ and 'c' or 'o'
411
412# byte-compiled file suffix
413_suffix = '.py' + _suffix_char
414
415def _compile(pathname, timestamp):
416 """Compile (and cache) a Python source file.
417
418 The file specified by <pathname> is compiled to a code object and
419 returned.
420
421 Presuming the appropriate privileges exist, the bytecodes will be
422 saved back to the filesystem for future imports. The source file's
423 modification timestamp must be provided as a Long value.
424 """
425 codestring = open(pathname, 'rU').read()
426 if codestring and codestring[-1] != '\n':
427 codestring = codestring + '\n'
428 code = __builtin__.compile(codestring, pathname, 'exec')
429
430 # try to cache the compiled code
431 try:
432 f = open(pathname + _suffix_char, 'wb')
433 except IOError:
434 pass
435 else:
436 f.write('\0\0\0\0')
437 f.write(struct.pack('<I', timestamp))
438 marshal.dump(code, f)
439 f.flush()
440 f.seek(0, 0)
441 f.write(imp.get_magic())
442 f.close()
443
444 return code
445
446_os_stat = _os_path_join = None
447def _os_bootstrap():
448 "Set up 'os' module replacement functions for use during import bootstrap."
449
450 names = sys.builtin_module_names
451
452 join = None
453 if 'posix' in names:
454 sep = '/'
455 from posix import stat
456 elif 'nt' in names:
457 sep = '\\'
458 from nt import stat
459 elif 'dos' in names:
460 sep = '\\'
461 from dos import stat
462 elif 'os2' in names:
463 sep = '\\'
464 from os2 import stat
465 else:
466 raise ImportError, 'no os specific module found'
467
468 if join is None:
469 def join(a, b, sep=sep):
470 if a == '':
471 return b
472 lastchar = a[-1:]
473 if lastchar == '/' or lastchar == sep:
474 return a + b
475 return a + sep + b
476
477 global _os_stat
478 _os_stat = stat
479
480 global _os_path_join
481 _os_path_join = join
482
483def _os_path_isdir(pathname):
484 "Local replacement for os.path.isdir()."
485 try:
486 s = _os_stat(pathname)
487 except OSError:
488 return None
489 return (s.st_mode & 0170000) == 0040000
490
491def _timestamp(pathname):
492 "Return the file modification time as a Long."
493 try:
494 s = _os_stat(pathname)
495 except OSError:
496 return None
497 return long(s.st_mtime)
498
499
500######################################################################
501#
502# Emulate the import mechanism for builtin and frozen modules
503#
504class BuiltinImporter(Importer):
505 def get_code(self, parent, modname, fqname):
506 if parent:
507 # these modules definitely do not occur within a package context
508 return None
509
510 # look for the module
511 if imp.is_builtin(modname):
512 type = imp.C_BUILTIN
513 elif imp.is_frozen(modname):
514 type = imp.PY_FROZEN
515 else:
516 # not found
517 return None
518
519 # got it. now load and return it.
520 module = imp.load_module(modname, None, modname, ('', '', type))
521 return 0, module, { }
522
523
524######################################################################
525#
526# Internal importer used for importing from the filesystem
527#
528class _FilesystemImporter(Importer):
529 def __init__(self):
530 self.suffixes = [ ]
531
532 def add_suffix(self, suffix, importFunc):
[391]533 assert hasattr(importFunc, '__call__')
[2]534 self.suffixes.append((suffix, importFunc))
535
536 def import_from_dir(self, dir, fqname):
537 result = self._import_pathname(_os_path_join(dir, fqname), fqname)
538 if result:
539 return self._process_result(result, fqname)
540 return None
541
542 def get_code(self, parent, modname, fqname):
543 # This importer is never used with an empty parent. Its existence is
544 # private to the ImportManager. The ImportManager uses the
545 # import_from_dir() method to import top-level modules/packages.
546 # This method is only used when we look for a module within a package.
547 assert parent
548
549 for submodule_path in parent.__path__:
550 code = self._import_pathname(_os_path_join(submodule_path, modname), fqname)
551 if code is not None:
552 return code
553 return self._import_pathname(_os_path_join(parent.__pkgdir__, modname),
554 fqname)
555
556 def _import_pathname(self, pathname, fqname):
557 if _os_path_isdir(pathname):
558 result = self._import_pathname(_os_path_join(pathname, '__init__'),
559 fqname)
560 if result:
561 values = result[2]
562 values['__pkgdir__'] = pathname
563 values['__path__'] = [ pathname ]
564 return 1, result[1], values
565 return None
566
567 for suffix, importFunc in self.suffixes:
568 filename = pathname + suffix
569 try:
570 finfo = _os_stat(filename)
571 except OSError:
572 pass
573 else:
574 return importFunc(filename, finfo, fqname)
575 return None
576
577######################################################################
578#
579# SUFFIX-BASED IMPORTERS
580#
581
582def py_suffix_importer(filename, finfo, fqname):
583 file = filename[:-3] + _suffix
584 t_py = long(finfo[8])
585 t_pyc = _timestamp(file)
586
587 code = None
588 if t_pyc is not None and t_pyc >= t_py:
589 f = open(file, 'rb')
590 if f.read(4) == imp.get_magic():
591 t = struct.unpack('<I', f.read(4))[0]
592 if t == t_py:
593 code = marshal.load(f)
594 f.close()
595 if code is None:
596 file = filename
597 code = _compile(file, t_py)
598
599 return 0, code, { '__file__' : file }
600
601class DynLoadSuffixImporter:
602 def __init__(self, desc):
603 self.desc = desc
604
605 def import_file(self, filename, finfo, fqname):
606 fp = open(filename, self.desc[1])
607 module = imp.load_module(fqname, fp, filename, self.desc)
608 module.__file__ = filename
609 return 0, module, { }
610
611
612######################################################################
613
614def _print_importers():
615 items = sys.modules.items()
616 items.sort()
617 for name, module in items:
618 if module:
619 print name, module.__dict__.get('__importer__', '-- no importer')
620 else:
621 print name, '-- non-existent module'
622
623def _test_revamp():
624 ImportManager().install()
625 sys.path.insert(0, BuiltinImporter())
626
627######################################################################
628
629#
630# TODO
631#
632# from Finn Bock:
633# type(sys) is not a module in Jython. what to use instead?
634# imp.C_EXTENSION is not in Jython. same for get_suffixes and new_module
635#
636# given foo.py of:
637# import sys
638# sys.modules['foo'] = sys
639#
640# ---- standard import mechanism
641# >>> import foo
642# >>> foo
643# <module 'sys' (built-in)>
644#
645# ---- revamped import mechanism
646# >>> import imputil
647# >>> imputil._test_revamp()
648# >>> import foo
649# >>> foo
650# <module 'foo' from 'foo.py'>
651#
652#
653# from MAL:
654# should BuiltinImporter exist in sys.path or hard-wired in ImportManager?
655# need __path__ processing
656# performance
657# move chaining to a subclass [gjs: it's been nuked]
658# deinstall should be possible
659# query mechanism needed: is a specific Importer installed?
660# py/pyc/pyo piping hooks to filter/process these files
661# wish list:
662# distutils importer hooked to list of standard Internet repositories
663# module->file location mapper to speed FS-based imports
664# relative imports
665# keep chaining so that it can play nice with other import hooks
666#
667# from Gordon:
668# push MAL's mapper into sys.path[0] as a cache (hard-coded for apps)
669#
670# from Guido:
671# need to change sys.* references for rexec environs
672# need hook for MAL's walk-me-up import strategy, or Tim's absolute strategy
673# watch out for sys.modules[...] is None
674# flag to force absolute imports? (speeds _determine_import_context and
675# checking for a relative module)
676# insert names of archives into sys.path (see quote below)
677# note: reload does NOT blast module dict
678# shift import mechanisms and policies around; provide for hooks, overrides
679# (see quote below)
680# add get_source stuff
681# get_topcode and get_subcode
682# CRLF handling in _compile
683# race condition in _compile
684# refactoring of os.py to deal with _os_bootstrap problem
685# any special handling to do for importing a module with a SyntaxError?
686# (e.g. clean up the traceback)
687# implement "domain" for path-type functionality using pkg namespace
688# (rather than FS-names like __path__)
689# don't use the word "private"... maybe "internal"
690#
691#
692# Guido's comments on sys.path caching:
693#
694# We could cache this in a dictionary: the ImportManager can have a
695# cache dict mapping pathnames to importer objects, and a separate
696# method for coming up with an importer given a pathname that's not yet
697# in the cache. The method should do a stat and/or look at the
698# extension to decide which importer class to use; you can register new
699# importer classes by registering a suffix or a Boolean function, plus a
700# class. If you register a new importer class, the cache is zapped.
701# The cache is independent from sys.path (but maintained per
702# ImportManager instance) so that rearrangements of sys.path do the
703# right thing. If a path is dropped from sys.path the corresponding
704# cache entry is simply no longer used.
705#
706# My/Guido's comments on factoring ImportManager and Importer:
707#
708# > However, we still have a tension occurring here:
709# >
710# > 1) implementing policy in ImportManager assists in single-point policy
711# > changes for app/rexec situations
712# > 2) implementing policy in Importer assists in package-private policy
713# > changes for normal, operating conditions
714# >
715# > I'll see if I can sort out a way to do this. Maybe the Importer class will
716# > implement the methods (which can be overridden to change policy) by
717# > delegating to ImportManager.
718#
719# Maybe also think about what kind of policies an Importer would be
720# likely to want to change. I have a feeling that a lot of the code
721# there is actually not so much policy but a *necessity* to get things
722# working given the calling conventions for the __import__ hook: whether
723# to return the head or tail of a dotted name, or when to do the "finish
724# fromlist" stuff.
725#
Note: See TracBrowser for help on using the repository browser.