301 lines
11 KiB
Python
301 lines
11 KiB
Python
|
"""Utility code for constructing importers, etc."""
|
||
|
|
from . import abc
|
||
|
|
from ._bootstrap import module_from_spec
|
||
|
|
from ._bootstrap import _resolve_name
|
||
|
|
from ._bootstrap import spec_from_loader
|
||
|
|
from ._bootstrap import _find_spec
|
||
|
|
from ._bootstrap_external import MAGIC_NUMBER
|
||
|
|
from ._bootstrap_external import _RAW_MAGIC_NUMBER
|
||
|
|
from ._bootstrap_external import cache_from_source
|
||
|
|
from ._bootstrap_external import decode_source
|
||
|
|
from ._bootstrap_external import source_from_cache
|
||
|
|
from ._bootstrap_external import spec_from_file_location
|
||
|
|
|
||
|
|
from contextlib import contextmanager
|
||
|
|
import _imp
|
||
|
|
import functools
|
||
|
|
import sys
|
||
|
|
import types
|
||
|
|
import warnings
|
||
|
|
|
||
|
|
|
||
|
|
def source_hash(source_bytes):
|
||
|
|
"Return the hash of *source_bytes* as used in hash-based pyc files."
|
||
|
|
return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes)
|
||
|
|
|
||
|
|
|
||
|
|
def resolve_name(name, package):
|
||
|
|
"""Resolve a relative module name to an absolute one."""
|
||
|
|
if not name.startswith('.'):
|
||
|
|
return name
|
||
|
|
elif not package:
|
||
|
|
raise ValueError(f'no package specified for {repr(name)} '
|
||
|
|
'(required for relative module names)')
|
||
|
|
level = 0
|
||
|
|
for character in name:
|
||
|
|
if character != '.':
|
||
|
|
break
|
||
|
|
level += 1
|
||
|
|
return _resolve_name(name[level:], package, level)
|
||
|
|
|
||
|
|
|
||
|
|
def _find_spec_from_path(name, path=None):
|
||
|
|
"""Return the spec for the specified module.
|
||
|
|
|
||
|
|
First, sys.modules is checked to see if the module was already imported. If
|
||
|
|
so, then sys.modules[name].__spec__ is returned. If that happens to be
|
||
|
|
set to None, then ValueError is raised. If the module is not in
|
||
|
|
sys.modules, then sys.meta_path is searched for a suitable spec with the
|
||
|
|
value of 'path' given to the finders. None is returned if no spec could
|
||
|
|
be found.
|
||
|
|
|
||
|
|
Dotted names do not have their parent packages implicitly imported. You will
|
||
|
|
most likely need to explicitly import all parent packages in the proper
|
||
|
|
order for a submodule to get the correct spec.
|
||
|
|
|
||
|
|
"""
|
||
|
|
if name not in sys.modules:
|
||
|
|
return _find_spec(name, path)
|
||
|
|
else:
|
||
|
|
module = sys.modules[name]
|
||
|
|
if module is None:
|
||
|
|
return None
|
||
|
|
try:
|
||
|
|
spec = module.__spec__
|
||
|
|
except AttributeError:
|
||
|
|
raise ValueError('{}.__spec__ is not set'.format(name)) from None
|
||
|
|
else:
|
||
|
|
if spec is None:
|
||
|
|
raise ValueError('{}.__spec__ is None'.format(name))
|
||
|
|
return spec
|
||
|
|
|
||
|
|
|
||
|
|
def find_spec(name, package=None):
|
||
|
|
"""Return the spec for the specified module.
|
||
|
|
|
||
|
|
First, sys.modules is checked to see if the module was already imported. If
|
||
|
|
so, then sys.modules[name].__spec__ is returned. If that happens to be
|
||
|
|
set to None, then ValueError is raised. If the module is not in
|
||
|
|
sys.modules, then sys.meta_path is searched for a suitable spec with the
|
||
|
|
value of 'path' given to the finders. None is returned if no spec could
|
||
|
|
be found.
|
||
|
|
|
||
|
|
If the name is for submodule (contains a dot), the parent module is
|
||
|
|
automatically imported.
|
||
|
|
|
||
|
|
The name and package arguments work the same as importlib.import_module().
|
||
|
|
In other words, relative module names (with leading dots) work.
|
||
|
|
|
||
|
|
"""
|
||
|
|
fullname = resolve_name(name, package) if name.startswith('.') else name
|
||
|
|
if fullname not in sys.modules:
|
||
|
|
parent_name = fullname.rpartition('.')[0]
|
||
|
|
if parent_name:
|
||
|
|
parent = __import__(parent_name, fromlist=['__path__'])
|
||
|
|
try:
|
||
|
|
parent_path = parent.__path__
|
||
|
|
except AttributeError as e:
|
||
|
|
raise ModuleNotFoundError(
|
||
|
|
f"__path__ attribute not found on {parent_name!r} "
|
||
|
|
f"while trying to find {fullname!r}", name=fullname) from e
|
||
|
|
else:
|
||
|
|
parent_path = None
|
||
|
|
return _find_spec(fullname, parent_path)
|
||
|
|
else:
|
||
|
|
module = sys.modules[fullname]
|
||
|
|
if module is None:
|
||
|
|
return None
|
||
|
|
try:
|
||
|
|
spec = module.__spec__
|
||
|
|
except AttributeError:
|
||
|
|
raise ValueError('{}.__spec__ is not set'.format(name)) from None
|
||
|
|
else:
|
||
|
|
if spec is None:
|
||
|
|
raise ValueError('{}.__spec__ is None'.format(name))
|
||
|
|
return spec
|
||
|
|
|
||
|
|
|
||
|
|
@contextmanager
|
||
|
|
def _module_to_load(name):
|
||
|
|
is_reload = name in sys.modules
|
||
|
|
|
||
|
|
module = sys.modules.get(name)
|
||
|
|
if not is_reload:
|
||
|
|
# This must be done before open() is called as the 'io' module
|
||
|
|
# implicitly imports 'locale' and would otherwise trigger an
|
||
|
|
# infinite loop.
|
||
|
|
module = type(sys)(name)
|
||
|
|
# This must be done before putting the module in sys.modules
|
||
|
|
# (otherwise an optimization shortcut in import.c becomes wrong)
|
||
|
|
module.__initializing__ = True
|
||
|
|
sys.modules[name] = module
|
||
|
|
try:
|
||
|
|
yield module
|
||
|
|
except Exception:
|
||
|
|
if not is_reload:
|
||
|
|
try:
|
||
|
|
del sys.modules[name]
|
||
|
|
except KeyError:
|
||
|
|
pass
|
||
|
|
finally:
|
||
|
|
module.__initializing__ = False
|
||
|
|
|
||
|
|
|
||
|
|
def set_package(fxn):
|
||
|
|
"""Set __package__ on the returned module.
|
||
|
|
|
||
|
|
This function is deprecated.
|
||
|
|
|
||
|
|
"""
|
||
|
|
@functools.wraps(fxn)
|
||
|
|
def set_package_wrapper(*args, **kwargs):
|
||
|
|
warnings.warn('The import system now takes care of this automatically.',
|
||
|
|
DeprecationWarning, stacklevel=2)
|
||
|
|
module = fxn(*args, **kwargs)
|
||
|
|
if getattr(module, '__package__', None) is None:
|
||
|
|
module.__package__ = module.__name__
|
||
|
|
if not hasattr(module, '__path__'):
|
||
|
|
module.__package__ = module.__package__.rpartition('.')[0]
|
||
|
|
return module
|
||
|
|
return set_package_wrapper
|
||
|
|
|
||
|
|
|
||
|
|
def set_loader(fxn):
|
||
|
|
"""Set __loader__ on the returned module.
|
||
|
|
|
||
|
|
This function is deprecated.
|
||
|
|
|
||
|
|
"""
|
||
|
|
@functools.wraps(fxn)
|
||
|
|
def set_loader_wrapper(self, *args, **kwargs):
|
||
|
|
warnings.warn('The import system now takes care of this automatically.',
|
||
|
|
DeprecationWarning, stacklevel=2)
|
||
|
|
module = fxn(self, *args, **kwargs)
|
||
|
|
if getattr(module, '__loader__', None) is None:
|
||
|
|
module.__loader__ = self
|
||
|
|
return module
|
||
|
|
return set_loader_wrapper
|
||
|
|
|
||
|
|
|
||
|
|
def module_for_loader(fxn):
|
||
|
|
"""Decorator to handle selecting the proper module for loaders.
|
||
|
|
|
||
|
|
The decorated function is passed the module to use instead of the module
|
||
|
|
name. The module passed in to the function is either from sys.modules if
|
||
|
|
it already exists or is a new module. If the module is new, then __name__
|
||
|
|
is set the first argument to the method, __loader__ is set to self, and
|
||
|
|
__package__ is set accordingly (if self.is_package() is defined) will be set
|
||
|
|
before it is passed to the decorated function (if self.is_package() does
|
||
|
|
not work for the module it will be set post-load).
|
||
|
|
|
||
|
|
If an exception is raised and the decorator created the module it is
|
||
|
|
subsequently removed from sys.modules.
|
||
|
|
|
||
|
|
The decorator assumes that the decorated function takes the module name as
|
||
|
|
the second argument.
|
||
|
|
|
||
|
|
"""
|
||
|
|
warnings.warn('The import system now takes care of this automatically.',
|
||
|
|
DeprecationWarning, stacklevel=2)
|
||
|
|
@functools.wraps(fxn)
|
||
|
|
def module_for_loader_wrapper(self, fullname, *args, **kwargs):
|
||
|
|
with _module_to_load(fullname) as module:
|
||
|
|
module.__loader__ = self
|
||
|
|
try:
|
||
|
|
is_package = self.is_package(fullname)
|
||
|
|
except (ImportError, AttributeError):
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
if is_package:
|
||
|
|
module.__package__ = fullname
|
||
|
|
else:
|
||
|
|
module.__package__ = fullname.rpartition('.')[0]
|
||
|
|
# If __package__ was not set above, __import__() will do it later.
|
||
|
|
return fxn(self, module, *args, **kwargs)
|
||
|
|
|
||
|
|
return module_for_loader_wrapper
|
||
|
|
|
||
|
|
|
||
|
|
class _LazyModule(types.ModuleType):
|
||
|
|
|
||
|
|
"""A subclass of the module type which triggers loading upon attribute access."""
|
||
|
|
|
||
|
|
def __getattribute__(self, attr):
|
||
|
|
"""Trigger the load of the module and return the attribute."""
|
||
|
|
# All module metadata must be garnered from __spec__ in order to avoid
|
||
|
|
# using mutated values.
|
||
|
|
# Stop triggering this method.
|
||
|
|
self.__class__ = types.ModuleType
|
||
|
|
# Get the original name to make sure no object substitution occurred
|
||
|
|
# in sys.modules.
|
||
|
|
original_name = self.__spec__.name
|
||
|
|
# Figure out exactly what attributes were mutated between the creation
|
||
|
|
# of the module and now.
|
||
|
|
attrs_then = self.__spec__.loader_state['__dict__']
|
||
|
|
original_type = self.__spec__.loader_state['__class__']
|
||
|
|
attrs_now = self.__dict__
|
||
|
|
attrs_updated = {}
|
||
|
|
for key, value in attrs_now.items():
|
||
|
|
# Code that set the attribute may have kept a reference to the
|
||
|
|
# assigned object, making identity more important than equality.
|
||
|
|
if key not in attrs_then:
|
||
|
|
attrs_updated[key] = value
|
||
|
|
elif id(attrs_now[key]) != id(attrs_then[key]):
|
||
|
|
attrs_updated[key] = value
|
||
|
|
self.__spec__.loader.exec_module(self)
|
||
|
|
# If exec_module() was used directly there is no guarantee the module
|
||
|
|
# object was put into sys.modules.
|
||
|
|
if original_name in sys.modules:
|
||
|
|
if id(self) != id(sys.modules[original_name]):
|
||
|
|
raise ValueError(f"module object for {original_name!r} "
|
||
|
|
"substituted in sys.modules during a lazy "
|
||
|
|
"load")
|
||
|
|
# Update after loading since that's what would happen in an eager
|
||
|
|
# loading situation.
|
||
|
|
self.__dict__.update(attrs_updated)
|
||
|
|
return getattr(self, attr)
|
||
|
|
|
||
|
|
def __delattr__(self, attr):
|
||
|
|
"""Trigger the load and then perform the deletion."""
|
||
|
|
# To trigger the load and raise an exception if the attribute
|
||
|
|
# doesn't exist.
|
||
|
|
self.__getattribute__(attr)
|
||
|
|
delattr(self, attr)
|
||
|
|
|
||
|
|
|
||
|
|
class LazyLoader(abc.Loader):
|
||
|
|
|
||
|
|
"""A loader that creates a module which defers loading until attribute access."""
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def __check_eager_loader(loader):
|
||
|
|
if not hasattr(loader, 'exec_module'):
|
||
|
|
raise TypeError('loader must define exec_module()')
|
||
|
|
|
||
|
|
@classmethod
|
||
|
|
def factory(cls, loader):
|
||
|
|
"""Construct a callable which returns the eager loader made lazy."""
|
||
|
|
cls.__check_eager_loader(loader)
|
||
|
|
return lambda *args, **kwargs: cls(loader(*args, **kwargs))
|
||
|
|
|
||
|
|
def __init__(self, loader):
|
||
|
|
self.__check_eager_loader(loader)
|
||
|
|
self.loader = loader
|
||
|
|
|
||
|
|
def create_module(self, spec):
|
||
|
|
return self.loader.create_module(spec)
|
||
|
|
|
||
|
|
def exec_module(self, module):
|
||
|
|
"""Make the module load lazily."""
|
||
|
|
module.__spec__.loader = self.loader
|
||
|
|
module.__loader__ = self.loader
|
||
|
|
# Don't need to worry about deep-copying as trying to set an attribute
|
||
|
|
# on an object would have triggered the load,
|
||
|
|
# e.g. ``module.__spec__.loader = None`` would trigger a load from
|
||
|
|
# trying to access module.__spec__.
|
||
|
|
loader_state = {}
|
||
|
|
loader_state['__dict__'] = module.__dict__.copy()
|
||
|
|
loader_state['__class__'] = module.__class__
|
||
|
|
module.__spec__.loader_state = loader_state
|
||
|
|
module.__class__ = _LazyModule
|