# -*- coding: utf-8 -*- # Copyright (c) 2014-2016 Claudiu Popa # Copyright (c) 2014 Google, Inc. # Copyright (c) 2015 Florian Bruhin # Copyright (c) 2015 Radosław Ganczarek # Copyright (c) 2016 Jakub Wilk # Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html # For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER """Python modules manipulation utility functions. :type PY_SOURCE_EXTS: tuple(str) :var PY_SOURCE_EXTS: list of possible python source file extension :type STD_LIB_DIRS: set of str :var STD_LIB_DIRS: directories where standard modules are located :type BUILTIN_MODULES: dict :var BUILTIN_MODULES: dictionary with builtin module names has key """ import imp import os import platform import sys from distutils.sysconfig import get_python_lib # pylint: disable=import-error # pylint: disable=import-error, no-name-in-module from distutils.errors import DistutilsPlatformError # distutils is replaced by virtualenv with a module that does # weird path manipulations in order to get to the # real distutils module. import six from .interpreter._import import spec from .interpreter._import import util if sys.platform.startswith('win'): PY_SOURCE_EXTS = ('py', 'pyw') PY_COMPILED_EXTS = ('dll', 'pyd') else: PY_SOURCE_EXTS = ('py',) PY_COMPILED_EXTS = ('so',) try: # The explicit sys.prefix is to work around a patch in virtualenv that # replaces the 'real' sys.prefix (i.e. the location of the binary) # with the prefix from which the virtualenv was created. This throws # off the detection logic for standard library modules, thus the # workaround. STD_LIB_DIRS = set([ get_python_lib(standard_lib=True, prefix=sys.prefix), # Take care of installations where exec_prefix != prefix. get_python_lib(standard_lib=True, prefix=sys.exec_prefix), get_python_lib(standard_lib=True)]) # get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to # non-valid path, see https://bugs.pypy.org/issue1164 except DistutilsPlatformError: STD_LIB_DIRS = set() if os.name == 'nt': STD_LIB_DIRS.add(os.path.join(sys.prefix, 'dlls')) try: # real_prefix is defined when running inside virtual environments, # created with the **virtualenv** library. STD_LIB_DIRS.add(os.path.join(sys.real_prefix, 'dlls')) except AttributeError: # sys.base_exec_prefix is always defined, but in a virtual environment # created with the stdlib **venv** module, it points to the original # installation, if the virtual env is activated. try: STD_LIB_DIRS.add(os.path.join(sys.base_exec_prefix, 'dlls')) except AttributeError: pass if platform.python_implementation() == 'PyPy': _root = os.path.join(sys.prefix, 'lib_pypy') STD_LIB_DIRS.add(_root) try: # real_prefix is defined when running inside virtualenv. STD_LIB_DIRS.add(os.path.join(sys.real_prefix, 'lib_pypy')) except AttributeError: pass del _root if os.name == 'posix': # Need the real prefix is we're under a virtualenv, otherwise # the usual one will do. try: prefix = sys.real_prefix except AttributeError: prefix = sys.prefix def _posix_path(path): base_python = 'python%d.%d' % sys.version_info[:2] return os.path.join(prefix, path, base_python) STD_LIB_DIRS.add(_posix_path('lib')) if sys.maxsize > 2**32: # This tries to fix a problem with /usr/lib64 builds, # where systems are running both 32-bit and 64-bit code # on the same machine, which reflects into the places where # standard library could be found. More details can be found # here http://bugs.python.org/issue1294959. # An easy reproducing case would be # https://github.com/PyCQA/pylint/issues/712#issuecomment-163178753 STD_LIB_DIRS.add(_posix_path('lib64')) EXT_LIB_DIR = get_python_lib() IS_JYTHON = platform.python_implementation() == 'Jython' BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True) class NoSourceFile(Exception): """exception raised when we are not able to get a python source file for a precompiled file """ def _normalize_path(path): return os.path.normcase(os.path.abspath(path)) def _canonicalize_path(path): return os.path.realpath(os.path.expanduser(path)) def _path_from_filename(filename, is_jython=IS_JYTHON): if not is_jython: if sys.version_info > (3, 0): return filename else: if filename.endswith(".pyc"): return filename[:-1] return filename head, has_pyclass, _ = filename.partition("$py.class") if has_pyclass: return head + ".py" return filename def _handle_blacklist(blacklist, dirnames, filenames): """remove files/directories in the black list dirnames/filenames are usually from os.walk """ for norecurs in blacklist: if norecurs in dirnames: dirnames.remove(norecurs) elif norecurs in filenames: filenames.remove(norecurs) _NORM_PATH_CACHE = {} def _cache_normalize_path(path): """abspath with caching""" # _module_file calls abspath on every path in sys.path every time it's # called; on a larger codebase this easily adds up to half a second just # assembling path components. This cache alleviates that. try: return _NORM_PATH_CACHE[path] except KeyError: if not path: # don't cache result for '' return _normalize_path(path) result = _NORM_PATH_CACHE[path] = _normalize_path(path) return result def load_module_from_name(dotted_name, path=None, use_sys=True): """Load a Python module from its name. :type dotted_name: str :param dotted_name: python name of a module or package :type path: list or None :param path: optional list of path where the module or package should be searched (use sys.path if nothing or None is given) :type use_sys: bool :param use_sys: boolean indicating whether the sys.modules dictionary should be used or not :raise ImportError: if the module or package is not found :rtype: module :return: the loaded module """ return load_module_from_modpath(dotted_name.split('.'), path, use_sys) def load_module_from_modpath(parts, path=None, use_sys=1): """Load a python module from its split name. :type parts: list(str) or tuple(str) :param parts: python name of a module or package split on '.' :type path: list or None :param path: optional list of path where the module or package should be searched (use sys.path if nothing or None is given) :type use_sys: bool :param use_sys: boolean indicating whether the sys.modules dictionary should be used or not :raise ImportError: if the module or package is not found :rtype: module :return: the loaded module """ if use_sys: try: return sys.modules['.'.join(parts)] except KeyError: pass modpath = [] prevmodule = None for part in parts: modpath.append(part) curname = '.'.join(modpath) module = None if len(modpath) != len(parts): # even with use_sys=False, should try to get outer packages from sys.modules module = sys.modules.get(curname) elif use_sys: # because it may have been indirectly loaded through a parent module = sys.modules.get(curname) if module is None: mp_file, mp_filename, mp_desc = imp.find_module(part, path) module = imp.load_module(curname, mp_file, mp_filename, mp_desc) # mp_file still needs to be closed. if mp_file: mp_file.close() if prevmodule: setattr(prevmodule, part, module) _file = getattr(module, '__file__', '') prevmodule = module if not _file and util.is_namespace(curname): continue if not _file and len(modpath) != len(parts): raise ImportError('no module in %s' % '.'.join(parts[len(modpath):])) path = [os.path.dirname(_file)] return module def load_module_from_file(filepath, path=None, use_sys=True, extrapath=None): """Load a Python module from it's path. :type filepath: str :param filepath: path to the python module or package :type path: list or None :param path: optional list of path where the module or package should be searched (use sys.path if nothing or None is given) :type use_sys: bool :param use_sys: boolean indicating whether the sys.modules dictionary should be used or not :raise ImportError: if the module or package is not found :rtype: module :return: the loaded module """ modpath = modpath_from_file(filepath, extrapath) return load_module_from_modpath(modpath, path, use_sys) def check_modpath_has_init(path, mod_path): """check there are some __init__.py all along the way""" modpath = [] for part in mod_path: modpath.append(part) path = os.path.join(path, part) if not _has_init(path): old_namespace = util.is_namespace('.'.join(modpath)) if not old_namespace: return False return True def modpath_from_file_with_callback(filename, extrapath=None, is_package_cb=None): filename = _path_from_filename(filename) filename = os.path.realpath(os.path.expanduser(filename)) base = os.path.splitext(filename)[0] if extrapath is not None: for path_ in six.moves.map(_canonicalize_path, extrapath): path = os.path.abspath(path_) if path and os.path.normcase(base[:len(path)]) == os.path.normcase(path): submodpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg] if is_package_cb(path, submodpath[:-1]): return extrapath[path_].split('.') + submodpath for path in six.moves.map(_canonicalize_path, sys.path): path = _cache_normalize_path(path) if path and os.path.normcase(base).startswith(path): modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg] if is_package_cb(path, modpath[:-1]): return modpath raise ImportError('Unable to find module for %s in %s' % ( filename, ', \n'.join(sys.path))) def modpath_from_file(filename, extrapath=None): """given a file path return the corresponding split module's name (i.e name of a module or package split on '.') :type filename: str :param filename: file's path for which we want the module's name :type extrapath: dict :param extrapath: optional extra search path, with path as key and package name for the path as value. This is usually useful to handle package split in multiple directories using __path__ trick. :raise ImportError: if the corresponding module's name has not been found :rtype: list(str) :return: the corresponding split module's name """ return modpath_from_file_with_callback(filename, extrapath, check_modpath_has_init) def file_from_modpath(modpath, path=None, context_file=None): return file_info_from_modpath(modpath, path, context_file).location def file_info_from_modpath(modpath, path=None, context_file=None): """given a mod path (i.e. split module / package name), return the corresponding file, giving priority to source file over precompiled file if it exists :type modpath: list or tuple :param modpath: split module's name (i.e name of a module or package split on '.') (this means explicit relative imports that start with dots have empty strings in this list!) :type path: list or None :param path: optional list of path where the module or package should be searched (use sys.path if nothing or None is given) :type context_file: str or None :param context_file: context file to consider, necessary if the identifier has been introduced using a relative import unresolvable in the actual context (i.e. modutils) :raise ImportError: if there is no such module in the directory :rtype: (str or None, import type) :return: the path to the module's file or None if it's an integrated builtin module such as 'sys' """ if context_file is not None: context = os.path.dirname(context_file) else: context = context_file if modpath[0] == 'xml': # handle _xmlplus try: return _spec_from_modpath(['_xmlplus'] + modpath[1:], path, context) except ImportError: return _spec_from_modpath(modpath, path, context) elif modpath == ['os', 'path']: # FIXME: currently ignoring search_path... return spec.ModuleSpec(name='os.path', location=os.path.__file__, module_type=imp.PY_SOURCE) return _spec_from_modpath(modpath, path, context) def get_module_part(dotted_name, context_file=None): """given a dotted name return the module part of the name : >>> get_module_part('astroid.as_string.dump') 'astroid.as_string' :type dotted_name: str :param dotted_name: full name of the identifier we are interested in :type context_file: str or None :param context_file: context file to consider, necessary if the identifier has been introduced using a relative import unresolvable in the actual context (i.e. modutils) :raise ImportError: if there is no such module in the directory :rtype: str or None :return: the module part of the name or None if we have not been able at all to import the given name XXX: deprecated, since it doesn't handle package precedence over module (see #10066) """ # os.path trick if dotted_name.startswith('os.path'): return 'os.path' parts = dotted_name.split('.') if context_file is not None: # first check for builtin module which won't be considered latter # in that case (path != None) if parts[0] in BUILTIN_MODULES: if len(parts) > 2: raise ImportError(dotted_name) return parts[0] # don't use += or insert, we want a new list to be created ! path = None starti = 0 if parts[0] == '': assert context_file is not None, \ 'explicit relative import, but no context_file?' path = [] # prevent resolving the import non-relatively starti = 1 while parts[starti] == '': # for all further dots: change context starti += 1 context_file = os.path.dirname(context_file) for i in range(starti, len(parts)): try: file_from_modpath(parts[starti:i+1], path=path, context_file=context_file) except ImportError: if i < max(1, len(parts) - 2): raise return '.'.join(parts[:i]) return dotted_name def get_module_files(src_directory, blacklist, list_all=False): """given a package directory return a list of all available python module's files in the package and its subpackages :type src_directory: str :param src_directory: path of the directory corresponding to the package :type blacklist: list or tuple :param blacklist: iterable list of files or directories to ignore. :type list_all: bool :param list_all: get files from all paths, including ones without __init__.py :rtype: list :return: the list of all available python module's files in the package and its subpackages """ files = [] for directory, dirnames, filenames in os.walk(src_directory): if directory in blacklist: continue _handle_blacklist(blacklist, dirnames, filenames) # check for __init__.py if not list_all and '__init__.py' not in filenames: dirnames[:] = () continue for filename in filenames: if _is_python_file(filename): src = os.path.join(directory, filename) files.append(src) return files def get_source_file(filename, include_no_ext=False): """given a python module's file name return the matching source file name (the filename will be returned identically if it's a already an absolute path to a python source file...) :type filename: str :param filename: python module's file name :raise NoSourceFile: if no source file exists on the file system :rtype: str :return: the absolute path of the source file if it exists """ filename = os.path.abspath(_path_from_filename(filename)) base, orig_ext = os.path.splitext(filename) for ext in PY_SOURCE_EXTS: source_path = '%s.%s' % (base, ext) if os.path.exists(source_path): return source_path if include_no_ext and not orig_ext and os.path.exists(base): return base raise NoSourceFile(filename) def is_python_source(filename): """ rtype: bool return: True if the filename is a python source file """ return os.path.splitext(filename)[1][1:] in PY_SOURCE_EXTS def is_standard_module(modname, std_path=None): """try to guess if a module is a standard python module (by default, see `std_path` parameter's description) :type modname: str :param modname: name of the module we are interested in :type std_path: list(str) or tuple(str) :param std_path: list of path considered has standard :rtype: bool :return: true if the module: - is located on the path listed in one of the directory in `std_path` - is a built-in module """ modname = modname.split('.')[0] try: filename = file_from_modpath([modname]) except ImportError: # import failed, i'm probably not so wrong by supposing it's # not standard... return False # modules which are not living in a file are considered standard # (sys and __builtin__ for instance) if filename is None: # we assume there are no namespaces in stdlib return not util.is_namespace(modname) filename = _normalize_path(filename) if filename.startswith(_cache_normalize_path(EXT_LIB_DIR)): return False if std_path is None: std_path = STD_LIB_DIRS for path in std_path: if filename.startswith(_cache_normalize_path(path)): return True return False def is_relative(modname, from_file): """return true if the given module name is relative to the given file name :type modname: str :param modname: name of the module we are interested in :type from_file: str :param from_file: path of the module from which modname has been imported :rtype: bool :return: true if the module has been imported relatively to `from_file` """ if not os.path.isdir(from_file): from_file = os.path.dirname(from_file) if from_file in sys.path: return False try: stream, _, _ = imp.find_module(modname.split('.')[0], [from_file]) # Close the stream to avoid ResourceWarnings. if stream: stream.close() return True except ImportError: return False # internal only functions ##################################################### def _spec_from_modpath(modpath, path=None, context=None): """given a mod path (i.e. split module / package name), return the corresponding spec this function is used internally, see `file_from_modpath`'s documentation for more information """ assert modpath location = None if context is not None: try: found_spec = spec.find_spec(modpath, [context]) location = found_spec.location except ImportError: found_spec = spec.find_spec(modpath, path) location = found_spec.location else: found_spec = spec.find_spec(modpath, path) if found_spec.type == spec.ModuleType.PY_COMPILED: try: location = get_source_file(found_spec.location) return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE) except NoSourceFile: return found_spec._replace(location=location) elif found_spec.type == spec.ModuleType.C_BUILTIN: # integrated builtin module return found_spec._replace(location=None) elif found_spec.type == spec.ModuleType.PKG_DIRECTORY: location = _has_init(found_spec.location) return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE) return found_spec def _is_python_file(filename): """return true if the given filename should be considered as a python file .pyc and .pyo are ignored """ for ext in ('.py', '.so', '.pyd', '.pyw'): if filename.endswith(ext): return True return False def _has_init(directory): """if the given directory has a valid __init__ file, return its path, else return None """ mod_or_pack = os.path.join(directory, '__init__') for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'): if os.path.exists(mod_or_pack + '.' + ext): return mod_or_pack + '.' + ext return None def is_namespace(specobj): return specobj.type == spec.ModuleType.PY_NAMESPACE def is_directory(specobj): return specobj.type == spec.ModuleType.PKG_DIRECTORY