Package logilab :: Package common :: Module modutils
[frames] | no frames]

Source Code for Module logilab.common.modutils

  1  # -*- coding: utf-8 -*- 
  2  # copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. 
  3  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr 
  4  # 
  5  # This file is part of logilab-common. 
  6  # 
  7  # logilab-common is free software: you can redistribute it and/or modify it under 
  8  # the terms of the GNU Lesser General Public License as published by the Free 
  9  # Software Foundation, either version 2.1 of the License, or (at your option) any 
 10  # later version. 
 11  # 
 12  # logilab-common is distributed in the hope that it will be useful, but WITHOUT 
 13  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
 14  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more 
 15  # details. 
 16  # 
 17  # You should have received a copy of the GNU Lesser General Public License along 
 18  # with logilab-common.  If not, see <http://www.gnu.org/licenses/>. 
 19  """Python modules manipulation utility functions. 
 20   
 21  :type PY_SOURCE_EXTS: tuple(str) 
 22  :var PY_SOURCE_EXTS: list of possible python source file extension 
 23   
 24  :type STD_LIB_DIR: str 
 25  :var STD_LIB_DIR: directory where standard modules are located 
 26   
 27  :type BUILTIN_MODULES: dict 
 28  :var BUILTIN_MODULES: dictionary with builtin module names as key 
 29  """ 
 30   
 31  __docformat__ = "restructuredtext en" 
 32   
 33  import sys 
 34  import os 
 35  from os.path import (splitext, join, abspath, isdir, dirname, exists, 
 36                       basename, expanduser, normcase, realpath) 
 37  from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY 
 38  from distutils.sysconfig import get_config_var, get_python_lib, get_python_version 
 39  from distutils.errors import DistutilsPlatformError 
 40   
 41  from six import PY3 
 42  from six.moves import map, range 
 43   
 44  try: 
 45      import zipimport 
 46  except ImportError: 
 47      zipimport = None 
 48   
 49  ZIPFILE = object() 
 50   
 51  from logilab.common import STD_BLACKLIST, _handle_blacklist 
 52   
 53  # Notes about STD_LIB_DIR 
 54  # Consider arch-specific installation for STD_LIB_DIR definition 
 55  # :mod:`distutils.sysconfig` contains to much hardcoded values to rely on 
 56  # 
 57  # :see: `Problems with /usr/lib64 builds <http://bugs.python.org/issue1294959>`_ 
 58  # :see: `FHS <http://www.pathname.com/fhs/pub/fhs-2.3.html#LIBLTQUALGTALTERNATEFORMATESSENTIAL>`_ 
 59  if sys.platform.startswith('win'): 
 60      PY_SOURCE_EXTS = ('py', 'pyw') 
 61      PY_COMPILED_EXTS = ('dll', 'pyd') 
 62  else: 
 63      PY_SOURCE_EXTS = ('py',) 
 64      PY_COMPILED_EXTS = ('so',) 
 65   
 66  try: 
 67      STD_LIB_DIR = get_python_lib(standard_lib=True) 
 68  # get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to 
 69  # non-valid path, see https://bugs.pypy.org/issue1164 
 70  except DistutilsPlatformError: 
 71      STD_LIB_DIR = '//' 
 72   
 73  EXT_LIB_DIR = get_python_lib() 
 74   
 75  BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True) 
 76   
 77   
78 -class NoSourceFile(Exception):
79 """exception raised when we are not able to get a python 80 source file for a precompiled file 81 """
82
83 -class LazyObject(object):
84 - def __init__(self, module, obj):
85 self.module = module 86 self.obj = obj 87 self._imported = None
88
89 - def _getobj(self):
90 if self._imported is None: 91 self._imported = getattr(load_module_from_name(self.module), 92 self.obj) 93 return self._imported
94
95 - def __getattribute__(self, attr):
96 try: 97 return super(LazyObject, self).__getattribute__(attr) 98 except AttributeError as ex: 99 return getattr(self._getobj(), attr)
100
101 - def __call__(self, *args, **kwargs):
102 return self._getobj()(*args, **kwargs)
103 104
105 -def load_module_from_name(dotted_name, path=None, use_sys=True):
106 """Load a Python module from its name. 107 108 :type dotted_name: str 109 :param dotted_name: python name of a module or package 110 111 :type path: list or None 112 :param path: 113 optional list of path where the module or package should be 114 searched (use sys.path if nothing or None is given) 115 116 :type use_sys: bool 117 :param use_sys: 118 boolean indicating whether the sys.modules dictionary should be 119 used or not 120 121 122 :raise ImportError: if the module or package is not found 123 124 :rtype: module 125 :return: the loaded module 126 """ 127 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
128 129
130 -def load_module_from_modpath(parts, path=None, use_sys=True):
131 """Load a python module from its splitted name. 132 133 :type parts: list(str) or tuple(str) 134 :param parts: 135 python name of a module or package splitted on '.' 136 137 :type path: list or None 138 :param path: 139 optional list of path where the module or package should be 140 searched (use sys.path if nothing or None is given) 141 142 :type use_sys: bool 143 :param use_sys: 144 boolean indicating whether the sys.modules dictionary should be used or not 145 146 :raise ImportError: if the module or package is not found 147 148 :rtype: module 149 :return: the loaded module 150 """ 151 if use_sys: 152 try: 153 return sys.modules['.'.join(parts)] 154 except KeyError: 155 pass 156 modpath = [] 157 prevmodule = None 158 for part in parts: 159 modpath.append(part) 160 curname = '.'.join(modpath) 161 module = None 162 if len(modpath) != len(parts): 163 # even with use_sys=False, should try to get outer packages from sys.modules 164 module = sys.modules.get(curname) 165 elif use_sys: 166 # because it may have been indirectly loaded through a parent 167 module = sys.modules.get(curname) 168 if module is None: 169 mp_file, mp_filename, mp_desc = find_module(part, path) 170 try: 171 module = load_module(curname, mp_file, mp_filename, mp_desc) 172 finally: 173 if mp_file is not None: 174 mp_file.close() 175 if prevmodule: 176 setattr(prevmodule, part, module) 177 _file = getattr(module, '__file__', '') 178 prevmodule = module 179 if not _file and _is_namespace(curname): 180 continue 181 if not _file and len(modpath) != len(parts): 182 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) ) 183 path = [dirname( _file )] 184 return module
185 186
187 -def load_module_from_file(filepath, path=None, use_sys=True, extrapath=None):
188 """Load a Python module from it's path. 189 190 :type filepath: str 191 :param filepath: path to the python module or package 192 193 :type path: list or None 194 :param path: 195 optional list of path where the module or package should be 196 searched (use sys.path if nothing or None is given) 197 198 :type use_sys: bool 199 :param use_sys: 200 boolean indicating whether the sys.modules dictionary should be 201 used or not 202 203 204 :raise ImportError: if the module or package is not found 205 206 :rtype: module 207 :return: the loaded module 208 """ 209 modpath = modpath_from_file(filepath, extrapath) 210 return load_module_from_modpath(modpath, path, use_sys)
211 212
213 -def _check_init(path, mod_path):
214 """check there are some __init__.py all along the way""" 215 modpath = [] 216 for part in mod_path: 217 modpath.append(part) 218 path = join(path, part) 219 if not _is_namespace('.'.join(modpath)) and not _has_init(path): 220 return False 221 return True
222 223
224 -def _canonicalize_path(path):
225 return realpath(expanduser(path))
226 227
228 -def _path_from_filename(filename):
229 if PY3: 230 return filename 231 else: 232 if filename.endswith(".pyc"): 233 return filename[:-1] 234 return filename
235 236
237 -def modpath_from_file(filename, extrapath=None):
238 """given a file path return the corresponding splitted module's name 239 (i.e name of a module or package splitted on '.') 240 241 :type filename: str 242 :param filename: file's path for which we want the module's name 243 244 :type extrapath: dict 245 :param extrapath: 246 optional extra search path, with path as key and package name for the path 247 as value. This is usually useful to handle package splitted in multiple 248 directories using __path__ trick. 249 250 251 :raise ImportError: 252 if the corresponding module's name has not been found 253 254 :rtype: list(str) 255 :return: the corresponding splitted module's name 256 """ 257 filename = _path_from_filename(filename) 258 filename = _canonicalize_path(filename) 259 base = os.path.splitext(filename)[0] 260 261 if extrapath is not None: 262 for path_ in map(_canonicalize_path, extrapath): 263 path = abspath(path_) 264 if path and normcase(base[:len(path)]) == normcase(path): 265 submodpath = [pkg for pkg in base[len(path):].split(os.sep) 266 if pkg] 267 if _check_init(path, submodpath[:-1]): 268 return extrapath[path_].split('.') + submodpath 269 270 for path in map(_canonicalize_path, sys.path): 271 if path and normcase(base).startswith(path): 272 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg] 273 if _check_init(path, modpath[:-1]): 274 return modpath 275 276 raise ImportError('Unable to find module for %s in %s' % ( 277 filename, ', \n'.join(sys.path)))
278 279
280 -def file_from_modpath(modpath, path=None, context_file=None):
281 """given a mod path (i.e. splitted module / package name), return the 282 corresponding file, giving priority to source file over precompiled 283 file if it exists 284 285 :type modpath: list or tuple 286 :param modpath: 287 splitted module's name (i.e name of a module or package splitted 288 on '.') 289 (this means explicit relative imports that start with dots have 290 empty strings in this list!) 291 292 :type path: list or None 293 :param path: 294 optional list of path where the module or package should be 295 searched (use sys.path if nothing or None is given) 296 297 :type context_file: str or None 298 :param context_file: 299 context file to consider, necessary if the identifier has been 300 introduced using a relative import unresolvable in the actual 301 context (i.e. modutils) 302 303 :raise ImportError: if there is no such module in the directory 304 305 :rtype: str or None 306 :return: 307 the path to the module's file or None if it's an integrated 308 builtin module such as 'sys' 309 """ 310 if context_file is not None: 311 context = dirname(context_file) 312 else: 313 context = context_file 314 if modpath[0] == 'xml': 315 # handle _xmlplus 316 try: 317 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context) 318 except ImportError: 319 return _file_from_modpath(modpath, path, context) 320 elif modpath == ['os', 'path']: 321 # FIXME: currently ignoring search_path... 322 return os.path.__file__ 323 return _file_from_modpath(modpath, path, context)
324 325 326
327 -def get_module_part(dotted_name, context_file=None):
328 """given a dotted name return the module part of the name : 329 330 >>> get_module_part('logilab.common.modutils.get_module_part') 331 'logilab.common.modutils' 332 333 :type dotted_name: str 334 :param dotted_name: full name of the identifier we are interested in 335 336 :type context_file: str or None 337 :param context_file: 338 context file to consider, necessary if the identifier has been 339 introduced using a relative import unresolvable in the actual 340 context (i.e. modutils) 341 342 343 :raise ImportError: if there is no such module in the directory 344 345 :rtype: str or None 346 :return: 347 the module part of the name or None if we have not been able at 348 all to import the given name 349 350 XXX: deprecated, since it doesn't handle package precedence over module 351 (see #10066) 352 """ 353 # os.path trick 354 if dotted_name.startswith('os.path'): 355 return 'os.path' 356 parts = dotted_name.split('.') 357 if context_file is not None: 358 # first check for builtin module which won't be considered latter 359 # in that case (path != None) 360 if parts[0] in BUILTIN_MODULES: 361 if len(parts) > 2: 362 raise ImportError(dotted_name) 363 return parts[0] 364 # don't use += or insert, we want a new list to be created ! 365 path = None 366 starti = 0 367 if parts[0] == '': 368 assert context_file is not None, \ 369 'explicit relative import, but no context_file?' 370 path = [] # prevent resolving the import non-relatively 371 starti = 1 372 while parts[starti] == '': # for all further dots: change context 373 starti += 1 374 context_file = dirname(context_file) 375 for i in range(starti, len(parts)): 376 try: 377 file_from_modpath(parts[starti:i+1], 378 path=path, context_file=context_file) 379 except ImportError: 380 if not i >= max(1, len(parts) - 2): 381 raise 382 return '.'.join(parts[:i]) 383 return dotted_name
384 385
386 -def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
387 """given a package directory return a list of all available python 388 modules in the package and its subpackages 389 390 :type package: str 391 :param package: the python name for the package 392 393 :type src_directory: str 394 :param src_directory: 395 path of the directory corresponding to the package 396 397 :type blacklist: list or tuple 398 :param blacklist: 399 optional list of files or directory to ignore, default to 400 the value of `logilab.common.STD_BLACKLIST` 401 402 :rtype: list 403 :return: 404 the list of all available python modules in the package and its 405 subpackages 406 """ 407 modules = [] 408 for directory, dirnames, filenames in os.walk(src_directory): 409 _handle_blacklist(blacklist, dirnames, filenames) 410 # check for __init__.py 411 if not '__init__.py' in filenames: 412 dirnames[:] = () 413 continue 414 if directory != src_directory: 415 dir_package = directory[len(src_directory):].replace(os.sep, '.') 416 modules.append(package + dir_package) 417 for filename in filenames: 418 if _is_python_file(filename) and filename != '__init__.py': 419 src = join(directory, filename) 420 module = package + src[len(src_directory):-3] 421 modules.append(module.replace(os.sep, '.')) 422 return modules
423 424 425
426 -def get_module_files(src_directory, blacklist=STD_BLACKLIST):
427 """given a package directory return a list of all available python 428 module's files in the package and its subpackages 429 430 :type src_directory: str 431 :param src_directory: 432 path of the directory corresponding to the package 433 434 :type blacklist: list or tuple 435 :param blacklist: 436 optional list of files or directory to ignore, default to the value of 437 `logilab.common.STD_BLACKLIST` 438 439 :rtype: list 440 :return: 441 the list of all available python module's files in the package and 442 its subpackages 443 """ 444 files = [] 445 for directory, dirnames, filenames in os.walk(src_directory): 446 _handle_blacklist(blacklist, dirnames, filenames) 447 # check for __init__.py 448 if not '__init__.py' in filenames: 449 dirnames[:] = () 450 continue 451 for filename in filenames: 452 if _is_python_file(filename): 453 src = join(directory, filename) 454 files.append(src) 455 return files
456 457
458 -def get_source_file(filename, include_no_ext=False):
459 """given a python module's file name return the matching source file 460 name (the filename will be returned identically if it's a already an 461 absolute path to a python source file...) 462 463 :type filename: str 464 :param filename: python module's file name 465 466 467 :raise NoSourceFile: if no source file exists on the file system 468 469 :rtype: str 470 :return: the absolute path of the source file if it exists 471 """ 472 base, orig_ext = splitext(abspath(filename)) 473 for ext in PY_SOURCE_EXTS: 474 source_path = '%s.%s' % (base, ext) 475 if exists(source_path): 476 return source_path 477 if include_no_ext and not orig_ext and exists(base): 478 return base 479 raise NoSourceFile(filename)
480 481
482 -def cleanup_sys_modules(directories):
483 """remove submodules of `directories` from `sys.modules`""" 484 cleaned = [] 485 for modname, module in list(sys.modules.items()): 486 modfile = getattr(module, '__file__', None) 487 if modfile: 488 for directory in directories: 489 if modfile.startswith(directory): 490 cleaned.append(modname) 491 del sys.modules[modname] 492 break 493 return cleaned
494 495
496 -def is_python_source(filename):
497 """ 498 rtype: bool 499 return: True if the filename is a python source file 500 """ 501 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
502 503
504 -def is_standard_module(modname, std_path=(STD_LIB_DIR,)):
505 """try to guess if a module is a standard python module (by default, 506 see `std_path` parameter's description) 507 508 :type modname: str 509 :param modname: name of the module we are interested in 510 511 :type std_path: list(str) or tuple(str) 512 :param std_path: list of path considered as standard 513 514 515 :rtype: bool 516 :return: 517 true if the module: 518 - is located on the path listed in one of the directory in `std_path` 519 - is a built-in module 520 521 Note: this function is known to return wrong values when inside virtualenv. 522 See https://www.logilab.org/ticket/294756. 523 """ 524 modname = modname.split('.')[0] 525 try: 526 filename = file_from_modpath([modname]) 527 except ImportError as ex: 528 # import failed, i'm probably not so wrong by supposing it's 529 # not standard... 530 return False 531 # modules which are not living in a file are considered standard 532 # (sys and __builtin__ for instance) 533 if filename is None: 534 # we assume there are no namespaces in stdlib 535 return not _is_namespace(modname) 536 filename = abspath(filename) 537 if filename.startswith(EXT_LIB_DIR): 538 return False 539 for path in std_path: 540 if filename.startswith(abspath(path)): 541 return True 542 return False
543 544 545
546 -def is_relative(modname, from_file):
547 """return true if the given module name is relative to the given 548 file name 549 550 :type modname: str 551 :param modname: name of the module we are interested in 552 553 :type from_file: str 554 :param from_file: 555 path of the module from which modname has been imported 556 557 :rtype: bool 558 :return: 559 true if the module has been imported relatively to `from_file` 560 """ 561 if not isdir(from_file): 562 from_file = dirname(from_file) 563 if from_file in sys.path: 564 return False 565 try: 566 find_module(modname.split('.')[0], [from_file]) 567 return True 568 except ImportError: 569 return False
570 571 572 # internal only functions ##################################################### 573
574 -def _file_from_modpath(modpath, path=None, context=None):
575 """given a mod path (i.e. splitted module / package name), return the 576 corresponding file 577 578 this function is used internally, see `file_from_modpath`'s 579 documentation for more information 580 """ 581 assert len(modpath) > 0 582 if context is not None: 583 try: 584 mtype, mp_filename = _module_file(modpath, [context]) 585 except ImportError: 586 mtype, mp_filename = _module_file(modpath, path) 587 else: 588 mtype, mp_filename = _module_file(modpath, path) 589 if mtype == PY_COMPILED: 590 try: 591 return get_source_file(mp_filename) 592 except NoSourceFile: 593 return mp_filename 594 elif mtype == C_BUILTIN: 595 # integrated builtin module 596 return None 597 elif mtype == PKG_DIRECTORY: 598 mp_filename = _has_init(mp_filename) 599 return mp_filename
600
601 -def _search_zip(modpath, pic):
602 for filepath, importer in pic.items(): 603 if importer is not None: 604 if importer.find_module(modpath[0]): 605 if not importer.find_module('/'.join(modpath)): 606 raise ImportError('No module named %s in %s/%s' % ( 607 '.'.join(modpath[1:]), filepath, modpath)) 608 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath 609 raise ImportError('No module named %s' % '.'.join(modpath))
610 611 try: 612 import pkg_resources 613 except ImportError: 614 pkg_resources = None 615 616
617 -def _is_namespace(modname):
618 return (pkg_resources is not None 619 and modname in pkg_resources._namespace_packages)
620 621
622 -def _module_file(modpath, path=None):
623 """get a module type / file path 624 625 :type modpath: list or tuple 626 :param modpath: 627 splitted module's name (i.e name of a module or package splitted 628 on '.'), with leading empty strings for explicit relative import 629 630 :type path: list or None 631 :param path: 632 optional list of path where the module or package should be 633 searched (use sys.path if nothing or None is given) 634 635 636 :rtype: tuple(int, str) 637 :return: the module type flag and the file path for a module 638 """ 639 # egg support compat 640 try: 641 pic = sys.path_importer_cache 642 _path = (path is None and sys.path or path) 643 for __path in _path: 644 if not __path in pic: 645 try: 646 pic[__path] = zipimport.zipimporter(__path) 647 except zipimport.ZipImportError: 648 pic[__path] = None 649 checkeggs = True 650 except AttributeError: 651 checkeggs = False 652 # pkg_resources support (aka setuptools namespace packages) 653 if (_is_namespace(modpath[0]) and modpath[0] in sys.modules): 654 # setuptools has added into sys.modules a module object with proper 655 # __path__, get back information from there 656 module = sys.modules[modpath.pop(0)] 657 # use list() to protect against _NamespacePath instance we get with python 3, which 658 # find_module later doesn't like 659 path = list(module.__path__) 660 if not modpath: 661 return C_BUILTIN, None 662 imported = [] 663 while modpath: 664 modname = modpath[0] 665 # take care to changes in find_module implementation wrt builtin modules 666 # 667 # Python 2.6.6 (r266:84292, Sep 11 2012, 08:34:23) 668 # >>> imp.find_module('posix') 669 # (None, 'posix', ('', '', 6)) 670 # 671 # Python 3.3.1 (default, Apr 26 2013, 12:08:46) 672 # >>> imp.find_module('posix') 673 # (None, None, ('', '', 6)) 674 try: 675 _, mp_filename, mp_desc = find_module(modname, path) 676 except ImportError: 677 if checkeggs: 678 return _search_zip(modpath, pic)[:2] 679 raise 680 else: 681 if checkeggs and mp_filename: 682 fullabspath = [abspath(x) for x in _path] 683 try: 684 pathindex = fullabspath.index(dirname(abspath(mp_filename))) 685 emtype, emp_filename, zippath = _search_zip(modpath, pic) 686 if pathindex > _path.index(zippath): 687 # an egg takes priority 688 return emtype, emp_filename 689 except ValueError: 690 # XXX not in _path 691 pass 692 except ImportError: 693 pass 694 checkeggs = False 695 imported.append(modpath.pop(0)) 696 mtype = mp_desc[2] 697 if modpath: 698 if mtype != PKG_DIRECTORY: 699 raise ImportError('No module %s in %s' % ('.'.join(modpath), 700 '.'.join(imported))) 701 # XXX guess if package is using pkgutil.extend_path by looking for 702 # those keywords in the first four Kbytes 703 try: 704 with open(join(mp_filename, '__init__.py')) as stream: 705 data = stream.read(4096) 706 except IOError: 707 path = [mp_filename] 708 else: 709 if 'pkgutil' in data and 'extend_path' in data: 710 # extend_path is called, search sys.path for module/packages 711 # of this name see pkgutil.extend_path documentation 712 path = [join(p, *imported) for p in sys.path 713 if isdir(join(p, *imported))] 714 else: 715 path = [mp_filename] 716 return mtype, mp_filename
717
718 -def _is_python_file(filename):
719 """return true if the given filename should be considered as a python file 720 721 .pyc and .pyo are ignored 722 """ 723 for ext in ('.py', '.so', '.pyd', '.pyw'): 724 if filename.endswith(ext): 725 return True 726 return False
727 728
729 -def _has_init(directory):
730 """if the given directory has a valid __init__ file, return its path, 731 else return None 732 """ 733 mod_or_pack = join(directory, '__init__') 734 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'): 735 if exists(mod_or_pack + '.' + ext): 736 return mod_or_pack + '.' + ext 737 return None
738