##############################################################################
# Institute for the Design of Advanced Energy Systems Process Systems
# Engineering Framework (IDAES PSE Framework) Copyright (c) 2018-2019, by the
# software owners: The Regents of the University of California, through
# Lawrence Berkeley National Laboratory, National Technology & Engineering
# Solutions of Sandia, LLC, Carnegie Mellon University, West Virginia
# University Research Corporation, et al. All rights reserved.
#
# Please see the files COPYRIGHT.txt and LICENSE.txt for full copyright and
# license information, respectively. Both files are also available online
# at the URL "https://github.com/IDAES/idaes-pse".
##############################################################################
"""
Search through the code and index static information in the DMF.
"""
# stdlib
import glob
import importlib
import inspect
import logging
import os
import pprint
import re
__author__ = 'Dan Gunter <dkgunter@lbl.gov>'
_log = logging.getLogger(__name__)
class Walker(object):
def walk(self, visitor):
"""Interface for walkers.
Args:
visitor (Visitor): Class whose `visit` method will be
called for each item.
Returns:
None
"""
pass
[docs]class ModuleClassWalker(Walker):
"""Walk modules from a given root (e.g. 'idaes'), and visit
all classes in those modules whose name matches a given pattern.
Example usage::
walker = ModuleClassWalker(from_pkg=idaes,
class_expr='_PropertyParameter.*')
walker.walk(PrintMetadataVisitor()) # see below
"""
def __init__(self, from_path=None, from_pkg=None, class_expr=None,
parent_class=None, suppress_warnings=False,
exclude_testdirs=True, exclude_tests=True,
exclude_init=True, exclude_setup=True,
exclude_dirs=None):
"""Constructor. Create from either a path or package root, or both.
If just one is given, the other is deduced. At least one must be
given.
Args:
from_path (str): Path to start looking for modules. Overrides
'from_pkg'.
from_pkg (module): Package from which to start looking for modules.
The path of the package will depend on the Python
environment and path.
class_expr (str): Regular expression for the classes to find.
parent_class (class): If given, filter for this parent class.
Will match class_expr OR parent_class.
suppress_warnings (bool): If True do not log any warning messages.
exclude_testdirs (bool): If True (the default), exclude test dirs
exclude_tests (bool): If True (the default), exclude test files
exclude_init (bool): If True (the default), exclude __init___.py
exclude_setup (bool): If True (the default), exclude setup.py
exclude_dirs (list): List of directory (regex) patterns to exclude.
These will be prefixed with a directory
separator (e.g. '/') but otherwise can be
any path expression.
"""
if from_path:
self._root = from_path
if from_pkg:
self._pkg = from_pkg.__name__
else:
self._pkg = str(os.path.basename(self._root))
elif from_pkg:
self._pkg = from_pkg.__name__
try:
self._root = from_pkg.__path__[0]
except AttributeError:
raise IOError('Root directory cannot be deduced from'
' package "{}": no __path__ attribute.'
.format(self._pkg))
else:
raise ValueError('Missing arguments: either "from_pkg" or '
'"from_path" must be given')
if not os.path.isdir(self._root):
raise IOError('Root directory "{}"'.format(self._root))
self._expr = re.compile(class_expr) if class_expr else None
self._warn = not suppress_warnings
self._parent = parent_class
# build regular expression of things to exclude
expr_list, psep = [], os.path.sep
if exclude_testdirs:
expr_list.append('{sl}tests?{sl}'.format(sl=psep))
if exclude_tests:
expr_list.append(r'tests?_.*.py')
if exclude_init:
expr_list.append(r'__init__\.py')
if exclude_setup:
expr_list.append(r'setup\.py')
if exclude_dirs:
for ed in exclude_dirs:
expr_list.append('{sl}{d}'.format(sl=psep, d=ed))
self._exclude_expr = re.compile('|'.join(expr_list))
self._history = []
# print('@@ exclude expr={}'.format(self._exclude_expr.pattern))
[docs] def walk(self, visitor):
modules = self._get_modules()
self._visit_subclasses(modules, visitor.visit)
def get_indexed_classes(self):
return self._history
def _get_modules(self):
_log.debug('getting modules from root: {}'.format(self._root))
# change file paths at 'root' to module paths from 'pkgroot'
n, module_list = len(self._root), []
for f in self._python_files():
module_path = os.path.splitext(f[n + 1:])[0]
module_path = module_path.replace(os.path.sep, '.')
module_list.append(self._pkg + '.' + module_path)
return module_list
def _python_files(self):
q = [self._root]
while q:
curdir = q.pop()
for f in glob.glob(os.path.join(curdir, '*.py')):
if not self._exclude_expr.search(f):
yield f
for d in os.listdir(curdir):
path = os.path.join(curdir, d)
if os.path.isdir(path):
q.append(path)
def _visit_subclasses(self, modules, visit):
for modname in modules:
_log.debug('visit module: {}'.format(modname))
try:
mod = importlib.import_module(modname)
except Exception:
if self._warn:
_log.warn('Error during import of module: {}. Ignoring.'
.format(modname))
continue
for item in dir(mod):
x = getattr(mod, item)
if inspect.isclass(x):
fullname = modname + '.' + x.__name__
if not self._expr and not self._parent:
if visit(x):
self._history.append(fullname)
elif self._expr:
if self._expr.match(x.__name__):
if visit(x):
self._history.append(fullname)
elif self._parent and issubclass(x, self._parent):
if visit(x):
self._history.append(fullname)
[docs]class Visitor(object):
"""Interface for the 'visitor' class passed to Walker subclasses'
`walk()` method.
"""
[docs] def visit(self, obj):
"""Visit one object.
Args:
obj (object): Some object to operate on.
Returns:
True if visit succeeded, else False
"""
pass
# This is for testing
class _TestClass(object):
pass
class _TestClass1(_TestClass):
@classmethod
def get_metadata(cls):
return {'x': 1}
class _TestClass2(_TestClass):
@classmethod
def get_metadata(cls):
return {'x': 2}
class _TestClass3(_TestClass):
@classmethod
def get_metadata(cls):
return {'x': 3}