I wrote a script to list imports of Python projects. It can print a JSON-ish dict of imported modules (keys) and members (values) as well as only the module names. It can also optionally exclude certain imports and those from the standard library.
#! /usr/bin/env python3
# lsimports.py - List python project imports
# Copyright (C) 2020 Richard Neumann <mail at richard dash neumann period de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""Lists imports of python modules and packages."""
from argparse import ArgumentParser, Namespace
from collections import defaultdict
from itertools import chain
from json import dumps
from logging import INFO, WARNING, basicConfig, getLogger
from pathlib import Path
from re import fullmatch
from typing import Iterable, Iterator, Set, Tuple
DESCRIPTION = __doc__
FROM_IMPORT = 'from ([\\w\\.]+) import (.*)'
IMPORT = 'import (.*)'
LOG_FORMAT = '[%(levelname)s] %(name)s: %(message)s'
LOGGER = getLogger(Path(__file__).stem)
STDLIB = Path('/usr/lib/python3.9')
def get_imports(filename: Path) -> dict:
"""Lists imports of the given file."""
imports = defaultdict(set)
with filename.open('r') as file:
for line in file:
if not (line := line.strip()):
continue
if (match := fullmatch(FROM_IMPORT, line)) is not None:
module, members = match.groups()
members = set(filter(None, members.split(',')))
elif (match := fullmatch(IMPORT, line)) is not None:
module = match.group(1)
members = set()
else:
continue
imports[module] |= members
return imports
def get_imports_from_files(paths: Iterable[Path]) -> dict:
"""Returns the imports from multiple files."""
imports = {}
for path in paths:
LOGGER.info('Checking: %s', path)
imports.update(get_imports(path))
return imports
def get_module_roots(imports: dict) -> Set[str]:
"""Returns a set of imported module roots."""
roots = set()
for module in imports.keys():
root, *_ = module.split('.')
roots.add(root)
return roots
def lsstdlib(root: Path) -> Iterator[str]:
"""Yields modules and packages of the standard library."""
for path in root.iterdir():
if path.is_file() and path.suffix == '.py' or path.is_dir():
yield path.stem
def iterfiles(path: Path) -> Iterator[Path]:
"""Recursively yields files in a directory."""
if path.is_dir():
for subdir in path.iterdir():
yield from iterfiles(subdir)
elif path.is_file():
yield path
def filterimports(excludes: set, imports: dict) -> Iterator[Tuple[str, set]]:
"""Yields filtered key/value pairs of imports."""
for module, members in imports.items():
root, *_ = module.split('.')
if root not in excludes:
yield (module, members)
def ispyfile(path: Path) -> bool:
"""Checks whether the file is a python file."""
return path.is_file() and path.suffix == '.py'
def jsonify(imports: dict) -> dict:
"""Makes a dict JSON-compliant."""
return {str(key): list(value) for key, value in imports.items()}
def get_args() -> Namespace:
"""Parses and returns the command line arguments."""
parser = ArgumentParser(description=DESCRIPTION)
parser.add_argument('path', nargs='*', type=Path, default=[Path.cwd()],
help='the files and folders to scan for imports')
parser.add_argument('-E', '--exclude-stdlib', action='store_true',
help='exclude imports from the standard library')
parser.add_argument('-e', '--exclude-modules', nargs='+', default=(),
help='exclude the specified imports')
parser.add_argument('-i', '--indent', type=int, metavar='spaces',
help='set indentation for JSON output')
parser.add_argument('-r', '--roots', action='store_true',
help='print only distinct root modules and packages')
parser.add_argument('-s', '--stdlib', type=Path, default=STDLIB,
help='specifies the root of the standard library')
parser.add_argument('-v', '--verbose', action='store_true',
help='print verbose messages')
return parser.parse_args()
def main():
"""Runs the script."""
args = get_args()
basicConfig(format=LOG_FORMAT, level=INFO if args.verbose else WARNING)
files = filter(ispyfile, chain(*map(iterfiles, args.path)))
imports = get_imports_from_files(files)
excludes = set(args.exclude_modules)
if args.exclude_stdlib:
excludes |= set(lsstdlib(args.stdlib))
if excludes:
imports = dict(filterimports(excludes, imports))
if args.roots:
for root in get_module_roots(imports):
print(root)
else:
print(dumps(jsonify(imports), indent=args.indent))
if __name__ == '__main__':
main()
How can I improve my code?
Update Gist: https://gist.github.com/conqp/cb41511869bba0d7c526e05df431aaf3
ast.parse()throws aSyntaxErrorhere: gist.github.com/conqp/…. There's little I can do about that, ifastthinks that the code is syntactically incorrect. \$\endgroup\$