| |
@@ -1,340 +0,0 @@
|
| |
- #! /usr/bin/python3
|
| |
-
|
| |
- """
|
| |
- Using 'csdiff', print newly added coding errors.
|
| |
- """
|
| |
-
|
| |
- import os
|
| |
- import sys
|
| |
- from subprocess import Popen, PIPE, check_output, check_call
|
| |
- import glob
|
| |
- import logging
|
| |
- import tempfile
|
| |
- import shutil
|
| |
- import argparse
|
| |
-
|
| |
-
|
| |
- logging.basicConfig(level=logging.ERROR, format='%(levelname)s: %(message)s')
|
| |
- log = logging.getLogger() # pylint: disable=invalid-name
|
| |
-
|
| |
- CSDIFF_PYLINT = os.path.realpath(os.path.join(os.path.dirname(__file__),
|
| |
- 'copr-csdiff-pylint'))
|
| |
-
|
| |
-
|
| |
- def _run_csdiff(old, new, msg):
|
| |
- popen_diff = Popen(['csdiff', '-c', old, new],
|
| |
- stdout=PIPE)
|
| |
- diff = popen_diff.communicate()[0].decode('utf-8')
|
| |
- if diff:
|
| |
- sep = "=" * (len(msg) + 2)
|
| |
- print(sep)
|
| |
- print(" {} ".format(msg))
|
| |
- print(sep + "\n")
|
| |
- sys.stdout.write(diff)
|
| |
- return int(bool(diff))
|
| |
-
|
| |
-
|
| |
- def file_type(filename):
|
| |
- """
|
| |
- Taking FILENAME (must exist), return it's type in string format. Current
|
| |
- supported formats are ('python',).
|
| |
- """
|
| |
- if filename.endswith(".py"):
|
| |
- return 'python'
|
| |
-
|
| |
- if os.path.islink(filename):
|
| |
- return 'unknown'
|
| |
-
|
| |
- with open(filename) as f_d:
|
| |
- try:
|
| |
- first_line = f_d.readline()
|
| |
- except UnicodeDecodeError:
|
| |
- return 'unknown'
|
| |
- if first_line.startswith('#!') and first_line.find('python') != -1:
|
| |
- return 'python'
|
| |
- return 'unknown'
|
| |
-
|
| |
-
|
| |
- class _Linter:
|
| |
- filetype = None
|
| |
- path_filters = None
|
| |
-
|
| |
- def __init__(self, gitroot, renames=None):
|
| |
- self.gitroot = gitroot
|
| |
- self.renames = renames
|
| |
-
|
| |
- @classmethod
|
| |
- def modify_rename(cls, old, new):
|
| |
- """ if the paths in linter output need adjustments """
|
| |
- return old, new
|
| |
-
|
| |
- def _sed_filter(self):
|
| |
- if not self.renames:
|
| |
- return None
|
| |
-
|
| |
- rules = []
|
| |
- for pair in self.renames:
|
| |
- old, new = self.modify_rename(pair[0], pair[1])
|
| |
- rule = "s|: \"{}\"|: \"{}\"|".format(old, new)
|
| |
- rules += ['-e', rule]
|
| |
-
|
| |
- return ['sed'] + rules
|
| |
-
|
| |
- def command(self, projectdir, filenames):
|
| |
- """
|
| |
- Given the list of FILENAMES, generate command that will be executed
|
| |
- by lint() method, and environment vars set. Return (CMD, ENV) pair.
|
| |
- """
|
| |
- raise NotImplementedError
|
| |
-
|
| |
- # pylint: disable=no-self-use,unused-argument
|
| |
- def is_compatible(self, file):
|
| |
- """ file contains 'filename' and 'type' attributes """
|
| |
- return True
|
| |
-
|
| |
- def lint(self, projectdir, files, logfd):
|
| |
- """ run the linter """
|
| |
- if not files:
|
| |
- return
|
| |
-
|
| |
- abs_projectdir = os.path.join(self.gitroot, projectdir)
|
| |
-
|
| |
- oldcwd = os.getcwd()
|
| |
-
|
| |
- try:
|
| |
- log.info("linting in %s", abs_projectdir)
|
| |
- os.chdir(abs_projectdir)
|
| |
- files = [f.filename for f in files if self.is_compatible(f)]
|
| |
- if not files:
|
| |
- return
|
| |
- linter_cmd, linter_env = self.command(projectdir, files)
|
| |
- log.debug("Running linter command: %s", linter_cmd)
|
| |
- env = os.environ.copy()
|
| |
- env.update(linter_env)
|
| |
- sed_cmd = self._sed_filter()
|
| |
- if sed_cmd:
|
| |
- linter = Popen(linter_cmd, env=env, stdout=PIPE)
|
| |
- sed = Popen(sed_cmd, stdout=logfd, stdin=linter.stdout)
|
| |
- sed.communicate()
|
| |
- else:
|
| |
- linter = Popen(linter_cmd, env=env, stdout=logfd)
|
| |
- linter.communicate()
|
| |
-
|
| |
- finally:
|
| |
- os.chdir(oldcwd)
|
| |
-
|
| |
-
|
| |
- class PylintLinter(_Linter):
|
| |
- """
|
| |
- Generate pyilnt error output that is compatible with 'csdiff'.
|
| |
- """
|
| |
- def is_compatible(self, file):
|
| |
- return file.type == 'python'
|
| |
-
|
| |
- def command(self, projectdir, filenames):
|
| |
- abs_pylintrc = os.path.join(self.gitroot, projectdir, 'pylintrc')
|
| |
- pylintrc = os.path.realpath(abs_pylintrc)
|
| |
- env = {}
|
| |
- if os.path.exists(pylintrc):
|
| |
- env['PYLINTRC'] = pylintrc
|
| |
- cmd = [CSDIFF_PYLINT] + filenames
|
| |
- return cmd, env
|
| |
-
|
| |
-
|
| |
- def get_rename_map(options, subdir):
|
| |
- """
|
| |
- Using the os.getcwd() and 'git diff --namestatus', generate list of
|
| |
- files to analyze with possible overrides. The returned format is
|
| |
- dict of format 'new_file' -> 'old_file'.
|
| |
- """
|
| |
- cmd = ['git', 'diff', '--name-status', '-C', options.compare_against,
|
| |
- '--numstat', '--relative', os.path.join(".", subdir)]
|
| |
-
|
| |
- log.debug("running: %s", " ".join(cmd))
|
| |
- # current file -> old_name
|
| |
- return_map = {}
|
| |
- output = check_output(cmd).decode('utf-8')
|
| |
- for line in output.split('\n'):
|
| |
- if not line:
|
| |
- continue
|
| |
-
|
| |
- parts = line.split('\t')
|
| |
- mode = parts[0]
|
| |
- if mode == '':
|
| |
- continue
|
| |
- if mode.startswith('R'):
|
| |
- log.debug("renamed: %s -> %s", parts[1], parts[2])
|
| |
- return_map[parts[2]] = parts[1]
|
| |
- elif mode.startswith('A'):
|
| |
- log.debug("new: %s", parts[1])
|
| |
- return_map[parts[1]] = None
|
| |
- elif mode == 'M':
|
| |
- log.debug("modified: %s", parts[1])
|
| |
- return_map[parts[1]] = parts[1]
|
| |
- else:
|
| |
- log.info("skipping diff mode %s for file %s", mode, parts[1])
|
| |
-
|
| |
- return return_map
|
| |
-
|
| |
-
|
| |
- class _Worker: # pylint: disable=too-few-public-methods
|
| |
- gitroot = None
|
| |
- # relative path within gitroot to the sub-project
|
| |
- projectdir = None
|
| |
- projectsubdir = None
|
| |
- workdir = None
|
| |
- checkout = None
|
| |
- linters = [PylintLinter]
|
| |
-
|
| |
- def __init__(self, options):
|
| |
- self.options = options
|
| |
-
|
| |
- def _analyze_projectdir(self):
|
| |
- """ find sub-directory in git repo which contains spec file """
|
| |
- gitroot = check_output(['git', 'rev-parse', '--show-toplevel'])
|
| |
- self.gitroot = gitroot.decode('utf-8').strip()
|
| |
-
|
| |
- checkout = check_output(['git', 'rev-parse',
|
| |
- self.options.compare_against])
|
| |
- self.checkout = checkout.decode('utf-8').strip()
|
| |
-
|
| |
- def rel_projdir(projdir):
|
| |
- gitroot_a = os.path.realpath(self.gitroot)
|
| |
- gitproj_a = os.path.realpath(projdir)
|
| |
- cwd_a = os.path.realpath(os.getcwd())
|
| |
- self.projectdir = gitproj_a.replace(gitroot_a + '/', '')
|
| |
- self.projectsubdir = (cwd_a + "/").replace(
|
| |
- "{}/{}/".format(gitroot_a, self.projectdir), "")
|
| |
- log.debug("relative projectdir: %s", self.projectdir)
|
| |
- log.debug("project subdir: %s", self.projectsubdir)
|
| |
-
|
| |
- path = os.getcwd()
|
| |
- while True:
|
| |
- log.info("checking for projectdir: %s", path)
|
| |
- if os.path.realpath(path) == '/':
|
| |
- raise Exception("project dir not found")
|
| |
- if os.path.isdir(os.path.join(path, '.git')):
|
| |
- rel_projdir(path)
|
| |
- return
|
| |
- if glob.glob(os.path.join(path, '*.spec')):
|
| |
- rel_projdir(path)
|
| |
- return
|
| |
- path = os.path.normpath(os.path.join(path, '..'))
|
| |
-
|
| |
- def _run_linters(self, old_report_fd, new_report_fd):
|
| |
- # pylint: disable=too-many-locals
|
| |
- lookup = get_rename_map(self.options, self.projectsubdir)
|
| |
- if not lookup:
|
| |
- return
|
| |
-
|
| |
- # prepare the old checkout
|
| |
- old_gitroot = os.path.join(self.workdir, 'old_dir')
|
| |
- origin_from = self.gitroot
|
| |
- check_call(['git', 'clone', '--quiet', origin_from, old_gitroot])
|
| |
- ret_cwd = os.getcwd()
|
| |
- try:
|
| |
- os.chdir(old_gitroot)
|
| |
- check_call(['git', 'checkout', '-q', self.checkout])
|
| |
- finally:
|
| |
- os.chdir(ret_cwd)
|
| |
-
|
| |
- def add_file(gitroot, files, filename):
|
| |
- if not filename:
|
| |
- return
|
| |
- git_file = os.path.join(gitroot, self.projectdir, filename)
|
| |
- if not os.path.isfile(git_file):
|
| |
- log.debug("skipping non-file %s", filename)
|
| |
- return
|
| |
- file = lambda: None # noqa: E731
|
| |
- file.filename = filename
|
| |
- file.type = file_type(git_file)
|
| |
- files.append(file)
|
| |
-
|
| |
- old_files = []
|
| |
- new_files = []
|
| |
- for filename in lookup:
|
| |
- add_file(self.gitroot, new_files, filename)
|
| |
- add_file(old_gitroot, old_files, lookup[filename])
|
| |
-
|
| |
- renames = []
|
| |
- for new in lookup:
|
| |
- old = lookup[new]
|
| |
- if old and old != new:
|
| |
- renames.append((old, new))
|
| |
-
|
| |
- for LinterClass in self.linters:
|
| |
- linter_new = LinterClass(self.gitroot)
|
| |
- linter_old = LinterClass(old_gitroot, renames)
|
| |
- linter_new.lint(self.projectdir, new_files, logfd=new_report_fd)
|
| |
- linter_old.lint(self.projectdir, old_files, logfd=old_report_fd)
|
| |
-
|
| |
- def run(self):
|
| |
- """
|
| |
- Run all the 'self.linters' against old sources and new sources,
|
| |
- and provide the diff.
|
| |
- """
|
| |
- self._analyze_projectdir()
|
| |
- self.workdir = tempfile.mkdtemp(prefix='copr-linter-')
|
| |
- try:
|
| |
- old_report = os.path.join(self.workdir, 'old')
|
| |
- new_report = os.path.join(self.workdir, 'new')
|
| |
-
|
| |
- with open(old_report, 'w') as old, open(new_report, 'w') as new:
|
| |
- oldcwd = os.getcwd()
|
| |
- try:
|
| |
- pd = os.path.join(self.gitroot, self.projectdir)
|
| |
- log.debug("Switching to project directory %s", pd)
|
| |
- os.chdir(pd)
|
| |
- self._run_linters(old, new)
|
| |
- finally:
|
| |
- os.chdir(oldcwd)
|
| |
-
|
| |
- if self.options.print_fixed_errors:
|
| |
- _run_csdiff(new_report, old_report, "Fixed warnings")
|
| |
- print()
|
| |
-
|
| |
- sys.exit(_run_csdiff(old_report, new_report, "Added warnings"))
|
| |
-
|
| |
- finally:
|
| |
- log.debug("removing workdir %s", self.workdir)
|
| |
- if not self.options.no_cleanup:
|
| |
- shutil.rmtree(self.workdir)
|
| |
-
|
| |
-
|
| |
- def _get_arg_parser():
|
| |
- parser = argparse.ArgumentParser()
|
| |
- parser.add_argument(
|
| |
- "--compare-against",
|
| |
- default="origin/main",
|
| |
- help=("What git ref to diff the linters' results against. Note "
|
| |
- "that the reference needs to be available in the current "
|
| |
- "git directory"))
|
| |
- parser.add_argument(
|
| |
- "--log-level",
|
| |
- default='error',
|
| |
- help="The python logging level, e.g. debug, error, ...")
|
| |
- parser.add_argument(
|
| |
- "--no-cleanup",
|
| |
- action='store_true',
|
| |
- default=False,
|
| |
- help=("Keep the temporary working directory, you can use the "
|
| |
- "--log-level=info switch to see where it is created"),
|
| |
- )
|
| |
- parser.add_argument(
|
| |
- "--print-fixed-errors",
|
| |
- action='store_true',
|
| |
- default=False,
|
| |
- help="Also print defects which were fixed by the changes",
|
| |
- )
|
| |
- return parser
|
| |
-
|
| |
-
|
| |
- def _main():
|
| |
- options = _get_arg_parser().parse_args()
|
| |
- log.setLevel(level=getattr(logging, options.log_level.upper()))
|
| |
- worker = _Worker(options)
|
| |
- worker.run()
|
| |
-
|
| |
-
|
| |
- if __name__ == "__main__":
|
| |
- _main()
|
| |
praiskup
commented 2 years ago
The linter can actually live on it's own in Fedora, and can be used
easily elsewhere.