lint-diff.py

#!/usr/bin/env python3
"""Filter warnings output, to only show output for changed lines."""

# Filter warnings output, to only show output for changed lines.
#
# This is useful when you want to enforce some style guideline, but
# making the changes globally in your project would be too burdensome.
# You can make the requirement only for new and changed lines, so your
# codebase will conform to the new standard gradually, as you edit it.
#
# Note: If the warnings output indicates a catastrophic failure of the
# program that produces warnings, then this program will issue no
# warnings because there will be no warnings on specific lines.
# Example:
#   Exception in thread "main" java.lang.IllegalAccessError

# Usage:  lint-diff.py [options] diff.txt [warnings.txt]
#         If warnings.txt is omitted, use standard input.
# Output: all lines in warnings.txt that are on a changed line.
#         Output status is 1 if it produced any output, 0 if not, 2 if error.
# Options: --guess-strip means guess values for --strip-diff and --strip-lint.
#          --strip-diff=N means to ignore N leading "/" in diff.txt.
#          --strip-warnings=N means to ignore N leading "/" in warnings.txt.
#              Affects matching, but not output, of lines.
#          --context=N is how many lines adjacent to the changed ones
#              are also considered changed; the default is 2.
#          --debug means to print diagnostic output.

# Here is how you could use this in continuous integration (Azure Pipelines,
# CircleCI, GitHub Actions, and Travis CI are currently supported) to require
# that pull requests satisfy the command `command-that-issues-warnings`:
#
#  if [ -d /tmp/$USER/plume-scripts ] ; then
#   git -C /tmp/$USER/plume-scripts pull -q > /dev/null 2>&1
#  else
#   mkdir -p /tmp/$USER \
#    && git -C /tmp/$USER clone --depth=1 -q https://github.com/plume-lib/plume-scripts.git
#  fi
# (command-that-issues-warnings > /tmp/warnings.txt 2>&1) || true
# /tmp/$USER/plume-scripts/ci-lint-diff /tmp/warnings.txt

# Implementation notes:
# 1. It may be possible to achieve a similar result using diff (but not `git diff`):
# https://unix.stackexchange.com/questions/34874/diff-output-line-numbers .
# 2. The documentation for diffFilter (https://github.com/exussum12/coverageChecker)
# suggests it has this same functionality, but my tests indicate it does not.

import argparse
import collections
import os
import re
import sys
from pathlib import Path
from typing import Any

PROGRAM = Path(__file__).name

DEBUG = False

PLUSPLUSPLUS_RE = re.compile(r"\+\+\+ (\S*).*")

# This cannot be multiline because files are read one line at a time.
FILENAME_LINENO_RE = re.compile(r"([^:]*):([0-9]+):.*")

INITIAL_WHITESPACE_RE = re.compile(r"[ \t]")


def eprint(*args: object, **kwargs: Any) -> None:
    """Print to stderr."""
    print(*args, file=sys.stderr, **kwargs)


def strip_dirs(filename: str, num_dirs: int) -> str:
    """Strip off `num_dirs` leading "/" characters.

    Returns:
        A subdirectory, with `num_dirs` top-level enclosing directories removed.
    """
    if num_dirs == 0:
        return filename
    return os.path.join(*(filename.split(os.path.sep)[num_dirs:]))  # noqa: PTH118


## Tests:
"""
import os
assert strip_dirs("/a/b/c/d", 0) == '/a/b/c/d'
assert strip_dirs("/a/b/c/d", 1) == 'a/b/c/d'
assert strip_dirs("/a/b/c/d", 2) == 'b/c/d'
assert strip_dirs("/a/b/c/d", 3) == 'c/d'
assert strip_dirs("/a/b/c/d", 4) == 'd'
assert strip_dirs("/a/b/c/", 0) == '/a/b/c/'
assert strip_dirs("/a/b/c/", 1) == 'a/b/c/'
assert strip_dirs("/a/b/c/", 2) == 'b/c/'
assert strip_dirs("/a/b/c/", 3) == 'c/'
assert strip_dirs("/a/b/c/", 4) == ''
"""


def min_strips(filename1: str, filename2: str) -> tuple[int, int, str, str]:
    """Find matching suffixes of the given filenames.

    Returns a 4-tuple of 2 integers and 2 strings.  The integers
    indicate the smallest strip values that make the two filenames equal,
    or a maximal pair if the files have different basenames.  The last two
    elements of the tuple are the argument strings.

    Returns:
        A 4-tuple of 2 integers and 2 strings.
    """
    components1 = filename1.split(os.path.sep)
    components2 = filename2.split(os.path.sep)
    if components1[-1] != components2[-1]:
        ## TODO: is this special case necessary?
        return (1000, 1000, filename1, filename2)
    while components1 and components2 and components1[-1] == components2[-1]:
        del components1[-1]
        del components2[-1]
    return (len(components1), len(components2), filename1, filename2)


## Tests:
"""
import os
assert min_strips("/a/b/c/d", "/a/b/c/d") == (0, 0, "/a/b/c/d", "/a/b/c/d")
assert min_strips("e1/e2/a/b/c/d", "/a/b/c/d") == (2, 1, "e1/e2/a/b/c/d", "/a/b/c/d")
assert min_strips("/e1/e2/a/b/c/d", "/a/b/c/d") == (3, 1, "/e1/e2/a/b/c/d", "/a/b/c/d")
assert min_strips("e1/e2/a/b/c/d", "a/b/c/d") == (2, 0, "e1/e2/a/b/c/d", "a/b/c/d")
assert min_strips("/e1/e2/a/b/c/d", "a/b/c/d") == (3, 0, "/e1/e2/a/b/c/d", "a/b/c/d")
assert min_strips("/a/b/c/d", "/e/f/g/h") == (1000, 1000, "/a/b/c/d", "/e/f/g/h")
"""


def pair_min(
    pair1: tuple[int, int, str, str], pair2: tuple[int, int, str, str]
) -> tuple[int, int, str, str]:
    """Given two tuples, returns the one that is pointwise lesser in its first two elements.

    Fails if neither is lesser.

    Returns:
        the argument that is pointwise lesser in its first two elements.
    """
    if pair1[0] <= pair2[0] and pair1[1] <= pair2[1]:
        return pair1
    if pair1[0] >= pair2[0] and pair1[1] >= pair2[1]:
        return pair2
    msg = f"incomparable pairs: {pair1} {pair2}"
    raise Exception(msg)


## Tests:
"""
import os
assert pair_min((3,4,"a","b"), (5,6,"c","d")) == (3,4,"a","b")
assert pair_min((4,3,"a","b"), (6,5,"c","d")) == (4,3,"a","b")
assert pair_min((30,40,"a","b"), (5,6,"c","d")) == (5,6,"c","d")
assert pair_min((40,30,"a","b"), (6,5,"c","d")) == (6,5,"c","d")
"""


def diff_filenames(diff_filename: str) -> set[str]:
    """All the filenames in the given diff file.

    Returns:
        All the filenames in the given diff file.
    """
    result = set()
    with Path(diff_filename).open(encoding="utf-8") as diff:
        for diff_line in diff:
            match = PLUSPLUSPLUS_RE.match(diff_line)
            if match:
                filename: str = match.group(1)
                if filename != "/dev/null":
                    result.add(filename)
    return result


def warning_filenames(warning_filename: str) -> set[str]:
    """All the filenames in the given warning file.

    Returns:
        All the filenames in the given warning file.
    """
    result = set()
    with Path(warning_filename).open(encoding="utf-8") as warnings:
        for warning_line in warnings:
            match = FILENAME_LINENO_RE.match(warning_line)
            if match:
                # lstrip is necessary because after Gradle outputs all warnings,
                # it prints "> Compilation failed; see the compiler output
                # below." and then prints one warning, indented by two spaces.
                result.add(match.group(1).lstrip())
    return result


def guess_strip_filenames(
    diff_filenames: set[str], warning_filenames: set[str]
) -> tuple[int, int, str, str]:
    """Match subdirectory structure.

    Arguments are two lists of file names.

    Returns:
        A pair of integers.
    """
    result = (1000, 1000, "no files seen yet", "no files seen yet")
    for diff_filename in diff_filenames:
        for warning_filename in warning_filenames:
            result = pair_min(result, min_strips(diff_filename, warning_filename))
    return result


def guess_strip_files(diff_file: str, warning_file: str) -> tuple[int, int, str, str]:
    """Match subdirectory structure.

    Arguments are files produced by diff and a lint tool, respectively.

    Returns:
        A pair of integers.
    """
    diff_files = diff_filenames(diff_file)
    warning_files = warning_filenames(warning_file)
    result = guess_strip_filenames(diff_files, warning_files)
    diff_prefix = commonpath(diff_files)
    try:
        warnings_prefix = commonpath(warning_files)
    except ValueError:
        with Path.open(Path(warning_file)) as file:
            for line in file:
                # rstrip is necessary because eprint adds a newline.
                eprint(line.rstrip())
        raise
    if result[0] > diff_prefix.count("/") or result[1] > warnings_prefix.count("/"):
        # This is not necessarily a problem.  It is possible that all the
        # diffs, or all the lint output, happens to be in one subdirectory.
        if DEBUG:
            eprint(
                "lint-diff.py: guess_strip_files all in one subdirectory: "
                f"result={result} diff_prefix={diff_prefix} warnings_prefix={warnings_prefix}"
            )
            eprint(f"diff_files={diff_files}")
            eprint(f"warning_files={warning_files}")
    return result


def commonpath(files: collections.abc.Iterable[str]) -> str:
    """Return the common prefix of the given paths.

    Argument is a set or list of paths.

    Returns:
         the common prefix of the given paths.
    """
    if not files:
        return ""
    try:
        # Remove leading null characters.
        files_list = [file.lstrip("\0") for file in files]
        return os.path.commonpath(files_list)
    except ValueError as err:
        raise ValueError(str(files)) from err


### Main routine


def parse_args() -> argparse.Namespace:
    """Parse and return the command-line arguments.

    Returns:
        The parsed command-line arguments.
    """
    global DEBUG

    parser = argparse.ArgumentParser(
        description="Filter warnings output, to only show output for changed lines"
    )
    parser.add_argument(
        "--guess-strip",
        dest="guess_strip",
        action="store_true",
        default=False,
        help="guess values for --strip-diff and --strip-warnings",
    )
    parser.add_argument(
        "--strip-diff",
        metavar="NUM_SLASHES",
        dest="strip_diff",
        action="store",
        type=int,
        default=0,
        help='ignore N leading "/" in filenames in diff.txt',
    )
    parser.add_argument(
        "--strip-warnings",
        metavar="NUM_SLASHES",
        dest="strip_warnings",
        action="store",
        type=int,
        default=0,
        help='ignore N leading "/" in filenames in warnings.txt',
    )
    # Deprecated; exists for backwards compatibility
    parser.add_argument(
        "--strip-lint",
        metavar="NUM_SLASHES",
        dest="strip_warnings",
        action="store",
        type=int,
        default=0,
        help='(DEPRECATED) ignore N leading "/" in filenames in warnings.txt',
    )
    parser.add_argument(
        "--context",
        metavar="NUM_LINES",
        dest="context_lines",
        action="store",
        type=int,
        default=2,
        help="how many lines around each changed one are also considered changed",
    )
    parser.add_argument(
        "--debug", dest="DEBUG", action="store_true", help="print diagnostic output"
    )
    parser.add_argument("diff_filename", metavar="diff.txt", default=Path.cwd())
    parser.add_argument("warning_filename", metavar="warnings.txt", default=None)

    args = parser.parse_args()
    DEBUG = args.DEBUG

    if args.guess_strip and args.strip_diff != 0:
        eprint(PROGRAM, ": don't supply both --guess-strip and --strip-diff")
        sys.exit(2)

    if args.guess_strip and args.strip_warnings != 0:
        eprint(PROGRAM, ": don't supply both --guess-strip and --strip-warnings")
        sys.exit(2)

    if args.guess_strip and args.warning_filename is None:
        eprint(PROGRAM, 'needs "warnings.txt" file argument when --guess-strip is provided')
        sys.exit(2)

    if args.guess_strip:
        guessed_strip = guess_strip_files(args.diff_filename, args.warning_filename)
        if guessed_strip[0] == 1000:
            if DEBUG:
                eprint(
                    "lint-diff.py: --guess-strip failed to guess values (maybe no files in common?)"
                )
        else:
            args.strip_diff = guessed_strip[0]
            args.strip_warnings = guessed_strip[1]
            if DEBUG:
                eprint(
                    "lint-diff.py inferred "
                    f"--strip-diff={args.strip_diff} --strip-warnings={args.strip_warnings}"
                )

    # A filename if the diff filenames start with "a/" and "b/", otherwise None.
    # Is set by changed_lines().
    args.relative_diff = None

    return args


def changed_lines(args: argparse.Namespace) -> dict[str, set[int]]:
    """Return a dictionary from file names to a set of ints (line numbers for changed lines).

    Returns:
        a dictionary from file names to a set of ints (line numbers for changed lines).
    """
    changed: dict[str, set[int]] = {}

    with Path(args.diff_filename).open(encoding="utf-8") as diff:
        atat_re = re.compile(r"@@ -([0-9]+)(,[0-9]+)? \+([0-9]+)(,[0-9]+)? @@.*")
        # content_re = re.compile("[ +-].*")

        filename = ""
        lineno = -1000000
        for diff_line in diff:
            if diff_line.startswith("---"):
                continue
            match = PLUSPLUSPLUS_RE.match(diff_line)
            if match:
                if match.group(1).startswith("b/"):  # heuristic
                    args.relative_diff = diff_line
                try:
                    filename = strip_dirs(match.group(1), args.strip_diff)
                except TypeError:
                    filename = "diff filename above common directory"
                    ## It's not an error; it just means this file doesn't appear in warnings output.
                    # eprint('Bad --strip-diff={0} ; line has fewer "/": {1}'.format(
                    #   strip_diff, match.group(1)))
                    # sys.exit(2)
                if filename not in changed:
                    changed[filename] = set()
                continue
            match = atat_re.match(diff_line)
            if match:
                lineno = int(match.group(3))
                continue
            if diff_line.startswith("+"):
                # Not just the changed line: changed[filename].add(lineno)
                for changed_lineno in range(
                    lineno - args.context_lines, lineno + args.context_lines + 1
                ):
                    changed[filename].add(changed_lineno)
                lineno += 1
            if diff_line.startswith(" "):
                lineno += 1
            if diff_line.startswith("-"):
                for changed_lineno in range(
                    lineno - args.context_lines, lineno + args.context_lines
                ):
                    changed[filename].add(changed_lineno)
                continue

    return changed


def warn_relative_diff(args: argparse.Namespace) -> bool:
    """Possibly warn about relative directories.

    Returns:
        a boolean.
    """
    result = False
    if args.relative_diff is not None and args.strip_diff == 0:
        # This is usually not an error, so don't warn.
        if DEBUG:
            eprint(
                "warning:",
                args.diff_filename,
                "may use relative paths (e.g.,",
                args.relative_diff.strip(),
                ") but --strip-diff=0",
                ("(guessed)" if args.guess_strip else ""),
            )
            eprint("warning: (Maybe there were no files in common.)")
        result = True
        if DEBUG:
            eprint(f"lint-diff.py: diff file {args.diff_filename}:")
            eprint("{}", Path(args.diff_filename).read_text(encoding="utf-8"))
            eprint(f"lint-diff.py: lint file {args.warning_filename}:")
            eprint("{}", Path(args.warning_filename).read_text(encoding="utf-8"))
            eprint("lint-diff.py: end of input files.")

    return result


def main() -> None:
    """Filter warnings output, to only show output for changed lines."""
    args = parse_args()

    # A dictionary from file names to a set of ints (line numbers for changed lines).
    changed = changed_lines(args)

    if DEBUG:
        for filename in sorted(changed):
            print(filename, sorted(changed[filename]))

    # True if a warning has been issued about relative directories.
    relative_diff_warned = warn_relative_diff(args)

    if args.warning_filename is None:
        args.warning_filename = "stdin"
        warnings = sys.stdin
    else:
        # pylint: disable=consider-using-with
        warnings = Path(args.warning_filename).open(encoding="utf-8")  # noqa: SIM115

    # 1 if this produced any output, 0 if not.
    status = 0
    # true if we just printed a warning and are looking for continuation lines to print.
    print_multiline_warning = False

    for warning_line in warnings:
        if print_multiline_warning and INITIAL_WHITESPACE_RE.match(warning_line):
            print(warning_line, end="")
            continue
        print_multiline_warning = False

        should_output = False

        # Special case for Java exception in the output.
        if warning_line.startswith("Exception in thread"):
            should_output = True

        match = FILENAME_LINENO_RE.match(warning_line)
        if match:
            try:
                filename = strip_dirs(match.group(1), args.strip_warnings)
            except TypeError:
                filename = "warnings filename above common directory"
                ## It's not an error; it just means this file doesn't appear in warnings output.
                # eprint('Bad --strip-warnings={0} ; line has fewer "/": {1}'.format(
                #   strip_warnings, match.group(1)))
                # sys.exit(2)
            if (
                filename.startswith("/")
                and args.relative_diff is not None
                and args.strip_warnings == 0
            ):
                if not relative_diff_warned:
                    eprint(
                        # No spaces around string literals becaues this is `eprint`.
                        "warning:",
                        args.diff_filename,
                        "uses relative paths but",
                        args.warning_filename,
                        "uses absolute paths",
                    )
                    relative_diff_warned = True
            lineno = int(match.group(2))
            if filename in changed and lineno in changed[filename]:
                should_output = True

        if should_output:
            print(warning_line, end="")
            status = 1
            print_multiline_warning = True

    if warnings is not sys.stdin:
        warnings.close()

    sys.exit(status)


if __name__ == "__main__":
    main()