From 45d089700378875bdedb106e9484c6643e381911 Mon Sep 17 00:00:00 2001 From: OLEGSHA Date: Sat, 17 Dec 2022 14:20:07 +0300 Subject: [PATCH] TMP / pre-commit.py rc1 --- .gitignore | 4 +- tools/clang-format/clang-format.json | 5 + tools/clang-format/clang-format.yml | 7 - tools/clang-tidy/clang-tidy.cmake | 36 ++- tools/clang-tidy/clang-tidy.yml | 3 +- tools/pre-commit.py | 425 ++++++++++++++++++++++----- 6 files changed, 388 insertions(+), 92 deletions(-) create mode 100644 tools/clang-format/clang-format.json delete mode 100644 tools/clang-format/clang-format.yml diff --git a/.gitignore b/.gitignore index d1078a3..a39d412 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,8 @@ build # Run directory run -# Local environment setup file -tools/private.sh +# Local settings for pre-commit.py +tools/pre-commit-settings.json # Prevent anyone from accidentally uploading CMakeFiles CMakeFiles diff --git a/tools/clang-format/clang-format.json b/tools/clang-format/clang-format.json new file mode 100644 index 0000000..4556c43 --- /dev/null +++ b/tools/clang-format/clang-format.json @@ -0,0 +1,5 @@ +{ + "BasedOnStyle": "LLVM", + "IndentWidth": 4, + "CommentPragmas": "NOLINT", +} diff --git a/tools/clang-format/clang-format.yml b/tools/clang-format/clang-format.yml deleted file mode 100644 index c78e906..0000000 --- a/tools/clang-format/clang-format.yml +++ /dev/null @@ -1,7 +0,0 @@ -BasedOnStyle: LLVM - -# Use larger indentation -IndentWidth: 4 - -# clang-tidy suppression markers -CommentPragmas: 'NOLINT' diff --git a/tools/clang-tidy/clang-tidy.cmake b/tools/clang-tidy/clang-tidy.cmake index 1ae80fd..96513ca 100644 --- a/tools/clang-tidy/clang-tidy.cmake +++ b/tools/clang-tidy/clang-tidy.cmake @@ -1,17 +1,41 @@ if (DEV_MODE) find_program(clang_tidy_EXECUTABLE NAMES clang-tidy-13 clang-tidy) - + find_package(Python3 COMPONENTS Interpreter REQUIRED) + + # Setup clang-tidy + set(clang_tidy_config_file "${CMAKE_CURRENT_LIST_DIR}/clang-tidy.yml") list(APPEND clang_tidy_command "${clang_tidy_EXECUTABLE}" - "--config-file=${CMAKE_CURRENT_LIST_DIR}/clang-tidy.yml" - "--format-style=${clang_format_style}" + "--config-file=${clang_tidy_config_file}" "--warnings-as-errors=*" "--use-color") set_target_properties(progressia PROPERTIES CXX_CLANG_TIDY "${clang_tidy_command}") - add_custom_command(TARGET progressia PRE_BUILD + # Display the marker for pre-commit.py at build time + add_custom_target(clang_tidy_marker ALL COMMAND ${CMAKE_COMMAND} -E echo - "Clang-tidy is enabled") - + "Clang-tidy is enabled (this is a marker for pre-commit.py)") + + # Notify pre-commit.py about CMAKE_BINARY_DIR + execute_process(COMMAND ${Python3_EXECUTABLE} ${tools}/tools/pre-commit.py + set-build-root -- "${CMAKE_BINARY_DIR}" + RESULT_VARIABLE set_build_root_RESULT) + + if(${set_build_root_RESULT}) + message(FATAL_ERROR "pre-commit.py set-build-root failed") + endif() + + # Setup pre-commit git hook + if (IS_DIRECTORY "${CMAKE_SOURCE_DIR}/.git/hooks") + if (NOT EXISTS "${CMAKE_SOURCE_DIR}/.git/hooks/pre-commit") + file(WRITE "${CMAKE_SOURCE_DIR}/.git/hooks/pre-commit" + "#!/bin/bash\n" + "# Progressia autogenerated pre-commit hook\n" + "# You may modify this hook freely " + "(just make sure the checks run)\n" + "/bin/env python3 ${CMAKE_SOURCE_DIR}/tools/pre-commit.py run") + endif() + endif() + endif() diff --git a/tools/clang-tidy/clang-tidy.yml b/tools/clang-tidy/clang-tidy.yml index 4a7f056..edc3d6b 100644 --- a/tools/clang-tidy/clang-tidy.yml +++ b/tools/clang-tidy/clang-tidy.yml @@ -3,5 +3,6 @@ Checks: "-*,\ cppcoreguidelines-*,\ modernize-*,\ performance-*,\ - readability-*" + readability-*,\ + clang-diagnostic-*" diff --git a/tools/pre-commit.py b/tools/pre-commit.py index 013970a..7a3fa94 100755 --- a/tools/pre-commit.py +++ b/tools/pre-commit.py @@ -1,116 +1,389 @@ #!/usr/bin/env python3 usage = \ -'''Usage: %(me)s [--dry-run] [--verbose] [--dont-update] - or: %(me)s restore - or: %(me)s update +'''Usage: %(me)s run [OPTIONS...] + or: %(me)s restore [OPTIONS...] + or: %(me)s set-build-root CMAKE_BINARY_DIR In the 1st form, run standard pre-commit procedure for Progressia. In the 2nd form, attempt to restore workspace if the pre-commit hook failed. -In the 3rd form, only update git pre-commit hook +In the 3rd form, only update git pre-commit hook. - --dry-run do not change anything in git or in the working tree + --dry-run do not change anything in git or in the filesystem; + implies --verbose --verbose print commands and diagnostics - --dont-update do not update git pre-commit hook --help display this help and exit + --version display version information and exit Currently, the pre-commit procedure performs the following: 1. format staged changes - 2. attempt to compile with staged changes only''' + 2. attempt to compile with staged changes only + +pre-commit-settings.json values: + build-root CMake binary dir to use (filled in by CMake) + parallelism threads to use, default is 1 + git git command, default is null + cmake cmake command, default is null + clang-format-diff clang-format-diff command, default is null + +Use semicolons to separate arguments in git, cmake and clang-format-diff''' + +# Script version. Increment when script logic changes significantly. +# Commit change separately. +version = 1 + +# Source directories to format +src_dirs = ['desktop', 'main'] + +# File extensions to format +exts = ['cpp', 'h', 'inl'] + import sys import os import subprocess +import shutil +import json -def fail(*args): + +STASH_NAME = 'progressia_pre_commit_stash' +# Paths are relative to this script's directory, tools/ +SETTINGS_PATH = 'pre-commit-settings.json' +CLANG_FORMAT_PATH = 'clang-format/clang-format.json' +CLANG_TIDY_CHECK_MARKER = 'Clang-tidy is enabled ' \ + '(this is a marker for pre-commit.py)' + + +def fail(*args, code=1): + """Print an error message and exit with given code (default 1)""" print(my_name + ':', *args, file=sys.stderr) sys.exit(1) -def invoke(*cmd, result_when_dry=None, quiet=True): - if verbose: - print(my_name + ': command "' + '" "'.join(cmd) + '"') +def verbose(*args): + """Print a message in verbose mode only.""" + if verbose_mode: + print(my_name + ':', *args) - if dry_run and result_when_dry != None: + +def long_print_iter(title, it): + """Print contents of iterable titled as specified. If iterable is empty, + print the string (nothing) instead. + """ + print(title + ':') + + if len(it) > 0: + print('\t' + '\n\t'.join(it) + '\n') + else: + print('\t(nothing)\n') + + +def invoke(*cmd, result_when_dry=None, quiet=True, text=True, stdin=None): + """Execute given system command and return its stdout. If command fails, + throw CalledProcessError. + + When in verbose mode, log command before execution. If in dry-run mode and + result_when_dry is not None, skip execution and return result_when_dry + instead. + + Keyword arguments: + result_when_dry -- unless None (default), skip execution and return this + quiet -- if False, print stdout (default True) + text -- treat stdin and stdout as text rather than bytes (default False) + stdin -- unless None (default), send this to stdin of spawned process + """ + verbose('command', *(repr(c) for c in cmd)) + + if dry_run and result_when_dry is not None: print(my_name + ': skipped: --dry-run') return result_when_dry - - output = '' + popen = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - text=True, - universal_newlines=True) - - for line in popen.stdout: - if (not quiet): - print(line, end='') - output += line - - popen.stdout.close() + stdout=subprocess.PIPE, + text=text, + universal_newlines=text, + stdin=subprocess.PIPE if stdin else subprocess.DEVNULL) + + stdout, _ = popen.communicate(input=stdin) + + if text and not quiet: + print(stdout, end='') return_code = popen.wait() if return_code != 0: raise subprocess.CalledProcessError(return_code, cmd) - return output + return stdout -STASH_NAME = 'progressia_pre_commit_stash' -def run_safety_checks(): - if invoke('git', 'stash', 'list', '--grep', f"^{STASH_NAME}$") != '': - fail(f"Cannot run pre-commit checks: stash {STASH_NAME} exists. " + - f"Use `{my_name} restore` to restore workspace and repository " + +def get_file_sets(): + """Return sets of indexed and unindexed files according to Git""" + def git_z(*cmd): + raw = invoke(*git, *cmd, '-z', text=False) + return set(f.decode() for f in raw.split(b'\x00') if len(f) != 0) + + indexed = git_z('diff', '--name-only', '--cached') + unindexed = git_z('diff', '--name-only') | \ + git_z('ls-files', '--other', '--exclude-standard') + + return indexed, unindexed + + +def run_safety_checks(indexed, unindexed): + if invoke(*git, 'stash', 'list', '--grep', f"\\b{STASH_NAME}$") != '': + fail(f"Cannot run pre-commit checks: stash {STASH_NAME} exists. " + f"Use `{my_name} restore` to restore workspace and repository " f"state") - # Let's hope there are no files with weird names - indexed_changes = \ - set(invoke('git', 'diff', '--name-only', '--cached') \ - .strip().split('\n')) - unindexed_changes = \ - set(invoke('git', 'diff', '--name-only') \ - .strip().split('\n')) - - both_changes = indexed_changes & unindexed_changes + both_changes = indexed & unindexed if len(both_changes) != 0: - fail(f"Cannot run pre-commit checks: files with indexed and " + - "unindexed changes exist:\n\n\t" + - "\n\t".join(both_changes) + - "\n") + fail(f"Cannot run pre-commit checks: files with indexed and unindexed " + 'changes exist:\n\n\t' + + '\n\t'.join(both_changes) + + '\n') + + +def do_restore(): + """Restore repository and filesystem. Fail if stash not found.""" + print('Redoing rolled back changes') + + git_list = invoke(*git, 'stash', 'list', '--grep', f"\\b{STASH_NAME}$") + + if len(git_list) == 0: + if dry_run: + stash_name = 'stash@{0}' + else: + fail(f"Cannot restore repository: stash {STASH_NAME} not found") + else: + stash_name, _, _ = git_list.partition(':') + + invoke(*git, 'stash', 'pop', '--index', '--quiet', stash_name, + result_when_dry='', quiet=False) + + +def format_project(): + """Format staged files with clang-format-diff.""" + print('Formatting code') + format_file = os.path.join(os.path.dirname(__file__), + CLANG_FORMAT_PATH) + + with open(format_file, encoding='utf-8') as f: + style = f.read() + + diff = invoke(*git, 'diff', '-U0', '--no-color', '--relative', 'HEAD', + *(f"{d}/*.{e}" for d in src_dirs for e in exts)) + + invoke(*clang_format_diff, '-p1', '-i', '--verbose', '-style=' + style, + stdin=diff, result_when_dry='', quiet=False) + + +def unformat_project(indexed): + """Undo formatting changes introduced by format_project().""" + print('Undoing formatting changes') + invoke(*git, 'restore', '--', *indexed) + + +def build_project(): + """Build project with cmake.""" + print('Building project') + build_log = invoke(*cmake, '--build', build_root, '--parallel', parallelism, + result_when_dry=CLANG_TIDY_CHECK_MARKER, + quiet=False) + + if CLANG_TIDY_CHECK_MARKER not in build_log.splitlines(): + fail('Project build was successful, but clang-tidy did not run. ' + 'Please make sure DEV_MODE is ON and regenerate CMake cache.') + + print('Success') + + +def pre_commit(): + """Run pre-commit checks.""" + + if build_root is None: + fail(f"build-root is not set in {SETTINGS_PATH}. Compile project " + 'manually to set this variable properly.') + if not os.path.exists(build_root): + fail(f"build-root {build_root} does not exist. Compile project " + 'manually to set this variable properly.') + + cmakeCache = os.path.join(build_root, 'CMakeCache.txt') + if not os.path.exists(cmakeCache): + fail(f"{cmakeCache} does not exist. build-root is likely invalid. " + 'Compile project manually to set this variable properly.') + + indexed, unindexed = get_file_sets() + if verbose_mode: + long_print_iter('Indexed changes', indexed) + long_print_iter('Unindexed changes', unindexed) + run_safety_checks(indexed, unindexed) + + undo_formatting = False + try: + if len(unindexed) != 0: + long_print_iter('Unindexed changes found in files', unindexed) + print('These changes will be rolled back temporarily') + + invoke(*git, 'stash', 'push', + '--keep-index', + '--include-untracked', + '--message', STASH_NAME, + result_when_dry='', quiet=False) + restore = True + + format_project() + undo_formatting = True + build_project() + undo_formatting = False + + finally: + if undo_formatting: + unformat_project(indexed) + if restore: + do_restore() + + print('Staging formatting changes') + invoke(*git, 'add', '--', *indexed, result_when_dry='', quiet=False) + + +def get_settings_path(): + return os.path.abspath(os.path.join(os.path.dirname(__file__), + SETTINGS_PATH)) + + +def set_build_root(): + """Set last build root in tools/pre-commit-settings.json.""" + path = get_settings_path() + verbose(f"Updating {path}") + if not dry_run: + settings['build_root'] = arg_build_root + with open(path, mode='w') as f: + json.dump(settings, f, indent=4) + else: + verbose(' skipped: --dry-run') + + +def parse_args(): + """Parse sys.argv and environment variables; set corresponding globals. + Return (action, argument for set-build-root). + """ + global action + global verbose_mode + global dry_run + global allow_update + + consider_options = True + action = None + arg_build_root = None + + for arg in sys.argv[1:]: + if arg == 'restore' or arg == 'set-build-root' or arg == 'run': + if action is not None: + fail(f"Cannot use '{arg}' and '{action}' together") + action = arg + elif consider_options and arg.startswith('-'): + if arg == '-h' or arg == '--help' or arg == 'help' or arg == '/?': + print(usage % {'me': my_name}) + sys.exit(0) + elif arg == '--version': + print(f"Progressia pre-commit script, version {version}") + sys.exit(0) + elif arg == '--verbose': + verbose_mode = True + elif arg == '--dry-run': + dry_run = True + verbose_mode = True + elif arg == '--': + consider_options = False + else: + fail(f"Unknown option '{arg}'") + elif action == 'set-build-root' and arg_build_root is None: + arg_build_root = arg + else: + fail(f"Unknown or unexpected argument '{arg}'") + + if not allow_update and action == 'update': + fail("Cannot use '--dont-update' and 'update' together") + + if action is None: + fail('No action specified') + + if action == 'set-build-root' and arg_build_root is None: + fail('No path given') + + return action, arg_build_root + + +def load_settings(): + """Load values from pre-commit-settings.json.""" + global settings + global build_root + global git + global cmake + global clang_format_diff + global parallelism + + path = get_settings_path() + if os.path.exists(path): + with open(path, mode='r') as f: + settings = json.load(f) + else: + verbose(f"{path} not found, using defaults") + settings = { + "__comment": "See `pre-commit.py --help` for documentation", + "build_root": None, + "git": None, + "cmake": None, + "clang_format_diff": None, + "parallelism": 1 + } + + build_root = settings['build_root'] + parallelism = settings['parallelism'] + + def find_command(hints, settings_name): + if settings[settings_name] is not None: + hints = [settings[settings_name]] + + cmds = (hint.split(';') for hint in hints) + res = next((cmd for cmd in cmds if shutil.which(cmd[0])), None) \ + or fail(f"Command {hints[0]} not found. Set {env_name} " + + f"in {path} or check PATH") + + verbose(f"Found command {hints[0]}:", *(repr(c) for c in res)) + return res + + git = find_command(['git'], 'git') + cmake = find_command(['cmake'], 'cmake') + clang_format_diff = find_command(['clang-format-diff-13', + 'clang-format-diff', + 'clang-format-diff.py'], + 'clang_format_diff') if __name__ == '__main__': my_name = os.path.basename(sys.argv[0]) - verbose = False + verbose_mode = False dry_run = False - - - - verbose = True - dry_run = True - run_safety_checks() - #update() - - unindexed_changes = invoke('git', 'diff', '--name-status') - if unindexed_changes != '': - print('Unindexed changes found in files:') - print(unindexed_changes) - print('These changes will be ignored') - - invoke('git', 'stash', 'push', - '--keep-index', - '--include-untracked', - '--message', STASH_NAME, - result_when_dry='') - - # check that ORIGINAL does not exist - # check that staged files & files with unstaged changes = 0 - # update pre-commit hook - # if any unstaged changes: - # stash ORIGINAL - # remove unstaged changes - # format staged files - # compile - # git add - # if any unstaged changes: - # unstash ORIGINAL - + allow_update = True + + action, arg_build_root = parse_args() + load_settings() + + if dry_run: + print('Running in dry mode: no changes to filesystem or git will ' + 'actually be performed') + + try: + if action == 'set-build-root': + set_build_root() + elif action == 'restore': + do_restore() + indexed, unindexed = get_file_sets() + if indexed & unindexed: + unformat_project(indexed) + else: + pre_commit() + except subprocess.CalledProcessError as e: + fail('Command', *(repr(c) for c in e.cmd), + f"exited with code {e.returncode}")