mass_compile.py (7648B)
1 """Compile all keyboards. 2 3 This will compile everything in parallel, for testing purposes. 4 """ 5 import os 6 from typing import List 7 from pathlib import Path 8 from subprocess import DEVNULL 9 from milc import cli 10 import shlex 11 12 from qmk.constants import QMK_FIRMWARE 13 from qmk.commands import find_make, get_make_parallel_args, build_environment 14 from qmk.search import search_keymap_targets, search_make_targets 15 from qmk.build_targets import BuildTarget, JsonKeymapBuildTarget 16 from qmk.util import maybe_exit_config 17 18 19 def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, no_temp: bool, parallel: int, print_failures: bool, **env): 20 if len(targets) == 0: 21 return 22 23 os.environ.setdefault('SKIP_SCHEMA_VALIDATION', '1') 24 25 make_cmd = find_make() 26 builddir = Path(QMK_FIRMWARE) / '.build' 27 makefile = builddir / 'parallel_kb_builds.mk' 28 29 if dry_run: 30 cli.log.info('Compilation targets:') 31 for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)): 32 extra_args = ' '.join([f"-e {shlex.quote(f'{k}={v}')}" for k, v in target.extra_args.items()]) 33 cli.log.info(f"{{fg_cyan}}qmk compile -kb {target.keyboard} -km {target.keymap} {extra_args}{{fg_reset}}") 34 else: 35 if clean: 36 cli.run([make_cmd, 'clean'], capture_output=False, stdin=DEVNULL) 37 38 builddir.mkdir(parents=True, exist_ok=True) 39 with open(makefile, "w") as f: 40 # yapf: disable 41 f.write( 42 f"""\ 43 # This file is auto-generated by qmk mass-compile 44 # Do not edit this file directly. 45 all: print_failures 46 .PHONY: all_targets print_failures 47 print_failures: all_targets 48 """# noqa 49 ) 50 if print_failures: 51 f.write( 52 f"""\ 53 @for f in $$(ls .build/failed.log.{os.getpid()}.* 2>/dev/null | sort); do \\ 54 echo; \\ 55 echo "======================================================================================"; \\ 56 echo "Failed build log: $$f"; \\ 57 echo "------------------------------------------------------"; \\ 58 cat $$f; \\ 59 echo "------------------------------------------------------"; \\ 60 done 61 """# noqa 62 ) 63 # yapf: enable 64 for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)): 65 keyboard_name = target.keyboard 66 keymap_name = target.keymap 67 keyboard_safe = keyboard_name.replace('/', '_') 68 target_filename = target.target_name(**env) 69 target.configure(parallel=1) # We ignore parallelism on a per-build basis as we defer to the parent make invocation 70 target.prepare_build(**env) # If we've got json targets, allow them to write out any extra info to .build before we kick off `make` 71 command = target.compile_command(**env) 72 command[0] = '+@$(MAKE)' # Override the make so that we can use jobserver to handle parallelism 73 extra_args = '_'.join([f"{k}_{v}" for k, v in target.extra_args.items()]) 74 build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" 75 failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" 76 target_suffix = '' 77 if len(extra_args) > 0: 78 build_log += f".{extra_args}" 79 failed_log += f".{extra_args}" 80 target_suffix = f"_{extra_args}" 81 # yapf: disable 82 f.write( 83 f"""\ 84 .PHONY: {target_filename}{target_suffix}_binary 85 all_targets: {target_filename}{target_suffix}_binary 86 {target_filename}{target_suffix}_binary: 87 @rm -f "{build_log}" || true 88 @echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}" 89 {' '.join(command)} \\ 90 >>"{build_log}" 2>&1 \\ 91 || cp "{build_log}" "{failed_log}" 92 @{{ grep '\\[ERRORS\\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \\e[1;31m[ERRORS]\\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\ 93 || {{ grep '\\[WARNINGS\\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \\e[1;33m[WARNINGS]\\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\ 94 || printf "Build %-64s \\e[1;32m[OK]\\e[0m\\n" "{keyboard_name}:{keymap_name}" 95 @rm -f "{build_log}" || true 96 """# noqa 97 ) 98 # yapf: enable 99 100 if no_temp: 101 # yapf: disable 102 f.write( 103 f"""\ 104 @rm -rf "{QMK_FIRMWARE}/.build/{target_filename}.elf" 2>/dev/null || true 105 @rm -rf "{QMK_FIRMWARE}/.build/{target_filename}.map" 2>/dev/null || true 106 @rm -rf "{QMK_FIRMWARE}/.build/obj_{target_filename}" || true 107 """# noqa 108 ) 109 # yapf: enable 110 f.write('\n') 111 112 cli.run([find_make(), *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) 113 114 # Check for failures 115 failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')] 116 if len(failures) > 0: 117 return False 118 119 120 @cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap> to compile in parallel. Specifying this overrides all other target search options.") 121 @cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.") 122 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") 123 @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") 124 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.") 125 @cli.argument('-p', '--print-failures', arg_only=True, action='store_true', help="Print failed builds.") 126 @cli.argument( 127 '-f', 128 '--filter', 129 arg_only=True, 130 action='append', 131 default=[], 132 help= # noqa: `format-python` and `pytest` don't agree here. 133 "Filter the list of keyboards based on the supplied value in rules.mk. Matches info.json structure, and accepts the formats 'features.rgblight=true' or 'exists(matrix_pins.direct)'. May be passed multiple times, all filters need to match. Value may include wildcards such as '*' and '?'." # noqa: `format-python` and `pytest` don't agree here. 134 ) 135 @cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.") 136 @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") 137 @cli.subcommand('Compile QMK Firmware for all keyboards.', hidden=False if cli.config.user.developer else True) 138 def mass_compile(cli): 139 """Compile QMK Firmware against all keyboards. 140 """ 141 maybe_exit_config(should_exit=False, should_reraise=True) 142 143 if len(cli.args.builds) > 0: 144 json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)]) 145 make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds)) 146 targets = search_make_targets(make_like_targets) 147 targets.extend([JsonKeymapBuildTarget(e) for e in json_like_targets]) 148 else: 149 targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter) 150 151 return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.args.no_temp, cli.config.mass_compile.parallel, cli.args.print_failures, **build_environment(cli.args.env))