build_targets.py (11832B)
1 # Copyright 2023-2024 Nick Brassel (@tzarc) 2 # SPDX-License-Identifier: GPL-2.0-or-later 3 import json 4 import shutil 5 from typing import Dict, List, Union 6 from pathlib import Path 7 from dotty_dict import dotty, Dotty 8 from milc import cli 9 from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX, HAS_QMK_USERSPACE, QMK_USERSPACE 10 from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json 11 from qmk.keyboard import keyboard_folder 12 from qmk.info import keymap_json 13 from qmk.keymap import locate_keymap 14 from qmk.path import is_under_qmk_firmware, is_under_qmk_userspace, unix_style_path 15 from qmk.compilation_database import write_compilation_database 16 17 # These must be kept in the order in which they're applied to $(TARGET) in the makefiles in order to ensure consistency. 18 TARGET_FILENAME_MODIFIERS = ['FORCE_LAYOUT', 'CONVERT_TO'] 19 20 21 class BuildTarget: 22 def __init__(self, keyboard: str, keymap: str, json: Union[dict, Dotty] = None): 23 self._keyboard = keyboard_folder(keyboard) 24 self._keyboard_safe = self._keyboard.replace('/', '_') 25 self._keymap = keymap 26 self._parallel = 1 27 self._clean = False 28 self._compiledb = False 29 self._extra_args = {} 30 self._json = json.to_dict() if isinstance(json, Dotty) else json 31 32 def __str__(self): 33 return f'{self.keyboard}:{self.keymap}' 34 35 def __repr__(self): 36 if len(self._extra_args.items()) > 0: 37 return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap}, extra_args={json.dumps(self._extra_args, sort_keys=True)})' 38 return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap})' 39 40 def __lt__(self, __value: object) -> bool: 41 return self.__repr__() < __value.__repr__() 42 43 def __eq__(self, __value: object) -> bool: 44 if not isinstance(__value, BuildTarget): 45 return False 46 return self.__repr__() == __value.__repr__() 47 48 def __hash__(self) -> int: 49 return self.__repr__().__hash__() 50 51 def configure(self, parallel: int = None, clean: bool = None, compiledb: bool = None) -> None: 52 if parallel is not None: 53 self._parallel = parallel 54 if clean is not None: 55 self._clean = clean 56 if compiledb is not None: 57 self._compiledb = compiledb 58 59 @property 60 def keyboard(self) -> str: 61 return self._keyboard 62 63 @property 64 def keymap(self) -> str: 65 return self._keymap 66 67 @property 68 def json(self) -> dict: 69 if not self._json: 70 self._load_json() 71 if not self._json: 72 return {} 73 return self._json 74 75 @property 76 def dotty(self) -> Dotty: 77 return dotty(self.json) 78 79 @property 80 def extra_args(self) -> Dict[str, str]: 81 return {k: v for k, v in self._extra_args.items()} 82 83 @extra_args.setter 84 def extra_args(self, ex_args: Dict[str, str]): 85 if ex_args is not None and isinstance(ex_args, dict): 86 self._extra_args = {k: v for k, v in ex_args.items()} 87 88 def target_name(self, **env_vars) -> str: 89 # Work out the intended target name 90 target = f'{self._keyboard_safe}_{self.keymap}' 91 vars = self._all_vars(**env_vars) 92 for modifier in TARGET_FILENAME_MODIFIERS: 93 if modifier in vars: 94 target += f"_{vars[modifier]}" 95 return target 96 97 def _all_vars(self, **env_vars) -> Dict[str, str]: 98 vars = {k: v for k, v in env_vars.items()} 99 for k, v in self._extra_args.items(): 100 vars[k] = v 101 return vars 102 103 def _intermediate_output(self, **env_vars) -> Path: 104 return Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self.target_name(**env_vars)}') 105 106 def _common_make_args(self, dry_run: bool = False, build_target: str = None, **env_vars): 107 compile_args = [ 108 find_make(), 109 *get_make_parallel_args(self._parallel), 110 '-r', 111 '-R', 112 '-f', 113 'builddefs/build_keyboard.mk', 114 ] 115 116 if not cli.config.general.verbose: 117 compile_args.append('-s') 118 119 verbose = 'true' if cli.config.general.verbose else 'false' 120 color = 'true' if cli.config.general.color else 'false' 121 122 if dry_run: 123 compile_args.append('-n') 124 125 if build_target: 126 compile_args.append(build_target) 127 128 compile_args.extend([ 129 f'KEYBOARD={self.keyboard}', 130 f'KEYMAP={self.keymap}', 131 f'KEYBOARD_FILESAFE={self._keyboard_safe}', 132 f'TARGET={self._keyboard_safe}_{self.keymap}', # don't use self.target_name() here, it's rebuilt on the makefile side 133 f'VERBOSE={verbose}', 134 f'COLOR={color}', 135 'SILENT=false', 136 'QMK_BIN="qmk"', 137 ]) 138 139 vars = self._all_vars(**env_vars) 140 for k, v in vars.items(): 141 compile_args.append(f'{k}={v}') 142 143 return compile_args 144 145 def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None: 146 raise NotImplementedError("prepare_build() not implemented in base class") 147 148 def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]: 149 raise NotImplementedError("compile_command() not implemented in base class") 150 151 def generate_compilation_database(self, build_target: str = None, skip_clean: bool = False, **env_vars) -> None: 152 self.prepare_build(build_target=build_target, **env_vars) 153 command = self.compile_command(build_target=build_target, dry_run=True, **env_vars) 154 output_path = QMK_FIRMWARE / 'compile_commands.json' 155 ret = write_compilation_database(command=command, output_path=output_path, skip_clean=skip_clean, **env_vars) 156 if ret and output_path.exists() and HAS_QMK_USERSPACE: 157 shutil.copy(str(output_path), str(QMK_USERSPACE / 'compile_commands.json')) 158 return ret 159 160 def compile(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None: 161 if self._clean or self._compiledb: 162 command = [find_make(), "clean"] 163 if dry_run: 164 command.append('-n') 165 cli.log.info('Cleaning with {fg_cyan}%s', ' '.join(command)) 166 cli.run(command, capture_output=False) 167 168 if self._compiledb and not dry_run: 169 self.generate_compilation_database(build_target=build_target, skip_clean=True, **env_vars) 170 171 self.prepare_build(build_target=build_target, dry_run=dry_run, **env_vars) 172 command = self.compile_command(build_target=build_target, **env_vars) 173 cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) 174 if not dry_run: 175 cli.echo('\n') 176 ret = cli.run(command, capture_output=False) 177 if ret.returncode: 178 return ret.returncode 179 180 181 class KeyboardKeymapBuildTarget(BuildTarget): 182 def __init__(self, keyboard: str, keymap: str, json: dict = None): 183 super().__init__(keyboard=keyboard, keymap=keymap, json=json) 184 185 def __repr__(self): 186 if len(self._extra_args.items()) > 0: 187 return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, extra_args={self._extra_args})' 188 return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap})' 189 190 def _load_json(self): 191 self._json = keymap_json(self.keyboard, self.keymap) 192 193 def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None: 194 pass 195 196 def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]: 197 compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target, **env_vars) 198 199 # Need to override the keymap path if the keymap is a userspace directory. 200 # This also ensures keyboard aliases as per `keyboard_aliases.hjson` still work if the userspace has the keymap 201 # in an equivalent historical location. 202 vars = self._all_vars(**env_vars) 203 keymap_location = locate_keymap(self.keyboard, self.keymap, force_layout=vars.get('FORCE_LAYOUT')) 204 if is_under_qmk_userspace(keymap_location) and not is_under_qmk_firmware(keymap_location): 205 keymap_directory = keymap_location.parent 206 compile_args.extend([ 207 f'MAIN_KEYMAP_PATH_1={unix_style_path(keymap_directory)}', 208 f'MAIN_KEYMAP_PATH_2={unix_style_path(keymap_directory)}', 209 f'MAIN_KEYMAP_PATH_3={unix_style_path(keymap_directory)}', 210 f'MAIN_KEYMAP_PATH_4={unix_style_path(keymap_directory)}', 211 f'MAIN_KEYMAP_PATH_5={unix_style_path(keymap_directory)}', 212 ]) 213 214 return compile_args 215 216 217 class JsonKeymapBuildTarget(BuildTarget): 218 def __init__(self, json_path): 219 if isinstance(json_path, Path): 220 self.json_path = json_path 221 else: 222 self.json_path = None 223 224 json = parse_configurator_json(json_path) # Will load from stdin if provided 225 226 # In case the user passes a keymap.json from a keymap directory directly to the CLI. 227 # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json 228 json["keymap"] = json.get("keymap", "default_json") 229 230 super().__init__(keyboard=json['keyboard'], keymap=json['keymap'], json=json) 231 232 def __repr__(self): 233 if len(self._extra_args.items()) > 0: 234 return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path}, extra_args={self._extra_args})' 235 return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})' 236 237 def _load_json(self): 238 pass # Already loaded in constructor 239 240 def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None: 241 intermediate_output = self._intermediate_output(**env_vars) 242 generated_files_path = intermediate_output / 'src' 243 keymap_json = generated_files_path / 'keymap.json' 244 245 if self._clean: 246 if intermediate_output.exists(): 247 shutil.rmtree(intermediate_output) 248 249 # begin with making the deepest folder in the tree 250 generated_files_path.mkdir(exist_ok=True, parents=True) 251 252 # Compare minified to ensure consistent comparison 253 new_content = json.dumps(self.json, separators=(',', ':')) 254 if keymap_json.exists(): 255 old_content = json.dumps(json.loads(keymap_json.read_text(encoding='utf-8')), separators=(',', ':')) 256 if old_content == new_content: 257 new_content = None 258 259 # Write the keymap.json file if different so timestamps are only updated 260 # if the content changes -- running `make` won't treat it as modified. 261 if new_content: 262 keymap_json.write_text(new_content, encoding='utf-8') 263 264 def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]: 265 compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target, **env_vars) 266 intermediate_output = self._intermediate_output(**env_vars) 267 generated_files_path = intermediate_output / 'src' 268 keymap_json = generated_files_path / 'keymap.json' 269 compile_args.extend([ 270 f'MAIN_KEYMAP_PATH_1={unix_style_path(intermediate_output)}', 271 f'MAIN_KEYMAP_PATH_2={unix_style_path(intermediate_output)}', 272 f'MAIN_KEYMAP_PATH_3={unix_style_path(intermediate_output)}', 273 f'MAIN_KEYMAP_PATH_4={unix_style_path(intermediate_output)}', 274 f'MAIN_KEYMAP_PATH_5={unix_style_path(intermediate_output)}', 275 f'KEYMAP_JSON={keymap_json}', 276 f'KEYMAP_PATH={generated_files_path}', 277 ]) 278 279 return compile_args