api.py (8326B)
1 """This script automates the generation of the QMK API data. 2 """ 3 from pathlib import Path 4 import shutil 5 import json 6 7 from milc import cli 8 9 import qmk.path 10 from qmk.datetime import current_datetime 11 from qmk.info import info_json 12 from qmk.json_schema import json_load 13 from qmk.keymap import list_keymaps 14 from qmk.keyboard import find_readme, list_keyboards, keyboard_alias_definitions 15 from qmk.keycodes import load_spec, list_versions, list_languages 16 17 DATA_PATH = Path('data') 18 TEMPLATE_PATH = DATA_PATH / 'templates/api/' 19 BUILD_API_PATH = Path('.build/api_data/') 20 21 22 def _list_constants(output_folder): 23 """Produce a map of available constants 24 """ 25 ret = {} 26 for file in (output_folder / 'constants').glob('**/*_[0-9].[0-9].[0-9].json'): 27 name, version = file.stem.rsplit('_', 1) 28 if name not in ret: 29 ret[name] = [] 30 ret[name].append(version) 31 32 # Ensure content is sorted 33 for name in ret: 34 ret[name] = sorted(ret[name]) 35 36 return ret 37 38 39 def _resolve_keycode_specs(output_folder): 40 """To make it easier for consumers, publish pre-merged spec files 41 """ 42 for version in list_versions(): 43 overall = load_spec(version) 44 45 output_file = output_folder / f'constants/keycodes_{version}.json' 46 output_file.write_text(json.dumps(overall, separators=(',', ':')), encoding='utf-8') 47 48 for lang in list_languages(): 49 for version in list_versions(lang): 50 overall = load_spec(version, lang) 51 52 output_file = output_folder / f'constants/keycodes_{lang}_{version}.json' 53 output_file.write_text(json.dumps(overall, separators=(',', ':')), encoding='utf-8') 54 55 # Purge files consumed by 'load_spec' 56 shutil.rmtree(output_folder / 'constants/keycodes/') 57 58 59 def _filtered_copy(src, dst): 60 src = Path(src) 61 dst = Path(dst) 62 63 if dst.suffix == '.hjson': 64 data = json_load(src) 65 66 dst = dst.with_suffix('.json') 67 dst.write_text(json.dumps(data, separators=(',', ':')), encoding='utf-8') 68 return dst 69 70 if dst.suffix == '.jsonschema': 71 data = json_load(src) 72 73 dst.write_text(json.dumps(data), encoding='utf-8') 74 return dst 75 76 return shutil.copy2(src, dst) 77 78 79 def _filtered_keyboard_list(): 80 """Perform basic filtering of list_keyboards 81 """ 82 keyboard_list = list_keyboards() 83 if cli.args.filter: 84 kb_list = [] 85 for keyboard_name in keyboard_list: 86 if any(i in keyboard_name for i in cli.args.filter): 87 kb_list.append(keyboard_name) 88 keyboard_list = kb_list 89 return keyboard_list 90 91 92 @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't write the data to disk.") 93 @cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on partial name matches the supplied value. May be passed multiple times.") 94 @cli.subcommand('Generate QMK API data', hidden=False if cli.config.user.developer else True) 95 def generate_api(cli): 96 """Generates the QMK API data. 97 """ 98 v1_dir = BUILD_API_PATH / 'v1' 99 keyboard_all_file = v1_dir / 'keyboards.json' # A massive JSON containing everything 100 keyboard_list_file = v1_dir / 'keyboard_list.json' # A simple list of keyboard targets 101 keyboard_aliases_file = v1_dir / 'keyboard_aliases.json' # A list of historical keyboard names and their new name 102 keyboard_metadata_file = v1_dir / 'keyboard_metadata.json' # All the data configurator/via needs for initialization 103 constants_metadata_file = v1_dir / 'constants_metadata.json' # Metadata for available constants 104 usb_file = v1_dir / 'usb.json' # A mapping of USB VID/PID -> keyboard target 105 106 if BUILD_API_PATH.exists(): 107 shutil.rmtree(BUILD_API_PATH) 108 109 shutil.copytree(TEMPLATE_PATH, BUILD_API_PATH) 110 shutil.copytree(DATA_PATH, v1_dir, copy_function=_filtered_copy) 111 112 # Filter down when required 113 keyboard_list = _filtered_keyboard_list() 114 115 kb_all = {} 116 usb_list = {} 117 118 # Generate and write keyboard specific JSON files 119 for keyboard_name in keyboard_list: 120 kb_json = info_json(keyboard_name) 121 kb_all[keyboard_name] = kb_json 122 123 keyboard_dir = v1_dir / 'keyboards' / keyboard_name 124 keyboard_info = keyboard_dir / 'info.json' 125 keyboard_readme = keyboard_dir / 'readme.md' 126 keyboard_readme_src = find_readme(keyboard_name) 127 128 # Populate the list of JSON keymaps 129 for keymap in list_keymaps(keyboard_name, c=False, fullpath=True): 130 keymap_rel = qmk.path.under_qmk_firmware(keymap) 131 if keymap_rel is None: 132 cli.log.debug('Skipping keymap %s (not in qmk_firmware)', keymap) 133 continue 134 135 if (keymap_rel / 'keymap.c').exists(): 136 cli.log.debug('Skipping keymap %s (not pure dd keymap)', keymap) 137 continue 138 139 kb_json['keymaps'][keymap.name] = { 140 # TODO: deprecate 'url' as consumer needs to know its potentially hjson 141 'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap_rel}/keymap.json', 142 143 # Instead consumer should grab from API and not repo directly 144 'path': (keymap_rel / 'keymap.json').as_posix(), 145 } 146 147 keyboard_dir.mkdir(parents=True, exist_ok=True) 148 keyboard_json = json.dumps({'last_updated': current_datetime(), 'keyboards': {keyboard_name: kb_json}}, separators=(',', ':')) 149 if not cli.args.dry_run: 150 keyboard_info.write_text(keyboard_json, encoding='utf-8') 151 cli.log.debug('Wrote file %s', keyboard_info) 152 153 if keyboard_readme_src: 154 shutil.copyfile(keyboard_readme_src, keyboard_readme) 155 cli.log.debug('Copied %s -> %s', keyboard_readme_src, keyboard_readme) 156 157 # resolve keymaps as json 158 for keymap in kb_json['keymaps']: 159 keymap_hjson = kb_json['keymaps'][keymap]['path'] 160 keymap_json = v1_dir / keymap_hjson 161 keymap_json.parent.mkdir(parents=True, exist_ok=True) 162 keymap_json.write_text(json.dumps(json_load(Path(keymap_hjson)), separators=(',', ':')), encoding='utf-8') 163 cli.log.debug('Wrote keymap %s', keymap_json) 164 165 if 'usb' in kb_json: 166 usb = kb_json['usb'] 167 168 if 'vid' in usb and usb['vid'] not in usb_list: 169 usb_list[usb['vid']] = {} 170 171 if 'pid' in usb and usb['pid'] not in usb_list[usb['vid']]: 172 usb_list[usb['vid']][usb['pid']] = {} 173 174 if 'vid' in usb and 'pid' in usb: 175 usb_list[usb['vid']][usb['pid']][keyboard_name] = usb 176 177 # Generate data for the global files 178 keyboard_list = sorted(kb_all) 179 keyboard_aliases = keyboard_alias_definitions() 180 keyboard_metadata = { 181 'last_updated': current_datetime(), 182 'keyboards': keyboard_list, 183 'keyboard_aliases': keyboard_aliases, 184 'usb': usb_list, 185 } 186 187 # Feature specific handling 188 _resolve_keycode_specs(v1_dir) 189 190 # Write the global JSON files 191 keyboard_all_json = json.dumps({'last_updated': current_datetime(), 'keyboards': kb_all}, separators=(',', ':')) 192 usb_json = json.dumps({'last_updated': current_datetime(), 'usb': usb_list}, separators=(',', ':')) 193 keyboard_list_json = json.dumps({'last_updated': current_datetime(), 'keyboards': keyboard_list}, separators=(',', ':')) 194 keyboard_aliases_json = json.dumps({'last_updated': current_datetime(), 'keyboard_aliases': keyboard_aliases}, separators=(',', ':')) 195 keyboard_metadata_json = json.dumps(keyboard_metadata, separators=(',', ':')) 196 constants_metadata_json = json.dumps({'last_updated': current_datetime(), 'constants': _list_constants(v1_dir)}, separators=(',', ':')) 197 198 if not cli.args.dry_run: 199 keyboard_all_file.write_text(keyboard_all_json, encoding='utf-8') 200 usb_file.write_text(usb_json, encoding='utf-8') 201 keyboard_list_file.write_text(keyboard_list_json, encoding='utf-8') 202 keyboard_aliases_file.write_text(keyboard_aliases_json, encoding='utf-8') 203 keyboard_metadata_file.write_text(keyboard_metadata_json, encoding='utf-8') 204 constants_metadata_file.write_text(constants_metadata_json, encoding='utf-8')