QMK Userspace (#22222)

Co-authored-by: Duncan Sutherland <dunk2k_2000@hotmail.com>

12 files changed, 282 insertions, 25 deletions

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index 695a180066..cf60903687 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -81,6 +81,11 @@ subcommands = [
     'qmk.cli.new.keymap',
     'qmk.cli.painter',
     'qmk.cli.pytest',
+    'qmk.cli.userspace.add',
+    'qmk.cli.userspace.compile',
+    'qmk.cli.userspace.doctor',
+    'qmk.cli.userspace.list',
+    'qmk.cli.userspace.remove',
     'qmk.cli.via2json',
 ]
 
diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py
index 71c1dec162..3c8f3664ea 100755
--- a/lib/python/qmk/cli/compile.py
+++ b/lib/python/qmk/cli/compile.py
@@ -37,7 +37,9 @@ def compile(cli):
         from .mass_compile import mass_compile
         cli.args.builds = []
         cli.args.filter = []
-        cli.args.no_temp = False
+        cli.config.mass_compile.keymap = cli.config.compile.keymap
+        cli.config.mass_compile.parallel = cli.config.compile.parallel
+        cli.config.mass_compile.no_temp = False
         return mass_compile(cli)
 
     # Build the environment vars
diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py
index 6a6feb87d1..dd8b58b2c7 100755
--- a/lib/python/qmk/cli/doctor/main.py
+++ b/lib/python/qmk/cli/doctor/main.py
@@ -9,10 +9,11 @@ from milc import cli
 from milc.questions import yesno
 
 from qmk import submodules
-from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM
+from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM, QMK_USERSPACE, HAS_QMK_USERSPACE
 from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules
 from qmk.git import git_check_repo, git_get_branch, git_get_tag, git_get_last_log_entry, git_get_common_ancestor, git_is_dirty, git_get_remotes, git_check_deviation
 from qmk.commands import in_virtualenv
+from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate, UserspaceValidationError
 
 
 def os_tests():
@@ -92,6 +93,25 @@ def output_submodule_status():
                 cli.log.error(f'- {sub_name}: <<< missing or unknown >>>')
 
 
+def userspace_tests(qmk_firmware):
+    if qmk_firmware:
+        cli.log.info(f'QMK home: {{fg_cyan}}{qmk_firmware}')
+
+    for path in qmk_userspace_paths():
+        try:
+            qmk_userspace_validate(path)
+            cli.log.info(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`')
+        except FileNotFoundError:
+            cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Missing `qmk.json`')
+        except UserspaceValidationError as err:
+            cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`')
+            cli.log.warn(f' -- {{fg_cyan}}{path}/qmk.json{{fg_reset}} validation error: {err}')
+
+    if QMK_USERSPACE is not None:
+        cli.log.info(f'QMK userspace: {{fg_cyan}}{QMK_USERSPACE}')
+    cli.log.info(f'Userspace enabled: {{fg_cyan}}{HAS_QMK_USERSPACE}')
+
+
 @cli.argument('-y', '--yes', action='store_true', arg_only=True, help='Answer yes to all questions.')
 @cli.argument('-n', '--no', action='store_true', arg_only=True, help='Answer no to all questions.')
 @cli.subcommand('Basic QMK environment checks')
@@ -108,6 +128,9 @@ def doctor(cli):
     cli.log.info('QMK home: {fg_cyan}%s', QMK_FIRMWARE)
 
     status = os_status = os_tests()
+
+    userspace_tests(None)
+
     git_status = git_tests()
 
     if git_status == CheckStatus.ERROR or (os_status == CheckStatus.OK and git_status == CheckStatus.WARNING):
diff --git a/lib/python/qmk/cli/format/json.py b/lib/python/qmk/cli/format/json.py
index 3299a0d807..283513254c 100755
--- a/lib/python/qmk/cli/format/json.py
+++ b/lib/python/qmk/cli/format/json.py
@@ -9,48 +9,74 @@ from milc import cli
 
 from qmk.info import info_json
 from qmk.json_schema import json_load, validate
-from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder
+from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder, UserspaceJSONEncoder
 from qmk.path import normpath
 
 
-@cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format')
-@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)')
-@cli.argument('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file')
-@cli.argument('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ')
-@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True)
-def format_json(cli):
-    """Format a json file.
+def _detect_json_format(file, json_data):
+    """Detect the format of a json file.
     """
-    json_file = json_load(cli.args.json_file)
-
-    if cli.args.format == 'auto':
+    json_encoder = None
+    try:
+        validate(json_data, 'qmk.user_repo.v1')
+        json_encoder = UserspaceJSONEncoder
+    except ValidationError:
+        pass
+
+    if json_encoder is None:
         try:
-            validate(json_file, 'qmk.keyboard.v1')
+            validate(json_data, 'qmk.keyboard.v1')
             json_encoder = InfoJSONEncoder
-
         except ValidationError as e:
-            cli.log.warning('File %s did not validate as a keyboard:\n\t%s', cli.args.json_file, e)
-            cli.log.info('Treating %s as a keymap file.', cli.args.json_file)
+            cli.log.warning('File %s did not validate as a keyboard info.json or userspace qmk.json:\n\t%s', file, e)
+            cli.log.info('Treating %s as a keymap file.', file)
             json_encoder = KeymapJSONEncoder
+
+    return json_encoder
+
+
+def _get_json_encoder(file, json_data):
+    """Get the json encoder for a file.
+    """
+    json_encoder = None
+    if cli.args.format == 'auto':
+        json_encoder = _detect_json_format(file, json_data)
     elif cli.args.format == 'keyboard':
         json_encoder = InfoJSONEncoder
     elif cli.args.format == 'keymap':
         json_encoder = KeymapJSONEncoder
+    elif cli.args.format == 'userspace':
+        json_encoder = UserspaceJSONEncoder
     else:
         # This should be impossible
         cli.log.error('Unknown format: %s', cli.args.format)
+    return json_encoder
+
+
+@cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format')
+@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap', 'userspace'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)')
+@cli.argument('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file')
+@cli.argument('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ')
+@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True)
+def format_json(cli):
+    """Format a json file.
+    """
+    json_data = json_load(cli.args.json_file)
+
+    json_encoder = _get_json_encoder(cli.args.json_file, json_data)
+    if json_encoder is None:
         return False
 
-    if json_encoder == KeymapJSONEncoder and 'layout' in json_file:
+    if json_encoder == KeymapJSONEncoder and 'layout' in json_data:
         # Attempt to format the keycodes.
-        layout = json_file['layout']
-        info_data = info_json(json_file['keyboard'])
+        layout = json_data['layout']
+        info_data = info_json(json_data['keyboard'])
 
         if layout in info_data.get('layout_aliases', {}):
-            layout = json_file['layout'] = info_data['layout_aliases'][layout]
+            layout = json_data['layout'] = info_data['layout_aliases'][layout]
 
         if layout in info_data.get('layouts'):
-            for layer_num, layer in enumerate(json_file['layers']):
+            for layer_num, layer in enumerate(json_data['layers']):
                 current_layer = []
                 last_row = 0
 
@@ -61,9 +87,9 @@ def format_json(cli):
 
                     current_layer.append(keymap_key)
 
-                json_file['layers'][layer_num] = current_layer
+                json_data['layers'][layer_num] = current_layer
 
-    output = json.dumps(json_file, cls=json_encoder, sort_keys=True)
+    output = json.dumps(json_data, cls=json_encoder, sort_keys=True)
 
     if cli.args.inplace:
         with open(cli.args.json_file, 'w+', encoding='utf-8') as outfile:
diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index 7968de53e7..b025f85701 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -72,7 +72,7 @@ all: {keyboard_safe}_{keymap_name}_binary
                     # yapf: enable
                 f.write('\n')
 
-        cli.run([make_cmd, *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)
+        cli.run([find_make(), *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)
 
         # Check for failures
         failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')]
diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py
index 9b0ac221a4..d4339bc9ef 100755
--- a/lib/python/qmk/cli/new/keymap.py
+++ b/lib/python/qmk/cli/new/keymap.py
@@ -5,10 +5,12 @@ import shutil
 from milc import cli
 from milc.questions import question
 
+from qmk.constants import HAS_QMK_USERSPACE, QMK_USERSPACE
 from qmk.path import is_keyboard, keymaps, keymap
 from qmk.git import git_get_username
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.userspace import UserspaceDefs
 
 
 def prompt_keyboard():
@@ -68,3 +70,9 @@ def new_keymap(cli):
     # end message to user
     cli.log.info(f'{{fg_green}}Created a new keymap called {{fg_cyan}}{user_name}{{fg_green}} in: {{fg_cyan}}{keymap_path_new}.{{fg_reset}}')
     cli.log.info(f"Compile a firmware with your new keymap by typing: {{fg_yellow}}qmk compile -kb {kb_name} -km {user_name}{{fg_reset}}.")
+
+    # Add to userspace compile if we have userspace available
+    if HAS_QMK_USERSPACE:
+        userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+        userspace.add_target(keyboard=kb_name, keymap=user_name, do_print=False)
+        return userspace.save()
diff --git a/lib/python/qmk/cli/userspace/__init__.py b/lib/python/qmk/cli/userspace/__init__.py
new file mode 100644
index 0000000000..5757d3a4c9
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/__init__.py
@@ -0,0 +1,5 @@
+from . import doctor
+from . import add
+from . import remove
+from . import list
+from . import compile
diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py
new file mode 100644
index 0000000000..8993d54dba
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/add.py
@@ -0,0 +1,51 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+from pathlib import Path
+from milc import cli
+
+from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.keyboard import keyboard_completer, keyboard_folder_or_all
+from qmk.keymap import keymap_completer, is_keymap_target
+from qmk.userspace import UserspaceDefs
+
+
+@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.")
+@cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.subcommand('Adds a build target to userspace `qmk.json`.')
+def userspace_add(cli):
+    if not HAS_QMK_USERSPACE:
+        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
+        return False
+
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+
+    if len(cli.args.builds) > 0:
+        json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
+        make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
+
+        for e in json_like_targets:
+            userspace.add_target(json_path=e)
+
+        for e in make_like_targets:
+            s = e.split(':')
+            userspace.add_target(keyboard=s[0], keymap=s[1])
+
+    else:
+        failed = False
+        try:
+            if not is_keymap_target(cli.args.keyboard, cli.args.keymap):
+                failed = True
+        except KeyError:
+            failed = True
+
+        if failed:
+            from qmk.cli.new.keymap import new_keymap
+            cli.config.new_keymap.keyboard = cli.args.keyboard
+            cli.config.new_keymap.keymap = cli.args.keymap
+            if new_keymap(cli) is not False:
+                userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
+        else:
+            userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
+
+    return userspace.save()
diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py
new file mode 100644
index 0000000000..0a42dd5bf5
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/compile.py
@@ -0,0 +1,38 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+from pathlib import Path
+from milc import cli
+
+from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.commands import build_environment
+from qmk.userspace import UserspaceDefs
+from qmk.build_targets import JsonKeymapBuildTarget
+from qmk.search import search_keymap_targets
+from qmk.cli.mass_compile import mass_compile_targets
+
+
+@cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
+@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
+@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
+@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.")
+@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
+@cli.subcommand('Compiles the build targets specified in userspace `qmk.json`.')
+def userspace_compile(cli):
+    if not HAS_QMK_USERSPACE:
+        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
+        return False
+
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+
+    build_targets = []
+    keyboard_keymap_targets = []
+    for e in userspace.build_targets:
+        if isinstance(e, Path):
+            build_targets.append(JsonKeymapBuildTarget(e))
+        elif isinstance(e, dict):
+            keyboard_keymap_targets.append((e['keyboard'], e['keymap']))
+
+    if len(keyboard_keymap_targets) > 0:
+        build_targets.extend(search_keymap_targets(keyboard_keymap_targets))
+
+    mass_compile_targets(list(set(build_targets)), cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, **build_environment(cli.args.env))
diff --git a/lib/python/qmk/cli/userspace/doctor.py b/lib/python/qmk/cli/userspace/doctor.py
new file mode 100644
index 0000000000..2b7e29aa7e
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/doctor.py
@@ -0,0 +1,11 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+from milc import cli
+
+from qmk.constants import QMK_FIRMWARE
+from qmk.cli.doctor.main import userspace_tests
+
+
+@cli.subcommand('Checks userspace configuration.')
+def userspace_doctor(cli):
+    userspace_tests(QMK_FIRMWARE)
diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py
new file mode 100644
index 0000000000..a63f669dd7
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/list.py
@@ -0,0 +1,51 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+from pathlib import Path
+from dotty_dict import Dotty
+from milc import cli
+
+from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.userspace import UserspaceDefs
+from qmk.build_targets import BuildTarget
+from qmk.keyboard import is_all_keyboards, keyboard_folder
+from qmk.keymap import is_keymap_target
+from qmk.search import search_keymap_targets
+
+
+@cli.argument('-e', '--expand', arg_only=True, action='store_true', help="Expands any use of `all` for either keyboard or keymap.")
+@cli.subcommand('Lists the build targets specified in userspace `qmk.json`.')
+def userspace_list(cli):
+    if not HAS_QMK_USERSPACE:
+        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
+        return False
+
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+
+    if cli.args.expand:
+        build_targets = []
+        for e in userspace.build_targets:
+            if isinstance(e, Path):
+                build_targets.append(e)
+            elif isinstance(e, dict) or isinstance(e, Dotty):
+                build_targets.extend(search_keymap_targets([(e['keyboard'], e['keymap'])]))
+    else:
+        build_targets = userspace.build_targets
+
+    for e in build_targets:
+        if isinstance(e, Path):
+            # JSON keymap from userspace
+            cli.log.info(f'JSON keymap: {{fg_cyan}}{e}{{fg_reset}}')
+            continue
+        elif isinstance(e, dict) or isinstance(e, Dotty):
+            # keyboard/keymap dict from userspace
+            keyboard = e['keyboard']
+            keymap = e['keymap']
+        elif isinstance(e, BuildTarget):
+            # BuildTarget from search_keymap_targets()
+            keyboard = e.keyboard
+            keymap = e.keymap
+
+        if is_all_keyboards(keyboard) or is_keymap_target(keyboard_folder(keyboard), keymap):
+            cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}')
+        else:
+            cli.log.warn(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}} -- not found!')
diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py
new file mode 100644
index 0000000000..c7d180bfd1
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/remove.py
@@ -0,0 +1,37 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+from pathlib import Path
+from milc import cli
+
+from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.keyboard import keyboard_completer, keyboard_folder_or_all
+from qmk.keymap import keymap_completer
+from qmk.userspace import UserspaceDefs
+
+
+@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.")
+@cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.subcommand('Removes a build target from userspace `qmk.json`.')
+def userspace_remove(cli):
+    if not HAS_QMK_USERSPACE:
+        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
+        return False
+
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+
+    if len(cli.args.builds) > 0:
+        json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
+        make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
+
+        for e in json_like_targets:
+            userspace.remove_target(json_path=e)
+
+        for e in make_like_targets:
+            s = e.split(':')
+            userspace.remove_target(keyboard=s[0], keymap=s[1])
+
+    else:
+        userspace.remove_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
+
+    return userspace.save()