summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorNick Brassel <nick@tzarc.org>2023-11-28 07:53:43 +1100
committerGitHub <noreply@github.com>2023-11-28 07:53:43 +1100
commit5501e804ff8d41ce656061b91896c4ac8c681d78 (patch)
tree6a655fbceaeab67cf727dbe4318721407dd31824
parent094357c40347e8a5db36578851f1af34a92e9f68 (diff)
QMK Userspace (#22222)
Co-authored-by: Duncan Sutherland <dunk2k_2000@hotmail.com>
-rw-r--r--Makefile30
-rw-r--r--builddefs/build_json.mk19
-rw-r--r--builddefs/build_keyboard.mk96
-rw-r--r--builddefs/build_layout.mk4
-rw-r--r--builddefs/common_rules.mk11
-rw-r--r--data/schemas/definitions.jsonschema18
-rw-r--r--data/schemas/user_repo_v0.jsonschema14
-rw-r--r--data/schemas/user_repo_v1.jsonschema22
-rw-r--r--docs/_summary.md2
-rw-r--r--docs/cli_commands.md125
-rw-r--r--docs/newbs_external_userspace.md96
-rw-r--r--lib/python/qmk/build_targets.py16
-rw-r--r--lib/python/qmk/cli/__init__.py5
-rwxr-xr-xlib/python/qmk/cli/compile.py4
-rwxr-xr-xlib/python/qmk/cli/doctor/main.py25
-rwxr-xr-xlib/python/qmk/cli/format/json.py70
-rwxr-xr-xlib/python/qmk/cli/mass_compile.py2
-rwxr-xr-xlib/python/qmk/cli/new/keymap.py8
-rw-r--r--lib/python/qmk/cli/userspace/__init__.py5
-rw-r--r--lib/python/qmk/cli/userspace/add.py51
-rw-r--r--lib/python/qmk/cli/userspace/compile.py38
-rw-r--r--lib/python/qmk/cli/userspace/doctor.py11
-rw-r--r--lib/python/qmk/cli/userspace/list.py51
-rw-r--r--lib/python/qmk/cli/userspace/remove.py37
-rw-r--r--lib/python/qmk/commands.py6
-rw-r--r--lib/python/qmk/constants.py8
-rwxr-xr-xlib/python/qmk/json_encoders.py18
-rw-r--r--lib/python/qmk/keyboard.py22
-rw-r--r--lib/python/qmk/keymap.py130
-rw-r--r--lib/python/qmk/path.py59
-rw-r--r--lib/python/qmk/userspace.py185
31 files changed, 1081 insertions, 107 deletions
diff --git a/Makefile b/Makefile
index 9ef406e420..ab30a17f58 100644
--- a/Makefile
+++ b/Makefile
@@ -38,6 +38,11 @@ $(info QMK Firmware $(QMK_VERSION))
38endif 38endif
39endif 39endif
40 40
41# Try to determine userspace from qmk config, if set.
42ifeq ($(QMK_USERSPACE),)
43 QMK_USERSPACE = $(shell qmk config -ro user.overlay_dir | cut -d= -f2 | sed -e 's@^None$$@@g')
44endif
45
41# Determine which qmk cli to use 46# Determine which qmk cli to use
42QMK_BIN := qmk 47QMK_BIN := qmk
43 48
@@ -191,9 +196,20 @@ define PARSE_KEYBOARD
191 KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(ROOT_DIR)/keyboards/$$(KEYBOARD_FOLDER_PATH_4)/keymaps/*/.))) 196 KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(ROOT_DIR)/keyboards/$$(KEYBOARD_FOLDER_PATH_4)/keymaps/*/.)))
192 KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(ROOT_DIR)/keyboards/$$(KEYBOARD_FOLDER_PATH_5)/keymaps/*/.))) 197 KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(ROOT_DIR)/keyboards/$$(KEYBOARD_FOLDER_PATH_5)/keymaps/*/.)))
193 198
199 ifneq ($(QMK_USERSPACE),)
200 KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_1)/keymaps/*/.)))
201 KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_2)/keymaps/*/.)))
202 KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_3)/keymaps/*/.)))
203 KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_4)/keymaps/*/.)))
204 KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_5)/keymaps/*/.)))
205 endif
206
194 KEYBOARD_LAYOUTS := $(shell $(QMK_BIN) list-layouts --keyboard $1) 207 KEYBOARD_LAYOUTS := $(shell $(QMK_BIN) list-layouts --keyboard $1)
195 LAYOUT_KEYMAPS := 208 LAYOUT_KEYMAPS :=
196 $$(foreach LAYOUT,$$(KEYBOARD_LAYOUTS),$$(eval LAYOUT_KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(ROOT_DIR)/layouts/*/$$(LAYOUT)/*/.))))) 209 $$(foreach LAYOUT,$$(KEYBOARD_LAYOUTS),$$(eval LAYOUT_KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(ROOT_DIR)/layouts/*/$$(LAYOUT)/*/.)))))
210 ifneq ($(QMK_USERSPACE),)
211 $$(foreach LAYOUT,$$(KEYBOARD_LAYOUTS),$$(eval LAYOUT_KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/layouts/$$(LAYOUT)/*/.)))))
212 endif
197 213
198 KEYMAPS := $$(sort $$(KEYMAPS) $$(LAYOUT_KEYMAPS)) 214 KEYMAPS := $$(sort $$(KEYMAPS) $$(LAYOUT_KEYMAPS))
199 215
@@ -431,8 +447,18 @@ clean:
431 rm -rf $(BUILD_DIR) 447 rm -rf $(BUILD_DIR)
432 echo 'done.' 448 echo 'done.'
433 449
434.PHONY: distclean 450.PHONY: distclean distclean_qmk
435distclean: clean 451distclean: distclean_qmk
452distclean_qmk: clean
436 echo -n 'Deleting *.bin, *.hex, and *.uf2 ... ' 453 echo -n 'Deleting *.bin, *.hex, and *.uf2 ... '
437 rm -f *.bin *.hex *.uf2 454 rm -f *.bin *.hex *.uf2
438 echo 'done.' 455 echo 'done.'
456
457ifneq ($(QMK_USERSPACE),)
458.PHONY: distclean_userspace
459distclean: distclean_userspace
460distclean_userspace: clean
461 echo -n 'Deleting userspace *.bin, *.hex, and *.uf2 ... '
462 rm -f $(QMK_USERSPACE)/*.bin $(QMK_USERSPACE)/*.hex $(QMK_USERSPACE)/*.uf2
463 echo 'done.'
464endif
diff --git a/builddefs/build_json.mk b/builddefs/build_json.mk
index e29d678e48..e9d1420f36 100644
--- a/builddefs/build_json.mk
+++ b/builddefs/build_json.mk
@@ -15,3 +15,22 @@ else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.json)","")
15 KEYMAP_JSON := $(MAIN_KEYMAP_PATH_1)/keymap.json 15 KEYMAP_JSON := $(MAIN_KEYMAP_PATH_1)/keymap.json
16 KEYMAP_JSON_PATH := $(MAIN_KEYMAP_PATH_1) 16 KEYMAP_JSON_PATH := $(MAIN_KEYMAP_PATH_1)
17endif 17endif
18
19ifneq ($(QMK_USERSPACE),)
20 ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/keymap.json)","")
21 KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/keymap.json
22 KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)
23 else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/keymap.json)","")
24 KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/keymap.json
25 KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)
26 else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/keymap.json)","")
27 KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/keymap.json
28 KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)
29 else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/keymap.json)","")
30 KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/keymap.json
31 KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)
32 else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/keymap.json)","")
33 KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/keymap.json
34 KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)
35 endif
36endif
diff --git a/builddefs/build_keyboard.mk b/builddefs/build_keyboard.mk
index 12a8c5b67b..f17171fe20 100644
--- a/builddefs/build_keyboard.mk
+++ b/builddefs/build_keyboard.mk
@@ -127,34 +127,60 @@ include $(INFO_RULES_MK)
127include $(BUILDDEFS_PATH)/build_json.mk 127include $(BUILDDEFS_PATH)/build_json.mk
128 128
129# Pull in keymap level rules.mk 129# Pull in keymap level rules.mk
130# Look through the possible keymap folders until we find a matching keymap.c 130ifeq ("$(wildcard $(KEYMAP_PATH))", "")
131ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","") 131 # Look through the possible keymap folders until we find a matching keymap.c
132 -include $(MAIN_KEYMAP_PATH_1)/rules.mk 132 ifneq ($(QMK_USERSPACE),)
133 KEYMAP_C := $(MAIN_KEYMAP_PATH_1)/keymap.c 133 ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/keymap.c)","")
134 KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1) 134 -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/rules.mk
135else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.c)","") 135 KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/keymap.c
136 -include $(MAIN_KEYMAP_PATH_2)/rules.mk 136 KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)
137 KEYMAP_C := $(MAIN_KEYMAP_PATH_2)/keymap.c 137 else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/keymap.c)","")
138 KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2) 138 -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/rules.mk
139else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.c)","") 139 KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/keymap.c
140 -include $(MAIN_KEYMAP_PATH_3)/rules.mk 140 KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)
141 KEYMAP_C := $(MAIN_KEYMAP_PATH_3)/keymap.c 141 else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/keymap.c)","")
142 KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3) 142 -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/rules.mk
143else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.c)","") 143 KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/keymap.c
144 -include $(MAIN_KEYMAP_PATH_4)/rules.mk 144 KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)
145 KEYMAP_C := $(MAIN_KEYMAP_PATH_4)/keymap.c 145 else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/keymap.c)","")
146 KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4) 146 -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/rules.mk
147else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","") 147 KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/keymap.c
148 -include $(MAIN_KEYMAP_PATH_5)/rules.mk 148 KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)
149 KEYMAP_C := $(MAIN_KEYMAP_PATH_5)/keymap.c 149 else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/keymap.c)","")
150 KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5) 150 -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/rules.mk
151else ifneq ($(LAYOUTS),) 151 KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/keymap.c
152 # If we haven't found a keymap yet fall back to community layouts 152 KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)
153 include $(BUILDDEFS_PATH)/build_layout.mk 153 endif
154# Not finding keymap.c is fine if we found a keymap.json 154 endif
155else ifeq ("$(wildcard $(KEYMAP_JSON_PATH))", "") 155 ifeq ($(KEYMAP_PATH),)
156 $(call CATASTROPHIC_ERROR,Invalid keymap,Could not find keymap) 156 ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","")
157 # this state should never be reached 157 -include $(MAIN_KEYMAP_PATH_1)/rules.mk
158 KEYMAP_C := $(MAIN_KEYMAP_PATH_1)/keymap.c
159 KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
160 else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.c)","")
161 -include $(MAIN_KEYMAP_PATH_2)/rules.mk
162 KEYMAP_C := $(MAIN_KEYMAP_PATH_2)/keymap.c
163 KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
164 else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.c)","")
165 -include $(MAIN_KEYMAP_PATH_3)/rules.mk
166 KEYMAP_C := $(MAIN_KEYMAP_PATH_3)/keymap.c
167 KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
168 else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.c)","")
169 -include $(MAIN_KEYMAP_PATH_4)/rules.mk
170 KEYMAP_C := $(MAIN_KEYMAP_PATH_4)/keymap.c
171 KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
172 else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
173 -include $(MAIN_KEYMAP_PATH_5)/rules.mk
174 KEYMAP_C := $(MAIN_KEYMAP_PATH_5)/keymap.c
175 KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
176 else ifneq ($(LAYOUTS),)
177 # If we haven't found a keymap yet fall back to community layouts
178 include $(BUILDDEFS_PATH)/build_layout.mk
179 else ifeq ("$(wildcard $(KEYMAP_JSON_PATH))", "") # Not finding keymap.c is fine if we found a keymap.json
180 $(call CATASTROPHIC_ERROR,Invalid keymap,Could not find keymap)
181 # this state should never be reached
182 endif
183 endif
158endif 184endif
159 185
160# Have we found a keymap.json? 186# Have we found a keymap.json?
@@ -364,6 +390,16 @@ ifeq ("$(USER_NAME)","")
364endif 390endif
365USER_PATH := users/$(USER_NAME) 391USER_PATH := users/$(USER_NAME)
366 392
393# If we have userspace, then add it to the lookup VPATH
394ifneq ($(wildcard $(QMK_USERSPACE)),)
395 VPATH += $(QMK_USERSPACE)
396endif
397
398# If the equivalent users directory exists in userspace, use that in preference to anything currently in the main repo
399ifneq ($(wildcard $(QMK_USERSPACE)/$(USER_PATH)),)
400 USER_PATH := $(QMK_USERSPACE)/$(USER_PATH)
401endif
402
367# Pull in user level rules.mk 403# Pull in user level rules.mk
368-include $(USER_PATH)/rules.mk 404-include $(USER_PATH)/rules.mk
369ifneq ("$(wildcard $(USER_PATH)/config.h)","") 405ifneq ("$(wildcard $(USER_PATH)/config.h)","")
@@ -404,6 +440,10 @@ ifneq ("$(KEYMAP_H)","")
404 CONFIG_H += $(KEYMAP_H) 440 CONFIG_H += $(KEYMAP_H)
405endif 441endif
406 442
443ifeq ($(KEYMAP_C),)
444 $(call CATASTROPHIC_ERROR,Invalid keymap,Could not find keymap)
445endif
446
407OPT_DEFS += -DKEYMAP_C=\"$(KEYMAP_C)\" 447OPT_DEFS += -DKEYMAP_C=\"$(KEYMAP_C)\"
408 448
409# If a keymap or userspace places their keymap array in another file instead, allow for it to be included 449# If a keymap or userspace places their keymap array in another file instead, allow for it to be included
diff --git a/builddefs/build_layout.mk b/builddefs/build_layout.mk
index 6166bd847c..9ff99cc221 100644
--- a/builddefs/build_layout.mk
+++ b/builddefs/build_layout.mk
@@ -1,6 +1,10 @@
1LAYOUTS_PATH := layouts 1LAYOUTS_PATH := layouts
2LAYOUTS_REPOS := $(patsubst %/,%,$(sort $(dir $(wildcard $(LAYOUTS_PATH)/*/)))) 2LAYOUTS_REPOS := $(patsubst %/,%,$(sort $(dir $(wildcard $(LAYOUTS_PATH)/*/))))
3 3
4ifneq ($(QMK_USERSPACE),)
5 LAYOUTS_REPOS += $(patsubst %/,%,$(QMK_USERSPACE)/$(LAYOUTS_PATH))
6endif
7
4define SEARCH_LAYOUTS_REPO 8define SEARCH_LAYOUTS_REPO
5 LAYOUT_KEYMAP_PATH := $$(LAYOUTS_REPO)/$$(LAYOUT)/$$(KEYMAP) 9 LAYOUT_KEYMAP_PATH := $$(LAYOUTS_REPO)/$$(LAYOUT)/$$(KEYMAP)
6 LAYOUT_KEYMAP_JSON := $$(LAYOUT_KEYMAP_PATH)/keymap.json 10 LAYOUT_KEYMAP_JSON := $$(LAYOUT_KEYMAP_PATH)/keymap.json
diff --git a/builddefs/common_rules.mk b/builddefs/common_rules.mk
index 52dccbe475..cfd261737c 100644
--- a/builddefs/common_rules.mk
+++ b/builddefs/common_rules.mk
@@ -191,7 +191,7 @@ DFU_SUFFIX_ARGS ?=
191elf: $(BUILD_DIR)/$(TARGET).elf 191elf: $(BUILD_DIR)/$(TARGET).elf
192hex: $(BUILD_DIR)/$(TARGET).hex 192hex: $(BUILD_DIR)/$(TARGET).hex
193uf2: $(BUILD_DIR)/$(TARGET).uf2 193uf2: $(BUILD_DIR)/$(TARGET).uf2
194cpfirmware: $(FIRMWARE_FORMAT) 194cpfirmware_qmk: $(FIRMWARE_FORMAT)
195 $(SILENT) || printf "Copying $(TARGET).$(FIRMWARE_FORMAT) to qmk_firmware folder" | $(AWK_CMD) 195 $(SILENT) || printf "Copying $(TARGET).$(FIRMWARE_FORMAT) to qmk_firmware folder" | $(AWK_CMD)
196 $(COPY) $(BUILD_DIR)/$(TARGET).$(FIRMWARE_FORMAT) $(TARGET).$(FIRMWARE_FORMAT) && $(PRINT_OK) 196 $(COPY) $(BUILD_DIR)/$(TARGET).$(FIRMWARE_FORMAT) $(TARGET).$(FIRMWARE_FORMAT) && $(PRINT_OK)
197eep: $(BUILD_DIR)/$(TARGET).eep 197eep: $(BUILD_DIR)/$(TARGET).eep
@@ -200,6 +200,15 @@ sym: $(BUILD_DIR)/$(TARGET).sym
200LIBNAME=lib$(TARGET).a 200LIBNAME=lib$(TARGET).a
201lib: $(LIBNAME) 201lib: $(LIBNAME)
202 202
203cpfirmware: cpfirmware_qmk
204
205ifneq ($(QMK_USERSPACE),)
206cpfirmware: cpfirmware_userspace
207cpfirmware_userspace: cpfirmware_qmk
208 $(SILENT) || printf "Copying $(TARGET).$(FIRMWARE_FORMAT) to userspace folder" | $(AWK_CMD)
209 $(COPY) $(BUILD_DIR)/$(TARGET).$(FIRMWARE_FORMAT) $(QMK_USERSPACE)/$(TARGET).$(FIRMWARE_FORMAT) && $(PRINT_OK)
210endif
211
203# Display size of file, modifying the output so people don't mistakenly grab the hex output 212# Display size of file, modifying the output so people don't mistakenly grab the hex output
204BINARY_SIZE = $(SIZE) --target=$(FORMAT) $(BUILD_DIR)/$(TARGET).hex | $(SED) -e 's/\.build\/.*$$/$(TARGET).$(FIRMWARE_FORMAT)/g' 213BINARY_SIZE = $(SIZE) --target=$(FORMAT) $(BUILD_DIR)/$(TARGET).hex | $(SED) -e 's/\.build\/.*$$/$(TARGET).$(FIRMWARE_FORMAT)/g'
205 214
diff --git a/data/schemas/definitions.jsonschema b/data/schemas/definitions.jsonschema
index 441e6395cf..ea29343d0a 100644
--- a/data/schemas/definitions.jsonschema
+++ b/data/schemas/definitions.jsonschema
@@ -177,5 +177,23 @@
177 "type": "integer", 177 "type": "integer",
178 "minimum": 0, 178 "minimum": 0,
179 "maximum": 1 179 "maximum": 1
180 },
181 "keyboard_keymap_tuple": {
182 "type": "array",
183 "prefixItems": [
184 { "$ref": "#/keyboard" },
185 { "$ref": "#/filename" }
186 ],
187 "unevaluatedItems": false
188 },
189 "json_file_path": {
190 "type": "string",
191 "pattern": "^[0-9a-z_/\\-]+\\.json$"
192 },
193 "build_target": {
194 "oneOf": [
195 { "$ref": "#/keyboard_keymap_tuple" },
196 { "$ref": "#/json_file_path" }
197 ]
180 } 198 }
181} 199}
diff --git a/data/schemas/user_repo_v0.jsonschema b/data/schemas/user_repo_v0.jsonschema
new file mode 100644
index 0000000000..b18ac50428
--- /dev/null
+++ b/data/schemas/user_repo_v0.jsonschema
@@ -0,0 +1,14 @@
1{
2 "$schema": "https://json-schema.org/draft/2020-12/schema#",
3 "$id": "qmk.user_repo.v0",
4 "title": "User Repository Information",
5 "type": "object",
6 "required": [
7 "userspace_version"
8 ],
9 "properties": {
10 "userspace_version": {
11 "type": "string",
12 },
13 }
14}
diff --git a/data/schemas/user_repo_v1.jsonschema b/data/schemas/user_repo_v1.jsonschema
new file mode 100644
index 0000000000..6cdf758685
--- /dev/null
+++ b/data/schemas/user_repo_v1.jsonschema
@@ -0,0 +1,22 @@
1{
2 "$schema": "https://json-schema.org/draft/2020-12/schema#",
3 "$id": "qmk.user_repo.v1",
4 "title": "User Repository Information",
5 "type": "object",
6 "required": [
7 "userspace_version",
8 "build_targets"
9 ],
10 "properties": {
11 "userspace_version": {
12 "type": "string",
13 "enum": ["1.0"]
14 },
15 "build_targets": {
16 "type": "array",
17 "items": {
18 "$ref": "qmk.definitions.v1#/build_target"
19 }
20 }
21 }
22}
diff --git a/docs/_summary.md b/docs/_summary.md
index 722c5f9c5d..36c90c5bb9 100644
--- a/docs/_summary.md
+++ b/docs/_summary.md
@@ -4,7 +4,7 @@
4 * [Building Your First Firmware](newbs_building_firmware.md) 4 * [Building Your First Firmware](newbs_building_firmware.md)
5 * [Flashing Firmware](newbs_flashing.md) 5 * [Flashing Firmware](newbs_flashing.md)
6 * [Getting Help/Support](support.md) 6 * [Getting Help/Support](support.md)
7 * [Building With GitHub Userspace](newbs_building_firmware_workflow.md) 7 * [External Userspace](newbs_external_userspace.md)
8 * [Other Resources](newbs_learn_more_resources.md) 8 * [Other Resources](newbs_learn_more_resources.md)
9 * [Syllabus](syllabus.md) 9 * [Syllabus](syllabus.md)
10 10
diff --git a/docs/cli_commands.md b/docs/cli_commands.md
index 79fd9de575..7b5ad5b13a 100644
--- a/docs/cli_commands.md
+++ b/docs/cli_commands.md
@@ -482,6 +482,131 @@ $ qmk import-kbfirmware ~/Downloads/gh62.json
482 482
483--- 483---
484 484
485# External Userspace Commands
486
487## `qmk userspace-add`
488
489This command adds a keyboard/keymap to the External Userspace build targets.
490
491**Usage**:
492
493```
494qmk userspace-add [-h] [-km KEYMAP] [-kb KEYBOARD] [builds ...]
495
496positional arguments:
497 builds List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.
498
499options:
500 -h, --help show this help message and exit
501 -km KEYMAP, --keymap KEYMAP
502 The keymap to build a firmware for. Ignored when a configurator export is supplied.
503 -kb KEYBOARD, --keyboard KEYBOARD
504 The keyboard to build a firmware for. Ignored when a configurator export is supplied.
505```
506
507**Example**:
508
509```
510$ qmk userspace-add -kb planck/rev6 -km default
511Ψ Added planck/rev6:default to userspace build targets
512Ψ Saved userspace file to /home/you/qmk_userspace/qmk.json
513```
514
515## `qmk userspace-remove`
516
517This command removes a keyboard/keymap from the External Userspace build targets.
518
519**Usage**:
520
521```
522qmk userspace-remove [-h] [-km KEYMAP] [-kb KEYBOARD] [builds ...]
523
524positional arguments:
525 builds List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.
526
527options:
528 -h, --help show this help message and exit
529 -km KEYMAP, --keymap KEYMAP
530 The keymap to build a firmware for. Ignored when a configurator export is supplied.
531 -kb KEYBOARD, --keyboard KEYBOARD
532 The keyboard to build a firmware for. Ignored when a configurator export is supplied.
533```
534
535**Example**:
536
537```
538$ qmk userspace-remove -kb planck/rev6 -km default
539Ψ Removed planck/rev6:default from userspace build targets
540Ψ Saved userspace file to /home/you/qmk_userspace/qmk.json
541```
542
543## `qmk userspace-list`
544
545This command lists the External Userspace build targets.
546
547**Usage**:
548
549```
550qmk userspace-list [-h] [-e]
551
552options:
553 -h, --help show this help message and exit
554 -e, --expand Expands any use of `all` for either keyboard or keymap.
555```
556
557**Example**:
558
559```
560$ qmk userspace-list
561Ψ Current userspace build targets:
562Ψ Keyboard: planck/rev6, keymap: you
563Ψ Keyboard: clueboard/66/rev3, keymap: you
564```
565
566## `qmk userspace-compile`
567
568This command compiles all the External Userspace build targets.
569
570**Usage**:
571
572```
573qmk userspace-compile [-h] [-e ENV] [-n] [-c] [-j PARALLEL] [-t]
574
575options:
576 -h, --help show this help message and exit
577 -e ENV, --env ENV Set a variable to be passed to make. May be passed multiple times.
578 -n, --dry-run Don't actually build, just show the commands to be run.
579 -c, --clean Remove object files before compiling.
580 -j PARALLEL, --parallel PARALLEL
581 Set the number of parallel make jobs; 0 means unlimited.
582 -t, --no-temp Remove temporary files during build.
583```
584
585**Example**:
586
587```
588$ qmk userspace-compile
589Ψ Preparing target list...
590Build planck/rev6:you [OK]
591Build clueboard/66/rev3:you [OK]
592```
593
594## `qmk userspace-doctor`
595
596This command examines your environment and alerts you to potential problems related to External Userspace.
597
598**Example**:
599
600```
601% qmk userspace-doctor
602Ψ QMK home: /home/you/qmk_userspace/qmk_firmware
603Ψ Testing userspace candidate: /home/you/qmk_userspace -- Valid `qmk.json`
604Ψ QMK userspace: /home/you/qmk_userspace
605Ψ Userspace enabled: True
606```
607
608---
609
485# Developer Commands 610# Developer Commands
486 611
487## `qmk format-text` 612## `qmk format-text`
diff --git a/docs/newbs_external_userspace.md b/docs/newbs_external_userspace.md
new file mode 100644
index 0000000000..9bdf4b0b18
--- /dev/null
+++ b/docs/newbs_external_userspace.md
@@ -0,0 +1,96 @@
1# External QMK Userspace
2
3QMK Firmware now officially supports storing user keymaps outside of the normal QMK Firmware repository, allowing users to maintain their own keymaps without having to fork, modify, and maintain a copy of QMK Firmware themselves.
4
5External Userspace mirrors the structure of the main QMK Firmware repository, but only contains the keymaps that you wish to build. You can still use `keyboards/<my keyboard>/keymaps/<my keymap>` to store your keymaps, or you can use the `layouts/<my layout>/<my keymap>` system as before -- they're just stored external to QMK Firmware.
6
7The build system will still honor the use of `users/<my keymap>` if you rely on the traditional QMK Firmware [userspace feature](feature_userspace.md) -- it's now supported externally too, using the same location inside the External Userspace directory.
8
9Additionally, there is first-class support for using GitHub Actions to build your keymaps, allowing you to automatically compile your keymaps whenever you push changes to your External Userspace repository.
10
11!> External Userspace is new functionality and may have issues. Tighter integration with the `qmk` command will occur over time.
12
13?> Historical keymap.json and GitHub-based firmware build instructions can be found [here](newbs_building_firmware_workflow.md). This document supersedes those instructions, but they should still function correctly.
14
15## Setting up QMK Locally
16
17If you wish to build on your local machine, you will need to set up QMK locally. This is a one-time process, and is documented in the [newbs setup guide](https://docs.qmk.fm/#/newbs).
18
19!> If you wish to use any QMK CLI commands related to manipulating External Userspace definitions, you will currently need a copy of QMK Firmware as well.
20
21!> Building locally has a much shorter turnaround time than waiting for GitHub Actions to complete.
22
23## External Userspace Repository Setup (forked on GitHub)
24
25A basic skeleton External Userspace repository can be found [here](https://github.com/qmk/qmk_userspace). If you wish to keep your keymaps on GitHub (strongly recommended!), you can fork the repository and use it as a base:
26
27![Userspace Fork](https://i.imgur.com/hcegguh.png)
28
29Going ahead with your fork will copy it to your account, at which point you can clone it to your local machine and begin adding your keymaps:
30
31![Userspace Clone](https://i.imgur.com/CWYmsk8.png)
32
33```sh
34cd $HOME
35git clone https://github.com/{myusername}/qmk_userspace.git
36qmk config user.overlay_dir="$(realpath qmk_userspace)"
37```
38
39## External Userspace Setup (locally stored only)
40
41If you don't want to use GitHub and prefer to keep everything local, you can clone a copy of the default External Userspace locally instead:
42
43```sh
44cd $HOME
45git clone https://github.com/qmk/qmk_userspace.git
46qmk config user.overlay_dir="$(realpath qmk_userspace)"
47```
48
49## Adding a Keymap
50
51_These instructions assume you have already set up QMK locally, and have a copy of the QMK Firmware repository on your machine._
52
53Keymaps within External Userspace are defined in the same way as they are in the main QMK repository. You can either use the `qmk new-keymap` command to create a new keymap, or manually create a new directory in the `keyboards` directory.
54
55Alternatively, you can use the `layouts` directory to store your keymaps, using the same layout system as the main QMK repository -- if you choose to do so you'll want to use the path `layouts/<layout name>/<keymap name>/keymap.*` to store your keymap files, where `layout name` matches an existing layout in QMK, such as `tkl_ansi`.
56
57After creating your new keymap, building the keymap matches normal QMK usage:
58
59```sh
60qmk compile -kb <keyboard> -km <keymap>
61```
62
63!> The `qmk config user.overlay_dir=...` command must have been run when cloning the External Userspace repository for this to work correctly.
64
65## Adding the keymap to External Userspace build targets
66
67Once you have created your keymap, if you want to use GitHub Actions to build your firmware, you will need to add it to the External Userspace build targets. This is done using the `qmk userspace-add` command:
68
69```sh
70# for a keyboard/keymap combo:
71qmk userspace-add -kb <keyboard> -km <keymap>
72# or, for a json-based keymap (if kept "loose"):
73qmk userspace-add <relative/path/to/my/keymap.json>
74```
75
76This updates the `qmk.json` file in the root of your External Userspace directory. If you're using a git repository to store your keymaps, now is a great time to commit and push to your own fork.
77
78## Compiling External Userspace build targets
79
80Once you have added your keymaps to the External Userspace build targets, you can compile all of them at once using the `qmk userspace-compile` command:
81
82```sh
83qmk userspace-compile
84```
85
86All firmware builds you've added to the External Userspace build targets will be built, and the resulting firmware files will be placed in the root of your External Userspace directory.
87
88## Using GitHub Actions
89
90GitHub Actions can be used to automatically build your keymaps whenever you push changes to your External Userspace repository. If you have set up your list of build targets, this is as simple as enabling workflows in the GitHub repository settings:
91
92![Repo Settings](https://i.imgur.com/EVkxOt1.png)
93
94Any push will result in compilation of all configured builds, and once completed a new release containing the newly-minted firmware files will be created on GitHub, which you can subsequently download and flash to your keyboard:
95
96![Releases](https://i.imgur.com/zmwOL5P.png)
diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
index 16a7ef87a2..1ab489cec3 100644
--- a/lib/python/qmk/build_targets.py
+++ b/lib/python/qmk/build_targets.py
@@ -10,6 +10,8 @@ from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX
10from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json 10from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json
11from qmk.keyboard import keyboard_folder 11from qmk.keyboard import keyboard_folder
12from qmk.info import keymap_json 12from qmk.info import keymap_json
13from qmk.keymap import locate_keymap
14from qmk.path import is_under_qmk_firmware, is_under_qmk_userspace
13 15
14 16
15class BuildTarget: 17class BuildTarget:
@@ -158,6 +160,20 @@ class KeyboardKeymapBuildTarget(BuildTarget):
158 for key, value in env_vars.items(): 160 for key, value in env_vars.items():
159 compile_args.append(f'{key}={value}') 161 compile_args.append(f'{key}={value}')
160 162
163 # Need to override the keymap path if the keymap is a userspace directory.
164 # This also ensures keyboard aliases as per `keyboard_aliases.hjson` still work if the userspace has the keymap
165 # in an equivalent historical location.
166 keymap_location = locate_keymap(self.keyboard, self.keymap)
167 if is_under_qmk_userspace(keymap_location) and not is_under_qmk_firmware(keymap_location):
168 keymap_directory = keymap_location.parent
169 compile_args.extend([
170 f'MAIN_KEYMAP_PATH_1={keymap_directory}',
171 f'MAIN_KEYMAP_PATH_2={keymap_directory}',
172 f'MAIN_KEYMAP_PATH_3={keymap_directory}',
173 f'MAIN_KEYMAP_PATH_4={keymap_directory}',
174 f'MAIN_KEYMAP_PATH_5={keymap_directory}',
175 ])
176
161 return compile_args 177 return compile_args
162 178
163 179
diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index 695a180066..cf60903687 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -81,6 +81,11 @@ subcommands = [
81 'qmk.cli.new.keymap', 81 'qmk.cli.new.keymap',
82 'qmk.cli.painter', 82 'qmk.cli.painter',
83 'qmk.cli.pytest', 83 'qmk.cli.pytest',
84 'qmk.cli.userspace.add',
85 'qmk.cli.userspace.compile',
86 'qmk.cli.userspace.doctor',
87 'qmk.cli.userspace.list',
88 'qmk.cli.userspace.remove',
84 'qmk.cli.via2json', 89 'qmk.cli.via2json',
85] 90]
86 91
diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py
index 71c1dec162..3c8f3664ea 100755
--- a/lib/python/qmk/cli/compile.py
+++ b/lib/python/qmk/cli/compile.py
@@ -37,7 +37,9 @@ def compile(cli):
37 from .mass_compile import mass_compile 37 from .mass_compile import mass_compile
38 cli.args.builds = [] 38 cli.args.builds = []
39 cli.args.filter = [] 39 cli.args.filter = []
40 cli.args.no_temp = False 40 cli.config.mass_compile.keymap = cli.config.compile.keymap
41 cli.config.mass_compile.parallel = cli.config.compile.parallel
42 cli.config.mass_compile.no_temp = False
41 return mass_compile(cli) 43 return mass_compile(cli)
42 44
43 # Build the environment vars 45 # Build the environment vars
diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py
index 6a6feb87d1..dd8b58b2c7 100755
--- a/lib/python/qmk/cli/doctor/main.py
+++ b/lib/python/qmk/cli/doctor/main.py
@@ -9,10 +9,11 @@ from milc import cli
9from milc.questions import yesno 9from milc.questions import yesno
10 10
11from qmk import submodules 11from qmk import submodules
12from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM 12from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM, QMK_USERSPACE, HAS_QMK_USERSPACE
13from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules 13from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules
14from qmk.git import git_check_repo, git_get_branch, git_get_tag, git_get_last_log_entry, git_get_common_ancestor, git_is_dirty, git_get_remotes, git_check_deviation 14from qmk.git import git_check_repo, git_get_branch, git_get_tag, git_get_last_log_entry, git_get_common_ancestor, git_is_dirty, git_get_remotes, git_check_deviation
15from qmk.commands import in_virtualenv 15from qmk.commands import in_virtualenv
16from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate, UserspaceValidationError
16 17
17 18
18def os_tests(): 19def os_tests():
@@ -92,6 +93,25 @@ def output_submodule_status():
92 cli.log.error(f'- {sub_name}: <<< missing or unknown >>>') 93 cli.log.error(f'- {sub_name}: <<< missing or unknown >>>')
93 94
94 95
96def userspace_tests(qmk_firmware):
97 if qmk_firmware:
98 cli.log.info(f'QMK home: {{fg_cyan}}{qmk_firmware}')
99
100 for path in qmk_userspace_paths():
101 try:
102 qmk_userspace_validate(path)
103 cli.log.info(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`')
104 except FileNotFoundError:
105 cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Missing `qmk.json`')
106 except UserspaceValidationError as err:
107 cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`')
108 cli.log.warn(f' -- {{fg_cyan}}{path}/qmk.json{{fg_reset}} validation error: {err}')
109
110 if QMK_USERSPACE is not None:
111 cli.log.info(f'QMK userspace: {{fg_cyan}}{QMK_USERSPACE}')
112 cli.log.info(f'Userspace enabled: {{fg_cyan}}{HAS_QMK_USERSPACE}')
113
114
95@cli.argument('-y', '--yes', action='store_true', arg_only=True, help='Answer yes to all questions.') 115@cli.argument('-y', '--yes', action='store_true', arg_only=True, help='Answer yes to all questions.')
96@cli.argument('-n', '--no', action='store_true', arg_only=True, help='Answer no to all questions.') 116@cli.argument('-n', '--no', action='store_true', arg_only=True, help='Answer no to all questions.')
97@cli.subcommand('Basic QMK environment checks') 117@cli.subcommand('Basic QMK environment checks')
@@ -108,6 +128,9 @@ def doctor(cli):
108 cli.log.info('QMK home: {fg_cyan}%s', QMK_FIRMWARE) 128 cli.log.info('QMK home: {fg_cyan}%s', QMK_FIRMWARE)
109 129
110 status = os_status = os_tests() 130 status = os_status = os_tests()
131
132 userspace_tests(None)
133
111 git_status = git_tests() 134 git_status = git_tests()
112 135
113 if git_status == CheckStatus.ERROR or (os_status == CheckStatus.OK and git_status == CheckStatus.WARNING): 136 if git_status == CheckStatus.ERROR or (os_status == CheckStatus.OK and git_status == CheckStatus.WARNING):
diff --git a/lib/python/qmk/cli/format/json.py b/lib/python/qmk/cli/format/json.py
index 3299a0d807..283513254c 100755
--- a/lib/python/qmk/cli/format/json.py
+++ b/lib/python/qmk/cli/format/json.py
@@ -9,48 +9,74 @@ from milc import cli
9 9
10from qmk.info import info_json 10from qmk.info import info_json
11from qmk.json_schema import json_load, validate 11from qmk.json_schema import json_load, validate
12from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder 12from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder, UserspaceJSONEncoder
13from qmk.path import normpath 13from qmk.path import normpath
14 14
15 15
16@cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format') 16def _detect_json_format(file, json_data):
17@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)') 17 """Detect the format of a json file.
18@cli.argument('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file')
19@cli.argument('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ')
20@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True)
21def format_json(cli):
22 """Format a json file.
23 """ 18 """
24 json_file = json_load(cli.args.json_file) 19 json_encoder = None
25 20 try:
26 if cli.args.format == 'auto': 21 validate(json_data, 'qmk.user_repo.v1')
22 json_encoder = UserspaceJSONEncoder
23 except ValidationError:
24 pass
25
26 if json_encoder is None:
27 try: 27 try:
28 validate(json_file, 'qmk.keyboard.v1') 28 validate(json_data, 'qmk.keyboard.v1')
29 json_encoder = InfoJSONEncoder 29 json_encoder = InfoJSONEncoder
30
31 except ValidationError as e: 30 except ValidationError as e:
32 cli.log.warning('File %s did not validate as a keyboard:\n\t%s', cli.args.json_file, e) 31 cli.log.warning('File %s did not validate as a keyboard info.json or userspace qmk.json:\n\t%s', file, e)
33 cli.log.info('Treating %s as a keymap file.', cli.args.json_file) 32 cli.log.info('Treating %s as a keymap file.', file)
34 json_encoder = KeymapJSONEncoder 33 json_encoder = KeymapJSONEncoder
34
35 return json_encoder
36
37
38def _get_json_encoder(file, json_data):
39 """Get the json encoder for a file.
40 """
41 json_encoder = None
42 if cli.args.format == 'auto':
43 json_encoder = _detect_json_format(file, json_data)
35 elif cli.args.format == 'keyboard': 44 elif cli.args.format == 'keyboard':
36 json_encoder = InfoJSONEncoder 45 json_encoder = InfoJSONEncoder
37 elif cli.args.format == 'keymap': 46 elif cli.args.format == 'keymap':
38 json_encoder = KeymapJSONEncoder 47 json_encoder = KeymapJSONEncoder
48 elif cli.args.format == 'userspace':
49 json_encoder = UserspaceJSONEncoder
39 else: 50 else:
40 # This should be impossible 51 # This should be impossible
41 cli.log.error('Unknown format: %s', cli.args.format) 52 cli.log.error('Unknown format: %s', cli.args.format)
53 return json_encoder
54
55
56@cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format')
57@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap', 'userspace'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)')
58@cli.argument('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file')
59@cli.argument('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ')
60@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True)
61def format_json(cli):
62 """Format a json file.
63 """
64 json_data = json_load(cli.args.json_file)
65
66 json_encoder = _get_json_encoder(cli.args.json_file, json_data)
67 if json_encoder is None:
42 return False 68 return False
43 69
44 if json_encoder == KeymapJSONEncoder and 'layout' in json_file: 70 if json_encoder == KeymapJSONEncoder and 'layout' in json_data:
45 # Attempt to format the keycodes. 71 # Attempt to format the keycodes.
46 layout = json_file['layout'] 72 layout = json_data['layout']
47 info_data = info_json(json_file['keyboard']) 73 info_data = info_json(json_data['keyboard'])
48 74
49 if layout in info_data.get('layout_aliases', {}): 75 if layout in info_data.get('layout_aliases', {}):
50 layout = json_file['layout'] = info_data['layout_aliases'][layout] 76 layout = json_data['layout'] = info_data['layout_aliases'][layout]
51 77
52 if layout in info_data.get('layouts'): 78 if layout in info_data.get('layouts'):
53 for layer_num, layer in enumerate(json_file['layers']): 79 for layer_num, layer in enumerate(json_data['layers']):
54 current_layer = [] 80 current_layer = []
55 last_row = 0 81 last_row = 0
56 82
@@ -61,9 +87,9 @@ def format_json(cli):
61 87
62 current_layer.append(keymap_key) 88 current_layer.append(keymap_key)
63 89
64 json_file['layers'][layer_num] = current_layer 90 json_data['layers'][layer_num] = current_layer
65 91
66 output = json.dumps(json_file, cls=json_encoder, sort_keys=True) 92 output = json.dumps(json_data, cls=json_encoder, sort_keys=True)
67 93
68 if cli.args.inplace: 94 if cli.args.inplace:
69 with open(cli.args.json_file, 'w+', encoding='utf-8') as outfile: 95 with open(cli.args.json_file, 'w+', encoding='utf-8') as outfile:
diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
index 7968de53e7..b025f85701 100755
--- a/lib/python/qmk/cli/mass_compile.py
+++ b/lib/python/qmk/cli/mass_compile.py
@@ -72,7 +72,7 @@ all: {keyboard_safe}_{keymap_name}_binary
72 # yapf: enable 72 # yapf: enable
73 f.write('\n') 73 f.write('\n')
74 74
75 cli.run([make_cmd, *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) 75 cli.run([find_make(), *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)
76 76
77 # Check for failures 77 # Check for failures
78 failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')] 78 failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')]
diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py
index 9b0ac221a4..d4339bc9ef 100755
--- a/lib/python/qmk/cli/new/keymap.py
+++ b/lib/python/qmk/cli/new/keymap.py
@@ -5,10 +5,12 @@ import shutil
5from milc import cli 5from milc import cli
6from milc.questions import question 6from milc.questions import question
7 7
8from qmk.constants import HAS_QMK_USERSPACE, QMK_USERSPACE
8from qmk.path import is_keyboard, keymaps, keymap 9from qmk.path import is_keyboard, keymaps, keymap
9from qmk.git import git_get_username 10from qmk.git import git_get_username
10from qmk.decorators import automagic_keyboard, automagic_keymap 11from qmk.decorators import automagic_keyboard, automagic_keymap
11from qmk.keyboard import keyboard_completer, keyboard_folder 12from qmk.keyboard import keyboard_completer, keyboard_folder
13from qmk.userspace import UserspaceDefs
12 14
13 15
14def prompt_keyboard(): 16def prompt_keyboard():
@@ -68,3 +70,9 @@ def new_keymap(cli):
68 # end message to user 70 # end message to user
69 cli.log.info(f'{{fg_green}}Created a new keymap called {{fg_cyan}}{user_name}{{fg_green}} in: {{fg_cyan}}{keymap_path_new}.{{fg_reset}}') 71 cli.log.info(f'{{fg_green}}Created a new keymap called {{fg_cyan}}{user_name}{{fg_green}} in: {{fg_cyan}}{keymap_path_new}.{{fg_reset}}')
70 cli.log.info(f"Compile a firmware with your new keymap by typing: {{fg_yellow}}qmk compile -kb {kb_name} -km {user_name}{{fg_reset}}.") 72 cli.log.info(f"Compile a firmware with your new keymap by typing: {{fg_yellow}}qmk compile -kb {kb_name} -km {user_name}{{fg_reset}}.")
73
74 # Add to userspace compile if we have userspace available
75 if HAS_QMK_USERSPACE:
76 userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
77 userspace.add_target(keyboard=kb_name, keymap=user_name, do_print=False)
78 return userspace.save()
diff --git a/lib/python/qmk/cli/userspace/__init__.py b/lib/python/qmk/cli/userspace/__init__.py
new file mode 100644
index 0000000000..5757d3a4c9
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/__init__.py
@@ -0,0 +1,5 @@
1from . import doctor
2from . import add
3from . import remove
4from . import list
5from . import compile
diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py
new file mode 100644
index 0000000000..8993d54dba
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/add.py
@@ -0,0 +1,51 @@
1# Copyright 2023 Nick Brassel (@tzarc)
2# SPDX-License-Identifier: GPL-2.0-or-later
3from pathlib import Path
4from milc import cli
5
6from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
7from qmk.keyboard import keyboard_completer, keyboard_folder_or_all
8from qmk.keymap import keymap_completer, is_keymap_target
9from qmk.userspace import UserspaceDefs
10
11
12@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.")
13@cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
14@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
15@cli.subcommand('Adds a build target to userspace `qmk.json`.')
16def userspace_add(cli):
17 if not HAS_QMK_USERSPACE:
18 cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
19 return False
20
21 userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
22
23 if len(cli.args.builds) > 0:
24 json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
25 make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
26
27 for e in json_like_targets:
28 userspace.add_target(json_path=e)
29
30 for e in make_like_targets:
31 s = e.split(':')
32 userspace.add_target(keyboard=s[0], keymap=s[1])
33
34 else:
35 failed = False
36 try:
37 if not is_keymap_target(cli.args.keyboard, cli.args.keymap):
38 failed = True
39 except KeyError:
40 failed = True
41
42 if failed:
43 from qmk.cli.new.keymap import new_keymap
44 cli.config.new_keymap.keyboard = cli.args.keyboard
45 cli.config.new_keymap.keymap = cli.args.keymap
46 if new_keymap(cli) is not False:
47 userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
48 else:
49 userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
50
51 return userspace.save()
diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py
new file mode 100644
index 0000000000..0a42dd5bf5
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/compile.py
@@ -0,0 +1,38 @@
1# Copyright 2023 Nick Brassel (@tzarc)
2# SPDX-License-Identifier: GPL-2.0-or-later
3from pathlib import Path
4from milc import cli
5
6from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
7from qmk.commands import build_environment
8from qmk.userspace import UserspaceDefs
9from qmk.build_targets import JsonKeymapBuildTarget
10from qmk.search import search_keymap_targets
11from qmk.cli.mass_compile import mass_compile_targets
12
13
14@cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
15@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
16@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
17@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.")
18@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
19@cli.subcommand('Compiles the build targets specified in userspace `qmk.json`.')
20def userspace_compile(cli):
21 if not HAS_QMK_USERSPACE:
22 cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
23 return False
24
25 userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
26
27 build_targets = []
28 keyboard_keymap_targets = []
29 for e in userspace.build_targets:
30 if isinstance(e, Path):
31 build_targets.append(JsonKeymapBuildTarget(e))
32 elif isinstance(e, dict):
33 keyboard_keymap_targets.append((e['keyboard'], e['keymap']))
34
35 if len(keyboard_keymap_targets) > 0:
36 build_targets.extend(search_keymap_targets(keyboard_keymap_targets))
37
38 mass_compile_targets(list(set(build_targets)), cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, **build_environment(cli.args.env))
diff --git a/lib/python/qmk/cli/userspace/doctor.py b/lib/python/qmk/cli/userspace/doctor.py
new file mode 100644
index 0000000000..2b7e29aa7e
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/doctor.py
@@ -0,0 +1,11 @@
1# Copyright 2023 Nick Brassel (@tzarc)
2# SPDX-License-Identifier: GPL-2.0-or-later
3from milc import cli
4
5from qmk.constants import QMK_FIRMWARE
6from qmk.cli.doctor.main import userspace_tests
7
8
9@cli.subcommand('Checks userspace configuration.')
10def userspace_doctor(cli):
11 userspace_tests(QMK_FIRMWARE)
diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py
new file mode 100644
index 0000000000..a63f669dd7
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/list.py
@@ -0,0 +1,51 @@
1# Copyright 2023 Nick Brassel (@tzarc)
2# SPDX-License-Identifier: GPL-2.0-or-later
3from pathlib import Path
4from dotty_dict import Dotty
5from milc import cli
6
7from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
8from qmk.userspace import UserspaceDefs
9from qmk.build_targets import BuildTarget
10from qmk.keyboard import is_all_keyboards, keyboard_folder
11from qmk.keymap import is_keymap_target
12from qmk.search import search_keymap_targets
13
14
15@cli.argument('-e', '--expand', arg_only=True, action='store_true', help="Expands any use of `all` for either keyboard or keymap.")
16@cli.subcommand('Lists the build targets specified in userspace `qmk.json`.')
17def userspace_list(cli):
18 if not HAS_QMK_USERSPACE:
19 cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
20 return False
21
22 userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
23
24 if cli.args.expand:
25 build_targets = []
26 for e in userspace.build_targets:
27 if isinstance(e, Path):
28 build_targets.append(e)
29 elif isinstance(e, dict) or isinstance(e, Dotty):
30 build_targets.extend(search_keymap_targets([(e['keyboard'], e['keymap'])]))
31 else:
32 build_targets = userspace.build_targets
33
34 for e in build_targets:
35 if isinstance(e, Path):
36 # JSON keymap from userspace
37 cli.log.info(f'JSON keymap: {{fg_cyan}}{e}{{fg_reset}}')
38 continue
39 elif isinstance(e, dict) or isinstance(e, Dotty):
40 # keyboard/keymap dict from userspace
41 keyboard = e['keyboard']
42 keymap = e['keymap']
43 elif isinstance(e, BuildTarget):
44 # BuildTarget from search_keymap_targets()
45 keyboard = e.keyboard
46 keymap = e.keymap
47
48 if is_all_keyboards(keyboard) or is_keymap_target(keyboard_folder(keyboard), keymap):
49 cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}')
50 else:
51 cli.log.warn(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}} -- not found!')
diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py
new file mode 100644
index 0000000000..c7d180bfd1
--- /dev/null
+++ b/lib/python/qmk/cli/userspace/remove.py
@@ -0,0 +1,37 @@
1# Copyright 2023 Nick Brassel (@tzarc)
2# SPDX-License-Identifier: GPL-2.0-or-later
3from pathlib import Path
4from milc import cli
5
6from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
7from qmk.keyboard import keyboard_completer, keyboard_folder_or_all
8from qmk.keymap import keymap_completer
9from qmk.userspace import UserspaceDefs
10
11
12@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.")
13@cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
14@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
15@cli.subcommand('Removes a build target from userspace `qmk.json`.')
16def userspace_remove(cli):
17 if not HAS_QMK_USERSPACE:
18 cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
19 return False
20
21 userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
22
23 if len(cli.args.builds) > 0:
24 json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
25 make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
26
27 for e in json_like_targets:
28 userspace.remove_target(json_path=e)
29
30 for e in make_like_targets:
31 s = e.split(':')
32 userspace.remove_target(keyboard=s[0], keymap=s[1])
33
34 else:
35 userspace.remove_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
36
37 return userspace.save()
diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
index 519cb4c708..d95ff5f923 100644
--- a/lib/python/qmk/commands.py
+++ b/lib/python/qmk/commands.py
@@ -3,10 +3,12 @@
3import os 3import os
4import sys 4import sys
5import shutil 5import shutil
6from pathlib import Path
6 7
7from milc import cli 8from milc import cli
8import jsonschema 9import jsonschema
9 10
11from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
10from qmk.json_schema import json_load, validate 12from qmk.json_schema import json_load, validate
11from qmk.keyboard import keyboard_alias_definitions 13from qmk.keyboard import keyboard_alias_definitions
12 14
@@ -75,6 +77,10 @@ def build_environment(args):
75 envs[key] = value 77 envs[key] = value
76 else: 78 else:
77 cli.log.warning('Invalid environment variable: %s', env) 79 cli.log.warning('Invalid environment variable: %s', env)
80
81 if HAS_QMK_USERSPACE:
82 envs['QMK_USERSPACE'] = Path(QMK_USERSPACE).resolve()
83
78 return envs 84 return envs
79 85
80 86
diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py
index 1967441fc8..90e4452f2b 100644
--- a/lib/python/qmk/constants.py
+++ b/lib/python/qmk/constants.py
@@ -4,9 +4,17 @@ from os import environ
4from datetime import date 4from datetime import date
5from pathlib import Path 5from pathlib import Path
6 6
7from qmk.userspace import detect_qmk_userspace
8
7# The root of the qmk_firmware tree. 9# The root of the qmk_firmware tree.
8QMK_FIRMWARE = Path.cwd() 10QMK_FIRMWARE = Path.cwd()
9 11
12# The detected userspace tree
13QMK_USERSPACE = detect_qmk_userspace()
14
15# Whether or not we have a separate userspace directory
16HAS_QMK_USERSPACE = True if QMK_USERSPACE is not None else False
17
10# Upstream repo url 18# Upstream repo url
11QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware' 19QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware'
12 20
diff --git a/lib/python/qmk/json_encoders.py b/lib/python/qmk/json_encoders.py
index 1e90f6a288..0e4ad1d220 100755
--- a/lib/python/qmk/json_encoders.py
+++ b/lib/python/qmk/json_encoders.py
@@ -217,3 +217,21 @@ class KeymapJSONEncoder(QMKJSONEncoder):
217 return '50' + str(key) 217 return '50' + str(key)
218 218
219 return key 219 return key
220
221
222class UserspaceJSONEncoder(QMKJSONEncoder):
223 """Custom encoder to make userspace qmk.json's a little nicer to work with.
224 """
225 def sort_dict(self, item):
226 """Sorts the hashes in a nice way.
227 """
228 key = item[0]
229
230 if self.indentation_level == 1:
231 if key == 'userspace_version':
232 return '00userspace_version'
233
234 if key == 'build_targets':
235 return '01build_targets'
236
237 return key
diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py
index 34257bee8d..b56505d649 100644
--- a/lib/python/qmk/keyboard.py
+++ b/lib/python/qmk/keyboard.py
@@ -78,13 +78,17 @@ def keyboard_alias_definitions():
78def is_all_keyboards(keyboard): 78def is_all_keyboards(keyboard):
79 """Returns True if the keyboard is an AllKeyboards object. 79 """Returns True if the keyboard is an AllKeyboards object.
80 """ 80 """
81 if isinstance(keyboard, str):
82 return (keyboard == 'all')
81 return isinstance(keyboard, AllKeyboards) 83 return isinstance(keyboard, AllKeyboards)
82 84
83 85
84def find_keyboard_from_dir(): 86def find_keyboard_from_dir():
85 """Returns a keyboard name based on the user's current directory. 87 """Returns a keyboard name based on the user's current directory.
86 """ 88 """
87 relative_cwd = qmk.path.under_qmk_firmware() 89 relative_cwd = qmk.path.under_qmk_userspace()
90 if not relative_cwd:
91 relative_cwd = qmk.path.under_qmk_firmware()
88 92
89 if relative_cwd and len(relative_cwd.parts) > 1 and relative_cwd.parts[0] == 'keyboards': 93 if relative_cwd and len(relative_cwd.parts) > 1 and relative_cwd.parts[0] == 'keyboards':
90 # Attempt to extract the keyboard name from the current directory 94 # Attempt to extract the keyboard name from the current directory
@@ -133,6 +137,22 @@ def keyboard_folder(keyboard):
133 return keyboard 137 return keyboard
134 138
135 139
140def keyboard_aliases(keyboard):
141 """Returns the list of aliases for the supplied keyboard.
142
143 Includes the keyboard itself.
144 """
145 aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
146
147 if keyboard in aliases:
148 keyboard = aliases[keyboard].get('target', keyboard)
149
150 keyboards = set(filter(lambda k: aliases[k].get('target', '') == keyboard, aliases.keys()))
151 keyboards.add(keyboard)
152 keyboards = list(sorted(keyboards))
153 return keyboards
154
155
136def keyboard_folder_or_all(keyboard): 156def keyboard_folder_or_all(keyboard):
137 """Returns the actual keyboard folder. 157 """Returns the actual keyboard folder.
138 158
diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
index 281c53cfda..b7bf897377 100644
--- a/lib/python/qmk/keymap.py
+++ b/lib/python/qmk/keymap.py
@@ -12,7 +12,8 @@ from pygments.token import Token
12from pygments import lex 12from pygments import lex
13 13
14import qmk.path 14import qmk.path
15from qmk.keyboard import find_keyboard_from_dir, keyboard_folder 15from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE
16from qmk.keyboard import find_keyboard_from_dir, keyboard_folder, keyboard_aliases
16from qmk.errors import CppError 17from qmk.errors import CppError
17from qmk.info import info_json 18from qmk.info import info_json
18 19
@@ -194,29 +195,38 @@ def _strip_any(keycode):
194def find_keymap_from_dir(*args): 195def find_keymap_from_dir(*args):
195 """Returns `(keymap_name, source)` for the directory provided (or cwd if not specified). 196 """Returns `(keymap_name, source)` for the directory provided (or cwd if not specified).
196 """ 197 """
197 relative_path = qmk.path.under_qmk_firmware(*args) 198 def _impl_find_keymap_from_dir(relative_path):
199 if relative_path and len(relative_path.parts) > 1:
200 # If we're in `qmk_firmware/keyboards` and `keymaps` is in our path, try to find the keyboard name.
201 if relative_path.parts[0] == 'keyboards' and 'keymaps' in relative_path.parts:
202 current_path = Path('/'.join(relative_path.parts[1:])) # Strip 'keyboards' from the front
198 203
199 if relative_path and len(relative_path.parts) > 1: 204 if 'keymaps' in current_path.parts and current_path.name != 'keymaps':
200 # If we're in `qmk_firmware/keyboards` and `keymaps` is in our path, try to find the keyboard name. 205 while current_path.parent.name != 'keymaps':
201 if relative_path.parts[0] == 'keyboards' and 'keymaps' in relative_path.parts: 206 current_path = current_path.parent
202 current_path = Path('/'.join(relative_path.parts[1:])) # Strip 'keyboards' from the front
203 207
204 if 'keymaps' in current_path.parts and current_path.name != 'keymaps': 208 return current_path.name, 'keymap_directory'
205 while current_path.parent.name != 'keymaps':
206 current_path = current_path.parent
207 209
208 return current_path.name, 'keymap_directory' 210 # If we're in `qmk_firmware/layouts` guess the name from the community keymap they're in
211 elif relative_path.parts[0] == 'layouts' and is_keymap_dir(relative_path):
212 return relative_path.name, 'layouts_directory'
209 213
210 # If we're in `qmk_firmware/layouts` guess the name from the community keymap they're in 214 # If we're in `qmk_firmware/users` guess the name from the userspace they're in
211 elif relative_path.parts[0] == 'layouts' and is_keymap_dir(relative_path): 215 elif relative_path.parts[0] == 'users':
212 return relative_path.name, 'layouts_directory' 216 # Guess the keymap name based on which userspace they're in
217 return relative_path.parts[1], 'users_directory'
218 return None, None
213 219
214 # If we're in `qmk_firmware/users` guess the name from the userspace they're in 220 if HAS_QMK_USERSPACE:
215 elif relative_path.parts[0] == 'users': 221 name, source = _impl_find_keymap_from_dir(qmk.path.under_qmk_userspace(*args))
216 # Guess the keymap name based on which userspace they're in 222 if name and source:
217 return relative_path.parts[1], 'users_directory' 223 return name, source
218 224
219 return None, None 225 name, source = _impl_find_keymap_from_dir(qmk.path.under_qmk_firmware(*args))
226 if name and source:
227 return name, source
228
229 return (None, None)
220 230
221 231
222def keymap_completer(prefix, action, parser, parsed_args): 232def keymap_completer(prefix, action, parser, parsed_args):
@@ -417,29 +427,45 @@ def locate_keymap(keyboard, keymap):
417 raise KeyError('Invalid keyboard: ' + repr(keyboard)) 427 raise KeyError('Invalid keyboard: ' + repr(keyboard))
418 428
419 # Check the keyboard folder first, last match wins 429 # Check the keyboard folder first, last match wins
420 checked_dirs = ''
421 keymap_path = '' 430 keymap_path = ''
422 431
423 for dir in keyboard_folder(keyboard).split('/'): 432 search_dirs = [QMK_FIRMWARE]
424 if checked_dirs: 433 keyboard_dirs = [keyboard_folder(keyboard)]
425 checked_dirs = '/'.join((checked_dirs, dir)) 434 if HAS_QMK_USERSPACE:
426 else: 435 # When we've got userspace, check there _last_ as we want them to override anything in the main repo.
427 checked_dirs = dir 436 search_dirs.append(QMK_USERSPACE)
437 # We also want to search for any aliases as QMK's folder structure may have changed, with an alias, but the user
438 # hasn't updated their keymap location yet.
439 keyboard_dirs.extend(keyboard_aliases(keyboard))
440 keyboard_dirs = list(set(keyboard_dirs))
441
442 for search_dir in search_dirs:
443 for keyboard_dir in keyboard_dirs:
444 checked_dirs = ''
445 for dir in keyboard_dir.split('/'):
446 if checked_dirs:
447 checked_dirs = '/'.join((checked_dirs, dir))
448 else:
449 checked_dirs = dir
428 450
429 keymap_dir = Path('keyboards') / checked_dirs / 'keymaps' 451 keymap_dir = Path(search_dir) / Path('keyboards') / checked_dirs / 'keymaps'
430 452
431 if (keymap_dir / keymap / 'keymap.c').exists(): 453 if (keymap_dir / keymap / 'keymap.c').exists():
432 keymap_path = keymap_dir / keymap / 'keymap.c' 454 keymap_path = keymap_dir / keymap / 'keymap.c'
433 if (keymap_dir / keymap / 'keymap.json').exists(): 455 if (keymap_dir / keymap / 'keymap.json').exists():
434 keymap_path = keymap_dir / keymap / 'keymap.json' 456 keymap_path = keymap_dir / keymap / 'keymap.json'
435 457
436 if keymap_path: 458 if keymap_path:
437 return keymap_path 459 return keymap_path
438 460
439 # Check community layouts as a fallback 461 # Check community layouts as a fallback
440 info = info_json(keyboard) 462 info = info_json(keyboard)
441 463
442 for community_parent in Path('layouts').glob('*/'): 464 community_parents = list(Path('layouts').glob('*/'))
465 if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists():
466 community_parents.append(Path(QMK_USERSPACE) / "layouts")
467
468 for community_parent in community_parents:
443 for layout in info.get("community_layouts", []): 469 for layout in info.get("community_layouts", []):
444 community_layout = community_parent / layout / keymap 470 community_layout = community_parent / layout / keymap
445 if community_layout.exists(): 471 if community_layout.exists():
@@ -449,6 +475,16 @@ def locate_keymap(keyboard, keymap):
449 return community_layout / 'keymap.c' 475 return community_layout / 'keymap.c'
450 476
451 477
478def is_keymap_target(keyboard, keymap):
479 if keymap == 'all':
480 return True
481
482 if locate_keymap(keyboard, keymap):
483 return True
484
485 return False
486
487
452def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=False): 488def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=False):
453 """List the available keymaps for a keyboard. 489 """List the available keymaps for a keyboard.
454 490
@@ -473,26 +509,30 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa
473 """ 509 """
474 names = set() 510 names = set()
475 511
476 keyboards_dir = Path('keyboards')
477 kb_path = keyboards_dir / keyboard
478
479 # walk up the directory tree until keyboards_dir 512 # walk up the directory tree until keyboards_dir
480 # and collect all directories' name with keymap.c file in it 513 # and collect all directories' name with keymap.c file in it
481 while kb_path != keyboards_dir: 514 for search_dir in [QMK_FIRMWARE, QMK_USERSPACE] if HAS_QMK_USERSPACE else [QMK_FIRMWARE]:
482 keymaps_dir = kb_path / "keymaps" 515 keyboards_dir = search_dir / Path('keyboards')
483 516 kb_path = keyboards_dir / keyboard
484 if keymaps_dir.is_dir(): 517
485 for keymap in keymaps_dir.iterdir(): 518 while kb_path != keyboards_dir:
486 if is_keymap_dir(keymap, c, json, additional_files): 519 keymaps_dir = kb_path / "keymaps"
487 keymap = keymap if fullpath else keymap.name 520 if keymaps_dir.is_dir():
488 names.add(keymap) 521 for keymap in keymaps_dir.iterdir():
522 if is_keymap_dir(keymap, c, json, additional_files):
523 keymap = keymap if fullpath else keymap.name
524 names.add(keymap)
489 525
490 kb_path = kb_path.parent 526 kb_path = kb_path.parent
491 527
492 # Check community layouts as a fallback 528 # Check community layouts as a fallback
493 info = info_json(keyboard) 529 info = info_json(keyboard)
494 530
495 for community_parent in Path('layouts').glob('*/'): 531 community_parents = list(Path('layouts').glob('*/'))
532 if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists():
533 community_parents.append(Path(QMK_USERSPACE) / "layouts")
534
535 for community_parent in community_parents:
496 for layout in info.get("community_layouts", []): 536 for layout in info.get("community_layouts", []):
497 cl_path = community_parent / layout 537 cl_path = community_parent / layout
498 if cl_path.is_dir(): 538 if cl_path.is_dir():
diff --git a/lib/python/qmk/path.py b/lib/python/qmk/path.py
index 94582a05e0..74364ee04b 100644
--- a/lib/python/qmk/path.py
+++ b/lib/python/qmk/path.py
@@ -5,7 +5,7 @@ import os
5import argparse 5import argparse
6from pathlib import Path 6from pathlib import Path
7 7
8from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE 8from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE
9from qmk.errors import NoSuchKeyboardError 9from qmk.errors import NoSuchKeyboardError
10 10
11 11
@@ -28,6 +28,40 @@ def under_qmk_firmware(path=Path(os.environ['ORIG_CWD'])):
28 return None 28 return None
29 29
30 30
31def under_qmk_userspace(path=Path(os.environ['ORIG_CWD'])):
32 """Returns a Path object representing the relative path under $QMK_USERSPACE, or None.
33 """
34 try:
35 if HAS_QMK_USERSPACE:
36 return path.relative_to(QMK_USERSPACE)
37 except ValueError:
38 pass
39 return None
40
41
42def is_under_qmk_firmware(path=Path(os.environ['ORIG_CWD'])):
43 """Returns a boolean if the input path is a child under qmk_firmware.
44 """
45 if path is None:
46 return False
47 try:
48 return Path(os.path.commonpath([Path(path), QMK_FIRMWARE])) == QMK_FIRMWARE
49 except ValueError:
50 return False
51
52
53def is_under_qmk_userspace(path=Path(os.environ['ORIG_CWD'])):
54 """Returns a boolean if the input path is a child under $QMK_USERSPACE.
55 """
56 if path is None:
57 return False
58 try:
59 if HAS_QMK_USERSPACE:
60 return Path(os.path.commonpath([Path(path), QMK_USERSPACE])) == QMK_USERSPACE
61 except ValueError:
62 return False
63
64
31def keyboard(keyboard_name): 65def keyboard(keyboard_name):
32 """Returns the path to a keyboard's directory relative to the qmk root. 66 """Returns the path to a keyboard's directory relative to the qmk root.
33 """ 67 """
@@ -45,11 +79,28 @@ def keymaps(keyboard_name):
45 keyboard_folder = keyboard(keyboard_name) 79 keyboard_folder = keyboard(keyboard_name)
46 found_dirs = [] 80 found_dirs = []
47 81
82 if HAS_QMK_USERSPACE:
83 this_keyboard_folder = Path(QMK_USERSPACE) / keyboard_folder
84 for _ in range(MAX_KEYBOARD_SUBFOLDERS):
85 if (this_keyboard_folder / 'keymaps').exists():
86 found_dirs.append((this_keyboard_folder / 'keymaps').resolve())
87
88 this_keyboard_folder = this_keyboard_folder.parent
89 if this_keyboard_folder.resolve() == QMK_USERSPACE.resolve():
90 break
91
92 # We don't have any relevant keymap directories in userspace, so we'll use the fully-qualified path instead.
93 if len(found_dirs) == 0:
94 found_dirs.append((QMK_USERSPACE / keyboard_folder / 'keymaps').resolve())
95
96 this_keyboard_folder = QMK_FIRMWARE / keyboard_folder
48 for _ in range(MAX_KEYBOARD_SUBFOLDERS): 97 for _ in range(MAX_KEYBOARD_SUBFOLDERS):
49 if (keyboard_folder / 'keymaps').exists(): 98 if (this_keyboard_folder / 'keymaps').exists():
50 found_dirs.append((keyboard_folder / 'keymaps').resolve()) 99 found_dirs.append((this_keyboard_folder / 'keymaps').resolve())
51 100
52 keyboard_folder = keyboard_folder.parent 101 this_keyboard_folder = this_keyboard_folder.parent
102 if this_keyboard_folder.resolve() == QMK_FIRMWARE.resolve():
103 break
53 104
54 if len(found_dirs) > 0: 105 if len(found_dirs) > 0:
55 return found_dirs 106 return found_dirs
diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
new file mode 100644
index 0000000000..3783568006
--- /dev/null
+++ b/lib/python/qmk/userspace.py
@@ -0,0 +1,185 @@
1# Copyright 2023 Nick Brassel (@tzarc)
2# SPDX-License-Identifier: GPL-2.0-or-later
3from os import environ
4from pathlib import Path
5import json
6import jsonschema
7
8from milc import cli
9
10from qmk.json_schema import validate, json_load
11from qmk.json_encoders import UserspaceJSONEncoder
12
13
14def qmk_userspace_paths():
15 test_dirs = []
16
17 # If we're already in a directory with a qmk.json and a keyboards or layouts directory, interpret it as userspace
18 current_dir = Path(environ['ORIG_CWD'])
19 while len(current_dir.parts) > 1:
20 if (current_dir / 'qmk.json').is_file():
21 test_dirs.append(current_dir)
22 current_dir = current_dir.parent
23
24 # If we have a QMK_USERSPACE environment variable, use that
25 if environ.get('QMK_USERSPACE') is not None:
26 current_dir = Path(environ.get('QMK_USERSPACE'))
27 if current_dir.is_dir():
28 test_dirs.append(current_dir)
29
30 # If someone has configured a directory, use that
31 if cli.config.user.overlay_dir is not None:
32 current_dir = Path(cli.config.user.overlay_dir)
33 if current_dir.is_dir():
34 test_dirs.append(current_dir)
35
36 return test_dirs
37
38
39def qmk_userspace_validate(path):
40 # Construct a UserspaceDefs object to ensure it validates correctly
41 if (path / 'qmk.json').is_file():
42 UserspaceDefs(path / 'qmk.json')
43 return
44
45 # No qmk.json file found
46 raise FileNotFoundError('No qmk.json file found.')
47
48
49def detect_qmk_userspace():
50 # Iterate through all the detected userspace paths and return the first one that validates correctly
51 test_dirs = qmk_userspace_paths()
52 for test_dir in test_dirs:
53 try:
54 qmk_userspace_validate(test_dir)
55 return test_dir
56 except FileNotFoundError:
57 continue
58 except UserspaceValidationError:
59 continue
60 return None
61
62
63class UserspaceDefs:
64 def __init__(self, userspace_json: Path):
65 self.path = userspace_json
66 self.build_targets = []
67 json = json_load(userspace_json)
68
69 exception = UserspaceValidationError()
70 success = False
71
72 try:
73 validate(json, 'qmk.user_repo.v0') # `qmk.json` must have a userspace_version at minimum
74 except jsonschema.ValidationError as err:
75 exception.add('qmk.user_repo.v0', err)
76 raise exception
77
78 # Iterate through each version of the schema, starting with the latest and decreasing to v1
79 try:
80 validate(json, 'qmk.user_repo.v1')
81 self.__load_v1(json)
82 success = True
83 except jsonschema.ValidationError as err:
84 exception.add('qmk.user_repo.v1', err)
85
86 if not success:
87 raise exception
88
89 def save(self):
90 target_json = {
91 "userspace_version": "1.0", # Needs to match latest version
92 "build_targets": []
93 }
94
95 for e in self.build_targets:
96 if isinstance(e, dict):
97 target_json['build_targets'].append([e['keyboard'], e['keymap']])
98 elif isinstance(e, Path):
99 target_json['build_targets'].append(str(e.relative_to(self.path.parent)))
100
101 try:
102 # Ensure what we're writing validates against the latest version of the schema
103 validate(target_json, 'qmk.user_repo.v1')
104 except jsonschema.ValidationError as err:
105 cli.log.error(f'Could not save userspace file: {err}')
106 return False
107
108 # Only actually write out data if it changed
109 old_data = json.dumps(json.loads(self.path.read_text()), cls=UserspaceJSONEncoder, sort_keys=True)
110 new_data = json.dumps(target_json, cls=UserspaceJSONEncoder, sort_keys=True)
111 if old_data != new_data:
112 self.path.write_text(new_data)
113 cli.log.info(f'Saved userspace file to {self.path}.')
114 return True
115
116 def add_target(self, keyboard=None, keymap=None, json_path=None, do_print=True):
117 if json_path is not None:
118 # Assume we're adding a json filename/path
119 json_path = Path(json_path)
120 if json_path not in self.build_targets:
121 self.build_targets.append(json_path)
122 if do_print:
123 cli.log.info(f'Added {json_path} to userspace build targets.')
124 else:
125 cli.log.info(f'{json_path} is already a userspace build target.')
126
127 elif keyboard is not None and keymap is not None:
128 # Both keyboard/keymap specified
129 e = {"keyboard": keyboard, "keymap": keymap}
130 if e not in self.build_targets:
131 self.build_targets.append(e)
132 if do_print:
133 cli.log.info(f'Added {keyboard}:{keymap} to userspace build targets.')
134 else:
135 if do_print:
136 cli.log.info(f'{keyboard}:{keymap} is already a userspace build target.')
137
138 def remove_target(self, keyboard=None, keymap=None, json_path=None, do_print=True):
139 if json_path is not None:
140 # Assume we're removing a json filename/path
141 json_path = Path(json_path)
142 if json_path in self.build_targets:
143 self.build_targets.remove(json_path)
144 if do_print:
145 cli.log.info(f'Removed {json_path} from userspace build targets.')
146 else:
147 cli.log.info(f'{json_path} is not a userspace build target.')
148
149 elif keyboard is not None and keymap is not None:
150 # Both keyboard/keymap specified
151 e = {"keyboard": keyboard, "keymap": keymap}
152 if e in self.build_targets:
153 self.build_targets.remove(e)
154 if do_print:
155 cli.log.info(f'Removed {keyboard}:{keymap} from userspace build targets.')
156 else:
157 if do_print:
158 cli.log.info(f'{keyboard}:{keymap} is not a userspace build target.')
159
160 def __load_v1(self, json):
161 for e in json['build_targets']:
162 if isinstance(e, list) and len(e) == 2:
163 self.add_target(keyboard=e[0], keymap=e[1], do_print=False)
164 if isinstance(e, str):
165 p = self.path.parent / e
166 if p.exists() and p.suffix == '.json':
167 self.add_target(json_path=p, do_print=False)
168
169
170class UserspaceValidationError(Exception):
171 def __init__(self, *args, **kwargs):
172 super().__init__(*args, **kwargs)
173 self.__exceptions = []
174
175 def __str__(self):
176 return self.message
177
178 @property
179 def exceptions(self):
180 return self.__exceptions
181
182 def add(self, schema, exception):
183 self.__exceptions.append((schema, exception))
184 errorlist = "\n\n".join([f"{schema}: {exception}" for schema, exception in self.__exceptions])
185 self.message = f'Could not validate against any version of the userspace schema. Errors:\n\n{errorlist}'