Unicode, Unicodemap and UCIS refactor (#21659)

13 files changed, 446 insertions, 2 deletions

diff --git a/tests/test_common/test_driver.cpp b/tests/test_common/test_driver.cpp
index f1c52cb7b6..0495da8205 100644
--- a/tests/test_common/test_driver.cpp
+++ b/tests/test_common/test_driver.cpp
@@ -60,7 +60,10 @@ void TestDriver::send_extra(report_extra_t* report) {
 namespace internal {
 void expect_unicode_code_point(TestDriver& driver, uint32_t code_point) {
     testing::InSequence seq;
-    EXPECT_REPORT(driver, (KC_LCTL, KC_LSFT, KC_U));
+    EXPECT_REPORT(driver, (KC_LEFT_CTRL, KC_LEFT_SHIFT));
+    EXPECT_REPORT(driver, (KC_LEFT_CTRL, KC_LEFT_SHIFT, KC_U));
+    EXPECT_REPORT(driver, (KC_LEFT_CTRL, KC_LEFT_SHIFT));
+    EXPECT_EMPTY_REPORT(driver);
 
     bool print_zero = false;
     for (int i = 7; i >= 0; --i) {
@@ -71,10 +74,12 @@ void expect_unicode_code_point(TestDriver& driver, uint32_t code_point) {
         const uint8_t digit = (code_point >> (i * 4)) & 0xf;
         if (digit || print_zero) {
             EXPECT_REPORT(driver, (hex_digit_to_keycode(digit)));
+            EXPECT_EMPTY_REPORT(driver);
             print_zero = true;
         }
     }
 
-    EXPECT_REPORT(driver, (KC_SPC));
+    EXPECT_REPORT(driver, (KC_SPACE));
+    EXPECT_EMPTY_REPORT(driver);
 }
 } // namespace internal
diff --git a/tests/unicode/config.h b/tests/unicode/config.h
new file mode 100644
index 0000000000..16f95f7f8d
--- /dev/null
+++ b/tests/unicode/config.h
@@ -0,0 +1,8 @@
+// Copyright 2023 QMK
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include "test_common.h"
+
+#define UNICODE_SELECTED_MODES UNICODE_MODE_LINUX, UNICODE_MODE_MACOS
diff --git a/tests/unicode/test.mk b/tests/unicode/test.mk
new file mode 100644
index 0000000000..1afc79be55
--- /dev/null
+++ b/tests/unicode/test.mk
@@ -0,0 +1,5 @@
+# --------------------------------------------------------------------------------
+# Keep this file, even if it is empty, as a marker that this folder contains tests
+# --------------------------------------------------------------------------------
+
+UNICODE_COMMON = yes
diff --git a/tests/unicode/test_unicode.cpp b/tests/unicode/test_unicode.cpp
new file mode 100644
index 0000000000..a8500e3ba1
--- /dev/null
+++ b/tests/unicode/test_unicode.cpp
@@ -0,0 +1,86 @@
+// Copyright 2023 QMK
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "keyboard_report_util.hpp"
+#include "keycode.h"
+#include "test_common.hpp"
+#include "test_keymap_key.hpp"
+
+using testing::_;
+
+class Unicode : public TestFixture {};
+
+TEST_F(Unicode, sends_bmp_unicode_sequence) {
+    TestDriver driver;
+
+    set_unicode_input_mode(UNICODE_MODE_LINUX);
+
+    EXPECT_UNICODE(driver, 0x03A8); // Ψ
+    register_unicode(0x03A8);
+
+    VERIFY_AND_CLEAR(driver);
+}
+
+TEST_F(Unicode, sends_smp_unicode_sequence) {
+    TestDriver driver;
+
+    set_unicode_input_mode(UNICODE_MODE_LINUX);
+
+    EXPECT_UNICODE(driver, 0x1F9D9); // 🧙
+    register_unicode(0x1F9D9);
+
+    VERIFY_AND_CLEAR(driver);
+}
+
+TEST_F(Unicode, sends_surrogate_pair_for_macos) {
+    TestDriver driver;
+
+    set_unicode_input_mode(UNICODE_MODE_MACOS);
+
+    // EXPECT_UNICODE() assumes Linux input mode
+    {
+        testing::InSequence s;
+
+        // Alt+D83EDDD9 🧙
+        EXPECT_REPORT(driver, (KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_D, KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_8, KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_3, KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_E, KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_D, KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_D, KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_D, KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_9, KC_LEFT_ALT));
+        EXPECT_REPORT(driver, (KC_LEFT_ALT));
+        EXPECT_EMPTY_REPORT(driver);
+    }
+
+    register_unicode(0x1F9D9);
+
+    VERIFY_AND_CLEAR(driver);
+}
+
+TEST_F(Unicode, sends_unicode_string) {
+    TestDriver driver;
+
+    set_unicode_input_mode(UNICODE_MODE_LINUX);
+
+    {
+        testing::InSequence s;
+
+        EXPECT_UNICODE(driver, 0xFF31);
+        EXPECT_UNICODE(driver, 0xFF2D);
+        EXPECT_UNICODE(driver, 0xFF2B);
+        EXPECT_UNICODE(driver, 0xFF01);
+    }
+    send_unicode_string("ＱＭＫ！");
+
+    VERIFY_AND_CLEAR(driver);
+}
diff --git a/tests/unicode/unicode_basic/config.h b/tests/unicode/unicode_basic/config.h
new file mode 100644
index 0000000000..16f95f7f8d
--- /dev/null
+++ b/tests/unicode/unicode_basic/config.h
@@ -0,0 +1,8 @@
+// Copyright 2023 QMK
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include "test_common.h"
+
+#define UNICODE_SELECTED_MODES UNICODE_MODE_LINUX, UNICODE_MODE_MACOS
diff --git a/tests/unicode/unicode_basic/test.mk b/tests/unicode/unicode_basic/test.mk
new file mode 100644
index 0000000000..f53051dce6
--- /dev/null
+++ b/tests/unicode/unicode_basic/test.mk
@@ -0,0 +1,5 @@
+# --------------------------------------------------------------------------------
+# Keep this file, even if it is empty, as a marker that this folder contains tests
+# --------------------------------------------------------------------------------
+
+UNICODE_ENABLE = yes
diff --git a/tests/unicode/unicode_basic/test_unicode_basic.cpp b/tests/unicode/unicode_basic/test_unicode_basic.cpp
new file mode 100644
index 0000000000..598b57277f
--- /dev/null
+++ b/tests/unicode/unicode_basic/test_unicode_basic.cpp
@@ -0,0 +1,26 @@
+// Copyright 2023 QMK
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "keyboard_report_util.hpp"
+#include "keycode.h"
+#include "test_common.hpp"
+#include "test_keymap_key.hpp"
+
+using testing::_;
+
+class UnicodeBasic : public TestFixture {};
+
+TEST_F(UnicodeBasic, sends_unicode_sequence) {
+    TestDriver driver;
+
+    set_unicode_input_mode(UNICODE_MODE_LINUX);
+
+    auto key_uc = KeymapKey(0, 0, 0, UC(0x03A8)); // Ψ
+
+    set_keymap({key_uc});
+
+    EXPECT_UNICODE(driver, 0x03A8);
+    tap_key(key_uc);
+
+    VERIFY_AND_CLEAR(driver);
+}
diff --git a/tests/unicode/unicode_map/config.h b/tests/unicode/unicode_map/config.h
new file mode 100644
index 0000000000..0d86922f0d
--- /dev/null
+++ b/tests/unicode/unicode_map/config.h
@@ -0,0 +1,8 @@
+// Copyright 2023 QMK
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include "test_common.h"
+
+#define UNICODE_SELECTED_MODES UNICODE_MODE_LINUX
diff --git a/tests/unicode/unicode_map/test.mk b/tests/unicode/unicode_map/test.mk
new file mode 100644
index 0000000000..e6d0034799
--- /dev/null
+++ b/tests/unicode/unicode_map/test.mk
@@ -0,0 +1,5 @@
+# --------------------------------------------------------------------------------
+# Keep this file, even if it is empty, as a marker that this folder contains tests
+# --------------------------------------------------------------------------------
+
+UNICODEMAP_ENABLE = yes
diff --git a/tests/unicode/unicode_map/test_unicode_map.cpp b/tests/unicode/unicode_map/test_unicode_map.cpp
new file mode 100644
index 0000000000..cacd8f3c75
--- /dev/null
+++ b/tests/unicode/unicode_map/test_unicode_map.cpp
@@ -0,0 +1,54 @@
+// Copyright 2023 QMK
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "keyboard_report_util.hpp"
+#include "keycode.h"
+#include "test_common.hpp"
+#include "test_keymap_key.hpp"
+
+using testing::_;
+
+const uint32_t PROGMEM unicode_map[] = {
+    0x03A8, // Ψ
+    0x2318  // ⌘
+};
+
+class UnicodeMap : public TestFixture {};
+
+TEST_F(UnicodeMap, sends_unicodemap_code_point_from_keycode) {
+    TestDriver driver;
+
+    auto key_um = KeymapKey(0, 0, 0, UM(0));
+
+    set_keymap({key_um});
+
+    EXPECT_UNICODE(driver, 0x03A8);
+    tap_key(key_um);
+
+    VERIFY_AND_CLEAR(driver);
+}
+
+TEST_F(UnicodeMap, sends_unicodemap_pair_from_keycode) {
+    TestDriver driver;
+
+    auto key_shift = KeymapKey(0, 0, 0, KC_LEFT_SHIFT);
+    auto key_up    = KeymapKey(0, 1, 0, UP(0, 1));
+
+    set_keymap({key_shift, key_up});
+
+    EXPECT_UNICODE(driver, 0x03A8);
+    tap_key(key_up);
+
+    EXPECT_REPORT(driver, (KC_LEFT_SHIFT));
+    key_shift.press();
+    run_one_scan_loop();
+
+    EXPECT_UNICODE(driver, 0x2318);
+    tap_key(key_up);
+
+    EXPECT_NO_REPORT(driver);
+    key_shift.release();
+    run_one_scan_loop();
+
+    VERIFY_AND_CLEAR(driver);
+}
diff --git a/tests/unicode/unicode_ucis/config.h b/tests/unicode/unicode_ucis/config.h
new file mode 100644
index 0000000000..0d86922f0d
--- /dev/null
+++ b/tests/unicode/unicode_ucis/config.h
@@ -0,0 +1,8 @@
+// Copyright 2023 QMK
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include "test_common.h"
+
+#define UNICODE_SELECTED_MODES UNICODE_MODE_LINUX
diff --git a/tests/unicode/unicode_ucis/test.mk b/tests/unicode/unicode_ucis/test.mk
new file mode 100644
index 0000000000..d7ac651758
--- /dev/null
+++ b/tests/unicode/unicode_ucis/test.mk
@@ -0,0 +1,5 @@
+# --------------------------------------------------------------------------------
+# Keep this file, even if it is empty, as a marker that this folder contains tests
+# --------------------------------------------------------------------------------
+
+UCIS_ENABLE = yes
diff --git a/tests/unicode/unicode_ucis/test_unicode_ucis.cpp b/tests/unicode/unicode_ucis/test_unicode_ucis.cpp
new file mode 100644
index 0000000000..acc6329f85
--- /dev/null
+++ b/tests/unicode/unicode_ucis/test_unicode_ucis.cpp
@@ -0,0 +1,221 @@
+// Copyright 2023 QMK
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "keyboard_report_util.hpp"
+#include "keycode.h"
+#include "test_common.hpp"
+#include "test_keymap_key.hpp"
+
+using testing::_;
+
+// clang-format off
+const ucis_symbol_t ucis_symbol_table[] = UCIS_TABLE(
+    UCIS_SYM("qmk", 0x03A8) // Ψ
+);
+// clang-format on
+
+class UnicodeUCIS : public TestFixture {};
+
+TEST_F(UnicodeUCIS, matches_sequence) {
+    TestDriver driver;
+
+    auto key_q     = KeymapKey(0, 0, 0, KC_Q);
+    auto key_m     = KeymapKey(0, 1, 0, KC_M);
+    auto key_k     = KeymapKey(0, 2, 0, KC_K);
+    auto key_enter = KeymapKey(0, 3, 0, KC_ENTER);
+
+    set_keymap({key_q, key_m, key_k, key_enter});
+
+    EXPECT_UNICODE(driver, 0x2328); // ⌨
+    ucis_start();
+
+    EXPECT_EQ(ucis_active(), true);
+    EXPECT_EQ(ucis_count(), 0);
+
+    EXPECT_REPORT(driver, (KC_Q));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_q);
+    EXPECT_EQ(ucis_count(), 1);
+
+    EXPECT_REPORT(driver, (KC_M));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_m);
+    EXPECT_EQ(ucis_count(), 2);
+
+    EXPECT_REPORT(driver, (KC_K));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_k);
+    EXPECT_EQ(ucis_count(), 3);
+
+    EXPECT_REPORT(driver, (KC_BACKSPACE)).Times(4);
+    EXPECT_EMPTY_REPORT(driver).Times(4);
+    EXPECT_UNICODE(driver, 0x03A8);
+    tap_key(key_enter);
+
+    EXPECT_EQ(ucis_active(), false);
+
+    VERIFY_AND_CLEAR(driver);
+}
+
+TEST_F(UnicodeUCIS, cancels_sequence) {
+    TestDriver driver;
+
+    auto key_q      = KeymapKey(0, 0, 0, KC_Q);
+    auto key_m      = KeymapKey(0, 1, 0, KC_M);
+    auto key_k      = KeymapKey(0, 2, 0, KC_K);
+    auto key_escape = KeymapKey(0, 3, 0, KC_ESCAPE);
+
+    set_keymap({key_q, key_m, key_k, key_escape});
+
+    EXPECT_UNICODE(driver, 0x2328); // ⌨
+    ucis_start();
+
+    EXPECT_EQ(ucis_active(), true);
+    EXPECT_EQ(ucis_count(), 0);
+
+    EXPECT_REPORT(driver, (KC_Q));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_q);
+    EXPECT_EQ(ucis_count(), 1);
+
+    EXPECT_REPORT(driver, (KC_M));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_m);
+    EXPECT_EQ(ucis_count(), 2);
+
+    EXPECT_REPORT(driver, (KC_K));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_k);
+    EXPECT_EQ(ucis_count(), 3);
+
+    EXPECT_NO_REPORT(driver);
+    tap_key(key_escape);
+
+    EXPECT_EQ(ucis_active(), false);
+
+    VERIFY_AND_CLEAR(driver);
+}
+
+TEST_F(UnicodeUCIS, matches_sequence_with_corrected_typo) {
+    TestDriver driver;
+
+    auto key_q         = KeymapKey(0, 0, 0, KC_Q);
+    auto key_m         = KeymapKey(0, 1, 0, KC_M);
+    auto key_j         = KeymapKey(0, 2, 0, KC_J);
+    auto key_k         = KeymapKey(0, 3, 0, KC_K);
+    auto key_backspace = KeymapKey(0, 4, 0, KC_BACKSPACE);
+    auto key_enter     = KeymapKey(0, 5, 0, KC_ENTER);
+
+    set_keymap({key_q, key_m, key_j, key_k, key_backspace, key_enter});
+
+    EXPECT_UNICODE(driver, 0x2328); // ⌨
+    ucis_start();
+
+    EXPECT_EQ(ucis_active(), true);
+    EXPECT_EQ(ucis_count(), 0);
+
+    EXPECT_REPORT(driver, (KC_Q));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_q);
+    EXPECT_EQ(ucis_count(), 1);
+
+    EXPECT_REPORT(driver, (KC_M));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_m);
+    EXPECT_EQ(ucis_count(), 2);
+
+    EXPECT_REPORT(driver, (KC_J));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_j);
+    EXPECT_EQ(ucis_count(), 3);
+
+    EXPECT_REPORT(driver, (KC_BACKSPACE));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_backspace);
+    EXPECT_EQ(ucis_count(), 2);
+
+    EXPECT_REPORT(driver, (KC_K));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_k);
+    EXPECT_EQ(ucis_count(), 3);
+
+    EXPECT_REPORT(driver, (KC_BACKSPACE)).Times(4);
+    EXPECT_EMPTY_REPORT(driver).Times(4);
+    EXPECT_UNICODE(driver, 0x03A8);
+    tap_key(key_enter);
+
+    EXPECT_EQ(ucis_active(), false);
+
+    VERIFY_AND_CLEAR(driver);
+}
+
+TEST_F(UnicodeUCIS, does_not_match_longer_sequence) {
+    TestDriver driver;
+
+    auto key_q     = KeymapKey(0, 0, 0, KC_Q);
+    auto key_m     = KeymapKey(0, 1, 0, KC_M);
+    auto key_k     = KeymapKey(0, 2, 0, KC_K);
+    auto key_enter = KeymapKey(0, 3, 0, KC_ENTER);
+
+    set_keymap({key_q, key_m, key_k, key_enter});
+
+    EXPECT_UNICODE(driver, 0x2328); // ⌨
+    ucis_start();
+
+    EXPECT_EQ(ucis_active(), true);
+    EXPECT_EQ(ucis_count(), 0);
+
+    EXPECT_REPORT(driver, (KC_Q));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_q);
+    EXPECT_EQ(ucis_count(), 1);
+
+    EXPECT_REPORT(driver, (KC_M));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_m);
+    EXPECT_EQ(ucis_count(), 2);
+
+    EXPECT_REPORT(driver, (KC_K));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_k);
+    EXPECT_EQ(ucis_count(), 3);
+
+    EXPECT_REPORT(driver, (KC_K));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_k);
+    EXPECT_EQ(ucis_count(), 4);
+
+    EXPECT_NO_REPORT(driver);
+    tap_key(key_enter);
+
+    EXPECT_EQ(ucis_active(), false);
+
+    VERIFY_AND_CLEAR(driver);
+}
+
+TEST_F(UnicodeUCIS, does_not_match_shorter_sequence) {
+    TestDriver driver;
+
+    auto key_q     = KeymapKey(0, 0, 0, KC_Q);
+    auto key_enter = KeymapKey(0, 1, 0, KC_ENTER);
+
+    set_keymap({key_q, key_enter});
+
+    EXPECT_UNICODE(driver, 0x2328); // ⌨
+    ucis_start();
+
+    EXPECT_EQ(ucis_active(), true);
+    EXPECT_EQ(ucis_count(), 0);
+
+    EXPECT_REPORT(driver, (KC_Q));
+    EXPECT_EMPTY_REPORT(driver);
+    tap_key(key_q);
+    EXPECT_EQ(ucis_count(), 1);
+
+    EXPECT_NO_REPORT(driver);
+    tap_key(key_enter);
+
+    EXPECT_EQ(ucis_active(), false);
+
+    VERIFY_AND_CLEAR(driver);
+}