wear_leveling_rp2040_flash.c (8609B)
1 /** 2 * Copyright (c) 2020 Raspberry Pi (Trading) Ltd. 3 * Copyright (c) 2022 Nick Brassel (@tzarc) 4 * Copyright (c) 2022 Stefan Kerkmann (@KarlK90) 5 * 6 * SPDX-License-Identifier: BSD-3-Clause 7 */ 8 9 #include <stdbool.h> 10 11 #include "pico/bootrom.h" 12 #include "hardware/flash.h" 13 #include "hardware/sync.h" 14 #include "hardware/structs/ssi.h" 15 #include "hardware/structs/ioqspi.h" 16 17 #include "compiler_support.h" 18 #include "timer.h" 19 #include "wear_leveling.h" 20 #include "wear_leveling_rp2040_flash_config.h" 21 #include "wear_leveling_internal.h" 22 23 #ifndef WEAR_LEVELING_RP2040_FLASH_BULK_COUNT 24 # define WEAR_LEVELING_RP2040_FLASH_BULK_COUNT 64 25 #endif // WEAR_LEVELING_RP2040_FLASH_BULK_COUNT 26 27 #define FLASHCMD_PAGE_PROGRAM 0x02 28 #define FLASHCMD_READ_STATUS 0x05 29 #define FLASHCMD_WRITE_ENABLE 0x06 30 31 extern const uint8_t BOOT2_ROM[256]; 32 static uint32_t BOOT2_ROM_RAM[64]; 33 34 static ssi_hw_t *const ssi = (ssi_hw_t *)XIP_SSI_BASE; 35 36 // Sanity check 37 check_hw_layout(ssi_hw_t, ssienr, SSI_SSIENR_OFFSET); 38 check_hw_layout(ssi_hw_t, spi_ctrlr0, SSI_SPI_CTRLR0_OFFSET); 39 40 static void __no_inline_not_in_flash_func(flash_enable_xip_via_boot2)(void) { 41 ((void (*)(void))BOOT2_ROM_RAM + 1)(); 42 } 43 44 // Bitbanging the chip select using IO overrides, in case RAM-resident IRQs 45 // are still running, and the FIFO bottoms out. (the bootrom does the same) 46 static void __no_inline_not_in_flash_func(flash_cs_force)(bool high) { 47 uint32_t field_val = high ? IO_QSPI_GPIO_QSPI_SS_CTRL_OUTOVER_VALUE_HIGH : IO_QSPI_GPIO_QSPI_SS_CTRL_OUTOVER_VALUE_LOW; 48 hw_write_masked(&ioqspi_hw->io[1].ctrl, field_val << IO_QSPI_GPIO_QSPI_SS_CTRL_OUTOVER_LSB, IO_QSPI_GPIO_QSPI_SS_CTRL_OUTOVER_BITS); 49 } 50 51 // Also allow any unbounded loops to check whether the above abort condition 52 // was asserted, and terminate early 53 static int __no_inline_not_in_flash_func(flash_was_aborted)(void) { 54 return *(io_rw_32 *)(IO_QSPI_BASE + IO_QSPI_GPIO_QSPI_SD1_CTRL_OFFSET) & IO_QSPI_GPIO_QSPI_SD1_CTRL_INOVER_BITS; 55 } 56 57 // Put bytes from one buffer, and get bytes into another buffer. 58 // These can be the same buffer. 59 // If tx is NULL then send zeroes. 60 // If rx is NULL then all read data will be dropped. 61 // 62 // If rx_skip is nonzero, this many bytes will first be consumed from the FIFO, 63 // before reading a further count bytes into *rx. 64 // E.g. if you have written a command+address just before calling this function. 65 static void __no_inline_not_in_flash_func(flash_put_get)(const uint8_t *tx, uint8_t *rx, size_t count, size_t rx_skip) { 66 // Make sure there is never more data in flight than the depth of the RX 67 // FIFO. Otherwise, when we are interrupted for long periods, hardware 68 // will overflow the RX FIFO. 69 const uint max_in_flight = 16 - 2; // account for data internal to SSI 70 size_t tx_count = count; 71 size_t rx_count = count; 72 while (tx_count || rx_skip || rx_count) { 73 // NB order of reads, for pessimism rather than optimism 74 uint32_t tx_level = ssi_hw->txflr; 75 uint32_t rx_level = ssi_hw->rxflr; 76 bool did_something = false; // Expect this to be folded into control flow, not register 77 if (tx_count && tx_level + rx_level < max_in_flight) { 78 ssi->dr0 = (uint32_t)(tx ? *tx++ : 0); 79 --tx_count; 80 did_something = true; 81 } 82 if (rx_level) { 83 uint8_t rxbyte = ssi->dr0; 84 did_something = true; 85 if (rx_skip) { 86 --rx_skip; 87 } else { 88 if (rx) *rx++ = rxbyte; 89 --rx_count; 90 } 91 } 92 // APB load costs 4 cycles, so only do it on idle loops (our budget is 93 // 48 cyc/byte) 94 if (!did_something && __builtin_expect(flash_was_aborted(), 0)) break; 95 } 96 flash_cs_force(1); 97 } 98 99 // Convenience wrapper for above 100 // (And it's hard for the debug host to get the tight timing between 101 // cmd DR0 write and the remaining data) 102 static void __no_inline_not_in_flash_func(_flash_do_cmd)(uint8_t cmd, const uint8_t *tx, uint8_t *rx, size_t count) { 103 flash_cs_force(0); 104 ssi->dr0 = cmd; 105 flash_put_get(tx, rx, count, 1); 106 } 107 108 // Timing of this one is critical, so do not expose the symbol to debugger etc 109 static void __no_inline_not_in_flash_func(flash_put_cmd_addr)(uint8_t cmd, uint32_t addr) { 110 flash_cs_force(0); 111 addr |= cmd << 24; 112 for (int i = 0; i < 4; ++i) { 113 ssi->dr0 = addr >> 24; 114 addr <<= 8; 115 } 116 } 117 118 // Poll the flash status register until the busy bit (LSB) clears 119 static void __no_inline_not_in_flash_func(flash_wait_ready)(void) { 120 uint8_t stat; 121 do { 122 _flash_do_cmd(FLASHCMD_READ_STATUS, NULL, &stat, 1); 123 } while (stat & 0x1 && !flash_was_aborted()); 124 } 125 126 // Set the WEL bit (needed before any program/erase operation) 127 static void __no_inline_not_in_flash_func(flash_enable_write)(void) { 128 _flash_do_cmd(FLASHCMD_WRITE_ENABLE, NULL, NULL, 0); 129 } 130 131 static void __no_inline_not_in_flash_func(pico_program_bulk)(uint32_t flash_address, backing_store_int_t *values, size_t item_count) { 132 rom_connect_internal_flash_fn connect_internal_flash = (rom_connect_internal_flash_fn)rom_func_lookup_inline(ROM_FUNC_CONNECT_INTERNAL_FLASH); 133 rom_flash_exit_xip_fn flash_exit_xip = (rom_flash_exit_xip_fn)rom_func_lookup_inline(ROM_FUNC_FLASH_EXIT_XIP); 134 rom_flash_flush_cache_fn flash_flush_cache = (rom_flash_flush_cache_fn)rom_func_lookup_inline(ROM_FUNC_FLASH_FLUSH_CACHE); 135 assert(connect_internal_flash && flash_exit_xip && flash_flush_cache); 136 137 static backing_store_int_t bulk_write_buffer[WEAR_LEVELING_RP2040_FLASH_BULK_COUNT]; 138 139 while (item_count) { 140 size_t batch_size = MIN(item_count, WEAR_LEVELING_RP2040_FLASH_BULK_COUNT); 141 for (size_t i = 0; i < batch_size; i++, values++, item_count--) { 142 bulk_write_buffer[i] = ~(*values); 143 } 144 __compiler_memory_barrier(); 145 146 connect_internal_flash(); 147 flash_exit_xip(); 148 flash_enable_write(); 149 150 flash_put_cmd_addr(FLASHCMD_PAGE_PROGRAM, flash_address); 151 flash_put_get((uint8_t *)bulk_write_buffer, NULL, batch_size * sizeof(backing_store_int_t), 4); 152 flash_wait_ready(); 153 flash_address += batch_size * sizeof(backing_store_int_t); 154 155 flash_flush_cache(); 156 flash_enable_xip_via_boot2(); 157 } 158 } 159 160 //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// 161 // QMK Wear-Leveling Backing Store implementation 162 163 static int interrupts; 164 165 bool backing_store_init(void) { 166 bs_dprintf("Init\n"); 167 memcpy(BOOT2_ROM_RAM, BOOT2_ROM, sizeof(BOOT2_ROM)); 168 __compiler_memory_barrier(); 169 return true; 170 } 171 172 bool backing_store_unlock(void) { 173 bs_dprintf("Unlock\n"); 174 return true; 175 } 176 177 bool backing_store_erase(void) { 178 #ifdef WEAR_LEVELING_DEBUG_OUTPUT 179 uint32_t start = timer_read32(); 180 #endif 181 182 // Ensure the backing size can be cleanly subtracted from the flash size without alignment issues. 183 STATIC_ASSERT((WEAR_LEVELING_BACKING_SIZE) % (FLASH_SECTOR_SIZE) == 0, "Backing size must be a multiple of FLASH_SECTOR_SIZE"); 184 185 interrupts = save_and_disable_interrupts(); 186 flash_range_erase((WEAR_LEVELING_RP2040_FLASH_BASE), (WEAR_LEVELING_BACKING_SIZE)); 187 restore_interrupts(interrupts); 188 189 bs_dprintf("Backing store erase took %ldms to complete\n", ((long)(timer_read32() - start))); 190 return true; 191 } 192 193 bool backing_store_write(uint32_t address, backing_store_int_t value) { 194 return backing_store_write_bulk(address, &value, 1); 195 } 196 197 bool backing_store_write_bulk(uint32_t address, backing_store_int_t *values, size_t item_count) { 198 uint32_t offset = (WEAR_LEVELING_RP2040_FLASH_BASE) + address; 199 bs_dprintf("Write "); 200 wl_dump(offset, values, sizeof(backing_store_int_t) * item_count); 201 interrupts = save_and_disable_interrupts(); 202 pico_program_bulk(offset, values, item_count); 203 restore_interrupts(interrupts); 204 return true; 205 } 206 207 bool backing_store_lock(void) { 208 return true; 209 } 210 211 bool backing_store_read(uint32_t address, backing_store_int_t *value) { 212 return backing_store_read_bulk(address, value, 1); 213 } 214 215 bool backing_store_read_bulk(uint32_t address, backing_store_int_t *values, size_t item_count) { 216 uint32_t offset = (WEAR_LEVELING_RP2040_FLASH_BASE) + address; 217 backing_store_int_t *loc = (backing_store_int_t *)((XIP_BASE) + offset); 218 for (size_t i = 0; i < item_count; ++i) { 219 values[i] = ~loc[i]; 220 } 221 bs_dprintf("Read "); 222 wl_dump(offset, values, item_count * sizeof(backing_store_int_t)); 223 return true; 224 }