From c445ec134b1997e62771e6808ba84c8353acb555 Mon Sep 17 00:00:00 2001 From: "radek.tandler" Date: Wed, 26 Jun 2024 11:50:14 +0200 Subject: [PATCH] feat(nvs_flash): Implemented basic nvs_flash support for bootloader --- components/esp_partition/CMakeLists.txt | 16 + .../esp_partition/partition_bootloader.c | 91 +++ components/nvs_flash/.build-test-rules.yml | 8 + components/nvs_flash/CMakeLists.txt | 95 +-- components/nvs_flash/include/nvs_bootloader.h | 109 ++++ .../private_include/nvs_bootloader_private.h | 184 ++++++ .../nvs_flash/private_include/nvs_constants.h | 68 ++ components/nvs_flash/src/nvs_bootloader.c | 604 ++++++++++++++++++ components/nvs_flash/src/nvs_page.hpp | 53 +- .../test_apps_bootloader/CMakeLists.txt | 13 + .../nvs_flash/test_apps_bootloader/README.md | 2 + .../test_apps_bootloader/main/CMakeLists.txt | 7 + .../test_apps_bootloader/main/test_app_main.c | 49 ++ .../main/test_nvs_bootloader.c | 75 +++ .../partition_nvs_data.csv | 13 + .../pytest_nvs_bootloader_support.py | 12 + .../test_apps_bootloader/sdkconfig.ci.default | 4 + .../test_apps_bootloader/sdkconfig.defaults | 4 + docs/doxygen/Doxyfile | 1 + docs/en/api-guides/bootloader.rst | 4 +- docs/en/api-reference/storage/index.rst | 3 + .../api-reference/storage/nvs_bootloader.rst | 31 + docs/en/api-reference/storage/nvs_flash.rst | 7 + docs/zh_CN/api-guides/bootloader.rst | 4 +- docs/zh_CN/api-reference/storage/index.rst | 3 + .../api-reference/storage/nvs_bootloader.rst | 4 + .../zh_CN/api-reference/storage/nvs_flash.rst | 7 + examples/custom_bootloader/README.md | 12 + examples/storage/README.md | 2 + .../storage/nvs_bootloader/CMakeLists.txt | 11 + examples/storage/nvs_bootloader/README.md | 125 ++++ .../nvs_bootloader_example/CMakeLists.txt | 10 + .../include/nvs_bootloader_example_utils.h | 15 + .../src/nvs_bootloader_example.c | 131 ++++ .../src/nvs_bootloader_example_utils.c | 216 +++++++ .../nvs_bootloader/main/CMakeLists.txt | 3 + .../main/bootloader_hooks_example_main.c | 16 + examples/storage/nvs_bootloader/nvs_data.csv | 15 + .../nvs_bootloader/pytest_nvs_bootloader.py | 13 + .../storage/nvs_bootloader/sdkconfig.defaults | 4 + tools/ci/check_rules_components_patterns.py | 6 +- 41 files changed, 1980 insertions(+), 70 deletions(-) create mode 100644 components/esp_partition/partition_bootloader.c create mode 100644 components/nvs_flash/include/nvs_bootloader.h create mode 100644 components/nvs_flash/private_include/nvs_bootloader_private.h create mode 100644 components/nvs_flash/private_include/nvs_constants.h create mode 100644 components/nvs_flash/src/nvs_bootloader.c create mode 100644 components/nvs_flash/test_apps_bootloader/CMakeLists.txt create mode 100644 components/nvs_flash/test_apps_bootloader/README.md create mode 100644 components/nvs_flash/test_apps_bootloader/main/CMakeLists.txt create mode 100644 components/nvs_flash/test_apps_bootloader/main/test_app_main.c create mode 100644 components/nvs_flash/test_apps_bootloader/main/test_nvs_bootloader.c create mode 100644 components/nvs_flash/test_apps_bootloader/partition_nvs_data.csv create mode 100644 components/nvs_flash/test_apps_bootloader/pytest_nvs_bootloader_support.py create mode 100644 components/nvs_flash/test_apps_bootloader/sdkconfig.ci.default create mode 100644 components/nvs_flash/test_apps_bootloader/sdkconfig.defaults create mode 100644 docs/en/api-reference/storage/nvs_bootloader.rst create mode 100644 docs/zh_CN/api-reference/storage/nvs_bootloader.rst create mode 100644 examples/storage/nvs_bootloader/CMakeLists.txt create mode 100644 examples/storage/nvs_bootloader/README.md create mode 100644 examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/CMakeLists.txt create mode 100644 examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/include/nvs_bootloader_example_utils.h create mode 100644 examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/src/nvs_bootloader_example.c create mode 100644 examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/src/nvs_bootloader_example_utils.c create mode 100644 examples/storage/nvs_bootloader/main/CMakeLists.txt create mode 100644 examples/storage/nvs_bootloader/main/bootloader_hooks_example_main.c create mode 100644 examples/storage/nvs_bootloader/nvs_data.csv create mode 100644 examples/storage/nvs_bootloader/pytest_nvs_bootloader.py create mode 100644 examples/storage/nvs_bootloader/sdkconfig.defaults diff --git a/components/esp_partition/CMakeLists.txt b/components/esp_partition/CMakeLists.txt index db5b1ec6db..624681a367 100644 --- a/components/esp_partition/CMakeLists.txt +++ b/components/esp_partition/CMakeLists.txt @@ -1,3 +1,17 @@ +# bootloader build simplified version +if(BOOTLOADER_BUILD) +set(srcs "partition_bootloader.c") +set(reqs "spi_flash") +set(priv_reqs "bootloader_support") + +idf_component_register(SRCS "${srcs}" + INCLUDE_DIRS "include" + PRIV_INCLUDE_DIRS ${private_include_dirs} + REQUIRES ${reqs} + PRIV_REQUIRES ${priv_reqs}) + +# regular, non bootloader build +else() set(srcs "partition.c") set(priv_reqs esp_system spi_flash partition_table) set(reqs) @@ -33,3 +47,5 @@ if(CMAKE_C_COMPILER_ID MATCHES "GNU") set_property(SOURCE ${cache_srcs} APPEND_STRING PROPERTY COMPILE_FLAGS " -fno-inline-small-functions -fno-inline-functions-called-once") endif() + +endif() diff --git a/components/esp_partition/partition_bootloader.c b/components/esp_partition/partition_bootloader.c new file mode 100644 index 0000000000..8cfc5ff48b --- /dev/null +++ b/components/esp_partition/partition_bootloader.c @@ -0,0 +1,91 @@ +/* + * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + * +*/ + +#include "esp_partition.h" +#include "esp_flash_partitions.h" // esp_partition_table_verify +#include "bootloader_flash_priv.h" // bootloader_ssupport/bootloader_flash/include +#include "esp_log.h" + +static const char* TAG = "partition_bootloader"; + +// populates the internal_partition_list from flash +const esp_partition_t* esp_partition_find_first(esp_partition_type_t type, esp_partition_subtype_t subtype, const char* label) +{ + static esp_partition_t internal_partition = {0}; + int partition_count = 0; + + // Inspired by the implementation in bootloader_support component in bootloader_support.c + const esp_partition_info_t *partitions = NULL; + + partitions = bootloader_mmap(ESP_PARTITION_TABLE_OFFSET, ESP_PARTITION_TABLE_MAX_LEN); + if (!partitions) { + ESP_LOGV(TAG, "bootloader_mmap(0x%x, 0x%x) failed", ESP_PARTITION_TABLE_OFFSET, ESP_PARTITION_TABLE_MAX_LEN); + return NULL; + } + ESP_LOGV(TAG, "mapped partition table 0x%x at 0x%x", ESP_PARTITION_TABLE_OFFSET, (intptr_t)partitions); + + esp_err_t err = esp_partition_table_verify(partitions, true, &partition_count); + if (err != ESP_OK) { + ESP_LOGV(TAG, "Failed to verify partition table"); + + bootloader_munmap(partitions); + return NULL; + } + + ESP_LOGV(TAG, "Partition Table:"); + + // in the loop, try to match the partition type, subtype and label + for (int i = 0; i < partition_count; i++) { + const esp_partition_info_t *partition = &partitions[i]; + ESP_LOGV(TAG, "Partition table entry 0x%x", (intptr_t)partition); + ESP_LOGV(TAG, "type=%x subtype=%x label=%-16s", partition->type, partition->subtype, partition->label); + + // if everything matches, populate the internal_partition + if (partition->type == type + && partition->subtype == subtype + && strncmp((char*) partition->label, label, sizeof(partition->label) - 1) == 0) { + + ESP_LOGV(TAG, "Matched", partition->type, partition->subtype, partition->label); + internal_partition.flash_chip = NULL; //esp_flash_default_chip; + internal_partition.type = partition->type; + internal_partition.subtype = partition->subtype; + internal_partition.address = partition->pos.offset; + internal_partition.size = partition->pos.size; + internal_partition.erase_size = SPI_FLASH_SEC_SIZE; + strncpy(internal_partition.label, (char*) partition->label, sizeof(internal_partition.label) - 1); + internal_partition.encrypted = partition->flags & PART_FLAG_ENCRYPTED; + internal_partition.readonly = partition->flags & PART_FLAG_READONLY; + + bootloader_munmap(partitions); + return &internal_partition; + } + } + ESP_LOGV(TAG, "Nothing matched, end of partition table"); + + bootloader_munmap(partitions); + + return NULL; +} + +esp_err_t esp_partition_read(const esp_partition_t *partition, + size_t src_offset, void *dst, size_t size) +{ + const void *buf; + + // log call to mmap + ESP_LOGV(TAG, "mmap(0x%x, 0x%x)", partition->address + src_offset, size); + + buf = bootloader_mmap(partition->address + src_offset, size); + + if (buf == NULL) { + ESP_LOGV(TAG, "bootloader_mmap(0x%x, 0x%x) failed", partition->address, size); + return ESP_ERR_NO_MEM; + } + memcpy(dst, buf, size); + bootloader_munmap(buf); + return ESP_OK; +} diff --git a/components/nvs_flash/.build-test-rules.yml b/components/nvs_flash/.build-test-rules.yml index 3902727a71..cdfd738050 100644 --- a/components/nvs_flash/.build-test-rules.yml +++ b/components/nvs_flash/.build-test-rules.yml @@ -15,3 +15,11 @@ components/nvs_flash/test_apps: - esp_partition disable_test: - if: IDF_TARGET not in ["esp32", "esp32c3"] + +components/nvs_flash/test_apps_bootloader: + depends_components: + - spi_flash + - nvs_flash + - esp_partition + disable_test: + - if: IDF_TARGET not in ["esp32", "esp32c3"] diff --git a/components/nvs_flash/CMakeLists.txt b/components/nvs_flash/CMakeLists.txt index bba0f5e42e..27f66cc9be 100644 --- a/components/nvs_flash/CMakeLists.txt +++ b/components/nvs_flash/CMakeLists.txt @@ -1,46 +1,63 @@ -idf_build_get_property(target IDF_TARGET) +if(BOOTLOADER_BUILD) + # bootloader build simplified version + set(srcs "src/nvs_bootloader.c") + set(requires "esp_partition") -set(srcs "src/nvs_api.cpp" - "src/nvs_cxx_api.cpp" - "src/nvs_item_hash_list.cpp" - "src/nvs_page.cpp" - "src/nvs_pagemanager.cpp" - "src/nvs_storage.cpp" - "src/nvs_handle_simple.cpp" - "src/nvs_handle_locked.cpp" - "src/nvs_partition.cpp" - "src/nvs_partition_lookup.cpp" - "src/nvs_partition_manager.cpp" - "src/nvs_types.cpp" - "src/nvs_platform.cpp") + idf_component_register(SRCS "${srcs}" + REQUIRES "${requires}" + INCLUDE_DIRS "include" + PRIV_INCLUDE_DIRS "private_include" + ) -set(requires esp_partition) -if(${target} STREQUAL "linux") - set(priv_requires spi_flash) else() - set(priv_requires spi_flash newlib) -endif() + # regular, non bootloader build + idf_build_get_property(target IDF_TARGET) -idf_component_register(SRCS "${srcs}" - REQUIRES "${requires}" - PRIV_REQUIRES "${priv_requires}" - INCLUDE_DIRS "include" - PRIV_INCLUDE_DIRS "private_include") + set(srcs "src/nvs_api.cpp" + "src/nvs_cxx_api.cpp" + "src/nvs_item_hash_list.cpp" + "src/nvs_page.cpp" + "src/nvs_pagemanager.cpp" + "src/nvs_storage.cpp" + "src/nvs_handle_simple.cpp" + "src/nvs_handle_locked.cpp" + "src/nvs_partition.cpp" + "src/nvs_partition_lookup.cpp" + "src/nvs_partition_manager.cpp" + "src/nvs_types.cpp" + "src/nvs_platform.cpp" + "src/nvs_bootloader.c") -# If we use the linux target, we need to redirect the crc functions to the linux -if(${target} STREQUAL "linux") - if(CONFIG_NVS_ENCRYPTION) - # mbedtls isn't configured for building with linux or as mock target. It will draw in all kind of dependencies - message(FATAL_ERROR "NVS currently doesn't support encryption if built for Linux.") + set(requires esp_partition) + if(${target} STREQUAL "linux") + set(priv_requires spi_flash) + else() + set(priv_requires spi_flash newlib) endif() - target_compile_options(${COMPONENT_LIB} PUBLIC "-DLINUX_TARGET") - target_compile_options(${COMPONENT_LIB} PUBLIC --coverage) - target_link_libraries(${COMPONENT_LIB} PUBLIC --coverage) -else() - target_sources(${COMPONENT_LIB} PRIVATE "src/nvs_encrypted_partition.cpp") - target_link_libraries(${COMPONENT_LIB} PRIVATE idf::mbedtls) -endif() -if(CONFIG_COMPILER_STATIC_ANALYZER AND CMAKE_C_COMPILER_ID STREQUAL "GNU") # TODO IDF-10088 - target_compile_options(${COMPONENT_LIB} PUBLIC "-fno-analyzer") -endif() + idf_component_register(SRCS "${srcs}" + REQUIRES "${requires}" + PRIV_REQUIRES "${priv_requires}" + INCLUDE_DIRS "include" + PRIV_INCLUDE_DIRS "private_include") + + # If we use the linux target, we need to redirect the crc functions to the linux + if(${target} STREQUAL "linux") + if(CONFIG_NVS_ENCRYPTION) + # mbedtls isn't configured for building with linux or as mock target. + # It will draw in all kind of dependencies + message(FATAL_ERROR "NVS currently doesn't support encryption if built for Linux.") + endif() + target_compile_options(${COMPONENT_LIB} PUBLIC "-DLINUX_TARGET") + target_compile_options(${COMPONENT_LIB} PUBLIC --coverage) + target_link_libraries(${COMPONENT_LIB} PUBLIC --coverage) + else() + target_sources(${COMPONENT_LIB} PRIVATE "src/nvs_encrypted_partition.cpp") + target_link_libraries(${COMPONENT_LIB} PRIVATE idf::mbedtls) + endif() + + if(CONFIG_COMPILER_STATIC_ANALYZER AND CMAKE_C_COMPILER_ID STREQUAL "GNU") # TODO IDF-10088 + target_compile_options(${COMPONENT_LIB} PUBLIC "-fno-analyzer") + endif() + +endif() #bootloader build diff --git a/components/nvs_flash/include/nvs_bootloader.h b/components/nvs_flash/include/nvs_bootloader.h new file mode 100644 index 0000000000..cdb11fb3f2 --- /dev/null +++ b/components/nvs_flash/include/nvs_bootloader.h @@ -0,0 +1,109 @@ +/* + * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include "esp_err.h" // esp_err_t +#include "nvs.h" // nvs entry data types + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @file nvs_bootloader.h + * @brief Definition of a structures and public functions provided for NVS bootloader operations. + */ + +/** + * @brief Placeholders for buffer pointer and length of string type + */ +typedef struct { + char* buff_ptr; /**< Pointer to the buffer where string and terminating zero character will be read */ + size_t buff_len; /**< Buffer length in bytes */ +} nvs_bootloader_str_value_placeholder_t; + +/** + * @brief Union of value placeholders for all nvs_type_t supported by bootloader code + */ +typedef union { + uint8_t u8_val; /**< Placeholder for unsigned 8 bit integer variable */ + int8_t i8_val; /**< Placeholder for signed 8 bit integer variable */ + uint16_t u16_val; /**< Placeholder for unsigned 16 bit integer variable */ + int16_t i16_val; /**< Placeholder for signed 16 bit integer variable */ + uint32_t u32_val; /**< Placeholder for unsigned 32 bit integer variable */ + int32_t i32_val; /**< Placeholder for signed 32 bit integer variable */ + uint64_t u64_val; /**< Placeholder for unsigned 64 bit integer variable */ + int64_t i64_val; /**< Placeholder for signed 64 bit integer variable */ + nvs_bootloader_str_value_placeholder_t str_val; /**< Placeholder for string buffer information */ +} nvs_bootloader_value_placeholder_t; + +/** + * @brief Structure representing one NVS bootloader entry. + * + * This structure serves as read operation input parameters and result value and status placeholder. + * Before calling the `nvs_bootloader_read` function, populate the namespace_name, key_name and value_type members. + * If string value has to be read, provide also buffer and its length in the `value.str_val` member. + * + * The result_code member will be populated by the function with the result of the read operation. + * There are 2 possible situations and interpretations of the result_code: + * If the return value of the `nvs_bootloader_read` was ESP_OK, the result_code will be one of the following: + * - `ESP_OK`: Entry found, value member contains the data. This is the only case when the value member is populated. + * - `ESP_ERR_NVS_TYPE_MISMATCH`: Entry was found, but requested datatype doesn't match datatype found in NVS + * - `ESP_ERR_NVS_NOT_FOUND`: Data was not found. + * - `ESP_ERR_INVALID_SIZE`: the value found for string is longer than the space provided in placeholder (str_val.buff_len) + * If the return value of the function was ESP_ERR_INVALID_ARG, the result_code will be one of the following: + * - `ESP_ERR_NVS_NOT_FOUND`: Check of this parameters was successful. + * - `ESP_ERR_NVS_INVALID_NAME`: namespace_name is NULL or too long + * - `ESP_ERR_NVS_KEY_TOO_LONG`: key_name NULL or too long + * - `ESP_ERR_INVALID_SIZE`: the size of the buffer provided for NVS_TYPE_STR in placeholder (str_val.buff_len) is zero or exceeds maximum value NVS_CONST_STR_LEN_MAX_SIZE + * - `ESP_ERR_INVALID_ARG`: Invalid datatype requested + * + */ +typedef struct { + const char* namespace_name; /**< Namespace of the entry */ + const char* key_name; /**< Key of the entry */ + nvs_type_t value_type; /**< Expected datatype to be read, can be any of NVS_TYPE_U*, NVS_TYPE_I* or NVS_TYPE_STR */ + esp_err_t result_code; /**< Result code of this entry. Explanation is in general description of the struct nvs_bootloader_read_list_t*/ + nvs_bootloader_value_placeholder_t value; /**< Placeholder for value read */ + uint8_t namespace_index; /**< Index of the namespace (internal variable, do not use) */ +} nvs_bootloader_read_list_t; + +/** + * @brief Reads data specified from the specified NVS partition. + * + * This function reads data from the NVS partition specified by `partition_name`. + * Multiple NVS entries can be read in a single call. The list of entries to read is specified in the `read_list` array. + * Function indicates overall success or failure by its return value. In case it is ESP_OK or ESP_ERR_INVALID_ARG, + * result of validation / reading of individual entry is returned in the `result_code` member of each element of the `read_list` array. + * + * @param partition_name The name of the NVS partition to read from. + * @param read_list_count The number of elements in the `read_list` array. + * @param read_list An array of `nvs_bootloader_read_list_t` structures specifying the keys and buffers for reading data. + * + * @return + * The return value of the function in this file can be one of the following: + * - `ESP_OK`: The function successfully checked all input parameters and executed successfully. + * The individual `result_code` in `read_list` indicates the result of the lookup for a particular requested key. + * - `ESP_ERR_INVALID_ARG`: The validity of all `read_list` input parameters was + * checked and failed for at least one of the parameters. The individual `result_code` + * in `read_list` provides the detailed reason. This error code is also returned when read_list is null or read_list_count is 0. + * - `ESP_ERR_NVS_INVALID_NAME`: The partition name specified is too long or is null. + * - `ESP_ERR_NVS_PART_NOT_FOUND`: The partition was not found in the partition table. + * - `ESP_ERR_NVS_CORRUPT_KEY_PART`: Encryption-related problems. + * - `ESP_ERR_NVS_WRONG_ENCRYPTION`: Encryption-related problems. + * - `ESP_ERR_INVALID_STATE`: NVS partition or pages related errors - wrong size of partition, header inconsistent / entries + * inconsistencies, multiple active pages, page in state INVALID. + * - `ESP_ERR_NO_MEM`: Cannot allocate memory required to perform the function. + * - Technical errors in underlying storage. + */ +esp_err_t nvs_bootloader_read(const char* partition_name, + const size_t read_list_count, + nvs_bootloader_read_list_t read_list[]); + +#ifdef __cplusplus +} +#endif diff --git a/components/nvs_flash/private_include/nvs_bootloader_private.h b/components/nvs_flash/private_include/nvs_bootloader_private.h new file mode 100644 index 0000000000..5b4929ffe0 --- /dev/null +++ b/components/nvs_flash/private_include/nvs_bootloader_private.h @@ -0,0 +1,184 @@ +/* + * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include "nvs_constants.h" // NVS_CONST_ENTRY_SIZE and all size related constants shared with cpp implementation of NVS +#include "nvs_bootloader.h" // nvs_bootloader_read_list_t and function prototypes +#include "esp_partition.h" // esp_partition_t + +#ifdef __cplusplus +extern "C" { +#endif + +// evaluates to true for NVS types fitting single NVS entry. At the moment all NVS_TYPE_U* or NVS_TYPE_I* */ +#define NVS_BOOTLOADER_TYPE_FITS_SINGLE_ENTRY(data_type) \ +( (data_type == NVS_TYPE_U8) \ + ||(data_type == NVS_TYPE_I8) \ + ||(data_type == NVS_TYPE_U16) \ + ||(data_type == NVS_TYPE_I16) \ + ||(data_type == NVS_TYPE_U32) \ + ||(data_type == NVS_TYPE_I32) \ + ||(data_type == NVS_TYPE_U64) \ + ||(data_type == NVS_TYPE_I64) \ +) + +// evaluates to true for NVS types supported by the nvs bootloader code*/ +#define NVS_BOOTLOADER_IS_SUPPORTED_TYPE(data_type) \ +( (data_type == NVS_TYPE_STR) || NVS_BOOTLOADER_TYPE_FITS_SINGLE_ENTRY(data_type) \ +) + +// returns length of supported data field in bytes +#define NVS_BOOTLOADER_GET_TYPE_LEN(data_type) \ +( (data_type == NVS_TYPE_U8) ? 1 \ + :(data_type == NVS_TYPE_I8) ? 1 \ + :(data_type == NVS_TYPE_U16) ? 2 \ + :(data_type == NVS_TYPE_I16) ? 2 \ + :(data_type == NVS_TYPE_U32) ? 4 \ + :(data_type == NVS_TYPE_I32) ? 4 \ + :(data_type == NVS_TYPE_U64) ? 8 \ + :(data_type == NVS_TYPE_I64) ? 8 \ + :0 \ +) + +esp_err_t nvs_bootloader_check_parameters(const char* partition_name, + const size_t read_list_count, + nvs_bootloader_read_list_t read_list[]); + +// NVS page header structure +typedef struct { + uint32_t page_state; // page state + uint32_t sequence_number; // sequence number of this page + uint8_t nvs_format_version; // nvs format version + uint8_t reserved[19]; // unused, must be 0xff + uint32_t crc32; // crc32 of everything except page_state +} nvs_bootloader_page_header_t; + +// NVS page entry state array structure +// Each entry state is represented by two bits +// There are 126 entries in a page +// The last 4 bits are reserved +typedef struct { + uint8_t entry_states[NVS_CONST_ENTRY_SIZE]; +} nvs_bootloader_page_entry_states_t; + +// Macro to extract the state of an entry from the entry state structure +#define NVS_BOOTLOADER_GET_ENTRY_STATE(page_entry_states, entry_index) \ + ((page_entry_states->entry_states[(entry_index) / 4] >> (((entry_index) % 4) * 2)) & 0x03) + +typedef struct { + uint8_t data[8]; +} nvs_bootloader_entry_data_primitive_type_t; + +typedef struct { + uint16_t size; + uint16_t reserved; + uint32_t crc32; +} nvs_bootloader_entry_data_var_len_type_t; + +typedef union { + nvs_bootloader_entry_data_primitive_type_t primitive_type; + nvs_bootloader_entry_data_var_len_type_t var_len_type; +} nvs_bootloader_entry_data_t; + +// NVS single entry item definition +typedef struct { + uint8_t namespace_index; // namespace index + uint8_t data_type; // data type + uint8_t span; // span + uint8_t chunk_index; // chunk index + uint32_t crc32; // crc32 of the entry + char key[NVS_KEY_NAME_MAX_SIZE]; // key + nvs_bootloader_entry_data_t data; // data, interpreted based on data_type +} nvs_bootloader_single_entry_t; + +#define MAX_KEY_LENGTH = (NVS_KEY_NAME_MAX_SIZE - 1); + +// read a single entry item from a page +// only reads entries marked as written +// skips items spanning multiple entries +// entry_index is the index of the first entry to read +// item is the output parameter +// entry_index is updated to the index of the next entry +// returns ESP_ERR_NOT_FOUND if no more entries are found +// returns ESP_OK if an entry is found +// returns other error codes if an error occurs +esp_err_t nvs_bootloader_read_next_single_entry_item(const esp_partition_t *partition, const size_t page_index, const nvs_bootloader_page_entry_states_t *entry_states, uint8_t *entry_index, nvs_bootloader_single_entry_t *item); + +// read a block of data from the entries from a page +// the block is read starting from the offset specified as entry_index of entry +// the length of the block is specified in bytes in parameter block_len +// the block is read into the buffer pointed to by block +// before accessing the entries, the entry states are checked and the entries are read only if they are marked as written +// if any of the entries that needs to be read is not in the state written, the function returns ESP_ERR_INVALID_STATE +// if the block is not fully read after reaching end of the page, the function returns ESP_ERR_INVALID_STATE +// if the block is fully read, the function returns ESP_OK +esp_err_t nvs_bootloader_read_entries_block(const esp_partition_t *partition, const size_t page_index, const nvs_bootloader_page_entry_states_t *entry_states, const size_t entry_index, const size_t block_len, uint8_t *block); + +// visitor function parameters for reading the page header +typedef struct { + size_t no_active_pages; + size_t no_freeing_pages; +} nvs_bootloader_page_visitor_param_get_page_states_t; + +// visitor function parameters for reading the namespaces from pages and the key - value pairs from pages +typedef struct { + const bool skip_active_page; + const size_t read_list_count; + nvs_bootloader_read_list_t *read_list; +} nvs_bootloader_page_visitor_param_read_entries_t; + +// visitor function parameter union type declaration +typedef union { + nvs_bootloader_page_visitor_param_get_page_states_t *get_page_states; + nvs_bootloader_page_visitor_param_read_entries_t *read_entries; +} nvs_bootloader_page_visitor_param_t; + +// visitor function type declaration +typedef esp_err_t (*nvs_bootloader_page_visitor_t)(nvs_bootloader_page_visitor_param_t *visitor_param, + const esp_partition_t *partition, + const size_t page_index, + const nvs_bootloader_page_header_t *page_header); + +// visit all pages in the partition +esp_err_t nvs_bootloader_visit_pages(const esp_partition_t *partition, + nvs_bootloader_page_visitor_t visitor, + nvs_bootloader_page_visitor_param_t *visitor_param); + +// read page header +esp_err_t nvs_bootloader_read_page_header(const esp_partition_t *partition, + const size_t page_index, + const nvs_bootloader_page_header_t *page_header); + +// read entry states from a page +esp_err_t nvs_bootloader_read_page_entry_states(const esp_partition_t *partition, + const size_t page_index, + nvs_bootloader_page_entry_states_t *entry_states); + +// populates the page header for a given page +esp_err_t nvs_bootloader_page_visitor_get_page_states(nvs_bootloader_page_visitor_param_t *visitor_param, + const esp_partition_t *partition, + const size_t page_index, + const nvs_bootloader_page_header_t *page_header); + +// populates the namespace indexes found on a page +esp_err_t nvs_bootloader_page_visitor_get_namespaces(nvs_bootloader_page_visitor_param_t *visitor_param, + const esp_partition_t *partition, + const size_t page_index, + const nvs_bootloader_page_header_t *page_header); + +// populates the key value pairs found on a page +esp_err_t nvs_bootloader_page_visitor_get_key_value_pairs(nvs_bootloader_page_visitor_param_t *visitor_param, + const esp_partition_t *partition, + const size_t page_index, + const nvs_bootloader_page_header_t *page_header); + +// check if the item header is consistent - crc32 matches, namespace index is valid, data type is valid and other fields are consistent +bool nvs_bootloader_check_item_header_consistency(const nvs_bootloader_single_entry_t *item, + const uint8_t entry_index); +#ifdef __cplusplus +} +#endif diff --git a/components/nvs_flash/private_include/nvs_constants.h b/components/nvs_flash/private_include/nvs_constants.h new file mode 100644 index 0000000000..0881156393 --- /dev/null +++ b/components/nvs_flash/private_include/nvs_constants.h @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef NVS_CONSTANTS_H +#define NVS_CONSTANTS_H + +#include "spi_flash_mmap.h" // for SPI_FLASH_SEC_SIZE + +// constants for the NVS to be used in the regular as well as bootloader implementations + +#define NVS_CONST_PSB_INIT 0x1 +#define NVS_CONST_PSB_FULL 0x2 +#define NVS_CONST_PSB_FREEING 0x4 +#define NVS_CONST_PSB_CORRUPT 0x8 + +#define NVS_CONST_ESB_WRITTEN 0x1 +#define NVS_CONST_ESB_ERASED 0x2 + +#define NVS_CONST_PAGE_SIZE SPI_FLASH_SEC_SIZE + +#define NVS_CONST_ENTRY_SIZE 32 +#define NVS_CONST_ENTRY_COUNT 126 +#define NVS_CONST_INVALID_ENTRY 0xffffffff + +#define NVS_CONST_CHUNK_MAX_SIZE (NVS_CONST_ENTRY_SIZE * (NVS_CONST_ENTRY_COUNT - 1)) +#define NVS_CONST_STR_LEN_MAX_SIZE (NVS_CONST_ENTRY_SIZE * (NVS_CONST_ENTRY_COUNT - 1)) + +#define NVS_CONST_NS_INDEX 0 +#define NVS_CONST_NS_ANY 255 + +#define NVS_CONST_NVS_VERSION 0xfe // Decrement to upgrade + +// Page States +// All bits set, default state after flash erase. Page has not been initialized yet. +#define NVS_CONST_PAGE_STATE_UNINITIALIZED 0xffffffff + +// Page is initialized, and will accept writes. +#define NVS_CONST_PAGE_STATE_ACTIVE (NVS_CONST_PAGE_STATE_UNINITIALIZED & ~NVS_CONST_PSB_INIT) + +// Page is marked as full and will not accept new writes. +#define NVS_CONST_PAGE_STATE_FULL (NVS_CONST_PAGE_STATE_ACTIVE & ~NVS_CONST_PSB_FULL) + +// Data is being moved from this page to a new one. +#define NVS_CONST_PAGE_STATE_FREEING (NVS_CONST_PAGE_STATE_FULL & ~NVS_CONST_PSB_FREEING) + +// Page was found to be in a corrupt and unrecoverable state. +// Instead of being erased immediately, it will be kept for diagnostics and data recovery. +// It will be erased once we run out out free pages. +#define NVS_CONST_PAGE_STATE_CORRUPT (NVS_CONST_PAGE_STATE_FREEING & ~PSB_CORRUPT) + +// Page object wasn't loaded from flash memory +#define NVS_CONST_PAGE_STATE_INVALID 0 + +// Entry States +#define NVS_CONST_ENTRY_STATE_EMPTY 0x3 // 0b11, default state after flash erase +#define NVS_CONST_ENTRY_STATE_WRITTEN (NVS_CONST_ENTRY_STATE_EMPTY & ~NVS_CONST_ESB_WRITTEN) // entry was written +#define NVS_CONST_ENTRY_STATE_ERASED (NVS_CONST_ENTRY_STATE_WRITTEN & ~NVS_CONST_ESB_ERASED) // entry was written and then erased +#define NVS_CONST_ENTRY_STATE_ILLEGAL 0x1 // only possible if flash is inconsistent +#define NVS_CONST_ENTRY_STATE_INVALID 0x4 // entry is in inconsistent state (write started but ESB_WRITTEN has not been set yet) + +// Offsets within a NVS page +#define NVS_CONST_PAGE_HEADER_OFFSET 0 +#define NVS_CONST_PAGE_ENTRY_TABLE_OFFSET (NVS_CONST_PAGE_HEADER_OFFSET + 32) +#define NVS_CONST_PAGE_ENTRY_DATA_OFFSET (NVS_CONST_PAGE_ENTRY_TABLE_OFFSET + 32) + +#endif diff --git a/components/nvs_flash/src/nvs_bootloader.c b/components/nvs_flash/src/nvs_bootloader.c new file mode 100644 index 0000000000..07126c966a --- /dev/null +++ b/components/nvs_flash/src/nvs_bootloader.c @@ -0,0 +1,604 @@ +/* + * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ +#include +#include +#include "esp_log.h" +#include "nvs_bootloader.h" +#include "nvs_bootloader_private.h" +#include "esp_assert.h" + +#include "esp_partition.h" +#include "nvs_constants.h" +#include + +static const char* TAG = "nvs_bootloader"; +const bool const_is_encrypted = false; + +// Static asserts ensuring that the size of the c structures match NVS physical footprint on the flash +ESP_STATIC_ASSERT(sizeof(nvs_bootloader_page_header_t) == NVS_CONST_ENTRY_SIZE, "nvs_bootloader_page_header_t size is not 32 bytes"); +ESP_STATIC_ASSERT(sizeof(nvs_bootloader_page_entry_states_t) == NVS_CONST_ENTRY_SIZE, "nvs_bootloader_page_entry_states_t size is not 32 bytes"); +ESP_STATIC_ASSERT(sizeof(nvs_bootloader_single_entry_t) == NVS_CONST_ENTRY_SIZE, "nvs_bootloader_single_entry_t size is not 32 bytes"); + +esp_err_t nvs_bootloader_read(const char* partition_name, + const size_t read_list_count, + nvs_bootloader_read_list_t read_list[]) +{ + + /* + The flow: + + 1. validate parameters and if there are any, report errors. + 2. check if the partition exists + 3. read the partition and browse all NVS pages to figure out whether there is any page in the state of "FREEING" + 3.1. if there is a page in the state of "FREEING", then reading from the page marked as "ACTIVE" will be skipped + 4. read entries from pages marked as "FULL" or ("ACTIVE" xor "FREEING") to identify namespace indexes of the requested namespaces + 5. read the requested entries, same skipping of pages in the state of "ACTIVE" as in step 4 + + */ + // load input parameters + ESP_LOGD(TAG, "nvs_bootloader_read called with partition_name: %s, read_list_count: %u", partition_name, (unsigned)read_list_count); + + // Placeholder return value, replace with actual error handling + esp_err_t ret = ESP_OK; + + // Check if the parameters are valid + ret = nvs_bootloader_check_parameters(partition_name, read_list_count, read_list); + if (ret != ESP_OK) { + ESP_LOGD(TAG, "nvs_bootloader_check_parameters failed"); + return ret; + } + + const esp_partition_t *partition = esp_partition_find_first(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS, partition_name); + if (partition == NULL) { + ESP_LOGV(TAG, "esp_partition_find_first failed"); + return ESP_ERR_NVS_PART_NOT_FOUND; + } + + // log the partition details + ESP_LOGV(TAG, "Partition %s found, size is: %" PRIu32 "", partition->label, partition->size); + + // visit pages to get the number of pages in the state of "ACTIVE" and "FREEING" + nvs_bootloader_page_visitor_param_get_page_states_t page_states_count = {0}; + + ret = nvs_bootloader_visit_pages(partition, nvs_bootloader_page_visitor_get_page_states, (nvs_bootloader_page_visitor_param_t*)&page_states_count); + + if (ret != ESP_OK) { + ESP_LOGV(TAG, "Failed reading page states"); + return ret; + } + if (page_states_count.no_freeing_pages > 1) { + ESP_LOGV(TAG, "Multiple pages in the state of FREEING"); + return ESP_ERR_INVALID_STATE; + } + if (page_states_count.no_active_pages > 1) { + ESP_LOGV(TAG, "Multiple pages in the state of ACTIVE"); + return ESP_ERR_INVALID_STATE; + } + + // Here we have at most 1 active page and at most 1 freeing page. If there exists a freeing page, we will skip reading from the active page + // Visit pages to get the namespace indexes of the requested namespaces + nvs_bootloader_page_visitor_param_read_entries_t read_entries = { .skip_active_page = page_states_count.no_freeing_pages > 0, .read_list_count = read_list_count, .read_list = read_list }; + + // log the visitor parameters + ESP_LOGV(TAG, "nvs_bootloader_read - visiting pages to get namespace indexes"); + + ret = nvs_bootloader_visit_pages(partition, nvs_bootloader_page_visitor_get_namespaces, (nvs_bootloader_page_visitor_param_t*) &read_entries); + + if (ret != ESP_OK) { + ESP_LOGV(TAG, "nvs_bootloader_page_visitor_get_namespaces failed"); + return ret; + } + + // log the visitor parameters + ESP_LOGV(TAG, "nvs_bootloader_read - visiting pages to get key - value pairs"); + + // Visit pages to read the requested key - value pairs + ret = nvs_bootloader_visit_pages(partition, nvs_bootloader_page_visitor_get_key_value_pairs, (nvs_bootloader_page_visitor_param_t*) &read_entries); + + return ret; +} + +esp_err_t nvs_bootloader_check_parameters(const char* partition_name, + const size_t read_list_count, + nvs_bootloader_read_list_t read_list[]) +{ + // return `ESP_ERR_NVS_INVALID_NAME` if the partition name specified is too long or is null. + if (partition_name == NULL || strlen(partition_name) > NVS_PART_NAME_MAX_SIZE) { + ESP_LOGD(TAG, "Invalid argument: partition_name is NULL or too long"); + return ESP_ERR_NVS_INVALID_NAME; + } + // immediately return `ESP_ERR_INVALID_ARG` if the read_list is NULL or read_list_count is 0 + if (read_list == NULL || read_list_count == 0) { + ESP_LOGD(TAG, "Invalid argument: read_list is NULL or read_list_count is 0"); + return ESP_ERR_INVALID_ARG; + } + esp_err_t ret = ESP_OK; + // iterate over all the read_list entries and check if the namespace name is too long or is null, + // or if the key name is too long or is null, + // or if the value type is invalid + // or if the value type is NVS_TYPE_STR and the buffer length is 0 or greater than the maximum allowed size + for (size_t i = 0; i < read_list_count; i++) { + // pre-fill the result code with `ESP_ERR_NVS_NOT_FOUND` and namespace index with 0 + read_list[i].result_code = ESP_ERR_NVS_NOT_FOUND; // default result code during the check_parameters phase + read_list[i].namespace_index = 0; // indicated namespace as not found + + // checks + if (read_list[i].namespace_name == NULL || strlen(read_list[i].namespace_name) > NVS_NS_NAME_MAX_SIZE - 1) { + ESP_LOGD(TAG, "Invalid argument: namespace_name is NULL or too long"); + read_list[i].result_code = ESP_ERR_NVS_INVALID_NAME; + } + if (read_list[i].key_name == NULL || strlen(read_list[i].key_name) > NVS_KEY_NAME_MAX_SIZE - 1) { + ESP_LOGD(TAG, "Invalid argument: key_name is NULL or too long"); + read_list[i].result_code = ESP_ERR_NVS_KEY_TOO_LONG; + } + if (!NVS_BOOTLOADER_IS_SUPPORTED_TYPE(read_list[i].value_type)) { + ESP_LOGD(TAG, "Invalid argument: value_type is invalid"); + read_list[i].result_code = ESP_ERR_INVALID_ARG; + } + if ((read_list[i].value_type) == NVS_TYPE_STR && (read_list[i].value.str_val.buff_len == 0 || read_list[i].value.str_val.buff_len > NVS_CONST_STR_LEN_MAX_SIZE)) { + ESP_LOGD(TAG, "Invalid argument: buffer size provided for NVS_TYPE_STR is invalid"); + read_list[i].result_code = ESP_ERR_INVALID_SIZE; + } + if ((read_list[i].value_type) == NVS_TYPE_STR && (read_list[i].value.str_val.buff_ptr == NULL)) { + ESP_LOGD(TAG, "Invalid argument: buffer pointer provided for NVS_TYPE_STR is null"); + read_list[i].result_code = ESP_ERR_INVALID_SIZE; + } + + // if the individual result code has changed from it's default value ESP_ERR_NVS_NOT_FOUND, set the result code of function to `ESP_ERR_INVALID_ARG` + if (read_list[i].result_code != ESP_ERR_NVS_NOT_FOUND) { + ret = ESP_ERR_INVALID_ARG; + } + + // clear the value placeholder for single entry data types. We do not touch the buffer type placeholder + if (NVS_BOOTLOADER_TYPE_FITS_SINGLE_ENTRY(read_list[i].value_type)) { + size_t val_len = sizeof(read_list[i].value); + memset((void*) &read_list[i].value, 0, val_len); // clear the value placeholder + } + } + return ret; +} + +// we are going to use the visitor pattern to process all NVS pages in three subsequent loops: +// 1. read the page header to figure out the existence of the page in state "FREEING" +// 2. depending on the state of the page, read the entries from the page to figure out the namespace indexes of the requested namespaces +// 3. read the requested key - value pairs from the pages + +// visitor function for reading the page header +esp_err_t nvs_bootloader_page_visitor_get_page_states(nvs_bootloader_page_visitor_param_t *visitor_param, + const esp_partition_t *partition, + const size_t page_index, + const nvs_bootloader_page_header_t *page_header) +{ + nvs_bootloader_page_visitor_param_get_page_states_t *param = (nvs_bootloader_page_visitor_param_get_page_states_t*) visitor_param; + switch (page_header->page_state) { + case NVS_CONST_PAGE_STATE_ACTIVE: + param->no_active_pages++; + break; + case NVS_CONST_PAGE_STATE_FREEING: + param->no_freeing_pages++; + break; + default: + break; + } + return ESP_OK; +} + +// visitor function for reading the namespaces from pages +esp_err_t nvs_bootloader_page_visitor_get_namespaces(nvs_bootloader_page_visitor_param_t *visitor_param, + const esp_partition_t *partition, + const size_t page_index, + const nvs_bootloader_page_header_t *page_header) +{ + // log the visitor parameters + ESP_LOGV(TAG, "nvs_bootloader_page_visitor_get_namespaces called with page_index: %u", (unsigned)page_index); + + nvs_bootloader_page_visitor_param_read_entries_t *param = (nvs_bootloader_page_visitor_param_read_entries_t*) visitor_param; + + // if the page is in the state of "ACTIVE" and we are skipping the active page, then return immediately + if (param->skip_active_page && page_header->page_state == NVS_CONST_PAGE_STATE_ACTIVE) { + return ESP_OK; + } + + // read entry state map from the page + WORD_ALIGNED_ATTR nvs_bootloader_page_entry_states_t page_entry_states = {0}; + + esp_err_t ret = nvs_bootloader_read_page_entry_states(partition, page_index, &page_entry_states); + + if (ret != ESP_OK) { + ESP_LOGV(TAG, "Error reading NVS page entry states"); + return ret; + } + + // iterate over all entries with state written + // if the entry is namespace entry, then iterate the read_list and populate the namespace index by matching the namespace name + uint8_t start_index = 0; + nvs_bootloader_single_entry_t item = {0}; + + // repeat finding single entry items on the page until all entries are processed or error occurs + while (ret == ESP_OK) { + ret = nvs_bootloader_read_next_single_entry_item(partition, page_index, &page_entry_states, &start_index, &item); + + if (ret != ESP_OK) { + break; + } + + // if the entry is a namespace entry, then iterate the read_list and populate the namespace index by matching the namespace name + if (item.namespace_index == 0 && item.data_type == NVS_TYPE_U8) { + + ESP_LOGV(TAG, "Namespace %s record found. NS index is: %d. Trying to match read list...", item.key, item.data.primitive_type.data[0]); + for (size_t i = 0; i < param->read_list_count; i++) { + // if the namespace index is already populated, skip the read list entry + if (param->read_list[i].namespace_index != 0) { + continue; + } + + if (strncmp(param->read_list[i].namespace_name, item.key, NVS_KEY_NAME_MAX_SIZE) == 0) { + param->read_list[i].namespace_index = item.data.primitive_type.data[0]; + ESP_LOGV(TAG, "Namespace %s found for read item: %u key: %s NS index is: %d", param->read_list[i].namespace_name,(unsigned)i, param->read_list[i].key_name, param->read_list[i].namespace_index); + } + } + } + } + + if (ret == ESP_ERR_NVS_NOT_FOUND) { + ret = ESP_OK; + } + return ret; +} + +// visitor function for reading the key - value pairs from pages +esp_err_t nvs_bootloader_page_visitor_get_key_value_pairs(nvs_bootloader_page_visitor_param_t *visitor_param, + const esp_partition_t *partition, + const size_t page_index, + const nvs_bootloader_page_header_t *page_header) +{ + nvs_bootloader_page_visitor_param_read_entries_t *param = (nvs_bootloader_page_visitor_param_read_entries_t*) visitor_param; + + // if the page is in the state of "ACTIVE" and we are skipping the active page, then return immediately + if (param->skip_active_page && page_header->page_state == NVS_CONST_PAGE_STATE_ACTIVE) { + return ESP_OK; + } + + // read entry state map from the page + WORD_ALIGNED_ATTR nvs_bootloader_page_entry_states_t page_entry_states = {0}; + esp_err_t ret = nvs_bootloader_read_page_entry_states(partition, page_index, &page_entry_states); + + if (ret != ESP_OK) { + ESP_LOGV(TAG, "Error reading NVS page entry states"); + return ret; + } + + // iterate over all entries with state written + // if the entry is not a namespace entry, then iterate the read_list and populate the value by matching the namespace index, key name and value type + uint8_t next_index = 0; // index of the next entry to read, updated to the next entry by the read_next_single_entry_item + uint8_t current_index = 0; // index of the actual entry being processed + nvs_bootloader_single_entry_t item = {0}; + + // repeat finding single entry items on the page until all entries are processed or error occurs + while (ret == ESP_OK) { + current_index = next_index; + ret = nvs_bootloader_read_next_single_entry_item(partition, page_index, &page_entry_states, &next_index, &item); + + if (ret != ESP_OK) { + break; + } + + ESP_LOGV(TAG, "nvs_bootloader_page_visitor_get_key_value_pairs - read item NS index: %d, key: %s, data type: %d, span: %d. Trying to match read list...", item.namespace_index, item.key, item.data_type, item.span); + + // skip namespace entry + if (item.namespace_index == 0) { + continue; + } + + // iterate the read_list and populate the value by matching the namespace index, key name and value type + for (size_t i = 0; i < param->read_list_count; i++) { + + // skip non matching namespace indexes + if (param->read_list[i].namespace_index != item.namespace_index) { + continue; + } + + // process only if the result code of the read list is still in the state of "NOT_FOUND" + if (param->read_list[i].result_code != ESP_ERR_NVS_NOT_FOUND) { + continue; + } + + // try to match the key name + if (strncmp(param->read_list[i].key_name, item.key, NVS_KEY_NAME_MAX_SIZE) != 0) { + continue; + } + + // check data type mismatch between the entry and the read_list, if data type requested and found differ, indicate it in the result code and skip the entry + if (param->read_list[i].value_type != item.data_type) { + param->read_list[i].result_code = ESP_ERR_NVS_TYPE_MISMATCH; + continue; + } + + ESP_LOGV(TAG, "nvs_bootloader_page_visitor_get_key_value_pairs - matched item with read_list index: %u", (unsigned)i); + + // for single entry data types, populate the value directly from the data of item + if (NVS_BOOTLOADER_TYPE_FITS_SINGLE_ENTRY(item.data_type)) { + memcpy((void*) ¶m->read_list[i].value, (void*) item.data.primitive_type.data, NVS_BOOTLOADER_GET_TYPE_LEN(item.data_type)); + param->read_list[i].result_code = ESP_OK; + + // we are done with this entry, continue with the next one + break; + } + // for a string, the data is stored in the entries following the metadata entry in item + else if (item.data_type == NVS_TYPE_STR) { + // check if the buffer length is long enough to hold the string data and if not, indicate it with ESP_ERR_INVALID_SIZE + if (param->read_list[i].value.str_val.buff_len < item.data.var_len_type.size) { + param->read_list[i].result_code = ESP_ERR_INVALID_SIZE; + // it does not make sense to continue with the next entry, as the the function doesn't support multiple entries for a single key + break; + } + + // to get the actual data, entries at the index current_index + 1 onwards have to be read. If the result is different from ESP_OK, treat the situation as not found - it is technically a mismatch + esp_err_t ret_str = nvs_bootloader_read_entries_block(partition, page_index, &page_entry_states, current_index + 1, item.data.var_len_type.size, (uint8_t*)param->read_list[i].value.str_val.buff_ptr); + if (ret_str != ESP_OK) { + // it does not make sense to continue with the next entry, as the data is not found + break; + } + + // calculate the crc32 of the data and compare it with the crc32 stored in the item. If they do not match, treat the situation as not found - it is technically a mismatch + uint32_t calc_crc = esp_rom_crc32_le(0xffffffff, (uint8_t*)(param->read_list[i].value.str_val.buff_ptr), item.data.var_len_type.size); + + if (calc_crc != item.data.var_len_type.crc32) { + ESP_LOGV(TAG, "CRC32 of string data failed"); + // it does not make sense to continue with the next entry, as the data is not found + break; + } + + // set the result code to ESP_OK + param->read_list[i].result_code = ESP_OK; + } + } + } + + if (ret == ESP_ERR_NVS_NOT_FOUND) { + ret = ESP_OK; + } + return ret; +} + +// visitor pattern for processing of all NVS pages of type "FULL" or "ACTIVE" xor "FREEING" +esp_err_t nvs_bootloader_visit_pages(const esp_partition_t *partition, + nvs_bootloader_page_visitor_t visitor, + nvs_bootloader_page_visitor_param_t *visitor_param) +{ + esp_err_t ret = ESP_OK; + WORD_ALIGNED_ATTR nvs_bootloader_page_header_t page_header = {0}; // has to be 32 bit aligned + + for (size_t page_index = 0; page_index < (partition->size / NVS_CONST_PAGE_SIZE); page_index++) { + // reading the NVS page header + ret = nvs_bootloader_read_page_header(partition, page_index, &page_header); + + if (ret != ESP_OK) { + ESP_LOGV(TAG, "Error %04x reading NVS page header of page %u", ret, (unsigned)page_index); + } + + // just skip page with invalid header as it can be the case of uninitialized page + if (ret == ESP_ERR_NVS_INVALID_STATE) { + ret = ESP_OK; + continue; + } + + // log the page header + ESP_LOGV(TAG, "nvs_bootloader_visit_pages - page %u, state: %" PRIu32 "", (unsigned)page_index, page_header.page_state); + + // skip pages not in states "FULL", "ACTIVE" or "FREEING" + if (page_header.page_state != NVS_CONST_PAGE_STATE_FULL + && page_header.page_state != NVS_CONST_PAGE_STATE_ACTIVE + && page_header.page_state != NVS_CONST_PAGE_STATE_FREEING) { + continue; + } + + // visit the page + ret = visitor(visitor_param, partition, page_index, &page_header); + if (ret != ESP_OK) { + ESP_LOGV(TAG, "Error %04x visiting NVS page %u", ret, (unsigned)page_index); + return ret; + } + } + + return ret; +} + +esp_err_t nvs_bootloader_read_page_header(const esp_partition_t *partition, + const size_t page_index, + const nvs_bootloader_page_header_t *page_header) +{ + + esp_err_t ret = ESP_OK; + + ret = esp_partition_read(partition, page_index * NVS_CONST_PAGE_SIZE + NVS_CONST_PAGE_HEADER_OFFSET, (void*)page_header, sizeof(nvs_bootloader_page_header_t)); + + if (ret != ESP_OK) { + return ret; + } + + // calculate the CRC32 of the page header + uint32_t calc_crc = esp_rom_crc32_le(0xffffffff, (uint8_t*)page_header + offsetof(nvs_bootloader_page_header_t, sequence_number), offsetof(nvs_bootloader_page_header_t, crc32) - offsetof(nvs_bootloader_page_header_t, sequence_number)); + if (calc_crc != page_header->crc32) { + ESP_LOGV(TAG, "CRC32 of page# %u header failed", (unsigned)page_index); + return ESP_ERR_NVS_INVALID_STATE; + } + + return ret; +} + +esp_err_t nvs_bootloader_read_page_entry_states(const esp_partition_t *partition, + const size_t page_index, + nvs_bootloader_page_entry_states_t *page_entry_states) +{ + // log the read_page_entry_states parameters + ESP_LOGV(TAG, "nvs_bootloader_page_visitor_get_namespaces - reading page entry states for page %u", (unsigned)page_index); + + return esp_partition_read(partition, page_index * NVS_CONST_PAGE_SIZE + NVS_CONST_PAGE_ENTRY_TABLE_OFFSET, (void*)page_entry_states, sizeof(nvs_bootloader_page_entry_states_t)); +} + +esp_err_t nvs_bootloader_read_next_single_entry_item(const esp_partition_t *partition, + const size_t page_index, + const nvs_bootloader_page_entry_states_t *page_entry_states, + uint8_t *entry_index, + nvs_bootloader_single_entry_t *item) +{ + + // log parameters + ESP_LOGV(TAG, "nvs_bootloader_read_next_single_entry_item called with page_index: %u, entry_index: %d", (unsigned)page_index, *entry_index); + + // check if the start index is valid. Not found is signal to the caller that there are no more entries to read + if (*entry_index >= NVS_CONST_ENTRY_COUNT) { + return ESP_ERR_NVS_NOT_FOUND; + } + + esp_err_t ret = ESP_OK; + + // find the next single entry item + while ((*entry_index) < NVS_CONST_ENTRY_COUNT) { + // If the entry is written, try to read and process it + if (NVS_BOOTLOADER_GET_ENTRY_STATE(page_entry_states, *entry_index) == NVS_CONST_ENTRY_STATE_WRITTEN) { + + ret = esp_partition_read(partition, page_index * NVS_CONST_PAGE_SIZE + NVS_CONST_PAGE_ENTRY_DATA_OFFSET + (*entry_index) * NVS_CONST_ENTRY_SIZE, (void*)item, sizeof(nvs_bootloader_single_entry_t)); + + if (ret != ESP_OK) { + return ret; + } + + // only look at item with consistent header + if (nvs_bootloader_check_item_header_consistency(item, *entry_index)) { + // advance the start index + (*entry_index) += item->span; + + // if the item is single entry value data type, return ESP_OK + if (NVS_BOOTLOADER_IS_SUPPORTED_TYPE(item->data_type)) { + return ESP_OK; + } + } + // inconsistent header, try next entry + else { + (*entry_index)++; + } + + } + // if the entry is not written just try next entry + else { + (*entry_index)++; + } + } + + return ESP_ERR_NVS_NOT_FOUND; +} + +esp_err_t nvs_bootloader_read_entries_block(const esp_partition_t *partition, + const size_t page_index, + const nvs_bootloader_page_entry_states_t *entry_states, + const size_t entry_index, + const size_t block_len, + uint8_t *block) +{ + // log parameters + ESP_LOGV(TAG, "nvs_bootloader_read_entries_block called with page_index: %u, entry_index: %u, block_len: %u", (unsigned)page_index, (unsigned)entry_index, (unsigned)block_len); + + esp_err_t ret = ESP_ERR_INVALID_STATE; + + do { + // check if the start index is valid + if (entry_index >= NVS_CONST_ENTRY_COUNT) { + ESP_LOGV(TAG, "entry index is beyond the page boundary"); + break; + } + + // calculate the number of entries necessary to read the block + size_t number_of_entries = block_len / NVS_CONST_ENTRY_SIZE; + if (block_len % NVS_CONST_ENTRY_SIZE != 0) { + number_of_entries++; + } + + // check if the highest entry to be read still fits the page size + if (entry_index + number_of_entries > NVS_CONST_ENTRY_COUNT) { + ESP_LOGV(TAG, "block length exceeds the page boundary"); + break; + } + + // check if all entries are in the state written + bool all_entries_written = true; + for (size_t i = 0; i < number_of_entries; i++) { + if (NVS_BOOTLOADER_GET_ENTRY_STATE(entry_states, entry_index + i) != NVS_CONST_ENTRY_STATE_WRITTEN) { + ESP_LOGV(TAG, "some entry is not in the state written"); + all_entries_written = false; + break; + } + } + if(!all_entries_written) { + break; + } + + ret = ESP_OK; + } while (false); + + if (ret != ESP_OK) { + return ret; + } + + size_t data_offset = page_index * NVS_CONST_PAGE_SIZE + NVS_CONST_PAGE_ENTRY_DATA_OFFSET + entry_index * NVS_CONST_ENTRY_SIZE ; + + return esp_partition_read(partition, data_offset, block, block_len); +} + +// validates item's header +// CRC is checked +// zero span is checked +// span is validated using the data type +// span is checked not to exceed the remaining number of entries in the page +bool nvs_bootloader_check_item_header_consistency(const nvs_bootloader_single_entry_t *item, + const uint8_t entry_index) +{ + + // check crc of item + uint32_t calc_crc = esp_rom_crc32_le(0xffffffff, (uint8_t*)item + offsetof(nvs_bootloader_single_entry_t, namespace_index), offsetof(nvs_bootloader_single_entry_t, crc32) - offsetof(nvs_bootloader_single_entry_t, namespace_index)); + calc_crc = esp_rom_crc32_le(calc_crc, (uint8_t*)item + offsetof(nvs_bootloader_single_entry_t, key), sizeof(item->key)); + calc_crc = esp_rom_crc32_le(calc_crc, (uint8_t*)item + offsetof(nvs_bootloader_single_entry_t, data), sizeof(item->data)); + + bool ret = false; + + do { + if (calc_crc != item->crc32) { + ESP_LOGV(TAG, "Header CRC32 failed for item with key %s", item->key); + break; + } + + // check zero span + if (item->span == 0) { + ESP_LOGV(TAG, "Span of key %s is zero", item->key); + break; + } + + // check span according to the data type + if (NVS_BOOTLOADER_TYPE_FITS_SINGLE_ENTRY(item->data_type) && item->span != 1) { + ESP_LOGV(TAG, "Span of single entry key %s is not 1", item->key); + break; + } + + // check span according to the remaining space in the page + if ((uint16_t)entry_index + (uint16_t)(item->span) > NVS_CONST_ENTRY_COUNT) { + ESP_LOGV(TAG, "Span of key %s exceeds remaining page space", item->key); + break; + } + + // check span for string data type. The span has to be able to hold the data length but should not occupy more than necessary entries + if (item->data_type == NVS_TYPE_STR) { + if( ((item->span - 1) * NVS_CONST_ENTRY_SIZE < item->data.var_len_type.size) + || ((item->span - 1) * NVS_CONST_ENTRY_SIZE > item->data.var_len_type.size + NVS_CONST_ENTRY_SIZE - 1) ) { + ESP_LOGV(TAG, "Span %u of key %s for string type doesn't match indicated string length %u", (unsigned)item->span, item->key, item->data.var_len_type.size); + break; + } + } + ret = true; + } while (false); + + return ret; +} diff --git a/components/nvs_flash/src/nvs_page.hpp b/components/nvs_flash/src/nvs_page.hpp index 8d9c11b002..7adb75bef7 100644 --- a/components/nvs_flash/src/nvs_page.hpp +++ b/components/nvs_flash/src/nvs_page.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -17,6 +17,7 @@ #include "nvs_item_hash_list.hpp" #include "nvs_memory_management.hpp" #include "partition.hpp" +#include "nvs_constants.h" namespace nvs { @@ -25,46 +26,46 @@ namespace nvs class Page : public intrusive_list_node, public ExceptionlessAllocatable { public: - static const uint32_t PSB_INIT = 0x1; - static const uint32_t PSB_FULL = 0x2; - static const uint32_t PSB_FREEING = 0x4; - static const uint32_t PSB_CORRUPT = 0x8; + static const uint32_t PSB_INIT = NVS_CONST_PSB_INIT; + static const uint32_t PSB_FULL = NVS_CONST_PSB_FULL; + static const uint32_t PSB_FREEING = NVS_CONST_PSB_FREEING; + static const uint32_t PSB_CORRUPT = NVS_CONST_PSB_CORRUPT; - static const uint32_t ESB_WRITTEN = 0x1; - static const uint32_t ESB_ERASED = 0x2; + static const uint32_t ESB_WRITTEN = NVS_CONST_ESB_WRITTEN; + static const uint32_t ESB_ERASED = NVS_CONST_ESB_ERASED; static const uint32_t SEC_SIZE; - static const size_t ENTRY_SIZE = 32; - static const size_t ENTRY_COUNT = 126; - static const uint32_t INVALID_ENTRY = 0xffffffff; + static const size_t ENTRY_SIZE = NVS_CONST_ENTRY_SIZE; + static const size_t ENTRY_COUNT = NVS_CONST_ENTRY_COUNT; + static const uint32_t INVALID_ENTRY = NVS_CONST_INVALID_ENTRY; static const size_t CHUNK_MAX_SIZE = ENTRY_SIZE * (ENTRY_COUNT - 1); - static const uint8_t NS_INDEX = 0; - static const uint8_t NS_ANY = 255; + static const uint8_t NS_INDEX = NVS_CONST_NS_INDEX; + static const uint8_t NS_ANY = NVS_CONST_NS_ANY; static const uint8_t CHUNK_ANY = Item::CHUNK_ANY; - static const uint8_t NVS_VERSION = 0xfe; // Decrement to upgrade + static const uint8_t NVS_VERSION = NVS_CONST_NVS_VERSION; // Decrement to upgrade enum class PageState : uint32_t { // All bits set, default state after flash erase. Page has not been initialized yet. - UNINITIALIZED = 0xffffffff, + UNINITIALIZED = NVS_CONST_PAGE_STATE_UNINITIALIZED, // Page is initialized, and will accept writes. - ACTIVE = UNINITIALIZED & ~PSB_INIT, + ACTIVE = NVS_CONST_PAGE_STATE_ACTIVE, // Page is marked as full and will not accept new writes. - FULL = ACTIVE & ~PSB_FULL, + FULL = NVS_CONST_PAGE_STATE_FULL, // Data is being moved from this page to a new one. - FREEING = FULL & ~PSB_FREEING, + FREEING = NVS_CONST_PAGE_STATE_FREEING, // Page was found to be in a corrupt and unrecoverable state. // Instead of being erased immediately, it will be kept for diagnostics and data recovery. // It will be erased once we run out out free pages. - CORRUPT = FREEING & ~PSB_CORRUPT, + CORRUPT = NVS_CONST_PAGE_STATE_CORRUPT, // Page object wasn't loaded from flash memory INVALID = 0 @@ -166,11 +167,11 @@ protected: }; enum class EntryState { - EMPTY = 0x3, // 0b11, default state after flash erase - WRITTEN = EMPTY & ~ESB_WRITTEN, // entry was written - ERASED = WRITTEN & ~ESB_ERASED, // entry was written and then erased - ILLEGAL = 0x1, // only possible if flash is inconsistent - INVALID = 0x4 // entry is in inconsistent state (write started but ESB_WRITTEN has not been set yet) + EMPTY = NVS_CONST_ENTRY_STATE_EMPTY, // 0b11, default state after flash erase + WRITTEN = NVS_CONST_ENTRY_STATE_WRITTEN, // entry was written + ERASED = NVS_CONST_ENTRY_STATE_ERASED, // entry was written and then erased + ILLEGAL = NVS_CONST_ENTRY_STATE_ILLEGAL, // only possible if flash is inconsistent + INVALID = NVS_CONST_ENTRY_STATE_INVALID // entry is in inconsistent state (write started but ESB_WRITTEN has not been set yet) }; esp_err_t mLoadEntryTable(); @@ -225,9 +226,9 @@ protected: Partition *mPartition; - static const uint32_t HEADER_OFFSET = 0; - static const uint32_t ENTRY_TABLE_OFFSET = HEADER_OFFSET + 32; - static const uint32_t ENTRY_DATA_OFFSET = ENTRY_TABLE_OFFSET + 32; + static const uint32_t HEADER_OFFSET = NVS_CONST_PAGE_HEADER_OFFSET; + static const uint32_t ENTRY_TABLE_OFFSET = NVS_CONST_PAGE_ENTRY_TABLE_OFFSET; + static const uint32_t ENTRY_DATA_OFFSET = NVS_CONST_PAGE_ENTRY_DATA_OFFSET; static_assert(sizeof(Header) == 32, "header size must be 32 bytes"); static_assert(ENTRY_TABLE_OFFSET % 32 == 0, "entry table offset should be aligned"); diff --git a/components/nvs_flash/test_apps_bootloader/CMakeLists.txt b/components/nvs_flash/test_apps_bootloader/CMakeLists.txt new file mode 100644 index 0000000000..d0cf30b7e6 --- /dev/null +++ b/components/nvs_flash/test_apps_bootloader/CMakeLists.txt @@ -0,0 +1,13 @@ +# The following lines of boilerplate have to be in your project's +# CMakeLists in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.16) + +set(EXTRA_COMPONENT_DIRS "$ENV{IDF_PATH}/tools/unit-test-app/components") + +# set(BOOT_DIR "$ENV{IDF_PATH}/components/nvs_flash/bootloader_test/bootloader_components/nvs_bootloader") +# set(BOOT_TEST_DIR "$ENV{IDF_PATH}/components/nvs_flash/bootloader_test/bootloader_components/nvs_bootloader_test") +list(PREPEND SDKCONFIG_DEFAULTS "$ENV{IDF_PATH}/tools/test_apps/configs/sdkconfig.debug_helpers" "sdkconfig.defaults") +list(APPEND EXTRA_COMPONENT_DIRS ${BOOT_DIR} ${BOOT_TEST_DIR}) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(nvs_bootloader_test) diff --git a/components/nvs_flash/test_apps_bootloader/README.md b/components/nvs_flash/test_apps_bootloader/README.md new file mode 100644 index 0000000000..7b96141437 --- /dev/null +++ b/components/nvs_flash/test_apps_bootloader/README.md @@ -0,0 +1,2 @@ +| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C5 | ESP32-C6 | ESP32-C61 | ESP32-H2 | ESP32-P4 | ESP32-S2 | ESP32-S3 | +| ----------------- | ----- | -------- | -------- | -------- | -------- | --------- | -------- | -------- | -------- | -------- | diff --git a/components/nvs_flash/test_apps_bootloader/main/CMakeLists.txt b/components/nvs_flash/test_apps_bootloader/main/CMakeLists.txt new file mode 100644 index 0000000000..978ea7e30f --- /dev/null +++ b/components/nvs_flash/test_apps_bootloader/main/CMakeLists.txt @@ -0,0 +1,7 @@ +idf_component_register(SRCS "test_app_main.c" "test_nvs_bootloader.c" + INCLUDE_DIRS "." + REQUIRES unity nvs_flash + WHOLE_ARCHIVE + ) + +nvs_create_partition_image(nvs ../partition_nvs_data.csv FLASH_IN_PROJECT) diff --git a/components/nvs_flash/test_apps_bootloader/main/test_app_main.c b/components/nvs_flash/test_apps_bootloader/main/test_app_main.c new file mode 100644 index 0000000000..8c2912aff9 --- /dev/null +++ b/components/nvs_flash/test_apps_bootloader/main/test_app_main.c @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "unity.h" +#include "unity_test_runner.h" +#include "esp_heap_caps.h" + +// There should be no memory allocated during the test. +#define TEST_MEMORY_INITIAL_LEAK_THRESHOLD (-700) + + +static size_t before_free_8bit; +static size_t before_free_32bit; +static size_t actual_leak_threshold = TEST_MEMORY_INITIAL_LEAK_THRESHOLD; + + +static void check_leak(size_t before_free, size_t after_free, const char *type) +{ + ssize_t delta = after_free - before_free; + printf("MALLOC_CAP_%s: Before %u bytes free, After %u bytes free (delta %d)\n", type, before_free, after_free, delta); + TEST_ASSERT_MESSAGE(delta >= actual_leak_threshold, "memory leak"); +} + +void setUp(void) +{ + // load the partition table before measuring the initial free heap size. + before_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT); + before_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT); +} + +void tearDown(void) +{ + size_t after_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT); + size_t after_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT); + check_leak(before_free_8bit, after_free_8bit, "8BIT"); + check_leak(before_free_32bit, after_free_32bit, "32BIT"); + + // Reset the threshold to the zero after initial check + actual_leak_threshold = 0; +} + +void app_main(void) +{ + printf("Running nvs bootloader support component tests\n"); + unity_run_menu(); +} diff --git a/components/nvs_flash/test_apps_bootloader/main/test_nvs_bootloader.c b/components/nvs_flash/test_apps_bootloader/main/test_nvs_bootloader.c new file mode 100644 index 0000000000..9d6a4843e9 --- /dev/null +++ b/components/nvs_flash/test_apps_bootloader/main/test_nvs_bootloader.c @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include "string.h" +#include "esp_err.h" + +#include "unity.h" +#include "nvs_bootloader.h" + +TEST_CASE("Verify nvs bootloader read_list result_code and value if bootloader read is successful", "[nvs_bootloader]") +{ + nvs_bootloader_read_list_t read_list[] = { +// {namespace_name, key_name, value_type, result_code, value, namespace_index}} + { .namespace_name = "storage", .key_name = "u8_key", .value_type = NVS_TYPE_U8 }, //0 OK + { .namespace_name = "storage", .key_name = "u16_key", .value_type = NVS_TYPE_U16 }, //1 OK + { .namespace_name = "storage", .key_name = "u32_key", .value_type = NVS_TYPE_U32 }, //2 OK + { .namespace_name = "storage", .key_name = "i32_key", .value_type = NVS_TYPE_I32 }, //3 OK + { .namespace_name = "storage", .key_name = "i8_key", .value_type = NVS_TYPE_U8 }, //4 Type mismatch + { .namespace_name = "storage", .key_name = "i16_key", .value_type = NVS_TYPE_I16 }, //5 Not found + }; + uint8_t size = sizeof(read_list) / sizeof(read_list[0]); + + TEST_ESP_OK(nvs_bootloader_read("nvs", size, read_list)); + + TEST_ASSERT_EQUAL(ESP_OK, read_list[0].result_code); + TEST_ASSERT_EQUAL(ESP_OK, read_list[1].result_code); + TEST_ASSERT_EQUAL(ESP_OK, read_list[2].result_code); + TEST_ASSERT_EQUAL(ESP_OK, read_list[3].result_code); + TEST_ASSERT_EQUAL(ESP_ERR_NVS_TYPE_MISMATCH, read_list[4].result_code); + TEST_ASSERT_EQUAL(ESP_ERR_NVS_NOT_FOUND, read_list[5].result_code); + + TEST_ASSERT_EQUAL(255, read_list[0].value.u8_val); + TEST_ASSERT_EQUAL(65535, read_list[1].value.u16_val); + TEST_ASSERT_EQUAL(4294967295, read_list[2].value.u32_val); + TEST_ASSERT_EQUAL(-2147483648, read_list[3].value.i32_val); +} + +TEST_CASE("Verify nvs bootloader read_list result_code if bootloader read fails", "[nvs_bootloader]") +{ + nvs_bootloader_read_list_t read_list[] = { +// {namespace_name, key_name, value_type, result_code, value, namespace_index}} + { .namespace_name = "too_long_namespace", .key_name = "i32_key", .value_type = NVS_TYPE_I32 }, //0 Invalid name + { .namespace_name = "nvs", .key_name = "too_long_key_name", .value_type = NVS_TYPE_I32 }, //1 Key too long + { .namespace_name = "nvs", .key_name = "str_key", .value_type = NVS_TYPE_BLOB }, //2 Invalid arg + { .namespace_name = "nvs", .key_name = "i32_key", .value_type = NVS_TYPE_I32 }, //3 Not found + }; + uint8_t size = sizeof(read_list) / sizeof(read_list[0]); + + esp_err_t ret = nvs_bootloader_read("nvs", size, read_list); + + TEST_ASSERT_EQUAL(ESP_ERR_INVALID_ARG, ret); + TEST_ASSERT_EQUAL(ESP_ERR_NVS_INVALID_NAME, read_list[0].result_code); + TEST_ASSERT_EQUAL(ESP_ERR_NVS_KEY_TOO_LONG, read_list[1].result_code); + TEST_ASSERT_EQUAL(ESP_ERR_INVALID_ARG, read_list[2].result_code); + TEST_ASSERT_EQUAL(ESP_ERR_NVS_NOT_FOUND, read_list[3].result_code); +} + +TEST_CASE("Verify nvs_bootloader_read failure cases", "[nvs_bootloader]") +{ + nvs_bootloader_read_list_t read_list[] = { +// {namespace_name, key_name, value_type, result_code, value, namespace_index}} + { "nvs", "i32_key", NVS_TYPE_I32, ESP_OK, {0}, 0} + }; + uint8_t size = sizeof(read_list) / sizeof(read_list[0]); + + esp_err_t ret = nvs_bootloader_read("nvs_partition_name_too_long", size, read_list); + TEST_ASSERT_EQUAL(ESP_ERR_NVS_INVALID_NAME, ret); + + ret = nvs_bootloader_read("nvs_part", size, read_list); + TEST_ASSERT_EQUAL(ESP_ERR_NVS_PART_NOT_FOUND, ret); +} diff --git a/components/nvs_flash/test_apps_bootloader/partition_nvs_data.csv b/components/nvs_flash/test_apps_bootloader/partition_nvs_data.csv new file mode 100644 index 0000000000..8e0cc35695 --- /dev/null +++ b/components/nvs_flash/test_apps_bootloader/partition_nvs_data.csv @@ -0,0 +1,13 @@ +# Sample csv file +key,type,encoding,value +storage,namespace,, +u8_key,data,u8,255 +i8_key,data,i8,-128 +u16_key,data,u16,65535 +u32_key,data,u32,4294967295 +i32_key,data,i32,-2147483648 +str_key,data,string,"Lorem ipsum dolor sit amet, consectetur adipiscing elit. +Fusce quis risus justo. +Suspendisse egestas in nisi sit amet auctor. +Pellentesque rhoncus dictum sodales. +In justo erat, viverra at interdum eget, interdum vel dui." diff --git a/components/nvs_flash/test_apps_bootloader/pytest_nvs_bootloader_support.py b/components/nvs_flash/test_apps_bootloader/pytest_nvs_bootloader_support.py new file mode 100644 index 0000000000..b7562bf5a0 --- /dev/null +++ b/components/nvs_flash/test_apps_bootloader/pytest_nvs_bootloader_support.py @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: CC0-1.0 +import pytest +from pytest_embedded import Dut + + +@pytest.mark.esp32 +@pytest.mark.esp32c3 +@pytest.mark.generic +@pytest.mark.parametrize('config', ['default'], indirect=True) +def test_nvs_bootloader_support(dut: Dut) -> None: + dut.run_all_single_board_cases() diff --git a/components/nvs_flash/test_apps_bootloader/sdkconfig.ci.default b/components/nvs_flash/test_apps_bootloader/sdkconfig.ci.default new file mode 100644 index 0000000000..afab2a1ab7 --- /dev/null +++ b/components/nvs_flash/test_apps_bootloader/sdkconfig.ci.default @@ -0,0 +1,4 @@ +# This "default" configuration is appended to all other configurations +# The contents of "sdkconfig.debug_helpers" is also appended to all other configurations (see CMakeLists.txt) +CONFIG_ESP_TASK_WDT_INIT=n +CONFIG_PARTITION_TABLE_OFFSET=0xA000 diff --git a/components/nvs_flash/test_apps_bootloader/sdkconfig.defaults b/components/nvs_flash/test_apps_bootloader/sdkconfig.defaults new file mode 100644 index 0000000000..afab2a1ab7 --- /dev/null +++ b/components/nvs_flash/test_apps_bootloader/sdkconfig.defaults @@ -0,0 +1,4 @@ +# This "default" configuration is appended to all other configurations +# The contents of "sdkconfig.debug_helpers" is also appended to all other configurations (see CMakeLists.txt) +CONFIG_ESP_TASK_WDT_INIT=n +CONFIG_PARTITION_TABLE_OFFSET=0xA000 diff --git a/docs/doxygen/Doxyfile b/docs/doxygen/Doxyfile index 8a6d91db27..d91ca083ed 100644 --- a/docs/doxygen/Doxyfile +++ b/docs/doxygen/Doxyfile @@ -284,6 +284,7 @@ INPUT = \ $(PROJECT_PATH)/components/mqtt/esp-mqtt/include/mqtt_client.h \ $(PROJECT_PATH)/components/nvs_flash/include/nvs_flash.h \ $(PROJECT_PATH)/components/nvs_flash/include/nvs.h \ + $(PROJECT_PATH)/components/nvs_flash/include/nvs_bootloader.h \ $(PROJECT_PATH)/components/nvs_sec_provider/include/nvs_sec_provider.h \ $(PROJECT_PATH)/components/openthread/include/esp_openthread_border_router.h \ $(PROJECT_PATH)/components/openthread/include/esp_openthread_lock.h \ diff --git a/docs/en/api-guides/bootloader.rst b/docs/en/api-guides/bootloader.rst index 8c19b5535c..75105fe08a 100644 --- a/docs/en/api-guides/bootloader.rst +++ b/docs/en/api-guides/bootloader.rst @@ -184,6 +184,8 @@ The current bootloader implementation allows a project to extend it or modify it * :example:`custom_bootloader/bootloader_hooks` presents how to connect some hooks to the bootloader initialization * :example:`custom_bootloader/bootloader_override` presents how to override the bootloader implementation -In the bootloader space, you cannot use the drivers and functions from other components. If necessary, then the required functionality should be placed in the project's `bootloader_components` directory (note that this will increase its size). +In the bootloader space, you cannot use the drivers and functions from other components unless they explicitly support run in bootloader. If necessary, then the required functionality should be placed in the project's `bootloader_components` directory (note that this will increase its size). Examples of components that can be used in the bootloader are: + +* :example:`storage/nvs_bootloader` If the bootloader grows too large then it can collide with the partition table, which is flashed at offset 0x8000 by default. Increase the :ref:`partition table offset ` value to place the partition table later in the flash. This increases the space available for the bootloader. diff --git a/docs/en/api-reference/storage/index.rst b/docs/en/api-reference/storage/index.rst index f3f184270f..618cec3217 100644 --- a/docs/en/api-reference/storage/index.rst +++ b/docs/en/api-reference/storage/index.rst @@ -27,6 +27,7 @@ For information about storage security, please refer to :doc:`Storage Security < fatfsgen mass_mfg.rst nvs_flash + nvs_bootloader nvs_encryption nvs_partition_gen.rst nvs_partition_parse.rst @@ -52,6 +53,8 @@ Examples - Shows the use of the C-style API to read and write integer data types in NVS flash. * - :example:`nvs_rw_value_cxx ` - Shows the use of the C++-style API to read and write integer data types in NVS flash. + * - :example:`nvs_bootloader ` + - Shows the use of the API available to the bootloader code to read NVS data. * - :example:`nvsgen ` - Demonstrates how to use the Python-based NVS image generation tool to create an NVS partition image from the contents of a CSV file. diff --git a/docs/en/api-reference/storage/nvs_bootloader.rst b/docs/en/api-reference/storage/nvs_bootloader.rst new file mode 100644 index 0000000000..fa0049b795 --- /dev/null +++ b/docs/en/api-reference/storage/nvs_bootloader.rst @@ -0,0 +1,31 @@ +NVS Bootloader +============== + +:link_to_translation:`zh_CN:[中文]` + +Overview +-------- + +This guide provides an overview of the NVS functionality available for custom bootloader code, along with its limitations. + +Due to the constraints of the bootloader runtime environment, custom bootloader code cannot use the full NVS API directly. Instead, NVS provides a simplified API that offers read-only access to NVS data. + +The API supports reading all NVS datatypes except for blobs. One call to the API can read multiple NVS entries at once. Values can be read from different namespaces within the same NVS partition. The array of input-output structures serves as placeholders for the data read from NVS, with a fixed size of up to 8 bytes. + +To read string entries, the API requires the caller to provide a buffer and its size, due to the heap memory allocation restriction in the bootloader. + +Application Example +------------------- + +You can find code examples in the :example:`storage` directory of ESP-IDF examples: + +:example:`storage/nvs_bootloader` + +This section demonstrates how to prepare data in the input-output structure for various data types, namespaces, and keys. It includes an example of reading string data from NVS. + +The example also shows how to check if a read operation was successful, or if there were inconsistencies in the data or if certain values were not found in NVS. The example prints the values (or error codes) returned by the API to the console. + +API Reference +------------- + +.. include-build-file:: inc/nvs_bootloader.inc diff --git a/docs/en/api-reference/storage/nvs_flash.rst b/docs/en/api-reference/storage/nvs_flash.rst index 572fe59c9d..a5025ae972 100644 --- a/docs/en/api-reference/storage/nvs_flash.rst +++ b/docs/en/api-reference/storage/nvs_flash.rst @@ -37,6 +37,13 @@ Although not recommended, NVS can store tens of thousands of keys and NVS partit This option is available in the nvs_flash component of the menuconfig menu when SPIRAM is enabled and and :ref:`CONFIG_SPIRAM_USE` is set to ``CONFIG_SPIRAM_USE_CAPS_ALLOC``. .. note:: Using SPI-connected PSRAM slows down NVS API for integer operations by an approximate factor of 2.5. +.. _nvs_bootloader: + +Use of NVS in Bootloader code +----------------------------- + +The standard NVS API described in this guide is available to the running application. It is also possible to read data from NVS in the custom bootloader code. More information can be found in the :doc:`nvs_bootloader` guide. + Keys and Values ^^^^^^^^^^^^^^^ diff --git a/docs/zh_CN/api-guides/bootloader.rst b/docs/zh_CN/api-guides/bootloader.rst index a8537c1b9f..dc680114f4 100644 --- a/docs/zh_CN/api-guides/bootloader.rst +++ b/docs/zh_CN/api-guides/bootloader.rst @@ -184,6 +184,8 @@ ESP-IDF 二级引导加载程序位于 flash 的 {IDF_TARGET_CONFIG_BOOTLOADER_O * :example:`custom_bootloader/bootloader_hooks` 介绍了如何将钩子与引导加载程序初始化连接。 * :example:`custom_bootloader/bootloader_override` 介绍了如何覆盖引导加载程序的实现。 -在引导加载程序的代码中，用户不能使用其他组件提供的驱动和函数，如果确实需要，请将该功能的实现部分放在项目的 `bootloader_components` 目录中（注意，这会增加引导加载程序的大小）。 +在引导加载程序的代码中，不能使用其他组件提供的驱动和函数，除非某个驱动或函数明确声明支持在引导加载程序中运行。如果确实需要，请将所需功能放在项目的 `bootloader_components` 目录中（注意，这会增加引导加载程序的大小）。以下是可以在引导加载程序中使用的组件示例： + +* :example:`storage/nvs_bootloader` 如果引导加载程序过大，则可能与内存中的分区表重叠，分区表默认烧录在偏移量 0x8000 处。增加 :ref:`分区表偏移量 ` ，将分区表放在 flash 中靠后的区域，这样可以增加引导加载程序的可用空间。 diff --git a/docs/zh_CN/api-reference/storage/index.rst b/docs/zh_CN/api-reference/storage/index.rst index 3b71db1be0..38024031aa 100644 --- a/docs/zh_CN/api-reference/storage/index.rst +++ b/docs/zh_CN/api-reference/storage/index.rst @@ -27,6 +27,7 @@ fatfsgen mass_mfg.rst nvs_flash + nvs_bootloader nvs_encryption nvs_partition_gen.rst nvs_partition_parse.rst @@ -52,6 +53,8 @@ - 演示了如何在 NVS flash 中使用 C 语言 API 读写整数数据类型。 * - :example:`nvs_rw_value ` - 演示了如何在 NVS flash 中使用 C++ 语言 API 读写整数数据类型。 + * - :example:`nvs_bootloader ` + - 演示了如何使用引导加载程序代码中可用的 API 来读取 NVS 数据。 * - :example:`nvsgen ` - 演示了如何使用基于 Python 的 NVS 镜像生成工具，根据 CSV 文件内容创建 NVS 分区镜像。 diff --git a/docs/zh_CN/api-reference/storage/nvs_bootloader.rst b/docs/zh_CN/api-reference/storage/nvs_bootloader.rst new file mode 100644 index 0000000000..dc6a151947 --- /dev/null +++ b/docs/zh_CN/api-reference/storage/nvs_bootloader.rst @@ -0,0 +1,4 @@ +NVS Bootloader +============== + +:link_to_translation:`en:[English]` diff --git a/docs/zh_CN/api-reference/storage/nvs_flash.rst b/docs/zh_CN/api-reference/storage/nvs_flash.rst index 65a2149195..59d131cf51 100644 --- a/docs/zh_CN/api-reference/storage/nvs_flash.rst +++ b/docs/zh_CN/api-reference/storage/nvs_flash.rst @@ -37,6 +37,13 @@ NVS 支持至多存储数万个键，且 NVS 分区的大小也可以达到几 启用 SPIRAM 且将 :ref:`CONFIG_SPIRAM_USE` 设为 ``CONFIG_SPIRAM_USE_CAPS_ALLOC`` 后，即可在 menuconfig 菜单的 nvs_flash 组件中启用 :ref:`CONFIG_NVS_ALLOCATE_CACHE_IN_SPIRAM` 选项。 .. note:: 使用带有 SPI 连接的 PSRAM 会导致 NVS API 的整数操作速度减慢约 2.5 倍。 +.. _nvs_bootloader: + +在引导加载程序代码中使用 NVS +--------------------------------- + +运行中的应用程序可使用本指南中描述的标准 NVS API。也可以在自定义引导加载程序代码中从 NVS 读取数据。更多信息请参阅 :doc:`nvs_bootloader` 指南。 + 键值对 ^^^^^^^^^^^^^^^ diff --git a/examples/custom_bootloader/README.md b/examples/custom_bootloader/README.md index df7112721d..4137a7c6cf 100644 --- a/examples/custom_bootloader/README.md +++ b/examples/custom_bootloader/README.md @@ -33,3 +33,15 @@ totally or partially re-write it, in order to include, remove or modify parts of it. Thanks to this, it will be fully customizable. This shall only be used if heavy changes are required and they cannot be done with hooks or within an application. + +## Component code supported in bootloader + +Links to the resources of component APIs supporting bootloader code are +listed below. + +### NVS Flash + +NVS flash component provides limited, read-only API. It is documented in +[NVS Flash Component for Bootloader](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/nvs_bootloader.html) + +Examples of it's use are located in `examples/storage/nvs_bootloader` diff --git a/examples/storage/README.md b/examples/storage/README.md index ebfb75833d..ff4d16417d 100644 --- a/examples/storage/README.md +++ b/examples/storage/README.md @@ -12,6 +12,8 @@ The examples are grouped into sub-directories by category. Each category directo * `custom_flash_driver` example demonstrates how to implement your own flash chip driver by overriding the default driver. * `emmc` example demonstrates how to use an eMMC chip with an ESP device. * `ext_flash_fatfs` example demonstrates how to use FATFS partition with external SPI FLASH chip. +* `fatfsgen` example demonstrates how to use FATFS partition generator +* `nvs_bootloader` example demonstrates how to read data from NVS in the bootloader code. * `nvs_rw_blob` example demonstrates how to read and write a single integer value and a blob (binary large object) using NVS to preserve them between ESP module restarts. * `nvs_rw_value` example demonstrates how to read and write a single integer value using NVS. * `nvs_rw_value_cxx` example demonstrates how to read and write a single integer value using NVS (it uses the C++ NVS handle API). diff --git a/examples/storage/nvs_bootloader/CMakeLists.txt b/examples/storage/nvs_bootloader/CMakeLists.txt new file mode 100644 index 0000000000..f0935b0e9f --- /dev/null +++ b/examples/storage/nvs_bootloader/CMakeLists.txt @@ -0,0 +1,11 @@ +# For more information about build system see +# https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html +# The following five lines of boilerplate have to be in your project's +# CMakeLists in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.16) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) + +idf_build_set_property(__BUILD_COMPONENT_DEPGRAPH_ENABLED 1) + +project(main) diff --git a/examples/storage/nvs_bootloader/README.md b/examples/storage/nvs_bootloader/README.md new file mode 100644 index 0000000000..a5247f64a3 --- /dev/null +++ b/examples/storage/nvs_bootloader/README.md @@ -0,0 +1,125 @@ +| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C5 | ESP32-C6 | ESP32-C61 | ESP32-H2 | ESP32-P4 | ESP32-S2 | ESP32-S3 | +| ----------------- | ----- | -------- | -------- | -------- | -------- | --------- | -------- | -------- | -------- | -------- | + +# NVS Bootloader + +The purpose of this example is to show how to use the simplified, read-only API of NVS flash that can be used as a part of bootloader + +## Usage of this example: + +Simply compile it: +``` +idf.py build +``` + +Then flash it and open the monitor with the following command: +``` +idf.py flash monitor +``` + +If everything went well, the console output should contain three blocks of log messages. +For detailed explanation, please see the comments in the source code file `nvs_bootloader_example.c` + +First block shows indication of invalid parameters. + +``` +I (765) nvs_bootloader_example_utils: Showing request data only +I (765) nvs_bootloader_example_utils: ## Namespace Key DT +I (766) nvs_bootloader_example_utils: 0 sunny_day u8 U8 +I (766) nvs_bootloader_example_utils: 1 too_long_sunny_day u8 I8 +I (766) nvs_bootloader_example_utils: 2 sunny_day too_long_dark_key I32 +I (767) nvs_bootloader_example_utils: 3 clowny_day blobeee BLOB +I (767) nvs_bootloader_example_utils: 4 sunny_day string_10_chars STR +I (767) nvs_bootloader_example_utils: 5 sunny_day string_10_chars STR +E (769) nvs_bootloader_example: Invalid arguments passed to the read function +I (770) nvs_bootloader_example_utils: Request data with invalid arguments. Function returned ESP_ERR_INVALID_ARG +I (770) nvs_bootloader_example_utils: ## Result Namespace Key DT +I (771) nvs_bootloader_example_utils: 0 ESP_ERR_NVS_NOT_FOUND sunny_day u8 U8 +E (772) nvs_bootloader_example_utils: 1 ESP_ERR_NVS_INVALID_NAME too_long_sunny_day u8 I8 +E (772) nvs_bootloader_example_utils: 2 ESP_ERR_NVS_KEY_TOO_LONG sunny_day too_long_dark_key I32 +E (772) nvs_bootloader_example_utils: 3 ESP_ERR_INVALID_ARG clowny_day blobeee BLOB +E (773) nvs_bootloader_example_utils: 4 ESP_ERR_INVALID_SIZE sunny_day string_10_chars STR +E (773) nvs_bootloader_example_utils: 5 ESP_ERR_INVALID_SIZE sunny_day string_10_chars STR +``` + +Second block shows indication of inconsistencies detected during the data reading. There can be mix of correctly read entries and entries with errors. + +``` +I (779) nvs_bootloader_example_utils: Showing request data only +I (779) nvs_bootloader_example_utils: ## Namespace Key DT +I (779) nvs_bootloader_example_utils: 0 sunny_day u8 I8 +I (780) nvs_bootloader_example_utils: 1 sunny_day i32_ I32 +I (780) nvs_bootloader_example_utils: 2 clowny_day i8 I8 +I (780) nvs_bootloader_example_utils: 3 sunny_day string_10_chars STR +I (780) nvs_bootloader_example_utils: 4 sunny_day u32 U32 +I (781) nvs_bootloader_example_utils: 5 sunny_day u32 U32 +I (1227) nvs_bootloader_example: Data read from NVS partition +I (1227) nvs_bootloader_example_utils: Result data. Function returned ESP_OK +I (1227) nvs_bootloader_example_utils: ## Result Namespace Key DT Value +E (1228) nvs_bootloader_example_utils: 0 ESP_ERR_NVS_TYPE_MISMATCH sunny_day u8 I8 +E (1228) nvs_bootloader_example_utils: 1 ESP_ERR_NVS_NOT_FOUND sunny_day i32_ I32 +E (1228) nvs_bootloader_example_utils: 2 ESP_ERR_NVS_NOT_FOUND clowny_day i8 I8 +E (1229) nvs_bootloader_example_utils: 3 ESP_ERR_INVALID_SIZE sunny_day string_10_chars STR +I (1229) nvs_bootloader_example_utils: 4 ESP_OK sunny_day u32 U32 4294967295 +E (1229) nvs_bootloader_example_utils: 5 ESP_ERR_NVS_NOT_FOUND sunny_day u32 U32 +``` + +Final block of log messages shows the successful reading of various data + +``` +I (1230) nvs_bootloader_example_utils: Showing request data only +I (1230) nvs_bootloader_example_utils: ## Namespace Key DT +I (1230) nvs_bootloader_example_utils: 0 sunny_day u8 U8 +I (1231) nvs_bootloader_example_utils: 1 sunny_day i32 I32 +I (1231) nvs_bootloader_example_utils: 2 cloudy_day i8 I8 +I (1231) nvs_bootloader_example_utils: 3 sunny_day u32 U32 +I (1231) nvs_bootloader_example_utils: 4 sunny_day i8 I8 +I (1232) nvs_bootloader_example_utils: 5 sunny_day u16 U16 +I (1232) nvs_bootloader_example_utils: 6 sunny_day i16 I16 +I (1232) nvs_bootloader_example_utils: 7 sunny_day string_10_chars STR +I (1665) nvs_bootloader_example: Data read from NVS partition +I (1665) nvs_bootloader_example_utils: Result data. Function returned ESP_OK +I (1666) nvs_bootloader_example_utils: ## Result Namespace Key DT Value +I (1666) nvs_bootloader_example_utils: 0 ESP_OK sunny_day u8 U8 255 +I (1666) nvs_bootloader_example_utils: 1 ESP_OK sunny_day i32 I32 -2147483648 +I (1667) nvs_bootloader_example_utils: 2 ESP_OK cloudy_day i8 I8 -13 +I (1667) nvs_bootloader_example_utils: 3 ESP_OK sunny_day u32 U32 4294967295 +I (1668) nvs_bootloader_example_utils: 4 ESP_OK sunny_day i8 I8 -128 +I (1668) nvs_bootloader_example_utils: 5 ESP_OK sunny_day u16 U16 65535 +I (1668) nvs_bootloader_example_utils: 6 ESP_OK sunny_day i16 I16 -20000 +I (1669) nvs_bootloader_example_utils: 7 ESP_OK sunny_day string_10_chars STR Text_67890 +``` + +At the end the firmware starts regular application and just prints the message: + +``` +I (3212) main_task: Calling app_main() +User application is loaded and running. +I (3212) main_task: Returned from app_main() +``` + +## Organisation of this example + +The code demonstrating the use of nvs in bootloader is in `bootloader_components/nvs_bootloader_example`. This code is executed during bootloader run. +Bootloader hooks technique is used to extend the default bootloader. The main function demonstrating the functionality is `bootloader_after_init()` +in the `src/nvs_bootloader_example` + +Below is a short explanation of files in the project folder. + +``` +├── CMakeLists.txt +├── main +│   ├── CMakeLists.txt +│   └── main.c Regular application, just prints message and ends +├── bootloader_components +│   └── nvs_bootloader_example +│   ├── CMakeLists.txt +│   ├── nvs_bootloader_example_utils.c Supplementary functions for easier logging the content of request / response data +│   └── nvs_bootloader_example.c Implementation of main test. All supplenentary information in the form of comments can be found here. +├── nvs_data.csv Initial content of the nvs partition. Data to be read by the nvs_bootloader_read are here +└── README.md This is the file you are currently reading +``` + +The example creates request/response array `read_list[]`, populates it with identifiers of the data to be read. +Function `nvs_bootloader_read()` tries to find respective data in the partition (here `"nvs"`) and if the data is found, it populates the request/response array with data. For nvs entries either not found or not matching are indicated in response array as well. +Function `log_nvs_bootloader_read_list()`is used before and after reading from nvs to show request/response data to the console. diff --git a/examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/CMakeLists.txt b/examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/CMakeLists.txt new file mode 100644 index 0000000000..91038a9e13 --- /dev/null +++ b/examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/CMakeLists.txt @@ -0,0 +1,10 @@ +idf_component_register( SRCS "src/nvs_bootloader_example.c" "src/nvs_bootloader_example_utils.c" + INCLUDE_DIRS "include" + REQUIRES "nvs_flash" ) + +# We need to force GCC to integrate this static library into the +# bootloader link. Indeed, by default, as the hooks in the bootloader are weak, +# the linker would just ignore the symbols in the extra. (i.e. not strictly +# required) +# To do so, we need to define the symbol (function) `bootloader_hooks_include` +# within hooks.c source file. diff --git a/examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/include/nvs_bootloader_example_utils.h b/examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/include/nvs_bootloader_example_utils.h new file mode 100644 index 0000000000..9b0ff5a610 --- /dev/null +++ b/examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/include/nvs_bootloader_example_utils.h @@ -0,0 +1,15 @@ +/* + * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "nvs_bootloader.h" + +// logs the result of the call to the nvs_bootloader_read function +void log_nvs_bootloader_read_list(const esp_err_t return_code, const nvs_bootloader_read_list_t read_list[], const size_t read_list_count); + +void log_nvs_bootloader_read_list_OK(const nvs_bootloader_read_list_t read_list[], const size_t read_list_count); +void log_nvs_bootloader_read_list_INVALID_ARG(const nvs_bootloader_read_list_t read_list[], const size_t read_list_count); +void log_nvs_bootloader_read_list_NOT_FINISHED(const nvs_bootloader_read_list_t read_list[], const size_t read_list_count); diff --git a/examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/src/nvs_bootloader_example.c b/examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/src/nvs_bootloader_example.c new file mode 100644 index 0000000000..cf83507aaf --- /dev/null +++ b/examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/src/nvs_bootloader_example.c @@ -0,0 +1,131 @@ +/* + * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ +#define LOG_LOCAL_LEVEL ESP_LOG_INFO +#include "esp_log.h" +#include "nvs_bootloader.h" +#include "nvs_bootloader_example_utils.h" + +static const char* TAG = "nvs_bootloader_example"; + +// Function used to tell the linker to include this file +// with all its symbols. +// +void bootloader_hooks_include(void){ +} + +// not used in this example +void bootloader_before_init(void) { +} + + +void log_request_call_read_evaluate_output(const char* nvs_partition_label, nvs_bootloader_read_list_t read_list[], const size_t read_list_count) { + // log the request structure before the read to see the requested keys and namespaces + // with the ESP_ERR_NOT_FINISHED return code we are just telling the log function to show the request data and omit printing the result data + // it is useful for debugging the request structure + log_nvs_bootloader_read_list(ESP_ERR_NOT_FINISHED, read_list, read_list_count); + + // call the read function + esp_err_t ret = nvs_bootloader_read(nvs_partition_label, read_list_count, read_list); + + // Error code ESP_OK means that the read function was successful and individual, per record results are stored in the read_list + if (ret == ESP_OK) { + ESP_LOGI(TAG, "Data read from NVS partition"); + + // log the request structure after the read to see the results + // some records may indicate problems with the read i.e. not found, type mismatch, etc. + log_nvs_bootloader_read_list(ret, read_list, read_list_count); + + // Error code ESP_ERR_INVALID_ARG means that the read function was called with invalid arguments + } else if(ret == ESP_ERR_INVALID_ARG) { + ESP_LOGE(TAG, "Invalid arguments passed to the function"); + + // some records may indicate problems with the input parameters + // application developer may evaluate the read_list to see what went wrong with input parameters + log_nvs_bootloader_read_list(ret, read_list, read_list_count); + + // Other error codes mean that the read function failed to read the data from the NVS partition + // The read list doesn't contain any useful data in this case + } else { + ESP_LOGE(TAG, "Failed to read NVS partition ret = %04x", ret); + } +} + + +// function hook called at the end of the bootloader standard code +// this is the 'main' function of the example +void bootloader_after_init(void) { + ESP_LOGI(TAG, "Before reading from NVS partition"); + + // we are going to read from the default nvs partition labelled 'nvs' + const char* nvs_partition_label = "nvs"; + + #define STR_BUFF_LEN 10+1 // 10 characters + null terminator + char str_buff[STR_BUFF_LEN]; + + // --- This is the request structure for the read function showing validation errors - function will return ESP_ERR_INVALID_ARG --- + nvs_bootloader_read_list_t bad_read_list_indicate_problems[] = { + { .namespace_name = "sunny_day", .key_name = "u8", .value_type = NVS_TYPE_U8 }, // ESP_ERR_NVS_NOT_FOUND + // this is correct request, not found is expected default result code + { .namespace_name = "too_long_sunny_day", .key_name = "u8", .value_type = NVS_TYPE_I8 }, // ESP_ERR_NVS_INVALID_NAME + // too long namespace name + { .namespace_name = "sunny_day", .key_name = "too_long_dark_key", .value_type = NVS_TYPE_I32 }, // ESP_ERR_NVS_KEY_TOO_LONG + // too long key name + { .namespace_name = "clowny_day", .key_name = "blobeee", .value_type = NVS_TYPE_BLOB }, // ESP_ERR_INVALID_ARG + // not supported data type + { .namespace_name = "sunny_day", .key_name = "string_10_chars", .value_type = NVS_TYPE_STR, .value.str_val = { .buff_ptr = str_buff, .buff_len = 0 } }, + // ESP_ERR_INVALID_SIZE + // buffer size is 0 + { .namespace_name = "sunny_day", .key_name = "string_10_chars", .value_type = NVS_TYPE_STR, .value.str_val = { .buff_ptr = NULL, .buff_len = 10 } } + // ESP_ERR_INVALID_SIZE + // buffer pointer is invalid + }; + + size_t bad_read_list_indicate_problems_count = sizeof(bad_read_list_indicate_problems) / sizeof(bad_read_list_indicate_problems[0]); + log_request_call_read_evaluate_output(nvs_partition_label, bad_read_list_indicate_problems, bad_read_list_indicate_problems_count); + + // --- This is the request structure for the read function showing runtime errors - function will return ESP_OK --- + // but some records will have result_code set to ESP_ERR_NVS_NOT_FOUND, ESP_ERR_NVS_TYPE_MISMATCH, ESP_ERR_INVALID_SIZE + nvs_bootloader_read_list_t good_read_list_bad_results[] = { + { .namespace_name = "sunny_day", .key_name = "u8", .value_type = NVS_TYPE_I8 }, // ESP_ERR_NVS_TYPE_MISMATCH + // data in the partition is of different type (NVS_TYPE_U8) + { .namespace_name = "sunny_day", .key_name = "i32_", .value_type = NVS_TYPE_I32 }, // ESP_ERR_NVS_NOT_FOUND + // data in the partition won't be found, because there is a typo in the key name + { .namespace_name = "clowny_day", .key_name = "i8", .value_type = NVS_TYPE_I8 }, // ESP_ERR_NVS_NOT_FOUND + // data in the partition won't be found, because there is typo in namespace name + { .namespace_name = "sunny_day", .key_name = "string_10_chars", .value_type = NVS_TYPE_STR, .value.str_val = { .buff_ptr = str_buff, .buff_len = 2 } }, + // ESP_ERR_INVALID_SIZE + // buffer is too small + { .namespace_name = "sunny_day", .key_name = "u32", .value_type = NVS_TYPE_U32 }, // ESP_OK + // this value will be read correctly + { .namespace_name = "sunny_day", .key_name = "u32", .value_type = NVS_TYPE_U32 } // ESP_ERR_NVS_NOT_FOUND + // this value won't be read as function doesn't support duplicate readings + }; + + size_t good_read_list_bad_results_count = sizeof(good_read_list_bad_results) / sizeof(good_read_list_bad_results[0]); + log_request_call_read_evaluate_output(nvs_partition_label, good_read_list_bad_results, good_read_list_bad_results_count); + + + // --- This is the request structure for the read function showing all records found--- + // function will return ESP_OK, all individual records will have result_code set to ESP_OK + // Order of the requested keys and namespaces is not important + // We mix different types of data to demonstrate the usage of different data types and also mix-in different namespaces + // For NVS_TYPE_I* and NVS_TYPE_U* the value field is used directly to store the read value + // For NVS_TYPE_STR the value field is a structure with a pointer to the buffer and the buffer length is povided + // In this case, the buffer is a stack allocated array of 10 characters plus space for the null terminator + + nvs_bootloader_read_list_t good_read_list[] = { + { .namespace_name = "sunny_day", .key_name = "u8", .value_type = NVS_TYPE_U8 }, + { .namespace_name = "sunny_day", .key_name = "i32", .value_type = NVS_TYPE_I32 }, + { .namespace_name = "cloudy_day", .key_name = "i8", .value_type = NVS_TYPE_I8 }, // mixed in different namespace + { .namespace_name = "sunny_day", .key_name = "u16", .value_type = NVS_TYPE_U16 }, + { .namespace_name = "sunny_day", .key_name = "string_10_chars", .value_type = NVS_TYPE_STR, .value.str_val = { .buff_ptr = str_buff, .buff_len = STR_BUFF_LEN } } + }; + + size_t good_read_list_count = sizeof(good_read_list) / sizeof(good_read_list[0]); + log_request_call_read_evaluate_output(nvs_partition_label, good_read_list, good_read_list_count); + + ESP_LOGI(TAG, "Finished bootloader part"); +} diff --git a/examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/src/nvs_bootloader_example_utils.c b/examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/src/nvs_bootloader_example_utils.c new file mode 100644 index 0000000000..167e4ea00b --- /dev/null +++ b/examples/storage/nvs_bootloader/bootloader_components/nvs_bootloader_example/src/nvs_bootloader_example_utils.c @@ -0,0 +1,216 @@ +/* + * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ +#define LOG_LOCAL_LEVEL ESP_LOG_INFO +#include "esp_log.h" +#include "nvs_bootloader_example_utils.h" + +static const char* TAG = "nvs_bootloader_example_utils"; +static const char* result_code_to_string(const esp_err_t result_code); + + +/* logs the result of the call to the nvs_bootloader_read function */ +void log_nvs_bootloader_read_list(const esp_err_t return_code, const nvs_bootloader_read_list_t read_list[], const size_t read_list_count) +{ + switch (return_code) { + // read list contains data after successful read + case ESP_OK: { + ESP_LOGI(TAG, "Result data. Return code: %s", result_code_to_string(return_code)); + log_nvs_bootloader_read_list_OK(read_list, read_list_count); + break; + } + + // read list contains data failing validation + case ESP_ERR_INVALID_ARG: { + ESP_LOGI(TAG, "Request data with invalid arguments. Return code: %s", result_code_to_string(return_code)); + log_nvs_bootloader_read_list_INVALID_ARG(read_list, read_list_count); + break; + } + + // request data only + case ESP_ERR_NOT_FINISHED: { + ESP_LOGI(TAG, "Showing request data only. Return code: %s", result_code_to_string(return_code)); + log_nvs_bootloader_read_list_NOT_FINISHED(read_list, read_list_count); + break; + } + default: { + ESP_LOGI(TAG, "Function returned: %04x %s", return_code, result_code_to_string(return_code)); + break; + } + } +} + +static const char* result_code_to_string(const esp_err_t result_code) +{ + switch (result_code) { + case ESP_ERR_NVS_NOT_FOUND: return "ESP_ERR_NVS_NOT_FOUND"; + case ESP_OK: return "ESP_OK"; + case ESP_ERR_NVS_TYPE_MISMATCH: return "ESP_ERR_NVS_TYPE_MISMATCH"; + case ESP_ERR_INVALID_ARG: return "ESP_ERR_INVALID_ARG"; + case ESP_ERR_NVS_INVALID_NAME: return "ESP_ERR_NVS_INVALID_NAME"; + case ESP_ERR_NVS_KEY_TOO_LONG: return "ESP_ERR_NVS_KEY_TOO_LONG"; + case ESP_ERR_INVALID_SIZE: return "ESP_ERR_INVALID_SIZE"; + default: return "Unknown"; + } +} + +static const char* value_type_to_string(const nvs_type_t value_type) +{ + switch (value_type) { + case NVS_TYPE_U8: return "U8"; + case NVS_TYPE_U16: return "U16"; + case NVS_TYPE_U32: return "U32"; + case NVS_TYPE_U64: return "U64"; + case NVS_TYPE_I8: return "I8"; + case NVS_TYPE_I16: return "I16"; + case NVS_TYPE_I32: return "I32"; + case NVS_TYPE_I64: return "I64"; + case NVS_TYPE_STR: return "STR"; + case NVS_TYPE_BLOB: return "BLOB"; + default: return "Unknown"; + } +} + +void log_nvs_bootloader_read_list_OK(const nvs_bootloader_read_list_t read_list[], const size_t read_list_count) +{ + bool i_type, u_type = false; + uint64_t u64_val = 0; + int64_t i64_val = 0; + + ESP_LOGI(TAG, "## Result Namespace Key DT Value"); + + for (size_t i = 0; i < read_list_count; i++) { + switch (read_list[i].result_code) { + case ESP_OK: { + switch (read_list[i].value_type) { + case NVS_TYPE_U8: { + u64_val = read_list[i].value.u8_val; + u_type = true; + break; + } + case NVS_TYPE_U16: { + u64_val = read_list[i].value.u16_val; + u_type = true; + break; + } + case NVS_TYPE_U32: { + u64_val = read_list[i].value.u32_val; + u_type = true; + break; + } + case NVS_TYPE_U64: { + u64_val = read_list[i].value.u64_val; + u_type = true; + break; + } + case NVS_TYPE_I8: { + i64_val = read_list[i].value.i8_val; + i_type = true; + break; + } + case NVS_TYPE_I16: { + i64_val = read_list[i].value.i16_val; + i_type = true; + break; + } + case NVS_TYPE_I32: { + i64_val = read_list[i].value.i32_val; + i_type = true; + break; + } + case NVS_TYPE_I64: { + i64_val = read_list[i].value.i64_val; + i_type = true; + break; + } + case NVS_TYPE_STR: { + ESP_LOGI(TAG, "%2d %-25s %-16s %-16s %-3s %s", + i, + result_code_to_string(read_list[i].result_code), + read_list[i].namespace_name, + read_list[i].key_name, + value_type_to_string(read_list[i].value_type), + read_list[i].value.str_val.buff_ptr); + continue; + } + default: { + u64_val = 0; + i64_val = 0; + break; + } + + } + if (i_type) { + ESP_LOGI(TAG, "%2d %-25s %-16s %-16s %-3s %lld", + i, + result_code_to_string(read_list[i].result_code), + read_list[i].namespace_name, + read_list[i].key_name, + value_type_to_string(read_list[i].value_type), + i64_val); + } else if (u_type) { + ESP_LOGI(TAG, "%2d %-25s %-16s %-16s %-3s %llu", + i, + result_code_to_string(read_list[i].result_code), + read_list[i].namespace_name, + read_list[i].key_name, + value_type_to_string(read_list[i].value_type), + u64_val); + } + i_type = false; + u_type = false; + break; + } + default: { + // data was not found or other error + ESP_LOGE(TAG, "%2d %-25s %-16s %-16s %-3s", + i, + result_code_to_string(read_list[i].result_code), + read_list[i].namespace_name, + read_list[i].key_name, + value_type_to_string(read_list[i].value_type)); + break; + } + } + } +} + +void log_nvs_bootloader_read_list_INVALID_ARG(const nvs_bootloader_read_list_t read_list[], const size_t read_list_count) +{ + + ESP_LOGI(TAG, "## Result Namespace Key DT"); + + for (size_t i = 0; i < read_list_count; i++) { + if (read_list[i].result_code != ESP_ERR_NVS_NOT_FOUND) { + ESP_LOGE(TAG, "%2d %-25s %-16s %-16s %-3s", + i, + result_code_to_string(read_list[i].result_code), + read_list[i].namespace_name, + read_list[i].key_name, + value_type_to_string(read_list[i].value_type)); + } else { + ESP_LOGI(TAG, "%2d %-25s %-16s %-16s %-3s", + i, + result_code_to_string(read_list[i].result_code), + read_list[i].namespace_name, + read_list[i].key_name, + value_type_to_string(read_list[i].value_type)); + } + } +} + +void log_nvs_bootloader_read_list_NOT_FINISHED(const nvs_bootloader_read_list_t read_list[], const size_t read_list_count) +{ + + ESP_LOGI(TAG, "## Namespace Key DT"); + + for (size_t i = 0; i < read_list_count; i++) { + ESP_LOGI(TAG, "%2d %-16s %-16s %-3s", + i, + read_list[i].namespace_name, + read_list[i].key_name, + value_type_to_string(read_list[i].value_type)); + } +} diff --git a/examples/storage/nvs_bootloader/main/CMakeLists.txt b/examples/storage/nvs_bootloader/main/CMakeLists.txt new file mode 100644 index 0000000000..d8d0fb7a76 --- /dev/null +++ b/examples/storage/nvs_bootloader/main/CMakeLists.txt @@ -0,0 +1,3 @@ +idf_component_register(SRCS "bootloader_hooks_example_main.c" + INCLUDE_DIRS ".") +nvs_create_partition_image(nvs ../nvs_data.csv FLASH_IN_PROJECT) diff --git a/examples/storage/nvs_bootloader/main/bootloader_hooks_example_main.c b/examples/storage/nvs_bootloader/main/bootloader_hooks_example_main.c new file mode 100644 index 0000000000..281f880e80 --- /dev/null +++ b/examples/storage/nvs_bootloader/main/bootloader_hooks_example_main.c @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ +#include + +void app_main(void) +{ + /** + * Nothing special is done here, everything interesting in this example + * is done in the custom bootloader code, located in: + * `bootloader_components/nvs_bootloader_test/nvs_bootloader_test.c` + */ + printf("User application is loaded and running.\n"); +} diff --git a/examples/storage/nvs_bootloader/nvs_data.csv b/examples/storage/nvs_bootloader/nvs_data.csv new file mode 100644 index 0000000000..5079deb33a --- /dev/null +++ b/examples/storage/nvs_bootloader/nvs_data.csv @@ -0,0 +1,15 @@ +# Sample csv file +# Defines sunny_day namespace holding data for example +# string entry contains 10 characters and terminating zero +key,type,encoding,value +sunny_day,namespace,, +u8,data,u8,255 +i8,data,i8,-128 +u16,data,u16,65535 +i16,data,i16,-20000 +u32,data,u32,4294967295 +i32,data,i32,-2147483648 +string_10_chars,data,string,"Text_67890" +# defines cloudy_day namespace to show it can be read at once with sunny_day one +cloudy_day,namespace,, +i8,data,i8,-13 diff --git a/examples/storage/nvs_bootloader/pytest_nvs_bootloader.py b/examples/storage/nvs_bootloader/pytest_nvs_bootloader.py new file mode 100644 index 0000000000..66e2a50286 --- /dev/null +++ b/examples/storage/nvs_bootloader/pytest_nvs_bootloader.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: CC0-1.0 +import pytest +from pytest_embedded import Dut + + +@pytest.mark.supported_targets +@pytest.mark.generic +def test_nvs_bootloader_example(dut: Dut) -> None: + # Expect to read hooks messages and data from NVS partition + dut.expect_exact('Before reading from NVS partition') + dut.expect_exact('Finished bootloader part') + dut.expect_exact('User application is loaded and running.') diff --git a/examples/storage/nvs_bootloader/sdkconfig.defaults b/examples/storage/nvs_bootloader/sdkconfig.defaults new file mode 100644 index 0000000000..bb48f6522e --- /dev/null +++ b/examples/storage/nvs_bootloader/sdkconfig.defaults @@ -0,0 +1,4 @@ +# The size of bootloader has been increased, so the partition table offset needs adjustment +CONFIG_PARTITION_TABLE_OFFSET=0xA000 +CONFIG_BOOTLOADER_LOG_LEVEL_WARN=y +CONFIG_BOOTLOADER_LOG_LEVEL=2 diff --git a/tools/ci/check_rules_components_patterns.py b/tools/ci/check_rules_components_patterns.py index 8c5794d4aa..8c8b709dac 100755 --- a/tools/ci/check_rules_components_patterns.py +++ b/tools/ci/check_rules_components_patterns.py @@ -1,16 +1,15 @@ #!/usr/bin/env python # SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - """ Check patterns-build_components including all components files except 'test*/' """ - import glob import os import subprocess from functools import lru_cache -from typing import List, Set +from typing import List +from typing import Set import yaml from idf_ci_utils import IDF_PATH @@ -56,6 +55,7 @@ def get_components_test_files() -> Set[str]: 'components/fatfs/test_fatfsgen/**/*', 'components/heap/test_multi_heap_host/**/*', 'components/nvs_flash/test_nvs_host/**/*', + 'components/nvs_flash/test_apps_bootloader/**/*', 'components/partition_table/test_gen_esp32part_host/**/*', 'components/spiffs/test_spiffsgen/**/*', ]