feat(bootloader): add an example that implements multiboot

Add a new example for the bootloader that shows how to override it to implement multiboot.

components/log/include/esp_log_color.h

@@ -39,10 +39,12 @@ extern "C" {
 #define LOG_ANSI_COLOR_STYLE_BOLD                                   "1"
 #define LOG_ANSI_COLOR_STYLE_ITALIC                                 "3"
 #define LOG_ANSI_COLOR_STYLE_UNDERLINE                              "4"
+#define LOG_ANSI_COLOR_STYLE_REVERSE                                "7"
 // Macros that form the starting sequence for setting the text color, background color, and reset all.
 #define LOG_ANSI_COLOR(TEXT_COLOR)                                  "\033[" TEXT_COLOR "m"
 #define LOG_ANSI_COLOR_BG(BG_COLOR)                                 "\033[" BG_COLOR "m"
 #define LOG_ANSI_COLOR_RESET                                        "\033[" LOG_ANSI_COLOR_STYLE_RESET "m"
+#define LOG_ANSI_COLOR_REVERSE                                      "\033[" LOG_ANSI_COLOR_STYLE_REVERSE "m"
 // Macros that form the starting sequence for text color + style + background colors
 #define LOG_ANSI_COLOR_REGULAR(COLOR)                               LOG_ANSI_COLOR(LOG_ANSI_COLOR_STYLE_RESET ";" COLOR)
 #define LOG_ANSI_COLOR_BOLD(COLOR)                                  LOG_ANSI_COLOR(LOG_ANSI_COLOR_STYLE_BOLD ";" COLOR)
@@ -54,6 +56,9 @@ extern "C" {
 #define LOG_ANSI_COLOR_ITALIC_BACKGROUND(TEXT_COLOR, BG_COLOR)      LOG_ANSI_COLOR_ITALIC(TEXT_COLOR ";" BG_COLOR)
 #define LOG_ANSI_COLOR_UNDERLINE_BACKGROUND(TEXT_COLOR, BG_COLOR)   LOG_ANSI_COLOR_UNDERLINE(TEXT_COLOR ";" BG_COLOR)
 #define LOG_ANSI_COLOR_FORMAT(TEXT_STYLE, TEXT_COLOR, BG_COLOR)     LOG_ANSI_COLOR(TEXT_STYLE ";" TEXT_COLOR ";" BG_COLOR)
+// Miscellaneous macros for screen and cursor manipulation
+#define LOG_ANSI_CLEAR_SCREEN                                       "\033[2J"
+#define LOG_ANSI_SET_CURSOR_HOME                                    "\033[H"
 
 /**
  * Usage example of ANSI color for logs:

examples/custom_bootloader/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/custom_bootloader/bootloader_multiboot:
+  disable_test:
+    - if: IDF_TARGET not in ["esp32s3", "esp32c3"]
+      reason: Testing on two diff architectures is sufficient

examples/custom_bootloader/bootloader_multiboot/CMakeLists.txt

@@ -0,0 +1,62 @@
+# For more information about build system see
+# https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html
+# The following five lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.16)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+# "Trim" the build. Include the minimal set of components, main, and anything it depends on.
+idf_build_set_property(MINIMAL_BUILD ON)
+project(main)
+
+# For this example, we need two binaries to embed in the flash, for the `ota` partitions defined
+# in the CSV partition table.
+# It is possible to provide binaries files directly to `esptool_py_flash_target_image`, but let's
+# be clean and build two subproject that will generate the binaries to embed in the two new
+# `ota` sections.
+idf_build_get_property(idf_path IDF_PATH)
+idf_build_get_property(idf_target IDF_TARGET)
+
+# Use two examples that are available in the `examples` directory.
+file(REMOVE $ENV{IDF_PATH}/examples/get-started/hello_world/sdkconfig)
+set(hello_world_build_dir "${CMAKE_CURRENT_BINARY_DIR}/build_hello_world")
+externalproject_add(hello_world
+    SOURCE_DIR $ENV{IDF_PATH}/examples/get-started/hello_world
+    CMAKE_ARGS -DIDF_PATH=${idf_path} -DIDF_TARGET=${idf_target} -DSDKCONFIG=${hello_world_build_dir}/sdkconfig
+    BINARY_DIR "${hello_world_build_dir}"
+    INSTALL_COMMAND ""
+    BUILD_BYPRODUCTS "${hello_world_build_dir}/hello_world.bin"
+)
+# Because the CI wants all the binaries to be in the `build/` directory, we need to create a
+# custom command to move the generated file
+add_custom_target(move_hello_world ALL
+    COMMAND ${CMAKE_COMMAND} -E copy
+            "${hello_world_build_dir}/hello_world.bin"
+            "${CMAKE_CURRENT_BINARY_DIR}/hello_world.bin"
+    DEPENDS hello_world
+)
+
+
+# Do the same thing for the console example
+file(REMOVE $ENV{IDF_PATH}/examples/system/console/basic/sdkconfig)
+set(console_build_dir "${CMAKE_CURRENT_BINARY_DIR}/build_console")
+externalproject_add(console
+    SOURCE_DIR $ENV{IDF_PATH}/examples/system/console/basic
+    CMAKE_ARGS -DIDF_PATH=${idf_path} -DIDF_TARGET=${idf_target} -DSDKCONFIG=${console_build_dir}/sdkconfig
+    BINARY_DIR "${console_build_dir}"
+    INSTALL_COMMAND ""
+    BUILD_BYPRODUCTS "${console_build_dir}/console.bin"
+)
+add_custom_target(move_console ALL
+    COMMAND ${CMAKE_COMMAND} -E copy
+            "${console_build_dir}/console.bin"
+            "${CMAKE_CURRENT_BINARY_DIR}/console.bin"
+    DEPENDS console
+)
+
+# Tell esptool about the generated binaries, it will flash them when using `idf.py flash`
+partition_table_get_partition_info(offset "--partition-name hello_world" "offset")
+esptool_py_flash_target_image(flash "hello_world" "${offset}" "${CMAKE_CURRENT_BINARY_DIR}/hello_world.bin")
+
+partition_table_get_partition_info(offset "--partition-name console" "offset")
+esptool_py_flash_target_image(flash "console" "${offset}" "${CMAKE_CURRENT_BINARY_DIR}/console.bin")

examples/custom_bootloader/bootloader_multiboot/README.md

@@ -0,0 +1,69 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C5 | ESP32-C6 | ESP32-C61 | ESP32-H2 | ESP32-P4 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- | --------- | -------- | -------- | -------- | -------- |
+
+# Bootloader override
+
+(See the README.md file in the upper level for more information about bootloader examples.)
+
+The purpose of this example is to show how to override the second-stage bootloader from a regular project and make it so that it can load any IDF-based project.
+
+This example will compile two other examples, `hello_world` and `console`, and embed them in the flash, in their own partition. The goal is to give the user the possibility to choose which image to boot. As such, this example will also compile these two other examples.
+
+A custom partition table is required to manage all the images, please check `partitions.csv`.
+
+## How to use this example
+
+Simply compile it:
+```
+idf.py build
+```
+
+And flash it with the following commands:
+```
+idf.py flash
+```
+
+This custom bootloader will clear the monitor and show all the bootable partitions found:
+```
+Please select a partition to boot.
+
+FOUND 3 BOOT PARTITIONS:
+     default
+     hello_world
+     console
+```
+
+Note that the names shown in the menu are the names of the projects (set when compiled) and not the partitions names.
+
+It is possible to choose the image to boot using the arrow keys and the `enter` to validate.
+
+## Manage the bootable images
+
+The bootable images are described in the `partitions.csv` file. A `factory` partition must be present at all time since it represents the default image to boot.
+
+Any additional image must be stored in its own partition which, of course, must be big enough to store the whole bootable image, be of type `app` and have `ota_n`, where `n` is between 0 and 15 included, as a subtype.
+
+## Organization of this example
+
+This project contains an application, in the `main` directory that represents a user program that does nothing more than printing a message on the standard output.
+It also contains a `bootloader_components` directory that, as its name states, contains a component that will override the current bootloader implementation.
+
+Below is a short explanation of the files in the project folder.
+
+```
+├── CMakeLists.txt
+│   partitions.csv                              Custom partition table, containing all the bootable images
+├── main
+│   ├── CMakeLists.txt
+│   └── bootloader_multiboot_example_main.c     Example bootable application (always present)
+├── bootloader_components
+│   └── main
+│       ├── CMakeLists.txt
+│       └── bootloader_start.c                  Implementation of the second stage bootloader
+└── README.md                                   This is the file you are currently reading
+```
+
+As stated in the `README.md` file in the upper level, when the bootloader components are named `main`, it overrides the whole second stage bootloader code.
+
+Please note that this example also relies on the `hello_world` and `console` examples, before building, make sure these two examples have a clean environment and a valid `sdkconfig`.
+

examples/custom_bootloader/bootloader_multiboot/bootloader_components/main/CMakeLists.txt

@@ -0,0 +1,6 @@
+idf_component_register(SRCS "bootloader_start.c"
+                    REQUIRES bootloader bootloader_support spi_flash esp_app_format)
+
+# Use the default linker scripts
+idf_build_get_property(scripts BOOTLOADER_LINKER_SCRIPT)
+target_linker_script(${COMPONENT_LIB} INTERFACE "${scripts}")

examples/custom_bootloader/bootloader_multiboot/bootloader_components/main/bootloader_start.c

@@ -0,0 +1,234 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <stdbool.h>
+#include <sys/reent.h>
+#include "sdkconfig.h"
+#include "esp_log.h"
+#include "esp_log_color.h"
+#include "bootloader_init.h"
+#include "bootloader_utility.h"
+#include "bootloader_common.h"
+#include "bootloader_flash_priv.h"
+#include "esp_app_desc.h"
+
+#define MAX_PARTITIONS  8
+#define LABEL_LENGTH    32
+
+/**
+ * @brief Helper macro to test the equality of two partition positions
+ */
+#define PART_POS_EQUAL(p0, p1)      (((p0).offset == (p1).offset) && ((p0).size == (p1).size))
+
+static const char* TAG = "boot";
+
+/**
+ * @brief Structure describing a partition, with its name label and its position, used as an identifier here.
+ */
+typedef struct {
+    const esp_partition_pos_t* pos;
+    int number;
+    char label[LABEL_LENGTH];
+} bootloader_part_info;
+
+
+/**
+ * @brief Enumeration of the possible input events
+ */
+typedef enum {
+    INPUT_UP,
+    INPUT_DOWN,
+    INPUT_VALIDATE,
+} bootloader_input_t;
+
+
+extern char uart_rx_one_char_block(void);
+extern esp_err_t bootloader_common_get_partition_description(const esp_partition_pos_t *partition, esp_app_desc_t *app_desc);
+
+static int bootloader_utility_list_partitions(const bootloader_state_t* st, bootloader_part_info *bp, int length);
+static bootloader_input_t bootloader_get_next_event(void);
+
+/*
+ * We arrive here after the ROM bootloader finished loading this second stage bootloader from flash.
+ * The hardware is mostly uninitialized, flash cache is down and the app CPU is in reset.
+ * We do have a stack, so we can do the initialization in C.
+ */
+void __attribute__((noreturn)) call_start_cpu0(void)
+{
+    bootloader_state_t bs = {0};
+    bootloader_part_info partitions[MAX_PARTITIONS];
+
+    // 1. Hardware initialization
+    if (bootloader_init() != ESP_OK) {
+        bootloader_reset();
+    }
+
+#ifdef CONFIG_BOOTLOADER_SKIP_VALIDATE_IN_DEEP_SLEEP
+    // If this boot is a wake up from the deep sleep then go to the short way,
+    // try to load the application which worked before deep sleep.
+    // It skips a lot of checks due to it was done before (while first boot).
+    bootloader_utility_load_boot_image_from_deep_sleep();
+    // If it is not successful try to load an application as usual.
+#endif
+
+    // 2. Load partition table
+    if (!bootloader_utility_load_partition_table(&bs)) {
+        ESP_LOGE(TAG, "load partition table error!");
+        bootloader_reset();
+    }
+
+    // 3. Show the available partitions menu
+    int count = bootloader_utility_list_partitions(&bs, partitions, MAX_PARTITIONS);
+    if (count == 0) {
+        esp_rom_printf("No boot partition available, rebooting...\n");
+        esp_rom_delay_us(1000000);
+        bootloader_reset();
+    }
+
+    // 4. Show all the partitions on screen and wait for the user to select one
+
+    /* Keep track of the select entry */
+    int select = 0;
+    int validate = 0;
+    while (validate == 0) {
+        /* Clear the screen and print the partition boot selector */
+        esp_rom_printf(LOG_ANSI_CLEAR_SCREEN LOG_ANSI_SET_CURSOR_HOME
+                       "\nPlease select a partition to boot.\n\n"
+                       "FOUND %d BOOT PARTITIONS:\n", count);
+
+        for (int i = 0; i < count; i++) {
+            if (i == select) {
+                esp_rom_printf("    " LOG_ANSI_COLOR_REVERSE " %s " LOG_ANSI_COLOR_RESET "\n", partitions[i].label);
+            } else {
+                esp_rom_printf("     %s \n", partitions[i].label);
+            }
+        }
+
+        /* Get the next input from the user */
+        switch(bootloader_get_next_event()) {
+            case INPUT_UP:
+                /* Select the option above, roll it back to the bottom of the menu if negative */
+                select = (select - 1 + count) % count;
+                break;
+            case INPUT_DOWN:
+                select = (select + 1) % count;
+                break;
+            case INPUT_VALIDATE:
+                /* The partition to boot has been selected! */
+                validate = 1;
+                break;
+        }
+    }
+    esp_rom_printf("BOOTING PARTITION %s\n", partitions[select].label);
+
+    // 5. Load the app image for booting
+    bootloader_utility_load_boot_image(&bs, partitions[select].number);
+}
+
+
+static esp_err_t bootloader_utility_fill_info(const esp_partition_pos_t* partition, bootloader_part_info* info)
+{
+    esp_image_metadata_t image_data = {0};
+    esp_app_desc_t app_desc;
+    esp_err_t ret;
+
+    ret = esp_image_verify(ESP_IMAGE_VERIFY_SILENT, partition, &image_data);
+
+    if (ret == ESP_OK) {
+        ret = bootloader_common_get_partition_description(partition, &app_desc);
+    }
+
+    if (ret == ESP_OK) {
+        info->pos = partition;
+        /* Make sure to always have a NULL-byte */
+        strncpy(info->label, app_desc.project_name, LABEL_LENGTH - 1);
+        info->label[LABEL_LENGTH - 1] = 0;
+    }
+
+    return ret;
+}
+
+
+/**
+ * @brief Get bootable partitions information.
+ * We could override the function `bootloader_utility_load_partition_table` to return these information at the same time
+ * when loading and populating `bootloader_state_t` structure. But let's keep it simple and avoid copy-paste.
+ */
+static int bootloader_utility_list_partitions(const bootloader_state_t* st, bootloader_part_info *bp, int length)
+{
+    int index = 0;
+
+    // Add factory partition if available and if `bp` is big enough
+    if (st->factory.offset && index < length) {
+        if (bootloader_utility_fill_info(&st->factory, bp + index) == ESP_OK) {
+            bp[index++].number = FACTORY_INDEX;
+        } else {
+            esp_rom_printf("Error getting the description for the factory partition\n");
+        }
+    }
+
+    // Add test partition if available
+    if (st->test.offset && index < length) {
+        if (bootloader_utility_fill_info(&st->test, bp + index) == ESP_OK) {
+            bp[index++].number = TEST_APP_INDEX;
+        } else {
+            esp_rom_printf("Error getting the description for the test partition\n");
+        }
+    }
+
+    // Add OTA partitions
+    for (int i = 0; i < st->app_count && index < length; i++) {
+        if (bootloader_utility_fill_info(&st->ota[i], bp + index) == ESP_OK) {
+            bp[index++].number = i;
+        } else {
+            esp_rom_printf("Error getting the description for the ota_%d partition\n", i);
+        }
+    }
+
+    return index;
+}
+
+
+/**
+ * @brief Get an input from the user, this is currently implemented with the UART interface but we can
+ * imagine implementing it with a GPIO or any other mean.
+ */
+static bootloader_input_t bootloader_get_next_event(void)
+{
+    /* Up arrow, down arrow and Enter will be used for up, down and validate events respectively */
+    for (;;) {
+        char input = uart_rx_one_char_block();
+
+        /* Check if the received character is an escaped character */
+        if (input == 0x1B) {
+            input = uart_rx_one_char_block();
+
+            /* Likely to be an arrow key */
+            if (input == '[') {
+                input = uart_rx_one_char_block();
+                switch (input) {
+                    case 'A':
+                        return INPUT_UP;
+                    case 'B':
+                        return INPUT_DOWN;
+                    default:
+                        /* Ignore unknown sequences */
+                        break;
+                }
+            }
+        } else if (input == '\n' || input == '\r') {
+            return INPUT_VALIDATE;
+        }
+    }
+}
+
+
+#if CONFIG_LIBC_NEWLIB
+// Return global reent struct if any newlib functions are linked to bootloader
+struct _reent *__getreent(void)
+{
+    return _GLOBAL_REENT;
+}
+#endif

examples/custom_bootloader/bootloader_multiboot/main/CMakeLists.txt

@@ -0,0 +1,3 @@
+idf_component_register(SRCS "bootloader_multiboot_example_main.c"
+                       INCLUDE_DIRS "."
+                       PRIV_REQUIRES newlib)

examples/custom_bootloader/bootloader_multiboot/main/bootloader_multiboot_example_main.c

@@ -0,0 +1,16 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <stdio.h>
+
+void app_main(void)
+{
+    /**
+     * Nothing special is done here, everything interesting in this example
+     * is done in the custom bootloader code, located in:
+     * `bootloader_components/main/bootloader_start.c`
+     */
+    printf("Application started!\n");
+}

examples/custom_bootloader/bootloader_multiboot/partitions.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap
+default,     app,  factory, 0x10000, 256K,
+hello_world, app,  ota_0,   , 256K,
+console,     app,  ota_1,   , 1M,
+# Required by the console example
+nvs,         data, nvs,     ,  32K,

examples/custom_bootloader/bootloader_multiboot/pytest_custom_bootloader_multiboot.py

@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+import pytest
+from pytest_embedded import Dut
+from pytest_embedded_idf.app import IdfApp
+
+
+@pytest.mark.esp32c3
+@pytest.mark.esp32s3
+@pytest.mark.generic
+def test_custom_bootloader_multiboot_example(app: IdfApp, dut: Dut) -> None:
+    # Expect to see all three partitions in the list
+    dut.expect_exact('default')
+    dut.expect_exact('hello_world')
+    dut.expect_exact('console')
+    # Send "down arrow" signal to select the second image, which should be `hello_world`
+    dut.write('\x1b[B')
+    # Send Enter to validate and start the image
+    dut.write('\n')
+    # Make sure the example booted properly
+    dut.expect('Hello world!')

examples/custom_bootloader/bootloader_multiboot/sdkconfig.defaults

@@ -0,0 +1,4 @@
+CONFIG_PARTITION_TABLE_CUSTOM=y
+CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
+CONFIG_PARTITION_TABLE_FILENAME="partitions.csv"
+CONFIG_BOOTLOADER_WDT_ENABLE=n